sago

package module
v1.1.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 19, 2020 License: Apache-2.0 Imports: 26 Imported by: 0

README

sago

西米露,一种富有营养的物质,手工制成

cli并行框架 提供一套轻量可控的脚本开发脚手架,降低并发程序开发难度

软件功能
- 支持测试/生产环境 配置文件区分
- 数据库组件 mysql,redis
- 日志组件 支（持错误分级输出,文件切割,rgb颜色输出）
- 工具组件 (文件,网络,变量转换等)
- 并发组件  (支持协程复用模式和单次释放模式,支持并发数和超时时间设置)
- cmd  脚手架工具等 (生成项目基本目录生成,model文件生成)
使用说明
  • go mod 安装
    • 需要安装 mod 包,安装成功后配置环境变量
  • IDE选择:建议使用goland版本 >= 2019.2,安装好后会自动检测并引入依赖,go mod是golang新出特性,旧版本不支持
新项目初始化
  • 1 利用脚手架工具生成一个新项目,如下
cd cmd
./sago_cmd -pname 项目名

   table参数如不指定，则不生成model下的文件

  • 2 将生成的目录拷贝到console下面,结束
项目部署
  • 交叉编译:mac,windows,linux之间是不兼容的,在mac下生成linux上可执行文件命令
GOOS=linux GOARCH=amd64 go build -o worker_pcpro  main.go
  • 生产环境:
//生产环境-prod.toml配置文件
./sago  -env=prod

//开发环境-对应etc下dev.toml配置文件
./sago  -env=dev
其他examples
常规使用（参考export项目）
package main

import (
	"fmt"
	"gitee.com/xiawucha365/sago"
	"strconv"
)

func main() {

defer logger.flush()
//使用mysql 
var priceSeekoriItems []models.THotelPriceSeekori
app.GMysql.Select("id,ctime").Where("ctime > ? ", lastTime, ).Order("id asc ").Find(&priceSeekoriItems)

//使用redis
app.GRedis.Set("test_key","message")

reply,err := app.GRedis.Get("test_key")

if(err != nill){
    panic(err)
}else{
    fmt.Println(reply)
}
//日志输出/记录到文件
logger.Warn("hello,world")
}
并发相关 （参考spider_worker）

注意:worker的定义需要满足interface中d约定,实现Task()与GetTaskID()两个方法,后面会支持匿名函数

package main

import (
	"fmt"
	"gitee.com/xiawucha365/sago"
	"time"
)

//并发池实例
var wpool *pipe.WPool

//任务实例
type worker struct {
	name string
}

//要执行的任务列表
var name_slices = []string{"001", "002", "003", "004", "005", "006", "007", "008", "009"}

//单任务的业务逻辑代码,
func (m *worker) Task() error {

	fmt.Println("job:" + m.name + " start")
	time.Sleep(time.Second * 3)
	fmt.Println("job:" + m.name + " end")
	return nil
}
//获取任务id
func (m *worker) GetTaskID() interface{} {
	return m.Name
}


//例子演示
func main() {

	logger.Warn("并发开始")
    //设置最大并发数与超时时间
	wool = pipe.NewWPool(4, len(name_slices),30)
	wool.Run()

	for _, name := range name_slices {
		np := worker{
			name: name,
		}
		wool.Commit(&np)
	}

	wool.Shutdown()

}

Documentation

Index

Constants

View Source 
const (
	GENV_CONST_DEV  = "dev"
	GENV_CONST_PROD = "prod"
)
View Source 
const Host = "smtp.exmail.qq.com"
View Source 
const Pass = "Chen7110363"
View Source 
const Port = 465
View Source 
const User = "chenweijie@mafengwo.com"

连接信息

Variables

View Source 
var (
	GMysql   *DbEngine
	GModel   *DbDialect
	GRedis   redis.Conn
	GMongodb Mongodb
	GMail    Mail
	Env      string
	Debug    bool
)
View Source 
var (
	MailEngine *gomail.Dialer
)

Functions

func CreateMongodbSessionList added in v1.0.1 

func CreateMongodbSessionList() []*mgo.Session

创建mongodb连接池

func GetCurrentPath 

func GetCurrentPath() string

func InitConfig 

func InitConfig(filename string) (err error)

加载配置

func InitMailConnect added in v1.0.1 

func InitMailConnect() *gomail.Dialer

初始化连接

func SelfPath 

func SelfPath() string

SelfPath gets compiled executable file absolute path

func Testing 

func Testing()

Types

type ComConfig 

type ComConfig struct {
	Appdir    string // 应用根目录
	ConfDir   string // 配置文件目录
	Logdir    string //日志目录
	VersionId int    // 版本号
	Debug     bool
	Env       string
}

type Config 

type Config struct {
	Common  ComConfig
	Mysql   MysqlConfig
	Redis   RedisConfig
	Mongodb MongodbConfig
}

程序配置 

var (
	// 单例
	GConfig *Config
)

type DbDialect 

type DbDialect struct{}
var MysqlDialect *DbDialect

提供封装的方式,保证调用风格统一

func (*DbDialect) Count 

func (m *DbDialect) Count(bean ...interface{}) (int64, error)

func (*DbDialect) Find 

func (m *DbDialect) Find(bean ...interface{}) error

func (*DbDialect) FindBySql 

func (m *DbDialect) FindBySql(sql string, items []interface{}) error

func (*DbDialect) GetById 

func (m *DbDialect) GetById(id int, item interface{}) (bool, error)

func (*DbDialect) Insert 

func (m *DbDialect) Insert(bean ...interface{}) (int64, error)

func (*DbDialect) Limit 

func (m *DbDialect) Limit(limit int) *xorm.Session

func (*DbDialect) OrderBy 

func (m *DbDialect) OrderBy(order string) *xorm.Session

func (*DbDialect) Select 

func (m *DbDialect) Select(str string) *xorm.Session

func (*DbDialect) UpdateById 

func (m *DbDialect) UpdateById(id int, item interface{}) (int64, error)

func (*DbDialect) Where 

func (m *DbDialect) Where(query interface{}, args ...interface{}) *xorm.Session

type DbEngine 

type DbEngine = xorm.Engine
var (

	//通用模式,可以调用xorm底层方法
	MysqlEngine *DbEngine
)

func CreateMysqlDialect 

func CreateMysqlDialect() *DbEngine

type Logger 

type Logger struct{}

程序配置 

var (
	// 单例
	Log *Logger
)

func (*Logger) Error 

func (l *Logger) Error(v ...interface{})

func (*Logger) Flush 

func (l *Logger) Flush(v ...interface{})

func (*Logger) Info 

func (l *Logger) Info(v ...interface{})

func (*Logger) Sucess 

func (l *Logger) Sucess(v ...interface{})

func (*Logger) Warn 

func (l *Logger) Warn(v ...interface{})

type Mail added in v1.0.1 

type Mail struct {
	MailEngine *gomail.Dialer
}

邮件对象

func (*Mail) SendEmail added in v1.0.1 

func (mailInfo *Mail) SendEmail(mailToStr string, title string, body string) error

发送邮件

type Mongodb added in v1.0.1 

type Mongodb struct {
	SessionList []*mgo.Session
}

func (*Mongodb) Select added in v1.0.1 

func (m *Mongodb) Select(key string) *mgo.Session

type MongodbConfig added in v1.0.1 

type MongodbConfig struct {
	Addr     string
	Username string
	Password string
	Dbname   string
}

type MysqlConfig 

type MysqlConfig struct {
	Addr     string
	Username string
	Password string
	Charset  string
	Dbname   string
}

type RedisConfig 

type RedisConfig struct {
	Addr     string
	Password string
}

type RedisDialect 

type RedisDialect struct{}

func (*RedisDialect) Get 

func (m *RedisDialect) Get(key string) (reply interface{}, err error)

func (*RedisDialect) RegisterDbConn 

func (m *RedisDialect) RegisterDbConn()

func (*RedisDialect) Set 

func (m *RedisDialect) Set(key string, val string) (err error)

type SPool 

type SPool struct {
	TotalNum    int
	Counter     int
	CounterFail int

	TimeStart     int64
	TimeOut       int
	MaxGoroutines int
	// contains filtered or unexported fields
}

/共享协程池

func NewSPool 

func NewSPool(maxGoroutines int) *SPool

协程池

func (*SPool) Commit 

func (p *SPool) Commit(w WorkerInterface)

提交任务

func (*SPool) CountFail 

func (p *SPool) CountFail()

计数器-失败

func (*SPool) CountOk 

func (p *SPool) CountOk()

计数器

func (*SPool) Run 

func (p *SPool) Run()

func (*SPool) Runtimelog 

func (p *SPool) Runtimelog()

log

func (*SPool) Shutdown 

func (p *SPool) Shutdown()

等待组 关闭channel

type Session 

type Session = xorm.Session

type Tooler 

type Tooler struct{}
var Tool *Tooler

func (*Tooler) ConvInt2Str 

func (t *Tooler) ConvInt2Str(str int) string

func (*Tooler) ConvStr2Int 

func (t *Tooler) ConvStr2Int(str string) int

func (*Tooler) CreateDir 

func (t *Tooler) CreateDir(dir string)

func (*Tooler) Exist 

func (t *Tooler) Exist(filename string) bool

func (*Tooler) FileExists 

func (t *Tooler) FileExists(path string) bool

FileExists reports whether the named file or directory exists. 判断所给路径文件/文件夹是否存在

func (*Tooler) GetCurrentPath 

func (t *Tooler) GetCurrentPath() string

func (*Tooler) GetExecDir 

func (t *Tooler) GetExecDir() string

func (*Tooler) GetRootPath 

func (t *Tooler) GetRootPath() string

获取项目根目录

func (*Tooler) GrepFile 

func (t *Tooler) GrepFile(patten string, filename string) (lines []string, err error)

GrepFile like command grep -E for example: GrepFile(`^hello`, "hello.txt") \n is striped while read

func (*Tooler) JsonDecode 

func (t *Tooler) JsonDecode(data string, v interface{}) error

func (*Tooler) JsonEncode 

func (t *Tooler) JsonEncode(v interface{}) (string, error)

文件相关

func (*Tooler) MathDecimal 

func (t *Tooler) MathDecimal(value float64) float64

浮点保留2位小数

func (*Tooler) PrintType 

func (t *Tooler) PrintType(s interface{})

打印值和类型

func (*Tooler) PrintVal 

func (t *Tooler) PrintVal(s interface{})

打印值

func (*Tooler) ReadFile 

func (t *Tooler) ReadFile(fileName string) (data []byte)

func (*Tooler) RequestGet 

func (t *Tooler) RequestGet(url string) string

http

func (*Tooler) RequestPostForm 

func (t *Tooler) RequestPostForm(url string, json_input interface{}) (string, *http.Response, []error)

func (*Tooler) RequestPostJson 

func (t *Tooler) RequestPostJson(url string, json_input interface{}) (string, *http.Response, []error)

func (*Tooler) SelfDir 

func (t *Tooler) SelfDir() string

SelfDir gets compiled executable file directory

func (*Tooler) WriteFile 

func (t *Tooler) WriteFile(filename string, data string) bool

WriteFile

type WPool 

type WPool struct {
	TotalNum    int
	Counter     int
	CounterFail int

	TimeStart int64
	TimeOut   int
	// contains filtered or unexported fields
}

非共享协程池

func NewWPool 

func NewWPool(maxGoroutines int, totalNum int, timeOut int) *WPool

协程池

func (*WPool) Commit 

func (p *WPool) Commit(w WorkerInterface)

提交任务

func (*WPool) CountFail 

func (p *WPool) CountFail()

计数器-失败

func (*WPool) CountOk 

func (p *WPool) CountOk()

计数器

func (*WPool) Run 

func (p *WPool) Run()

控制最大并发数

func (*WPool) Runtimelog 

func (p *WPool) Runtimelog()

log

func (*WPool) Shutdown 

func (p *WPool) Shutdown()

等待组 关闭channel

type WorkerInterface 

type WorkerInterface interface {
	Task() error
	GetTaskID() interface{}
}

任务接口

Source Files

View all Source files

Directories

Path Synopsis
cmd
test
test/context
Package context defines the Context type, which carries deadlines, cancelation signals, and other request-scoped values across API boundaries and between processes.
 Package context defines the Context type, which carries deadlines, cancelation signals, and other request-scoped values across API boundaries and between processes.
internal
db
logger
pipe
tool

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.