初始化项目

trunk/framework/sqlAsyncMgr/sqlAsyncIdentityStatistics.go

@@ -0,0 +1,77 @@
+package sqlAsyncMgr
+
+import (
+	"sync"
+	"sync/atomic"
+)
+
+// SqlAsyncIdentityStatistics sql标识统计类
+type SqlAsyncIdentityStatistics struct {
+	// 锁对象
+	m sync.Mutex
+
+	// 当前正在同步的sql数量
+	syncCount int32
+
+	// 待同步的统计
+	statisticData map[string]*StatisticModel
+}
+
+// newSqlAsyncIdentityStatistics  创建新StatisticModel对象
+func newSqlAsyncIdentityStatistics() *SqlAsyncIdentityStatistics {
+	return &SqlAsyncIdentityStatistics{
+		statisticData: make(map[string]*StatisticModel),
+	}
+}
+
+// getItem 获取指定统计对象
+func (this *SqlAsyncIdentityStatistics) getItem(_identityId string, isCreate bool) *StatisticModel {
+	this.m.Lock()
+	defer this.m.Unlock()
+
+	if item, exists := this.statisticData[_identityId]; exists {
+		return item
+	}
+
+	if isCreate == false {
+		return nil
+	}
+
+	newItem := newStatisticModel(_identityId)
+	this.statisticData[_identityId] = newItem
+
+	return newItem
+}
+
+// AddCount 指定标识添加数量
+func (this *SqlAsyncIdentityStatistics) AddCount(_identityId string, count int32) {
+	model := this.getItem(_identityId, true)
+	model.AddCount(count)
+
+	// 原子操作相加
+	atomic.AddInt32(&this.syncCount, count)
+}
+
+// Reduce 指定标识减少数量
+func (this *SqlAsyncIdentityStatistics) Reduce(_identityId string, count int32) {
+	if model := this.getItem(_identityId, false); model != nil {
+		model.ReduceCount(count)
+	}
+
+	// 原子操作相减
+	atomic.AddInt32(&this.syncCount, -count)
+}
+
+// GetCount 获取指定标识的数量
+func (this *SqlAsyncIdentityStatistics) GetCount(_identityId string) int32 {
+	if model := this.getItem(_identityId, false); model != nil {
+		return model.Count
+	}
+
+	return 0
+}
+
+// GetAllCount 获取总的待写入数据
+func (this *SqlAsyncIdentityStatistics) GetAllCount() int32 {
+	return this.syncCount
+}

trunk/framework/sqlAsyncMgr/sqlAsyncItemModel.go

@@ -0,0 +1,22 @@
+package sqlAsyncMgr
+
+// SqlAsyncItemModel 异步sql模型
+type SqlAsyncItemModel struct {
+	// 表名
+	TableName string
+
+	// 唯一标识
+	IdentityId string
+
+	// 待执行的sql
+	Sql string
+}
+
+// newSqlAsyncItemModel 构造异步sql模型
+func newSqlAsyncItemModel(tableName, identityId, sql string) *SqlAsyncItemModel {
+	return &SqlAsyncItemModel{
+		TableName:  tableName,
+		IdentityId: identityId,
+		Sql:        sql,
+	}
+}

trunk/framework/sqlAsyncMgr/sqlAsyncListModel.go

@@ -0,0 +1,75 @@
+package sqlAsyncMgr
+
+import (
+	"sync"
+)
+
+// SqlAsyncListModel 待同步sql列表模型对象
+type SqlAsyncListModel struct {
+	// 锁对象
+	m sync.Mutex
+
+	// 待同步的sql对象
+	sqlData []*SqlAsyncItemModel
+}
+
+// newSqlAsyncListModel  创建新SqlAsyncListModel对象
+func newSqlAsyncListModel() *SqlAsyncListModel {
+	return &SqlAsyncListModel{
+		sqlData: make([]*SqlAsyncItemModel, 0, 4),
+	}
+}
+
+// SqlCount 待同步的sql数量
+func (this *SqlAsyncListModel) SqlCount() int32 {
+	this.m.Lock()
+	defer this.m.Unlock()
+
+	return int32(len(this.sqlData))
+}
+
+// Add 添加同步sql
+func (this *SqlAsyncListModel) Add(newSqlModel *SqlAsyncItemModel) {
+	this.m.Lock()
+	defer this.m.Unlock()
+
+	this.sqlData = append(this.sqlData, newSqlModel)
+}
+
+// GetItem 获取同步对象
+func (this *SqlAsyncListModel) GetItem() (*SqlAsyncItemModel, bool) {
+	this.m.Lock()
+	defer this.m.Unlock()
+
+	if len(this.sqlData) > 0 {
+		return this.sqlData[0], true
+	}
+
+	return nil, false
+}
+
+// RemoveFirst 移除首位元素
+func (this *SqlAsyncListModel) RemoveFirst() {
+	this.m.Lock()
+	defer this.m.Unlock()
+
+	if len(this.sqlData) == 0 {
+		return
+	}
+
+	this.sqlData = this.sqlData[1:]
+}
+
+// GetAllSqlModel 获取所有的sql
+func (this *SqlAsyncListModel) GetAllSqlModel() []*SqlAsyncItemModel {
+	this.m.Lock()
+	defer this.m.Unlock()
+
+	if len(this.sqlData) == 0 {
+		return nil
+	}
+
+	r := this.sqlData[0:]
+
+	return r
+}

trunk/framework/sqlAsyncMgr/sqlAsyncUtil.go

@@ -0,0 +1,243 @@
+package sqlAsyncMgr
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/jinzhu/gorm"
+	_ "github.com/jinzhu/gorm/dialects/mysql"
+	"goutil/logUtil"
+	"goutil/stringUtil"
+)
+
+// sql异步util类
+type SqlAsyncUtil struct {
+	// 文件路径
+	filePath string
+
+	// 名称
+	name string
+
+	// 工作者最大数量
+	max_Worker_Nums int32
+
+	// 工作池
+	workerPool *SqlAsyncWorkerPool
+
+	// 缓存统计
+	sqlStatistics *SqlAsyncIdentityStatistics
+
+	// 日志处理方法
+	logAction func(logUtil.LogType, string)
+
+	// db驱动
+	db *gorm.DB
+}
+
+//  NewSqlAsyncUtil 创建新SqlAsyncUtil对象
+func NewSqlAsyncUtil(_name, _filePath string, maxWorker int32, dbConnectionStr string, _logAction func(logUtil.LogType, string)) *SqlAsyncUtil {
+	if stringUtil.IsEmpty(_name) || stringUtil.IsEmpty(_filePath) || stringUtil.IsEmpty(dbConnectionStr) {
+		panic(fmt.Sprintf("NewSqlAsyncUtil方法_name/_filePath/dbConnectionStr参数不能为空"))
+	}
+	if maxWorker <= 0 {
+		panic(fmt.Sprintf("NewSqlAsyncUtil方法maxWorker参数必须>0"))
+	}
+	if _logAction == nil {
+		panic(fmt.Sprintf("NewSqlAsyncUtil方法_logAction参数并能为nil"))
+	}
+
+	result := &SqlAsyncUtil{
+		name:            _name,
+		filePath:        _filePath,
+		max_Worker_Nums: maxWorker,
+		logAction:       _logAction,
+	}
+	result.db = newGormDb(dbConnectionStr, maxWorker)
+	result.workerPool = newSqlAsyncWorkerPool(maxWorker, result.reduceStatistics, _logAction, func() string { return result.name }, result.db)
+	result.sqlStatistics = newSqlAsyncIdentityStatistics()
+
+	return result
+}
+
+//  NewSqlAsyncUtil2 创建新SqlAsyncUtil对象
+func NewSqlAsyncUtil2(_name, _filePath string, maxWorker int32, db *gorm.DB, _logAction func(logUtil.LogType, string)) *SqlAsyncUtil {
+	if stringUtil.IsEmpty(_name) || stringUtil.IsEmpty(_filePath) || db == nil {
+		panic(fmt.Sprintf("NewSqlAsyncUtil方法_name/_filePath/db参数不能为空"))
+	}
+	if maxWorker <= 0 {
+		panic(fmt.Sprintf("NewSqlAsyncUtil方法maxWorker参数必须>0"))
+	}
+	if _logAction == nil {
+		panic(fmt.Sprintf("NewSqlAsyncUtil方法_logAction参数并能为nil"))
+	}
+
+	result := &SqlAsyncUtil{
+		name:            _name,
+		filePath:        _filePath,
+		max_Worker_Nums: maxWorker,
+		logAction:       _logAction,
+	}
+	result.db = db
+	result.workerPool = newSqlAsyncWorkerPool(maxWorker, result.reduceStatistics, _logAction, func() string { return result.name }, result.db)
+	result.sqlStatistics = newSqlAsyncIdentityStatistics()
+
+	return result
+}
+
+// Start 启动
+func (this *SqlAsyncUtil) Start() {
+	// 启动工作线程
+	this.workerPool.Start()
+
+	// 读取待同步sql
+	waitSyncSqls := this.readWaitSql()
+	if waitSyncSqls == nil {
+		return
+	}
+
+	// 将数据加入写入队列
+	for _, item := range waitSyncSqls {
+		if item == nil {
+			continue
+		}
+		this.Write(item)
+	}
+}
+
+// Stop 停止
+func (this *SqlAsyncUtil) Stop(save bool) {
+	saveList := this.workerPool.Stop()
+	if saveList == nil || len(saveList) == 0 {
+		return
+	}
+
+	if save == false {
+		return
+	}
+
+	this.waitSqlFlushFile(saveList)
+}
+
+// WaitSqlSyncDone 等待剩余sql同步完成
+func (this *SqlAsyncUtil) WaitSqlSyncDone() {
+	for {
+		num := this.GetAllCount()
+		this.logAction(logUtil.Debug, fmt.Sprintf("SqlAsyncUtil(%s)当前剩余sql数量为:%v", this.name, num))
+
+		if num > 0 {
+			time.Sleep(time.Millisecond * 100)
+			continue
+		}
+
+		// 删除文件
+		this.delFile()
+		return
+	}
+}
+
+// Write1 写入Sql
+func (this *SqlAsyncUtil) Write1(tableName, identityId, sql string) {
+	newItem := newSqlAsyncItemModel(tableName, identityId, sql)
+	this.Write(newItem)
+}
+
+// Write 写入Sql
+func (this *SqlAsyncUtil) Write(item *SqlAsyncItemModel) {
+	worker, err := this.workerPool.GetWork(item.TableName)
+	if err != nil {
+		this.logAction(logUtil.Error, err.Error())
+		return
+	}
+
+	worker.Add(item)
+	this.sqlStatistics.AddCount(item.IdentityId, 1)
+}
+
+// GetCount 获取指定标识的待执行sql
+func (this *SqlAsyncUtil) GetCount(identityId string) int32 {
+	return this.sqlStatistics.GetCount(identityId)
+}
+
+// GetAllCount 获取总的待执行sql
+func (this *SqlAsyncUtil) GetAllCount() int32 {
+	return this.workerPool.GetWaitSyncCount()
+}
+
+// ReduceStatistics 减少统计
+func (this *SqlAsyncUtil) reduceStatistics(item *SqlAsyncItemModel) {
+	this.sqlStatistics.Reduce(item.IdentityId, 1)
+}
+
+// readWaitSql 读取待同步sql
+func (this *SqlAsyncUtil) readWaitSql() []*SqlAsyncItemModel {
+	_, err := os.Stat(this.filePath)
+	if err != nil {
+		return nil
+	}
+
+	var bytes []byte
+	bytes, err = ioutil.ReadFile(this.filePath)
+	if err != nil {
+		panic(fmt.Sprintf("readWaitSql错误 file:%s err:%s", this.filePath, err.Error()))
+	}
+
+	result := make([]*SqlAsyncItemModel, 0)
+	err = json.Unmarshal(bytes, &result)
+	if err != nil {
+		panic(fmt.Sprintf("readWaitSql json反序列化错误 file:%s err:%s", this.filePath, err.Error()))
+	}
+
+	return result
+}
+
+// waitSqlFlushFile 待同步sql写入文件
+func (this *SqlAsyncUtil) waitSqlFlushFile(waitSqlList []*SqlAsyncItemModel) {
+	dir := filepath.Dir(this.filePath)
+
+	// 创建文件夹
+	_, err := os.Stat(dir)
+	if err != nil {
+		os.MkdirAll(dir, os.ModePerm)
+	}
+
+	data, err := json.Marshal(waitSqlList)
+	if err != nil {
+		this.logAction(logUtil.Error, fmt.Sprintf("SqlAsyncUtil保存sql时,json出错,name:%s err:%s", this.name, err.Error()))
+		return
+	}
+
+	err = ioutil.WriteFile(this.filePath, data, os.ModePerm)
+	if err != nil {
+		this.logAction(logUtil.Error, fmt.Sprintf("SqlAsyncUtil保存sql时,写入出错,name:%s err:%s", this.name, err.Error()))
+	}
+}
+
+// delFile 删除文件
+func (this *SqlAsyncUtil) delFile() {
+	// 创建文件夹
+	_, err := os.Stat(this.filePath)
+	if err == nil {
+		os.Remove(this.filePath)
+	}
+}
+
+// @title newGormDb
+// @description 构造新gorm.DB对象
+// @Param connectionString 数据库连接字符串
+// @Param maxOpenCount 最大打开连接数
+func newGormDb(connectionString string, maxOpenCount int32) *gorm.DB {
+	dbObj, err := gorm.Open("mysql", connectionString)
+	if err != nil {
+		panic(fmt.Sprintf("连接mysql出错 connectionString:%s  err:%s", connectionString, err))
+	}
+
+	dbObj.DB().SetMaxOpenConns(int(maxOpenCount))
+	dbObj.DB().SetMaxIdleConns(int(maxOpenCount))
+	dbObj.DB().SetConnMaxLifetime(time.Minute * 4)
+
+	return dbObj
+}

trunk/framework/sqlAsyncMgr/sqlAsyncWorker.go

@@ -0,0 +1,143 @@
+package sqlAsyncMgr
+
+import (
+	"fmt"
+	"time"
+
+	"framework/goroutineMgr"
+	"github.com/jinzhu/gorm"
+	"goutil/logUtil"
+)
+
+const (
+	// 同步多少次打印待同步sql数量
+	LogSyncCount = 1000
+)
+
+// sql同步的worker
+type SqlAsyncWorker struct {
+	// 工作者Id
+	Id int32
+
+	// 是否停止工作
+	IsTop bool
+
+	// 当前同步的条数
+	CurSyncCount int32
+
+	// 待执行对象
+	cachModel *SqlAsyncListModel
+
+	// 刷新到时候的回调方法
+	flushDbCallBack func(*SqlAsyncItemModel)
+
+	// 日志处理方法
+	logAction func(logUtil.LogType, string)
+
+	// 获取同步标识
+	getSyncName func() string
+
+	// 数据库驱动
+	db *gorm.DB
+
+	tempCount int
+}
+
+// newSqlAsyncWorker  创建新SqlAsyncWorker对象
+func newSqlAsyncWorker(_id int32, _flushCallBack func(*SqlAsyncItemModel), _logAction func(logUtil.LogType, string), _getSyncName func() string, _db *gorm.DB) *SqlAsyncWorker {
+	return &SqlAsyncWorker{
+		Id:              _id,
+		cachModel:       newSqlAsyncListModel(),
+		flushDbCallBack: _flushCallBack,
+		logAction:       _logAction,
+		getSyncName:     _getSyncName,
+		db:              _db,
+	}
+}
+
+// Start 启动工作
+func (this *SqlAsyncWorker) Start() {
+	go this.handler()
+}
+
+// Stop 停止工作
+func (this *SqlAsyncWorker) Stop() []*SqlAsyncItemModel {
+	this.IsTop = true
+
+	return this.cachModel.GetAllSqlModel()
+}
+
+// Add 添加执行的sql对象
+func (this *SqlAsyncWorker) Add(item *SqlAsyncItemModel) {
+	if this.IsTop {
+		return
+	}
+
+	this.cachModel.Add(item)
+}
+
+// WaitSqlCount 获取待同步的sql数量
+func (this *SqlAsyncWorker) WaitSqlCount() int32 {
+	return this.cachModel.SqlCount()
+}
+
+// FlushDB 刷新到DB
+func (this *SqlAsyncWorker) handler() {
+	// 处理goroutine数量
+	goroutineName := fmt.Sprintf("%s-%v", this.getSyncName(), this.Id)
+	goroutineMgr.Monitor(goroutineName)
+	defer goroutineMgr.ReleaseMonitor(goroutineName)
+
+	for {
+		if this.IsTop {
+			this.logAction(logUtil.Debug, fmt.Sprintf("sql异步线程停止工作 name=%s", goroutineName))
+			return
+		}
+
+		if this.flushItem() == false {
+			time.Sleep(time.Millisecond * 100)
+		}
+	}
+}
+
+// flushItem 刷新到DB
+func (this *SqlAsyncWorker) flushItem() bool {
+	defer func() {
+		if r := recover(); r != nil {
+			this.logAction(logUtil.Error, fmt.Sprintf("flushItem err:%s", r))
+		}
+	}()
+
+	// 获取同步对象
+	item, exists := this.cachModel.GetItem()
+	if exists == false {
+		return false
+	}
+
+	this.tempCount = this.tempCount + 1
+
+	// 写入数据库
+	for i := 0; i < 3; i++ {
+		err := this.db.Exec(item.Sql).Error
+		if err == nil {
+			break
+		}
+
+		this.logAction(logUtil.Error, fmt.Sprintf("写入数据库出现错误 sql:%s  err:%s", item.Sql, err.Error()))
+	}
+
+	// 移除缓存
+	this.cachModel.RemoveFirst()
+
+	// 减少统计
+	this.flushDbCallBack(item)
+
+	// 统计当前待同步数量
+	this.CurSyncCount = this.CurSyncCount + 1
+	if this.CurSyncCount%LogSyncCount == 0 {
+		this.logAction(logUtil.Debug, fmt.Sprintf("当前待同步的sql数量为:%v Name:%s  id:%v", this.cachModel.SqlCount(), this.getSyncName(), this.Id))
+		this.CurSyncCount = 0
+	}
+
+	return true
+}

trunk/framework/sqlAsyncMgr/sqlAsyncWorkerPool.go

@@ -0,0 +1,72 @@
+package sqlAsyncMgr
+
+import (
+	"fmt"
+
+	"github.com/jinzhu/gorm"
+	"goutil/logUtil"
+	"goutil/stringUtil"
+)
+
+type SqlAsyncWorkerPool struct {
+	// 工作者数量
+	wcount int32
+
+	// 工作者列表
+	workers []*SqlAsyncWorker
+}
+
+// newSqlAsyncWorkerPool 构造同步池
+func newSqlAsyncWorkerPool(workerCount int32, _flushCallBack func(*SqlAsyncItemModel), _logAction func(logUtil.LogType, string), _getSyncName func() string, db *gorm.DB) *SqlAsyncWorkerPool {
+	result := &SqlAsyncWorkerPool{
+		wcount:  workerCount,
+		workers: make([]*SqlAsyncWorker, workerCount, workerCount),
+	}
+
+	var i int32
+	for i = 0; i < workerCount; i++ {
+		newWorker := newSqlAsyncWorker(i, _flushCallBack, _logAction, _getSyncName, db)
+		result.workers[i] = newWorker
+	}
+
+	return result
+}
+
+// Start 启动
+func (this *SqlAsyncWorkerPool) Start() {
+	for _, w := range this.workers {
+		w.Start()
+	}
+}
+
+// Stop 停止工作
+func (this *SqlAsyncWorkerPool) Stop() []*SqlAsyncItemModel {
+	result := make([]*SqlAsyncItemModel, 0)
+	for _, w := range this.workers {
+		if tempList := w.Stop(); tempList != nil {
+			result = append(result, tempList...)
+		}
+	}
+
+	return result
+}
+
+// GetWork 获取指定work
+func (this *SqlAsyncWorkerPool) GetWork(tableName string) (*SqlAsyncWorker, error) {
+	index := stringUtil.HashCode(tableName) % int(this.wcount)
+	if index < 0 || index >= int(this.wcount) {
+		return nil, fmt.Errorf("SqlAsyncWorkerPool中work数量为:%v,不存在id=%v的worker", this.wcount, index)
+	}
+
+	return this.workers[index], nil
+}
+
+// GetWaitSyncCount 获取待同步的sql数量
+func (this *SqlAsyncWorkerPool) GetWaitSyncCount() int32 {
+	var count int32
+	for _, w := range this.workers {
+		count = count + w.WaitSqlCount()
+	}
+
+	return count
+}

trunk/framework/sqlAsyncMgr/statisticModel.go

@@ -0,0 +1,40 @@
+package sqlAsyncMgr
+
+import (
+	"sync"
+)
+
+// 标识内容模型对象
+type StatisticModel struct {
+	// 锁对象
+	m sync.Mutex
+
+	// 标识
+	IdentityId string
+
+	// 待同步的数量
+	Count int32
+}
+
+// AddCount 添加指定数量
+func (this *StatisticModel) AddCount(addcount int32) {
+	this.m.Lock()
+	defer this.m.Unlock()
+
+	this.Count = this.Count + addcount
+}
+
+// ReduceCount 添加指定数量
+func (this *StatisticModel) ReduceCount(recount int32) {
+	this.m.Lock()
+	defer this.m.Unlock()
+
+	this.Count = this.Count - recount
+}
+
+// newStatisticModel  创建新StatisticModel对象
+func newStatisticModel(ident string) *StatisticModel {
+	return &StatisticModel{
+		IdentityId: ident,
+	}
+}