初始化项目

trunk/goutil/logUtil/impl-es/api.go

@@ -0,0 +1,63 @@
+package impl_es
+
+// InfoLog
+// @description: 信息日志记录
+// parameter:
+//		@format:日志格式
+//		@args:参数列表
+// return:
+func (l *Logger) InfoLog(format string, args ...interface{}) {
+	info := l.buildEsLog("Info", format, args...)
+	l.cache.addCacheLog(info)
+}
+
+// DebugLog
+// @description: 调试日志记录
+// parameter:
+//		@format:日志格式
+//		@args:参数列表
+// return:
+func (l *Logger) DebugLog(format string, args ...interface{}) {
+	info := l.buildEsLog("Debug", format, args...)
+	l.cache.addCacheLog(info)
+}
+
+// WarnLog
+// @description: 警告日志记录
+// parameter:
+//		@format:日志格式
+//		@args:参数列表
+// return:
+func (l *Logger) WarnLog(format string, args ...interface{}) {
+	info := l.buildEsLog("Warn", format, args...)
+	l.cache.addCacheLog(info)
+}
+
+// ErrorLog
+// @description: 错误日志记录
+// parameter:
+//		@format:日志格式
+//		@args:参数列表
+// return:
+func (l *Logger) ErrorLog(format string, args ...interface{}) {
+	info := l.buildEsLog("Error", format, args...)
+	l.cache.addCacheLog(info)
+}
+
+// FatalLog
+// @description: 致命错误日志记录
+// parameter:
+//		@format:日志格式
+//		@args:参数列表
+// return:
+func (l *Logger) FatalLog(format string, args ...interface{}) {
+	info := l.buildEsLog("Fatal", format, args...)
+	l.cache.addCacheLog(info)
+}
+
+// CloseLog
+// @description: 关闭日志
+// parameter:
+//		@waitFinish:是否等待日志
+// return:
+func (l *Logger) CloseLog(waitFinish bool) {}

trunk/goutil/logUtil/impl-es/cache.go

@@ -0,0 +1,72 @@
+package impl_es
+
+import "sync"
+
+const (
+	// 批量保存的消息数量
+	con_MAX_NUMBER_OF_MESSAGE = 20
+)
+
+// logCache
+// @description: 日志缓存
+type logCache struct {
+	logCacheList  []*EsLogModel // logCacheList 日志缓存对象
+	logCacheMutex sync.RWMutex  // logCacheMutex 锁
+}
+
+// newlogCache
+// @description: 构造日志缓存
+// parameter:
+// return:
+//		@*logCache:
+func newlogCache() *logCache {
+	m := &logCache{
+		logCacheList:  make([]*EsLogModel, 0, 256),
+		logCacheMutex: sync.RWMutex{},
+	}
+
+	return m
+}
+
+// 写入在线日志
+// 参数:
+//		日志信息对象
+// 返回值：
+//		无
+func (c *logCache) addCacheLog(logObj *EsLogModel) {
+	c.logCacheMutex.Lock()
+	defer c.logCacheMutex.Unlock()
+
+	c.logCacheList = append(c.logCacheList, logObj)
+}
+
+// 获取日志数量
+// 参数:
+//		无
+// 返回值：
+//		缓存中的日志数量
+func (c *logCache) getCacheLogCount() int {
+	c.logCacheMutex.RLock()
+	defer c.logCacheMutex.RUnlock()
+
+	return len(c.logCacheList)
+}
+
+// 获取缓存中的日志
+// 参数:
+//		数量
+// 返回值：
+//		日志列表对象
+func (c *logCache) getCacheLog(logCount int) (logList []*EsLogModel) {
+	if logCount > con_MAX_NUMBER_OF_MESSAGE {
+		logCount = con_MAX_NUMBER_OF_MESSAGE
+	}
+
+	c.logCacheMutex.Lock()
+	defer c.logCacheMutex.Unlock()
+
+	logList = c.logCacheList[:logCount]
+	c.logCacheList = c.logCacheList[logCount:]
+
+	return
+}

trunk/goutil/logUtil/impl-es/doc.go

@@ -0,0 +1 @@
+package impl_es

trunk/goutil/logUtil/impl-es/logger.go

@@ -0,0 +1,236 @@
+package impl_es
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/elastic/go-elasticsearch/v8"
+	"github.com/elastic/go-elasticsearch/v8/esutil"
+	"golang.org/x/net/context"
+)
+
+const (
+	// 如果缓存中的日志数量超过阈值，则记录日志
+	con_WRITE_LOG_THRESHOLD_NUMBER = 1000
+)
+
+// Logger
+// @description: es日志处理对象
+type Logger struct {
+	esClient        *elasticsearch.Client // esClient es客户端
+	blukIndexer     esutil.BulkIndexer    // blukIndexer es批量索引
+	indexNamePrefix string                // indexNamePrefix 索引前缀
+	curIndexName    string                // curIndexName 当前索引名字
+	innerId         string                // innerId 系统唯一标识
+	extendCb        func() string         // extendCb 扩展信息获取方法
+	cache           *logCache             // cache 日志缓存
+}
+
+// NewLogger
+// @description: 构造es日志对象
+// parameter:
+//		@urls:
+//		@username:
+//		@pwd:
+//		@esIndexName:
+//		@_innerId:
+//		@_extendCb:
+// return:
+//		@*Logger:
+//		@error:
+func NewLogger(urls []string, username, pwd string, esIndexName, _innerId string, _extendCb func() string) (*Logger, error) {
+	if _extendCb == nil {
+		_extendCb = func() string {
+			return ""
+		}
+	}
+
+	esClient, err := elasticsearch.NewClient(elasticsearch.Config{
+		Addresses: urls,
+		Username:  username,
+		Password:  pwd,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	l := &Logger{
+		esClient:        esClient,
+		blukIndexer:     nil,
+		indexNamePrefix: esIndexName,
+		curIndexName:    "",
+		innerId:         _innerId,
+		extendCb:        _extendCb,
+		cache:           newlogCache(),
+	}
+
+	l.curIndexName = l.getCurIndexName()
+	l.blukIndexer, err = l.newBlukIndexer()
+	if err != nil {
+		return nil, err
+	}
+
+	//  启动日志处理
+	l.start()
+
+	return l, nil
+}
+
+// start
+// @description: 启动日志处理
+// parameter:
+//		@receiver l:
+// return:
+func (l *Logger) start() {
+	go l.logHandlerStart()
+	go l.indexNameCheck()
+}
+
+// getCurIndexName
+// @description: 获取索引名字
+// parameter:
+//		@receiver l:
+// return:
+//		@string:
+func (l *Logger) getCurIndexName() string {
+	//获取当天日期
+	return fmt.Sprintf("%s_%s", l.indexNamePrefix, time.Now().Format("20060102"))
+}
+
+// newBlukIndexer
+// @description: 创建新的BlukIndexer
+// parameter:
+//		@receiver l:
+// return:
+//		@bulkIndexer:
+//		@err:
+func (l *Logger) newBlukIndexer() (bulkIndexer esutil.BulkIndexer, err error) {
+	bulkIndexer, err = esutil.NewBulkIndexer(esutil.BulkIndexerConfig{
+		Index:         l.getCurIndexName(), // The default index name
+		Client:        l.esClient,          // The Elasticsearch client
+		FlushInterval: time.Second,         // The periodic flush interval
+	})
+	if err != nil {
+		fmt.Println("[es log]: Creating the indexer err: ", err)
+	}
+
+	return
+}
+
+// 日志处理
+func (l *Logger) logHandlerStart() {
+	defer func() {
+		if r := recover(); r != nil {
+			fmt.Println("[es log]: Creating the indexer err: ", r)
+			go l.logHandlerStart()
+		}
+	}()
+
+	time.Sleep(1 * time.Second)
+	for {
+		// 达到指定的时间或者指定的日志数量，则保存到消息队列中去
+		logCount := l.cache.getCacheLogCount()
+		if logCount == 0 || l.esClient == nil {
+			time.Sleep(time.Second * 1)
+			continue
+		}
+
+		// 记录在线日志待发送数量
+		if logCount > con_WRITE_LOG_THRESHOLD_NUMBER {
+			fmt.Printf("[es log]: 当前缓存中共有%d条未发送到在线日志系统的日志", logCount)
+		}
+
+		// 执行刷新
+		l.bulkPushToOnlineLogSystem(l.cache.getCacheLog(logCount))
+	}
+}
+
+// indexNameCheck
+// @description: index名字校验
+// parameter:
+//		@receiver l:
+// return:
+func (l *Logger) indexNameCheck() {
+	defer func() {
+		if r := recover(); r != nil {
+			fmt.Println("[es log]: indexNameCheck  err: ", r)
+			go l.indexNameCheck()
+		}
+	}()
+
+	time.Sleep(1 * time.Second)
+	for {
+		time.Sleep(time.Second * 1)
+
+		tempIndexName := l.getCurIndexName()
+		if tempIndexName == l.curIndexName {
+			continue
+		}
+
+		// 关闭老的blukIndexer+建立新的blukIndexer
+		newBlukIndexer, err := l.newBlukIndexer()
+		if err != nil {
+			continue
+		}
+
+		tempBlukIndexer := l.blukIndexer
+		l.blukIndexer = newBlukIndexer
+		l.curIndexName = tempIndexName
+
+		// 暂停3s，等待正在写入的数据写入完成
+		time.Sleep(3 * time.Second)
+		err = tempBlukIndexer.Close(context.Background())
+		if err != nil {
+			fmt.Println("[es log]:BlukIndexer close err：", err.Error())
+		}
+	}
+}
+
+// bulkPushToOnlineLogSystem
+// @description: 批量保存到在线日志系统
+// parameter:
+//		@receiver l:
+//		@logList:
+// return:
+func (l *Logger) bulkPushToOnlineLogSystem(logList []*EsLogModel) {
+	for _, logObj := range logList {
+		message, err := json.Marshal(logObj)
+		if err != nil {
+			fmt.Println("[es log]: Marshal failed. Err:", err)
+			continue
+		}
+
+		err = l.blukIndexer.Add(
+			context.Background(),
+			esutil.BulkIndexerItem{
+				Action: "index",
+				Body:   bytes.NewReader(message),
+			})
+		if err != nil {
+			fmt.Println("[es log]: Add data err：", err.Error())
+		}
+	}
+}
+
+// buildOnlineLog
+// @description: 组装es日志对象
+// parameter:
+//		@receiver l:
+//		@logType:
+//		@format:
+//		@args:
+// return:
+//		@newLogObj:
+func (l *Logger) buildEsLog(logType, format string, args ...interface{}) (newLogObj *EsLogModel) {
+	msg := format
+	if len(args) > 0 {
+		msg = fmt.Sprintf(format, args...)
+	}
+
+	// 构造新的日志对象
+	newLogObj = newEsLogModel(logType, msg, l.innerId, l.extendCb)
+
+	return
+}

trunk/goutil/logUtil/impl-es/logger_test.go

@@ -0,0 +1,25 @@
+package impl_es
+
+import (
+	"testing"
+	"time"
+)
+
+func TestLog(t *testing.T) {
+	urls := []string{"http://10.1.0.71:9101/"}
+	log, err := NewLogger(urls, "", "", "es_log_test", "les_log_test", nil)
+	if err != nil {
+		t.Error("esLog 创建失败")
+	}
+
+	for i := 1; i <= 1000; i++ {
+		log.DebugLog("Debug test %v", i)
+		log.InfoLog("Info test %v", i)
+		log.WarnLog("Warn test %v", i)
+		log.ErrorLog("Error test %v", i)
+		log.FatalLog("Fatal test %v", i)
+		time.Sleep(time.Millisecond * 100)
+	}
+
+	time.Sleep(time.Second * 5)
+}

trunk/goutil/logUtil/impl-es/model.go

@@ -0,0 +1,53 @@
+package impl_es
+
+import (
+	"sync"
+	"time"
+)
+
+var (
+	logPool = sync.Pool{
+		New: func() interface{} {
+			return &EsLogModel{}
+		},
+	}
+)
+
+// 日志对象
+type EsLogModel struct {
+	// 日志类型
+	LogType string
+
+	// 日志消息
+	Message string
+
+	// 程序标识
+	InnerId string
+
+	// Extendstr 扩展信息
+	Extendstr string
+
+	// 日志时间
+	CrTime time.Time
+}
+
+// newEsLogModel
+// @description: 构造esLog对象
+// parameter:
+//		@logType:日志类型
+//		@msg:日志内容
+//		@innerId:内部唯一id
+//		@extendCb:扩展信息回调方法
+// return:
+//		@*EsLogModel:
+func newEsLogModel(logType string, msg string, innerId string, extendCb func() string) *EsLogModel {
+	logObj := logPool.Get().(*EsLogModel)
+
+	logObj.CrTime = time.Now()
+	logObj.LogType = logType
+	logObj.Message = msg
+	logObj.InnerId = innerId
+	logObj.Extendstr = extendCb()
+
+	return logObj
+}