初始化项目

trunk/goutil/esLogUtil/esIndexHandler.go

@@ -0,0 +1,113 @@
+package esLogUtil
+
+import (
+	"bytes"
+	"encoding/json"
+	"sync"
+	"time"
+
+	"github.com/elastic/go-elasticsearch/v8/esutil"
+	"golang.org/x/net/context"
+	"goutil/logUtil"
+)
+
+var (
+	indexMutex sync.Mutex
+	indexer    esutil.BulkIndexer
+)
+
+func timedReindex() {
+	go func() {
+		defer func() {
+			if r := recover(); r != nil {
+				logUtil.LogUnknownError(r)
+
+				timedReindex()
+				time.Sleep(time.Second * 1)
+			}
+		}()
+
+		currentIndexName := getIndexName()
+		for {
+			//设置休眠
+			time.Sleep(time.Second * 1)
+
+			newIndexName := getIndexName()
+			if currentIndexName != getIndexName() {
+				newIndexer(newIndexName)
+				currentIndexName = newIndexName
+			}
+		}
+	}()
+}
+
+func newIndexer(newIndexName string) {
+	if indexer != nil {
+		closeIndex()
+	}
+
+	//wait until get the lock
+	indexMutex.Lock()
+	defer indexMutex.Unlock()
+
+	var err error
+	indexer, err = esutil.NewBulkIndexer(esutil.BulkIndexerConfig{
+		Index:         getIndexName(), // The default index name
+		Client:        esClient,       // The Elasticsearch client
+		FlushInterval: time.Second,    // The periodic flush interval
+	})
+	if err != nil {
+		logUtil.ErrorLog("[%s]: Creating the indexer err: %s", serverModuleName, err)
+	}
+}
+
+func closeIndex() {
+	//wait until get the lock
+	indexMutex.Lock()
+	defer indexMutex.Unlock()
+
+	if indexer == nil {
+		return
+	}
+
+	err := indexer.Close(context.Background())
+	if err != nil {
+		logUtil.ErrorLog("[%s]:Close err：%s", serverModuleName, err.Error())
+	}
+
+	indexer = nil
+}
+
+// 批量保存到在线日志系统
+// 参数:
+//
+//	数量
+//
+// 返回值：
+//
+//	日志列表对象
+func bulkSendHandler(logObj EsLog) {
+	if esClient == nil || indexer == nil {
+		return
+	}
+
+	//try to get the lock in 10000 milliseconds,if cant obtain it,return false
+	indexMutex.Lock()
+	defer indexMutex.Unlock()
+
+	message, err := json.Marshal(logObj)
+	if err != nil {
+		logUtil.ErrorLog("[%s]: Marshal failed. Err:%s", serverModuleName, err)
+		return
+	}
+
+	err = indexer.Add(
+		context.Background(),
+		esutil.BulkIndexerItem{
+			Action: "index",
+			Body:   bytes.NewReader(message),
+		})
+	if err != nil {
+		logUtil.ErrorLog("[%s]: Add data err：%s", serverModuleName, err.Error())
+	}
+}

trunk/goutil/esLogUtil/esLogUtil.go

@@ -0,0 +1,269 @@
+package esLogUtil
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/elastic/go-elasticsearch/v8"
+	"goutil/logUtil"
+	"goutil/stringUtil"
+)
+
+var (
+	serverModuleName = "esLog"
+	esClient         *elasticsearch.Client
+	indexName        string //Index名
+	strServerGroupId string //区服
+	isStop           bool
+	logChan          = make(chan EsLog, 2048)
+	warnCount        = 2000
+	closedChan       = make(chan struct{})
+	logPool          = sync.Pool{
+		New: func() interface{} {
+			return &EsLog{}
+		},
+	}
+)
+
+// 启动ES日志系统
+// 参数:
+//
+//	esUrls：ES地址(多个地址使用,分割)
+//	name：IndexName
+//	serverGroupId：服务器组Id
+//
+// 返回值：
+//
+//	结果状态
+func Start(esUrls string, name string, serverGroupId int32) {
+	if stringUtil.IsEmpty(esUrls) {
+		return
+	}
+
+	//构造Es客户端
+	var err error
+	esClient, err = elasticsearch.NewClient(elasticsearch.Config{
+		Addresses: stringUtil.Split(esUrls, []string{","}),
+		// Retry on 429 TooManyRequests statuses
+		//
+		RetryOnStatus: []int{502, 503, 504, 429},
+
+		// A simple incremental backoff function
+		//
+		RetryBackoff: func(i int) time.Duration { return time.Duration(i) * 100 * time.Millisecond },
+
+		// Retry up to 5 attempts
+		//
+		MaxRetries: 5,
+	})
+	if err != nil {
+		panic(fmt.Sprintf("构造es对象出错，err:%s", err.Error()))
+	}
+
+	indexName = name
+	strServerGroupId = fmt.Sprintf("%d", serverGroupId)
+
+	//初始化ES
+	newIndexer(getIndexName())
+
+	timedReindex()
+
+	startSendProcessor()
+
+	guardProcessor()
+	return
+}
+
+// 停止服务
+func Stop() {
+	//停止接受日志
+	isStop = true
+
+	if indexer == nil {
+		return
+	}
+
+	if len(logChan) == 0 {
+		close(logChan)
+	}
+
+	<-closedChan
+}
+
+//#region 内部方法
+
+func guardProcessor() {
+	go func() {
+		defer func() {
+			if r := recover(); r != nil {
+				logUtil.LogUnknownError(r)
+
+				time.Sleep(1 * time.Second)
+				guardProcessor()
+			}
+		}()
+
+		for {
+			time.Sleep(5 * time.Second)
+
+			count := len(logChan)
+			if count < warnCount {
+				continue
+			}
+
+			logUtil.NormalLog(fmt.Sprintf("ES日志通道中当前有%d条消息待消费。", count), logUtil.Warn)
+		}
+	}()
+}
+
+func startSendProcessor() {
+	go func() {
+		defer func() {
+			if r := recover(); r != nil {
+				logUtil.LogUnknownError(r)
+
+				time.Sleep(1 * time.Second)
+				startSendProcessor()
+			}
+		}()
+
+		for {
+			select {
+			case logObj, ok := <-logChan:
+				if ok {
+					bulkSendHandler(logObj) // 执行刷新
+				}
+
+				if len(logChan) == 0 && isStop {
+					// is closed
+					closeIndex()
+
+					closedChan <- struct{}{}
+					return
+				}
+			}
+		}
+	}()
+}
+
+func getIndexName() string {
+	//获取当天日期
+	return fmt.Sprintf("%s_%s", indexName, time.Now().Format("20060102"))
+}
+
+// 写入在线日志
+// 参数:
+//
+//	日志信息对象
+//
+// 返回值：
+//
+//	无
+func writeLog(logObj *EsLog) {
+	if isStop || indexer == nil {
+		return
+	}
+
+	logChan <- *logObj
+}
+
+// 组装ES日志对象
+// 参数:
+//
+//	logType 日志类型
+//	format 日志格式
+//	args 参数列表
+//
+// 返回值：
+//
+//	结果状态
+func buildLog(logType, format string, args ...interface{}) (newLogObj *EsLog) {
+	msg := format
+	if len(args) > 0 {
+		msg = fmt.Sprintf(format, args...)
+	}
+
+	//构造新的日志对象
+	newLogObj = new(logType, msg, strServerGroupId)
+	return
+}
+
+//#endregion
+
+//#region 外部方法
+
+// 日志记录
+//
+//	format:日志格式
+//	logType:日志类型
+//	args:参数列表
+//
+// 返回值
+//
+//	无
+func NormalLog(format string, logType logUtil.LogType, args ...interface{}) {
+	writeLog(buildLog(logType.String(), format, args...))
+}
+
+// 消息日志记录
+//
+//	format:日志格式
+//	args:参数列表
+//
+// 返回值
+//
+//	无
+func InfoLog(format string, args ...interface{}) {
+	writeLog(buildLog("Info", format, args...))
+}
+
+// 警告日志记录
+//
+//	format:日志格式
+//	args:参数列表
+//
+// 返回值
+//
+//	无
+func WarnLog(format string, args ...interface{}) {
+	writeLog(buildLog("Warn", format, args...))
+}
+
+// 调试日志记录
+//
+//	format:日志格式
+//	args:参数列表
+//
+// 返回值
+//
+//	无
+func DebugLog(format string, args ...interface{}) {
+	writeLog(buildLog("Debug", format, args...))
+}
+
+// 错误日志记录
+//
+//	format:日志格式
+//	args:参数列表
+//
+// 返回值
+//
+//	无
+func ErrorLog(format string, args ...interface{}) {
+	writeLog(buildLog("Error", format, args...))
+}
+
+// 致命错误日志记录
+//
+//	format:日志格式
+//	args:参数列表
+//
+// 返回值
+//
+//	无
+func FatalLog(format string, args ...interface{}) {
+	writeLog(buildLog("Fatal", format, args...))
+}
+
+//#endregion

trunk/goutil/esLogUtil/esLogUtil_test.go

@@ -0,0 +1,35 @@
+package esLogUtil
+
+import (
+	"testing"
+	"time"
+)
+
+func TestWrite(t *testing.T) {
+	Start("http://10.1.0.86:9200", "dzg_gs_log_gmc2", 20008)
+	for i := 0; i < 10000; i++ {
+		InfoLog("ES在线日志测试")
+		WarnLog("ES在线日志测试")
+		DebugLog("ES在线日志测试")
+		ErrorLog("ES在线日志测试")
+		FatalLog("ES在线日志测试")
+	}
+	Stop()
+}
+
+func BenchmarkWrite(b *testing.B) {
+	Start("http://106.52.100.147:14001", "20008_gs_log", 20008)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		InfoLog("ES在线日志测试%d", i)
+		WarnLog("ES在线日志测试%d", i)
+		DebugLog("ES在线日志测试%d", i)
+		ErrorLog("ES在线日志测试%d", i)
+		FatalLog("ES在线日志测试%d", i)
+	}
+	b.StopTimer()
+
+	time.Sleep(30 * time.Second)
+	Stop()
+}

trunk/goutil/esLogUtil/model.go

@@ -0,0 +1,39 @@
+package esLogUtil
+
+import (
+	"time"
+)
+
+// 日志对象
+type EsLog struct {
+	// 日志类型
+	LogType string
+
+	// 日志消息
+	Message string
+
+	// 程序标识
+	InnerId string
+
+	// 日志时间
+	CrTime time.Time
+}
+
+func new(logType string, msg string, innerId string) *EsLog {
+	logObj := logPool.Get().(*EsLog)
+
+	logObj.CrTime = time.Now()
+	logObj.LogType = logType
+	logObj.Message = msg
+	logObj.InnerId = innerId
+
+	return logObj
+
+	//
+	//return EsLog{
+	//	LogType: logType,
+	//	Message: msg,
+	//	InnerId: innerId,
+	//	CrTime:  time.Now(),
+	//}
+}