初始化项目

2025-01-06 16:01:02 +08:00
commit 1b77f62820
575 changed files with 69193 additions and 0 deletions
--- a/trunk/goutil/ensureSendUtil/bytesSendUtil/baseSender.go
+++ b/trunk/goutil/ensureSendUtil/bytesSendUtil/baseSender.go
@@ -0,0 +1,54 @@
+package bytesSendUtil
+
+import (
+	"fmt"
+)
+
+/*
+实现sender接口
+*/
+
+type baseSender struct {
+	// 待发送的数据channel
+	waitingDataChan chan dataItem
+
+	// 失败数据缓存
+	cachedDataChan chan dataItem
+
+	// 用于停止协程
+	done chan struct{}
+}
+
+func newBaseSender() *baseSender {
+	return &baseSender{
+		waitingDataChan: make(chan dataItem, 1024),
+		cachedDataChan:  make(chan dataItem, 1024000),
+		done:            make(chan struct{}),
+	}
+}
+
+// Sender接口
+// Send:
+func (this *baseSender) Send() error {
+	// baseSender不实现发送
+	// 由tcpSender和httpSender实现发送
+	return fmt.Errorf("baseSender dose not have Send Method")
+}
+
+// Sender接口
+// Data: 返回待发送的数据channel
+func (this *baseSender) Data() <-chan dataItem {
+	return this.waitingDataChan
+}
+
+// Sender接口
+// Cache：返回失败数据缓存channel
+func (this *baseSender) Cache() chan dataItem {
+	return this.cachedDataChan
+}
+
+// Sender接口
+// Done：返回channel用于判断是否关闭
+func (this *baseSender) Done() <-chan struct{} {
+	return this.done
+}
--- a/trunk/goutil/ensureSendUtil/bytesSendUtil/dataItem.go
+++ b/trunk/goutil/ensureSendUtil/bytesSendUtil/dataItem.go
@@ -0,0 +1,101 @@
+package bytesSendUtil
+
+import (
+	"goutil/zlibUtil"
+)
+
+type dataItem interface {
+	// 返回原始数据
+	OriginData() []byte
+
+	// 返回发送字节流
+	Bytes() []byte
+
+	// 设置发送次数
+	SetCount(uint)
+
+	// 返回发送次数
+	Count() uint
+}
+
+/////////////////////////////////////////////////
+// httpDataItem
+
+type httpDataItem struct {
+	// 数据
+	data []byte
+
+	// 发送次数
+	count uint
+}
+
+func newHTTPData(_data []byte) dataItem {
+	return &httpDataItem{
+		data:  _data,
+		count: 0,
+	}
+}
+
+// 返回原始数据
+func (this *httpDataItem) OriginData() []byte {
+	return this.data
+}
+
+// 返回原始数据用于发送
+func (this *httpDataItem) Bytes() []byte {
+	return this.data
+}
+
+func (this *httpDataItem) SetCount(cnt uint) {
+	this.count = cnt
+}
+
+func (this *httpDataItem) Count() uint {
+	return this.count
+}
+
+/////////////////////////////////////////////////
+// tcpDataItem
+
+type tcpDataItem struct {
+	// 原始数据
+	origin []byte
+
+	// 压缩后数据
+	data []byte
+
+	// 重试次数
+	count uint
+}
+
+func newTCPDataItem(_data []byte) (dataItem, error) {
+	compressed, err := zlibUtil.Compress([]byte(_data), 5)
+	if err != nil {
+		return nil, err
+	}
+
+	item := &tcpDataItem{
+		origin: _data,
+		data:   compressed,
+		count:  0,
+	}
+	return item, nil
+}
+
+// 返回原始数据
+func (this *tcpDataItem) OriginData() []byte {
+	return this.origin
+}
+
+// 返回压缩数据用于发送
+func (this *tcpDataItem) Bytes() []byte {
+	return this.data
+}
+
+func (this *tcpDataItem) SetCount(cnt uint) {
+	this.count = cnt
+}
+
+func (this *tcpDataItem) Count() uint {
+	return this.count
+}
--- a/trunk/goutil/ensureSendUtil/bytesSendUtil/ensureSendUtil.go
+++ b/trunk/goutil/ensureSendUtil/bytesSendUtil/ensureSendUtil.go
@@ -0,0 +1,29 @@
+package bytesSendUtil
+
+/*
+ensureSendUtil 用于推送数据
+支持TCP和HTTP两种形式，在发送失败时会缓存数据，并在一定时间间隔后重试
+
+通过NewTCPSender和NewHTTPSender两个接口分别创建TCP和HTTP模式的EnsureSender
+
+type EnsureSender interface {
+    // 用于发送数据
+    Write([]byte) error
+
+    // 用于停止发送，此时会自动保存未发送数据
+    Close() error
+}
+
+// 创建一个tcp数据发送器
+// 参数：
+//      _dataFolder  数据存放目录
+//      _address     连接地址
+func NewTCPSender(_dataFolder, _address string) (EnsureSender, error) {
+
+
+// 创建一个http数据发送器
+// 参数：
+//      _dataFolder  数据存放目录
+//      _url         发送地址
+func NewHTTPSender(_dataFolder, _url string) (EnsureSender, error) {
+*/
--- a/trunk/goutil/ensureSendUtil/bytesSendUtil/ensureSender.go
+++ b/trunk/goutil/ensureSendUtil/bytesSendUtil/ensureSender.go
@@ -0,0 +1,24 @@
+package bytesSendUtil
+
+type EnsureSender interface {
+	// use Write to send data
+	Write([]byte) error
+
+	// stop sender
+	Close() error
+}
+
+// resend和dataSaver通过此接口调用tcpSender与httpSender
+type sender interface {
+	// 发送数据
+	Send(dataItem) error
+
+	// 返回待发送的数据channel
+	Data() <-chan dataItem
+
+	// 返回失败数据缓存channel
+	Cache() chan dataItem
+
+	// 用于判断是否关闭
+	Done() <-chan struct{}
+}
--- a/trunk/goutil/ensureSendUtil/bytesSendUtil/httpSender.go
+++ b/trunk/goutil/ensureSendUtil/bytesSendUtil/httpSender.go
@@ -0,0 +1,96 @@
+package bytesSendUtil
+
+import (
+	"fmt"
+
+	"goutil/webUtil"
+)
+
+// 实现 EnsureSender和sender接口
+type httpSender struct {
+	// 需要实现的接口
+	EnsureSender
+
+	// 包含sender接口部分实现
+	*baseSender
+
+	// 数据目录
+	dataFolder string
+
+	// 发送地址
+	url string
+
+	// 用于sendLoop和resendLoop发送退出信号
+	closeSignal chan struct{}
+}
+
+// 创建一个http数据发送器
+// 参数：
+//
+//	_dataFolder  数据存放目录
+//	_url         发送地址
+func NewHTTPSender(_dataFolder, _url string) (EnsureSender, error) {
+	this := &httpSender{
+		dataFolder:  _dataFolder,
+		url:         _url,
+		baseSender:  newBaseSender(),
+		closeSignal: make(chan struct{}),
+	}
+
+	// 新开协程发送数据
+	go sendLoop(this, this.closeSignal)
+
+	// 定时重发
+	go resendLoop(this, _dataFolder, this.closeSignal)
+
+	return this, nil
+}
+
+// EnsureSender接口
+// Write：写入数据
+func (this *httpSender) Write(data []byte) error {
+	item := newHTTPData(data)
+
+	this.waitingDataChan <- item
+
+	return nil
+}
+
+// EnsureSender接口
+// Close：关闭
+func (this *httpSender) Close() error {
+	close(this.done)
+
+	// 等待sendLoop和resendLoop退出
+	<-this.closeSignal
+	<-this.closeSignal
+
+	// 保存数据
+	_, e1 := saveData(this.Cache(), this.dataFolder)
+	_, e2 := saveData(this.Data(), this.dataFolder)
+
+	if e2 != nil {
+		if e1 != nil {
+			return fmt.Errorf("%s %s", e1, e2)
+		}
+		return e2
+	} else {
+		return e1
+	}
+}
+
+// sender接口
+// Send：发送数据
+func (this *httpSender) Send(item dataItem) error {
+	statusCode, _, err := webUtil.PostByteData2(this.url, item.Bytes(), nil, nil)
+	if err != nil || statusCode != 200 {
+		if err == nil {
+			err = fmt.Errorf("StatusCode is not 200")
+		}
+
+		// 发送失败时发送次数+1
+		item.SetCount(item.Count() + 1)
+	}
+
+	return err
+}
--- a/trunk/goutil/ensureSendUtil/bytesSendUtil/httpSender_test.go
+++ b/trunk/goutil/ensureSendUtil/bytesSendUtil/httpSender_test.go
@@ -0,0 +1,86 @@
+package bytesSendUtil
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"testing"
+	"time"
+
+	"goutil/debugUtil"
+)
+
+// 保存接收的数据用于校验
+var http_recv_msg = make([]byte, 0)
+
+func init() {
+	debugUtil.SetDebug(true)
+}
+
+type httpHandler struct {
+	cnt int
+}
+
+func (ctx *httpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+
+	defer r.Body.Close()
+	result, _ := ioutil.ReadAll(r.Body)
+
+	if string(result) == "http-msg-failed" {
+		http.NotFound(w, r)
+	} else {
+		ctx.cnt++
+		// 模拟一次失败
+		if ctx.cnt == 2 {
+			http.NotFound(w, r)
+		} else {
+			http_recv_msg = append(http_recv_msg, result...)
+		}
+	}
+
+}
+
+func Test_http(t *testing.T) {
+	http.Handle("/test", new(httpHandler))
+	go http.ListenAndServe("127.0.0.1:9560", nil)
+
+	httpSender, err := NewHTTPSender("./test_http", "http://127.0.0.1:9560/test")
+	if err != nil {
+		t.Error(err)
+	}
+
+	time.Sleep(time.Millisecond * 50)
+
+	// 第一次应该成功
+	httpSender.Write([]byte("http-msg-1"))
+
+	time.Sleep(time.Millisecond)
+
+	// 发送消息，此数据会多次失败，被丢弃到giveup目录
+	httpSender.Write([]byte("http-msg-failed"))
+
+	time.Sleep(time.Second * 4)
+
+	// 第二次应该失败
+	httpSender.Write([]byte("http-msg-2"))
+
+	time.Sleep(time.Millisecond)
+
+	// 保存数据
+	httpSender.Close()
+
+	// 重启之后应该会重发数据
+	httpSender, err = NewHTTPSender("./test_http", "http://127.0.0.1:9560/test")
+	if err != nil {
+		t.Error(err)
+	}
+	time.Sleep(time.Second * 2)
+
+	httpSender.Close()
+
+	if string(http_recv_msg) != "http-msg-1http-msg-2" {
+		t.Error("message error. got " + string(http_recv_msg))
+	} else {
+		fmt.Println("HTTP OK")
+	}
+}
--- a/trunk/goutil/ensureSendUtil/bytesSendUtil/saveData.go
+++ b/trunk/goutil/ensureSendUtil/bytesSendUtil/saveData.go
@@ -0,0 +1,58 @@
+package bytesSendUtil
+
+import (
+	"fmt"
+
+	"goutil/fileUtil"
+	"goutil/logUtil"
+	"goutil/stringUtil"
+)
+
+// 从目录加载缓存数据并发送
+func loadData(s EnsureSender, folder string) error {
+	if fileList, err := fileUtil.GetFileList(folder); err != nil {
+		return err
+	} else {
+		for _, filename := range fileList {
+			// 读取发送内容
+			if fileContent, err := fileUtil.ReadFileBytes(filename); err != nil {
+				// 打印错误
+				log := fmt.Sprintf("ensureSendUtil.loadData: Failed To Read File: %s %s\n", err, filename)
+				logUtil.NormalLog(log, logUtil.Error)
+			} else if err = fileUtil.DeleteFile(filename); err != nil {
+				// 删除文件，如果成功则将内容添加到通道中，否则不处理
+				log := fmt.Sprintf("ensureSendUtil.loadData: Failed To Delete File: %s %s", err, filename)
+				logUtil.NormalLog(log, logUtil.Error)
+			} else {
+				// 发送数据
+				s.Write(fileContent)
+			}
+		}
+	}
+
+	return nil
+}
+
+// 保存数据到文件中(通常在退出时调用)
+func saveData(datas <-chan dataItem, folder string) (failed []dataItem, err error) {
+	defer func() {
+		if len(failed) > 0 {
+			err = fmt.Errorf("保存数据时有%d个失败数据", len(failed))
+		}
+	}()
+
+	for {
+		select {
+		case v := <-datas:
+			filename := stringUtil.GetNewGUID()
+			if e := fileUtil.WriteFile4Byte(folder, filename, false, v.OriginData()); e != nil {
+				failed = append(failed, v)
+				log := fmt.Sprintf("ensureSendUtil.saveData: 写入错误\n目录：%s，文件：%s，错误信息为：%s, Data Len:%v",
+					folder, filename, err, len(v.OriginData()))
+				logUtil.NormalLog(log, logUtil.Error)
+			}
+		default:
+			return
+		}
+	}
+}
--- a/trunk/goutil/ensureSendUtil/bytesSendUtil/send.go
+++ b/trunk/goutil/ensureSendUtil/bytesSendUtil/send.go
@@ -0,0 +1,111 @@
+package bytesSendUtil
+
+import (
+	"fmt"
+	"time"
+
+	"goutil/debugUtil"
+	"goutil/logUtil"
+)
+
+// 负责发送数据的协程
+func sendLoop(s sender, closeSignal chan struct{}) {
+	defer func() {
+		if r := recover(); r != nil {
+			logUtil.LogUnknownError(r)
+		}
+	}()
+
+	for {
+		select {
+		case <-s.Done():
+			closeSignal <- struct{}{}
+			return
+		case v := <-s.Data():
+			if err := s.Send(v); err != nil {
+				// 发送失败存入缓存
+				s.Cache() <- v
+			}
+		}
+	}
+}
+
+// 定时重发失败的数据
+func resendLoop(s sender, folder string, closeSignal chan struct{}) {
+	defer func() {
+		if r := recover(); r != nil {
+			logUtil.LogUnknownError(r)
+		}
+	}()
+
+	// debug模式每秒重试1次
+	var delay time.Duration
+	if debugUtil.IsDebug() {
+		delay = time.Second
+	} else {
+		delay = time.Minute * 5
+	}
+
+	// 定时重发失败数据
+	for {
+		select {
+		case <-s.Done():
+			closeSignal <- struct{}{}
+			return
+		case <-time.After(delay):
+			sendCacheData(s, folder)
+			loadData(s.(EnsureSender), folder)
+		}
+	}
+}
+
+// 从sender获取失败数据重发
+func sendCacheData(s sender, folder string) {
+	failed := make([]dataItem, 0)
+	length := len(s.Cache())
+
+	defer func() {
+		// 用于记录多次失败后放弃发送的数据
+		giveUpItems := make(chan dataItem, len(failed))
+
+		for _, v := range failed {
+			if v.Count() >= 3 {
+				// 失败次数太多的数据准备存放到磁盘中
+				giveUpItems <- v
+			} else {
+				s.Cache() <- v
+			}
+		}
+
+		giveUpLen := len(giveUpItems)
+		if giveUpLen > 0 {
+			// 将多次失败的数据保存到文件中
+			if folder[len(folder)-1] == '/' {
+				folder = folder[:len(folder)-1]
+			}
+			saveData(giveUpItems, folder+"_giveup")
+
+			if giveUpLen >= 5 {
+				log := fmt.Sprintf("ensureSendUtil: 有%d条数据多次发送失败", giveUpLen)
+				logUtil.NormalLog(log, logUtil.Error)
+			}
+		}
+
+		// 输出信息
+		log := fmt.Sprintf("ensureSendUtil: 重发%d条数据，失败%d条，存盘%d条\n", length, len(failed), giveUpLen)
+		logUtil.NormalLog(log, logUtil.Info)
+	}()
+
+	for {
+		select {
+		case v := <-s.Cache():
+			// 重发数据
+			if e := s.Send(v); e != nil {
+				// 记录失败的数据
+				failed = append(failed, v)
+			}
+		default:
+			return
+		}
+	}
+}
--- a/trunk/goutil/ensureSendUtil/bytesSendUtil/tcpSender.go
+++ b/trunk/goutil/ensureSendUtil/bytesSendUtil/tcpSender.go
@@ -0,0 +1,208 @@
+package bytesSendUtil
+
+import (
+	"encoding/binary"
+	"fmt"
+	"net"
+	"sync"
+	"time"
+
+	"goutil/intAndBytesUtil"
+	"goutil/logUtil"
+)
+
+var (
+	errConnectEmpty = fmt.Errorf("scoket reconnecting...")
+	byterOrder      = binary.LittleEndian
+)
+
+// 实现 EnsureSender和sender接口
+type tcpSender struct {
+	// 需要实现的接口
+	EnsureSender
+
+	// 包含sender接口部分实现
+	*baseSender
+
+	// 数据目录
+	dataFolder string
+
+	// 服务器地址
+	address string
+
+	// 连接
+	conn net.Conn
+
+	// 用于重连时互斥
+	mutex sync.Mutex
+
+	// 用于sendLoop和resendLoop发送退出信号
+	closeSignal chan struct{}
+}
+
+// 创建一个tcp数据发送器
+// 参数：
+//
+//	_dataFolder  数据存放目录
+//	_address     连接地址
+func NewTCPSender(_dataFolder, _address string) (EnsureSender, error) {
+	// 连接服务器
+	conn, err := net.DialTimeout("tcp", _address, 5*time.Second)
+	if err != nil {
+		return nil, err
+	}
+
+	this := &tcpSender{
+		dataFolder:  _dataFolder,
+		baseSender:  newBaseSender(),
+		address:     _address,
+		conn:        conn,
+		closeSignal: make(chan struct{}),
+	}
+
+	// 新开协程发送数据
+	go sendLoop(this, this.closeSignal)
+
+	// 定时重发
+	go resendLoop(this, _dataFolder, this.closeSignal)
+
+	// 发送心跳包
+	go this.heartBeat()
+
+	return this, nil
+}
+
+// 每隔15秒发送心跳包
+func (this *tcpSender) heartBeat() {
+	defer func() {
+		if r := recover(); r != nil {
+			logUtil.LogUnknownError(r)
+		}
+	}()
+
+	tick := time.Tick(time.Second * 15)
+
+	for {
+		select {
+		case <-this.Done():
+			return
+		case <-tick:
+			this.sendBytes([]byte{})
+		}
+	}
+}
+
+// EnsureSender接口
+// Write：写入数据
+func (this *tcpSender) Write(data []byte) error {
+	item, err := newTCPDataItem(data)
+	if err != nil {
+		return err
+	}
+
+	this.waitingDataChan <- item
+
+	return nil
+}
+
+// EnsureSender接口
+// Close：关闭
+func (this *tcpSender) Close() error {
+	close(this.done)
+
+	// 关闭socket连接
+	conn := this.conn
+	if conn != nil {
+		conn.Close()
+	}
+
+	// 等待sendLoop和resendLoop退出
+	<-this.closeSignal
+	<-this.closeSignal
+
+	// 保存数据
+	_, e1 := saveData(this.Cache(), this.dataFolder)
+	_, e2 := saveData(this.Data(), this.dataFolder)
+
+	if e2 != nil {
+		if e1 != nil {
+			return fmt.Errorf("%s %s", e1, e2)
+		}
+		return e2
+	} else {
+		return e1
+	}
+}
+
+// Sender接口
+// Send：发送dataItem
+func (this *tcpSender) Send(item dataItem) error {
+	err := this.sendBytes(item.Bytes())
+	if err != nil && err != errConnectEmpty {
+		// 发送失败时发送次数+1
+		item.SetCount(item.Count() + 1)
+	}
+
+	return err
+}
+
+// 发送字节数据
+// 发送格式：[lenght+data]
+func (this *tcpSender) sendBytes(data []byte) error {
+	conn := this.conn
+	if conn == nil {
+		return errConnectEmpty
+	}
+
+	// 将长度转化为字节数组
+	header := intAndBytesUtil.Int32ToBytes(int32(len(data)), byterOrder)
+
+	if len(data) > 0 {
+		data = append(header, data...)
+	} else {
+		data = header
+	}
+
+	_, err := conn.Write(data)
+	if err != nil {
+		this.mutex.Lock()
+		// 发送失败
+		// 检查失败的conn是否this.conn（避免多个线程失败后均调用reconnect）
+		// 是则关闭并重连
+		if conn == this.conn {
+			this.conn.Close()
+			this.conn = nil
+			this.mutex.Unlock()
+
+			// 重连
+			go this.reconnect()
+		} else {
+			this.mutex.Unlock()
+		}
+	}
+
+	return err
+}
+
+// 重连服务器
+func (this *tcpSender) reconnect() error {
+	// lock-it
+	this.mutex.Lock()
+	defer this.mutex.Unlock()
+
+	for {
+		// 检查是否已经重连
+		if this.conn != nil {
+			return nil
+		}
+
+		conn, err := net.DialTimeout("tcp", this.address, 5*time.Second)
+		if err != nil {
+			// 连接失败，5秒后重试
+			<-time.After(time.Second * 5)
+			continue
+		}
+
+		this.conn = conn
+	}
+}
--- a/trunk/goutil/ensureSendUtil/bytesSendUtil/tcpSender_test.go
+++ b/trunk/goutil/ensureSendUtil/bytesSendUtil/tcpSender_test.go
@@ -0,0 +1,95 @@
+package bytesSendUtil
+
+import (
+	"fmt"
+	"net"
+	"testing"
+	"time"
+
+	"goutil/debugUtil"
+	"goutil/zlibUtil"
+)
+
+// 保存接收的数据用于校验
+var tcp_recv_msg = make([]byte, 0)
+
+func init() {
+	debugUtil.SetDebug(true)
+}
+
+// 创建socket服务器，保存收到的数据
+func server(addr string) net.Listener {
+	listener, err := net.Listen("tcp", addr)
+	if err != nil {
+		panic(err)
+	}
+
+	go func() {
+		for {
+			conn, err := listener.Accept()
+			if err != nil {
+				return
+			}
+
+			for {
+				buff := make([]byte, 512)
+				_, err := conn.Read(buff)
+				if err != nil {
+					break
+				} else {
+					decompressed, err := zlibUtil.Decompress(buff[4:])
+					if err != nil {
+						panic(err)
+					} else {
+						tcp_recv_msg = append(tcp_recv_msg, decompressed...)
+					}
+				}
+			}
+		}
+	}()
+
+	return listener
+}
+
+func Test_tcp(t *testing.T) {
+	// 开启服务器
+	l := server("127.0.0.1:9559")
+
+	tcp, err := NewTCPSender("./test_tcp", "127.0.0.1:9559")
+	if err != nil {
+		t.Error(err)
+	}
+
+	// 发送消息
+	tcp.Write([]byte("tcp-msg-1"))
+	time.Sleep(time.Millisecond * 50) // 等待协程发送数据
+
+	// 关闭连接和服务器
+	l.Close()
+	(tcp.(*tcpSender)).conn.Close()
+
+	// 发送消息，此数据会失败
+	tcp.Write([]byte("tcp-msg-2"))
+	// time.Sleep(time.Millisecond * 50)
+
+	// 保存数据
+	tcp.Close()
+
+	// 重启，检查是否重发tcp-msg-2
+	l = server("127.0.0.1:9559")
+	tcp, err = NewTCPSender("./test_tcp", "127.0.0.1:9559")
+	if err != nil {
+		t.Error(err)
+	}
+
+	time.Sleep(time.Second * 2)
+
+	if string(tcp_recv_msg) != "tcp-msg-1tcp-msg-2" {
+		t.Error("message error. got " + string(tcp_recv_msg))
+	} else {
+		fmt.Println("TCP OK")
+	}
+
+	tcp.Close()
+	l.Close()
+}