+ Hello,This is an example for gxpath. +
+ + + + ` + var root *xmlUtil.Node + root, errMsg = xmlUtil.LoadFromString(content) + if errMsg == nil { + xmlConfigData = NewXmlConfig() + xmlConfigData.LoadFromXmlNode(root) + } + + return +} diff --git a/trunk/goutil/configUtil/xmlConfig_test.go b/trunk/goutil/configUtil/xmlConfig_test.go new file mode 100644 index 0000000..6790f46 --- /dev/null +++ b/trunk/goutil/configUtil/xmlConfig_test.go @@ -0,0 +1,321 @@ +package configUtil + +import ( + "fmt" + "testing" + + "goutil/xmlUtil" +) + +// bool值读取测试 +func TestBool(t *testing.T) { + xmlConfigData, errMsg := getxmlConfigData() + if errMsg != nil { + t.Error(errMsg) + t.Fail() + + return + } + + var ispost bool + ispost, errMsg = xmlConfigData.Bool("html/body", "IsPost") + if errMsg != nil { + t.Error(errMsg) + t.Fail() + return + } + fmt.Println("读取到的值:", ispost) + if ispost == false { + t.Error("html/body的isPost读取错误") + t.Fail() + return + } + + ispost = xmlConfigData.DefaultBool("html/body", "IsPost", false) + if ispost == false { + t.Error("html/body的isPost读取错误") + t.Fail() + } +} + +// int值读取测试 +func TestInt(t *testing.T) { + xmlConfigData, errMsg := getxmlConfigData() + if errMsg != nil { + t.Error(errMsg) + t.Fail() + + return + } + + var id int + id, errMsg = xmlConfigData.Int("html/body/ul/li/a[@id=1]", "id") + if errMsg != nil { + t.Error(errMsg) + t.Fail() + return + } + + if id != 1 { + t.Errorf("html/body的isPost读取错误，读取到的值:%v", id) + t.Fail() + return + } + + id = xmlConfigData.DefaultInt("html/body", "id", 2) + if id != 2 { + t.Error("TestInt html/body的id读取错误") + t.Fail() + } +} + +// int64值读取测试 +func TestInt64(t *testing.T) { + xmlConfigData, errMsg := getxmlConfigData() + if errMsg != nil { + t.Error(errMsg) + t.Fail() + + return + } + + var id int64 + id, errMsg = xmlConfigData.Int64("html/body/ul/li/a[@id=1]", "id") + if errMsg != nil { + t.Error(errMsg) + t.Fail() + return + } + + if id != 1 { + t.Errorf("TestInt64 html/body/ul/li/a[@id=1]的id读取错误，读取到的值:%v", id) + t.Fail() + return + } + + id = xmlConfigData.DefaultInt64("html/body", "id", 2) + if id != 2 { + t.Error("TestInt64 html/body的id读取错误") + t.Fail() + } +} + +// Float值读取测试 +func TestFloat(t *testing.T) { + xmlConfigData, errMsg := getxmlConfigData() + if errMsg != nil { + t.Error(errMsg) + t.Fail() + + return + } + + var id float64 + id, errMsg = xmlConfigData.Float("html/body/ul/li/a[@id=1]", "dd") + if errMsg != nil { + t.Error(errMsg) + t.Fail() + return + } + + if id != 1.1 { + t.Errorf("TestFloat html/body/ul/li/a[@id=1]的id读取错误，读取到的值:%v", id) + t.Fail() + return + } + + id = xmlConfigData.DefaultFloat("html/body", "id", 2) + if id != 2 { + t.Error("TestFloat html/body的id读取错误") + t.Fail() + } +} + +// 字符串读取测试 +func TestString(t *testing.T) { + xmlConfigData, errMsg := getxmlConfigData() + if errMsg != nil { + t.Error(errMsg) + t.Fail() + + return + } + + var id string + id, errMsg = xmlConfigData.String("html/body/ul/li/a[@id=1]", "dd") + if errMsg != nil { + t.Error(errMsg) + t.Fail() + return + } + + if id != "1.1" { + t.Errorf("TestString html/body/ul/li/a[@id=1]的id读取错误，读取到的值:%v", id) + t.Fail() + return + } + + id = xmlConfigData.DefaultString("html/body", "id", "2") + if id != "2" { + t.Error("TestString html/body的id读取错误") + t.Fail() + } +} + +type HelloStruct struct { + // 连接字符串 + ConnectionString string + + // 最大开启连接数量 + MaxOpenConns int `xml:",attr"` + + // 最大空闲连接数量 + MaxIdleConns int `xml:",attr"` +} + +func (this *HelloStruct) Equal(other *HelloStruct) bool { + return this.MaxOpenConns == other.MaxOpenConns && this.MaxIdleConns == other.MaxIdleConns +} + +func TestUnmarshal(t *testing.T) { + data, _ := getxmlConfigData2(` ++ Hello,This is an example for gxpath. +
+ + + + ` + var root *xmlUtil.Node + root, errMsg = xmlUtil.LoadFromString(content) + if errMsg == nil { + xmlConfigData = NewXmlConfig() + xmlConfigData.LoadFromXmlNode(root) + } + + return +} + +func getxmlConfigData2(xml string) (xmlConfigData *XmlConfig, errMsg error) { + var root *xmlUtil.Node + root, errMsg = xmlUtil.LoadFromString(xml) + if errMsg == nil { + xmlConfigData = NewXmlConfig() + xmlConfigData.LoadFromXmlNode(root) + } + + return +} diff --git a/trunk/goutil/coroutine-timer/.gitignore b/trunk/goutil/coroutine-timer/.gitignore new file mode 100644 index 0000000..1a1566e --- /dev/null +++ b/trunk/goutil/coroutine-timer/.gitignore @@ -0,0 +1,2 @@ +Log/* +logs/* \ No newline at end of file diff --git a/trunk/goutil/coroutine-timer/cmd.go b/trunk/goutil/coroutine-timer/cmd.go new file mode 100644 index 0000000..1ddcb42 --- /dev/null +++ b/trunk/goutil/coroutine-timer/cmd.go @@ -0,0 +1,47 @@ +package coroutine_timer + +const ( + // 添加 + cmd_add = 1 + + // 删除 + cmd_del = 2 +) + +// cmdModel +// @description: 命令对象 +type cmdModel struct { + // cmd 指令 + cmd int + + // paramObj 指令参数 + paramObj interface{} + + // resObj 指令返回对象 + resObj interface{} + + // err 指令返回的错误 + err error + + // waitChan 等待channel + waitChan chan struct{} +} + +// newCmdModel +// @description: 创建cmd模型对象 +// parameter: +// @c:cmd命令 +// @po:参数 +// return: +// @*cmdModel: +func newCmdModel(c int, po interface{}) *cmdModel { + result := &cmdModel{ + cmd: c, + paramObj: po, + resObj: nil, + err: nil, + waitChan: make(chan struct{}, 1), + } + + return result +} diff --git a/trunk/goutil/coroutine-timer/model.go b/trunk/goutil/coroutine-timer/model.go new file mode 100644 index 0000000..58209b8 --- /dev/null +++ b/trunk/goutil/coroutine-timer/model.go @@ -0,0 +1,73 @@ +package coroutine_timer + +// timersModel +// @description: timer卡槽对象 +type timersModel struct { + timers map[string]*timerObj +} + +// newTimersModel +// @description: 构造timer卡槽对象 +// parameter: +// return: +// @*timersModel: +func newTimersModel() *timersModel { + return &timersModel{timers: map[string]*timerObj{}} +} + +// addTimer +// @description: 添加定时器 +// parameter: +// @receiver this: +// @t: +// return: +func (this *timersModel) addTimer(t *timerObj) { + this.timers[t.id] = t +} + +// delTimer +// @description: 删除定时器 +// parameter: +// @receiver this: +// @id: +// return: +func (this *timersModel) delTimer(id string) { + delete(this.timers, id) +} + +// exist +// @description: 判断id是否存在 +// parameter: +// @receiver this: +// @id: +// return: +// @exist: +func (this *timersModel) exist(id string) (exist bool) { + _, exist = this.timers[id] + return +} + +// getAllTimers +// @description: 获取所有定时器 +// parameter: +// @receiver this: +// return: +// @map[string]*timerObj: +func (this *timersModel) getAllTimers() map[string]*timerObj { + return this.timers +} + +// getAllTimers2 +// @description: 获取所有定时器 +// parameter: +// @receiver this: +// return: +// @result: +func (this *timersModel) getAllTimers2() (result []*timerObj) { + result = make([]*timerObj, 0, len(this.timers)) + for _, v := range this.timers { + result = append(result, v) + } + + return +} diff --git a/trunk/goutil/coroutine-timer/readme.md b/trunk/goutil/coroutine-timer/readme.md new file mode 100644 index 0000000..02eaad6 --- /dev/null +++ b/trunk/goutil/coroutine-timer/readme.md @@ -0,0 +1,20 @@ +coroutine-timer支持如下工作： +定时触发设定的回调，最小精度秒级 + +## 使用方式 + +### 增加回调 + +> 1. 导入包 +> 2. 调用AddTimerx添加定时回调，传入相关参数 + +ps: +> 1. AddTimer1，AddTimer2，AddTimer3是内部自动生成的id，内部保证唯一性。外部如果后续要删除该添加的timer，需要持有返回的id信息 +> 2. AddTimer4 需要外部传入id，外部需要保证id的唯一性。并且这个方法会在内部校验id是否已经存在，所以性能上会比其他AddTimer方法慢 + +### 删除回调 + +```go + DeleteTimer(id) +``` + diff --git a/trunk/goutil/coroutine-timer/timer-mgr.go b/trunk/goutil/coroutine-timer/timer-mgr.go new file mode 100644 index 0000000..5808105 --- /dev/null +++ b/trunk/goutil/coroutine-timer/timer-mgr.go @@ -0,0 +1,409 @@ +package coroutine_timer + +import ( + "fmt" + "math" + "time" + + "goutil/logUtil" + "goutil/stringUtil" +) + +const ( + // 启动暂停时间 + con_STAR_SLEEP_NUM = 3 + + // 秒级定时器卡槽数量 + con_SECOND_SLOT_NUM = 60 + + //分钟级定时器卡槽数量 + con_MINUTES_SLOT_NUM = 60 +) + +var ( + // 秒级定时器下标 + secIndex = 0 + + // 秒级定时器当前开始时间 + secondStarTime int64 + + // 秒级定时器槽 + secondsTimers [con_SECOND_SLOT_NUM]*timersModel + + // 分钟级定时器下标 + minIndex = 0 + + // 分钟级定时器当前开始时间 + minStarTime int64 + + // 分钟级定时器槽 + minutesTimers [con_MINUTES_SLOT_NUM]*timersModel + + // 其他定时器存放槽 + otherTimers *timersModel + + // 操作通道 + cmdChan chan *cmdModel +) + +func init() { + for i := 0; i < con_SECOND_SLOT_NUM; i++ { + secondsTimers[i] = newTimersModel() + } + for i := 0; i < con_MINUTES_SLOT_NUM; i++ { + minutesTimers[i] = newTimersModel() + } + + otherTimers = newTimersModel() + cmdChan = make(chan *cmdModel, 1000) + secondStarTime = time.Now().Unix() + minStarTime = secondStarTime + con_SECOND_SLOT_NUM + + go chanHandler() +} + +// AddTimer +// @description: 添加定时回调 +// parameter: +// +// @afterSecond:延后多少时间执行 +// @exfun:执行方法 +// @obj:执行传入的参数 +// +// return: +// +// @string: +func AddTimer(afterSecond int, exfun func(interface{}), obj interface{}) string { + tick := time.Now().Unix() + int64(afterSecond) + return AddTimer3(tick, exfun, obj) +} + +// AddTimer2 +// @description: 添加定时回调 +// parameter: +// +// @t:执行时间点 +// @exfun:执行方法 +// @obj:执行传入的参数 +// +// return: +// +// @string: +func AddTimer2(t time.Time, exfun func(interface{}), obj interface{}) string { + tick := t.Unix() + return AddTimer3(tick, exfun, obj) +} + +// AddTimer3 +// @description: 添加定时回调 +// parameter: +// +// @tick:执行时间点 +// @exfun:执行方法 +// @obj:执行传入的参数 +// +// return: +// +// @newId: +func AddTimer3(tick int64, exfun func(interface{}), obj interface{}) (newId string) { + newId = stringUtil.GetNewUUID() + newObj := newTimerObj(newId, tick, exfun, obj) + + cnm := newCmdModel(cmd_add, newObj) + cmdChan <- cnm + + return +} + +// AddTimer4 +// @description: 添加定时回调(此方法会在内部校验id，所以性能会比其他AddTimer方法低) +// parameter: +// +// @id:定时id(外部需要自行保证id唯一) +// @tick:执行时间点 +// @exfun:执行方法 +// @obj:执行传入的参数 +// +// return: +// +// @err: +func AddTimer4(id string, tick int64, exfun func(interface{}), obj interface{}) (err error) { + newObj := newTimerObj(id, tick, exfun, obj) + newObj.needCheckId = true + + // 加入处理队列 + cnm := newCmdModel(cmd_add, newObj) + cmdChan <- cnm + + // 等待处理结束 + <-cnm.waitChan + + // 返回处理结果 + err = cnm.err + + return +} + +// DeleteTimer +// @description: 删除定时器 +// parameter: +// +// @id: +// +// return: +func DeleteTimer(id string) { + cnm := newCmdModel(cmd_del, id) + cmdChan <- cnm +} + +// chanHandler +// @description: channel处理 +// parameter: +// return: +func chanHandler() { + defer func() { + if err := recover(); err != nil { + logUtil.ErrorLog("coroutine-timer.excute err:%s", err) + } + }() + + // 暂停一下再处理，避免启动立即处理，其他数据还没准备好 + time.Sleep(con_STAR_SLEEP_NUM * time.Second) + + at := time.After(time.Second * 1) + for { + select { + case cm := <-cmdChan: + switch cm.cmd { + case cmd_add: + cmdAdd(cm) + case cmd_del: + cmdDel(cm) + } + case <-at: + // byron:需要处理时间后调导致跳时间的问题：调整后应该马上执行的 + + // 计算需要执行的次数 + n := time.Now().Unix() - secondStarTime - int64(secIndex) + if n > 0 { + + // 执行对应次数的方法 --- 正常应该只执行1此，调时间后，此处会追时间 + var i int64 + for i = 0; i < n; i++ { + cmdRun() + } + } + + at = time.After(time.Second * 1) + } + } +} + +// cmdAdd +// @description: 添加定时器 +// parameter: +// +// @cm: +// +// return: +func cmdAdd(cm *cmdModel) { + newObj := cm.paramObj.(*timerObj) + if newObj.needCheckId && checkTimerExist(newObj.id) { + cm.err = fmt.Errorf("已经存在id=%s的timer", newObj.id) + cm.waitChan <- struct{}{} + return + } + + // 如果执行时间比当前时间小，则放入最近的调度卡槽，以便尽快执行 + tick := newObj.tick + if tick <= (secondStarTime + int64(secIndex)) { + tick = (secondStarTime + int64(secIndex)) + 1 + } + + // 落在秒钟级别定时器上 + if tick < (secondStarTime + con_SECOND_SLOT_NUM) { + index := (int)(tick - secondStarTime) + secondsTimers[index].addTimer(newObj) + cm.waitChan <- struct{}{} + return + } + + // 落在分钟级别定时器上 + if tick < (minStarTime + con_MINUTES_SLOT_NUM*con_SECOND_SLOT_NUM) { + index := (int)(tick-minStarTime) / con_SECOND_SLOT_NUM + minutesTimers[index].addTimer(newObj) + cm.waitChan <- struct{}{} + return + } + + //落在小时级别定时器上 + otherTimers.addTimer(newObj) + + // 返回操作完成 + cm.waitChan <- struct{}{} +} + +// cmdDel +// @description: 删除timer +// parameter: +// +// @cm: +// +// return: +func cmdDel(cm *cmdModel) { + id := cm.paramObj.(string) + + // 移除秒级别定时器 + for _, item := range secondsTimers { + item.delTimer(id) + } + + // 移除分种级定时器 + for _, item := range minutesTimers { + item.delTimer(id) + } + + // 移除时钟级定时器 + otherTimers.delTimer(id) + + // 返回操作完成 + cm.waitChan <- struct{}{} +} + +// cmdRun +// @description: 运行定时器 +// parameter: +// return: +func cmdRun() { + defer func() { + if err := recover(); err != nil { + logUtil.ErrorLog("coroutine-timer.inExcute err:%s", err) + } + }() + + // 执行秒级定时器 + timers := getSencondTimers() + if len(timers) == 0 { + return + } + + for _, t := range timers { + go safeRun(t) + } +} + +// checkTimerExist +// @description: 校验timer是否存在 +// parameter: +// +// @id:id +// +// return: +// +// @bool: +func checkTimerExist(id string) bool { + // 秒级别定时器检测 + for _, item := range secondsTimers { + if item.exist(id) { + return true + } + } + + // 分种级定时器检测 + for _, item := range minutesTimers { + if item.exist(id) { + return true + } + } + + // 时钟级定时器检测 + return otherTimers.exist(id) +} + +// getSencondTimers +// @description: 获取秒级定时器 +// parameter: +// return: +// +// @result: +func getSencondTimers() (result []*timerObj) { + // 获取对应slot里面的定时对象 + result = secondsTimers[secIndex].getAllTimers2() + secondsTimers[secIndex] = newTimersModel() + secIndex++ + + // 如果达到最大，则重新填装新的调度对象 + if secIndex == con_SECOND_SLOT_NUM { + secIndex = 0 + secondStarTime = secondStarTime + con_SECOND_SLOT_NUM + minTaskList := getMinutesTasks() + for _, t := range minTaskList { + index := t.tick - secondStarTime + secondsTimers[index].addTimer(t) + } + } + + return +} + +// getMinutesTasks +// @description: 获取分钟级定时器 +// parameter: +// return: +// +// @result: +func getMinutesTasks() (result []*timerObj) { + // 获取对应slot里面的定时对象 + result = minutesTimers[minIndex].getAllTimers2() + minutesTimers[minIndex] = newTimersModel() + minIndex++ + + // 如果达到最大，则重新填装新的调度对象 + if minIndex == con_MINUTES_SLOT_NUM { + reInputMin() + } + + return +} + +// reInputMin +// @description: 重新填入分钟级定时器 +// parameter: +// return: +func reInputMin() { + minIndex = 0 + minStarTime = minStarTime + con_MINUTES_SLOT_NUM*con_SECOND_SLOT_NUM + + delMap := make(map[string]struct{}) + for _, t := range otherTimers.getAllTimers() { + index := (t.tick - minStarTime) / con_SECOND_SLOT_NUM + if index > math.MaxInt || index >= con_MINUTES_SLOT_NUM { + continue + } + minutesTimers[index].addTimer(t) + delMap[t.id] = struct{}{} + } + + if len(delMap) > 0 { + for k := range delMap { + otherTimers.delTimer(k) + } + } +} + +// safeRun +// @description: 安全运行定时器回调 +// parameter: +// +// @t: +// +// return: +func safeRun(t *timerObj) { + defer func() { + if err := recover(); err != nil { + logUtil.ErrorLog("coroutine-timer.safeRun id:%s err:%s", t.id, err) + } + }() + + t.excuteAction(t.paramObj) +} diff --git a/trunk/goutil/coroutine-timer/timer-obj.go b/trunk/goutil/coroutine-timer/timer-obj.go new file mode 100644 index 0000000..a60cc69 --- /dev/null +++ b/trunk/goutil/coroutine-timer/timer-obj.go @@ -0,0 +1,40 @@ +package coroutine_timer + +// timerObj +// @description: 定时调度对象 +type timerObj struct { + // id 调度id + id string + + // tick 执行时间 + tick int64 + + // excuteAction 执行方法 + excuteAction func(interface{}) + + // paramObj 携带的参数 + paramObj interface{} + + // needCheckId 是否需要校验id + needCheckId bool +} + +// newTimerObj +// @description: 构造调度对象 +// parameter: +// @_id:id +// @t:调度时间 +// @ea:调度方法 +// @pm:调度参数 +// return: +// @*timerObj: +func newTimerObj(_id string, t int64, ea func(interface{}), pm interface{}) *timerObj { + result := &timerObj{ + id: _id, + tick: t, + excuteAction: ea, + paramObj: pm, + } + + return result +} diff --git a/trunk/goutil/coroutine-timer/timer_test.go b/trunk/goutil/coroutine-timer/timer_test.go new file mode 100644 index 0000000..869b391 --- /dev/null +++ b/trunk/goutil/coroutine-timer/timer_test.go @@ -0,0 +1,114 @@ +package coroutine_timer + +import ( + "sync" + "testing" + "time" + + "goutil/mathUtil" + "goutil/stringUtil" +) + +func init() { +} + +func Test_Method1(t *testing.T) { + imap := make(map[int]struct{}) + var lockObj sync.Mutex + + cb := func(obj interface{}) { + i := obj.(int) + + lockObj.Lock() + defer lockObj.Unlock() + + if _, exist := imap[i]; exist == false { + t.Error(i, "应该删除，不应该回调 Test_Method1") + } + + delete(imap, i) + } + + for i := 0; i < 20000; i++ { + tick := i % 20 + isdel := false + if tick > 1 { + isdel = mathUtil.GetRand().GetRandInt(100) < 50 + } + if isdel == false { + lockObj.Lock() + imap[i] = struct{}{} + lockObj.Unlock() + } + id := AddTimer(tick, cb, i) + if isdel { + DeleteTimer(id) + } + } + + newN := 10000000 + newId := stringUtil.GetNewUUID() + + lockObj.Lock() + imap[newN] = struct{}{} + lockObj.Unlock() + + err := AddTimer4(newId, 3, cb, newN) + if err != nil { + t.Error(err) + } + + err = AddTimer4(newId, 3, cb, newN) + if err == nil { + t.Error("未检测到重复id") + } + + for { + if len(imap) == 0 { + break + } + + t.Log("剩余回调次数：", len(imap)) + time.Sleep(time.Second) + } +} + +func Test_Method2(t *testing.T) { + imap := make(map[int64]struct{}) + var lockObj sync.Mutex + + cb := func(obj interface{}) { + i := obj.(int64) + n := time.Now().Unix() + x := n - i + // 此处因为启动有暂停5s，所以启动后最近的执行偏差在5s内 + if x > 6 || x < -6 { + t.Errorf("错误的时间执行了回调函数 tick:%v now:%v", i, n) + } + + lockObj.Lock() + defer lockObj.Unlock() + + if _, exist := imap[i]; exist == false { + t.Error(i, "应该删除，不应该回调 Test_Method2") + } + + delete(imap, i) + } + + for i := 0; i < 20; i++ { + tick := time.Now().Unix() + int64(i) + imap[tick] = struct{}{} + AddTimer3(tick, cb, tick) + } + + for { + if len(imap) == 0 { + break + } + + t.Log("剩余回调次数：", len(imap)) + time.Sleep(time.Second) + } + +} diff --git a/trunk/goutil/counter-util/counter.go b/trunk/goutil/counter-util/counter.go new file mode 100644 index 0000000..4250b89 --- /dev/null +++ b/trunk/goutil/counter-util/counter.go @@ -0,0 +1,60 @@ +package counter_util + +import ( + "time" +) + +// CounterUtil +// @description: 固定窗口计数器辅助类 +type CounterUtil struct { + tag string // tag counter标识 + num int // num 当前计数 + warnNum int // warnNum 警告数量 + windowsTime time.Time // windowsTime 窗口时间 + checkSameWindowsFun func(t1, t2 time.Time) bool // checkSameWindowsFun 比较是否同一个时间窗口 + warnAction func(tag string, num int, t time.Time) // warnAction 监控回调方法 +} + +// NewCounterUtil +// @description: 构造计数器 +// parameter: +// @_tag:tag标识，会在WarnAction中传递回来 +// @_warnNum:告警数量 +// @_checkWindowFun:比较是否同一个时间窗口 +// @_warnAction:指定数量触发回调 +// return: +// @*CounterUtil:固定窗口计数器辅助类 +func NewCounterUtil(_tag string, _warnNum int, _checkWindowFun func(t1, t2 time.Time) bool, _warnAction func(tag string, num int, t time.Time)) *CounterUtil { + r := &CounterUtil{ + tag: _tag, + warnNum: _warnNum, + windowsTime: time.Now(), + checkSameWindowsFun: _checkWindowFun, + warnAction: _warnAction, + } + + return r +} + +// AddNum +// @description: 添加数量 +// parameter: +// @receiver c:计数器 +// @n:增加的数量 +// return: +// @int:计数器当前的数量 +func (c *CounterUtil) AddNum(n int) int { + if !c.checkSameWindowsFun(c.windowsTime, time.Now()) { + c.num = 0 + c.windowsTime = time.Now() + } + + // 增加次数 + c.num += n + if c.num >= c.warnNum { + c.warnAction(c.tag, c.num, time.Now()) + c.num = 0 + } + + return c.num +} diff --git a/trunk/goutil/counter-util/counter_test.go b/trunk/goutil/counter-util/counter_test.go new file mode 100644 index 0000000..0bcfe2e --- /dev/null +++ b/trunk/goutil/counter-util/counter_test.go @@ -0,0 +1,33 @@ +package counter_util + +import ( + "fmt" + "testing" + "time" +) + +func TestInfoLog(t *testing.T) { + var iserr bool = true + c := NewCounterUtil("test", 2, checkId, func(tag string, num int, ti time.Time) { + msg := fmt.Sprintf("tag:%s 当前数量为num:%v ti:%v", tag, num, ti) + if iserr { + t.Error(msg) + } else { + t.Log(msg) + } + + }) + + c.AddNum(1) + iserr = false + c.AddNum(1) + time.Sleep(time.Second * 1) + iserr = true + c.AddNum(1) + time.Sleep(time.Second * 1) + c.AddNum(1) +} + +func checkId(t1, t2 time.Time) bool { + return t1.Second() == t2.Second() +} diff --git a/trunk/goutil/counter-util/readme.md b/trunk/goutil/counter-util/readme.md new file mode 100644 index 0000000..316ff00 --- /dev/null +++ b/trunk/goutil/counter-util/readme.md @@ -0,0 +1,35 @@ +### 窗口周期计数器 +窗口周期计数类，用于记录一个窗口周期数量，并且触发某个操作的场景。 +在下一个窗口周期会自动重置次数 + +#### =======================>使用方法说明<========================= + +1.引入包 +2.构造对象并次有 +3.调用对象的增加次数方法 + +```go +package demo + +import ( + "time" + + "goutil/counter_util" +) + +func main() { + // 构造名字叫test的，窗口间隔为1s，计数达到2就会触发警告的窗口计数器 + c := counter_util.NewCounterUtil("test", 2, checkId, func(tag string, num int, ti time.Time) { + //自定义触发动作 + }) + + c.AddNum(1) + c.AddNum(10) +} + +// 窗口周期设定为1s +func checkId(t1, t2 time.Time) bool { + return t1.Second() == t2.Second() +} + +``` \ No newline at end of file diff --git a/trunk/goutil/dbUtil/dataRow.go b/trunk/goutil/dbUtil/dataRow.go new file mode 100644 index 0000000..36cc8d9 --- /dev/null +++ b/trunk/goutil/dbUtil/dataRow.go @@ -0,0 +1,104 @@ +package dbUtil + +import ( + "errors" + "time" +) + +// 数据行结果 +type DataRow struct { + // 所属数据表 + table *DataTable + + // 行的所有值 + cells []interface{} +} + +// 行的所有原始值 +func (this *DataRow) CellOriginValues() []interface{} { + return this.cells +} + +// 值的个数 +func (this *DataRow) Len() int { + return len(this.cells) +} + +// 单元格的字符串值（可能为nil）,如果有设置连接字符串：parseTime=true，则会有time.Time +// celIndex:单元格序号 +// 返回值: +// interface{}:单元格的字符串值 +// error:错误信息 +func (this *DataRow) CellValue(celIndex int) (interface{}, error) { + if len(this.cells) <= celIndex { + return nil, errors.New("cell out of range") + } + + // 检查是否为nil + if this.cells[celIndex] == nil { + return nil, nil + } + + // 转换为字符串 + switch this.cells[celIndex].(type) { + case []byte: + return string(this.cells[celIndex].([]byte)), nil + case string: + return this.cells[celIndex].(string), nil + case time.Time: + return this.cells[celIndex].(time.Time), nil + } + + return nil, errors.New("unknown value type") +} + +// 单元格的字符串值（可能为nil）,如果有设置连接字符串：parseTime=true，则会有time.Time +// cellName:单元格名称 +// 返回值: +// interface{}:单元格的字符串值 +// error:错误信息 +func (this *DataRow) CellValueByName(cellName string) (interface{}, error) { + celIndex := this.table.cellIndex(cellName) + if celIndex < 0 { + return nil, errors.New("cell name no exist") + } + + return this.CellValue(celIndex) +} + +// 单元格的原始值 +// celIndex:单元格序号 +// 返回值: +// interface{}:单元格的字符串值 +// error:错误信息 +func (this *DataRow) OriginCellValue(celIndex int) (interface{}, error) { + if len(this.cells) <= celIndex { + return nil, errors.New("cell out of range") + } + + return this.cells[celIndex], nil +} + +// 单元格的原始值 +// cellName:单元格名称 +// 返回值: +// interface{}:单元格的字符串值 +// error:错误信息 +func (this *DataRow) OriginCellValueByName(cellName string) (interface{}, error) { + celIndex := this.table.cellIndex(cellName) + if celIndex < 0 { + return nil, errors.New("cell name no exist") + } + + return this.OriginCellValue(celIndex) +} + +// 创建单元格对象 +// _table:所属表对象 +// _cells:单元格的值集合 +func newDataRow(_table *DataTable, _cells []interface{}) *DataRow { + return &DataRow{ + table: _table, + cells: _cells, + } +} diff --git a/trunk/goutil/dbUtil/dataTable.go b/trunk/goutil/dbUtil/dataTable.go new file mode 100644 index 0000000..6e7f3c5 --- /dev/null +++ b/trunk/goutil/dbUtil/dataTable.go @@ -0,0 +1,189 @@ +package dbUtil + +import ( + "database/sql" + "errors" +) + +// 数据表结构 +type DataTable struct { + // 行对象集合 + rowData []*DataRow + + // 列名称集合 + columnNames map[string]int +} + +// 数据表初始化 +// rows:原始的数据行信息 +// 返回值： +// error:初始化的错误信息 +func (this *DataTable) init(rows *sql.Rows) error { + defer func() { + rows.Close() + }() + + // 读取列信息和保存列名称 + tmpColumns, errMsg := rows.Columns() + if errMsg != nil { + return errMsg + } + this.columnNames = make(map[string]int) + for index, val := range tmpColumns { + this.columnNames[val] = index + } + + // 读取行数据 + this.rowData = make([]*DataRow, 0) + columnCount := len(this.columnNames) + + args := make([]interface{}, columnCount) + for rows.Next() { + values := make([]interface{}, columnCount) + for i := 0; i < columnCount; i++ { + args[i] = &values[i] + } + rows.Scan(args...) + + this.rowData = append(this.rowData, newDataRow(this, values)) + } + + return nil +} + +// 获取原始单元格值（一般为:string或[]byte） +// rowIndex:行序号 +// cellIndex:单元格序号 +// 返回值: +// interface{}:原始单元格值（一般为:string或[]byte） +// error:获取错误信息 +func (this *DataTable) OriginCellValueByIndex(rowIndex int, cellIndex int) (interface{}, error) { + if len(this.rowData) <= rowIndex { + return nil, errors.New("row out of range") + } + + rowItem := this.rowData[rowIndex] + if len(rowItem.cells) <= cellIndex { + return nil, errors.New("column out of range") + } + + return rowItem.OriginCellValue(cellIndex) +} + +// 获取原始单元格值（一般为:string或[]byte） +// rowIndex:行序号 +// cellIndex:单元格序号 +// 返回值: +// interface{}:原始单元格值（一般为:string或[]byte） +// error:获取错误信息 +func (this *DataTable) OriginCellValueByCellName(rowIndex int, cellName string) (interface{}, error) { + if len(this.rowData) <= rowIndex { + return nil, errors.New("row out of range") + } + + rowItem := this.rowData[rowIndex] + + return rowItem.OriginCellValueByName(cellName) +} + +// 获取字符串的单元格值（有可能为nil） +// rowIndex:行序号 +// cellIndex:单元格序号 +// 返回值: +// interface{}:字符串的单元格值（有可能为nil） +// error:获取错误信息 +func (this *DataTable) CellValueByIndex(rowIndex int, cellIndex int) (interface{}, error) { + if len(this.rowData) <= rowIndex { + return nil, errors.New("row out of range") + } + + rowItem := this.rowData[rowIndex] + if len(rowItem.cells) <= cellIndex { + return nil, errors.New("column out of range") + } + + return rowItem.CellValue(cellIndex) +} + +// 获取字符串的单元格值（有可能为nil） +// rowIndex:行序号 +// cellIndex:单元格序号 +// 返回值: +// interface{}:字符串的单元格值（有可能为nil） +// error:获取错误信息 +func (this *DataTable) CellValueByName(rowIndex int, cellName string) (interface{}, error) { + if len(this.rowData) <= rowIndex { + return nil, errors.New("row out of range") + } + + rowItem := this.rowData[rowIndex] + + return rowItem.CellValueByName(cellName) +} + +// 获取行对象 +// rowIndex:行序号 +// 返回值: +// *DataRow:行对象 +// error:错误信息 +func (this *DataTable) Row(rowIndex int) (*DataRow, error) { + if len(this.rowData) <= rowIndex { + return nil, errors.New("row out of range") + } + + return this.rowData[rowIndex], nil +} + +// 根据列名获取列序号 +// cellName:列名 +// 返回值: +// int:列序号 +func (this *DataTable) cellIndex(cellName string) int { + cellIndex, isExist := this.columnNames[cellName] + if isExist == false { + return -1 + } + + return cellIndex +} + +// 获取所有列的名字 +// 返回值: +// []string:列字段名集合 +func (this *DataTable) Columns() []string { + result := make([]string, len(this.columnNames)) + for key, val := range this.columnNames { + result[val] = key + } + + return result +} + +// 获取列数量 +// 返回值: +// int:列数量 +func (this *DataTable) ColumnCount() int { + return len(this.columnNames) +} + +// 获取数据行数 +// 返回值: +// int:行数 +func (this *DataTable) RowCount() int { + return len(this.rowData) +} + +// 新建数据表对象 +// rows:数据行对象 +// 返回值： +// *DataTable:数据表对象 +// error:错误信息 +func NewDataTable(rows *sql.Rows) (*DataTable, error) { + table := &DataTable{} + errMsg := table.init(rows) + if errMsg != nil { + return nil, errMsg + } + + return table, nil +} diff --git a/trunk/goutil/dbUtil/valueConvert.go b/trunk/goutil/dbUtil/valueConvert.go new file mode 100644 index 0000000..3d0d496 --- /dev/null +++ b/trunk/goutil/dbUtil/valueConvert.go @@ -0,0 +1,195 @@ +package dbUtil + +import ( + "fmt" + "time" + + "goutil/typeUtil" +) + +// 类型转换为byte +// 返回值: +// byte:结果 +// error:错误数据 +func Byte(row *DataRow, key string) (byte, error) { + val, errMsg := row.CellValueByName(key) + if errMsg != nil { + return 0, errMsg + } + + if val == nil { + return 0, fmt.Errorf("value is nil") + } + + return typeUtil.Byte(val) +} + +// 类型转换为int +// 返回值: +// int:结果 +// error:错误数据 +func Int32(row *DataRow, key string) (int32, error) { + val, errMsg := row.CellValueByName(key) + if errMsg != nil { + return 0, errMsg + } + + if val == nil { + return 0, fmt.Errorf("value is nil") + } + + return typeUtil.Int32(val) +} + +// 类型转换为uint32 +// 返回值: +// int:结果 +// error:错误数据 +func Uint32(row *DataRow, key string) (uint32, error) { + val, errMsg := row.CellValueByName(key) + if errMsg != nil { + return 0, errMsg + } + + if val == nil { + return 0, fmt.Errorf("value is nil") + } + + return typeUtil.Uint32(val) +} + +// 类型转换为int +// 返回值: +// int:结果 +// error:错误数据 +func Int(row *DataRow, key string) (int, error) { + val, errMsg := row.CellValueByName(key) + if errMsg != nil { + return 0, errMsg + } + + if val == nil { + return 0, fmt.Errorf("value is nil") + } + + return typeUtil.Int(val) +} + +// 类型转换为int +// 返回值: +// int:结果 +// error:错误数据 +func Uint(row *DataRow, key string) (uint, error) { + val, errMsg := row.CellValueByName(key) + if errMsg != nil { + return 0, errMsg + } + + if val == nil { + return 0, fmt.Errorf("value is nil") + } + + return typeUtil.Uint(val) +} + +// 类型转换为int +// 返回值: +// int:结果 +// error:错误数据 +func Int64(row *DataRow, key string) (int64, error) { + val, errMsg := row.CellValueByName(key) + if errMsg != nil { + return 0, errMsg + } + + if val == nil { + return 0, fmt.Errorf("value is nil") + } + + return typeUtil.Int64(val) +} + +// 类型转换为int +// 返回值: +// int:结果 +// error:错误数据 +func Uint64(row *DataRow, key string) (uint64, error) { + val, errMsg := row.CellValueByName(key) + if errMsg != nil { + return 0, errMsg + } + + if val == nil { + return 0, fmt.Errorf("value is nil") + } + + return typeUtil.Uint64(val) +} + +// 类型转换为int +// 返回值: +// float64:结果 +// error:错误数据 +func Float64(row *DataRow, key string) (float64, error) { + val, errMsg := row.CellValueByName(key) + if errMsg != nil { + return 0, errMsg + } + + if val == nil { + return 0, fmt.Errorf("value is nil") + } + + return typeUtil.Float64(val) +} + +// 类型转换为bool +// 返回值: +// bool:结果 +// error:错误信息 +func Bool(row *DataRow, key string) (bool, error) { + val, errMsg := row.CellValueByName(key) + if errMsg != nil { + return false, errMsg + } + + if val == nil { + return false, fmt.Errorf("value is nil") + } + + return typeUtil.Bool(val) +} + +// 类型转换为字符串 +// 返回值: +// string:结果 +// error:错误信息 +func String(row *DataRow, key string) (string, error) { + val, errMsg := row.CellValueByName(key) + if errMsg != nil { + return "", errMsg + } + + if val == nil { + return "", fmt.Errorf("value is nil") + } + + return typeUtil.String(val) +} + +// 转换为时间格式，如果是字符串，则要求内容格式形如：2017-02-14 05:20:00 +// 返回值: +// bool:结果 +// error:错误信息 +func DateTime(row *DataRow, key string) (time.Time, error) { + val, errMsg := row.CellValueByName(key) + if errMsg != nil { + return time.Time{}, errMsg + } + + if val == nil { + return time.Time{}, fmt.Errorf("value is nil") + } + + return typeUtil.DateTime(val) +} diff --git a/trunk/goutil/debugUtil/debug.go b/trunk/goutil/debugUtil/debug.go new file mode 100644 index 0000000..0b20060 --- /dev/null +++ b/trunk/goutil/debugUtil/debug.go @@ -0,0 +1,66 @@ +package debugUtil + +import ( + "github.com/fatih/color" +) + +var ( + isDebug = false + code Code = Code_Bold + foregroundColor ForegroundColor = Foreground_Purple + backgroundColor BackgroundColor = BackgroundColor_Black + + colorObj = color.New(foregroundColor, code) +) + +// 设置DEBUG状态 +// _isDebug：是否是DEBUG +func SetDebug(_isDebug bool) { + isDebug = _isDebug +} + +// 是否处于调试状态 +func IsDebug() bool { + return isDebug +} + +// 设置显示信息 +func SetDisplayInfo(_code Code, _foregroundColor ForegroundColor, _backgroundColor BackgroundColor) { + code = _code + foregroundColor = _foregroundColor + backgroundColor = _backgroundColor + + colorObj = color.New(foregroundColor, code) +} + +// Print formats using the default formats for its operands and writes to standard output. +// Spaces are added between operands when neither is a string. +// It returns the number of bytes written and any write error encountered. +func Print(a ...interface{}) { + if !isDebug { + return + } + + _, _ = colorObj.Print(a...) +} + +// Printf formats according to a format specifier and writes to standard output. +// It returns the number of bytes written and any write error encountered. +func Printf(format string, a ...interface{}) { + if !isDebug { + return + } + + _, _ = colorObj.Printf(format, a...) +} + +// Println formats using the default formats for its operands and writes to standard output. +// Spaces are always added between operands and a newline is appended. +// It returns the number of bytes written and any write error encountered. +func Println(a ...interface{}) { + if !isDebug { + return + } + + _, _ = colorObj.Println(a...) +} diff --git a/trunk/goutil/debugUtil/display.go b/trunk/goutil/debugUtil/display.go new file mode 100644 index 0000000..071d1a4 --- /dev/null +++ b/trunk/goutil/debugUtil/display.go @@ -0,0 +1,47 @@ +package debugUtil + +import "github.com/fatih/color" + +// Code 显示代码 +type Code = color.Attribute + +const ( + Code_Reset Code = color.Reset + Code_Bold Code = color.Bold + Code_Faint Code = color.Faint + Code_Italic Code = color.Italic + Code_Underline Code = color.Underline + Code_BlinkSlow Code = color.BlinkSlow + Code_BlinkRapid Code = color.BlinkRapid + Code_ReverseVideo Code = color.ReverseVideo + Code_Concealed Code = color.Concealed + Code_CrossedOut Code = color.CrossedOut +) + +// ForegroundColor 前景色 +type ForegroundColor = color.Attribute + +const ( + Foreground_Black ForegroundColor = color.FgBlack + Foreground_Red ForegroundColor = color.FgRed + Foreground_Green ForegroundColor = color.FgGreen + Foreground_Yellow ForegroundColor = color.FgYellow + Foreground_Blue ForegroundColor = color.FgBlue + Foreground_Purple ForegroundColor = color.FgMagenta + Foreground_Cyan ForegroundColor = color.FgCyan + Foreground_White ForegroundColor = color.FgWhite +) + +// BackgroundColor 背景色 +type BackgroundColor = color.Attribute + +const ( + BackgroundColor_Black = color.BgBlack + BackgroundColor_Red = color.BgRed + BackgroundColor_Green = color.BgGreen + BackgroundColor_Yellow = color.BgYellow + BackgroundColor_Blue = color.BgBlue + BackgroundColor_Purple = color.BgMagenta + BackgroundColor_Cyan = color.BgCyan + BackgroundColor_White = color.BgWhite +) diff --git a/trunk/goutil/debugUtil/doc.go b/trunk/goutil/debugUtil/doc.go new file mode 100644 index 0000000..6e476fb --- /dev/null +++ b/trunk/goutil/debugUtil/doc.go @@ -0,0 +1,4 @@ +/* +提供调试功能的助手包 +*/ +package debugUtil diff --git a/trunk/goutil/deviceUtil/deviceUtil.go b/trunk/goutil/deviceUtil/deviceUtil.go new file mode 100644 index 0000000..28fa721 --- /dev/null +++ b/trunk/goutil/deviceUtil/deviceUtil.go @@ -0,0 +1,81 @@ +package deviceUtil + +import ( + "strings" +) + +// 将MAC地址转化为标准格式 +func ConvertMacToStandardFormat(mac string) string { + if mac == "" || mac == "00:00:00:00:00:00" || mac == "02:00:00:00:00:00" { + return "" + } + + //如果mac的长度不为12或17，则是不正确的格式 + if len(mac) != 12 && len(mac) != 17 { + return "" + } + + + //转化为大写 + mac = strings.ToUpper(mac) + + //如果mac地址的长度为17（已经有:），则直接返回 + if len(mac) == 17 { + return mac + } + + //如果没有分隔符，则添加分隔符 + newMac := make([]rune, 0, 17) + for i, v := range []rune(mac) { + newMac = append(newMac, v) + if i < len(mac) - 1 && i % 2 == 1 { + newMac = append(newMac, ':') + } + } + + return string(newMac) +} + +func ConvertIdfaToStandardFormat(idfa string) string { + //如果是空或默认值，则返回String.Empty + if idfa == "" || idfa == "00000000-0000-0000-0000-000000000000" { + return "" + } + + //如果idfa的长度不为32或36，则代表是Android的数据，则可以直接返回 + if len(idfa) != 32 && len(idfa) != 36 { + return idfa + } + + //转化为大写 + idfa = strings.ToUpper(idfa); + + //如果idfa地址的长度为36（已经有:），则直接返回 + if len(idfa) == 36 { + return idfa + } + + //如果没有分隔符，则添加分隔符 + newIdfa := make([]rune, 0, 36) + for i, v := range []rune(idfa) { + newIdfa = append(newIdfa, v) + if i == 7 || i == 11 || i == 15 || i == 19 { + newIdfa = append(newIdfa, '-') + } + } + + return string(newIdfa) +} + +// 根据MAC和IDFA获取唯一标识 +func GetIdentifier(mac, idfa string) string { + mac = ConvertMacToStandardFormat(mac) + idfa = ConvertIdfaToStandardFormat(idfa); + + //如果idfa不为空，则使用idfa，否则使用mac + if idfa != "" { + return idfa + } else { + return mac + } +} diff --git a/trunk/goutil/deviceUtil/deviceUtil_test.go b/trunk/goutil/deviceUtil/deviceUtil_test.go new file mode 100644 index 0000000..271273f --- /dev/null +++ b/trunk/goutil/deviceUtil/deviceUtil_test.go @@ -0,0 +1,131 @@ +package deviceUtil + +import ( + "testing" +) + +func TestConvertMacToStarndardFormat(t *testing.T) { + mac := "" + expected := "" + got := ConvertMacToStandardFormat(mac) + if got != expected { + t.Errorf("Expected: %s, but got:%s", expected, got) + } + + mac = "00:00:00:00:00:00" + expected = "" + got = ConvertMacToStandardFormat(mac) + if got != expected { + t.Errorf("Expected: %s, but got:%s", expected, got) + } + + mac = "02:00:00:00:00:00" + expected = "" + got = ConvertMacToStandardFormat(mac) + if got != expected { + t.Errorf("Expected: %s, but got:%s", expected, got) + } + + mac = "02:00:00:00:00" + expected = "" + got = ConvertMacToStandardFormat(mac) + if got != expected { + t.Errorf("Expected: %s, but got:%s", expected, got) + } + + mac = "020000000020" + expected = "02:00:00:00:00:20" + got = ConvertMacToStandardFormat(mac) + if got != expected { + t.Errorf("Expected: %s, but got:%s", expected, got) + } + + mac = "02:00:00:00:00:20" + expected = "02:00:00:00:00:20" + got = ConvertMacToStandardFormat(mac) + if got != expected { + t.Errorf("Expected: %s, but got:%s", expected, got) + } +} + +func TestConvertIdfaToStandardFormat(t *testing.T) { + idfa := "" + expected := "" + got := ConvertIdfaToStandardFormat(idfa) + if got != expected { + t.Errorf("Expected: %s, but got:%s", expected, got) + } + + idfa = "00000000-0000-0000-0000-000000000000" + expected = "" + got = ConvertIdfaToStandardFormat(idfa) + if got != expected { + t.Errorf("Expected: %s, but got:%s", expected, got) + } + + idfa = "00000000-0000-0000-0000-000000000000-123" + expected = "00000000-0000-0000-0000-000000000000-123" + got = ConvertIdfaToStandardFormat(idfa) + if got != expected { + t.Errorf("Expected: %s, but got:%s", expected, got) + } + + idfa = "00000000-1234-5678-0000-000000000000" + expected = "00000000-1234-5678-0000-000000000000" + got = ConvertIdfaToStandardFormat(idfa) + if got != expected { + t.Errorf("Expected: %s, but got:%s", expected, got) + } + + idfa = "00000000123456780000000000000000" + expected = "00000000-1234-5678-0000-000000000000" + got = ConvertIdfaToStandardFormat(idfa) + if got != expected { + t.Errorf("Expected: %s, but got:%s", expected, got) + } +} + +func TestGetIdentifier(t *testing.T) { + mac := "" + idfa := "" + expected := "" + got := GetIdentifier(mac, idfa) + if got != expected { + t.Errorf("Expected: %s, but got:%s", expected, got) + } + + mac = "00:00:00:00:00:00" + expected = "" + got = GetIdentifier(mac, idfa) + if got != expected { + t.Errorf("Expected: %s, but got:%s", expected, got) + } + + mac = "02:00:00:00:00:00" + expected = "" + got = GetIdentifier(mac, idfa) + if got != expected { + t.Errorf("Expected: %s, but got:%s", expected, got) + } + + mac = "020000000020" + expected = "02:00:00:00:00:20" + got = GetIdentifier(mac, idfa) + if got != expected { + t.Errorf("Expected: %s, but got:%s", expected, got) + } + + mac = "02:00:00:00:00:20" + expected = "02:00:00:00:00:20" + got = GetIdentifier(mac, idfa) + if got != expected { + t.Errorf("Expected: %s, but got:%s", expected, got) + } + + idfa = "00000000123456780000000000000000" + expected = "00000000-1234-5678-0000-000000000000" + got = GetIdentifier(mac, idfa) + if got != expected { + t.Errorf("Expected: %s, but got:%s", expected, got) + } +} diff --git a/trunk/goutil/dfaExUtil/dfaEx.go b/trunk/goutil/dfaExUtil/dfaEx.go new file mode 100644 index 0000000..98967e3 --- /dev/null +++ b/trunk/goutil/dfaExUtil/dfaEx.go @@ -0,0 +1,240 @@ +package dfaExUtil + +/* + * 扩展DFA算法 + * + * 一种二层树实现类DFA算法（DFA为多层树结构；go语言特性中的map结构过于“重量级”导致内存占用很大；此外还可能存在大量相同字符结点） + * + * 第一层map为所有字母/汉字作为key；value为第二层map + * 第二层map为第一层冲突字母/汉字的自定义hash作为key；value指示是否为敏感词结束标识 + * + * 测试结果：50万+的敏感词： + * 构造树耗时稍优于原始DFA； + * 内存使用为原DFA的不到1/4：原DFA占用495M内存，此算法使用111M； + * 查询效率比原DFA低10%~20%左右；主要是多一次map查询和多一次hash计算； + * + */ + +/* + * 注意使用[]rune的问题(此问题已通过使用固定位数hash解决)： + * []rune中文使用的是unicode编码；若“中”编码为#4E2D；而#4E2D对应“N-”; + * 即："N-"与"中"unicode编码均为#4E2D，即会产生hash冲突 + * + */ + +import ( + "fmt" + "strings" + + hash "goutil/dfaExUtil/hash64" +) + +// hash使用的类型(uint64对应hash64函数；uint32对应hash32函数) +type hashType = uint64 + +type DFAEx struct { + // 忽略大小写；true-忽略大小写；false-大小写敏感 + ignoreCase bool + // hash冲突个数 + hashCollisions int + // 树根 + // 字符/hash/uint8(b10000000)(最高字符表示是否结束/低7位表示字符位置) + root map[rune]map[hashType]uint8 +} + +// 新建敏感词对象 +// wordList - 敏感词列表 +// ignoreCase - [可选；默认false] 是否忽略大小写 +func NewDFAEx(wordList []string, ignoreCase ...bool) (dfaEx *DFAEx) { + var iCase bool + var mapSize int + + if len(ignoreCase) > 0 { + iCase = ignoreCase[0] + } + + mapSize = len(wordList) * 10 + // 防止过小 + if mapSize < 1_000 { + mapSize = 1_000 + } + // 通常各语言的单rune非重复数不超过1万 + if mapSize > 10_000 { + mapSize = 10_000 + } + + dfaEx = &DFAEx{ + ignoreCase: iCase, + root: make(map[rune]map[hashType]uint8, mapSize), + } + + for _, v := range wordList { + word := v + if iCase { + // 忽略大小写；所有字母转大写 + word = strings.ToUpper(word) + } + wordRune := []rune(word) + if len(wordRune) > 0 { + dfaEx.InsertWord(wordRune) + } + } + + return dfaEx +} + +// 添加敏感词 +func (dfaEx *DFAEx) InsertWord(word []rune) { + var hs hashType + var lastWord rune + var lastHash hashType + for i, v := range word { + lastWord = v + lastHash = hs + if wdInfo, ok := dfaEx.root[v]; ok { + // "字"已存在 + if hsV, ok := wdInfo[hs]; !ok { + // hash不存在，添加hash + wdInfo[hs] = uint8(i & 0x7F) // 第i位 + } else { + // hash已存在，检测是否冲突 + if (hsV & 0x7F) != uint8(i) { + // hash冲突 + dfaEx.hashCollisions++ + // fmt.Printf("hash冲突 %s %016X %d %d\n", string(v), hs, i+1, hsV&0x7F+1) + } + } + } else { + // "字"不存在，添加"字"和hash + wdInfo = make(map[hashType]uint8) + wdInfo[hs] = uint8(i & 0x7F) // 第i位 + dfaEx.root[v] = wdInfo + } + hs = hash.FastSumByRune2(v, hs) // hash更新 + } + + // 敏感词结束标志(uint8最高位置1) + dfaEx.root[lastWord][lastHash] |= 0x80 +} + +// 字符串查找敏感词 +func (dfaEx *DFAEx) IsMatch(str string) bool { + starts, _ := dfaEx.SearchSentence(str, true) + return len(starts) > 0 +} + +// 指定字符替换敏感词 +func (dfaEx *DFAEx) HandleWord(str string, replace rune) string { + starts, ends := dfaEx.SearchSentence(str) + if len(starts) == 0 { + return str + } + + strRune := []rune(str) + for i := 0; i < len(starts); i++ { + for idx := starts[i]; idx <= ends[i]; idx++ { + strRune[idx] = replace + } + } + + return string(strRune) +} + +// 字符串查找敏感词 +func (dfaEx *DFAEx) SearchSentence(str string, firstOpt ...bool) (starts, ends []int) { + var first bool // 是否首次匹配就返回 + if len(firstOpt) > 0 { + first = firstOpt[0] + } + strBak := str + if dfaEx.ignoreCase { + // 忽略大小写；所有字母转大写 + strBak = strings.ToUpper(str) + } + runeStr := []rune(strBak) + for i := 0; i < len(runeStr); { + end := dfaEx.searchByStart(i, runeStr) + if end < 0 { + // 继续下一个进行匹配 + i++ + } else { + // 记录匹配位置；从匹配到的下一个位置继续 + starts = append(starts, i) + ends = append(ends, end) + if first { + // 首次匹配就返回 + break + } + + i = end + 1 + } + } + + return +} + +// 从指定的开始位置搜索语句 +// start - 开始匹配的位置 +// str - 待检测字符串 +// 返回：匹配到的结束位置，未匹配到返回-1 +func (dfaEx *DFAEx) searchByStart(start int, runeStr []rune) (end int) { + var hs hashType + end = -1 // 未匹配到返回值 + + for i := start; i < len(runeStr); i++ { + wd := runeStr[i] + wdInfo, ok := dfaEx.root[wd] + if !ok { + // "字"不存在 + break + } + + hsV, ok := wdInfo[hs] + if !ok { + // hash不存在 + break + } + + // 检测是否句尾 + if (hsV & 0x80) != 0 { + // 找到句尾，继续匹配，直到匹配到最长敏感词为止 + end = i + } + + hs = hash.FastSumByRune2(wd, hs) // hash更新 + } + + return +} + +// 获取hash冲突数 +func (dfaEx *DFAEx) GetHashCollisions() int { + return dfaEx.hashCollisions +} + +// 调试接口 +func (dfaEx *DFAEx) Print() { + fmt.Println(dfaEx) +} + +// 调试接口 +func (dfaEx *DFAEx) PrintFmt(verbose bool) { + var keys int + var hashs int + for k, v := range dfaEx.root { + keys++ + if verbose { + fmt.Println("---------------------------") + fmt.Println(string(k)) + } + for kk, vv := range v { + hashs++ + if verbose { + fmt.Printf("%016X %02X\n", kk, vv) + } + } + } + + fmt.Println("================================") + fmt.Println("keys:", keys, "hashs", hashs, "map count", keys+1, "hashCollisions Count", dfaEx.hashCollisions) +} diff --git a/trunk/goutil/dfaExUtil/dfaEx_test.go b/trunk/goutil/dfaExUtil/dfaEx_test.go new file mode 100644 index 0000000..e7496b7 --- /dev/null +++ b/trunk/goutil/dfaExUtil/dfaEx_test.go @@ -0,0 +1,15 @@ +package dfaExUtil + +import "testing" + +func TestHandleWord(t *testing.T) { + strs := []string{"ABC", "1234", "测试", "测试代码", "测试一下"} + + dfaEx1 := NewDFAEx(strs) + str := dfaEx1.HandleWord("abc按了数字12345来测试代码是否正常，结果测试出了bug", '*') + t.Log(str) + + dfaEx2 := NewDFAEx(strs, true) + str = dfaEx2.HandleWord("abc按了数字12345来测试代码是否正常，结果测试出了bug", '*') + t.Log(str) +} diff --git a/trunk/goutil/dfaExUtil/hash32/hash32.go b/trunk/goutil/dfaExUtil/hash32/hash32.go new file mode 100644 index 0000000..fa84949 --- /dev/null +++ b/trunk/goutil/dfaExUtil/hash32/hash32.go @@ -0,0 +1,98 @@ +package hash32 + +/* + * ***注意*** + * + * Sum 使用的是[]byte参数；string中文是utf-8编码 + * SumByRune 使用的是[]rune参数；中文使用的是unicode编码 + * + * 两种参数中文编码不同；同一个string调用两个接口得到的hash是不同的！！！ 这是要特别注意的 + * + * 对string进行for range操作会自动被[]rune化；一定要注意！！！ + * + */ + +const shiftBit = 11 // 每个字节移位数(测试经验值11) +const reverseBit = 32 - shiftBit + +// 快速Hash计算(*** 注意：只取了rune低16位进行hash计算；计算结果与SumByRune不一致 ***) +func FastSumByRune2(in rune, hs uint32) (out uint32) { + out = ((hs << shiftBit) | (hs >> reverseBit)) + uint32(byte(in>>8)) + out = ((out << shiftBit) | (out >> reverseBit)) + uint32(byte(in)) + return +} + +// 快速Hash计算(*** 此计算结果与SumByRune一致 ***) +func FastSumByRune4(in rune, hs uint32) (out uint32) { + out = ((hs << shiftBit) | (hs >> reverseBit)) + uint32(byte(in>>24)) + out = ((out << shiftBit) | (out >> reverseBit)) + uint32(byte(in>>16)) + out = ((out << shiftBit) | (out >> reverseBit)) + uint32(byte(in>>8)) + out = ((out << shiftBit) | (out >> reverseBit)) + uint32(byte(in)) + return +} + +// 原Hash值参数重 +func hsArg(hsOpt ...uint32) (out uint32) { + out = uint32(0) + if len(hsOpt) > 0 { + out = hsOpt[0] + } + return +} + +// Hash计算 +// in - 待hash串 +// hsOpt - 原hash值(在此基础上继续hash) +func Sum(in []byte, hsOpt ...uint32) (out uint32) { + out = hsArg(hsOpt...) + for _, v := range in { + out = ((out << shiftBit) | (out >> reverseBit)) + uint32(v) + } + + return +} + +// Hash计算 +func SumByRune(in rune, hsOpt ...uint32) (out uint32) { + // rune转[]byte + inVal := make([]byte, 4) + inVal[0] = byte(in >> 24) + inVal[1] = byte(in >> 16) + inVal[2] = byte(in >> 8) + inVal[3] = byte(in) + + // *** 经实际测试：不加以下代码运行效率更高 *** + + // 去除前面多余的\x00 + // for { + // if len(inVal) <= 1 { + // // 以免全0异常；至少要保留1位 + // break + // } + // if inVal[0] == 0 { + // inVal = inVal[1:] + // } else { + // break + // } + // } + // 规避hash冲突，如："N-"与"中"unicode编码均为#4E2D，即会产生hash冲突 + // 若长度>1(即非常规ASCII)，所有字节最高位置1 + // if len(inVal) > 1 { + // for i := 0; i < len(inVal); i++ { + // inVal[i] |= 0x80 + // } + // } + out = Sum(inVal, hsOpt...) + + return +} + +// Hash计算 +func SumByRunes(in []rune, hsOpt ...uint32) (out uint32) { + out = hsArg(hsOpt...) + for _, v := range in { + out = SumByRune(v, out) + } + + return +} diff --git a/trunk/goutil/dfaExUtil/hash64/hash64.go b/trunk/goutil/dfaExUtil/hash64/hash64.go new file mode 100644 index 0000000..17eb0fa --- /dev/null +++ b/trunk/goutil/dfaExUtil/hash64/hash64.go @@ -0,0 +1,98 @@ +package hash64 + +/* + * ***注意*** + * + * Sum 使用的是[]byte参数；string中文是utf-8编码 + * SumByRune 使用的是[]rune参数；中文使用的是unicode编码 + * + * 两种参数中文编码不同；同一个string调用两个接口得到的hash是不同的！！！ 这是要特别注意的 + * + * 对string进行for range操作会自动被[]rune化；一定要注意！！！ + * + */ + +const shiftBit = 23 // 每个字节移位数(测试经验值23) +const reverseBit = 64 - shiftBit + +// 快速Hash计算(*** 注意：只取了rune低16位进行hash计算；计算结果与SumByRune不一致 ***) +func FastSumByRune2(in rune, hs uint64) (out uint64) { + out = ((hs << shiftBit) | (hs >> reverseBit)) + uint64(byte(in>>8)) + out = ((out << shiftBit) | (out >> reverseBit)) + uint64(byte(in)) + return +} + +// 快速Hash计算(*** 此计算结果与SumByRune一致 ***) +func FastSumByRune4(in rune, hs uint64) (out uint64) { + out = ((hs << shiftBit) | (hs >> reverseBit)) + uint64(byte(in>>24)) + out = ((out << shiftBit) | (out >> reverseBit)) + uint64(byte(in>>16)) + out = ((out << shiftBit) | (out >> reverseBit)) + uint64(byte(in>>8)) + out = ((out << shiftBit) | (out >> reverseBit)) + uint64(byte(in)) + return +} + +// 原Hash值参数重 +func hsArg(hsOpt ...uint64) (out uint64) { + out = uint64(0) + if len(hsOpt) > 0 { + out = hsOpt[0] + } + return +} + +// Hash计算 +// in - 待hash串 +// hsOpt - 原hash值(在此基础上继续hash) +func Sum(in []byte, hsOpt ...uint64) (out uint64) { + out = hsArg(hsOpt...) + for _, v := range in { + out = ((out << shiftBit) | (out >> reverseBit)) + uint64(v) + } + + return +} + +// Hash计算 +func SumByRune(in rune, hsOpt ...uint64) (out uint64) { + // rune转[]byte + inVal := make([]byte, 4) + inVal[0] = byte(in >> 24) + inVal[1] = byte(in >> 16) + inVal[2] = byte(in >> 8) + inVal[3] = byte(in) + + // *** 经实际测试：不加以下代码运行效率更高 *** + + // 去除前面多余的\x00 + // for { + // if len(inVal) <= 1 { + // // 以免全0异常；至少要保留1位 + // break + // } + // if inVal[0] == 0 { + // inVal = inVal[1:] + // } else { + // break + // } + // } + // 规避hash冲突，如："N-"与"中"unicode编码均为#4E2D，即会产生hash冲突 + // 若长度>1(即非常规ASCII)，所有字节最高位置1 + // if len(inVal) > 1 { + // for i := 0; i < len(inVal); i++ { + // inVal[i] |= 0x80 + // } + // } + out = Sum(inVal, hsOpt...) + + return +} + +// Hash计算 +func SumByRunes(in []rune, hsOpt ...uint64) (out uint64) { + out = hsArg(hsOpt...) + for _, v := range in { + out = SumByRune(v, out) + } + + return +} diff --git a/trunk/goutil/dfaExUtil/三维树转二维树.bmp b/trunk/goutil/dfaExUtil/三维树转二维树.bmp new file mode 100644 index 0000000000000000000000000000000000000000..f3af9ddcfc29fa287786d95ad46d964dea4f363b GIT binary patch literal 2239542 zcmeI*J+354wjkz9f*4RDMi^m)0tO&Tpg?s-xFW>@2-FU!