gomod/uid
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add test & fix cocurrent bug

Browse Source
This commit is contained in:
bryanqiu 2023-04-20 16:16:25 +08:00
parent 229bfb80cd
commit cc94c074cc
5 changed files with 186 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
go.mod
View File

@ -2,4 +2,16 @@ module qoobing.com/gomod/uid
go 1.19
require github.com/gammazero/deque v0.2.1
require (
github.com/gammazero/deque v0.2.1
gorm.io/gorm v1.25.0
qoobing.com/gomod/log v1.2.2
)
require (
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/tylerb/gls v0.0.0-20150407001822-e606233f194d // indirect
github.com/tylerb/is v2.1.4+incompatible // indirect
qoobing.com/gomod/str v1.0.1 // indirect
)

14
go.sum
View File

@ -1,2 +1,16 @@
github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0=
github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/tylerb/gls v0.0.0-20150407001822-e606233f194d h1:yYYPFFlbqxF5mrj5sEfETtM/Ssz2LTy0/VKlDdXYctc=
github.com/tylerb/gls v0.0.0-20150407001822-e606233f194d/go.mod h1:0MwyId/pXK5wkYYEXe7NnVknX+aNBuF73fLV3U0reU8=
github.com/tylerb/is v2.1.4+incompatible h1:BMf2zP0kY2Ykzx2W1fDrjwKj1x1B4E0mELkpjaNy1tM=
github.com/tylerb/is v2.1.4+incompatible/go.mod h1:3Bw2NWEEe8Kx7/etYqgm9ug53iNDgabnloch75jjOSc=
gorm.io/gorm v1.25.0 h1:+KtYtb2roDz14EQe4bla8CbQlmb9dN3VejSai3lprfU=
gorm.io/gorm v1.25.0/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
qoobing.com/gomod/log v1.2.2 h1:6h7cgfhIm4cQNASC7KDJ8TyLn3pb7eYJV9JO+vXYwDA=
qoobing.com/gomod/log v1.2.2/go.mod h1:/ZTN/ukAbSqRb4eMlF9LpfkVgM21xwprbd5y3tcQxpM=
qoobing.com/gomod/str v1.0.1 h1:X+JOigE9xA6cTNph7/s1KeD4zLYM9XTLPPHQcpHFoog=
qoobing.com/gomod/str v1.0.1/go.mod h1:gbhN2dba/P5gFRGVJvEI57KEJLlMHHAd6Kuuxn4GlMY=

0
test/logs/.gitkeep Normal file
View File

85
test/main.go Normal file
View File

@ -0,0 +1,85 @@
package main
import (
"errors"
"fmt"
"sync"
"sync/atomic"
"time"
"qoobing.com/gomod/log"
"qoobing.com/gomod/uid"
)
type MemCreatorHelper struct {
start int64
length int
lock sync.Mutex
}
func NewMemCreatorHelper() *MemCreatorHelper {
return &MemCreatorHelper{
lock: sync.Mutex{},
start: 0,
length: 100,
}
}
// CreatePrefix create id prefix
func (mch *MemCreatorHelper) CreatePrefix(parts ...interface{}) string {
return "XXXPPPYYY"
}
// CreateRange create id range from database table t_id.
func (mch *MemCreatorHelper) CreateRange(prefix string) (start int64, length int, err error) {
mch.lock.Lock()
defer mch.lock.Unlock()
fmt.Println("CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCreateRange")
mch.start += int64(mch.length)
if mch.start == 300 {
panic("test panic: 3")
} else if mch.start == 7000 {
return mch.start, mch.length, errors.New("tesk error: 7")
}
return mch.start, mch.length, nil
}
// // CreateMixedId create finnal transaction id
func (mch *MemCreatorHelper) CreateMixedId(prefix string, id int64, parts ...interface{}) string {
return fmt.Sprintf("id-%s-%d", prefix, id)
}
var helper = NewMemCreatorHelper()
var idcreator = uid.NewIdCreator("test", helper)
func getId(i, ii int) {
defer func() {
if r := recover(); r != nil {
fmt.Println("panic recover:", r)
}
}()
fmt.Printf("%d-%d: %s\n", i, ii, idcreator.GetId())
}
func main() {
c := atomic.Uint64{}
st := time.Now()
wg := sync.WaitGroup{}
wg.Add(1000)
for i := 0; i < 1000; i++ {
go func(i int) {
log.SetLogid(fmt.Sprintf("main idcreator%07d", i))
for ii := 0; ii < 1000; ii++ {
getId(i, ii)
c.Add(uint64(1))
}
wg.Done()
}(i)
}
wg.Wait()
et := time.Now()
fmt.Println("cost:", et.Sub(st))
fmt.Println("helper:", helper)
fmt.Println("all done, total id number is:", c.Load())
}

80
uid.go
View File

@ -49,6 +49,7 @@ type rangeQueue struct {
const (
RANGE_STATUS_OK = 1
RANGE_STATUS_USEOUT = 2
RANGE_STATUS_REMOVED = 3
)
// idCreator
@ -78,8 +79,29 @@ func NewIdCreator(typ string, helper IdCreatorHelper) IdCreator {
// Step 3. backgroud task for generate range
go func() {
for {
log.Infof("start backgroud range generater, type=%s", typ)
idg.backgroudTaskForGenerateRange()
log.Infof("unexpect backgroud range generater exit")
time.Sleep(1000 * time.Millisecond)
}
}()
return idg
}
func (idg *idCreator) backgroudTaskForGenerateRange() {
// rocover panic
defer func() {
if r := recover(); r != nil {
log.Warningf("backgroudTaskForGenerateRange panic: [%v]", r)
}
}()
// doing range generator
for i := 0; ; i++ {
log.SetLogid(fmt.Sprintf("uidcreator%07d", i))
log.PrintPretty("backgroud info, ranges:", idg.Ranges)
for prefix, queue := range idg.Ranges {
var qlen = queue.Len()
var timeoutMs = 1 * time.Millisecond
@ -94,21 +116,19 @@ func NewIdCreator(typ string, helper IdCreatorHelper) IdCreator {
if int(timeoutMs) > 0 {
idg.createNewRangeWithErrorPanic(prefix)
time.Sleep(timeoutMs * time.Millisecond)
time.Sleep(timeoutMs)
} else if int(timeoutMs) < 0 {
time.Sleep((-timeoutMs) * time.Millisecond)
time.Sleep(-timeoutMs)
}
}
time.Sleep(3000 * time.Millisecond)
}
}()
return idg
}
func (idg *idCreator) GetId(idparts ...interface{}) string {
var retry int
var prefix = idg.Helper.CreatePrefix(idparts...)
for {
for retry = 0; retry < 50; retry++ {
var id int64 = 0
var currange = idg.getCurrentRange(prefix)
log.Infof("success get current range:(%s, %d-%d)",
@ -118,6 +138,7 @@ func (idg *idCreator) GetId(idparts ...interface{}) string {
id = atomic.LoadInt64(&currange.Next)
if id > currange.End {
//next range
currange.Status = RANGE_STATUS_USEOUT
break
}
if atomic.CompareAndSwapInt64(&currange.Next, id, id+1) {
@ -126,7 +147,7 @@ func (idg *idCreator) GetId(idparts ...interface{}) string {
}
}
// Step 2. No lock add failed, try add id in next range.
// Step 2. Presious [no lock] adding failed, try add id in next range.
if id > currange.End {
//next range
log.Infof("failed get id from current range, try next")
@ -138,6 +159,8 @@ func (idg *idCreator) GetId(idparts ...interface{}) string {
log.Infof("success get id(not miexed):(%s, %d)", prefix, id)
return idg.Helper.CreateMixedId(prefix, id, idparts...)
}
log.Errorf("getid from range retry too much time(%d)", retry)
panic("getid from range retry too much time")
}
func (idg *idCreator) createDefaultPrefix() (prefix string, err error) {
@ -165,17 +188,7 @@ func (idg *idCreator) removeOldRangeWithErrorPanic(prefix string) {
}
func (idg *idCreator) createNewRange(prefix string) error {
// Step 1. create range from db or something else.
start, length, err := idg.Helper.CreateRange(prefix)
if err != nil {
str := fmt.Sprintf(
"idCreator '%s' generate range err:'%s'",
idg.Type, err.Error())
log.Errorf("%s", str)
return errors.New(str)
}
// Step 2. lock generator.
// Step 1. lock generator.
idg.lock.Lock()
defer idg.lock.Unlock()
if _, ok := idg.RangesLocks[prefix]; !ok {
@ -186,6 +199,21 @@ func (idg *idCreator) createNewRange(prefix string) error {
var lock = idg.RangesLocks[prefix]
lock.Lock()
defer lock.Unlock()
var queueLen = queue.Len()
if queueLen >= 30 {
log.Infof("ignore createNewRange beacuse of queue is long enough(%d)", queueLen)
return nil
}
// Step 2. create range from db or something else.
start, length, err := idg.Helper.CreateRange(prefix)
if err != nil {
str := fmt.Sprintf(
"idCreator '%s' generate range err:'%s'",
idg.Type, err.Error())
log.Errorf("%s", str)
return errors.New(str)
}
// Step 3. push to ranges queue.
var r = &Range{
@ -210,17 +238,20 @@ func (idg *idCreator) removeOldRange(prefix string) error {
for queue.Len() > 0 {
r := queue.Front()
if r.Status == RANGE_STATUS_OK {
break
log.Infof("success remove all useout ranges for prefix:%s", prefix)
return nil
} else if r.Status == RANGE_STATUS_USEOUT {
queue.PopFront()
log.Infof("success remove old range:(%s,%d-%d)",
if rr := queue.PopFront(); rr != r {
panic("queue head is changed, maybe somethong bug")
}
log.Infof("success remove old range:(%s,%d-%d), try remove next",
prefix, r.Start, r.End)
} else {
panic("invalid status, maybe somethong bug")
}
}
} else {
log.Infof("rare case: rangequeue:(%s) have remove by others", prefix)
log.Infof("rare case: rangequeue:(%s) have removed by others", prefix)
}
return nil
}
@ -270,11 +301,10 @@ func (idg *idCreator) turnToNextRange(r *Range) {
// We donot lock the range, because there is no too much
// side effect even if 'createNewRange' execute twice.
switch r.Status {
case RANGE_STATUS_OK:
r.Status = RANGE_STATUS_USEOUT
case RANGE_STATUS_USEOUT:
idg.createNewRangeWithErrorPanic(r.Prefix)
idg.removeOldRangeWithErrorPanic(r.Prefix)
case RANGE_STATUS_USEOUT:
case RANGE_STATUS_OK:
// have turn by others, we do nothing.
log.Debugf("have turn by others, we do nothing.")
default: