From cc94c074cc2bce9024957cc6e1be8c74607d6f8b Mon Sep 17 00:00:00 2001 From: bryanqiu Date: Thu, 20 Apr 2023 16:16:25 +0800 Subject: [PATCH] add test & fix cocurrent bug --- go.mod | 14 +++++- go.sum | 14 ++++++ test/logs/.gitkeep | 0 test/main.go | 85 ++++++++++++++++++++++++++++++++ uid.go | 118 ++++++++++++++++++++++++++++----------------- 5 files changed, 186 insertions(+), 45 deletions(-) create mode 100644 test/logs/.gitkeep create mode 100644 test/main.go diff --git a/go.mod b/go.mod index 0840974..fc55474 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,16 @@ module qoobing.com/gomod/uid go 1.19 -require github.com/gammazero/deque v0.2.1 +require ( + github.com/gammazero/deque v0.2.1 + gorm.io/gorm v1.25.0 + qoobing.com/gomod/log v1.2.2 +) + +require ( + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/tylerb/gls v0.0.0-20150407001822-e606233f194d // indirect + github.com/tylerb/is v2.1.4+incompatible // indirect + qoobing.com/gomod/str v1.0.1 // indirect +) diff --git a/go.sum b/go.sum index 2df5de4..9c8f988 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,16 @@ github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0= github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/tylerb/gls v0.0.0-20150407001822-e606233f194d h1:yYYPFFlbqxF5mrj5sEfETtM/Ssz2LTy0/VKlDdXYctc= +github.com/tylerb/gls v0.0.0-20150407001822-e606233f194d/go.mod h1:0MwyId/pXK5wkYYEXe7NnVknX+aNBuF73fLV3U0reU8= +github.com/tylerb/is v2.1.4+incompatible h1:BMf2zP0kY2Ykzx2W1fDrjwKj1x1B4E0mELkpjaNy1tM= +github.com/tylerb/is v2.1.4+incompatible/go.mod h1:3Bw2NWEEe8Kx7/etYqgm9ug53iNDgabnloch75jjOSc= +gorm.io/gorm v1.25.0 h1:+KtYtb2roDz14EQe4bla8CbQlmb9dN3VejSai3lprfU= +gorm.io/gorm v1.25.0/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= +qoobing.com/gomod/log v1.2.2 h1:6h7cgfhIm4cQNASC7KDJ8TyLn3pb7eYJV9JO+vXYwDA= +qoobing.com/gomod/log v1.2.2/go.mod h1:/ZTN/ukAbSqRb4eMlF9LpfkVgM21xwprbd5y3tcQxpM= +qoobing.com/gomod/str v1.0.1 h1:X+JOigE9xA6cTNph7/s1KeD4zLYM9XTLPPHQcpHFoog= +qoobing.com/gomod/str v1.0.1/go.mod h1:gbhN2dba/P5gFRGVJvEI57KEJLlMHHAd6Kuuxn4GlMY= diff --git a/test/logs/.gitkeep b/test/logs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/test/main.go b/test/main.go new file mode 100644 index 0000000..ba84248 --- /dev/null +++ b/test/main.go @@ -0,0 +1,85 @@ +package main + +import ( + "errors" + "fmt" + "sync" + "sync/atomic" + "time" + + "qoobing.com/gomod/log" + "qoobing.com/gomod/uid" +) + +type MemCreatorHelper struct { + start int64 + length int + lock sync.Mutex +} + +func NewMemCreatorHelper() *MemCreatorHelper { + return &MemCreatorHelper{ + lock: sync.Mutex{}, + start: 0, + length: 100, + } +} + +// CreatePrefix create id prefix +func (mch *MemCreatorHelper) CreatePrefix(parts ...interface{}) string { + return "XXXPPPYYY" +} + +// CreateRange create id range from database table t_id. +func (mch *MemCreatorHelper) CreateRange(prefix string) (start int64, length int, err error) { + mch.lock.Lock() + defer mch.lock.Unlock() + fmt.Println("CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCreateRange") + + mch.start += int64(mch.length) + if mch.start == 300 { + panic("test panic: 3") + } else if mch.start == 7000 { + return mch.start, mch.length, errors.New("tesk error: 7") + } + return mch.start, mch.length, nil +} + +// // CreateMixedId create finnal transaction id +func (mch *MemCreatorHelper) CreateMixedId(prefix string, id int64, parts ...interface{}) string { + return fmt.Sprintf("id-%s-%d", prefix, id) +} + +var helper = NewMemCreatorHelper() +var idcreator = uid.NewIdCreator("test", helper) + +func getId(i, ii int) { + defer func() { + if r := recover(); r != nil { + fmt.Println("panic recover:", r) + } + }() + fmt.Printf("%d-%d: %s\n", i, ii, idcreator.GetId()) +} + +func main() { + c := atomic.Uint64{} + st := time.Now() + wg := sync.WaitGroup{} + wg.Add(1000) + for i := 0; i < 1000; i++ { + go func(i int) { + log.SetLogid(fmt.Sprintf("main idcreator%07d", i)) + for ii := 0; ii < 1000; ii++ { + getId(i, ii) + c.Add(uint64(1)) + } + wg.Done() + }(i) + } + wg.Wait() + et := time.Now() + fmt.Println("cost:", et.Sub(st)) + fmt.Println("helper:", helper) + fmt.Println("all done, total id number is:", c.Load()) +} diff --git a/uid.go b/uid.go index a17c48d..a7be50b 100644 --- a/uid.go +++ b/uid.go @@ -47,8 +47,9 @@ type rangeQueue struct { } const ( - RANGE_STATUS_OK = 1 - RANGE_STATUS_USEOUT = 2 + RANGE_STATUS_OK = 1 + RANGE_STATUS_USEOUT = 2 + RANGE_STATUS_REMOVED = 3 ) // idCreator @@ -78,37 +79,56 @@ func NewIdCreator(typ string, helper IdCreatorHelper) IdCreator { // Step 3. backgroud task for generate range go func() { - for i := 0; ; i++ { - log.SetLogid(fmt.Sprintf("uidcreator%07d", i)) - for prefix, queue := range idg.Ranges { - var qlen = queue.Len() - var timeoutMs = 1 * time.Millisecond - - if qlen <= 0 { - timeoutMs = 1 * time.Millisecond - } else if qlen <= 2 { - timeoutMs = 50 * time.Millisecond - } else { - timeoutMs = -1000 * time.Millisecond - } - - if int(timeoutMs) > 0 { - idg.createNewRangeWithErrorPanic(prefix) - time.Sleep(timeoutMs * time.Millisecond) - } else if int(timeoutMs) < 0 { - time.Sleep((-timeoutMs) * time.Millisecond) - } - } - time.Sleep(3000 * time.Millisecond) + for { + log.Infof("start backgroud range generater, type=%s", typ) + idg.backgroudTaskForGenerateRange() + log.Infof("unexpect backgroud range generater exit") + time.Sleep(1000 * time.Millisecond) } }() return idg } +func (idg *idCreator) backgroudTaskForGenerateRange() { + // rocover panic + defer func() { + if r := recover(); r != nil { + log.Warningf("backgroudTaskForGenerateRange panic: [%v]", r) + } + }() + + // doing range generator + for i := 0; ; i++ { + log.SetLogid(fmt.Sprintf("uidcreator%07d", i)) + log.PrintPretty("backgroud info, ranges:", idg.Ranges) + for prefix, queue := range idg.Ranges { + var qlen = queue.Len() + var timeoutMs = 1 * time.Millisecond + + if qlen <= 0 { + timeoutMs = 1 * time.Millisecond + } else if qlen <= 2 { + timeoutMs = 50 * time.Millisecond + } else { + timeoutMs = -1000 * time.Millisecond + } + + if int(timeoutMs) > 0 { + idg.createNewRangeWithErrorPanic(prefix) + time.Sleep(timeoutMs) + } else if int(timeoutMs) < 0 { + time.Sleep(-timeoutMs) + } + } + time.Sleep(3000 * time.Millisecond) + } +} + func (idg *idCreator) GetId(idparts ...interface{}) string { + var retry int var prefix = idg.Helper.CreatePrefix(idparts...) - for { + for retry = 0; retry < 50; retry++ { var id int64 = 0 var currange = idg.getCurrentRange(prefix) log.Infof("success get current range:(%s, %d-%d)", @@ -118,6 +138,7 @@ func (idg *idCreator) GetId(idparts ...interface{}) string { id = atomic.LoadInt64(&currange.Next) if id > currange.End { //next range + currange.Status = RANGE_STATUS_USEOUT break } if atomic.CompareAndSwapInt64(&currange.Next, id, id+1) { @@ -126,7 +147,7 @@ func (idg *idCreator) GetId(idparts ...interface{}) string { } } - // Step 2. No lock add failed, try add id in next range. + // Step 2. Presious [no lock] adding failed, try add id in next range. if id > currange.End { //next range log.Infof("failed get id from current range, try next") @@ -138,6 +159,8 @@ func (idg *idCreator) GetId(idparts ...interface{}) string { log.Infof("success get id(not miexed):(%s, %d)", prefix, id) return idg.Helper.CreateMixedId(prefix, id, idparts...) } + log.Errorf("getid from range retry too much time(%d)", retry) + panic("getid from range retry too much time") } func (idg *idCreator) createDefaultPrefix() (prefix string, err error) { @@ -165,17 +188,7 @@ func (idg *idCreator) removeOldRangeWithErrorPanic(prefix string) { } func (idg *idCreator) createNewRange(prefix string) error { - // Step 1. create range from db or something else. - start, length, err := idg.Helper.CreateRange(prefix) - if err != nil { - str := fmt.Sprintf( - "idCreator '%s' generate range err:'%s'", - idg.Type, err.Error()) - log.Errorf("%s", str) - return errors.New(str) - } - - // Step 2. lock generator. + // Step 1. lock generator. idg.lock.Lock() defer idg.lock.Unlock() if _, ok := idg.RangesLocks[prefix]; !ok { @@ -186,6 +199,21 @@ func (idg *idCreator) createNewRange(prefix string) error { var lock = idg.RangesLocks[prefix] lock.Lock() defer lock.Unlock() + var queueLen = queue.Len() + if queueLen >= 30 { + log.Infof("ignore createNewRange beacuse of queue is long enough(%d)", queueLen) + return nil + } + + // Step 2. create range from db or something else. + start, length, err := idg.Helper.CreateRange(prefix) + if err != nil { + str := fmt.Sprintf( + "idCreator '%s' generate range err:'%s'", + idg.Type, err.Error()) + log.Errorf("%s", str) + return errors.New(str) + } // Step 3. push to ranges queue. var r = &Range{ @@ -210,17 +238,20 @@ func (idg *idCreator) removeOldRange(prefix string) error { for queue.Len() > 0 { r := queue.Front() if r.Status == RANGE_STATUS_OK { - break + log.Infof("success remove all useout ranges for prefix:%s", prefix) + return nil } else if r.Status == RANGE_STATUS_USEOUT { - queue.PopFront() - log.Infof("success remove old range:(%s,%d-%d)", + if rr := queue.PopFront(); rr != r { + panic("queue head is changed, maybe somethong bug") + } + log.Infof("success remove old range:(%s,%d-%d), try remove next", prefix, r.Start, r.End) } else { panic("invalid status, maybe somethong bug") } } } else { - log.Infof("rare case: rangequeue:(%s) have remove by others", prefix) + log.Infof("rare case: rangequeue:(%s) have removed by others", prefix) } return nil } @@ -270,11 +301,10 @@ func (idg *idCreator) turnToNextRange(r *Range) { // We donot lock the range, because there is no too much // side effect even if 'createNewRange' execute twice. switch r.Status { - case RANGE_STATUS_OK: - r.Status = RANGE_STATUS_USEOUT + case RANGE_STATUS_USEOUT: idg.createNewRangeWithErrorPanic(r.Prefix) idg.removeOldRangeWithErrorPanic(r.Prefix) - case RANGE_STATUS_USEOUT: + case RANGE_STATUS_OK: // have turn by others, we do nothing. log.Debugf("have turn by others, we do nothing.") default: