add test & fix cocurrent bug

go.mod

@@ -2,4 +2,16 @@ module qoobing.com/gomod/uid
 
 go 1.19
 
-require github.com/gammazero/deque v0.2.1
+require (
+	github.com/gammazero/deque v0.2.1
+	gorm.io/gorm v1.25.0
+	qoobing.com/gomod/log v1.2.2
+)
+
+require (
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
+	github.com/tylerb/gls v0.0.0-20150407001822-e606233f194d // indirect
+	github.com/tylerb/is v2.1.4+incompatible // indirect
+	qoobing.com/gomod/str v1.0.1 // indirect
+)

go.sum

@@ -1,2 +1,16 @@
 github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0=
 github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
+github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/tylerb/gls v0.0.0-20150407001822-e606233f194d h1:yYYPFFlbqxF5mrj5sEfETtM/Ssz2LTy0/VKlDdXYctc=
+github.com/tylerb/gls v0.0.0-20150407001822-e606233f194d/go.mod h1:0MwyId/pXK5wkYYEXe7NnVknX+aNBuF73fLV3U0reU8=
+github.com/tylerb/is v2.1.4+incompatible h1:BMf2zP0kY2Ykzx2W1fDrjwKj1x1B4E0mELkpjaNy1tM=
+github.com/tylerb/is v2.1.4+incompatible/go.mod h1:3Bw2NWEEe8Kx7/etYqgm9ug53iNDgabnloch75jjOSc=
+gorm.io/gorm v1.25.0 h1:+KtYtb2roDz14EQe4bla8CbQlmb9dN3VejSai3lprfU=
+gorm.io/gorm v1.25.0/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
+qoobing.com/gomod/log v1.2.2 h1:6h7cgfhIm4cQNASC7KDJ8TyLn3pb7eYJV9JO+vXYwDA=
+qoobing.com/gomod/log v1.2.2/go.mod h1:/ZTN/ukAbSqRb4eMlF9LpfkVgM21xwprbd5y3tcQxpM=
+qoobing.com/gomod/str v1.0.1 h1:X+JOigE9xA6cTNph7/s1KeD4zLYM9XTLPPHQcpHFoog=
+qoobing.com/gomod/str v1.0.1/go.mod h1:gbhN2dba/P5gFRGVJvEI57KEJLlMHHAd6Kuuxn4GlMY=

test/main.go

@@ -0,0 +1,85 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"qoobing.com/gomod/log"
+	"qoobing.com/gomod/uid"
+)
+
+type MemCreatorHelper struct {
+	start  int64
+	length int
+	lock   sync.Mutex
+}
+
+func NewMemCreatorHelper() *MemCreatorHelper {
+	return &MemCreatorHelper{
+		lock:   sync.Mutex{},
+		start:  0,
+		length: 100,
+	}
+}
+
+// CreatePrefix create id prefix
+func (mch *MemCreatorHelper) CreatePrefix(parts ...interface{}) string {
+	return "XXXPPPYYY"
+}
+
+// CreateRange create id range from database table t_id.
+func (mch *MemCreatorHelper) CreateRange(prefix string) (start int64, length int, err error) {
+	mch.lock.Lock()
+	defer mch.lock.Unlock()
+	fmt.Println("CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCreateRange")
+
+	mch.start += int64(mch.length)
+	if mch.start == 300 {
+		panic("test panic: 3")
+	} else if mch.start == 7000 {
+		return mch.start, mch.length, errors.New("tesk error: 7")
+	}
+	return mch.start, mch.length, nil
+}
+
+// // CreateMixedId create finnal transaction id
+func (mch *MemCreatorHelper) CreateMixedId(prefix string, id int64, parts ...interface{}) string {
+	return fmt.Sprintf("id-%s-%d", prefix, id)
+}
+
+var helper = NewMemCreatorHelper()
+var idcreator = uid.NewIdCreator("test", helper)
+
+func getId(i, ii int) {
+	defer func() {
+		if r := recover(); r != nil {
+			fmt.Println("panic recover:", r)
+		}
+	}()
+	fmt.Printf("%d-%d: %s\n", i, ii, idcreator.GetId())
+}
+
+func main() {
+	c := atomic.Uint64{}
+	st := time.Now()
+	wg := sync.WaitGroup{}
+	wg.Add(1000)
+	for i := 0; i < 1000; i++ {
+		go func(i int) {
+			log.SetLogid(fmt.Sprintf("main idcreator%07d", i))
+			for ii := 0; ii < 1000; ii++ {
+				getId(i, ii)
+				c.Add(uint64(1))
+			}
+			wg.Done()
+		}(i)
+	}
+	wg.Wait()
+	et := time.Now()
+	fmt.Println("cost:", et.Sub(st))
+	fmt.Println("helper:", helper)
+	fmt.Println("all done, total id number is:", c.Load())
+}

uid.go

@@ -47,8 +47,9 @@ type rangeQueue struct {
 }
 
 const (
-	RANGE_STATUS_OK     = 1
+	RANGE_STATUS_OK      = 1
-	RANGE_STATUS_USEOUT = 2
+	RANGE_STATUS_USEOUT  = 2
+	RANGE_STATUS_REMOVED = 3
 )
 
 // idCreator
@@ -78,37 +79,56 @@ func NewIdCreator(typ string, helper IdCreatorHelper) IdCreator {
 
 	// Step 3. backgroud task for generate range
 	go func() {
-		for i := 0; ; i++ {
+		for {
-			log.SetLogid(fmt.Sprintf("uidcreator%07d", i))
+			log.Infof("start backgroud range generater, type=%s", typ)
-			for prefix, queue := range idg.Ranges {
+			idg.backgroudTaskForGenerateRange()
-				var qlen = queue.Len()
+			log.Infof("unexpect backgroud range generater exit")
-				var timeoutMs = 1 * time.Millisecond
+			time.Sleep(1000 * time.Millisecond)
-
-				if qlen <= 0 {
-					timeoutMs = 1 * time.Millisecond
-				} else if qlen <= 2 {
-					timeoutMs = 50 * time.Millisecond
-				} else {
-					timeoutMs = -1000 * time.Millisecond
-				}
-
-				if int(timeoutMs) > 0 {
-					idg.createNewRangeWithErrorPanic(prefix)
-					time.Sleep(timeoutMs * time.Millisecond)
-				} else if int(timeoutMs) < 0 {
-					time.Sleep((-timeoutMs) * time.Millisecond)
-				}
-			}
-			time.Sleep(3000 * time.Millisecond)
 		}
 	}()
 
 	return idg
 }
 
+func (idg *idCreator) backgroudTaskForGenerateRange() {
+	// rocover panic
+	defer func() {
+		if r := recover(); r != nil {
+			log.Warningf("backgroudTaskForGenerateRange panic: [%v]", r)
+		}
+	}()
+
+	// doing range generator
+	for i := 0; ; i++ {
+		log.SetLogid(fmt.Sprintf("uidcreator%07d", i))
+		log.PrintPretty("backgroud info, ranges:", idg.Ranges)
+		for prefix, queue := range idg.Ranges {
+			var qlen = queue.Len()
+			var timeoutMs = 1 * time.Millisecond
+
+			if qlen <= 0 {
+				timeoutMs = 1 * time.Millisecond
+			} else if qlen <= 2 {
+				timeoutMs = 50 * time.Millisecond
+			} else {
+				timeoutMs = -1000 * time.Millisecond
+			}
+
+			if int(timeoutMs) > 0 {
+				idg.createNewRangeWithErrorPanic(prefix)
+				time.Sleep(timeoutMs)
+			} else if int(timeoutMs) < 0 {
+				time.Sleep(-timeoutMs)
+			}
+		}
+		time.Sleep(3000 * time.Millisecond)
+	}
+}
+
 func (idg *idCreator) GetId(idparts ...interface{}) string {
+	var retry int
 	var prefix = idg.Helper.CreatePrefix(idparts...)
-	for {
+	for retry = 0; retry < 50; retry++ {
 		var id int64 = 0
 		var currange = idg.getCurrentRange(prefix)
 		log.Infof("success get current range:(%s, %d-%d)",
@@ -118,6 +138,7 @@ func (idg *idCreator) GetId(idparts ...interface{}) string {
 			id = atomic.LoadInt64(&currange.Next)
 			if id > currange.End {
 				//next range
+				currange.Status = RANGE_STATUS_USEOUT
 				break
 			}
 			if atomic.CompareAndSwapInt64(&currange.Next, id, id+1) {
@@ -126,7 +147,7 @@ func (idg *idCreator) GetId(idparts ...interface{}) string {
 			}
 		}
 
-		// Step 2. No lock add failed, try add id in next range.
+		// Step 2. Presious [no lock] adding failed, try add id in next range.
 		if id > currange.End {
 			//next range
 			log.Infof("failed get id from current range, try next")
@@ -138,6 +159,8 @@ func (idg *idCreator) GetId(idparts ...interface{}) string {
 		log.Infof("success get id(not miexed):(%s, %d)", prefix, id)
 		return idg.Helper.CreateMixedId(prefix, id, idparts...)
 	}
+	log.Errorf("getid from range retry too much time(%d)", retry)
+	panic("getid from range retry too much time")
 }
 
 func (idg *idCreator) createDefaultPrefix() (prefix string, err error) {
@@ -165,17 +188,7 @@ func (idg *idCreator) removeOldRangeWithErrorPanic(prefix string) {
 }
 
 func (idg *idCreator) createNewRange(prefix string) error {
-	// Step 1. create range from db or something else.
+	// Step 1. lock generator.
-	start, length, err := idg.Helper.CreateRange(prefix)
-	if err != nil {
-		str := fmt.Sprintf(
-			"idCreator '%s' generate range err:'%s'",
-			idg.Type, err.Error())
-		log.Errorf("%s", str)
-		return errors.New(str)
-	}
-
-	// Step 2. lock generator.
 	idg.lock.Lock()
 	defer idg.lock.Unlock()
 	if _, ok := idg.RangesLocks[prefix]; !ok {
@@ -186,6 +199,21 @@ func (idg *idCreator) createNewRange(prefix string) error {
 	var lock = idg.RangesLocks[prefix]
 	lock.Lock()
 	defer lock.Unlock()
+	var queueLen = queue.Len()
+	if queueLen >= 30 {
+		log.Infof("ignore createNewRange beacuse of queue is long enough(%d)", queueLen)
+		return nil
+	}
+
+	// Step 2. create range from db or something else.
+	start, length, err := idg.Helper.CreateRange(prefix)
+	if err != nil {
+		str := fmt.Sprintf(
+			"idCreator '%s' generate range err:'%s'",
+			idg.Type, err.Error())
+		log.Errorf("%s", str)
+		return errors.New(str)
+	}
 
 	// Step 3. push to ranges queue.
 	var r = &Range{
@@ -210,17 +238,20 @@ func (idg *idCreator) removeOldRange(prefix string) error {
 		for queue.Len() > 0 {
 			r := queue.Front()
 			if r.Status == RANGE_STATUS_OK {
-				break
+				log.Infof("success remove all useout ranges for prefix:%s", prefix)
+				return nil
 			} else if r.Status == RANGE_STATUS_USEOUT {
-				queue.PopFront()
+				if rr := queue.PopFront(); rr != r {
-				log.Infof("success remove old range:(%s,%d-%d)",
+					panic("queue head is changed, maybe somethong bug")
+				}
+				log.Infof("success remove old range:(%s,%d-%d), try remove next",
 					prefix, r.Start, r.End)
 			} else {
 				panic("invalid status, maybe somethong bug")
 			}
 		}
 	} else {
-		log.Infof("rare case: rangequeue:(%s) have remove by others", prefix)
+		log.Infof("rare case: rangequeue:(%s) have removed by others", prefix)
 	}
 	return nil
 }
@@ -270,11 +301,10 @@ func (idg *idCreator) turnToNextRange(r *Range) {
 	// We donot lock the range, because there is no too much
 	// side effect even if 'createNewRange' execute twice.
 	switch r.Status {
-	case RANGE_STATUS_OK:
+	case RANGE_STATUS_USEOUT:
-		r.Status = RANGE_STATUS_USEOUT
 		idg.createNewRangeWithErrorPanic(r.Prefix)
 		idg.removeOldRangeWithErrorPanic(r.Prefix)
-	case RANGE_STATUS_USEOUT:
+	case RANGE_STATUS_OK:
 		// have turn by others, we do nothing.
 		log.Debugf("have turn by others, we do nothing.")
 	default: