change log

go.mod

@@ -2,4 +2,16 @@ module qoobing.com/gomod/uid
 
 go 1.19
 
-require github.com/gammazero/deque v0.2.1
+require (
+	github.com/gammazero/deque v0.2.1
+	gorm.io/gorm v1.25.0
+	qoobing.com/gomod/log v1.2.2
+)
+
+require (
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
+	github.com/tylerb/gls v0.0.0-20150407001822-e606233f194d // indirect
+	github.com/tylerb/is v2.1.4+incompatible // indirect
+	qoobing.com/gomod/str v1.0.1 // indirect
+)

go.sum

@@ -1,2 +1,16 @@
 github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0=
 github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
+github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/tylerb/gls v0.0.0-20150407001822-e606233f194d h1:yYYPFFlbqxF5mrj5sEfETtM/Ssz2LTy0/VKlDdXYctc=
+github.com/tylerb/gls v0.0.0-20150407001822-e606233f194d/go.mod h1:0MwyId/pXK5wkYYEXe7NnVknX+aNBuF73fLV3U0reU8=
+github.com/tylerb/is v2.1.4+incompatible h1:BMf2zP0kY2Ykzx2W1fDrjwKj1x1B4E0mELkpjaNy1tM=
+github.com/tylerb/is v2.1.4+incompatible/go.mod h1:3Bw2NWEEe8Kx7/etYqgm9ug53iNDgabnloch75jjOSc=
+gorm.io/gorm v1.25.0 h1:+KtYtb2roDz14EQe4bla8CbQlmb9dN3VejSai3lprfU=
+gorm.io/gorm v1.25.0/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
+qoobing.com/gomod/log v1.2.2 h1:6h7cgfhIm4cQNASC7KDJ8TyLn3pb7eYJV9JO+vXYwDA=
+qoobing.com/gomod/log v1.2.2/go.mod h1:/ZTN/ukAbSqRb4eMlF9LpfkVgM21xwprbd5y3tcQxpM=
+qoobing.com/gomod/str v1.0.1 h1:X+JOigE9xA6cTNph7/s1KeD4zLYM9XTLPPHQcpHFoog=
+qoobing.com/gomod/str v1.0.1/go.mod h1:gbhN2dba/P5gFRGVJvEI57KEJLlMHHAd6Kuuxn4GlMY=

test/main.go

@@ -0,0 +1,85 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"qoobing.com/gomod/log"
+	"qoobing.com/gomod/uid"
+)
+
+type MemCreatorHelper struct {
+	start  int64
+	length int
+	lock   sync.Mutex
+}
+
+func NewMemCreatorHelper() *MemCreatorHelper {
+	return &MemCreatorHelper{
+		lock:   sync.Mutex{},
+		start:  0,
+		length: 100,
+	}
+}
+
+// CreatePrefix create id prefix
+func (mch *MemCreatorHelper) CreatePrefix(parts ...interface{}) string {
+	return "XXXPPPYYY"
+}
+
+// CreateRange create id range from database table t_id.
+func (mch *MemCreatorHelper) CreateRange(prefix string) (start int64, length int, err error) {
+	mch.lock.Lock()
+	defer mch.lock.Unlock()
+	fmt.Println("CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCreateRange")
+
+	mch.start += int64(mch.length)
+	if mch.start == 300 {
+		panic("test panic: 3")
+	} else if mch.start == 7000 {
+		return mch.start, mch.length, errors.New("tesk error: 7")
+	}
+	return mch.start, mch.length, nil
+}
+
+// // CreateMixedId create finnal transaction id
+func (mch *MemCreatorHelper) CreateMixedId(prefix string, id int64, parts ...interface{}) string {
+	return fmt.Sprintf("id-%s-%d", prefix, id)
+}
+
+var helper = NewMemCreatorHelper()
+var idcreator = uid.NewIdCreator("test", helper)
+
+func getId(i, ii int) {
+	defer func() {
+		if r := recover(); r != nil {
+			fmt.Println("panic recover:", r)
+		}
+	}()
+	fmt.Printf("%d-%d: %s\n", i, ii, idcreator.GetId())
+}
+
+func main() {
+	c := atomic.Uint64{}
+	st := time.Now()
+	wg := sync.WaitGroup{}
+	wg.Add(1000)
+	for i := 0; i < 1000; i++ {
+		go func(i int) {
+			log.SetLogid(fmt.Sprintf("main idcreator%07d", i))
+			for ii := 0; ii < 1000; ii++ {
+				getId(i, ii)
+				c.Add(uint64(1))
+			}
+			wg.Done()
+		}(i)
+	}
+	wg.Wait()
+	et := time.Now()
+	fmt.Println("cost:", et.Sub(st))
+	fmt.Println("helper:", helper)
+	fmt.Println("all done, total id number is:", c.Load())
+}

uid.go

@@ -33,10 +33,25 @@ type IdCreatorHelper interface {
 	CreateMixedId(prefix string, id int64, parts ...interface{}) string
 }
 
+// Range Range define the range [start, end] of ids.
+type Range struct {
+	Next   int64  // next id
+	Status int    // 0:invlid; 1:ok; 2:useout
+	Prefix string // prefix
+	Start  int64  // begin id of this range(include)
+	End    int64  // end id of this range(include)
+	Length int    // lenght of this range
+}
 type rangeQueue struct {
 	deque.Deque[*Range]
 }
 
+const (
+	RANGE_STATUS_OK      = 1
+	RANGE_STATUS_USEOUT  = 2
+	RANGE_STATUS_REMOVED = 3
+)
+
 // idCreator
 type idCreator struct {
 	lock        sync.Mutex
@@ -59,34 +74,15 @@ func NewIdCreator(typ string, helper IdCreatorHelper) IdCreator {
 
 	// Step 2. generate the first range.
 	if prefix, err := idg.createDefaultPrefix(); err == nil {
-		if err := idg.createNewRange(prefix); err != nil {
-			log.Errorf("generate the first range failed: %s", err)
-			panic("NewIdCreator error")
-		}
+		idg.createNewRangeWithErrorPanic(prefix)
 	}
 
 	// Step 3. backgroud task for generate range
 	go func() {
 		for {
-			for prefix, queue := range idg.Ranges {
-				var qlen = queue.Len()
-				var timeoutMs = 1 * time.Millisecond
-
-				if qlen <= 0 {
-					timeoutMs = 1 * time.Millisecond
-				} else if qlen <= 2 {
-					timeoutMs = 50 * time.Millisecond
-				} else {
-					timeoutMs = -1000 * time.Millisecond
-				}
-
-				if int(timeoutMs) > 0 {
-					idg.createNewRange(prefix)
-					time.Sleep(timeoutMs * time.Millisecond)
-				} else if int(timeoutMs) < 0 {
-					time.Sleep((-timeoutMs) * time.Millisecond)
-				}
-			}
+			log.Infof("start backgroud range generater, type=%s", typ)
+			idg.backgroudTaskForGenerateRange()
+			log.Infof("unexpect backgroud range generater exit")
 			time.Sleep(1000 * time.Millisecond)
 		}
 	}()
@@ -94,17 +90,61 @@ func NewIdCreator(typ string, helper IdCreatorHelper) IdCreator {
 	return idg
 }
 
+func (idg *idCreator) backgroudTaskForGenerateRange() {
+	// rocover panic
+	defer func() {
+		if r := recover(); r != nil {
+			log.Warningf("backgroudTaskForGenerateRange panic: [%v]", r)
+		}
+	}()
+
+	// doing range generator
+	for i := 0; ; i++ {
+		logStr := ""
+		log.SetLogid(fmt.Sprintf("uidcreator%07d", i))
+		for prefix, queue := range idg.Ranges {
+			var qlen = queue.Len()
+			var timeoutMs = 1 * time.Millisecond
+
+			if qlen <= 0 {
+				timeoutMs = 1 * time.Millisecond
+			} else if qlen <= 2 {
+				timeoutMs = 50 * time.Millisecond
+			} else {
+				timeoutMs = -1000 * time.Millisecond
+			}
+
+			if int(timeoutMs) > 0 {
+				idg.createNewRangeWithErrorPanic(prefix)
+				time.Sleep(timeoutMs)
+			} else if int(timeoutMs) < 0 {
+				time.Sleep(-timeoutMs)
+			}
+			if i%50 == 0 {
+				logStr += fmt.Sprintf("queue-legth:(%s,%d);", prefix, queue.Len())
+			}
+		}
+		if i%50 == 0 {
+			log.Infof("backgroud idcreator info: %s", logStr)
+		}
+		time.Sleep(3000 * time.Millisecond)
+	}
+}
+
 func (idg *idCreator) GetId(idparts ...interface{}) string {
+	var retry int
 	var prefix = idg.Helper.CreatePrefix(idparts...)
-	for {
+	for retry = 0; retry < 50; retry++ {
 		var id int64 = 0
 		var currange = idg.getCurrentRange(prefix)
-		log.Infof("success get current range:(%s, %d)", prefix, currange.Start)
+		log.Infof("success get current range:(%s, %d-%d)",
+			prefix, currange.Start, currange.End)
 		// Step 1. Optimistic[no lock] to add id in current range.
 		for {
 			id = atomic.LoadInt64(&currange.Next)
 			if id > currange.End {
 				//next range
+				currange.Status = RANGE_STATUS_USEOUT
 				break
 			}
 			if atomic.CompareAndSwapInt64(&currange.Next, id, id+1) {
@@ -113,7 +153,7 @@ func (idg *idCreator) GetId(idparts ...interface{}) string {
 			}
 		}
 
-		// Step 2. No lock add failed, try add id in next range.
+		// Step 2. Presious [no lock] adding failed, try add id in next range.
 		if id > currange.End {
 			//next range
 			log.Infof("failed get id from current range, try next")
@@ -125,16 +165,8 @@ func (idg *idCreator) GetId(idparts ...interface{}) string {
 		log.Infof("success get id(not miexed):(%s, %d)", prefix, id)
 		return idg.Helper.CreateMixedId(prefix, id, idparts...)
 	}
-}
-
-// Range Range define the range [start, end] of ids.
-type Range struct {
-	Next   int64  // next id
-	Status int    // 0:invlid; 1:ok; 2:used
-	Prefix string // prefix
-	Start  int64  // begin id of this range(include)
-	End    int64  // end id of this range(include)
-	Length int    // lenght of this range
+	log.Errorf("getid from range retry too much time(%d)", retry)
+	panic("getid from range retry too much time")
 }
 
 func (idg *idCreator) createDefaultPrefix() (prefix string, err error) {
@@ -149,27 +181,37 @@ func (idg *idCreator) createDefaultPrefix() (prefix string, err error) {
 	return prefix, err
 }
 
-func (idg *idCreator) removeOldRange(prefix string) error {
-	if lock, ok := idg.RangesLocks[prefix]; ok {
-		lock.Lock()
-		defer lock.Unlock()
-
-		queue := idg.Ranges[prefix]
-		for {
-			r := queue.Front()
-			if r.Status == 1 {
-				break
-			} else if r.Status == 2 {
-				queue.PopFront()
-			} else {
-				panic("invalid status, maybe somethong bug")
-			}
-		}
+func (idg *idCreator) createNewRangeWithErrorPanic(prefix string) {
+	if err := idg.createNewRange(prefix); err != nil {
+		panic("create new range error:" + err.Error())
 	}
-	return nil
 }
+
+func (idg *idCreator) removeOldRangeWithErrorPanic(prefix string) {
+	if err := idg.removeOldRange(prefix); err != nil {
+		panic("remove old range error:" + err.Error())
+	}
+}
+
 func (idg *idCreator) createNewRange(prefix string) error {
-	// Step 1. create range from db or something else.
+	// Step 1. lock generator.
+	idg.lock.Lock()
+	defer idg.lock.Unlock()
+	if _, ok := idg.RangesLocks[prefix]; !ok {
+		idg.Ranges[prefix] = &rangeQueue{Deque: *deque.New[*Range]()}
+		idg.RangesLocks[prefix] = &sync.Mutex{}
+	}
+	var queue = idg.Ranges[prefix]
+	var lock = idg.RangesLocks[prefix]
+	lock.Lock()
+	defer lock.Unlock()
+	var queueLen = queue.Len()
+	if queueLen >= 30 {
+		log.Infof("ignore createNewRange beacuse of queue is long enough(%d)", queueLen)
+		return nil
+	}
+
+	// Step 2. create range from db or something else.
 	start, length, err := idg.Helper.CreateRange(prefix)
 	if err != nil {
 		str := fmt.Sprintf(
@@ -179,54 +221,66 @@ func (idg *idCreator) createNewRange(prefix string) error {
 		return errors.New(str)
 	}
 
-	// Step 2. lock generator.
-	idg.lock.Lock()
-	defer idg.lock.Unlock()
-	if _, ok := idg.RangesLocks[prefix]; !ok {
-		idg.Ranges[prefix] = &rangeQueue{Deque: *deque.New[*Range]()}
-		idg.RangesLocks[prefix] = &sync.Mutex{}
-	}
-	var lock = idg.RangesLocks[prefix]
-	var queue = idg.Ranges[prefix]
-	lock.Lock()
-	defer lock.Unlock()
-
 	// Step 3. push to ranges queue.
-	queue.PushBack(
-		&Range{
-			Next:   start,
-			Status: 1,
-			Prefix: prefix,
-			Start:  start,
-			End:    start + int64(length) - 1,
-			Length: length,
-		})
+	var r = &Range{
+		Next:   start,
+		Status: RANGE_STATUS_OK,
+		Prefix: prefix,
+		Start:  start,
+		End:    start + int64(length) - 1,
+		Length: length,
+	}
+	queue.PushBack(r)
+	log.Infof("success createNewRange (%s, %d-%d)", prefix, start, r.End)
+	return nil
+}
+
+func (idg *idCreator) removeOldRange(prefix string) error {
+	if lock, ok := idg.RangesLocks[prefix]; ok {
+		lock.Lock()
+		defer lock.Unlock()
+
+		queue := idg.Ranges[prefix]
+		for queue.Len() > 0 {
+			r := queue.Front()
+			if r.Status == RANGE_STATUS_OK {
+				log.Infof("success remove all useout ranges for prefix:%s", prefix)
+				return nil
+			} else if r.Status == RANGE_STATUS_USEOUT {
+				if rr := queue.PopFront(); rr != r {
+					panic("queue head is changed, maybe somethong bug")
+				}
+				log.Infof("success remove old range:(%s,%d-%d), try remove next",
+					prefix, r.Start, r.End)
+			} else {
+				panic("invalid status, maybe somethong bug")
+			}
+		}
+	} else {
+		log.Infof("rare case: rangequeue:(%s) have removed by others", prefix)
+	}
 	return nil
 }
 
 func (idg *idCreator) getCurrentRange(prefix string) (r *Range) {
-	// Step 0. get ranges queue by prefix.
 	var (
-		trycount             = 0
-		queue    *rangeQueue = nil
-		ok                   = false
+		ok                = false
+		lock  *sync.Mutex = nil
+		queue *rangeQueue = nil
 	)
-	for trycount = 0; !ok && trycount < 3; trycount++ {
-		if queue, ok = idg.Ranges[prefix]; !ok {
-			idg.createNewRange(prefix)
-		}
-	}
-	if !ok {
-		panic("maybe something bug, cannot found range queue after createNewRange")
-	}
 
-	queue, ok = idg.Ranges[prefix]
-	if !ok {
+	// Step 0. get ranges queue by prefix.
+	for trycount := 0; !ok && trycount < 3; trycount++ {
+		if queue, ok = idg.Ranges[prefix]; ok {
+			lock = idg.RangesLocks[prefix]
+			break
+		}
+		// try create new range
 		idg.createNewRange(prefix)
 	}
-	var lock, okk = idg.RangesLocks[prefix]
-	if !okk {
-		panic("unreachable code: lock is not exist")
+	if !ok {
+		panic("maybe something bug or too many pressures," +
+			"cannot found range queue after createNewRange retry 3 times")
 	}
 
 	// Step 1. try read from queue directly.
@@ -238,7 +292,7 @@ func (idg *idCreator) getCurrentRange(prefix string) (r *Range) {
 	}
 	lock.Unlock()
 
-	// Step 2. try create a new range & try again.
+	// Step 2. retry create a new range & try again.
 	idg.createNewRange(prefix)
 	lock.Lock()
 	defer lock.Unlock()
@@ -253,31 +307,13 @@ func (idg *idCreator) turnToNextRange(r *Range) {
 	// We donot lock the range, because there is no too much
 	// side effect even if 'createNewRange' execute twice.
 	switch r.Status {
-	case 1:
-		r.Status = 2
-		idg.removeOldRange(r.Prefix)
-		idg.createNewRange(r.Prefix)
-	case 2:
-		//have turn by others, we do nothing.
+	case RANGE_STATUS_USEOUT:
+		idg.createNewRangeWithErrorPanic(r.Prefix)
+		idg.removeOldRangeWithErrorPanic(r.Prefix)
+	case RANGE_STATUS_OK:
+		// have turn by others, we do nothing.
 		log.Debugf("have turn by others, we do nothing.")
 	default:
 		panic("invalid status, maybe somethong bug")
 	}
 }
-
-// demo demo range creator, return current timestamp % 86400 * lenght.
-// **DONOT** use in product enviroment.
-// **DONOT** use in product enviroment.
-// **DONOT** use in product enviroment.
-type demo struct{}
-
-func (d *demo) CreateRange(typ, prefix string) (start int64, length int, err error) {
-	var curdate string = time.Now().Format("20060102")
-	if prefix != curdate {
-		return 0, 0, fmt.Errorf("demo just support prefix is current date")
-	}
-
-	start = int64((time.Now().Unix() % 86400) * 10000)
-	length = 10000
-	return start, length, nil
-}