add instancee

api.go

@@ -111,6 +111,9 @@ func middlewareMyApiEngine() gin.HandlerFunc {
 
 		// Step 3. do request
 		c.Next()
+
+		// Step 4. metrics record
+		cc.recordMetric()
 	}
 }
 
@@ -143,6 +146,17 @@ func ApiHandler(c *gin.Context) Handler {
 	return newc.Keys["handler"].(Handler)
 }
 
+func RecordMetrics(api, errcode, appid string, costms float64) {
+	var (
+		counter = metricApiCounter.WithLabelValues(api, errcode, appid)
+		summary = metricApiSummary.WithLabelValues(api, errcode, appid)
+	)
+	// Metric 1. api request counter
+	counter.Inc()
+	// Metric 2. api request cost/latency
+	summary.Observe(costms)
+}
+
 func DumpHandler(c *Context) error {
 	errmsg := "api not implemented"
 	errcode := errCodeUnimplementApi

context.go

@@ -149,9 +149,6 @@ func (c *Context) RESULT(output interface{}) error {
 		cost := time.Now().Sub(c.starttime).Milliseconds()
 		noticeStr := fmt.Sprintf("cost: %dms, output: '%s'", cost, string(b))
 		log.NoticefWithDepth(calldepth, noticeStr)
-
-		//// metric ////////////////////////////////////////
-		c.recordMetric()
 	}(&output)
 
 	if _, ok := t.FieldByName("ErrCode"); !ok {
@@ -227,12 +224,6 @@ func (c *Context) recordMetric() {
 		costus  = time.Now().Sub(c.starttime).Microseconds()
 		costms  = float64(costus) / 1000.0
 		errcode = strconv.Itoa(c.errcode)
-		counter = metricApiCounter.WithLabelValues(api, errcode, appid)
-		summary = metricApiSummary.WithLabelValues(api, errcode, appid)
 	)
-
-	// Metric 1. api request counter
-	counter.Inc()
-	// Metric 2. api request cost/latency
-	summary.Observe(costms)
+	RecordMetrics(api, errcode, appid, costms)
 }

prometheus.go

@@ -1,16 +1,18 @@
 package api
 
 import (
+	"errors"
 	"fmt"
 	"net/http"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/prometheus/client_golang/prometheus/push"
+	"qoobing.com/gomod/log"
 )
 
 var (
-	MetricExporterPusher                         = func() {}
+	MetricExporterPusher                         = func() error { return nil }
 	MetricExporterHandler Handler                = nil
 	metricRegistry        *prometheus.Registry   = nil
 	metricApiCounter      *prometheus.CounterVec = nil
@@ -33,11 +35,18 @@ func (m *metricExporterHandler) HandlerFunc() HandlerFunc {
 	}
 }
 
+func dumpMetricExporterPusher() error {
+	return errors.New("not initilized, forgot call 'SetupMetricsExporterPusher'?")
+}
+
 func InitMetrics() {
+	instance := getInstanceIpAddress()
+	log.Infof("self instance ip: [%s]", instance)
 	metricApiCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Name:        "api_requests_total",
 			Help:        "How many HTTP requests processed",
+			ConstLabels: prometheus.Labels{"instance": instance},
 		},
 		[]string{"api", "errcode", "appid"},
 	)
@@ -46,6 +55,7 @@ func InitMetrics() {
 			Name:        "api_requests_summary",
 			Help:        "The api request summary of cost.",
 			Objectives:  map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+			ConstLabels: prometheus.Labels{"instance": instance},
 		},
 		[]string{"api", "errcode", "appid"},
 	)
@@ -72,7 +82,8 @@ func SetupMetricsExporterPusher(pushgateway string, jobname string) {
 	}
 
 	var pusher = push.New(pushgateway, jobname).Gatherer(metricRegistry)
-	MetricExporterPusher = func() {
-		pusher.Push()
+	MetricExporterPusher = func() error {
+		RecordMetrics("push_metrics_to_prometheus", "0", "selfmonitoring", 1.0)
+		return pusher.Push()
 	}
 }

util.go

@@ -1,8 +1,11 @@
 package api
 
 import (
+	"fmt"
+	"net"
 	"os"
 	"path/filepath"
+	"regexp"
 )
 
 // getExeFilename
@@ -13,3 +16,55 @@ func getExeFilename() string {
 	}
 	return logfilename
 }
+
+// preferencesMatch
+func preferencesMatch(s string, preferences ...string) bool {
+	for _, pstr := range preferences {
+		reg := regexp.MustCompile(pstr)
+		if reg.Match([]byte(s)) {
+			return true
+		}
+	}
+	return false
+}
+
+// getInstanceIpAddress
+func getInstanceIpAddress(preferences ...string) string {
+	addrs, err := net.InterfaceAddrs()
+	if err != nil {
+		panic(fmt.Sprintf("getInstanceIpAddress failed: [%s]", err))
+	}
+
+	prefer := func() func(string) bool {
+		if len(preferences) <= 0 {
+			return nil
+		}
+		return func(ip string) bool {
+			return preferencesMatch(ip, preferences...)
+		}
+	}()
+
+	var ipnets = []*net.IPNet{}
+	for _, address := range addrs {
+		if ipnet, ok := address.(*net.IPNet); !ok {
+			continue
+		} else if ipnet.IP.IsLoopback() {
+			continue
+		} else if ipv4 := ipnet.IP.To4(); ipv4 == nil {
+			continue
+		} else if prefer == nil {
+			var n = *ipnet
+			ipnets = append(ipnets, &n)
+			continue
+		} else if prefer != nil && prefer(ipv4.String()) {
+			return ipv4.String()
+		}
+	}
+
+	// TODO: sort & return first
+	if len(ipnets) > 0 {
+		return ipnets[0].IP.To4().String()
+	}
+	panic(fmt.Sprintf("getInstanceIpAddreass failed,"+
+		"preferences=[%v], ipnets=[%v]", preferences, ipnets))
+}