Merge pull request #16 from zr-hebo/realtime-qps

Realtime qps

communicator/config.go

@@ -9,6 +9,7 @@ import (
 
 const (
 	CAPTURE_PACKET_RATE = "capture_packet_rate"
+	QPS = "qps"
 )
 
 var (
@@ -38,4 +39,5 @@ func init() {
 
 func regsiterConfig()  {
 	configMap[CAPTURE_PACKET_RATE] = catpurePacketRate
+	configMap[QPS] = &qpsConfig{}
 }

communicator/qps.go

@@ -0,0 +1,90 @@
+package communicator
+
+import (
+	"fmt"
+	"math"
+	"sync"
+	"time"
+)
+
+const (
+	CACHE_SIZE              = 1024
+	STATISTIC_SECONDS int64 = 5
+)
+
+var (
+	execTimeChan  chan int64
+	execTimeCache []int64
+	qpsLock       sync.Mutex
+)
+
+func init() {
+	execTimeChan = make(chan int64, 256)
+	execTimeCache = make([]int64, 0, CACHE_SIZE)
+	go updateCachedExecTime()
+}
+
+type qpsConfig struct{}
+
+func (qc *qpsConfig) setVal(val interface{}) (err error) {
+	err = fmt.Errorf("cannot set QPS on sniffer")
+	return
+}
+
+func (qc *qpsConfig) getVal() (val interface{}) {
+	return computeQPS()
+}
+
+func ReceiveExecTime(execTime int64) {
+	select {
+	case execTimeChan <- execTime:
+	default:
+		return
+	}
+}
+
+func updateCachedExecTime() {
+	for et := range execTimeChan {
+		qpsLock.Lock()
+		execTimeCache = append(execTimeCache, et)
+		if len(execTimeCache) > CACHE_SIZE {
+			execTimeCache = execTimeCache[1:]
+		}
+		qpsLock.Unlock()
+	}
+}
+
+func computeQPS() (qps int64) {
+	if catpurePacketRate.mysqlCPR <= 0 {
+		return 0
+	}
+
+	qpsLock.Lock()
+	defer qpsLock.Unlock()
+
+	// only deal execute time last 10 second
+	nowNano := time.Now().UnixNano()
+	lastTimeNano := nowNano - time.Second.Nanoseconds()*STATISTIC_SECONDS
+	minExecTimeNano := nowNano
+	var recentRecordNum int64
+	for _, et := range execTimeCache {
+		// ignore execute time before 10 second
+		if et < lastTimeNano {
+			continue
+		}
+
+		recentRecordNum += 1
+		if et < minExecTimeNano {
+			minExecTimeNano = et
+		}
+	}
+
+	if recentRecordNum < 1 || nowNano == minExecTimeNano {
+		return 0
+	}
+
+	qpsVal := float64(time.Second.Nanoseconds() /
+		((nowNano - minExecTimeNano) / recentRecordNum)) /
+		catpurePacketRate.mysqlCPR
+	return int64(math.Floor(qpsVal))
+}

docs/capture_rate.md

@@ -21,3 +21,8 @@ curl 'http://127.0.0.1:8088/get_config?config_name=capture_packet_rate'
 curl -XPOST -d'{"config_name":"capture_packet_rate","value":0.01}' 'http://127.0.0.1:8088/set_config?config_name=capture_packet_rate'
 ```
 
+#### Get QPS
+为了调整抓包率，sniffer提供了实时查询qps的功能
+```
+curl  'http://127.0.0.1:8088/get_config?config_name=qps'
+```

model/query_piece_pool.go

@@ -12,11 +12,10 @@ type PooledMysqlQueryPiece struct {
 
 func NewPooledMysqlQueryPiece(
 	sessionID, clientIP, visitUser, visitDB, serverIP *string,
-	clientPort, serverPort int, throwPacketRate float64, stmtBeginTime int64) (
+	clientPort, serverPort int, throwPacketRate float64, stmtBeginTimeNano int64) (
 	pmqp *PooledMysqlQueryPiece) {
 	pmqp = mqpp.Dequeue()
 
-	nowInMS := time.Now().UnixNano() / millSecondUnit
 	pmqp.SessionID = sessionID
 	pmqp.ClientHost = clientIP
 	pmqp.ClientPort = clientPort
@@ -26,8 +25,8 @@ func NewPooledMysqlQueryPiece(
 	pmqp.VisitDB = visitDB
 	pmqp.SyncSend = false
 	pmqp.CapturePacketRate = throwPacketRate
-	pmqp.EventTime = stmtBeginTime
+	pmqp.EventTime = stmtBeginTimeNano / millSecondUnit
-	pmqp.CostTimeInMS = nowInMS - stmtBeginTime
+	pmqp.CostTimeInMS = (time.Now().UnixNano() - stmtBeginTimeNano) / millSecondUnit
 	pmqp.recoverPool = mqpp
 
 	return

session-dealer/mysql/session.go

@@ -12,16 +12,16 @@ import (
 )
 
 type MysqlSession struct {
-	connectionID  *string
+	connectionID      *string
-	visitUser     *string
+	visitUser         *string
-	visitDB       *string
+	visitDB           *string
-	clientIP      *string
+	clientIP          *string
-	clientPort    int
+	clientPort        int
-	srcIP         *string
+	srcIP             *string
-	srcPort       int
+	srcPort           int
-	serverIP      *string
+	serverIP          *string
-	serverPort    int
+	serverPort        int
-	stmtBeginTime int64
+	stmtBeginTimeNano int64
 	// packageOffset            int64
 	beginSeqID               int64
 	endSeqID                 int64
@@ -58,7 +58,7 @@ func NewMysqlSession(
 		srcPort:            srcPort,
 		serverIP:           serverIP,
 		serverPort:         serverPort,
-		stmtBeginTime:      time.Now().UnixNano() / millSecondUnit,
+		stmtBeginTimeNano:  time.Now().UnixNano(),
 		cachedPrepareStmt:  make(map[int][]byte, 8),
 		queryPieceReceiver: receiver,
 		closeConn:          make(chan bool, 1),
@@ -100,7 +100,7 @@ func (ms *MysqlSession) ReceiveTCPPacket(newPkt *model.TCPPacket) {
 }
 
 func (ms *MysqlSession) resetBeginTime() {
-	ms.stmtBeginTime = time.Now().UnixNano() / millSecondUnit
+	ms.stmtBeginTimeNano = time.Now().UnixNano()
 }
 
 func (ms *MysqlSession) readFromServer(respSeq int64, bytes []byte) {
@@ -318,7 +318,8 @@ func (ms *MysqlSession) GenerateQueryPiece() (qp model.QueryPiece) {
 	if mqp == nil {
 		return nil
 	}
-	mqp.GenerateJsonBytes()
+
+	communicator.ReceiveExecTime(ms.stmtBeginTimeNano)
 	return mqp
 }
 
@@ -346,5 +347,5 @@ func (ms *MysqlSession) composeQueryPiece() (mqp *model.PooledMysqlQueryPiece) {
 	}
 	return model.NewPooledMysqlQueryPiece(
 		ms.connectionID, clientIP, ms.visitUser, ms.visitDB, ms.serverIP,
-		clientPort, ms.serverPort, communicator.GetMysqlCapturePacketRate(), ms.stmtBeginTime)
+		clientPort, ms.serverPort, communicator.GetMysqlCapturePacketRate(), ms.stmtBeginTimeNano)
 }