From 5b220838d9b2e7133c207f83fcd81ed401d1d82d Mon Sep 17 00:00:00 2001 From: lanrenwo Date: Wed, 14 Sep 2022 19:55:03 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=9E=E7=8E=B0=E5=90=8E=E5=8F=B0=E9=A6=96?= =?UTF-8?q?=E9=A1=B5=E5=9B=BE=E8=A1=A8(=E7=94=A8=E6=88=B7=E5=9C=A8?= =?UTF-8?q?=E7=BA=BF=E6=95=B0=E3=80=81=E7=BD=91=E7=BB=9C=E5=90=9E=E9=87=8F?= =?UTF-8?q?=E9=87=8F=E3=80=81CPU=E3=80=81=E5=86=85=E5=AD=98)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/admin/api_statsinfo.go | 33 ++++ server/admin/server.go | 4 + server/dbdata/db.go | 2 +- server/dbdata/db_orm.go | 9 ++ server/dbdata/statsinfo.go | 231 ++++++++++++++++++++++++++++ server/dbdata/tables.go | 28 ++++ server/go.mod | 8 +- server/sessdata/start.go | 1 + server/sessdata/statsinfo.go | 102 +++++++++++++ web/src/components/LineChart.vue | 62 ++++++-- web/src/pages/Home.vue | 249 +++++++++++++++++++++++++++---- 11 files changed, 680 insertions(+), 49 deletions(-) create mode 100644 server/admin/api_statsinfo.go create mode 100644 server/dbdata/statsinfo.go create mode 100644 server/sessdata/statsinfo.go diff --git a/server/admin/api_statsinfo.go b/server/admin/api_statsinfo.go new file mode 100644 index 0000000..2ef347a --- /dev/null +++ b/server/admin/api_statsinfo.go @@ -0,0 +1,33 @@ +package admin + +import ( + "errors" + "net/http" + + "github.com/bjdgyc/anylink/dbdata" +) + +func StatsInfoList(w http.ResponseWriter, r *http.Request) { + var ok bool + _ = r.ParseForm() + action := r.FormValue("action") + scope := r.FormValue("scope") + ok = dbdata.StatsInfoIns.ValidAction(action) + if !ok { + RespError(w, RespParamErr, errors.New("不存在的图表类别")) + return + } + ok = dbdata.StatsInfoIns.ValidScope(scope) + if !ok { + RespError(w, RespParamErr, errors.New("不存在的日期范围")) + return + } + datas, err := dbdata.StatsInfoIns.GetData(action, scope) + if err != nil { + RespError(w, RespInternalErr, err) + return + } + data := make(map[string]interface{}) + data["datas"] = datas + RespSucess(w, data) +} diff --git a/server/admin/server.go b/server/admin/server.go index 807efbd..482ddf0 100644 --- a/server/admin/server.go +++ b/server/admin/server.go @@ -8,6 +8,7 @@ import ( "net/http/pprof" "github.com/bjdgyc/anylink/base" + "github.com/gorilla/handlers" "github.com/gorilla/mux" ) @@ -18,6 +19,7 @@ func StartAdmin() { r := mux.NewRouter() r.Use(authMiddleware) + r.Use(handlers.CompressHandler) // 监控检测 r.HandleFunc("/status.html", func(w http.ResponseWriter, r *http.Request) { @@ -64,6 +66,8 @@ func StartAdmin() { r.HandleFunc("/group/set", GroupSet) r.HandleFunc("/group/del", GroupDel) + r.HandleFunc("/statsinfo/list", StatsInfoList) + // pprof if base.Cfg.Pprof { r.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline).Name("debug") diff --git a/server/dbdata/db.go b/server/dbdata/db.go index 994651b..00c992c 100644 --- a/server/dbdata/db.go +++ b/server/dbdata/db.go @@ -25,7 +25,7 @@ func initDb() { } // 初始化数据库 - err = xdb.Sync2(&User{}, &Setting{}, &Group{}, &IpMap{}, &AccessAudit{}, &Policy{}) + err = xdb.Sync2(&User{}, &Setting{}, &Group{}, &IpMap{}, &AccessAudit{}, &Policy{}, &StatsNetwork{}, &StatsCpu{}, &StatsMem{}, &StatsOnline{}) if err != nil { base.Fatal(err) } diff --git a/server/dbdata/db_orm.go b/server/dbdata/db_orm.go index c120b69..656ff01 100644 --- a/server/dbdata/db_orm.go +++ b/server/dbdata/db_orm.go @@ -70,6 +70,15 @@ func Find(data interface{}, limit, page int) error { return xdb.Limit(limit, start).Find(data) } +func FindWhere(data interface{}, limit int, page int, where string, args ...interface{}) error { + if limit == 0 { + return xdb.Where(where, args...).Find(data) + } + + start := (page - 1) * limit + return xdb.Where(where, args...).Limit(limit, start).Find(data) +} + func CountPrefix(fieldName string, prefix string, data interface{}) int { n, _ := xdb.Where(fieldName+" like ?", prefix+"%").Count(data) return int(n) diff --git a/server/dbdata/statsinfo.go b/server/dbdata/statsinfo.go new file mode 100644 index 0000000..dd53b2c --- /dev/null +++ b/server/dbdata/statsinfo.go @@ -0,0 +1,231 @@ +package dbdata + +import ( + "container/list" + "errors" + "fmt" + "strconv" + "time" + + "github.com/bjdgyc/anylink/base" +) + +const ( + LayoutTimeFormat = "2006-01-02 15:04:05" + LayoutTimeFormatMin = "2006-01-02 15:04" + RealTimeMaxSize = 120 // 实时数据最大保存条数 +) + +type StatsInfo struct { + RealtimeData map[string]*list.List + Actions []string + Scopes []string +} + +type ScopeDetail struct { + sTime time.Time + eTime time.Time + minutes int + fsTime string + feTime string +} + +var StatsInfoIns *StatsInfo + +func init() { + StatsInfoIns = &StatsInfo{ + Actions: []string{"online", "network", "cpu", "mem"}, + Scopes: []string{"rt", "1h", "24h", "3d", "7d", "30d"}, + RealtimeData: make(map[string]*list.List), + } + for _, v := range StatsInfoIns.Actions { + StatsInfoIns.RealtimeData[v] = list.New() + } +} + +// 校验统计类型值 +func (s *StatsInfo) ValidAction(action string) bool { + for _, item := range s.Actions { + if item == action { + return true + } + } + return false +} + +// 校验日期范围值 +func (s *StatsInfo) ValidScope(scope string) bool { + for _, item := range s.Scopes { + if item == scope { + return true + } + } + return false +} + +// 设置实时统计数据 +func (s *StatsInfo) SetRealTime(action string, val interface{}) { + if s.RealtimeData[action].Len() >= RealTimeMaxSize { + ele := s.RealtimeData[action].Front() + s.RealtimeData[action].Remove(ele) + } + s.RealtimeData[action].PushBack(val) +} + +// 获取实时统计数据 +func (s *StatsInfo) GetRealTime(action string) (res []interface{}) { + for e := s.RealtimeData[action].Front(); e != nil; e = e.Next() { + res = append(res, e.Value) + } + return +} + +// 保存数据至数据库 +func (s *StatsInfo) SaveStatsInfo(so *StatsOnline, sn *StatsNetwork, sc *StatsCpu, sm *StatsMem) { + if so.Num != 0 { + _ = Add(so) + } + if sn.Up != 0 || sn.Down != 0 { + _ = Add(sn) + } + if sc.Percent != 0 { + _ = Add(sc) + } + if sm.Percent != 0 { + _ = Add(sm) + } +} + +// 获取统计数据 +func (s *StatsInfo) GetData(action string, scope string) (res []interface{}, err error) { + if scope == "rt" { + return s.GetRealTime(action), nil + } + statsMaps := make(map[string]interface{}) + currSec := fmt.Sprintf("%02d", time.Now().Second()) + + // 获取时间段数据 + sd := s.getScopeDetail(scope) + timeList := s.getTimeList(sd) + res = make([]interface{}, len(timeList)) + + // 获取数据库查询条件 + where := s.getStatsWhere(sd) + if where == "" { + return nil, errors.New("不支持的数据库类型: " + base.Cfg.DbType) + } + // 查询数据表 + switch action { + case "online": + statsRes := []StatsOnline{} + FindWhere(&statsRes, 0, 0, where, sd.fsTime, sd.feTime) + for _, v := range statsRes { + t := v.CreatedAt.Format(LayoutTimeFormatMin) + statsMaps[t] = v + } + case "network": + statsRes := []StatsNetwork{} + FindWhere(&statsRes, 0, 0, where, sd.fsTime, sd.feTime) + for _, v := range statsRes { + t := v.CreatedAt.Format(LayoutTimeFormatMin) + statsMaps[t] = v + } + case "cpu": + statsRes := []StatsCpu{} + FindWhere(&statsRes, 0, 0, where, sd.fsTime, sd.feTime) + for _, v := range statsRes { + t := v.CreatedAt.Format(LayoutTimeFormatMin) + statsMaps[t] = v + } + case "mem": + statsRes := []StatsMem{} + FindWhere(&statsRes, 0, 0, where, sd.fsTime, sd.feTime) + for _, v := range statsRes { + t := v.CreatedAt.Format(LayoutTimeFormatMin) + statsMaps[t] = v + } + } + // 整合数据 + for i, v := range timeList { + if mv, ok := statsMaps[v]; ok { + res[i] = mv + continue + } + t, _ := time.ParseInLocation(LayoutTimeFormat, v+":"+currSec, time.Local) + switch action { + case "online": + res[i] = StatsOnline{CreatedAt: t} + case "network": + res[i] = StatsNetwork{CreatedAt: t} + case "cpu": + res[i] = StatsCpu{CreatedAt: t} + case "mem": + res[i] = StatsMem{CreatedAt: t} + } + } + return +} + +// 获取日期范围的明细值 +func (s *StatsInfo) getScopeDetail(scope string) (sd *ScopeDetail) { + sd = &ScopeDetail{} + t := time.Now() + sd.eTime = time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), 59, t.Nanosecond(), time.Local) + sd.minutes = 0 + switch scope { + case "1h": + sd.sTime = sd.eTime.Add(-time.Minute * 60) + sd.minutes = 1 + case "24h": + sd.sTime = sd.eTime.AddDate(0, 0, -1) + sd.minutes = 5 + case "7d": + sd.sTime = sd.eTime.AddDate(0, 0, -7) + sd.minutes = 30 + case "30d": + sd.sTime = sd.eTime.AddDate(0, 0, -30) + sd.minutes = 150 + } + if sd.minutes != 0 { + sd.sTime = sd.sTime.Add(-time.Minute * time.Duration(sd.minutes)) + } + sd.fsTime = sd.sTime.Format(LayoutTimeFormat) + sd.feTime = sd.eTime.Format(LayoutTimeFormat) + // UTC + switch base.Cfg.DbType { + case "sqlite3", "postgres": + sd.fsTime = sd.sTime.UTC().Format(LayoutTimeFormat) + sd.feTime = sd.eTime.UTC().Format(LayoutTimeFormat) + } + return +} + +// 针对日期范围进行拆解 +func (s *StatsInfo) getTimeList(sd *ScopeDetail) []string { + subSec := int64(60 * sd.minutes) + count := (sd.eTime.Unix()-sd.sTime.Unix())/subSec - 1 + eTime := sd.eTime.Unix() - subSec + timeLists := make([]string, count) + for i := count - 1; i >= 0; i-- { + timeLists[i] = time.Unix(eTime, 0).Format(LayoutTimeFormatMin) + eTime = eTime - subSec + } + return timeLists +} + +// 获取where条件 +func (s *StatsInfo) getStatsWhere(sd *ScopeDetail) (where string) { + where = "created_at BETWEEN ? AND ?" + min := strconv.Itoa(sd.minutes) + switch base.Cfg.DbType { + case "mysql": + where += " AND floor(TIMESTAMPDIFF(SECOND, created_at, '" + sd.feTime + "') / 60) % " + min + " = 0" + case "sqlite3": + where += " AND CAST(ROUND((JULIANDAY('" + sd.feTime + "') - JULIANDAY(created_at)) * 86400) / 60 as integer) % " + min + " = 0" + case "postgres": + where += " AND floor((EXTRACT(EPOCH FROM " + sd.feTime + ") - EXTRACT(EPOCH FROM created_at)) / 60) % " + min + " = 0" + default: + where = "" + } + return +} diff --git a/server/dbdata/tables.go b/server/dbdata/tables.go index 6bb0bea..9249643 100644 --- a/server/dbdata/tables.go +++ b/server/dbdata/tables.go @@ -84,3 +84,31 @@ type Policy struct { CreatedAt time.Time `json:"created_at" xorm:"DateTime created"` UpdatedAt time.Time `json:"updated_at" xorm:"DateTime updated"` } + +type StatsOnline struct { + Id int `json:"id" xorm:"pk autoincr not null"` + Num int `json:"num" xorm:"Int"` + NumGroups string `json:"num_groups" xorm:"varchar(500) not null"` + CreatedAt time.Time `json:"created_at" xorm:"DateTime created index"` +} + +type StatsNetwork struct { + Id int `json:"id" xorm:"pk autoincr not null"` + Up uint32 `json:"up" xorm:"Int"` + Down uint32 `json:"down" xorm:"Int"` + UpGroups string `json:"up_groups" xorm:"varchar(500) not null"` + DownGroups string `json:"down_groups" xorm:"varchar(500) not null"` + CreatedAt time.Time `json:"created_at" xorm:"DateTime created index"` +} + +type StatsCpu struct { + Id int `json:"id" xorm:"pk autoincr not null"` + Percent float64 `json:"percent" xorm:"Float"` + CreatedAt time.Time `json:"created_at" xorm:"DateTime created index"` +} + +type StatsMem struct { + Id int `json:"id" xorm:"pk autoincr not null"` + Percent float64 `json:"percent" xorm:"Float"` + CreatedAt time.Time `json:"created_at" xorm:"DateTime created index"` +} diff --git a/server/go.mod b/server/go.mod index 1ef67ff..e6fa303 100644 --- a/server/go.mod +++ b/server/go.mod @@ -4,16 +4,16 @@ go 1.16 require ( github.com/StackExchange/wmi v1.2.1 // indirect - github.com/go-ldap/ldap v3.0.3+incompatible // indirect - github.com/go-ldap/ldap/v3 v3.4.3 // indirect + github.com/go-ldap/ldap v3.0.3+incompatible github.com/go-sql-driver/mysql v1.6.0 - github.com/gocarina/gocsv v0.0.0-20220712153207-8b2118da4570 // indirect + github.com/gocarina/gocsv v0.0.0-20220712153207-8b2118da4570 github.com/golang-jwt/jwt/v4 v4.0.0 github.com/google/gopacket v1.1.19 + github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 github.com/lib/pq v1.10.2 github.com/mattn/go-sqlite3 v1.14.8 - github.com/orcaman/concurrent-map v1.0.0 // indirect + github.com/orcaman/concurrent-map v1.0.0 github.com/pion/dtls/v2 v2.0.9 github.com/pion/logging v0.2.2 github.com/shirou/gopsutil v3.21.7+incompatible diff --git a/server/sessdata/start.go b/server/sessdata/start.go index 73773ce..1d4243c 100644 --- a/server/sessdata/start.go +++ b/server/sessdata/start.go @@ -3,4 +3,5 @@ package sessdata func Start() { initIpPool() checkSession() + saveStatsInfo() } diff --git a/server/sessdata/statsinfo.go b/server/sessdata/statsinfo.go new file mode 100644 index 0000000..5a583b4 --- /dev/null +++ b/server/sessdata/statsinfo.go @@ -0,0 +1,102 @@ +package sessdata + +import ( + "encoding/json" + "sync/atomic" + "time" + + "github.com/bjdgyc/anylink/dbdata" + "github.com/shirou/gopsutil/cpu" + "github.com/shirou/gopsutil/mem" +) + +const ( + StatsCycleSec = 5 // 统计周期（秒） + AddCycleSec = 60 // 记录到数据表周期（秒） +) + +func saveStatsInfo() { + go func() { + tick := time.NewTicker(time.Second * StatsCycleSec) + count := 0 + for range tick.C { + up := uint32(0) + down := uint32(0) + upGroups := make(map[string]uint32) + downGroups := make(map[string]uint32) + numGroups := make(map[string]int) + onlineNum := 0 + sessMux.Lock() + for _, v := range sessions { + v.mux.Lock() + if v.IsActive { + // 在线人数 + onlineNum += 1 + numGroups[v.Group] += 1 + // 网络吞吐 + userUp := atomic.LoadUint32(&v.CSess.BandwidthUpPeriod) + userDown := atomic.LoadUint32(&v.CSess.BandwidthDownPeriod) + upGroups[v.Group] += userUp + downGroups[v.Group] += userDown + up += userUp + down += userDown + } + v.mux.Unlock() + } + sessMux.Unlock() + + tNow := time.Now() + // online + numData, _ := json.Marshal(numGroups) + so := &dbdata.StatsOnline{Num: onlineNum, NumGroups: string(numData), CreatedAt: tNow} + // network + upData, _ := json.Marshal(upGroups) + downData, _ := json.Marshal(downGroups) + sn := &dbdata.StatsNetwork{Up: up, Down: down, UpGroups: string(upData), DownGroups: string(downData), CreatedAt: tNow} + // cpu + sc := &dbdata.StatsCpu{Percent: getCpuPercent(), CreatedAt: tNow} + // mem + sm := &dbdata.StatsMem{Percent: getMemPercent(), CreatedAt: tNow} + count++ + // 是否保存至数据库 + save := count*StatsCycleSec >= AddCycleSec + // 历史数据 + if save { + count = 0 + } + // 设置统计数据 + setStatsData(save, so, sn, sc, sm) + } + }() +} + +func setStatsData(save bool, so *dbdata.StatsOnline, sn *dbdata.StatsNetwork, sc *dbdata.StatsCpu, sm *dbdata.StatsMem) { + // 实时数据 + dbdata.StatsInfoIns.SetRealTime("online", so) + dbdata.StatsInfoIns.SetRealTime("network", sn) + dbdata.StatsInfoIns.SetRealTime("cpu", sc) + dbdata.StatsInfoIns.SetRealTime("mem", sm) + if !save { + return + } + dbdata.StatsInfoIns.SaveStatsInfo(so, sn, sc, sm) +} + +func getCpuPercent() float64 { + cpuUsedPercent, _ := cpu.Percent(0, false) + percent := cpuUsedPercent[0] + if percent == 0 { + percent = 1 + } + return decimal(percent) +} + +func getMemPercent() float64 { + m, _ := mem.VirtualMemory() + return decimal(m.UsedPercent) +} + +func decimal(f float64) float64 { + i := int(f * 100) + return float64(i) / 100 +} diff --git a/web/src/components/LineChart.vue b/web/src/components/LineChart.vue index 09cd46a..ac881cd 100644 --- a/web/src/components/LineChart.vue +++ b/web/src/components/LineChart.vue @@ -1,5 +1,5 @@ diff --git a/web/src/pages/Home.vue b/web/src/pages/Home.vue index 10b7629..e66a75a 100644 --- a/web/src/pages/Home.vue +++ b/web/src/pages/Home.vue @@ -39,18 +39,63 @@ - + - - - - - - + + + +
+ + 实时 + 1小时 + 24小时 + 7天 + 30天 + +
+ + + +
+ + 实时 + 1小时 + 24小时 + 7天 + 30天 + +
+ + + + +
+ + 实时 + 1小时 + 24小时 + 7天 + 30天 + +
+ + + +
+ + 实时 + 1小时 + 24小时 + 7天 + 30天 + +
+ + @@ -60,24 +105,6 @@ import countTo from 'vue-count-to'; import LineChart from "@/components/LineChart"; import axios from "axios"; -const lineChartUser = { - title: '每日在线统计', - xname: ['2019-12-13', '2019-12-14', '2019-12-15', '2019-12-16', '2019-12-17', '2019-12-18', '2019-12-19'], - xdata: { - 'test1': [10, 120, 11, 134, 105, 10, 15], - 'test2': [10, 82, 91, 14, 162, 10, 15] - } -} - -const lineChartOrder = { - title: '每日流量统计', - xname: ['2019-12-13', '2019-12-14', '2019-12-15', '2019-12-16', '2019-12-17', '2019-12-18', '2019-12-19'], - xdata: { - 'test1': [100, 120, 161, 134, 105, 160, 165], - 'test2': [120, 82, 91, 154, 162, 140, 145] - } -} - export default { name: "Home", components: { @@ -92,8 +119,48 @@ export default { group: 0, ip_map: 0, }, - lineChartUser: lineChartUser, - lineChartOrder: lineChartOrder, + lineChart: { + online: { + title: '用户在线数', + xname: [], + xdata: { + '在线人数': [], + }, + yminInterval: 1, + yname:"人数" + }, + network: { + title: '网络吞吐量', + xname: [], + xdata: { + '下行流量': [], + '上行流量': [], + }, + yname:"Mbps" + }, + cpu: { + title: 'CPU占用比例', + xname: [], + xdata: { + 'CPU': [], + }, + yname:"%" + }, + mem: { + title: '内存占用比例', + xname: [], + xdata: { + '内存': [], + }, + yname:"%" + } + }, + lineChartScope : { + online: "rt", + network : "rt", + cpu : "rt", + mem : "rt" + }, } }, created() { @@ -102,6 +169,13 @@ export default { }, mounted() { this.getData() + this.getAllStats() + const chartsTimer = setInterval(() => { + this.getAllStats() + }, 5000); + this.$once('hook:beforeDestroy', () => { + clearInterval(chartsTimer); + }) }, methods: { getData() { @@ -114,9 +188,108 @@ export default { console.log(error); }); }, + getAllStats() { + for (var action in this.lineChartScope){ + if (this.lineChartScope[action] == "rt") { + this.getStatsData(action); + } + } + }, + getStatsData(action, scope) { + if (!scope) { + scope = "rt" + } + let getData = {params:{"action": action, "scope": scope}} + axios.get('/statsinfo/list', getData).then(resp => { + var data = resp.data.data + if (! data.datas) return ; + data.action = action + data.scope = scope + switch(action) { + case "online": this.formatOnline(data); break; + case "network": this.formatNetwork(data); break; + case "cpu": this.formatCpu(data); break; + case "mem": this.formatMem(data); break; + } + }).catch(error => { + this.$message.error('哦，请求出错'); + console.log(error); + }); + }, + formatOnline(data) { + let timeFormat = data.scope == "rt" || data.scope == "1h" || data.scope == "24h" ? "h:i:s" : "m/d h:i:s" + let datas = data.datas + this.lineChart.online.xname = [] + this.lineChart.online.xdata["在线人数"] = [] + for(var i=0; i @@ -124,6 +297,7 @@ export default {