重构路由监控

This commit is contained in:
tanghc
2020-10-31 18:02:02 +08:00
parent 4ce2fc826d
commit 80003c44d3
58 changed files with 1966 additions and 215 deletions

View File

@@ -0,0 +1,67 @@
package com.gitee.sop.gateway.entity;
import lombok.Data;
import javax.persistence.Column;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import java.time.LocalDateTime;
import java.util.Date;
/**
* 表名monitor_info
* 备注:接口监控信息
*
* @author tanghc
*/
@Table(name = "monitor_info")
@Data
public class MonitorInfo {
/** 数据库字段id */
@Id
@Column(name = "id")
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
/** 路由id, 数据库字段route_id */
private String routeId;
/** 接口名, 数据库字段name */
private String name;
/** 版本号, 数据库字段version */
private String version;
/** 数据库字段service_id */
private String serviceId;
/** 数据库字段instance_id */
private String instanceId;
/** 请求耗时最长时间, 数据库字段max_time */
private Integer maxTime;
/** 请求耗时最小时间, 数据库字段min_time */
private Integer minTime;
/** 总时长,毫秒, 数据库字段total_time */
private Long totalTime;
/** 总调用次数, 数据库字段total_request_count */
private Long totalRequestCount;
/** 成功次数, 数据库字段success_count */
private Long successCount;
/** 失败次数(业务主动抛出的异常算作成功,如参数校验,未知的错误算失败), 数据库字段error_count */
private Long errorCount;
/** 数据库字段gmt_create */
private Date gmtCreate;
/** 数据库字段gmt_modified */
private Date gmtModified;
}

View File

@@ -0,0 +1,55 @@
package com.gitee.sop.gateway.entity;
import lombok.Data;
import javax.persistence.Column;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import java.time.LocalDateTime;
import java.util.Date;
/**
* 表名monitor_info_error
*
* @author tanghc
*/
@Table(name = "monitor_info_error")
@Data
public class MonitorInfoError {
/** 数据库字段id */
@Id
@Column(name = "id")
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
/** 错误id,md5Hex(instanceId + routeId + errorMsg), 数据库字段error_id */
private String errorId;
/** 实例id, 数据库字段instance_id */
private String instanceId;
/** 数据库字段route_id */
private String routeId;
/** 数据库字段error_msg */
private String errorMsg;
/** http status非200错误, 数据库字段error_status */
private Integer errorStatus;
/** 错误次数, 数据库字段count */
private Integer count;
/** 数据库字段is_deleted */
@com.gitee.fastmybatis.core.annotation.LogicDelete
private Byte isDeleted;
/** 数据库字段gmt_create */
private Date gmtCreate;
/** 数据库字段gmt_modified */
private Date gmtModified;
}

View File

@@ -0,0 +1,43 @@
package com.gitee.sop.gateway.interceptor;
import com.gitee.sop.gatewaycommon.interceptor.RouteInterceptor;
import com.gitee.sop.gatewaycommon.interceptor.RouteInterceptorContext;
import com.gitee.sop.gatewaycommon.param.ApiParam;
import com.gitee.sop.gatewaycommon.sync.SopAsyncConfigurer;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
/**
* 用于收集监控数据
*
* @author tanghc
*/
@Component
@Slf4j
public class MonitorRouteInterceptor implements RouteInterceptor {
@Autowired
SopAsyncConfigurer sopAsyncConfigurer;
@Autowired
MonitorRouteInterceptorService monitorRouteInterceptorService;
@Override
public void preRoute(RouteInterceptorContext context) {
}
@Override
public void afterRoute(RouteInterceptorContext context) {
sopAsyncConfigurer.getAsyncExecutor().execute(()-> {
monitorRouteInterceptorService.storeRequestInfo(context);
});
}
@Override
public int getOrder() {
return -1000;
}
}

View File

@@ -0,0 +1,164 @@
package com.gitee.sop.gateway.interceptor;
import com.gitee.sop.gateway.mapper.DbMonitorInfoManager;
import com.gitee.sop.gatewaycommon.bean.LRUCache;
import com.gitee.sop.gatewaycommon.interceptor.RouteInterceptorContext;
import com.gitee.sop.gatewaycommon.monitor.MonitorDTO;
import com.gitee.sop.gatewaycommon.monitor.MonitorData;
import com.gitee.sop.gatewaycommon.monitor.MonitorManager;
import com.gitee.sop.gatewaycommon.param.ApiParam;
import com.gitee.sop.gatewaycommon.sync.MyNamedThreadFactory;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cloud.client.ServiceInstance;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
/**
* @author tanghc
*/
@Service
@Slf4j
public class MonitorRouteInterceptorService {
/**
* 刷新到数据库时间间隔,秒
*/
@Value("${sop.monitor.flush-period-seconds:30}")
int flushPeriodSeconds;
/**
* 定时任务每n秒执行一次
*/
@Value("${sop.monitor.schedule-period-seconds:30}")
int schedulePeriodSeconds;
/**
* 错误数量容量
*/
@Value("${sop.monitor.error-count-capacity:50}")
int monitorErrorCapacity;
@Autowired
DbMonitorInfoManager dbMonitorInfoManager;
@Autowired
MonitorManager monitorManager;
/**
* 记录接口调用流量,最大时间,最小时间,总时长,平均时长,调用次数,成功次数,失败次数.
* 需要考虑并发情况。
*/
public synchronized void storeRequestInfo(RouteInterceptorContext context) {
ApiParam apiParam = context.getApiParam();
ServiceInstance serviceInstance = context.getServiceInstance();
String routeId = apiParam.getRouteId();
int spendTime = (int)(context.getFinishTimeMillis() - context.getBeginTimeMillis());
// 这步操作是线程安全的底层调用了ConcurrentHashMap.computeIfAbsent
String key = getMonitorKey(routeId, serviceInstance.getInstanceId());
MonitorData monitorData = monitorManager.getMonitorInfo(key, (k) -> this.createMonitorInfo(apiParam, serviceInstance));
monitorData.storeMaxTime(spendTime);
monitorData.storeMinTime(spendTime);
monitorData.getTotalRequestCount().incrementAndGet();
monitorData.getTotalTime().addAndGet(spendTime);
if (context.isSuccessRequest()) {
monitorData.getSuccessCount().incrementAndGet();
} else {
monitorData.getErrorCount().incrementAndGet();
String errorMsg = context.getServiceErrorMsg();
monitorData.addErrorMsg(errorMsg, context.getResponseStatus());
}
}
private String getMonitorKey(String routeId, String instanceId) {
return routeId + instanceId;
}
/**
* 刷新到数据库
*/
private synchronized void flushDb() {
Map<String, MonitorData> monitorData = monitorManager.getMonitorData();
if (monitorData.isEmpty()) {
return;
}
LocalDateTime checkTime = LocalDateTime.now();
List<String> tobeRemoveKeys = new ArrayList<>();
List<MonitorDTO> tobeSaveBatch = new ArrayList<>(monitorData.size());
monitorData.forEach((key, value) -> {
LocalDateTime flushTime = value.getFlushTime();
if (flushTime.isEqual(checkTime) || flushTime.isBefore(checkTime)) {
log.debug("刷新监控数据到数据库, MonitorData:{}", value);
tobeRemoveKeys.add(key);
MonitorDTO monitorDTO = getMonitorDTO(value);
tobeSaveBatch.add(monitorDTO);
}
});
dbMonitorInfoManager.saveMonitorInfoBatch(tobeSaveBatch);
for (String key : tobeRemoveKeys) {
monitorData.remove(key);
}
}
private MonitorDTO getMonitorDTO(MonitorData monitorData) {
MonitorDTO monitorDTO = new MonitorDTO();
monitorDTO.setRouteId(monitorData.getRouteId());
monitorDTO.setName(monitorData.getName());
monitorDTO.setVersion(monitorData.getVersion());
monitorDTO.setServiceId(monitorData.getServiceId());
monitorDTO.setInstanceId(monitorData.getInstanceId());
monitorDTO.setMaxTime(monitorData.getMaxTime());
monitorDTO.setMinTime(monitorData.getMinTime());
monitorDTO.setTotalTime(monitorData.getTotalTime().longValue());
monitorDTO.setTotalRequestCount(monitorData.getTotalRequestCount().longValue());
monitorDTO.setSuccessCount(monitorData.getSuccessCount().longValue());
monitorDTO.setErrorCount(monitorData.getErrorCount().longValue());
monitorDTO.setErrorMsgList(monitorData.getMonitorErrorMsgMap().values());
return monitorDTO;
}
private MonitorData createMonitorInfo(ApiParam apiParam, ServiceInstance serviceInstance) {
MonitorData monitorData = new MonitorData();
monitorData.setRouteId(apiParam.getRouteId());
monitorData.setName(apiParam.fetchName());
monitorData.setVersion(apiParam.fetchVersion());
monitorData.setServiceId(apiParam.fetchServiceId());
monitorData.setInstanceId(serviceInstance.getInstanceId());
monitorData.setTotalTime(new AtomicInteger());
monitorData.setMaxTime(0);
monitorData.setMinTime(0);
monitorData.setSuccessCount(new AtomicInteger());
monitorData.setTotalRequestCount(new AtomicInteger());
monitorData.setErrorCount(new AtomicInteger());
monitorData.setFlushTime(getFlushTime());
monitorData.setMonitorErrorMsgMap(new LRUCache<>(monitorErrorCapacity));
return monitorData;
}
private LocalDateTime getFlushTime() {
return LocalDateTime.now()
.plusSeconds(flushPeriodSeconds);
}
@PostConstruct
public void after() {
// 每隔schedulePeriodSeconds秒执行一次
ScheduledThreadPoolExecutor scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor(1, new MyNamedThreadFactory("monitorSchedule"));
// 延迟执行随机5~14秒
int delay = 5 + new Random().nextInt(10);
scheduledThreadPoolExecutor.scheduleWithFixedDelay(this::flushDb, delay, schedulePeriodSeconds, TimeUnit.SECONDS);
}
}

View File

@@ -0,0 +1,62 @@
package com.gitee.sop.gateway.mapper;
import com.gitee.sop.gatewaycommon.monitor.MonitorDTO;
import com.gitee.sop.gatewaycommon.monitor.MonitorErrorMsg;
import com.gitee.sop.gatewaycommon.monitor.RouteErrorCount;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
/**
* @author tanghc
*/
@Service
public class DbMonitorInfoManager {
@Autowired
private MonitorInfoMapper monitorInfoMapper;
@Autowired
private MonitorInfoErrorMapper monitorInfoErrorMapper;
@Value("${sop.monitor.error-count-capacity:50}")
int limitCount;
public void saveMonitorInfoBatch(List<MonitorDTO> list) {
if (CollectionUtils.isEmpty(list)) {
return;
}
monitorInfoMapper.saveMonitorInfoBatch(list);
this.saveMonitorInfoErrorBatch(list);
}
private void saveMonitorInfoErrorBatch(List<MonitorDTO> list) {
List<RouteErrorCount> routeErrorCounts = monitorInfoErrorMapper.listRouteErrorCountAll();
// 路由id对应的错误次数keyrouteIdvalue错误次数
Map<String, Integer> routeErrorCountsMap = routeErrorCounts.stream()
.collect(Collectors.toMap(RouteErrorCount::getRouteId, RouteErrorCount::getCount));
List<MonitorErrorMsg> monitorErrorMsgList = list.stream()
.filter(monitorDTO -> CollectionUtils.isNotEmpty(monitorDTO.getErrorMsgList()))
.flatMap(monitorDTO -> {
int limit = limitCount - routeErrorCountsMap.getOrDefault(monitorDTO.getRouteId(), 0);
// 容量已满
if (limit <= 0) {
return null;
}
// 截取剩余
return monitorDTO.getErrorMsgList().stream().limit(limit);
})
.filter(Objects::nonNull)
.collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(monitorErrorMsgList)) {
monitorInfoErrorMapper.saveMonitorInfoErrorBatch(monitorErrorMsgList);
}
}
}

View File

@@ -0,0 +1,36 @@
package com.gitee.sop.gateway.mapper;
import com.gitee.fastmybatis.core.mapper.CrudMapper;
import com.gitee.sop.gateway.entity.MonitorInfoError;
import com.gitee.sop.gatewaycommon.monitor.MonitorErrorMsg;
import com.gitee.sop.gatewaycommon.monitor.RouteErrorCount;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import org.apache.ibatis.annotations.Update;
import java.util.List;
/**
* @author tanghc
*/
public interface MonitorInfoErrorMapper extends CrudMapper<MonitorInfoError, Long> {
@Update("UPDATE monitor_info_error " +
"SET is_deleted=0 " +
",count=count + 1 " +
"WHERE route_id=#{routeId} AND error_id=#{errorId}")
int updateError(@Param("routeId") String routeId,@Param("errorId") String errorId);
int saveMonitorInfoErrorBatch(@Param("list") List<MonitorErrorMsg> list);
@Select("SELECT route_id routeId, count(*) `count` FROM monitor_info_error \n" +
"WHERE is_deleted=0 \n" +
"GROUP BY route_id")
List<RouteErrorCount> listRouteErrorCount();
@Select("SELECT route_id routeId, count(*) `count` FROM monitor_info_error \n" +
"WHERE is_deleted=0 \n" +
"GROUP BY route_id")
List<RouteErrorCount> listRouteErrorCountAll();
}

View File

@@ -0,0 +1,39 @@
package com.gitee.sop.gateway.mapper;
import com.gitee.fastmybatis.core.mapper.CrudMapper;
import com.gitee.sop.gateway.entity.MonitorInfo;
import com.gitee.sop.gatewaycommon.monitor.MonitorDTO;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Update;
import java.util.List;
/**
* @author tanghc
*/
public interface MonitorInfoMapper extends CrudMapper<MonitorInfo, Long> {
/**
* 更新监控状态
*
* @return 返回影响行数
*/
@Update("UPDATE monitor_info " +
"set max_time=case when max_time < #{maxTime} then #{maxTime} else max_time end " +
",min_time=case when min_time > #{minTime} then #{minTime} else min_time end " +
",total_request_count=total_request_count + #{totalRequestCount} " +
",total_time=total_time + #{totalTime} " +
",success_count=success_count + #{successCount} " +
",error_count=error_count + #{errorCount} " +
"where route_id=#{routeId}")
int updateMonitorInfo(MonitorDTO monitorDTO);
/**
* 批量插入监控数据
* @param list 监控数据
* @return 返回影响行数
*/
int saveMonitorInfoBatch(@Param("list") List<MonitorDTO> list);
}

View File

@@ -29,6 +29,7 @@ spring.cloud.gateway.discovery.locator.lower-case-service-id=true
spring.cloud.gateway.discovery.locator.enabled=true
# 不用改
mybatis.mapper-locations=classpath:mybatis/mapper/*.xml
mybatis.fill.com.gitee.fastmybatis.core.support.DateFillInsert=gmt_create
mybatis.fill.com.gitee.fastmybatis.core.support.DateFillUpdate=gmt_modified

View File

@@ -0,0 +1,30 @@
<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<!-- 注意文件名必须跟Dao类名字一致因为是根据文件名做关联。 -->
<mapper namespace="com.gitee.sop.gateway.mapper.MonitorInfoErrorMapper">
<insert id="saveMonitorInfoErrorBatch">
INSERT INTO `monitor_info_error` (
`error_id`,
`instance_id`,
`route_id`,
`error_msg`,
`error_status`,
`count`)
VALUES
<foreach collection="list" item="data" separator="," >
(#{data.errorId},
#{data.instanceId},
#{data.routeId},
#{data.errorMsg},
#{data.errorStatus},
#{data.count})
</foreach>
ON DUPLICATE KEY UPDATE
error_msg = VALUES(error_msg)
, error_status = VALUES(error_status)
, `count`= `count` + VALUES(count)
, is_deleted = 0
</insert>
</mapper>

View File

@@ -0,0 +1,43 @@
<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<!-- 注意文件名必须跟Dao类名字一致因为是根据文件名做关联。 -->
<mapper namespace="com.gitee.sop.gateway.mapper.MonitorInfoMapper">
<insert id="saveMonitorInfoBatch">
INSERT INTO `monitor_info` (
`route_id`,
`name`,
`version`,
`service_id`,
`instance_id`,
`max_time`,
`min_time`,
`total_time`,
`total_request_count`,
`success_count`,
`error_count`)
VALUES
<foreach collection="list" item="data" separator="," >
(#{data.routeId},
#{data.name},
#{data.version},
#{data.serviceId},
#{data.instanceId},
#{data.maxTime},
#{data.minTime},
#{data.totalTime},
#{data.totalRequestCount},
#{data.successCount},
#{data.errorCount})
</foreach>
<![CDATA[
ON DUPLICATE KEY UPDATE
max_time = case when max_time < VALUES(max_time) then VALUES(max_time) else max_time end
,min_time = case when min_time > VALUES(min_time) then VALUES(min_time) else min_time end
,total_time = total_time + VALUES(total_time)
,total_request_count = total_request_count + VALUES(total_request_count)
,success_count = success_count + VALUES(success_count)
,error_count = error_count + VALUES(error_count)
]]></insert>
</mapper>