From cc67b41339cf6f1544986d604fd1c8634ed80152 Mon Sep 17 00:00:00 2001
From: pycook <pycook@126.com>
Date: Tue, 2 Jul 2024 20:18:43 +0800
Subject: [PATCH] feat(api): auto discovery supports mapping
---
README.md | 24 ++++---
cmdb-api/Pipfile | 1 +
.../lib/cmdb/auto_discovery/auto_discovery.py | 64 ++++++++++++++++---
cmdb-api/api/lib/cmdb/auto_discovery/const.py | 16 ++---
cmdb-api/api/lib/cmdb/cache.py | 19 ++++++
cmdb-api/api/lib/cmdb/utils.py | 11 +++-
cmdb-api/api/views/cmdb/auto_discovery.py | 5 ++
cmdb-api/requirements.txt | 1 +
8 files changed, 116 insertions(+), 25 deletions(-)
diff --git a/README.md b/README.md
index 06a66ed..eda331c 100644
--- a/README.md
+++ b/README.md
@@ -73,7 +73,8 @@
## 安装
### Docker 一键快速构建
-> 方法一
+
+[//]: # (> 方法一)
- 第一步: 先安装 Docker 环境, 以及Docker Compose (v2)
- 第二步: 拷贝项目
```shell
@@ -83,13 +84,20 @@ git clone https://github.com/veops/cmdb.git
```
docker compose up -d
```
-> 方法二, 该方法适用于linux系统
-- 第一步： 先安装 Docker 环境, 以及Docker Compose (v2)
-- 第二步： 直接使用项目根目录下的install.sh 文件进行 `安装`、`启动`、`暂停`、`查状态`、`删除`、`卸载`
-```shell
-curl -so install.sh https://raw.githubusercontent.com/veops/cmdb/deploy_on_kylin_docker/install.sh
-sh install.sh install
-```
+
+[//]: # (> 方法二, 该方法适用于linux系统)
+
+[//]: # (- 第一步： 先安装 Docker 环境, 以及Docker Compose (v2))
+
+[//]: # (- 第二步： 直接使用项目根目录下的install.sh 文件进行 `安装`、`启动`、`暂停`、`查状态`、`删除`、`卸载`)
+
+[//]: # (```shell)
+
+[//]: # (curl -so install.sh https://raw.githubusercontent.com/veops/cmdb/deploy_on_kylin_docker/install.sh)
+
+[//]: # (sh install.sh install)
+
+[//]: # (```)
### [本地开发环境搭建](docs/local.md)
diff --git a/cmdb-api/Pipfile b/cmdb-api/Pipfile
index 6442409..a807215 100644
--- a/cmdb-api/Pipfile
+++ b/cmdb-api/Pipfile
@@ -67,6 +67,7 @@ colorama = ">=0.4.6"
pycryptodomex = ">=3.19.0"
lz4 = ">=4.3.2"
python-magic = "==0.4.27"
+jsonpath = "==0.82.2"
[dev-packages]
# Testing
diff --git a/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py b/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py
index 8d203fc..8c0bc2c 100644
--- a/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py
+++ b/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py
@@ -2,6 +2,7 @@
import copy
import datetime
import json
+import jsonpath
import os
from flask import abort
from flask import current_app
@@ -9,10 +10,11 @@ from flask_login import current_user
from sqlalchemy import func
from api.extensions import db
-from api.lib.cmdb.auto_discovery.const import ClOUD_MAP
+from api.lib.cmdb.auto_discovery.const import CLOUD_MAP
from api.lib.cmdb.auto_discovery.const import DEFAULT_INNER
from api.lib.cmdb.auto_discovery.const import PRIVILEGED_USERS
from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.cache import AutoDiscoveryMappingCache
from api.lib.cmdb.cache import CITypeAttributeCache
from api.lib.cmdb.cache import CITypeCache
from api.lib.cmdb.ci import CIManager
@@ -279,7 +281,6 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
result.append(rule)
-
ad_rules_updated_at = (SystemConfigManager.get('ad_rules_updated_at') or {}).get('option', {}).get('v') or ""
new_last_update_at = ""
for i in result:
@@ -361,10 +362,11 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
en_name = i['en']
break
if en_name and kwargs['extra_option'].get('category'):
- for item in ClOUD_MAP[en_name]:
+ for item in CLOUD_MAP[en_name]:
if item["collect_key_map"].get(kwargs['extra_option']['category']):
kwargs["extra_option"]["collect_key"] = item["collect_key_map"][
kwargs['extra_option']['category']]
+ kwargs["extra_option"]["provider"] = en_name
break
if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
@@ -399,10 +401,11 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
en_name = i['en']
break
if en_name and kwargs['extra_option'].get('category'):
- for item in ClOUD_MAP[en_name]:
+ for item in CLOUD_MAP[en_name]:
if item["collect_key_map"].get(kwargs['extra_option']['category']):
kwargs["extra_option"]["collect_key"] = item["collect_key_map"][
kwargs['extra_option']['category']]
+ kwargs["extra_option"]["provider"] = en_name
break
if 'attributes' in kwargs:
@@ -479,6 +482,7 @@ class AutoDiscoveryCITypeRelationCRUD(DBMixin):
def get_all(cls, type_ids=None):
res = cls.cls.get_by(to_dict=False)
return [i for i in res if type_ids is None or i.ad_type_id in type_ids]
+
@classmethod
def get_by_type_id(cls, type_id, to_dict=False):
return cls.cls.get_by(ad_type_id=type_id, to_dict=to_dict)
@@ -543,7 +547,6 @@ class AutoDiscoveryCICRUD(DBMixin):
adts = AutoDiscoveryCITypeCRUD.get_by_type_id(type_id)
for adt in adts:
attr_names |= set((adt.attributes or {}).values())
-
return [attr for attr in attributes if attr['name'] in attr_names]
@classmethod
@@ -674,7 +677,16 @@ class AutoDiscoveryCICRUD(DBMixin):
ci_id = None
if adt.attributes:
- ci_dict = {adt.attributes[k]: v for k, v in adc.instance.items() if k in adt.attributes}
+ ci_dict = {adt.attributes[k]: None if not v and isinstance(v, (list, dict)) else v
+ for k, v in adc.instance.items() if k in adt.attributes}
+ extra_option = adt.extra_option or {}
+ mapping, path_mapping = AutoDiscoveryHTTPManager.get_predefined_value_mapping(
+ extra_option.get('provider'), extra_option.get('category'))
+ if mapping:
+ ci_dict = {k: (mapping.get(k) or {}).get(str(v), v) for k, v in ci_dict.items()}
+ if path_mapping:
+ ci_dict = {k: jsonpath.jsonpath(v, path_mapping[k]) if k in path_mapping else v
+ for k, v in ci_dict.items()}
ci_id = CIManager.add(adc.type_id, is_auto_discovery=True, _is_admin=True, **ci_dict)
AutoDiscoveryExecHistoryCRUD().add(type_id=adt.type_id,
stdout="accept resource: {}".format(adc.unique_value))
@@ -715,7 +727,7 @@ class AutoDiscoveryCICRUD(DBMixin):
class AutoDiscoveryHTTPManager(object):
@staticmethod
def get_categories(name):
- categories = (ClOUD_MAP.get(name) or {}) or []
+ categories = (CLOUD_MAP.get(name) or {}) or []
for item in copy.deepcopy(categories):
item.pop('map', None)
item.pop('collect_key_map', None)
@@ -738,16 +750,52 @@ class AutoDiscoveryHTTPManager(object):
@staticmethod
def get_attributes(provider, resource):
- for item in (ClOUD_MAP.get(provider) or {}):
+ for item in (CLOUD_MAP.get(provider) or {}):
for _resource in (item.get('map') or {}):
if _resource == resource:
tpt = item['map'][_resource]
+ if isinstance(tpt, dict):
+ tpt = tpt.get('template')
if tpt and os.path.exists(os.path.join(PWD, tpt)):
with open(os.path.join(PWD, tpt)) as f:
return json.loads(f.read())
return []
+ @staticmethod
+ def get_mapping(provider, resource):
+ for item in (CLOUD_MAP.get(provider) or {}):
+ for _resource in (item.get('map') or {}):
+ if _resource == resource:
+ mapping = item['map'][_resource]
+ if not isinstance(mapping, dict):
+ return {}
+ name = mapping.get('mapping')
+ mapping = AutoDiscoveryMappingCache.get(name)
+ if isinstance(mapping, dict):
+ return {mapping[key][provider]['key'].split('.')[0]: key for key in mapping if
+ mapping[key].get(provider, {}).get('key')}
+
+ return {}
+
+ @staticmethod
+ def get_predefined_value_mapping(provider, resource):
+ for item in (CLOUD_MAP.get(provider) or {}):
+ for _resource in (item.get('map') or {}):
+ if _resource == resource:
+ mapping = item['map'][_resource]
+ if not isinstance(mapping, dict):
+ return {}, {}
+ name = mapping.get('mapping')
+ mapping = AutoDiscoveryMappingCache.get(name)
+ if isinstance(mapping, dict):
+ return ({key: mapping[key][provider].get('map') for key in mapping if
+ mapping[key].get(provider, {}).get('map')},
+ {key: mapping[key][provider]['key'].split('.', 1)[1] for key in mapping if
+ (mapping[key].get(provider, {}).get('key') or '').split('.')[1:]})
+
+ return {}, {}
+
class AutoDiscoverySNMPManager(object):
diff --git a/cmdb-api/api/lib/cmdb/auto_discovery/const.py b/cmdb-api/api/lib/cmdb/auto_discovery/const.py
index 0d16666..a22aed0 100644
--- a/cmdb-api/api/lib/cmdb/auto_discovery/const.py
+++ b/cmdb-api/api/lib/cmdb/auto_discovery/const.py
@@ -12,7 +12,7 @@ DEFAULT_INNER = [
dict(name="华为云", en="huaweicloud", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
option={'icon': {'name': 'caise-huaweiyun'}, "en": "huaweicloud"}),
dict(name="AWS", en="aws", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
- option={'icon': {'name': 'caise-aws'}}),
+ option={'icon': {'name': 'caise-aws'}, "en": "aws"}),
dict(name="VCenter", en="vcenter", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
option={'icon': {'name': 'cmdb-vcenter'}, "category": "private_cloud", "en": "vcenter"}),
@@ -34,14 +34,14 @@ DEFAULT_INNER = [
option={'icon': {'name': 'caise-dayinji'}}),
]
-ClOUD_MAP = {
+CLOUD_MAP = {
"aliyun": [
{
"category": "计算",
"items": ["云服务器 ECS", "云服务器 Disk"],
"map": {
- "云服务器 ECS": "templates/aliyun_ecs.json",
- "云服务器 Disk": "templates/aliyun_ecs_disk2.json",
+ "云服务器 ECS": {"template": "templates/aliyun_ecs.json", "mapping": "ecs"},
+ "云服务器 Disk": {"template": "templates/aliyun_ecs_disk2.json", "mapping": "evs"},
},
"collect_key_map": {
"云服务器 ECS": "ali.ecs",
@@ -57,10 +57,10 @@ ClOUD_MAP = {
"交换机Switch",
],
"map": {
- "内容分发CDN": "templates/aliyun_cdn.json",
- "负载均衡SLB": "templates/aliyun_slb.json",
- "专有网络VPC": "templates/aliyun_vpc.json",
- "交换机Switch": "templates/aliyun_switch.json",
+ "内容分发CDN": {"template": "templates/aliyun_cdn.json", "mapping": "CDN"},
+ "负载均衡SLB": {"template": "templates/aliyun_slb.json", "mapping": "loadbalancer"},
+ "专有网络VPC": {"template": "templates/aliyun_vpc.json", "mapping": "vpc"},
+ "交换机Switch": {"template": "templates/aliyun_switch.json", "mapping": "vswitch"},
},
"collect_key_map": {
"内容分发CDN": "ali.cdn",
diff --git a/cmdb-api/api/lib/cmdb/cache.py b/cmdb-api/api/lib/cmdb/cache.py
index f521cd0..586f78a 100644
--- a/cmdb-api/api/lib/cmdb/cache.py
+++ b/cmdb-api/api/lib/cmdb/cache.py
@@ -3,6 +3,8 @@
from __future__ import unicode_literals
import datetime
+import os
+import yaml
from flask import current_app
@@ -546,3 +548,20 @@ class CMDBCounterCache(object):
@classmethod
def get_sub_counter(cls):
return cache.get(cls.KEY3) or cls.flush_sub_counter()
+
+
+class AutoDiscoveryMappingCache(object):
+ PREFIX = 'CMDB::AutoDiscovery::Mapping::{}'
+
+ @classmethod
+ def get(cls, name):
+ res = cache.get(cls.PREFIX.format(name)) or {}
+ if not res:
+ path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "auto_discovery/mapping/{}.yaml".format(name))
+ if os.path.exists(path):
+ with open(path, 'r') as f:
+ mapping = yaml.safe_load(f)
+ res = mapping.get('mapping') or {}
+ res and cache.set(cls.PREFIX.format(name), res, timeout=0)
+
+ return res
\ No newline at end of file
diff --git a/cmdb-api/api/lib/cmdb/utils.py b/cmdb-api/api/lib/cmdb/utils.py
index 9358d23..7b0ff2a 100644
--- a/cmdb-api/api/lib/cmdb/utils.py
+++ b/cmdb-api/api/lib/cmdb/utils.py
@@ -29,12 +29,21 @@ def string2int(x):
def str2datetime(x):
+
+ x = x.replace('T', ' ')
+ x = x.replace('Z', '')
+
try:
return datetime.datetime.strptime(x, "%Y-%m-%d").date()
except ValueError:
pass
- return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")
+ try:
+ return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S").date()
+ except ValueError:
+ pass
+
+ return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M")
class ValueTypeMap(object):
diff --git a/cmdb-api/api/views/cmdb/auto_discovery.py b/cmdb-api/api/views/cmdb/auto_discovery.py
index eb2b70a..8a79492 100644
--- a/cmdb-api/api/views/cmdb/auto_discovery.py
+++ b/cmdb-api/api/views/cmdb/auto_discovery.py
@@ -111,6 +111,7 @@ class AutoDiscoveryRuleTemplateFileView(APIView):
class AutoDiscoveryRuleHTTPView(APIView):
url_prefix = ("/adr/http/<string:name>/categories",
"/adr/http/<string:name>/attributes",
+ "/adr/http/<string:name>/mapping",
"/adr/snmp/<string:name>/attributes",
"/adr/components/<string:name>/attributes",)
@@ -125,6 +126,10 @@ class AutoDiscoveryRuleHTTPView(APIView):
resource = request.values.get('resource')
return self.jsonify(AutoDiscoveryHTTPManager.get_attributes(name, resource))
+ if "mapping" in request.url:
+ resource = request.values.get('resource')
+ return self.jsonify(AutoDiscoveryHTTPManager.get_mapping(name, resource))
+
return self.jsonify(AutoDiscoveryHTTPManager.get_categories(name))
diff --git a/cmdb-api/requirements.txt b/cmdb-api/requirements.txt
index 0bd688b..0ae7166 100644
--- a/cmdb-api/requirements.txt
+++ b/cmdb-api/requirements.txt
@@ -55,3 +55,4 @@ pycryptodomex>=3.19.0
colorama>=0.4.6
lz4>=4.3.2
python-magic==0.4.27
+jsonpath==0.82.2