feat(api): auto discovery supports mapping (#569)

README.md

@@ -73,7 +73,8 @@
 ## 安装
 
 ### Docker 一键快速构建
-> 方法一
+
+[//]: # (> 方法一)
 - 第一步: 先安装 Docker 环境, 以及Docker Compose (v2)
 - 第二步: 拷贝项目
 ```shell 
@@ -83,13 +84,20 @@ git clone https://github.com/veops/cmdb.git
 ```
 docker compose up -d
 ```
-> 方法二, 该方法适用于linux系统
+
-- 第一步： 先安装 Docker 环境, 以及Docker Compose (v2)
+[//]: # (> 方法二, 该方法适用于linux系统)
-- 第二步： 直接使用项目根目录下的install.sh 文件进行 `安装`、`启动`、`暂停`、`查状态`、`删除`、`卸载`
+
-```shell
+[//]: # (- 第一步： 先安装 Docker 环境, 以及Docker Compose (v2))
-curl -so install.sh https://raw.githubusercontent.com/veops/cmdb/deploy_on_kylin_docker/install.sh
+
-sh install.sh install
+[//]: # (- 第二步： 直接使用项目根目录下的install.sh 文件进行 `安装`、`启动`、`暂停`、`查状态`、`删除`、`卸载`)
-```
+
+[//]: # (```shell)
+
+[//]: # (curl -so install.sh https://raw.githubusercontent.com/veops/cmdb/deploy_on_kylin_docker/install.sh)
+
+[//]: # (sh install.sh install)
+
+[//]: # (```)
 
 ### [本地开发环境搭建](docs/local.md)
 

cmdb-api/Pipfile

@@ -67,6 +67,7 @@ colorama = ">=0.4.6"
 pycryptodomex = ">=3.19.0"
 lz4 = ">=4.3.2"
 python-magic = "==0.4.27"
+jsonpath = "==0.82.2"
 
 [dev-packages]
 # Testing

cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py

@@ -2,6 +2,7 @@
 import copy
 import datetime
 import json
+import jsonpath
 import os
 from flask import abort
 from flask import current_app
@@ -9,10 +10,11 @@ from flask_login import current_user
 from sqlalchemy import func
 
 from api.extensions import db
-from api.lib.cmdb.auto_discovery.const import ClOUD_MAP
+from api.lib.cmdb.auto_discovery.const import CLOUD_MAP
 from api.lib.cmdb.auto_discovery.const import DEFAULT_INNER
 from api.lib.cmdb.auto_discovery.const import PRIVILEGED_USERS
 from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.cache import AutoDiscoveryMappingCache
 from api.lib.cmdb.cache import CITypeAttributeCache
 from api.lib.cmdb.cache import CITypeCache
 from api.lib.cmdb.ci import CIManager
@@ -279,7 +281,6 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
 
                 result.append(rule)
 
-
         ad_rules_updated_at = (SystemConfigManager.get('ad_rules_updated_at') or {}).get('option', {}).get('v') or ""
         new_last_update_at = ""
         for i in result:
@@ -361,10 +362,11 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
                         en_name = i['en']
                         break
                 if en_name and kwargs['extra_option'].get('category'):
-                    for item in ClOUD_MAP[en_name]:
+                    for item in CLOUD_MAP[en_name]:
                         if item["collect_key_map"].get(kwargs['extra_option']['category']):
                             kwargs["extra_option"]["collect_key"] = item["collect_key_map"][
                                 kwargs['extra_option']['category']]
+                            kwargs["extra_option"]["provider"] = en_name
                             break
 
         if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
@@ -399,10 +401,11 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
                     en_name = i['en']
                     break
             if en_name and kwargs['extra_option'].get('category'):
-                for item in ClOUD_MAP[en_name]:
+                for item in CLOUD_MAP[en_name]:
                     if item["collect_key_map"].get(kwargs['extra_option']['category']):
                         kwargs["extra_option"]["collect_key"] = item["collect_key_map"][
                             kwargs['extra_option']['category']]
+                        kwargs["extra_option"]["provider"] = en_name
                         break
 
         if 'attributes' in kwargs:
@@ -479,6 +482,7 @@ class AutoDiscoveryCITypeRelationCRUD(DBMixin):
     def get_all(cls, type_ids=None):
         res = cls.cls.get_by(to_dict=False)
         return [i for i in res if type_ids is None or i.ad_type_id in type_ids]
+
     @classmethod
     def get_by_type_id(cls, type_id, to_dict=False):
         return cls.cls.get_by(ad_type_id=type_id, to_dict=to_dict)
@@ -543,7 +547,6 @@ class AutoDiscoveryCICRUD(DBMixin):
         adts = AutoDiscoveryCITypeCRUD.get_by_type_id(type_id)
         for adt in adts:
             attr_names |= set((adt.attributes or {}).values())
-
         return [attr for attr in attributes if attr['name'] in attr_names]
 
     @classmethod
@@ -674,7 +677,16 @@ class AutoDiscoveryCICRUD(DBMixin):
 
         ci_id = None
         if adt.attributes:
-            ci_dict = {adt.attributes[k]: v for k, v in adc.instance.items() if k in adt.attributes}
+            ci_dict = {adt.attributes[k]: None if not v and isinstance(v, (list, dict)) else v
+                       for k, v in adc.instance.items() if k in adt.attributes}
+            extra_option = adt.extra_option or {}
+            mapping, path_mapping = AutoDiscoveryHTTPManager.get_predefined_value_mapping(
+                extra_option.get('provider'), extra_option.get('category'))
+            if mapping:
+                ci_dict = {k: (mapping.get(k) or {}).get(str(v), v) for k, v in ci_dict.items()}
+            if path_mapping:
+                ci_dict = {k: jsonpath.jsonpath(v, path_mapping[k]) if k in path_mapping else v
+                           for k, v in ci_dict.items()}
             ci_id = CIManager.add(adc.type_id, is_auto_discovery=True, _is_admin=True, **ci_dict)
             AutoDiscoveryExecHistoryCRUD().add(type_id=adt.type_id,
                                                stdout="accept resource: {}".format(adc.unique_value))
@@ -715,7 +727,7 @@ class AutoDiscoveryCICRUD(DBMixin):
 class AutoDiscoveryHTTPManager(object):
     @staticmethod
     def get_categories(name):
-        categories = (ClOUD_MAP.get(name) or {}) or []
+        categories = (CLOUD_MAP.get(name) or {}) or []
         for item in copy.deepcopy(categories):
             item.pop('map', None)
             item.pop('collect_key_map', None)
@@ -738,16 +750,52 @@ class AutoDiscoveryHTTPManager(object):
 
     @staticmethod
     def get_attributes(provider, resource):
-        for item in (ClOUD_MAP.get(provider) or {}):
+        for item in (CLOUD_MAP.get(provider) or {}):
             for _resource in (item.get('map') or {}):
                 if _resource == resource:
                     tpt = item['map'][_resource]
+                    if isinstance(tpt, dict):
+                        tpt = tpt.get('template')
                     if tpt and os.path.exists(os.path.join(PWD, tpt)):
                         with open(os.path.join(PWD, tpt)) as f:
                             return json.loads(f.read())
 
         return []
 
+    @staticmethod
+    def get_mapping(provider, resource):
+        for item in (CLOUD_MAP.get(provider) or {}):
+            for _resource in (item.get('map') or {}):
+                if _resource == resource:
+                    mapping = item['map'][_resource]
+                    if not isinstance(mapping, dict):
+                        return {}
+                    name = mapping.get('mapping')
+                    mapping = AutoDiscoveryMappingCache.get(name)
+                    if isinstance(mapping, dict):
+                        return {mapping[key][provider]['key'].split('.')[0]: key for key in mapping if
+                                mapping[key].get(provider, {}).get('key')}
+
+        return {}
+
+    @staticmethod
+    def get_predefined_value_mapping(provider, resource):
+        for item in (CLOUD_MAP.get(provider) or {}):
+            for _resource in (item.get('map') or {}):
+                if _resource == resource:
+                    mapping = item['map'][_resource]
+                    if not isinstance(mapping, dict):
+                        return {}, {}
+                    name = mapping.get('mapping')
+                    mapping = AutoDiscoveryMappingCache.get(name)
+                    if isinstance(mapping, dict):
+                        return ({key: mapping[key][provider].get('map') for key in mapping if
+                                 mapping[key].get(provider, {}).get('map')},
+                                {key: mapping[key][provider]['key'].split('.', 1)[1] for key in mapping if
+                                 (mapping[key].get(provider, {}).get('key') or '').split('.')[1:]})
+
+        return {}, {}
+
 
 class AutoDiscoverySNMPManager(object):
 

cmdb-api/api/lib/cmdb/auto_discovery/const.py

@@ -12,7 +12,7 @@ DEFAULT_INNER = [
     dict(name="华为云", en="huaweicloud", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
          option={'icon': {'name': 'caise-huaweiyun'}, "en": "huaweicloud"}),
     dict(name="AWS", en="aws", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
-         option={'icon': {'name': 'caise-aws'}}),
+         option={'icon': {'name': 'caise-aws'}, "en": "aws"}),
 
     dict(name="VCenter", en="vcenter", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
          option={'icon': {'name': 'cmdb-vcenter'}, "category": "private_cloud", "en": "vcenter"}),
@@ -34,14 +34,14 @@ DEFAULT_INNER = [
          option={'icon': {'name': 'caise-dayinji'}}),
 ]
 
-ClOUD_MAP = {
+CLOUD_MAP = {
     "aliyun": [
         {
             "category": "计算",
             "items": ["云服务器 ECS", "云服务器 Disk"],
             "map": {
-                "云服务器 ECS": "templates/aliyun_ecs.json",
+                "云服务器 ECS": {"template": "templates/aliyun_ecs.json", "mapping": "ecs"},
-                "云服务器 Disk": "templates/aliyun_ecs_disk2.json",
+                "云服务器 Disk": {"template": "templates/aliyun_ecs_disk2.json", "mapping": "evs"},
             },
             "collect_key_map": {
                 "云服务器 ECS": "ali.ecs",
@@ -57,10 +57,10 @@ ClOUD_MAP = {
                 "交换机Switch",
             ],
             "map": {
-                "内容分发CDN": "templates/aliyun_cdn.json",
+                "内容分发CDN": {"template": "templates/aliyun_cdn.json", "mapping": "CDN"},
-                "负载均衡SLB": "templates/aliyun_slb.json",
+                "负载均衡SLB": {"template": "templates/aliyun_slb.json", "mapping": "loadbalancer"},
-                "专有网络VPC": "templates/aliyun_vpc.json",
+                "专有网络VPC": {"template": "templates/aliyun_vpc.json", "mapping": "vpc"},
-                "交换机Switch": "templates/aliyun_switch.json",
+                "交换机Switch": {"template": "templates/aliyun_switch.json", "mapping": "vswitch"},
             },
             "collect_key_map": {
                 "内容分发CDN": "ali.cdn",

cmdb-api/api/lib/cmdb/cache.py

@@ -3,6 +3,8 @@
 from __future__ import unicode_literals
 
 import datetime
+import os
+import yaml
 
 from flask import current_app
 
@@ -546,3 +548,20 @@ class CMDBCounterCache(object):
     @classmethod
     def get_sub_counter(cls):
         return cache.get(cls.KEY3) or cls.flush_sub_counter()
+
+
+class AutoDiscoveryMappingCache(object):
+    PREFIX = 'CMDB::AutoDiscovery::Mapping::{}'
+
+    @classmethod
+    def get(cls, name):
+        res = cache.get(cls.PREFIX.format(name)) or {}
+        if not res:
+            path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "auto_discovery/mapping/{}.yaml".format(name))
+            if os.path.exists(path):
+                with open(path, 'r') as f:
+                    mapping = yaml.safe_load(f)
+                    res = mapping.get('mapping') or {}
+                    res and cache.set(cls.PREFIX.format(name), res, timeout=0)
+
+        return res

cmdb-api/api/lib/cmdb/utils.py

@@ -29,12 +29,21 @@ def string2int(x):
 
 
 def str2datetime(x):
+
+    x = x.replace('T', ' ')
+    x = x.replace('Z', '')
+
     try:
         return datetime.datetime.strptime(x, "%Y-%m-%d").date()
     except ValueError:
         pass
 
-    return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")
+    try:
+        return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S").date()
+    except ValueError:
+        pass
+
+    return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M")
 
 
 class ValueTypeMap(object):

cmdb-api/api/views/cmdb/auto_discovery.py

@@ -111,6 +111,7 @@ class AutoDiscoveryRuleTemplateFileView(APIView):
 class AutoDiscoveryRuleHTTPView(APIView):
     url_prefix = ("/adr/http/<string:name>/categories",
                   "/adr/http/<string:name>/attributes",
+                  "/adr/http/<string:name>/mapping",
                   "/adr/snmp/<string:name>/attributes",
                   "/adr/components/<string:name>/attributes",)
 
@@ -125,6 +126,10 @@ class AutoDiscoveryRuleHTTPView(APIView):
             resource = request.values.get('resource')
             return self.jsonify(AutoDiscoveryHTTPManager.get_attributes(name, resource))
 
+        if "mapping" in request.url:
+            resource = request.values.get('resource')
+            return self.jsonify(AutoDiscoveryHTTPManager.get_mapping(name, resource))
+
         return self.jsonify(AutoDiscoveryHTTPManager.get_categories(name))
 
 

cmdb-api/requirements.txt

@@ -55,3 +55,4 @@ pycryptodomex>=3.19.0
 colorama>=0.4.6
 lz4>=4.3.2
 python-magic==0.4.27
+jsonpath==0.82.2