From d5db68d7d08637d952410e6ed51fd422b1769866 Mon Sep 17 00:00:00 2001 From: pycook Date: Tue, 2 Jul 2024 20:19:50 +0800 Subject: [PATCH] feat(api): auto discovery supports mapping (#569) --- README.md | 24 ++++--- cmdb-api/Pipfile | 1 + .../lib/cmdb/auto_discovery/auto_discovery.py | 64 ++++++++++++++++--- cmdb-api/api/lib/cmdb/auto_discovery/const.py | 16 ++--- cmdb-api/api/lib/cmdb/cache.py | 19 ++++++ cmdb-api/api/lib/cmdb/utils.py | 11 +++- cmdb-api/api/views/cmdb/auto_discovery.py | 5 ++ cmdb-api/requirements.txt | 1 + 8 files changed, 116 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 06a66ed..eda331c 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,8 @@ ## 安装 ### Docker 一键快速构建 -> 方法一 + +[//]: # (> 方法一) - 第一步: 先安装 Docker 环境, 以及Docker Compose (v2) - 第二步: 拷贝项目 ```shell @@ -83,13 +84,20 @@ git clone https://github.com/veops/cmdb.git ``` docker compose up -d ``` -> 方法二, 该方法适用于linux系统 -- 第一步： 先安装 Docker 环境, 以及Docker Compose (v2) -- 第二步： 直接使用项目根目录下的install.sh 文件进行 `安装`、`启动`、`暂停`、`查状态`、`删除`、`卸载` -```shell -curl -so install.sh https://raw.githubusercontent.com/veops/cmdb/deploy_on_kylin_docker/install.sh -sh install.sh install -``` + +[//]: # (> 方法二, 该方法适用于linux系统) + +[//]: # (- 第一步： 先安装 Docker 环境, 以及Docker Compose (v2)) + +[//]: # (- 第二步： 直接使用项目根目录下的install.sh 文件进行 `安装`、`启动`、`暂停`、`查状态`、`删除`、`卸载`) + +[//]: # (```shell) + +[//]: # (curl -so install.sh https://raw.githubusercontent.com/veops/cmdb/deploy_on_kylin_docker/install.sh) + +[//]: # (sh install.sh install) + +[//]: # (```) ### [本地开发环境搭建](docs/local.md) diff --git a/cmdb-api/Pipfile b/cmdb-api/Pipfile index 6442409..a807215 100644 --- a/cmdb-api/Pipfile +++ b/cmdb-api/Pipfile @@ -67,6 +67,7 @@ colorama = ">=0.4.6" pycryptodomex = ">=3.19.0" lz4 = ">=4.3.2" python-magic = "==0.4.27" +jsonpath = "==0.82.2" [dev-packages] # Testing diff --git a/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py b/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py index 8d203fc..8c0bc2c 100644 --- a/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py +++ b/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py @@ -2,6 +2,7 @@ import copy import datetime import json +import jsonpath import os from flask import abort from flask import current_app @@ -9,10 +10,11 @@ from flask_login import current_user from sqlalchemy import func from api.extensions import db -from api.lib.cmdb.auto_discovery.const import ClOUD_MAP +from api.lib.cmdb.auto_discovery.const import CLOUD_MAP from api.lib.cmdb.auto_discovery.const import DEFAULT_INNER from api.lib.cmdb.auto_discovery.const import PRIVILEGED_USERS from api.lib.cmdb.cache import AttributeCache +from api.lib.cmdb.cache import AutoDiscoveryMappingCache from api.lib.cmdb.cache import CITypeAttributeCache from api.lib.cmdb.cache import CITypeCache from api.lib.cmdb.ci import CIManager @@ -279,7 +281,6 @@ class AutoDiscoveryCITypeCRUD(DBMixin): result.append(rule) - ad_rules_updated_at = (SystemConfigManager.get('ad_rules_updated_at') or {}).get('option', {}).get('v') or "" new_last_update_at = "" for i in result: @@ -361,10 +362,11 @@ class AutoDiscoveryCITypeCRUD(DBMixin): en_name = i['en'] break if en_name and kwargs['extra_option'].get('category'): - for item in ClOUD_MAP[en_name]: + for item in CLOUD_MAP[en_name]: if item["collect_key_map"].get(kwargs['extra_option']['category']): kwargs["extra_option"]["collect_key"] = item["collect_key_map"][ kwargs['extra_option']['category']] + kwargs["extra_option"]["provider"] = en_name break if kwargs.get('is_plugin') and kwargs.get('plugin_script'): @@ -399,10 +401,11 @@ class AutoDiscoveryCITypeCRUD(DBMixin): en_name = i['en'] break if en_name and kwargs['extra_option'].get('category'): - for item in ClOUD_MAP[en_name]: + for item in CLOUD_MAP[en_name]: if item["collect_key_map"].get(kwargs['extra_option']['category']): kwargs["extra_option"]["collect_key"] = item["collect_key_map"][ kwargs['extra_option']['category']] + kwargs["extra_option"]["provider"] = en_name break if 'attributes' in kwargs: @@ -479,6 +482,7 @@ class AutoDiscoveryCITypeRelationCRUD(DBMixin): def get_all(cls, type_ids=None): res = cls.cls.get_by(to_dict=False) return [i for i in res if type_ids is None or i.ad_type_id in type_ids] + @classmethod def get_by_type_id(cls, type_id, to_dict=False): return cls.cls.get_by(ad_type_id=type_id, to_dict=to_dict) @@ -543,7 +547,6 @@ class AutoDiscoveryCICRUD(DBMixin): adts = AutoDiscoveryCITypeCRUD.get_by_type_id(type_id) for adt in adts: attr_names |= set((adt.attributes or {}).values()) - return [attr for attr in attributes if attr['name'] in attr_names] @classmethod @@ -674,7 +677,16 @@ class AutoDiscoveryCICRUD(DBMixin): ci_id = None if adt.attributes: - ci_dict = {adt.attributes[k]: v for k, v in adc.instance.items() if k in adt.attributes} + ci_dict = {adt.attributes[k]: None if not v and isinstance(v, (list, dict)) else v + for k, v in adc.instance.items() if k in adt.attributes} + extra_option = adt.extra_option or {} + mapping, path_mapping = AutoDiscoveryHTTPManager.get_predefined_value_mapping( + extra_option.get('provider'), extra_option.get('category')) + if mapping: + ci_dict = {k: (mapping.get(k) or {}).get(str(v), v) for k, v in ci_dict.items()} + if path_mapping: + ci_dict = {k: jsonpath.jsonpath(v, path_mapping[k]) if k in path_mapping else v + for k, v in ci_dict.items()} ci_id = CIManager.add(adc.type_id, is_auto_discovery=True, _is_admin=True, **ci_dict) AutoDiscoveryExecHistoryCRUD().add(type_id=adt.type_id, stdout="accept resource: {}".format(adc.unique_value)) @@ -715,7 +727,7 @@ class AutoDiscoveryCICRUD(DBMixin): class AutoDiscoveryHTTPManager(object): @staticmethod def get_categories(name): - categories = (ClOUD_MAP.get(name) or {}) or [] + categories = (CLOUD_MAP.get(name) or {}) or [] for item in copy.deepcopy(categories): item.pop('map', None) item.pop('collect_key_map', None) @@ -738,16 +750,52 @@ class AutoDiscoveryHTTPManager(object): @staticmethod def get_attributes(provider, resource): - for item in (ClOUD_MAP.get(provider) or {}): + for item in (CLOUD_MAP.get(provider) or {}): for _resource in (item.get('map') or {}): if _resource == resource: tpt = item['map'][_resource] + if isinstance(tpt, dict): + tpt = tpt.get('template') if tpt and os.path.exists(os.path.join(PWD, tpt)): with open(os.path.join(PWD, tpt)) as f: return json.loads(f.read()) return [] + @staticmethod + def get_mapping(provider, resource): + for item in (CLOUD_MAP.get(provider) or {}): + for _resource in (item.get('map') or {}): + if _resource == resource: + mapping = item['map'][_resource] + if not isinstance(mapping, dict): + return {} + name = mapping.get('mapping') + mapping = AutoDiscoveryMappingCache.get(name) + if isinstance(mapping, dict): + return {mapping[key][provider]['key'].split('.')[0]: key for key in mapping if + mapping[key].get(provider, {}).get('key')} + + return {} + + @staticmethod + def get_predefined_value_mapping(provider, resource): + for item in (CLOUD_MAP.get(provider) or {}): + for _resource in (item.get('map') or {}): + if _resource == resource: + mapping = item['map'][_resource] + if not isinstance(mapping, dict): + return {}, {} + name = mapping.get('mapping') + mapping = AutoDiscoveryMappingCache.get(name) + if isinstance(mapping, dict): + return ({key: mapping[key][provider].get('map') for key in mapping if + mapping[key].get(provider, {}).get('map')}, + {key: mapping[key][provider]['key'].split('.', 1)[1] for key in mapping if + (mapping[key].get(provider, {}).get('key') or '').split('.')[1:]}) + + return {}, {} + class AutoDiscoverySNMPManager(object): diff --git a/cmdb-api/api/lib/cmdb/auto_discovery/const.py b/cmdb-api/api/lib/cmdb/auto_discovery/const.py index 0d16666..a22aed0 100644 --- a/cmdb-api/api/lib/cmdb/auto_discovery/const.py +++ b/cmdb-api/api/lib/cmdb/auto_discovery/const.py @@ -12,7 +12,7 @@ DEFAULT_INNER = [ dict(name="华为云", en="huaweicloud", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False, option={'icon': {'name': 'caise-huaweiyun'}, "en": "huaweicloud"}), dict(name="AWS", en="aws", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False, - option={'icon': {'name': 'caise-aws'}}), + option={'icon': {'name': 'caise-aws'}, "en": "aws"}), dict(name="VCenter", en="vcenter", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False, option={'icon': {'name': 'cmdb-vcenter'}, "category": "private_cloud", "en": "vcenter"}), @@ -34,14 +34,14 @@ DEFAULT_INNER = [ option={'icon': {'name': 'caise-dayinji'}}), ] -ClOUD_MAP = { +CLOUD_MAP = { "aliyun": [ { "category": "计算", "items": ["云服务器 ECS", "云服务器 Disk"], "map": { - "云服务器 ECS": "templates/aliyun_ecs.json", - "云服务器 Disk": "templates/aliyun_ecs_disk2.json", + "云服务器 ECS": {"template": "templates/aliyun_ecs.json", "mapping": "ecs"}, + "云服务器 Disk": {"template": "templates/aliyun_ecs_disk2.json", "mapping": "evs"}, }, "collect_key_map": { "云服务器 ECS": "ali.ecs", @@ -57,10 +57,10 @@ ClOUD_MAP = { "交换机Switch", ], "map": { - "内容分发CDN": "templates/aliyun_cdn.json", - "负载均衡SLB": "templates/aliyun_slb.json", - "专有网络VPC": "templates/aliyun_vpc.json", - "交换机Switch": "templates/aliyun_switch.json", + "内容分发CDN": {"template": "templates/aliyun_cdn.json", "mapping": "CDN"}, + "负载均衡SLB": {"template": "templates/aliyun_slb.json", "mapping": "loadbalancer"}, + "专有网络VPC": {"template": "templates/aliyun_vpc.json", "mapping": "vpc"}, + "交换机Switch": {"template": "templates/aliyun_switch.json", "mapping": "vswitch"}, }, "collect_key_map": { "内容分发CDN": "ali.cdn", diff --git a/cmdb-api/api/lib/cmdb/cache.py b/cmdb-api/api/lib/cmdb/cache.py index f521cd0..586f78a 100644 --- a/cmdb-api/api/lib/cmdb/cache.py +++ b/cmdb-api/api/lib/cmdb/cache.py @@ -3,6 +3,8 @@ from __future__ import unicode_literals import datetime +import os +import yaml from flask import current_app @@ -546,3 +548,20 @@ class CMDBCounterCache(object): @classmethod def get_sub_counter(cls): return cache.get(cls.KEY3) or cls.flush_sub_counter() + + +class AutoDiscoveryMappingCache(object): + PREFIX = 'CMDB::AutoDiscovery::Mapping::{}' + + @classmethod + def get(cls, name): + res = cache.get(cls.PREFIX.format(name)) or {} + if not res: + path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "auto_discovery/mapping/{}.yaml".format(name)) + if os.path.exists(path): + with open(path, 'r') as f: + mapping = yaml.safe_load(f) + res = mapping.get('mapping') or {} + res and cache.set(cls.PREFIX.format(name), res, timeout=0) + + return res \ No newline at end of file diff --git a/cmdb-api/api/lib/cmdb/utils.py b/cmdb-api/api/lib/cmdb/utils.py index 9358d23..7b0ff2a 100644 --- a/cmdb-api/api/lib/cmdb/utils.py +++ b/cmdb-api/api/lib/cmdb/utils.py @@ -29,12 +29,21 @@ def string2int(x): def str2datetime(x): + + x = x.replace('T', ' ') + x = x.replace('Z', '') + try: return datetime.datetime.strptime(x, "%Y-%m-%d").date() except ValueError: pass - return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S") + try: + return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S").date() + except ValueError: + pass + + return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M") class ValueTypeMap(object): diff --git a/cmdb-api/api/views/cmdb/auto_discovery.py b/cmdb-api/api/views/cmdb/auto_discovery.py index eb2b70a..8a79492 100644 --- a/cmdb-api/api/views/cmdb/auto_discovery.py +++ b/cmdb-api/api/views/cmdb/auto_discovery.py @@ -111,6 +111,7 @@ class AutoDiscoveryRuleTemplateFileView(APIView): class AutoDiscoveryRuleHTTPView(APIView): url_prefix = ("/adr/http//categories", "/adr/http//attributes", + "/adr/http//mapping", "/adr/snmp//attributes", "/adr/components//attributes",) @@ -125,6 +126,10 @@ class AutoDiscoveryRuleHTTPView(APIView): resource = request.values.get('resource') return self.jsonify(AutoDiscoveryHTTPManager.get_attributes(name, resource)) + if "mapping" in request.url: + resource = request.values.get('resource') + return self.jsonify(AutoDiscoveryHTTPManager.get_mapping(name, resource)) + return self.jsonify(AutoDiscoveryHTTPManager.get_categories(name)) diff --git a/cmdb-api/requirements.txt b/cmdb-api/requirements.txt index 0bd688b..0ae7166 100644 --- a/cmdb-api/requirements.txt +++ b/cmdb-api/requirements.txt @@ -55,3 +55,4 @@ pycryptodomex>=3.19.0 colorama>=0.4.6 lz4>=4.3.2 python-magic==0.4.27 +jsonpath==0.82.2