7x31
/
cmdb
mirror of https://github.com/veops/cmdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(api): auto discovery supports mapping (#569)

This commit is contained in:
pycook 2024-07-02 20:19:50 +08:00 committed by GitHub
parent b22b8b286b
commit d5db68d7d0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 116 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
README.md
View File

@ -73,7 +73,8 @@
## 安装
### Docker 一键快速构建
> 方法一
[//]: # (> 方法一)
- 第一步: 先安装 Docker 环境, 以及Docker Compose (v2)
- 第二步: 拷贝项目
```shell
@ -83,13 +84,20 @@ git clone https://github.com/veops/cmdb.git
```
docker compose up -d
```
> 方法二, 该方法适用于linux系统
- 第一步： 先安装 Docker 环境, 以及Docker Compose (v2)
- 第二步： 直接使用项目根目录下的install.sh 文件进行 `安装`、`启动`、`暂停`、`查状态`、`删除`、`卸载`
```shell
curl -so install.sh https://raw.githubusercontent.com/veops/cmdb/deploy_on_kylin_docker/install.sh
sh install.sh install
```
[//]: # (> 方法二, 该方法适用于linux系统)
[//]: # (- 第一步： 先安装 Docker 环境, 以及Docker Compose (v2))
[//]: # (- 第二步： 直接使用项目根目录下的install.sh 文件进行 `安装`、`启动`、`暂停`、`查状态`、`删除`、`卸载`)
[//]: # (```shell)
[//]: # (curl -so install.sh https://raw.githubusercontent.com/veops/cmdb/deploy_on_kylin_docker/install.sh)
[//]: # (sh install.sh install)
[//]: # (```)
### [本地开发环境搭建](docs/local.md)

1
cmdb-api/Pipfile
View File

@ -67,6 +67,7 @@ colorama = ">=0.4.6"
pycryptodomex = ">=3.19.0"
lz4 = ">=4.3.2"
python-magic = "==0.4.27"
jsonpath = "==0.82.2"
[dev-packages]
# Testing

64
cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py
View File

@ -2,6 +2,7 @@
import copy
import datetime
import json
import jsonpath
import os
from flask import abort
from flask import current_app
@ -9,10 +10,11 @@ from flask_login import current_user
from sqlalchemy import func
from api.extensions import db
from api.lib.cmdb.auto_discovery.const import ClOUD_MAP
from api.lib.cmdb.auto_discovery.const import CLOUD_MAP
from api.lib.cmdb.auto_discovery.const import DEFAULT_INNER
from api.lib.cmdb.auto_discovery.const import PRIVILEGED_USERS
from api.lib.cmdb.cache import AttributeCache
from api.lib.cmdb.cache import AutoDiscoveryMappingCache
from api.lib.cmdb.cache import CITypeAttributeCache
from api.lib.cmdb.cache import CITypeCache
from api.lib.cmdb.ci import CIManager
@ -279,7 +281,6 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
result.append(rule)
ad_rules_updated_at = (SystemConfigManager.get('ad_rules_updated_at') or {}).get('option', {}).get('v') or ""
new_last_update_at = ""
for i in result:
@ -361,10 +362,11 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
en_name = i['en']
break
if en_name and kwargs['extra_option'].get('category'):
for item in ClOUD_MAP[en_name]:
for item in CLOUD_MAP[en_name]:
if item["collect_key_map"].get(kwargs['extra_option']['category']):
kwargs["extra_option"]["collect_key"] = item["collect_key_map"][
kwargs['extra_option']['category']]
kwargs["extra_option"]["provider"] = en_name
break
if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
@ -399,10 +401,11 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
en_name = i['en']
break
if en_name and kwargs['extra_option'].get('category'):
for item in ClOUD_MAP[en_name]:
for item in CLOUD_MAP[en_name]:
if item["collect_key_map"].get(kwargs['extra_option']['category']):
kwargs["extra_option"]["collect_key"] = item["collect_key_map"][
kwargs['extra_option']['category']]
kwargs["extra_option"]["provider"] = en_name
break
if 'attributes' in kwargs:
@ -479,6 +482,7 @@ class AutoDiscoveryCITypeRelationCRUD(DBMixin):
def get_all(cls, type_ids=None):
res = cls.cls.get_by(to_dict=False)
return [i for i in res if type_ids is None or i.ad_type_id in type_ids]
@classmethod
def get_by_type_id(cls, type_id, to_dict=False):
return cls.cls.get_by(ad_type_id=type_id, to_dict=to_dict)
@ -543,7 +547,6 @@ class AutoDiscoveryCICRUD(DBMixin):
adts = AutoDiscoveryCITypeCRUD.get_by_type_id(type_id)
for adt in adts:
attr_names |= set((adt.attributes or {}).values())
return [attr for attr in attributes if attr['name'] in attr_names]
@classmethod
@ -674,7 +677,16 @@ class AutoDiscoveryCICRUD(DBMixin):
ci_id = None
if adt.attributes:
ci_dict = {adt.attributes[k]: v for k, v in adc.instance.items() if k in adt.attributes}
ci_dict = {adt.attributes[k]: None if not v and isinstance(v, (list, dict)) else v
for k, v in adc.instance.items() if k in adt.attributes}
extra_option = adt.extra_option or {}
mapping, path_mapping = AutoDiscoveryHTTPManager.get_predefined_value_mapping(
extra_option.get('provider'), extra_option.get('category'))
if mapping:
ci_dict = {k: (mapping.get(k) or {}).get(str(v), v) for k, v in ci_dict.items()}
if path_mapping:
ci_dict = {k: jsonpath.jsonpath(v, path_mapping[k]) if k in path_mapping else v
for k, v in ci_dict.items()}
ci_id = CIManager.add(adc.type_id, is_auto_discovery=True, _is_admin=True, **ci_dict)
AutoDiscoveryExecHistoryCRUD().add(type_id=adt.type_id,
stdout="accept resource: {}".format(adc.unique_value))
@ -715,7 +727,7 @@ class AutoDiscoveryCICRUD(DBMixin):
class AutoDiscoveryHTTPManager(object):
@staticmethod
def get_categories(name):
categories = (ClOUD_MAP.get(name) or {}) or []
categories = (CLOUD_MAP.get(name) or {}) or []
for item in copy.deepcopy(categories):
item.pop('map', None)
item.pop('collect_key_map', None)
@ -738,16 +750,52 @@ class AutoDiscoveryHTTPManager(object):
@staticmethod
def get_attributes(provider, resource):
for item in (ClOUD_MAP.get(provider) or {}):
for item in (CLOUD_MAP.get(provider) or {}):
for _resource in (item.get('map') or {}):
if _resource == resource:
tpt = item['map'][_resource]
if isinstance(tpt, dict):
tpt = tpt.get('template')
if tpt and os.path.exists(os.path.join(PWD, tpt)):
with open(os.path.join(PWD, tpt)) as f:
return json.loads(f.read())
return []
@staticmethod
def get_mapping(provider, resource):
for item in (CLOUD_MAP.get(provider) or {}):
for _resource in (item.get('map') or {}):
if _resource == resource:
mapping = item['map'][_resource]
if not isinstance(mapping, dict):
return {}
name = mapping.get('mapping')
mapping = AutoDiscoveryMappingCache.get(name)
if isinstance(mapping, dict):
return {mapping[key][provider]['key'].split('.')[0]: key for key in mapping if
mapping[key].get(provider, {}).get('key')}
return {}
@staticmethod
def get_predefined_value_mapping(provider, resource):
for item in (CLOUD_MAP.get(provider) or {}):
for _resource in (item.get('map') or {}):
if _resource == resource:
mapping = item['map'][_resource]
if not isinstance(mapping, dict):
return {}, {}
name = mapping.get('mapping')
mapping = AutoDiscoveryMappingCache.get(name)
if isinstance(mapping, dict):
return ({key: mapping[key][provider].get('map') for key in mapping if
mapping[key].get(provider, {}).get('map')},
{key: mapping[key][provider]['key'].split('.', 1)[1] for key in mapping if
(mapping[key].get(provider, {}).get('key') or '').split('.')[1:]})
return {}, {}
class AutoDiscoverySNMPManager(object):

16
cmdb-api/api/lib/cmdb/auto_discovery/const.py
View File

@ -12,7 +12,7 @@ DEFAULT_INNER = [
dict(name="华为云", en="huaweicloud", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
option={'icon': {'name': 'caise-huaweiyun'}, "en": "huaweicloud"}),
dict(name="AWS", en="aws", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
option={'icon': {'name': 'caise-aws'}}),
option={'icon': {'name': 'caise-aws'}, "en": "aws"}),
dict(name="VCenter", en="vcenter", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
option={'icon': {'name': 'cmdb-vcenter'}, "category": "private_cloud", "en": "vcenter"}),
@ -34,14 +34,14 @@ DEFAULT_INNER = [
option={'icon': {'name': 'caise-dayinji'}}),
]
ClOUD_MAP = {
CLOUD_MAP = {
"aliyun": [
{
"category": "计算",
"items": ["云服务器 ECS", "云服务器 Disk"],
"map": {
"云服务器 ECS": "templates/aliyun_ecs.json",
"云服务器 Disk": "templates/aliyun_ecs_disk2.json",
"云服务器 ECS": {"template": "templates/aliyun_ecs.json", "mapping": "ecs"},
"云服务器 Disk": {"template": "templates/aliyun_ecs_disk2.json", "mapping": "evs"},
},
"collect_key_map": {
"云服务器 ECS": "ali.ecs",
@ -57,10 +57,10 @@ ClOUD_MAP = {
"交换机Switch",
],
"map": {
"内容分发CDN": "templates/aliyun_cdn.json",
"负载均衡SLB": "templates/aliyun_slb.json",
"专有网络VPC": "templates/aliyun_vpc.json",
"交换机Switch": "templates/aliyun_switch.json",
"内容分发CDN": {"template": "templates/aliyun_cdn.json", "mapping": "CDN"},
"负载均衡SLB": {"template": "templates/aliyun_slb.json", "mapping": "loadbalancer"},
"专有网络VPC": {"template": "templates/aliyun_vpc.json", "mapping": "vpc"},
"交换机Switch": {"template": "templates/aliyun_switch.json", "mapping": "vswitch"},
},
"collect_key_map": {
"内容分发CDN": "ali.cdn",

19
cmdb-api/api/lib/cmdb/cache.py
View File

@ -3,6 +3,8 @@
from __future__ import unicode_literals
import datetime
import os
import yaml
from flask import current_app
@ -546,3 +548,20 @@ class CMDBCounterCache(object):
@classmethod
def get_sub_counter(cls):
return cache.get(cls.KEY3) or cls.flush_sub_counter()
class AutoDiscoveryMappingCache(object):
PREFIX = 'CMDB::AutoDiscovery::Mapping::{}'
@classmethod
def get(cls, name):
res = cache.get(cls.PREFIX.format(name)) or {}
if not res:
path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "auto_discovery/mapping/{}.yaml".format(name))
if os.path.exists(path):
with open(path, 'r') as f:
mapping = yaml.safe_load(f)
res = mapping.get('mapping') or {}
res and cache.set(cls.PREFIX.format(name), res, timeout=0)
return res

11
cmdb-api/api/lib/cmdb/utils.py
View File

@ -29,12 +29,21 @@ def string2int(x):
def str2datetime(x):
x = x.replace('T', ' ')
x = x.replace('Z', '')
try:
return datetime.datetime.strptime(x, "%Y-%m-%d").date()
except ValueError:
pass
return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")
try:
return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S").date()
except ValueError:
pass
return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M")
class ValueTypeMap(object):

5
cmdb-api/api/views/cmdb/auto_discovery.py
View File

@ -111,6 +111,7 @@ class AutoDiscoveryRuleTemplateFileView(APIView):
class AutoDiscoveryRuleHTTPView(APIView):
url_prefix = ("/adr/http/<string:name>/categories",
"/adr/http/<string:name>/attributes",
"/adr/http/<string:name>/mapping",
"/adr/snmp/<string:name>/attributes",
"/adr/components/<string:name>/attributes",)
@ -125,6 +126,10 @@ class AutoDiscoveryRuleHTTPView(APIView):
resource = request.values.get('resource')
return self.jsonify(AutoDiscoveryHTTPManager.get_attributes(name, resource))
if "mapping" in request.url:
resource = request.values.get('resource')
return self.jsonify(AutoDiscoveryHTTPManager.get_mapping(name, resource))
return self.jsonify(AutoDiscoveryHTTPManager.get_categories(name))

1
cmdb-api/requirements.txt
View File

@ -55,3 +55,4 @@ pycryptodomex>=3.19.0
colorama>=0.4.6
lz4>=4.3.2
python-magic==0.4.27
jsonpath==0.82.2