diff --git a/cmdb-api/api/commands/click_cmdb.py b/cmdb-api/api/commands/click_cmdb.py index 5c95697..e71606c 100644 --- a/cmdb-api/api/commands/click_cmdb.py +++ b/cmdb-api/api/commands/click_cmdb.py @@ -540,6 +540,19 @@ def cmdb_patch(version): peer_attr_id=peer_attr_id, commit=False) if hasattr(adt, 'interval') and adt.interval and not adt.cron: - adt.cron = "*/{} * * * *".format(adt.interval // 60) + adt.cron = "*/{} * * * *".format(adt.interval // 60 or 1) + + db.session.commit() + + if version >= "2.4.7": + from api.lib.cmdb.auto_discovery.const import DEFAULT_INNER + from api.models.cmdb import AutoDiscoveryRule + for i in DEFAULT_INNER: + existed = AutoDiscoveryRule.get_by(name=i['name'], first=True, to_dict=False) + if existed is not None: + if "en" in i['option'] and 'en' not in (existed.option or {}): + option = copy.deepcopy(existed.option) + option['en'] = i['option']['en'] + existed.update(option=option, commit=False) db.session.commit() diff --git a/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py b/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py index 8b0ed9a..8d203fc 100644 --- a/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py +++ b/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py @@ -3,7 +3,6 @@ import copy import datetime import json import os - from flask import abort from flask import current_app from flask_login import current_user @@ -11,7 +10,8 @@ from sqlalchemy import func from api.extensions import db from api.lib.cmdb.auto_discovery.const import ClOUD_MAP -from api.lib.cmdb.auto_discovery.const import DEFAULT_HTTP +from api.lib.cmdb.auto_discovery.const import DEFAULT_INNER +from api.lib.cmdb.auto_discovery.const import PRIVILEGED_USERS from api.lib.cmdb.cache import AttributeCache from api.lib.cmdb.cache import CITypeAttributeCache from api.lib.cmdb.cache import CITypeCache @@ -22,6 +22,7 @@ from api.lib.cmdb.const import AutoDiscoveryType from api.lib.cmdb.const import CMDB_QUEUE from api.lib.cmdb.const import PermEnum from api.lib.cmdb.const import ResourceTypeEnum +from api.lib.cmdb.custom_dashboard import SystemConfigManager from api.lib.cmdb.resp_format import ErrFormat from api.lib.cmdb.search import SearchError from api.lib.cmdb.search.ci import search as ci_search @@ -109,9 +110,9 @@ class AutoDiscoveryRuleCRUD(DBMixin): else: self.cls.create(**rule) - def _can_add(self, **kwargs): + def _can_add(self, valid=True, **kwargs): self.cls.get_by(name=kwargs['name']) and abort(400, ErrFormat.adr_duplicate.format(kwargs['name'])) - if kwargs.get('is_plugin') and kwargs.get('plugin_script'): + if kwargs.get('is_plugin') and kwargs.get('plugin_script') and valid: kwargs = check_plugin_script(**kwargs) acl = ACLManager(app_cli.app_name) has_perm = True @@ -132,7 +133,7 @@ class AutoDiscoveryRuleCRUD(DBMixin): return kwargs - def _can_update(self, **kwargs): + def _can_update(self, valid=True, **kwargs): existed = self.cls.get_by_id(kwargs['_id']) or abort( 404, ErrFormat.adr_not_found.format("id={}".format(kwargs['_id']))) @@ -144,7 +145,7 @@ class AutoDiscoveryRuleCRUD(DBMixin): if other and other.id != existed.id: return abort(400, ErrFormat.adr_duplicate.format(kwargs['name'])) - if existed.is_plugin: + if existed.is_plugin and valid: acl = ACLManager(app_cli.app_name) has_perm = True try: @@ -202,8 +203,9 @@ class AutoDiscoveryCITypeCRUD(DBMixin): cls = AutoDiscoveryCIType @classmethod - def get_all(cls): - return cls.cls.get_by(to_dict=False) + def get_all(cls, type_ids=None): + res = cls.cls.get_by(to_dict=False) + return [i for i in res if type_ids is None or i.type_id in type_ids] @classmethod def get_by_id(cls, _id): @@ -222,7 +224,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin): if not adr: continue if adr.type == "http": - for i in DEFAULT_HTTP: + for i in DEFAULT_INNER: if adr.name == i['name']: attrs = AutoDiscoveryHTTPManager.get_attributes( i['en'], (adt.extra_option or {}).get('category')) or [] @@ -243,11 +245,17 @@ class AutoDiscoveryCITypeCRUD(DBMixin): for rule in rules: if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('secret'): - if not (current_user.username == "cmdb_agent" or current_user.uid == rule['uid']): + if not (current_user.username in PRIVILEGED_USERS or current_user.uid == rule['uid']): rule['extra_option'].pop('secret', None) else: rule['extra_option']['secret'] = AESCrypto.decrypt(rule['extra_option']['secret']) + if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('password'): + if not (current_user.username in PRIVILEGED_USERS or current_user.uid == rule['uid']): + rule['extra_option'].pop('password', None) + else: + rule['extra_option']['password'] = AESCrypto.decrypt(rule['extra_option']['password']) + if oneagent_id and rule['agent_id'] == oneagent_id: result.append(rule) elif rule['query_expr']: @@ -271,17 +279,19 @@ class AutoDiscoveryCITypeCRUD(DBMixin): result.append(rule) + + ad_rules_updated_at = (SystemConfigManager.get('ad_rules_updated_at') or {}).get('option', {}).get('v') or "" new_last_update_at = "" for i in result: i['adr'] = AutoDiscoveryRule.get_by_id(i['adr_id']).to_dict() + i['adr'].pop("attributes", None) __last_update_at = max([i['updated_at'] or "", i['created_at'] or "", - i['adr']['created_at'] or "", i['adr']['updated_at'] or ""]) + i['adr']['created_at'] or "", i['adr']['updated_at'] or "", ad_rules_updated_at]) if new_last_update_at < __last_update_at: new_last_update_at = __last_update_at write_ad_rule_sync_history.apply_async(args=(result, oneagent_id, oneagent_name, datetime.datetime.now()), queue=CMDB_QUEUE) - if not last_update_at or new_last_update_at > last_update_at: return result, new_last_update_at else: @@ -346,7 +356,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin): if adr.type == "http": kwargs.setdefault('extra_option', dict) en_name = None - for i in DEFAULT_HTTP: + for i in DEFAULT_INNER: if i['name'] == adr.name: en_name = i['en'] break @@ -362,10 +372,13 @@ class AutoDiscoveryCITypeCRUD(DBMixin): if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'): kwargs['extra_option']['secret'] = AESCrypto.encrypt(kwargs['extra_option']['secret']) + if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('password'): + kwargs['extra_option']['password'] = AESCrypto.encrypt(kwargs['extra_option']['password']) ci_type = CITypeCache.get(kwargs['type_id']) unique = AttributeCache.get(ci_type.unique_id) if unique and unique.name not in (kwargs.get('attributes') or {}).values(): + current_app.logger.warning((unique.name, kwargs.get('attributes'), ci_type.alias)) return abort(400, ErrFormat.ad_not_unique_key.format(unique.name)) kwargs['uid'] = current_user.uid @@ -381,7 +394,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin): if adr.type == "http": kwargs.setdefault('extra_option', dict) en_name = None - for i in DEFAULT_HTTP: + for i in DEFAULT_INNER: if i['name'] == adr.name: en_name = i['en'] break @@ -398,11 +411,15 @@ class AutoDiscoveryCITypeCRUD(DBMixin): ci_type = CITypeCache.get(existed.type_id) unique = AttributeCache.get(ci_type.unique_id) if unique and unique.name not in (kwargs.get('attributes') or {}).values(): + current_app.logger.warning((unique.name, kwargs.get('attributes'), ci_type.alias)) return abort(400, ErrFormat.ad_not_unique_key.format(unique.name)) if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'): if current_user.uid != existed.uid: return abort(403, ErrFormat.adt_secret_no_permission) + if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('password'): + if current_user.uid != existed.uid: + return abort(403, ErrFormat.adt_secret_no_permission) return existed @@ -413,6 +430,8 @@ class AutoDiscoveryCITypeCRUD(DBMixin): if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'): kwargs['extra_option']['secret'] = AESCrypto.encrypt(kwargs['extra_option']['secret']) + if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('password'): + kwargs['extra_option']['password'] = AESCrypto.encrypt(kwargs['extra_option']['password']) inst = self._can_update(_id=_id, **kwargs) if inst.agent_id != kwargs.get('agent_id') or inst.query_expr != kwargs.get('query_expr'): @@ -420,6 +439,9 @@ class AutoDiscoveryCITypeCRUD(DBMixin): item.delete(commit=False) db.session.commit() + SystemConfigManager.create_or_update("ad_rules_updated_at", + dict(v=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))) + obj = inst.update(_id=_id, filter_none=False, **kwargs) return obj @@ -453,6 +475,10 @@ class AutoDiscoveryCITypeCRUD(DBMixin): class AutoDiscoveryCITypeRelationCRUD(DBMixin): cls = AutoDiscoveryCITypeRelation + @classmethod + def get_all(cls, type_ids=None): + res = cls.cls.get_by(to_dict=False) + return [i for i in res if type_ids is None or i.ad_type_id in type_ids] @classmethod def get_by_type_id(cls, type_id, to_dict=False): return cls.cls.get_by(ad_type_id=type_id, to_dict=to_dict) @@ -692,12 +718,13 @@ class AutoDiscoveryHTTPManager(object): categories = (ClOUD_MAP.get(name) or {}) or [] for item in copy.deepcopy(categories): item.pop('map', None) + item.pop('collect_key_map', None) return categories def get_resources(self, name): en_name = None - for i in DEFAULT_HTTP: + for i in DEFAULT_INNER: if i['name'] == name: en_name = i['en'] break @@ -733,6 +760,17 @@ class AutoDiscoverySNMPManager(object): return [] +class AutoDiscoveryComponentsManager(object): + + @staticmethod + def get_attributes(name): + if os.path.exists(os.path.join(PWD, "templates/{}.json".format(name))): + with open(os.path.join(PWD, "templates/{}.json".format(name))) as f: + return json.loads(f.read()) + + return [] + + class AutoDiscoveryRuleSyncHistoryCRUD(DBMixin): cls = AutoDiscoveryRuleSyncHistory diff --git a/cmdb-api/api/lib/cmdb/auto_discovery/const.py b/cmdb-api/api/lib/cmdb/auto_discovery/const.py index 9ca229f..0d16666 100644 --- a/cmdb-api/api/lib/cmdb/auto_discovery/const.py +++ b/cmdb-api/api/lib/cmdb/auto_discovery/const.py @@ -2,16 +2,28 @@ from api.lib.cmdb.const import AutoDiscoveryType -DEFAULT_HTTP = [ +PRIVILEGED_USERS = ("cmdb_agent", "worker", "admin") + +DEFAULT_INNER = [ dict(name="阿里云", en="aliyun", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False, - option={'icon': {'name': 'caise-aliyun'}}), + option={'icon': {'name': 'caise-aliyun'}, "en": "aliyun"}), dict(name="腾讯云", en="tencentcloud", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False, - option={'icon': {'name': 'caise-tengxunyun'}}), + option={'icon': {'name': 'caise-tengxunyun'}, "en": "tencentcloud"}), dict(name="华为云", en="huaweicloud", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False, - option={'icon': {'name': 'caise-huaweiyun'}}), + option={'icon': {'name': 'caise-huaweiyun'}, "en": "huaweicloud"}), dict(name="AWS", en="aws", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False, option={'icon': {'name': 'caise-aws'}}), + dict(name="VCenter", en="vcenter", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False, + option={'icon': {'name': 'cmdb-vcenter'}, "category": "private_cloud", "en": "vcenter"}), + dict(name="KVM", en="kvm", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False, + option={'icon': {'name': 'ops-KVM'}, "category": "private_cloud", "en": "kvm"}), + + dict(name="Nginx", en="nginx", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False, + option={'icon': {'name': 'caise-nginx'}, "en": "nginx"}), + dict(name="Redis", en="redis", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False, + option={'icon': {'name': 'caise-redis'}, "en": "redis"}), + dict(name="交换机", type=AutoDiscoveryType.SNMP, is_inner=True, is_plugin=False, option={'icon': {'name': 'caise-jiaohuanji'}}), dict(name="路由器", type=AutoDiscoveryType.SNMP, is_inner=True, is_plugin=False, @@ -23,47 +35,306 @@ DEFAULT_HTTP = [ ] ClOUD_MAP = { - "aliyun": [{ - "category": "计算", - "items": ["云服务器 ECS"], - "map": { - "云服务器 ECS": "templates/aliyun_ecs.json", + "aliyun": [ + { + "category": "计算", + "items": ["云服务器 ECS", "云服务器 Disk"], + "map": { + "云服务器 ECS": "templates/aliyun_ecs.json", + "云服务器 Disk": "templates/aliyun_ecs_disk2.json", + }, + "collect_key_map": { + "云服务器 ECS": "ali.ecs", + "云服务器 Disk": "ali.ecs_disk", + }, }, - "collect_key_map": { - "云服务器 ECS": "ali.ecs", - } - }], - - "tencentcloud": [{ - "category": "计算", - "items": ["云服务器 CVM"], - "map": { - "云服务器 CVM": "templates/tencent_cvm.json", + { + "category": "网络与CDN", + "items": [ + "内容分发CDN", + "负载均衡SLB", + "专有网络VPC", + "交换机Switch", + ], + "map": { + "内容分发CDN": "templates/aliyun_cdn.json", + "负载均衡SLB": "templates/aliyun_slb.json", + "专有网络VPC": "templates/aliyun_vpc.json", + "交换机Switch": "templates/aliyun_switch.json", + }, + "collect_key_map": { + "内容分发CDN": "ali.cdn", + "负载均衡SLB": "ali.slb", + "专有网络VPC": "ali.vpc", + "交换机Switch": "ali.switch", + }, }, - "collect_key_map": { - "云服务器 CVM": "tencent.cvm", - } - }], - - "huaweicloud": [{ - "category": "计算", - "items": ["云服务器 ECS"], - "map": { - "云服务器 ECS": "templates/huaweicloud_ecs.json", + { + "category": "存储", + "items": ["块存储EBS", "对象存储OSS"], + "map": { + "块存储EBS": "templates/aliyun_ebs.json", + "对象存储OSS": "templates/aliyun_oss.json", + }, + "collect_key_map": { + "块存储EBS": "ali.ebs", + "对象存储OSS": "ali.oss", + }, }, - "collect_key_map": { - "云服务器 ECS": "huawei.ecs", - } - }], - - "aws": [{ - "category": "计算", - "items": ["云服务器 EC2"], - "map": { - "云服务器 EC2": "templates/aws_ec2.json", + { + "category": "数据库", + "items": ["云数据库RDS MySQL", "云数据库RDS PostgreSQL", "云数据库 Redis"], + "map": { + "云数据库RDS MySQL": "templates/aliyun_rds_mysql.json", + "云数据库RDS PostgreSQL": "templates/aliyun_rds_postgre.json", + "云数据库 Redis": "templates/aliyun_redis.json", + }, + "collect_key_map": { + "云数据库RDS MySQL": "ali.rds_mysql", + "云数据库RDS PostgreSQL": "ali.rds_postgre", + "云数据库 Redis": "ali.redis", + }, }, - "collect_key_map": { - "云服务器 EC2": "aws.ec2", - } - }], + ], + "tencentcloud": [ + { + "category": "计算", + "items": ["云服务器 CVM"], + "map": { + "云服务器 CVM": "templates/tencent_cvm.json", + }, + "collect_key_map": { + "云服务器 CVM": "tencent.cvm", + }, + }, + { + "category": "CDN与边缘", + "items": ["内容分发CDN"], + "map": { + "内容分发CDN": "templates/tencent_cdn.json", + }, + "collect_key_map": { + "内容分发CDN": "tencent.cdn", + }, + }, + { + "category": "网络", + "items": ["负载均衡CLB", "私有网络VPC", "子网"], + "map": { + "负载均衡CLB": "templates/tencent_clb.json", + "私有网络VPC": "templates/tencent_vpc.json", + "子网": "templates/tencent_subnet.json", + }, + "collect_key_map": { + "负载均衡CLB": "tencent.clb", + "私有网络VPC": "tencent.vpc", + "子网": "tencent.subnet", + }, + }, + { + "category": "存储", + "items": ["云硬盘CBS", "对象存储COS"], + "map": { + "云硬盘CBS": "templates/tencent_cbs.json", + "对象存储OSS": "templates/tencent_cos.json", + }, + "collect_key_map": { + "云硬盘CBS": "tencent.cbs", + "对象存储OSS": "tencent.cos", + }, + }, + { + "category": "数据库", + "items": ["云数据库 MySQL", "云数据库 PostgreSQL", "云数据库 Redis"], + "map": { + "云数据库 MySQL": "templates/tencent_rdb.json", + "云数据库 PostgreSQL": "templates/tencent_postgres.json", + "云数据库 Redis": "templates/tencent_redis.json", + }, + "collect_key_map": { + "云数据库 MySQL": "tencent.rdb", + "云数据库 PostgreSQL": "tencent.rds_postgres", + "云数据库 Redis": "tencent.redis", + }, + }, + ], + "huaweicloud": [ + { + "category": "计算", + "items": ["云服务器 ECS"], + "map": { + "云服务器 ECS": "templates/huaweicloud_ecs.json", + }, + "collect_key_map": { + "云服务器 ECS": "huawei.ecs", + }, + }, + { + "category": "CDN与智能边缘", + "items": ["内容分发网络CDN"], + "map": { + "内容分发网络CDN": "templates/huawei_cdn.json", + }, + "collect_key_map": { + "内容分发网络CDN": "huawei.cdn", + }, + }, + { + "category": "网络", + "items": ["弹性负载均衡ELB", "虚拟私有云VPC", "子网"], + "map": { + "弹性负载均衡ELB": "templates/huawei_elb.json", + "虚拟私有云VPC": "templates/huawei_vpc.json", + "子网": "templates/huawei_subnet.json", + }, + "collect_key_map": { + "弹性负载均衡ELB": "huawei.elb", + "虚拟私有云VPC": "huawei.vpc", + "子网": "huawei.subnet", + }, + }, + { + "category": "存储", + "items": ["云硬盘EVS", "对象存储OBS"], + "map": { + "云硬盘EVS": "templates/huawei_evs.json", + "对象存储OBS": "templates/huawei_obs.json", + }, + "collect_key_map": { + "云硬盘EVS": "huawei.evs", + "对象存储OBS": "huawei.obs", + }, + }, + { + "category": "数据库", + "items": ["云数据库RDS MySQL", "云数据库RDS PostgreSQL"], + "map": { + "云数据库RDS MySQL": "templates/huawei_rds_mysql.json", + "云数据库RDSPostgreSQL": "templates/huaweirds_postgre.json", + }, + "collect_key_map": { + "云数据库RDS MySQL": "huawei.rds_mysql", + "云数据库RDS PostgreSQL": "huawei.rds_postgre", + }, + }, + { + "category": "应用中间件", + "items": ["分布式缓存Redis"], + "map": { + "分布式缓存Redis": "templates/huawei_dcs.json", + }, + "collect_key_map": { + "分布式缓存Redis": "huawei.dcs", + }, + }, + ], + "aws": [ + { + "category": "计算", + "items": ["云服务器 EC2"], + "map": { + "云服务器 EC2": "templates/aws_ec2.json", + }, + "collect_key_map": { + "云服务器 EC2": "aws.ec2", + }, + }, + {"category": "网络与CDN", "items": [], "map": {}, "collect_key_map": {}}, + ], + "vcenter": [ + { + "category": "计算", + "items": [ + "主机", + "虚拟机", + "主机集群" + ], + "map": { + "主机": "templates/vsphere_host.json", + "虚拟机": "templates/vsphere_vm.json", + "主机集群": "templates/vsphere_cluster.json", + }, + "collect_key_map": { + "主机": "vsphere.host", + "虚拟机": "vsphere.vm", + "主机集群": "vsphere.cluster", + }, + }, + { + "category": "网络", + "items": [ + "网络", + "标准交换机", + "分布式交换机", + ], + "map": { + "网络": "templates/vsphere_network.json", + "标准交换机": "templates/vsphere_standard_switch.json", + "分布式交换机": "templates/vsphere_distributed_switch.json", + }, + "collect_key_map": { + "网络": "vsphere.network", + "标准交换机": "vsphere.standard_switch", + "分布式交换机": "vsphere.distributed_switch", + }, + }, + { + "category": "存储", + "items": ["数据存储", "数据存储集群"], + "map": { + "数据存储": "templates/vsphere_datastore.json", + "数据存储集群": "templates/vsphere.storage_pod.json", + }, + "collect_key_map": { + "数据存储": "vsphere.datastore", + "数据存储集群": "vsphere.storage_pod", + }, + }, + { + "category": "其他", + "items": ["资源池", "数据中心", "文件夹"], + "map": { + "资源池": "templates/vsphere_datastore.json", + "数据中心": "templates/vsphere_datacenter.json", + "文件夹": "templates/vsphere_folder.json", + }, + "collect_key_map": { + "资源池": "vsphere.pool", + "数据中心": "vsphere.datacenter", + "文件夹": "vsphere.folder", + }, + }, + ], + "kvm": [ + { + "category": "计算", + "items": ["虚拟机"], + "map": { + "虚拟机": "templates/kvm_vm.json", + }, + "collect_key_map": { + "虚拟机": "kvm.vm", + }, + }, + { + "category": "存储", + "items": ["存储"], + "map": { + "存储": "templates/kvm_storage.json", + }, + "collect_key_map": { + "存储": "kvm.storage", + }, + }, + { + "category": "network", + "items": ["网络"], + "map": { + "网络": "templates/kvm_network.json", + }, + "collect_key_map": { + "网络": "kvm.network", + }, + }, + ], } diff --git a/cmdb-api/api/lib/cmdb/ci_type.py b/cmdb-api/api/lib/cmdb/ci_type.py index 3b66302..847ef98 100644 --- a/cmdb-api/api/lib/cmdb/ci_type.py +++ b/cmdb-api/api/lib/cmdb/ci_type.py @@ -1,7 +1,6 @@ # -*- coding:utf-8 -*- import copy - import toposort from flask import abort from flask import current_app @@ -84,7 +83,7 @@ class CITypeManager(object): self.cls.id, self.cls.icon, self.cls.name).filter(self.cls.deleted.is_(False))} @staticmethod - def get_ci_types(type_name=None, like=True): + def get_ci_types(type_name=None, like=True, type_ids=None): resources = None if current_app.config.get('USE_ACL') and not is_app_admin('cmdb'): resources = set([i.get('name') for i in ACLManager().get_resources(ResourceTypeEnum.CI_TYPE)]) @@ -93,6 +92,9 @@ class CITypeManager(object): CIType.get_by_like(name=type_name) if like else CIType.get_by(name=type_name)) res = list() for type_dict in ci_types: + if type_ids is not None and type_dict['id'] not in type_ids: + continue + attr = AttributeCache.get(type_dict["unique_id"]) type_dict["unique_key"] = attr and attr.name if type_dict.get('show_id'): @@ -292,6 +294,12 @@ class CITypeManager(object): class CITypeInheritanceManager(object): cls = CITypeInheritance + @classmethod + def get_all(cls, type_ids=None): + res = cls.cls.get_by(to_dict=True) + + return [i for i in res if type_ids is None or (i['parent_id'] in type_ids and i['child_id'] in type_ids)] + @classmethod def get_parents(cls, type_id): return [i.parent_id for i in cls.cls.get_by(child_id=type_id, to_dict=False)] @@ -387,7 +395,7 @@ class CITypeGroupManager(object): cls = CITypeGroup @staticmethod - def get(need_other=None, config_required=True): + def get(need_other=None, config_required=True, type_ids=None, ci_types=None): resources = None if current_app.config.get('USE_ACL'): resources = ACLManager('cmdb').get_resources(ResourceTypeEnum.CI) @@ -401,6 +409,8 @@ class CITypeGroupManager(object): for group in groups: for t in sorted(CITypeGroupItem.get_by(group_id=group['id']), key=lambda x: x['order'] or 0): ci_type = CITypeCache.get(t['type_id']).to_dict() + if type_ids is not None and ci_type['id'] not in type_ids: + continue if resources is None or (ci_type and ci_type['name'] in resources): ci_type['permissions'] = resources[ci_type['name']] if resources is not None else None ci_type['inherited'] = True if CITypeInheritanceManager.get_parents(ci_type['id']) else False @@ -408,7 +418,7 @@ class CITypeGroupManager(object): group_types.add(t["type_id"]) if need_other: - ci_types = CITypeManager.get_ci_types() + ci_types = CITypeManager.get_ci_types(type_ids=type_ids) if ci_types is None else ci_types other_types = dict(ci_types=[]) for ci_type in ci_types: if ci_type["id"] not in group_types and (resources is None or ci_type['name'] in resources): @@ -529,6 +539,8 @@ class CITypeAttributeManager(object): attrs = CITypeAttributesCache.get(_type_id) for attr in sorted(attrs, key=lambda x: (x.order, x.id)): attr_dict = AttributeManager().get_attribute(attr.attr_id, choice_web_hook_parse, choice_other_parse) + if not attr_dict: + continue attr_dict["is_required"] = attr.is_required attr_dict["order"] = attr.order attr_dict["default_show"] = attr.default_show @@ -537,7 +549,6 @@ class CITypeAttributeManager(object): if not has_config_perm: attr_dict.pop('choice_web_hook', None) attr_dict.pop('choice_other', None) - if attr_dict['id'] not in id2pos: id2pos[attr_dict['id']] = len(result) result.append(attr_dict) @@ -602,7 +613,6 @@ class CITypeAttributeManager(object): if existed is not None: continue - current_app.logger.debug(attr_id) CITypeAttribute.create(type_id=type_id, attr_id=attr_id, **kwargs) attr = AttributeCache.get(attr_id) @@ -769,11 +779,15 @@ class CITypeRelationManager(object): """ @staticmethod - def get(): + def get(type_ids=None): res = CITypeRelation.get_by(to_dict=False) type2attributes = dict() + result = [] for idx, item in enumerate(res): _item = item.to_dict() + if type_ids is not None and _item['parent_id'] not in type_ids and _item['child_id'] not in type_ids: + continue + res[idx] = _item res[idx]['parent'] = item.parent.to_dict() if item.parent_id not in type2attributes: @@ -785,7 +799,9 @@ class CITypeRelationManager(object): CITypeAttributeManager.get_all_attributes(item.child_id)] res[idx]['relation_type'] = item.relation_type.to_dict() - return res, type2attributes + result.append(res[idx]) + + return result, type2attributes @staticmethod def get_child_type_ids(type_id, level): @@ -1034,7 +1050,8 @@ class CITypeAttributeGroupManager(object): parent_ids = CITypeInheritanceManager.base(type_id) groups = [] - id2type = {i: CITypeCache.get(i).alias for i in parent_ids} + id2type = {i: CITypeCache.get(i) for i in parent_ids} + id2type = {k: v.alias for k, v in id2type.items() if v} for _type_id in parent_ids + [type_id]: _groups = CITypeAttributeGroup.get_by(type_id=_type_id) _groups = sorted(_groups, key=lambda x: x["order"] or 0) @@ -1308,13 +1325,14 @@ class CITypeTemplateManager(object): attributes = [attr for type_id in type2attributes for attr in type2attributes[type_id]] attrs = [] for i in copy.deepcopy(attributes): + if i.pop('inherited', None): + continue i.pop('default_show', None) i.pop('is_required', None) i.pop('order', None) i.pop('choice_web_hook', None) i.pop('choice_other', None) i.pop('order', None) - i.pop('inherited', None) i.pop('inherited_from', None) choice_value = i.pop('choice_value', None) if not choice_value: @@ -1334,6 +1352,7 @@ class CITypeTemplateManager(object): for i in ci_types: i.pop("unique_key", None) i.pop("show_name", None) + i.pop("parent_ids", None) i['unique_id'] = attr_id_map.get(i['unique_id'], i['unique_id']) if i.get('show_id'): i['show_id'] = attr_id_map.get(i['show_id'], i['show_id']) @@ -1371,7 +1390,7 @@ class CITypeTemplateManager(object): return self.__import(RelationType, relation_types) @staticmethod - def _import_ci_type_relations(ci_type_relations, type_id_map, relation_type_id_map): + def _import_ci_type_relations(ci_type_relations, type_id_map, relation_type_id_map, attr_id_map): for i in ci_type_relations: i.pop('parent', None) i.pop('child', None) @@ -1381,15 +1400,32 @@ class CITypeTemplateManager(object): i['child_id'] = type_id_map.get(i['child_id'], i['child_id']) i['relation_type_id'] = relation_type_id_map.get(i['relation_type_id'], i['relation_type_id']) + i['parent_attr_ids'] = [attr_id_map.get(attr_id, attr_id) for attr_id in i.get('parent_attr_ids') or []] + i['child_attr_ids'] = [attr_id_map.get(attr_id, attr_id) for attr_id in i.get('child_attr_ids') or []] try: CITypeRelationManager.add(i.get('parent_id'), i.get('child_id'), i.get('relation_type_id'), i.get('constraint'), + parent_attr_ids=i.get('parent_attr_ids', []), + child_attr_ids=i.get('child_attr_ids', []), ) - except BadRequest: + except Exception: pass + @staticmethod + def _import_ci_type_inheritance(ci_type_inheritance, type_id_map): + + for i in ci_type_inheritance: + i['parent_id'] = type_id_map.get(i['parent_id']) + i['child_id'] = type_id_map.get(i['child_id']) + + if i['parent_id'] and i['child_id']: + try: + CITypeInheritanceManager.add([i.get('parent_id')], i.get('child_id')) + except BadRequest: + pass + @staticmethod def _import_type_attributes(type2attributes, type_id_map, attr_id_map): for type_id in type2attributes: @@ -1401,6 +1437,9 @@ class CITypeTemplateManager(object): handled = set() for attr in type2attributes[type_id]: + if attr.get('inherited'): + continue + payload = dict(type_id=type_id_map.get(int(type_id), type_id), attr_id=attr_id_map.get(attr['id'], attr['id']), default_show=attr['default_show'], @@ -1464,6 +1503,9 @@ class CITypeTemplateManager(object): for rule in rules: ci_type = CITypeCache.get(rule.pop('type_name', None)) + if ci_type is None: + continue + adr = rule.pop('adr', {}) or {} if ci_type: @@ -1476,10 +1518,10 @@ class CITypeTemplateManager(object): if ad_rule: rule['adr_id'] = ad_rule.id - ad_rule.update(**adr) + ad_rule.update(valid=False, **adr) elif adr: - ad_rule = AutoDiscoveryRuleCRUD().add(**adr) + ad_rule = AutoDiscoveryRuleCRUD().add(valid=False, **adr) rule['adr_id'] = ad_rule.id else: continue @@ -1508,6 +1550,23 @@ class CITypeTemplateManager(object): except Exception as e: current_app.logger.warning("import auto discovery rules failed: {}".format(e)) + @staticmethod + def _import_auto_discovery_relation_rules(rules): + from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeRelationCRUD + + for rule in rules: + ad_ci_type = CITypeCache.get(rule.pop('ad_type_name', None)) + peer_ci_type = CITypeCache.get(rule.pop('peer_type_name', None)) + peer_attr = AttributeCache.get(rule.pop('peer_attr_name', None)) + if ad_ci_type and peer_attr and peer_ci_type: + if not AutoDiscoveryCITypeRelation.get_by( + ad_type_id=ad_ci_type.id, ad_key=rule.get('ad_key'), + peer_attr_id=peer_attr.id, peer_type_id=peer_ci_type.id): + AutoDiscoveryCITypeRelationCRUD().add(ad_type_id=ad_ci_type.id, + ad_key=rule.get('ad_key'), + peer_attr_id=peer_attr.id, + peer_type_id=peer_ci_type.id) + @staticmethod def _import_icons(icons): from api.lib.common_setting.upload_file import CommonFileCRUD @@ -1519,6 +1578,8 @@ class CITypeTemplateManager(object): current_app.logger.warning("save icon failed: {}".format(e)) def import_template(self, tpt): + db.session.commit() + import time s = time.time() attr_id_map = self._import_attributes(tpt.get('type2attributes') or {}) @@ -1537,9 +1598,14 @@ class CITypeTemplateManager(object): current_app.logger.info('import relation_types cost: {}'.format(time.time() - s)) s = time.time() - self._import_ci_type_relations(tpt.get('ci_type_relations') or [], ci_type_id_map, relation_type_id_map) + self._import_ci_type_relations(tpt.get('ci_type_relations') or [], + ci_type_id_map, relation_type_id_map, attr_id_map) current_app.logger.info('import ci_type_relations cost: {}'.format(time.time() - s)) + s = time.time() + self._import_ci_type_inheritance(tpt.get('ci_type_inheritance') or [], ci_type_id_map) + current_app.logger.info('import ci_type_inheritance cost: {}'.format(time.time() - s)) + s = time.time() self._import_type_attributes(tpt.get('type2attributes') or {}, ci_type_id_map, attr_id_map) current_app.logger.info('import type2attributes cost: {}'.format(time.time() - s)) @@ -1552,26 +1618,73 @@ class CITypeTemplateManager(object): self._import_auto_discovery_rules(tpt.get('ci_type_auto_discovery_rules') or []) current_app.logger.info('import ci_type_auto_discovery_rules cost: {}'.format(time.time() - s)) + s = time.time() + self._import_auto_discovery_relation_rules(tpt.get('ci_type_auto_discovery_relation_rules') or []) + current_app.logger.info('import ci_type_auto_discovery_relation_rules cost: {}'.format(time.time() - s)) + s = time.time() self._import_icons(tpt.get('icons') or {}) current_app.logger.info('import icons cost: {}'.format(time.time() - s)) @staticmethod - def export_template(): + def export_template(type_ids=None): from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeCRUD + from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeRelationCRUD from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryRuleCRUD from api.lib.common_setting.upload_file import CommonFileCRUD + ci_types = CITypeManager.get_ci_types(type_ids=type_ids) + extend_type_ids = [] + for ci_type in ci_types: + if ci_type.get('parent_ids'): + extend_type_ids.extend(CITypeInheritanceManager.base(ci_type['id'])) + extend_type_ids = list(set(extend_type_ids) - set(type_ids)) + + ci_type_relations = CITypeRelationManager.get(type_ids=type_ids)[0] + for i in ci_type_relations: + if i['parent_id'] not in type_ids: + extend_type_ids.append(i['parent_id']) + if i['child_id'] not in type_ids: + extend_type_ids.append(i['child_id']) + + ad_relation_rules = AutoDiscoveryCITypeRelationCRUD.get_all(type_ids=type_ids) + rules = [] + for r in ad_relation_rules: + if r.peer_type_id not in type_ids: + extend_type_ids.append(r.peer_type_id) + + r = r.to_dict() + r['ad_type_name'] = CITypeCache.get(r.pop('ad_type_id')).name + peer_type_id = r.pop("peer_type_id") + peer_type_name = CITypeCache.get(peer_type_id).name + if not peer_type_name: + peer_type = CITypeCache.get(peer_type_id) + peer_type_name = peer_type and peer_type.name + r['peer_type_name'] = peer_type_name + peer_attr_id = r.pop("peer_attr_id") + peer_attr = AttributeCache.get(peer_attr_id) + r['peer_attr_name'] = peer_attr and peer_attr.name + + rules.append(r) + ci_type_auto_discovery_relation_rules = rules + + if extend_type_ids: + extend_type_ids = list(set(extend_type_ids)) + type_ids.extend(extend_type_ids) + ci_types.extend(CITypeManager.get_ci_types(type_ids=extend_type_ids)) + tpt = dict( - ci_types=CITypeManager.get_ci_types(), - ci_type_groups=CITypeGroupManager.get(), + ci_types=ci_types, relation_types=[i.to_dict() for i in RelationTypeManager.get_all()], - ci_type_relations=CITypeRelationManager.get()[0], + ci_type_relations=ci_type_relations, + ci_type_inheritance=CITypeInheritanceManager.get_all(type_ids=type_ids), ci_type_auto_discovery_rules=list(), + ci_type_auto_discovery_relation_rules=ci_type_auto_discovery_relation_rules, type2attributes=dict(), type2attribute_group=dict(), icons=dict() ) + tpt['ci_type_groups'] = CITypeGroupManager.get(ci_types=tpt['ci_types'], type_ids=type_ids) def get_icon_value(icon): try: @@ -1579,12 +1692,13 @@ class CITypeTemplateManager(object): except: return "" - ad_rules = AutoDiscoveryCITypeCRUD.get_all() + type_id2name = {i['id']: i['name'] for i in tpt['ci_types']} + ad_rules = AutoDiscoveryCITypeCRUD.get_all(type_ids=type_ids) rules = [] for r in ad_rules: r = r.to_dict() - ci_type = CITypeCache.get(r.pop('type_id')) - r['type_name'] = ci_type and ci_type.name + + r['type_name'] = type_id2name.get(r.pop('type_id')) if r.get('adr_id'): adr = AutoDiscoveryRuleCRUD.get_by_id(r.pop('adr_id')) r['adr_name'] = adr and adr.name @@ -1618,65 +1732,6 @@ class CITypeTemplateManager(object): return tpt - @staticmethod - def export_template_by_type(type_id): - ci_type = CITypeCache.get(type_id) or abort(404, ErrFormat.ci_type_not_found2.format("id={}".format(type_id))) - - from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeCRUD - from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryRuleCRUD - from api.lib.common_setting.upload_file import CommonFileCRUD - - tpt = dict( - ci_types=CITypeManager.get_ci_types(type_name=ci_type.name, like=False), - ci_type_auto_discovery_rules=list(), - type2attributes=dict(), - type2attribute_group=dict(), - icons=dict() - ) - - def get_icon_value(icon): - try: - return CommonFileCRUD().get_file_binary_str(icon) - except: - return "" - - ad_rules = AutoDiscoveryCITypeCRUD.get_by_type_id(ci_type.id) - rules = [] - for r in ad_rules: - r = r.to_dict() - r['type_name'] = ci_type and ci_type.name - if r.get('adr_id'): - adr = AutoDiscoveryRuleCRUD.get_by_id(r.pop('adr_id')) - r['adr_name'] = adr and adr.name - r['adr'] = adr and adr.to_dict() or {} - - icon_url = r['adr'].get('option', {}).get('icon', {}).get('url') - if icon_url and icon_url not in tpt['icons']: - tpt['icons'][icon_url] = get_icon_value(icon_url) - - rules.append(r) - tpt['ci_type_auto_discovery_rules'] = rules - - for ci_type in tpt['ci_types']: - if ci_type['icon'] and len(ci_type['icon'].split('$$')) > 3: - icon_url = ci_type['icon'].split('$$')[3] - if icon_url not in tpt['icons']: - tpt['icons'][icon_url] = get_icon_value(icon_url) - - tpt['type2attributes'][ci_type['id']] = CITypeAttributeManager.get_attributes_by_type_id( - ci_type['id'], choice_web_hook_parse=False, choice_other_parse=False) - - for attr in tpt['type2attributes'][ci_type['id']]: - for i in (attr.get('choice_value') or []): - if (i[1] or {}).get('icon', {}).get('url') and len(i[1]['icon']['url'].split('$$')) > 3: - icon_url = i[1]['icon']['url'].split('$$')[3] - if icon_url not in tpt['icons']: - tpt['icons'][icon_url] = get_icon_value(icon_url) - - tpt['type2attribute_group'][ci_type['id']] = CITypeAttributeGroupManager.get_by_type_id(ci_type['id']) - - return tpt - class CITypeUniqueConstraintManager(object): @staticmethod diff --git a/cmdb-api/api/lib/cmdb/const.py b/cmdb-api/api/lib/cmdb/const.py index 3452129..d0302be 100644 --- a/cmdb-api/api/lib/cmdb/const.py +++ b/cmdb-api/api/lib/cmdb/const.py @@ -93,7 +93,8 @@ class RoleEnum(BaseEnum): class AutoDiscoveryType(BaseEnum): AGENT = "agent" SNMP = "snmp" - HTTP = "http" + HTTP = "http" # cloud + COMPONENTS = "components" class AttributeDefaultValueEnum(BaseEnum): diff --git a/cmdb-api/api/views/cmdb/auto_discovery.py b/cmdb-api/api/views/cmdb/auto_discovery.py index 5e3ead8..eb2b70a 100644 --- a/cmdb-api/api/views/cmdb/auto_discovery.py +++ b/cmdb-api/api/views/cmdb/auto_discovery.py @@ -2,23 +2,24 @@ import copy import json import uuid -from io import BytesIO - from flask import abort from flask import current_app from flask import request from flask_login import current_user +from io import BytesIO from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCICRUD from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeCRUD from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeRelationCRUD +from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryComponentsManager from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCounterCRUD from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryExecHistoryCRUD from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryHTTPManager from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryRuleCRUD from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryRuleSyncHistoryCRUD from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoverySNMPManager -from api.lib.cmdb.auto_discovery.const import DEFAULT_HTTP +from api.lib.cmdb.auto_discovery.const import DEFAULT_INNER +from api.lib.cmdb.auto_discovery.const import PRIVILEGED_USERS from api.lib.cmdb.const import PermEnum from api.lib.cmdb.const import ResourceTypeEnum from api.lib.cmdb.resp_format import ErrFormat @@ -42,7 +43,7 @@ class AutoDiscoveryRuleView(APIView): rebuild = False exists = {i['name'] for i in res} - for i in copy.deepcopy(DEFAULT_HTTP): + for i in copy.deepcopy(DEFAULT_INNER): if i['name'] not in exists: i.pop('en', None) AutoDiscoveryRuleCRUD().add(**i) @@ -110,12 +111,16 @@ class AutoDiscoveryRuleTemplateFileView(APIView): class AutoDiscoveryRuleHTTPView(APIView): url_prefix = ("/adr/http//categories", "/adr/http//attributes", - "/adr/snmp//attributes") + "/adr/snmp//attributes", + "/adr/components//attributes",) def get(self, name): if "snmp" in request.url: return self.jsonify(AutoDiscoverySNMPManager.get_attributes()) + if "components" in request.url: + return self.jsonify(AutoDiscoveryComponentsManager.get_attributes(name)) + if "attributes" in request.url: resource = request.values.get('resource') return self.jsonify(AutoDiscoveryHTTPManager.get_attributes(name, resource)) @@ -250,7 +255,7 @@ class AutoDiscoveryRuleSyncView(APIView): url_prefix = ("/adt/sync",) def get(self): - if current_user.username not in ("cmdb_agent", "worker", "admin"): + if current_user.username not in PRIVILEGED_USERS: return abort(403) oneagent_name = request.values.get('oneagent_name') diff --git a/cmdb-api/api/views/cmdb/ci.py b/cmdb-api/api/views/cmdb/ci.py index 08cebe7..9afd217 100644 --- a/cmdb-api/api/views/cmdb/ci.py +++ b/cmdb-api/api/views/cmdb/ci.py @@ -268,6 +268,7 @@ class CIBaselineView(APIView): return self.jsonify(CIManager().baseline(list(map(int, ci_ids)), before_date)) @args_required("before_date") + @has_perm_for_ci("ci_id", ResourceTypeEnum.CI, PermEnum.UPDATE, CIManager.get_type) def post(self, ci_id): if 'rollback' in request.url: before_date = request.values.get('before_date') diff --git a/cmdb-api/api/views/cmdb/ci_type.py b/cmdb-api/api/views/cmdb/ci_type.py index 7375afb..bd17a97 100644 --- a/cmdb-api/api/views/cmdb/ci_type.py +++ b/cmdb-api/api/views/cmdb/ci_type.py @@ -51,7 +51,11 @@ class CITypeView(APIView): q = request.args.get("type_name") if type_id is not None: - ci_type = CITypeCache.get(type_id).to_dict() + ci_type = CITypeCache.get(type_id) + if ci_type is None: + return abort(404, ErrFormat.ci_type_not_found) + + ci_type = ci_type.to_dict() ci_type['parent_ids'] = CITypeInheritanceManager.get_parents(type_id) ci_types = [ci_type] elif type_name is not None: @@ -357,15 +361,13 @@ class CITypeAttributeGroupView(APIView): class CITypeTemplateView(APIView): - url_prefix = ("/ci_types/template/import", "/ci_types/template/export", "/ci_types//template/export") + url_prefix = ("/ci_types/template/import", "/ci_types/template/export") @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Model_Configuration, app_cli.op.download_CIType, app_cli.admin_name) - def get(self, type_id=None): # export - if type_id is not None: - return self.jsonify(dict(ci_type_template=CITypeTemplateManager.export_template_by_type(type_id))) - - return self.jsonify(dict(ci_type_template=CITypeTemplateManager.export_template())) + def get(self): # export + type_ids = list(map(int, handle_arg_list(request.values.get('type_ids')))) or None + return self.jsonify(dict(ci_type_template=CITypeTemplateManager.export_template(type_ids=type_ids))) @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Model_Configuration, app_cli.op.download_CIType, app_cli.admin_name)