perf(api): CIType templates download (#567)

cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py

@@ -3,7 +3,6 @@ import copy
 import datetime
 import json
 import os
-
 from flask import abort
 from flask import current_app
 from flask_login import current_user
@@ -11,7 +10,8 @@ from sqlalchemy import func
 
 from api.extensions import db
 from api.lib.cmdb.auto_discovery.const import ClOUD_MAP
-from api.lib.cmdb.auto_discovery.const import DEFAULT_HTTP
+from api.lib.cmdb.auto_discovery.const import DEFAULT_INNER
+from api.lib.cmdb.auto_discovery.const import PRIVILEGED_USERS
 from api.lib.cmdb.cache import AttributeCache
 from api.lib.cmdb.cache import CITypeAttributeCache
 from api.lib.cmdb.cache import CITypeCache
@@ -22,6 +22,7 @@ from api.lib.cmdb.const import AutoDiscoveryType
 from api.lib.cmdb.const import CMDB_QUEUE
 from api.lib.cmdb.const import PermEnum
 from api.lib.cmdb.const import ResourceTypeEnum
+from api.lib.cmdb.custom_dashboard import SystemConfigManager
 from api.lib.cmdb.resp_format import ErrFormat
 from api.lib.cmdb.search import SearchError
 from api.lib.cmdb.search.ci import search as ci_search
@@ -109,9 +110,9 @@ class AutoDiscoveryRuleCRUD(DBMixin):
             else:
                 self.cls.create(**rule)
 
-    def _can_add(self, **kwargs):
+    def _can_add(self, valid=True, **kwargs):
         self.cls.get_by(name=kwargs['name']) and abort(400, ErrFormat.adr_duplicate.format(kwargs['name']))
-        if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
+        if kwargs.get('is_plugin') and kwargs.get('plugin_script') and valid:
             kwargs = check_plugin_script(**kwargs)
             acl = ACLManager(app_cli.app_name)
             has_perm = True
@@ -132,7 +133,7 @@ class AutoDiscoveryRuleCRUD(DBMixin):
 
         return kwargs
 
-    def _can_update(self, **kwargs):
+    def _can_update(self, valid=True, **kwargs):
         existed = self.cls.get_by_id(kwargs['_id']) or abort(
             404, ErrFormat.adr_not_found.format("id={}".format(kwargs['_id'])))
 
@@ -144,7 +145,7 @@ class AutoDiscoveryRuleCRUD(DBMixin):
             if other and other.id != existed.id:
                 return abort(400, ErrFormat.adr_duplicate.format(kwargs['name']))
 
-        if existed.is_plugin:
+        if existed.is_plugin and valid:
             acl = ACLManager(app_cli.app_name)
             has_perm = True
             try:
@@ -202,8 +203,9 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
     cls = AutoDiscoveryCIType
 
     @classmethod
-    def get_all(cls):
-        return cls.cls.get_by(to_dict=False)
+    def get_all(cls, type_ids=None):
+        res = cls.cls.get_by(to_dict=False)
+        return [i for i in res if type_ids is None or i.type_id in type_ids]
 
     @classmethod
     def get_by_id(cls, _id):
@@ -222,7 +224,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
             if not adr:
                 continue
             if adr.type == "http":
-                for i in DEFAULT_HTTP:
+                for i in DEFAULT_INNER:
                     if adr.name == i['name']:
                         attrs = AutoDiscoveryHTTPManager.get_attributes(
                             i['en'], (adt.extra_option or {}).get('category')) or []
@@ -243,11 +245,17 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
 
         for rule in rules:
             if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('secret'):
-                if not (current_user.username == "cmdb_agent" or current_user.uid == rule['uid']):
+                if not (current_user.username in PRIVILEGED_USERS or current_user.uid == rule['uid']):
                     rule['extra_option'].pop('secret', None)
                 else:
                     rule['extra_option']['secret'] = AESCrypto.decrypt(rule['extra_option']['secret'])
 
+            if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('password'):
+                if not (current_user.username in PRIVILEGED_USERS or current_user.uid == rule['uid']):
+                    rule['extra_option'].pop('password', None)
+                else:
+                    rule['extra_option']['password'] = AESCrypto.decrypt(rule['extra_option']['password'])
+
             if oneagent_id and rule['agent_id'] == oneagent_id:
                 result.append(rule)
             elif rule['query_expr']:
@@ -271,17 +279,19 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
 
                 result.append(rule)
 
+
+        ad_rules_updated_at = (SystemConfigManager.get('ad_rules_updated_at') or {}).get('option', {}).get('v') or ""
         new_last_update_at = ""
         for i in result:
             i['adr'] = AutoDiscoveryRule.get_by_id(i['adr_id']).to_dict()
+            i['adr'].pop("attributes", None)
             __last_update_at = max([i['updated_at'] or "", i['created_at'] or "",
-                                    i['adr']['created_at'] or "", i['adr']['updated_at'] or ""])
+                                    i['adr']['created_at'] or "", i['adr']['updated_at'] or "", ad_rules_updated_at])
             if new_last_update_at < __last_update_at:
                 new_last_update_at = __last_update_at
 
         write_ad_rule_sync_history.apply_async(args=(result, oneagent_id, oneagent_name, datetime.datetime.now()),
                                                queue=CMDB_QUEUE)
-
         if not last_update_at or new_last_update_at > last_update_at:
             return result, new_last_update_at
         else:
@@ -346,7 +356,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
             if adr.type == "http":
                 kwargs.setdefault('extra_option', dict)
                 en_name = None
-                for i in DEFAULT_HTTP:
+                for i in DEFAULT_INNER:
                     if i['name'] == adr.name:
                         en_name = i['en']
                         break
@@ -362,10 +372,13 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
 
         if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'):
             kwargs['extra_option']['secret'] = AESCrypto.encrypt(kwargs['extra_option']['secret'])
+        if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('password'):
+            kwargs['extra_option']['password'] = AESCrypto.encrypt(kwargs['extra_option']['password'])
 
         ci_type = CITypeCache.get(kwargs['type_id'])
         unique = AttributeCache.get(ci_type.unique_id)
         if unique and unique.name not in (kwargs.get('attributes') or {}).values():
+            current_app.logger.warning((unique.name, kwargs.get('attributes'), ci_type.alias))
             return abort(400, ErrFormat.ad_not_unique_key.format(unique.name))
 
         kwargs['uid'] = current_user.uid
@@ -381,7 +394,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
         if adr.type == "http":
             kwargs.setdefault('extra_option', dict)
             en_name = None
-            for i in DEFAULT_HTTP:
+            for i in DEFAULT_INNER:
                 if i['name'] == adr.name:
                     en_name = i['en']
                     break
@@ -398,11 +411,15 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
             ci_type = CITypeCache.get(existed.type_id)
             unique = AttributeCache.get(ci_type.unique_id)
             if unique and unique.name not in (kwargs.get('attributes') or {}).values():
+                current_app.logger.warning((unique.name, kwargs.get('attributes'), ci_type.alias))
                 return abort(400, ErrFormat.ad_not_unique_key.format(unique.name))
 
         if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'):
             if current_user.uid != existed.uid:
                 return abort(403, ErrFormat.adt_secret_no_permission)
+        if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('password'):
+            if current_user.uid != existed.uid:
+                return abort(403, ErrFormat.adt_secret_no_permission)
 
         return existed
 
@@ -413,6 +430,8 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
 
         if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'):
             kwargs['extra_option']['secret'] = AESCrypto.encrypt(kwargs['extra_option']['secret'])
+        if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('password'):
+            kwargs['extra_option']['password'] = AESCrypto.encrypt(kwargs['extra_option']['password'])
 
         inst = self._can_update(_id=_id, **kwargs)
         if inst.agent_id != kwargs.get('agent_id') or inst.query_expr != kwargs.get('query_expr'):
@@ -420,6 +439,9 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
                 item.delete(commit=False)
             db.session.commit()
 
+            SystemConfigManager.create_or_update("ad_rules_updated_at",
+                                                 dict(v=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
+
         obj = inst.update(_id=_id, filter_none=False, **kwargs)
 
         return obj
@@ -453,6 +475,10 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
 class AutoDiscoveryCITypeRelationCRUD(DBMixin):
     cls = AutoDiscoveryCITypeRelation
 
+    @classmethod
+    def get_all(cls, type_ids=None):
+        res = cls.cls.get_by(to_dict=False)
+        return [i for i in res if type_ids is None or i.ad_type_id in type_ids]
     @classmethod
     def get_by_type_id(cls, type_id, to_dict=False):
         return cls.cls.get_by(ad_type_id=type_id, to_dict=to_dict)
@@ -692,12 +718,13 @@ class AutoDiscoveryHTTPManager(object):
         categories = (ClOUD_MAP.get(name) or {}) or []
         for item in copy.deepcopy(categories):
             item.pop('map', None)
+            item.pop('collect_key_map', None)
 
         return categories
 
     def get_resources(self, name):
         en_name = None
-        for i in DEFAULT_HTTP:
+        for i in DEFAULT_INNER:
             if i['name'] == name:
                 en_name = i['en']
                 break
@@ -733,6 +760,17 @@ class AutoDiscoverySNMPManager(object):
         return []
 
 
+class AutoDiscoveryComponentsManager(object):
+
+    @staticmethod
+    def get_attributes(name):
+        if os.path.exists(os.path.join(PWD, "templates/{}.json".format(name))):
+            with open(os.path.join(PWD, "templates/{}.json".format(name))) as f:
+                return json.loads(f.read())
+
+        return []
+
+
 class AutoDiscoveryRuleSyncHistoryCRUD(DBMixin):
     cls = AutoDiscoveryRuleSyncHistory
 

cmdb-api/api/lib/cmdb/auto_discovery/const.py

@@ -2,16 +2,28 @@
 
 from api.lib.cmdb.const import AutoDiscoveryType
 
-DEFAULT_HTTP = [
+PRIVILEGED_USERS = ("cmdb_agent", "worker", "admin")
+
+DEFAULT_INNER = [
     dict(name="阿里云", en="aliyun", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
-         option={'icon': {'name': 'caise-aliyun'}}),
+         option={'icon': {'name': 'caise-aliyun'}, "en": "aliyun"}),
     dict(name="腾讯云", en="tencentcloud", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
-         option={'icon': {'name': 'caise-tengxunyun'}}),
+         option={'icon': {'name': 'caise-tengxunyun'}, "en": "tencentcloud"}),
     dict(name="华为云", en="huaweicloud", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
-         option={'icon': {'name': 'caise-huaweiyun'}}),
+         option={'icon': {'name': 'caise-huaweiyun'}, "en": "huaweicloud"}),
     dict(name="AWS", en="aws", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
          option={'icon': {'name': 'caise-aws'}}),
 
+    dict(name="VCenter", en="vcenter", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'cmdb-vcenter'}, "category": "private_cloud", "en": "vcenter"}),
+    dict(name="KVM", en="kvm", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'ops-KVM'}, "category": "private_cloud", "en": "kvm"}),
+
+    dict(name="Nginx", en="nginx", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-nginx'}, "en": "nginx"}),
+    dict(name="Redis", en="redis", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-redis'}, "en": "redis"}),
+
     dict(name="交换机", type=AutoDiscoveryType.SNMP, is_inner=True, is_plugin=False,
          option={'icon': {'name': 'caise-jiaohuanji'}}),
     dict(name="路由器", type=AutoDiscoveryType.SNMP, is_inner=True, is_plugin=False,
@@ -23,47 +35,306 @@ DEFAULT_HTTP = [
 ]
 
 ClOUD_MAP = {
-    "aliyun": [{
-        "category": "计算",
-        "items": ["云服务器 ECS"],
-        "map": {
-            "云服务器 ECS": "templates/aliyun_ecs.json",
+    "aliyun": [
+        {
+            "category": "计算",
+            "items": ["云服务器 ECS", "云服务器 Disk"],
+            "map": {
+                "云服务器 ECS": "templates/aliyun_ecs.json",
+                "云服务器 Disk": "templates/aliyun_ecs_disk2.json",
+            },
+            "collect_key_map": {
+                "云服务器 ECS": "ali.ecs",
+                "云服务器 Disk": "ali.ecs_disk",
+            },
         },
-        "collect_key_map": {
-            "云服务器 ECS": "ali.ecs",
-        }
-    }],
-
-    "tencentcloud": [{
-        "category": "计算",
-        "items": ["云服务器 CVM"],
-        "map": {
-            "云服务器 CVM": "templates/tencent_cvm.json",
+        {
+            "category": "网络与CDN",
+            "items": [
+                "内容分发CDN",
+                "负载均衡SLB",
+                "专有网络VPC",
+                "交换机Switch",
+            ],
+            "map": {
+                "内容分发CDN": "templates/aliyun_cdn.json",
+                "负载均衡SLB": "templates/aliyun_slb.json",
+                "专有网络VPC": "templates/aliyun_vpc.json",
+                "交换机Switch": "templates/aliyun_switch.json",
+            },
+            "collect_key_map": {
+                "内容分发CDN": "ali.cdn",
+                "负载均衡SLB": "ali.slb",
+                "专有网络VPC": "ali.vpc",
+                "交换机Switch": "ali.switch",
+            },
         },
-        "collect_key_map": {
-            "云服务器 CVM": "tencent.cvm",
-        }
-    }],
-
-    "huaweicloud": [{
-        "category": "计算",
-        "items": ["云服务器 ECS"],
-        "map": {
-            "云服务器 ECS": "templates/huaweicloud_ecs.json",
+        {
+            "category": "存储",
+            "items": ["块存储EBS", "对象存储OSS"],
+            "map": {
+                "块存储EBS": "templates/aliyun_ebs.json",
+                "对象存储OSS": "templates/aliyun_oss.json",
+            },
+            "collect_key_map": {
+                "块存储EBS": "ali.ebs",
+                "对象存储OSS": "ali.oss",
+            },
         },
-        "collect_key_map": {
-            "云服务器 ECS": "huawei.ecs",
-        }
-    }],
-
-    "aws": [{
-        "category": "计算",
-        "items": ["云服务器 EC2"],
-        "map": {
-            "云服务器 EC2": "templates/aws_ec2.json",
+        {
+            "category": "数据库",
+            "items": ["云数据库RDS MySQL", "云数据库RDS PostgreSQL", "云数据库 Redis"],
+            "map": {
+                "云数据库RDS MySQL": "templates/aliyun_rds_mysql.json",
+                "云数据库RDS PostgreSQL": "templates/aliyun_rds_postgre.json",
+                "云数据库 Redis": "templates/aliyun_redis.json",
+            },
+            "collect_key_map": {
+                "云数据库RDS MySQL": "ali.rds_mysql",
+                "云数据库RDS PostgreSQL": "ali.rds_postgre",
+                "云数据库 Redis": "ali.redis",
+            },
         },
-        "collect_key_map": {
-            "云服务器 EC2": "aws.ec2",
-        }
-    }],
+    ],
+    "tencentcloud": [
+        {
+            "category": "计算",
+            "items": ["云服务器 CVM"],
+            "map": {
+                "云服务器 CVM": "templates/tencent_cvm.json",
+            },
+            "collect_key_map": {
+                "云服务器 CVM": "tencent.cvm",
+            },
+        },
+        {
+            "category": "CDN与边缘",
+            "items": ["内容分发CDN"],
+            "map": {
+                "内容分发CDN": "templates/tencent_cdn.json",
+            },
+            "collect_key_map": {
+                "内容分发CDN": "tencent.cdn",
+            },
+        },
+        {
+            "category": "网络",
+            "items": ["负载均衡CLB", "私有网络VPC", "子网"],
+            "map": {
+                "负载均衡CLB": "templates/tencent_clb.json",
+                "私有网络VPC": "templates/tencent_vpc.json",
+                "子网": "templates/tencent_subnet.json",
+            },
+            "collect_key_map": {
+                "负载均衡CLB": "tencent.clb",
+                "私有网络VPC": "tencent.vpc",
+                "子网": "tencent.subnet",
+            },
+        },
+        {
+            "category": "存储",
+            "items": ["云硬盘CBS", "对象存储COS"],
+            "map": {
+                "云硬盘CBS": "templates/tencent_cbs.json",
+                "对象存储OSS": "templates/tencent_cos.json",
+            },
+            "collect_key_map": {
+                "云硬盘CBS": "tencent.cbs",
+                "对象存储OSS": "tencent.cos",
+            },
+        },
+        {
+            "category": "数据库",
+            "items": ["云数据库 MySQL", "云数据库 PostgreSQL", "云数据库 Redis"],
+            "map": {
+                "云数据库 MySQL": "templates/tencent_rdb.json",
+                "云数据库 PostgreSQL": "templates/tencent_postgres.json",
+                "云数据库 Redis": "templates/tencent_redis.json",
+            },
+            "collect_key_map": {
+                "云数据库 MySQL": "tencent.rdb",
+                "云数据库 PostgreSQL": "tencent.rds_postgres",
+                "云数据库 Redis": "tencent.redis",
+            },
+        },
+    ],
+    "huaweicloud": [
+        {
+            "category": "计算",
+            "items": ["云服务器 ECS"],
+            "map": {
+                "云服务器 ECS": "templates/huaweicloud_ecs.json",
+            },
+            "collect_key_map": {
+                "云服务器 ECS": "huawei.ecs",
+            },
+        },
+        {
+            "category": "CDN与智能边缘",
+            "items": ["内容分发网络CDN"],
+            "map": {
+                "内容分发网络CDN": "templates/huawei_cdn.json",
+            },
+            "collect_key_map": {
+                "内容分发网络CDN": "huawei.cdn",
+            },
+        },
+        {
+            "category": "网络",
+            "items": ["弹性负载均衡ELB", "虚拟私有云VPC", "子网"],
+            "map": {
+                "弹性负载均衡ELB": "templates/huawei_elb.json",
+                "虚拟私有云VPC": "templates/huawei_vpc.json",
+                "子网": "templates/huawei_subnet.json",
+            },
+            "collect_key_map": {
+                "弹性负载均衡ELB": "huawei.elb",
+                "虚拟私有云VPC": "huawei.vpc",
+                "子网": "huawei.subnet",
+            },
+        },
+        {
+            "category": "存储",
+            "items": ["云硬盘EVS", "对象存储OBS"],
+            "map": {
+                "云硬盘EVS": "templates/huawei_evs.json",
+                "对象存储OBS": "templates/huawei_obs.json",
+            },
+            "collect_key_map": {
+                "云硬盘EVS": "huawei.evs",
+                "对象存储OBS": "huawei.obs",
+            },
+        },
+        {
+            "category": "数据库",
+            "items": ["云数据库RDS MySQL", "云数据库RDS PostgreSQL"],
+            "map": {
+                "云数据库RDS MySQL": "templates/huawei_rds_mysql.json",
+                "云数据库RDSPostgreSQL": "templates/huaweirds_postgre.json",
+            },
+            "collect_key_map": {
+                "云数据库RDS MySQL": "huawei.rds_mysql",
+                "云数据库RDS PostgreSQL": "huawei.rds_postgre",
+            },
+        },
+        {
+            "category": "应用中间件",
+            "items": ["分布式缓存Redis"],
+            "map": {
+                "分布式缓存Redis": "templates/huawei_dcs.json",
+            },
+            "collect_key_map": {
+                "分布式缓存Redis": "huawei.dcs",
+            },
+        },
+    ],
+    "aws": [
+        {
+            "category": "计算",
+            "items": ["云服务器 EC2"],
+            "map": {
+                "云服务器 EC2": "templates/aws_ec2.json",
+            },
+            "collect_key_map": {
+                "云服务器 EC2": "aws.ec2",
+            },
+        },
+        {"category": "网络与CDN", "items": [], "map": {}, "collect_key_map": {}},
+    ],
+    "vcenter": [
+        {
+            "category": "计算",
+            "items": [
+                "主机",
+                "虚拟机",
+                "主机集群"
+            ],
+            "map": {
+                "主机": "templates/vsphere_host.json",
+                "虚拟机": "templates/vsphere_vm.json",
+                "主机集群": "templates/vsphere_cluster.json",
+            },
+            "collect_key_map": {
+                "主机": "vsphere.host",
+                "虚拟机": "vsphere.vm",
+                "主机集群": "vsphere.cluster",
+            },
+        },
+        {
+            "category": "网络",
+            "items": [
+                "网络",
+                "标准交换机",
+                "分布式交换机",
+            ],
+            "map": {
+                "网络": "templates/vsphere_network.json",
+                "标准交换机": "templates/vsphere_standard_switch.json",
+                "分布式交换机": "templates/vsphere_distributed_switch.json",
+            },
+            "collect_key_map": {
+                "网络": "vsphere.network",
+                "标准交换机": "vsphere.standard_switch",
+                "分布式交换机": "vsphere.distributed_switch",
+            },
+        },
+        {
+            "category": "存储",
+            "items": ["数据存储", "数据存储集群"],
+            "map": {
+                "数据存储": "templates/vsphere_datastore.json",
+                "数据存储集群": "templates/vsphere.storage_pod.json",
+            },
+            "collect_key_map": {
+                "数据存储": "vsphere.datastore",
+                "数据存储集群": "vsphere.storage_pod",
+            },
+        },
+        {
+            "category": "其他",
+            "items": ["资源池", "数据中心", "文件夹"],
+            "map": {
+                "资源池": "templates/vsphere_datastore.json",
+                "数据中心": "templates/vsphere_datacenter.json",
+                "文件夹": "templates/vsphere_folder.json",
+            },
+            "collect_key_map": {
+                "资源池": "vsphere.pool",
+                "数据中心": "vsphere.datacenter",
+                "文件夹": "vsphere.folder",
+            },
+        },
+    ],
+    "kvm": [
+        {
+            "category": "计算",
+            "items": ["虚拟机"],
+            "map": {
+                "虚拟机": "templates/kvm_vm.json",
+            },
+            "collect_key_map": {
+                "虚拟机": "kvm.vm",
+            },
+        },
+        {
+            "category": "存储",
+            "items": ["存储"],
+            "map": {
+                "存储": "templates/kvm_storage.json",
+            },
+            "collect_key_map": {
+                "存储": "kvm.storage",
+            },
+        },
+        {
+            "category": "network",
+            "items": ["网络"],
+            "map": {
+                "网络": "templates/kvm_network.json",
+            },
+            "collect_key_map": {
+                "网络": "kvm.network",
+            },
+        },
+    ],
 }