perf(api): auto discovery has been upgraded (#559)

cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py

@@ -1,34 +1,47 @@
 # -*- coding:utf-8 -*-
+import copy
 import datetime
 import json
 import os
 
+from flask import abort
+from flask import current_app
+from flask_login import current_user
+from sqlalchemy import func
+
 from api.extensions import db
 from api.lib.cmdb.auto_discovery.const import ClOUD_MAP
+from api.lib.cmdb.auto_discovery.const import DEFAULT_HTTP
+from api.lib.cmdb.cache import AttributeCache
 from api.lib.cmdb.cache import CITypeAttributeCache
 from api.lib.cmdb.cache import CITypeCache
 from api.lib.cmdb.ci import CIManager
 from api.lib.cmdb.ci import CIRelationManager
 from api.lib.cmdb.ci_type import CITypeGroupManager
 from api.lib.cmdb.const import AutoDiscoveryType
+from api.lib.cmdb.const import CMDB_QUEUE
 from api.lib.cmdb.const import PermEnum
 from api.lib.cmdb.const import ResourceTypeEnum
 from api.lib.cmdb.resp_format import ErrFormat
 from api.lib.cmdb.search import SearchError
-from api.lib.cmdb.search.ci import search
+from api.lib.cmdb.search.ci import search as ci_search
+from api.lib.common_setting.role_perm_base import CMDBApp
 from api.lib.mixin import DBMixin
+from api.lib.perm.acl.acl import ACLManager
 from api.lib.perm.acl.acl import is_app_admin
 from api.lib.perm.acl.acl import validate_permission
 from api.lib.utils import AESCrypto
 from api.models.cmdb import AutoDiscoveryCI
 from api.models.cmdb import AutoDiscoveryCIType
+from api.models.cmdb import AutoDiscoveryCITypeRelation
+from api.models.cmdb import AutoDiscoveryCounter
+from api.models.cmdb import AutoDiscoveryExecHistory
 from api.models.cmdb import AutoDiscoveryRule
-from flask import abort
-from flask import current_app
-from flask_login import current_user
-from sqlalchemy import func
+from api.models.cmdb import AutoDiscoveryRuleSyncHistory
+from api.tasks.cmdb import write_ad_rule_sync_history
 
 PWD = os.path.abspath(os.path.dirname(__file__))
+app_cli = CMDBApp()
 
 
 def parse_plugin_script(script):
@@ -100,6 +113,14 @@ class AutoDiscoveryRuleCRUD(DBMixin):
         self.cls.get_by(name=kwargs['name']) and abort(400, ErrFormat.adr_duplicate.format(kwargs['name']))
         if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
             kwargs = check_plugin_script(**kwargs)
+            acl = ACLManager(app_cli.app_name)
+            if not acl.has_permission(app_cli.op.Auto_Discovery,
+                                      app_cli.resource_type_name,
+                                      app_cli.op.create_plugin) and not is_app_admin(app_cli.app_name):
+                return abort(403, ErrFormat.no_permission.format(
+                    app_cli.op.Auto_Discovery, app_cli.op.create_plugin))
+
+        kwargs['owner'] = current_user.uid
 
         return kwargs
 
@@ -115,6 +136,14 @@ class AutoDiscoveryRuleCRUD(DBMixin):
             if other and other.id != existed.id:
                 return abort(400, ErrFormat.adr_duplicate.format(kwargs['name']))
 
+        if existed.is_plugin:
+            acl = ACLManager(app_cli.app_name)
+            if not acl.has_permission(app_cli.op.Auto_Discovery,
+                                      app_cli.resource_type_name,
+                                      app_cli.op.update_plugin) and not is_app_admin(app_cli.app_name):
+                return abort(403, ErrFormat.no_permission.format(
+                    app_cli.op.Auto_Discovery, app_cli.op.update_plugin))
+
         return existed
 
     def update(self, _id, **kwargs):
@@ -122,13 +151,27 @@ class AutoDiscoveryRuleCRUD(DBMixin):
         if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
             kwargs = check_plugin_script(**kwargs)
 
+            for item in AutoDiscoveryCIType.get_by(adr_id=_id, to_dict=False):
+                item.update(updated_at=datetime.datetime.now())
+
         return super(AutoDiscoveryRuleCRUD, self).update(_id, filter_none=False, **kwargs)
 
     def _can_delete(self, **kwargs):
         if AutoDiscoveryCIType.get_by(adr_id=kwargs['_id'], first=True):
             return abort(400, ErrFormat.adr_referenced)
 
-        return self._can_update(**kwargs)
+        existed = self.cls.get_by_id(kwargs['_id']) or abort(
+            404, ErrFormat.adr_not_found.format("id={}".format(kwargs['_id'])))
+
+        if existed.is_plugin:
+            acl = ACLManager(app_cli.app_name)
+            if not acl.has_permission(app_cli.op.Auto_Discovery,
+                                      app_cli.resource_type_name,
+                                      app_cli.op.delete_plugin) and not is_app_admin(app_cli.app_name):
+                return abort(403, ErrFormat.no_permission.format(
+                    app_cli.op.Auto_Discovery, app_cli.op.delete_plugin))
+
+        return existed
 
 
 class AutoDiscoveryCITypeCRUD(DBMixin):
@@ -147,14 +190,34 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
         return cls.cls.get_by(type_id=type_id, to_dict=False)
 
     @classmethod
-    def get(cls, ci_id, oneagent_id, last_update_at=None):
+    def get_ad_attributes(cls, type_id):
+        result = []
+        adts = cls.get_by_type_id(type_id)
+        for adt in adts:
+            adr = AutoDiscoveryRuleCRUD.get_by_id(adt.adr_id)
+            if not adr:
+                continue
+            if adr.type == "http":
+                for i in DEFAULT_HTTP:
+                    if adr.name == i['name']:
+                        attrs = AutoDiscoveryHTTPManager.get_attributes(
+                            i['en'], (adt.extra_option or {}).get('category')) or []
+                        result.extend([i.get('name') for i in attrs])
+                        break
+            elif adr.type == "snmp":
+                attributes = AutoDiscoverySNMPManager.get_attributes()
+                result.extend([i.get('name') for i in (attributes or [])])
+            else:
+                result.extend([i.get('name') for i in (adr.attributes or [])])
+
+        return sorted(list(set(result)))
+
+    @classmethod
+    def get(cls, ci_id, oneagent_id, oneagent_name, last_update_at=None):
         result = []
         rules = cls.cls.get_by(to_dict=True)
 
         for rule in rules:
-            if rule.get('relation'):
-                continue
-
             if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('secret'):
                 if not (current_user.username == "cmdb_agent" or current_user.uid == rule['uid']):
                     rule['extra_option'].pop('secret', None)
@@ -165,7 +228,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
                 result.append(rule)
             elif rule['query_expr']:
                 query = rule['query_expr'].lstrip('q').lstrip('=')
-                s = search(query, fl=['_id'], count=1000000)
+                s = ci_search(query, fl=['_id'], count=1000000)
                 try:
                     response, _, _, _, _, _ = s.search()
                 except SearchError as e:
@@ -182,9 +245,6 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
                 if adr.type in (AutoDiscoveryType.SNMP, AutoDiscoveryType.HTTP):
                     continue
 
-                if not rule['updated_at']:
-                    continue
-
                 result.append(rule)
 
         new_last_update_at = ""
@@ -195,6 +255,9 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
             if new_last_update_at < __last_update_at:
                 new_last_update_at = __last_update_at
 
+        write_ad_rule_sync_history.apply_async(args=(result, oneagent_id, oneagent_name, datetime.datetime.now()),
+                                               queue=CMDB_QUEUE)
+
         if not last_update_at or new_last_update_at > last_update_at:
             return result, new_last_update_at
         else:
@@ -213,7 +276,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
             agent_id = agent_id.strip()
             q = "op_duty:{0},-rd_duty:{0},oneagent_id:{1}"
 
-            s = search(q.format(current_user.username, agent_id.strip()))
+            s = ci_search(q.format(current_user.username, agent_id.strip()))
             try:
                 response, _, _, _, _, _ = s.search()
                 if response:
@@ -222,7 +285,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
                 current_app.logger.warning(e)
                 return abort(400, str(e))
 
-            s = search(q.format(current_user.nickname, agent_id.strip()))
+            s = ci_search(q.format(current_user.nickname, agent_id.strip()))
             try:
                 response, _, _, _, _, _ = s.search()
                 if response:
@@ -236,7 +299,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
             if query_expr.startswith('q='):
                 query_expr = query_expr[2:]
 
-            s = search(query_expr, count=1000000)
+            s = ci_search(query_expr, count=1000000)
             try:
                 response, _, _, _, _, _ = s.search()
                 for i in response:
@@ -254,13 +317,21 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
     def _can_add(**kwargs):
 
         if kwargs.get('adr_id'):
-            AutoDiscoveryRule.get_by_id(kwargs['adr_id']) or abort(
+            adr = AutoDiscoveryRule.get_by_id(kwargs['adr_id']) or abort(
                 404, ErrFormat.adr_not_found.format("id={}".format(kwargs['adr_id'])))
-            # if not adr.is_plugin:
-            #     other = self.cls.get_by(adr_id=adr.id, first=True, to_dict=False)
-            #     if other:
-            #         ci_type = CITypeCache.get(other.type_id)
-            #         return abort(400, ErrFormat.adr_default_ref_once.format(ci_type.alias))
+            if adr.type == "http":
+                kwargs.setdefault('extra_option', dict)
+                en_name = None
+                for i in DEFAULT_HTTP:
+                    if i['name'] == adr.name:
+                        en_name = i['en']
+                        break
+                if en_name and kwargs['extra_option'].get('category'):
+                    for item in ClOUD_MAP[en_name]:
+                        if item["collect_key_map"].get(kwargs['extra_option']['category']):
+                            kwargs["extra_option"]["collect_key"] = item["collect_key_map"][
+                                kwargs['extra_option']['category']]
+                            break
 
         if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
             kwargs = check_plugin_script(**kwargs)
@@ -268,6 +339,11 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
         if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'):
             kwargs['extra_option']['secret'] = AESCrypto.encrypt(kwargs['extra_option']['secret'])
 
+        ci_type = CITypeCache.get(kwargs['type_id'])
+        unique = AttributeCache.get(ci_type.unique_id)
+        if unique and unique.name not in (kwargs.get('attributes') or {}).values():
+            return abort(400, ErrFormat.ad_not_unique_key.format(unique.name))
+
         kwargs['uid'] = current_user.uid
 
         return kwargs
@@ -276,7 +352,29 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
         existed = self.cls.get_by_id(kwargs['_id']) or abort(
             404, ErrFormat.ad_not_found.format("id={}".format(kwargs['_id'])))
 
-        self.__valid_exec_target(kwargs.get('agent_id'), kwargs.get('query_expr'))
+        adr = AutoDiscoveryRule.get_by_id(existed.adr_id) or abort(
+            404, ErrFormat.adr_not_found.format("id={}".format(existed.adr_id)))
+        if adr.type == "http":
+            kwargs.setdefault('extra_option', dict)
+            en_name = None
+            for i in DEFAULT_HTTP:
+                if i['name'] == adr.name:
+                    en_name = i['en']
+                    break
+            if en_name and kwargs['extra_option'].get('category'):
+                for item in ClOUD_MAP[en_name]:
+                    if item["collect_key_map"].get(kwargs['extra_option']['category']):
+                        kwargs["extra_option"]["collect_key"] = item["collect_key_map"][
+                            kwargs['extra_option']['category']]
+                        break
+
+        if 'attributes' in kwargs:
+            self.__valid_exec_target(kwargs.get('agent_id'), kwargs.get('query_expr'))
+
+            ci_type = CITypeCache.get(existed.type_id)
+            unique = AttributeCache.get(ci_type.unique_id)
+            if unique and unique.name not in (kwargs.get('attributes') or {}).values():
+                return abort(400, ErrFormat.ad_not_unique_key.format(unique.name))
 
         if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'):
             if current_user.uid != existed.uid:
@@ -292,7 +390,15 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
         if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'):
             kwargs['extra_option']['secret'] = AESCrypto.encrypt(kwargs['extra_option']['secret'])
 
-        return super(AutoDiscoveryCITypeCRUD, self).update(_id, filter_none=False, **kwargs)
+        inst = self._can_update(_id=_id, **kwargs)
+        if inst.agent_id != kwargs.get('agent_id') or inst.query_expr != kwargs.get('query_expr'):
+            for item in AutoDiscoveryRuleSyncHistory.get_by(adt_id=inst.id, to_dict=False):
+                item.delete(commit=False)
+            db.session.commit()
+
+        obj = inst.update(_id=_id, filter_none=False, **kwargs)
+
+        return obj
 
     def _can_delete(self, **kwargs):
         if AutoDiscoveryCICRUD.get_by_adt_id(kwargs['_id']):
@@ -303,6 +409,56 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
 
         return existed
 
+    def delete(self, _id):
+        inst = self._can_delete(_id=_id)
+
+        inst.soft_delete()
+
+        for item in AutoDiscoveryRuleSyncHistory.get_by(adt_id=inst.id, to_dict=False):
+            item.delete(commit=False)
+        db.session.commit()
+
+        attributes = self.get_ad_attributes(inst.type_id)
+        for item in AutoDiscoveryCITypeRelationCRUD.get_by_type_id(inst.type_id):
+            if item.ad_key not in attributes:
+                item.soft_delete()
+
+        return inst
+
+
+class AutoDiscoveryCITypeRelationCRUD(DBMixin):
+    cls = AutoDiscoveryCITypeRelation
+
+    @classmethod
+    def get_by_type_id(cls, type_id, to_dict=False):
+        return cls.cls.get_by(ad_type_id=type_id, to_dict=to_dict)
+
+    def upsert(self, ad_type_id, relations):
+        existed = self.cls.get_by(ad_type_id=ad_type_id, to_dict=False)
+        existed = {(i.ad_key, i.peer_type_id, i.peer_attr_id): i for i in existed}
+
+        new = []
+        for r in relations:
+            k = (r.get('ad_key'), r.get('peer_type_id'), r.get('peer_attr_id'))
+            if len(list(filter(lambda x: x, k))) == 3 and k not in existed:
+                self.cls.create(ad_type_id=ad_type_id, **r)
+
+            new.append(k)
+
+        for deleted in set(existed.keys()) - set(new):
+            existed[deleted].soft_delete()
+
+        return self.get_by_type_id(ad_type_id, to_dict=True)
+
+    def _can_add(self, **kwargs):
+        pass
+
+    def _can_update(self, **kwargs):
+        pass
+
+    def _can_delete(self, **kwargs):
+        pass
+
 
 class AutoDiscoveryCICRUD(DBMixin):
     cls = AutoDiscoveryCI
@@ -391,16 +547,24 @@ class AutoDiscoveryCICRUD(DBMixin):
         changed = False
         if existed is not None:
             if existed.instance != kwargs['instance']:
+                instance = copy.deepcopy(existed.instance) or {}
+                instance.update(kwargs['instance'])
+                kwargs['instance'] = instance
                 existed.update(filter_none=False, **kwargs)
+                AutoDiscoveryExecHistoryCRUD().add(type_id=adt.type_id,
+                                                   stdout="update resource: {}".format(kwargs.get('unique_value')))
                 changed = True
         else:
             existed = self.cls.create(**kwargs)
+            AutoDiscoveryExecHistoryCRUD().add(type_id=adt.type_id,
+                                               stdout="add resource: {}".format(kwargs.get('unique_value')))
             changed = True
 
         if adt.auto_accept and changed:
             try:
                 self.accept(existed)
             except Exception as e:
+                current_app.logger.error(e)
                 return abort(400, str(e))
         elif changed:
             existed.update(is_accept=False, accept_time=None, accept_by=None, filter_none=False)
@@ -420,6 +584,13 @@ class AutoDiscoveryCICRUD(DBMixin):
 
         inst.delete()
 
+        adt = AutoDiscoveryCIType.get_by_id(inst.adt_id)
+        if adt:
+            adt.update(updated_at=datetime.datetime.now())
+
+        AutoDiscoveryExecHistoryCRUD().add(type_id=inst.type_id,
+                                           stdout="delete resource: {}".format(inst.unique_value))
+
         self._after_delete(inst)
 
         return inst
@@ -435,6 +606,13 @@ class AutoDiscoveryCICRUD(DBMixin):
             not is_app_admin("cmdb") and validate_permission(ci_type.name, ResourceTypeEnum.CI, PermEnum.DELETE, "cmdb")
 
         existed.delete()
+
+        adt = AutoDiscoveryCIType.get_by_id(existed.adt_id)
+        if adt:
+            adt.update(updated_at=datetime.datetime.now())
+
+        AutoDiscoveryExecHistoryCRUD().add(type_id=type_id,
+                                           stdout="delete resource: {}".format(unique_value))
         # TODO: delete ci
 
     @classmethod
@@ -447,32 +625,34 @@ class AutoDiscoveryCICRUD(DBMixin):
         ci_id = None
         if adt.attributes:
             ci_dict = {adt.attributes[k]: v for k, v in adc.instance.items() if k in adt.attributes}
-            ci_id = CIManager.add(adc.type_id, is_auto_discovery=True, **ci_dict)
+            ci_id = CIManager.add(adc.type_id, is_auto_discovery=True, _is_admin=True, **ci_dict)
+            AutoDiscoveryExecHistoryCRUD().add(type_id=adt.type_id,
+                                               stdout="accept resource: {}".format(adc.unique_value))
 
-        relation_adts = AutoDiscoveryCIType.get_by(type_id=adt.type_id, adr_id=None, to_dict=False)
-        for r_adt in relation_adts:
-            if not r_adt.relation or ci_id is None:
+        relation_ads = AutoDiscoveryCITypeRelation.get_by(ad_type_id=adt.type_id, to_dict=False)
+        for r_adt in relation_ads:
+            ad_key = r_adt.ad_key
+            if not adc.instance.get(ad_key):
                 continue
-            for ad_key in r_adt.relation:
-                if not adc.instance.get(ad_key):
-                    continue
-                cmdb_key = r_adt.relation[ad_key]
-                query = "_type:{},{}:{}".format(cmdb_key.get('type_name'), cmdb_key.get('attr_name'),
-                                                adc.instance.get(ad_key))
-                s = search(query)
+
+            ad_key_values = [adc.instance.get(ad_key)] if not isinstance(
+                adc.instance.get(ad_key), list) else adc.instance.get(ad_key)
+            for ad_key_value in ad_key_values:
+                query = "_type:{},{}:{}".format(r_adt.peer_type_id, r_adt.peer_attr_id, ad_key_value)
+                s = ci_search(query, use_ci_filter=False, count=1000000)
                 try:
                     response, _, _, _, _, _ = s.search()
                 except SearchError as e:
                     current_app.logger.warning(e)
                     return abort(400, str(e))
 
-                relation_ci_id = response and response[0]['_id']
-                if relation_ci_id:
+                for relation_ci in response:
+                    relation_ci_id = relation_ci['_id']
                     try:
-                        CIRelationManager.add(ci_id, relation_ci_id)
+                        CIRelationManager.add(ci_id, relation_ci_id, valid=False)
                     except:
                         try:
-                            CIRelationManager.add(relation_ci_id, ci_id)
+                            CIRelationManager.add(relation_ci_id, ci_id, valid=False)
                         except:
                             pass
 
@@ -485,14 +665,35 @@ class AutoDiscoveryCICRUD(DBMixin):
 class AutoDiscoveryHTTPManager(object):
     @staticmethod
     def get_categories(name):
-        return (ClOUD_MAP.get(name) or {}).get('categories') or []
+        categories = (ClOUD_MAP.get(name) or {}) or []
+        for item in copy.deepcopy(categories):
+            item.pop('map', None)
+
+        return categories
+
+    def get_resources(self, name):
+        en_name = None
+        for i in DEFAULT_HTTP:
+            if i['name'] == name:
+                en_name = i['en']
+                break
+
+        if en_name:
+            categories = self.get_categories(en_name)
+
+            return [j for i in categories for j in i['items']]
+
+        return []
 
     @staticmethod
-    def get_attributes(name, category):
-        tpt = ((ClOUD_MAP.get(name) or {}).get('map') or {}).get(category)
-        if tpt and os.path.exists(os.path.join(PWD, tpt)):
-            with open(os.path.join(PWD, tpt)) as f:
-                return json.loads(f.read())
+    def get_attributes(provider, resource):
+        for item in (ClOUD_MAP.get(provider) or {}):
+            for _resource in (item.get('map') or {}):
+                if _resource == resource:
+                    tpt = item['map'][_resource]
+                    if tpt and os.path.exists(os.path.join(PWD, tpt)):
+                        with open(os.path.join(PWD, tpt)) as f:
+                            return json.loads(f.read())
 
         return []
 
@@ -506,3 +707,62 @@ class AutoDiscoverySNMPManager(object):
                 return json.loads(f.read())
 
         return []
+
+
+class AutoDiscoveryRuleSyncHistoryCRUD(DBMixin):
+    cls = AutoDiscoveryRuleSyncHistory
+
+    def _can_add(self, **kwargs):
+        pass
+
+    def _can_update(self, **kwargs):
+        pass
+
+    def _can_delete(self, **kwargs):
+        pass
+
+    def upsert(self, **kwargs):
+        existed = self.cls.get_by(adt_id=kwargs.get('adt_id'),
+                                  oneagent_id=kwargs.get('oneagent_id'),
+                                  oneagent_name=kwargs.get('oneagent_name'),
+                                  first=True,
+                                  to_dict=False)
+
+        if existed is not None:
+            existed.update(**kwargs)
+        else:
+            self.cls.create(**kwargs)
+
+
+class AutoDiscoveryExecHistoryCRUD(DBMixin):
+    cls = AutoDiscoveryExecHistory
+
+    def _can_add(self, **kwargs):
+        pass
+
+    def _can_update(self, **kwargs):
+        pass
+
+    def _can_delete(self, **kwargs):
+        pass
+
+
+class AutoDiscoveryCounterCRUD(DBMixin):
+    cls = AutoDiscoveryCounter
+
+    def get(self, type_id):
+        res = self.cls.get_by(type_id=type_id, first=True, to_dict=True)
+        if res is None:
+            return dict(rule_count=0, exec_target_count=0, instance_count=0, accept_count=0,
+                        this_month_count=0, this_week_count=0, last_month_count=0, last_week_count=0)
+
+        return res
+
+    def _can_add(self, **kwargs):
+        pass
+
+    def _can_update(self, **kwargs):
+        pass
+
+    def _can_delete(self, **kwargs):
+        pass

cmdb-api/api/lib/cmdb/auto_discovery/const.py

@@ -3,13 +3,13 @@
 from api.lib.cmdb.const import AutoDiscoveryType
 
 DEFAULT_HTTP = [
-    dict(name="阿里云", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
+    dict(name="阿里云", en="aliyun", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
          option={'icon': {'name': 'caise-aliyun'}}),
-    dict(name="腾讯云", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
+    dict(name="腾讯云", en="tencentcloud", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
          option={'icon': {'name': 'caise-tengxunyun'}}),
-    dict(name="华为云", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
+    dict(name="华为云", en="huaweicloud", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
          option={'icon': {'name': 'caise-huaweiyun'}}),
-    dict(name="AWS", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
+    dict(name="AWS", en="aws", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
          option={'icon': {'name': 'caise-aws'}}),
 
     dict(name="交换机", type=AutoDiscoveryType.SNMP, is_inner=True, is_plugin=False,
@@ -23,31 +23,47 @@ DEFAULT_HTTP = [
 ]
 
 ClOUD_MAP = {
-    "aliyun": {
-        "categories": ["云服务器 ECS"],
+    "aliyun": [{
+        "category": "计算",
+        "items": ["云服务器 ECS"],
         "map": {
             "云服务器 ECS": "templates/aliyun_ecs.json",
+        },
+        "collect_key_map": {
+            "云服务器 ECS": "ali.ecs",
         }
-    },
+    }],
 
-    "tencentcloud": {
-        "categories": ["云服务器 CVM"],
+    "tencentcloud": [{
+        "category": "计算",
+        "items": ["云服务器 CVM"],
         "map": {
             "云服务器 CVM": "templates/tencent_cvm.json",
+        },
+        "collect_key_map": {
+            "云服务器 CVM": "tencent.cvm",
         }
-    },
+    }],
 
-    "huaweicloud": {
-        "categories": ["云服务器 ECS"],
+    "huaweicloud": [{
+        "category": "计算",
+        "items": ["云服务器 ECS"],
         "map": {
             "云服务器 ECS": "templates/huaweicloud_ecs.json",
+        },
+        "collect_key_map": {
+            "云服务器 ECS": "huawei.ecs",
         }
-    },
+    }],
 
-    "aws": {
-        "categories": ["云服务器 EC2"],
+    "aws": [{
+        "category": "计算",
+        "items": ["云服务器 EC2"],
         "map": {
             "云服务器 EC2": "templates/aws_ec2.json",
+        },
+        "collect_key_map": {
+            "云服务器 EC2": "aws.ec2",
         }
-    },
+    }],
 }

cmdb-api/api/lib/cmdb/cache.py

@@ -2,12 +2,19 @@
 
 from __future__ import unicode_literals
 
+import datetime
+
 from flask import current_app
 
 from api.extensions import cache
 from api.extensions import db
 from api.lib.cmdb.custom_dashboard import CustomDashboardManager
-from api.models.cmdb import Attribute
+from api.models.cmdb import Attribute, AutoDiscoveryExecHistory
+from api.models.cmdb import AutoDiscoveryCI
+from api.models.cmdb import AutoDiscoveryCIType
+from api.models.cmdb import AutoDiscoveryCITypeRelation
+from api.models.cmdb import AutoDiscoveryCounter
+from api.models.cmdb import AutoDiscoveryRuleSyncHistory
 from api.models.cmdb import CI
 from api.models.cmdb import CIType
 from api.models.cmdb import CITypeAttribute
@@ -448,7 +455,67 @@ class CMDBCounterCache(object):
 
         cache.set(cls.KEY2, result, timeout=0)
 
-        return result
+        res = db.session.query(AutoDiscoveryCI.created_at,
+                               AutoDiscoveryCI.updated_at,
+                               AutoDiscoveryCI.adt_id,
+                               AutoDiscoveryCI.type_id,
+                               AutoDiscoveryCI.is_accept).filter(AutoDiscoveryCI.deleted.is_(False))
+
+        today = datetime.datetime.today()
+        this_month = datetime.datetime(today.year, today.month, 1)
+        last_month = this_month - datetime.timedelta(days=1)
+        last_month = datetime.datetime(last_month.year, last_month.month, 1)
+        this_week = today - datetime.timedelta(days=datetime.date.weekday(today))
+        this_week = datetime.datetime(this_week.year, this_week.month, this_week.day)
+        last_week = this_week - datetime.timedelta(days=7)
+        last_week = datetime.datetime(last_week.year, last_week.month, last_week.day)
+        result = dict()
+        for i in res:
+            if i.type_id not in result:
+                result[i.type_id] = dict(instance_count=0, accept_count=0,
+                                         this_month_count=0, this_week_count=0, last_month_count=0, last_week_count=0)
+
+                adts = AutoDiscoveryCIType.get_by(type_id=i.type_id, to_dict=False)
+                result[i.type_id]['rule_count'] = len(adts) + AutoDiscoveryCITypeRelation.get_by(
+                    ad_type_id=i.type_id, only_query=True).count()
+                result[i.type_id]['exec_target_count'] = len(
+                    set([i.oneagent_id for adt in adts for i in db.session.query(
+                        AutoDiscoveryRuleSyncHistory.oneagent_id).filter(
+                        AutoDiscoveryRuleSyncHistory.adt_id == adt.id)]))
+
+            result[i.type_id]['instance_count'] += 1
+            if i.is_accept:
+                result[i.type_id]['accept_count'] += 1
+
+            if last_month <= i.created_at < this_month:
+                result[i.type_id]['last_month_count'] += 1
+            elif i.created_at >= this_month:
+                result[i.type_id]['this_month_count'] += 1
+
+            if last_week <= i.created_at < this_week:
+                result[i.type_id]['last_week_count'] += 1
+            elif i.created_at >= this_week:
+                result[i.type_id]['this_week_count'] += 1
+
+        for type_id in result:
+            existed = AutoDiscoveryCounter.get_by(type_id=type_id, first=True, to_dict=False)
+            if existed is None:
+                AutoDiscoveryCounter.create(type_id=type_id, **result[type_id])
+            else:
+                existed.update(**result[type_id])
+
+        for i in AutoDiscoveryCounter.get_by(to_dict=False):
+            if i.type_id not in result:
+                i.delete()
+
+    @classmethod
+    def clear_ad_exec_history(cls):
+        ci_types = CIType.get_by(to_dict=False)
+        for ci_type in ci_types:
+            for i in AutoDiscoveryExecHistory.get_by(type_id=ci_type.id, only_query=True).order_by(
+                    AutoDiscoveryExecHistory.id.desc()).offset(50000):
+                i.delete(commit=False)
+            db.session.commit()
 
     @classmethod
     def get_adc_counter(cls):

cmdb-api/api/lib/cmdb/ci.py

@@ -223,7 +223,7 @@ class CIManager(object):
     def ci_is_exist(unique_key, unique_value, type_id):
         """
 
-        :param unique_key: is a attribute
+        :param unique_key: is an attribute
         :param unique_value:
         :param type_id:
         :return:
@@ -432,7 +432,7 @@ class CIManager(object):
             for attr_id in password_dict:
                 record_id = cls.save_password(ci.id, attr_id, password_dict[attr_id], record_id, ci_type.id)
 
-        if record_id or has_dynamic:  # has change
+        if record_id or has_dynamic:  # has changed
             ci_cache.apply_async(args=(ci.id, operate_type, record_id), queue=CMDB_QUEUE)
 
         if ref_ci_dict:  # add relations
@@ -504,7 +504,7 @@ class CIManager(object):
             for attr_id in password_dict:
                 record_id = self.save_password(ci.id, attr_id, password_dict[attr_id], record_id, ci.type_id)
 
-        if record_id or has_dynamic:  # has change
+        if record_id or has_dynamic:  # has changed
             if not __sync:
                 ci_cache.apply_async(args=(ci_id, OperateType.UPDATE, record_id), queue=CMDB_QUEUE)
             else:
@@ -737,7 +737,7 @@ class CIManager(object):
                        fields=None, value_tables=None, unique_required=False, excludes=None):
         """
 
-        :param ci_ids: list of CI instance ID, eg. ['1', '2']
+        :param ci_ids: list of CI instance ID, e.g. ['1', '2']
         :param ret_key: name, id or alias
         :param fields:
         :param value_tables:
@@ -1296,7 +1296,7 @@ class CIRelationManager(object):
                     relations = _relations
                 else:
                     relations &= _relations
-            for parent_ci_id, child_ci_id in relations:
+            for parent_ci_id, child_ci_id in (relations or []):
                 CIRelationManager.add(parent_ci_id, child_ci_id, valid=False)
 
         parent_items = CITypeRelation.get_by(child_id=type_id, only_query=True).filter(
@@ -1316,7 +1316,7 @@ class CIRelationManager(object):
                     relations = _relations
                 else:
                     relations &= _relations
-            for parent_ci_id, child_ci_id in relations:
+            for parent_ci_id, child_ci_id in (relations or []):
                 CIRelationManager.add(parent_ci_id, child_ci_id, valid=False)
 
     @classmethod

cmdb-api/api/lib/cmdb/ci_type.py

@@ -229,6 +229,9 @@ class CITypeManager(object):
         if CI.get_by(type_id=type_id, first=True, to_dict=False) is not None:
             return abort(400, ErrFormat.ci_exists_and_cannot_delete_type)
 
+        if CITypeInheritance.get_by(parent_id=type_id, first=True):
+            return abort(400, ErrFormat.ci_type_inheritance_cannot_delete)
+
         relation_views = PreferenceRelationView.get_by(to_dict=False)
         for rv in relation_views:
             for item in (rv.cr_ids or []):
@@ -253,21 +256,21 @@ class CITypeManager(object):
             item.delete(commit=False)
 
         for item in AutoDiscoveryCITypeRelation.get_by(ad_type_id=type_id, to_dict=False):
-            item.delete(commit=False)
+            item.soft_delete(commit=False)
 
         for item in AutoDiscoveryCITypeRelation.get_by(peer_type_id=type_id, to_dict=False):
-            item.delete(commit=False)
+            item.soft_delete(commit=False)
 
         for item in CITypeInheritance.get_by(parent_id=type_id, to_dict=False):
-            item.delete(commit=False)
+            item.soft_delete(commit=False)
 
         for item in CITypeInheritance.get_by(child_id=type_id, to_dict=False):
-            item.delete(commit=False)
+            item.soft_delete(commit=False)
 
         try:
             from api.models.cmdb import CITypeReconciliation
             for item in CITypeReconciliation.get_by(type_id=type_id, to_dict=False):
-                item.delete(commit=False)
+                item.soft_delete(commit=False)
         except Exception:
             pass
 

cmdb-api/api/lib/cmdb/const.py

@@ -55,9 +55,9 @@ class CITypeOperateType(BaseEnum):
     DELETE_UNIQUE_CONSTRAINT = "11"  # 删除联合唯一
     ADD_RELATION = "12"  # 新增关系
     DELETE_RELATION = "13"  # 删除关系
-    ADD_RECONCILIATION = "14"  # 删除关系
-    UPDATE_RECONCILIATION = "15"  # 删除关系
-    DELETE_RECONCILIATION = "16"  # 删除关系
+    ADD_RECONCILIATION = "14"  # 新增数据合规
+    UPDATE_RECONCILIATION = "15"  # 修改数据合规
+    DELETE_RECONCILIATION = "16"  # 删除数据合规
 
 
 class RetKey(BaseEnum):

cmdb-api/api/lib/cmdb/history.py

@@ -13,6 +13,7 @@ from api.lib.cmdb.const import OperateType
 from api.lib.cmdb.perms import CIFilterPermsCRUD
 from api.lib.cmdb.resp_format import ErrFormat
 from api.lib.perm.acl.cache import UserCache
+from api.models.cmdb import CI
 from api.models.cmdb import Attribute
 from api.models.cmdb import AttributeHistory
 from api.models.cmdb import CIRelationHistory
@@ -306,7 +307,7 @@ class CITriggerHistoryManager(object):
     def get(page, page_size, type_id=None, trigger_id=None, operate_type=None):
         query = CITriggerHistory.get_by(only_query=True)
         if type_id:
-            query = query.filter(CITriggerHistory.type_id == type_id)
+            query = query.join(CI, CI.id == CITriggerHistory.ci_id).filter(CI.type_id == type_id)
 
         if trigger_id:
             query = query.filter(CITriggerHistory.trigger_id == trigger_id)

cmdb-api/api/lib/cmdb/resp_format.py

@@ -62,6 +62,7 @@ class ErrFormat(CommonErrFormat):
         "The model cannot be deleted because the CI already exists")  # 因为CI已经存在，不能删除模型
     ci_exists_and_cannot_delete_inheritance = _l(
         "The inheritance cannot be deleted because the CI already exists")  # 因为CI已经存在，不能删除继承关系
+    ci_type_inheritance_cannot_delete = _l("The model is inherited and cannot be deleted")  # 该模型被继承, 不能删除
 
     # 因为关系视图 {} 引用了该模型，不能删除模型
     ci_relation_view_exists_and_cannot_delete_type = _l(

cmdb-api/api/lib/cmdb/topology.py

@@ -16,7 +16,7 @@ from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
 from api.lib.cmdb.const import ResourceTypeEnum
 from api.lib.cmdb.resp_format import ErrFormat
 from api.lib.cmdb.search import SearchError
-from api.lib.cmdb.search.ci import search
+from api.lib.cmdb.search.ci import search as ci_search
 from api.lib.perm.acl.acl import ACLManager
 from api.lib.perm.acl.acl import is_app_admin
 from api.models.cmdb import TopologyView
@@ -178,7 +178,7 @@ class TopologyViewManager(object):
 
         q = (central_node_instances[2:] if central_node_instances.startswith('q=') else
              central_node_instances)
-        s = search(q, fl=['_id', show_key.name], use_id_filter=False, use_ci_filter=False, count=1000000)
+        s = ci_search(q, fl=['_id', show_key.name], use_id_filter=False, use_ci_filter=False, count=1000000)
         try:
             response, _, _, _, _, _ = s.search()
         except SearchError as e:
@@ -238,7 +238,7 @@ class TopologyViewManager(object):
                     type2show[type_id] = attr.name
 
         if id2node:
-            s = search("_id:({})".format(';'.join(id2node.keys())), fl=list(fl),
+            s = ci_search("_id:({})".format(';'.join(id2node.keys())), fl=list(fl),
                        use_id_filter=False, use_ci_filter=False, count=1000000)
             try:
                 response, _, _, _, _, _ = s.search()

cmdb-api/api/lib/mixin.py

@@ -8,6 +8,8 @@ from api.extensions import db
 from api.lib.utils import get_page
 from api.lib.utils import get_page_size
 
+__author__ = 'pycook'
+
 
 class DBMixin(object):
     cls = None
@@ -17,13 +19,18 @@ class DBMixin(object):
         page = get_page(page)
         page_size = get_page_size(page_size)
         if fl is None:
-            query = db.session.query(cls.cls).filter(cls.cls.deleted.is_(False))
+            query = db.session.query(cls.cls)
         else:
-            query = db.session.query(*[getattr(cls.cls, i) for i in fl]).filter(cls.cls.deleted.is_(False))
+            query = db.session.query(*[getattr(cls.cls, i) for i in fl])
 
         _query = None
         if count_query:
-            _query = db.session.query(func.count(cls.cls.id)).filter(cls.cls.deleted.is_(False))
+            _query = db.session.query(func.count(cls.cls.id))
+
+        if hasattr(cls.cls, 'deleted'):
+            query = query.filter(cls.cls.deleted.is_(False))
+            if _query:
+                _query = _query.filter(cls.cls.deleted.is_(False))
 
         for k in kwargs:
             if hasattr(cls.cls, k):

cmdb-api/api/lib/perm/acl/acl.py

@@ -253,9 +253,6 @@ def is_app_admin(app=None):
     if app is None:
         return False
 
-    if hasattr(current_user, 'username') and current_user.username == 'worker':
-        return True
-
     app_id = app.id
     if 'acl_admin' in session.get("acl", {}).get("parentRoles", []):
         return True

cmdb-api/api/lib/resp_format.py

@@ -29,6 +29,6 @@ class CommonErrFormat(object):
 
     role_required = _l("Role {} can only operate!")  # 角色 {} 才能操作!
     user_not_found = _l("User {} does not exist")  # 用户 {} 不存在
-    no_permission = _l("You do not have {} permission for resource: {}!")  # 您没有资源: {} 的{}权限!
+    no_permission = _l("For resource: {}, you do not have {} permission!")  # 您没有资源: {} 的{}权限!
     no_permission2 = _l("You do not have permission to operate!")  # 您没有操作权限!
     no_permission_only_owner = _l("Only the creator or administrator has permission!")  # 只有创建人或者管理员才有权限!