perf(api): auto discovery has been upgraded (#559)

cmdb-api/api/commands/click_cmdb.py

@@ -190,6 +190,7 @@ def cmdb_counter():
     login_user(UserCache.get('worker'))
 
     i = 0
+    today = datetime.date.today()
     while True:
         try:
             db.session.remove()
@@ -200,6 +201,10 @@ def cmdb_counter():
                 CMDBCounterCache.flush_adc_counter()
                 i = 0
 
+            if datetime.date.today() != today:
+                CMDBCounterCache.clear_ad_exec_history()
+                today = datetime.date.today()
+
             CMDBCounterCache.flush_sub_counter()
 
             i += 1
@@ -493,3 +498,48 @@ def cmdb_agent_init():
 
     click.echo("Key   : {}".format(click.style(user.key, bg='red')))
     click.echo("Secret: {}".format(click.style(user.secret, bg='red')))
+
+
+@click.command()
+@click.option(
+    '-v',
+    '--version',
+    help='input cmdb version, e.g. 2.4.6',
+    required=True,
+)
+@with_appcontext
+def cmdb_patch(version):
+    """
+    CMDB upgrade patch
+    """
+
+    version = version[1:] if version.lower().startswith("v") else version
+
+    if version >= '2.4.6':
+
+        from api.models.cmdb import CITypeRelation
+        for cr in CITypeRelation.get_by(to_dict=False):
+            if hasattr(cr, 'parent_attr_id') and cr.parent_attr_id and not cr.parent_attr_ids:
+                parent_attr_ids, child_attr_ids = [cr.parent_attr_id], [cr.child_attr_id]
+                cr.update(parent_attr_ids=parent_attr_ids, child_attr_ids=child_attr_ids, commit=False)
+        db.session.commit()
+
+        from api.models.cmdb import AutoDiscoveryCIType, AutoDiscoveryCITypeRelation
+        from api.lib.cmdb.cache import CITypeCache, AttributeCache
+        for adt in AutoDiscoveryCIType.get_by(to_dict=False):
+            if adt.relation:
+                if not AutoDiscoveryCITypeRelation.get_by(ad_type_id=adt.type_id):
+                    peer_type = CITypeCache.get(list(adt.relation.values())['type_name'])
+                    peer_type_id = peer_type and peer_type.id
+                    peer_attr = AttributeCache.get(list(adt.relation.values())['attr_name'])
+                    peer_attr_id = peer_attr and peer_attr.id
+                    if peer_type_id and peer_attr_id:
+                        AutoDiscoveryCITypeRelation.create(ad_type_id=adt.type_id,
+                                                           ad_key=list(adt.relation.keys())[0],
+                                                           peer_type_id=peer_type_id,
+                                                           peer_attr_id=peer_attr_id,
+                                                           commit=False)
+            if hasattr(adt, 'interval') and adt.interval and not adt.cron:
+                adt.cron = "*/{} * * * *".format(adt.interval // 60)
+
+        db.session.commit()

cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py

@@ -1,34 +1,47 @@
 # -*- coding:utf-8 -*-
+import copy
 import datetime
 import json
 import os
 
+from flask import abort
+from flask import current_app
+from flask_login import current_user
+from sqlalchemy import func
+
 from api.extensions import db
 from api.lib.cmdb.auto_discovery.const import ClOUD_MAP
+from api.lib.cmdb.auto_discovery.const import DEFAULT_HTTP
+from api.lib.cmdb.cache import AttributeCache
 from api.lib.cmdb.cache import CITypeAttributeCache
 from api.lib.cmdb.cache import CITypeCache
 from api.lib.cmdb.ci import CIManager
 from api.lib.cmdb.ci import CIRelationManager
 from api.lib.cmdb.ci_type import CITypeGroupManager
 from api.lib.cmdb.const import AutoDiscoveryType
+from api.lib.cmdb.const import CMDB_QUEUE
 from api.lib.cmdb.const import PermEnum
 from api.lib.cmdb.const import ResourceTypeEnum
 from api.lib.cmdb.resp_format import ErrFormat
 from api.lib.cmdb.search import SearchError
-from api.lib.cmdb.search.ci import search
+from api.lib.cmdb.search.ci import search as ci_search
+from api.lib.common_setting.role_perm_base import CMDBApp
 from api.lib.mixin import DBMixin
+from api.lib.perm.acl.acl import ACLManager
 from api.lib.perm.acl.acl import is_app_admin
 from api.lib.perm.acl.acl import validate_permission
 from api.lib.utils import AESCrypto
 from api.models.cmdb import AutoDiscoveryCI
 from api.models.cmdb import AutoDiscoveryCIType
+from api.models.cmdb import AutoDiscoveryCITypeRelation
+from api.models.cmdb import AutoDiscoveryCounter
+from api.models.cmdb import AutoDiscoveryExecHistory
 from api.models.cmdb import AutoDiscoveryRule
-from flask import abort
+from api.models.cmdb import AutoDiscoveryRuleSyncHistory
-from flask import current_app
+from api.tasks.cmdb import write_ad_rule_sync_history
-from flask_login import current_user
-from sqlalchemy import func
 
 PWD = os.path.abspath(os.path.dirname(__file__))
+app_cli = CMDBApp()
 
 
 def parse_plugin_script(script):
@@ -100,6 +113,14 @@ class AutoDiscoveryRuleCRUD(DBMixin):
         self.cls.get_by(name=kwargs['name']) and abort(400, ErrFormat.adr_duplicate.format(kwargs['name']))
         if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
             kwargs = check_plugin_script(**kwargs)
+            acl = ACLManager(app_cli.app_name)
+            if not acl.has_permission(app_cli.op.Auto_Discovery,
+                                      app_cli.resource_type_name,
+                                      app_cli.op.create_plugin) and not is_app_admin(app_cli.app_name):
+                return abort(403, ErrFormat.no_permission.format(
+                    app_cli.op.Auto_Discovery, app_cli.op.create_plugin))
+
+        kwargs['owner'] = current_user.uid
 
         return kwargs
 
@@ -115,6 +136,14 @@ class AutoDiscoveryRuleCRUD(DBMixin):
             if other and other.id != existed.id:
                 return abort(400, ErrFormat.adr_duplicate.format(kwargs['name']))
 
+        if existed.is_plugin:
+            acl = ACLManager(app_cli.app_name)
+            if not acl.has_permission(app_cli.op.Auto_Discovery,
+                                      app_cli.resource_type_name,
+                                      app_cli.op.update_plugin) and not is_app_admin(app_cli.app_name):
+                return abort(403, ErrFormat.no_permission.format(
+                    app_cli.op.Auto_Discovery, app_cli.op.update_plugin))
+
         return existed
 
     def update(self, _id, **kwargs):
@@ -122,13 +151,27 @@ class AutoDiscoveryRuleCRUD(DBMixin):
         if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
             kwargs = check_plugin_script(**kwargs)
 
+            for item in AutoDiscoveryCIType.get_by(adr_id=_id, to_dict=False):
+                item.update(updated_at=datetime.datetime.now())
+
         return super(AutoDiscoveryRuleCRUD, self).update(_id, filter_none=False, **kwargs)
 
     def _can_delete(self, **kwargs):
         if AutoDiscoveryCIType.get_by(adr_id=kwargs['_id'], first=True):
             return abort(400, ErrFormat.adr_referenced)
 
-        return self._can_update(**kwargs)
+        existed = self.cls.get_by_id(kwargs['_id']) or abort(
+            404, ErrFormat.adr_not_found.format("id={}".format(kwargs['_id'])))
+
+        if existed.is_plugin:
+            acl = ACLManager(app_cli.app_name)
+            if not acl.has_permission(app_cli.op.Auto_Discovery,
+                                      app_cli.resource_type_name,
+                                      app_cli.op.delete_plugin) and not is_app_admin(app_cli.app_name):
+                return abort(403, ErrFormat.no_permission.format(
+                    app_cli.op.Auto_Discovery, app_cli.op.delete_plugin))
+
+        return existed
 
 
 class AutoDiscoveryCITypeCRUD(DBMixin):
@@ -147,14 +190,34 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
         return cls.cls.get_by(type_id=type_id, to_dict=False)
 
     @classmethod
-    def get(cls, ci_id, oneagent_id, last_update_at=None):
+    def get_ad_attributes(cls, type_id):
+        result = []
+        adts = cls.get_by_type_id(type_id)
+        for adt in adts:
+            adr = AutoDiscoveryRuleCRUD.get_by_id(adt.adr_id)
+            if not adr:
+                continue
+            if adr.type == "http":
+                for i in DEFAULT_HTTP:
+                    if adr.name == i['name']:
+                        attrs = AutoDiscoveryHTTPManager.get_attributes(
+                            i['en'], (adt.extra_option or {}).get('category')) or []
+                        result.extend([i.get('name') for i in attrs])
+                        break
+            elif adr.type == "snmp":
+                attributes = AutoDiscoverySNMPManager.get_attributes()
+                result.extend([i.get('name') for i in (attributes or [])])
+            else:
+                result.extend([i.get('name') for i in (adr.attributes or [])])
+
+        return sorted(list(set(result)))
+
+    @classmethod
+    def get(cls, ci_id, oneagent_id, oneagent_name, last_update_at=None):
         result = []
         rules = cls.cls.get_by(to_dict=True)
 
         for rule in rules:
-            if rule.get('relation'):
-                continue
-
             if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('secret'):
                 if not (current_user.username == "cmdb_agent" or current_user.uid == rule['uid']):
                     rule['extra_option'].pop('secret', None)
@@ -165,7 +228,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
                 result.append(rule)
             elif rule['query_expr']:
                 query = rule['query_expr'].lstrip('q').lstrip('=')
-                s = search(query, fl=['_id'], count=1000000)
+                s = ci_search(query, fl=['_id'], count=1000000)
                 try:
                     response, _, _, _, _, _ = s.search()
                 except SearchError as e:
@@ -182,9 +245,6 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
                 if adr.type in (AutoDiscoveryType.SNMP, AutoDiscoveryType.HTTP):
                     continue
 
-                if not rule['updated_at']:
-                    continue
-
                 result.append(rule)
 
         new_last_update_at = ""
@@ -195,6 +255,9 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
             if new_last_update_at < __last_update_at:
                 new_last_update_at = __last_update_at
 
+        write_ad_rule_sync_history.apply_async(args=(result, oneagent_id, oneagent_name, datetime.datetime.now()),
+                                               queue=CMDB_QUEUE)
+
         if not last_update_at or new_last_update_at > last_update_at:
             return result, new_last_update_at
         else:
@@ -213,7 +276,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
             agent_id = agent_id.strip()
             q = "op_duty:{0},-rd_duty:{0},oneagent_id:{1}"
 
-            s = search(q.format(current_user.username, agent_id.strip()))
+            s = ci_search(q.format(current_user.username, agent_id.strip()))
             try:
                 response, _, _, _, _, _ = s.search()
                 if response:
@@ -222,7 +285,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
                 current_app.logger.warning(e)
                 return abort(400, str(e))
 
-            s = search(q.format(current_user.nickname, agent_id.strip()))
+            s = ci_search(q.format(current_user.nickname, agent_id.strip()))
             try:
                 response, _, _, _, _, _ = s.search()
                 if response:
@@ -236,7 +299,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
             if query_expr.startswith('q='):
                 query_expr = query_expr[2:]
 
-            s = search(query_expr, count=1000000)
+            s = ci_search(query_expr, count=1000000)
             try:
                 response, _, _, _, _, _ = s.search()
                 for i in response:
@@ -254,13 +317,21 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
     def _can_add(**kwargs):
 
         if kwargs.get('adr_id'):
-            AutoDiscoveryRule.get_by_id(kwargs['adr_id']) or abort(
+            adr = AutoDiscoveryRule.get_by_id(kwargs['adr_id']) or abort(
                 404, ErrFormat.adr_not_found.format("id={}".format(kwargs['adr_id'])))
-            # if not adr.is_plugin:
+            if adr.type == "http":
-            #     other = self.cls.get_by(adr_id=adr.id, first=True, to_dict=False)
+                kwargs.setdefault('extra_option', dict)
-            #     if other:
+                en_name = None
-            #         ci_type = CITypeCache.get(other.type_id)
+                for i in DEFAULT_HTTP:
-            #         return abort(400, ErrFormat.adr_default_ref_once.format(ci_type.alias))
+                    if i['name'] == adr.name:
+                        en_name = i['en']
+                        break
+                if en_name and kwargs['extra_option'].get('category'):
+                    for item in ClOUD_MAP[en_name]:
+                        if item["collect_key_map"].get(kwargs['extra_option']['category']):
+                            kwargs["extra_option"]["collect_key"] = item["collect_key_map"][
+                                kwargs['extra_option']['category']]
+                            break
 
         if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
             kwargs = check_plugin_script(**kwargs)
@@ -268,6 +339,11 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
         if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'):
             kwargs['extra_option']['secret'] = AESCrypto.encrypt(kwargs['extra_option']['secret'])
 
+        ci_type = CITypeCache.get(kwargs['type_id'])
+        unique = AttributeCache.get(ci_type.unique_id)
+        if unique and unique.name not in (kwargs.get('attributes') or {}).values():
+            return abort(400, ErrFormat.ad_not_unique_key.format(unique.name))
+
         kwargs['uid'] = current_user.uid
 
         return kwargs
@@ -276,7 +352,29 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
         existed = self.cls.get_by_id(kwargs['_id']) or abort(
             404, ErrFormat.ad_not_found.format("id={}".format(kwargs['_id'])))
 
-        self.__valid_exec_target(kwargs.get('agent_id'), kwargs.get('query_expr'))
+        adr = AutoDiscoveryRule.get_by_id(existed.adr_id) or abort(
+            404, ErrFormat.adr_not_found.format("id={}".format(existed.adr_id)))
+        if adr.type == "http":
+            kwargs.setdefault('extra_option', dict)
+            en_name = None
+            for i in DEFAULT_HTTP:
+                if i['name'] == adr.name:
+                    en_name = i['en']
+                    break
+            if en_name and kwargs['extra_option'].get('category'):
+                for item in ClOUD_MAP[en_name]:
+                    if item["collect_key_map"].get(kwargs['extra_option']['category']):
+                        kwargs["extra_option"]["collect_key"] = item["collect_key_map"][
+                            kwargs['extra_option']['category']]
+                        break
+
+        if 'attributes' in kwargs:
+            self.__valid_exec_target(kwargs.get('agent_id'), kwargs.get('query_expr'))
+
+            ci_type = CITypeCache.get(existed.type_id)
+            unique = AttributeCache.get(ci_type.unique_id)
+            if unique and unique.name not in (kwargs.get('attributes') or {}).values():
+                return abort(400, ErrFormat.ad_not_unique_key.format(unique.name))
 
         if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'):
             if current_user.uid != existed.uid:
@@ -292,7 +390,15 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
         if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'):
             kwargs['extra_option']['secret'] = AESCrypto.encrypt(kwargs['extra_option']['secret'])
 
-        return super(AutoDiscoveryCITypeCRUD, self).update(_id, filter_none=False, **kwargs)
+        inst = self._can_update(_id=_id, **kwargs)
+        if inst.agent_id != kwargs.get('agent_id') or inst.query_expr != kwargs.get('query_expr'):
+            for item in AutoDiscoveryRuleSyncHistory.get_by(adt_id=inst.id, to_dict=False):
+                item.delete(commit=False)
+            db.session.commit()
+
+        obj = inst.update(_id=_id, filter_none=False, **kwargs)
+
+        return obj
 
     def _can_delete(self, **kwargs):
         if AutoDiscoveryCICRUD.get_by_adt_id(kwargs['_id']):
@@ -303,6 +409,56 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
 
         return existed
 
+    def delete(self, _id):
+        inst = self._can_delete(_id=_id)
+
+        inst.soft_delete()
+
+        for item in AutoDiscoveryRuleSyncHistory.get_by(adt_id=inst.id, to_dict=False):
+            item.delete(commit=False)
+        db.session.commit()
+
+        attributes = self.get_ad_attributes(inst.type_id)
+        for item in AutoDiscoveryCITypeRelationCRUD.get_by_type_id(inst.type_id):
+            if item.ad_key not in attributes:
+                item.soft_delete()
+
+        return inst
+
+
+class AutoDiscoveryCITypeRelationCRUD(DBMixin):
+    cls = AutoDiscoveryCITypeRelation
+
+    @classmethod
+    def get_by_type_id(cls, type_id, to_dict=False):
+        return cls.cls.get_by(ad_type_id=type_id, to_dict=to_dict)
+
+    def upsert(self, ad_type_id, relations):
+        existed = self.cls.get_by(ad_type_id=ad_type_id, to_dict=False)
+        existed = {(i.ad_key, i.peer_type_id, i.peer_attr_id): i for i in existed}
+
+        new = []
+        for r in relations:
+            k = (r.get('ad_key'), r.get('peer_type_id'), r.get('peer_attr_id'))
+            if len(list(filter(lambda x: x, k))) == 3 and k not in existed:
+                self.cls.create(ad_type_id=ad_type_id, **r)
+
+            new.append(k)
+
+        for deleted in set(existed.keys()) - set(new):
+            existed[deleted].soft_delete()
+
+        return self.get_by_type_id(ad_type_id, to_dict=True)
+
+    def _can_add(self, **kwargs):
+        pass
+
+    def _can_update(self, **kwargs):
+        pass
+
+    def _can_delete(self, **kwargs):
+        pass
+
 
 class AutoDiscoveryCICRUD(DBMixin):
     cls = AutoDiscoveryCI
@@ -391,16 +547,24 @@ class AutoDiscoveryCICRUD(DBMixin):
         changed = False
         if existed is not None:
             if existed.instance != kwargs['instance']:
+                instance = copy.deepcopy(existed.instance) or {}
+                instance.update(kwargs['instance'])
+                kwargs['instance'] = instance
                 existed.update(filter_none=False, **kwargs)
+                AutoDiscoveryExecHistoryCRUD().add(type_id=adt.type_id,
+                                                   stdout="update resource: {}".format(kwargs.get('unique_value')))
                 changed = True
         else:
             existed = self.cls.create(**kwargs)
+            AutoDiscoveryExecHistoryCRUD().add(type_id=adt.type_id,
+                                               stdout="add resource: {}".format(kwargs.get('unique_value')))
             changed = True
 
         if adt.auto_accept and changed:
             try:
                 self.accept(existed)
             except Exception as e:
+                current_app.logger.error(e)
                 return abort(400, str(e))
         elif changed:
             existed.update(is_accept=False, accept_time=None, accept_by=None, filter_none=False)
@@ -420,6 +584,13 @@ class AutoDiscoveryCICRUD(DBMixin):
 
         inst.delete()
 
+        adt = AutoDiscoveryCIType.get_by_id(inst.adt_id)
+        if adt:
+            adt.update(updated_at=datetime.datetime.now())
+
+        AutoDiscoveryExecHistoryCRUD().add(type_id=inst.type_id,
+                                           stdout="delete resource: {}".format(inst.unique_value))
+
         self._after_delete(inst)
 
         return inst
@@ -435,6 +606,13 @@ class AutoDiscoveryCICRUD(DBMixin):
             not is_app_admin("cmdb") and validate_permission(ci_type.name, ResourceTypeEnum.CI, PermEnum.DELETE, "cmdb")
 
         existed.delete()
+
+        adt = AutoDiscoveryCIType.get_by_id(existed.adt_id)
+        if adt:
+            adt.update(updated_at=datetime.datetime.now())
+
+        AutoDiscoveryExecHistoryCRUD().add(type_id=type_id,
+                                           stdout="delete resource: {}".format(unique_value))
         # TODO: delete ci
 
     @classmethod
@@ -447,32 +625,34 @@ class AutoDiscoveryCICRUD(DBMixin):
         ci_id = None
         if adt.attributes:
             ci_dict = {adt.attributes[k]: v for k, v in adc.instance.items() if k in adt.attributes}
-            ci_id = CIManager.add(adc.type_id, is_auto_discovery=True, **ci_dict)
+            ci_id = CIManager.add(adc.type_id, is_auto_discovery=True, _is_admin=True, **ci_dict)
+            AutoDiscoveryExecHistoryCRUD().add(type_id=adt.type_id,
+                                               stdout="accept resource: {}".format(adc.unique_value))
 
-        relation_adts = AutoDiscoveryCIType.get_by(type_id=adt.type_id, adr_id=None, to_dict=False)
+        relation_ads = AutoDiscoveryCITypeRelation.get_by(ad_type_id=adt.type_id, to_dict=False)
-        for r_adt in relation_adts:
+        for r_adt in relation_ads:
-            if not r_adt.relation or ci_id is None:
+            ad_key = r_adt.ad_key
+            if not adc.instance.get(ad_key):
                 continue
-            for ad_key in r_adt.relation:
+
-                if not adc.instance.get(ad_key):
+            ad_key_values = [adc.instance.get(ad_key)] if not isinstance(
-                    continue
+                adc.instance.get(ad_key), list) else adc.instance.get(ad_key)
-                cmdb_key = r_adt.relation[ad_key]
+            for ad_key_value in ad_key_values:
-                query = "_type:{},{}:{}".format(cmdb_key.get('type_name'), cmdb_key.get('attr_name'),
+                query = "_type:{},{}:{}".format(r_adt.peer_type_id, r_adt.peer_attr_id, ad_key_value)
-                                                adc.instance.get(ad_key))
+                s = ci_search(query, use_ci_filter=False, count=1000000)
-                s = search(query)
                 try:
                     response, _, _, _, _, _ = s.search()
                 except SearchError as e:
                     current_app.logger.warning(e)
                     return abort(400, str(e))
 
-                relation_ci_id = response and response[0]['_id']
+                for relation_ci in response:
-                if relation_ci_id:
+                    relation_ci_id = relation_ci['_id']
                     try:
-                        CIRelationManager.add(ci_id, relation_ci_id)
+                        CIRelationManager.add(ci_id, relation_ci_id, valid=False)
                     except:
                         try:
-                            CIRelationManager.add(relation_ci_id, ci_id)
+                            CIRelationManager.add(relation_ci_id, ci_id, valid=False)
                         except:
                             pass
 
@@ -485,14 +665,35 @@ class AutoDiscoveryCICRUD(DBMixin):
 class AutoDiscoveryHTTPManager(object):
     @staticmethod
     def get_categories(name):
-        return (ClOUD_MAP.get(name) or {}).get('categories') or []
+        categories = (ClOUD_MAP.get(name) or {}) or []
+        for item in copy.deepcopy(categories):
+            item.pop('map', None)
+
+        return categories
+
+    def get_resources(self, name):
+        en_name = None
+        for i in DEFAULT_HTTP:
+            if i['name'] == name:
+                en_name = i['en']
+                break
+
+        if en_name:
+            categories = self.get_categories(en_name)
+
+            return [j for i in categories for j in i['items']]
+
+        return []
 
     @staticmethod
-    def get_attributes(name, category):
+    def get_attributes(provider, resource):
-        tpt = ((ClOUD_MAP.get(name) or {}).get('map') or {}).get(category)
+        for item in (ClOUD_MAP.get(provider) or {}):
-        if tpt and os.path.exists(os.path.join(PWD, tpt)):
+            for _resource in (item.get('map') or {}):
-            with open(os.path.join(PWD, tpt)) as f:
+                if _resource == resource:
-                return json.loads(f.read())
+                    tpt = item['map'][_resource]
+                    if tpt and os.path.exists(os.path.join(PWD, tpt)):
+                        with open(os.path.join(PWD, tpt)) as f:
+                            return json.loads(f.read())
 
         return []
 
@@ -506,3 +707,62 @@ class AutoDiscoverySNMPManager(object):
                 return json.loads(f.read())
 
         return []
+
+
+class AutoDiscoveryRuleSyncHistoryCRUD(DBMixin):
+    cls = AutoDiscoveryRuleSyncHistory
+
+    def _can_add(self, **kwargs):
+        pass
+
+    def _can_update(self, **kwargs):
+        pass
+
+    def _can_delete(self, **kwargs):
+        pass
+
+    def upsert(self, **kwargs):
+        existed = self.cls.get_by(adt_id=kwargs.get('adt_id'),
+                                  oneagent_id=kwargs.get('oneagent_id'),
+                                  oneagent_name=kwargs.get('oneagent_name'),
+                                  first=True,
+                                  to_dict=False)
+
+        if existed is not None:
+            existed.update(**kwargs)
+        else:
+            self.cls.create(**kwargs)
+
+
+class AutoDiscoveryExecHistoryCRUD(DBMixin):
+    cls = AutoDiscoveryExecHistory
+
+    def _can_add(self, **kwargs):
+        pass
+
+    def _can_update(self, **kwargs):
+        pass
+
+    def _can_delete(self, **kwargs):
+        pass
+
+
+class AutoDiscoveryCounterCRUD(DBMixin):
+    cls = AutoDiscoveryCounter
+
+    def get(self, type_id):
+        res = self.cls.get_by(type_id=type_id, first=True, to_dict=True)
+        if res is None:
+            return dict(rule_count=0, exec_target_count=0, instance_count=0, accept_count=0,
+                        this_month_count=0, this_week_count=0, last_month_count=0, last_week_count=0)
+
+        return res
+
+    def _can_add(self, **kwargs):
+        pass
+
+    def _can_update(self, **kwargs):
+        pass
+
+    def _can_delete(self, **kwargs):
+        pass

cmdb-api/api/lib/cmdb/auto_discovery/const.py

@@ -3,13 +3,13 @@
 from api.lib.cmdb.const import AutoDiscoveryType
 
 DEFAULT_HTTP = [
-    dict(name="阿里云", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
+    dict(name="阿里云", en="aliyun", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
          option={'icon': {'name': 'caise-aliyun'}}),
-    dict(name="腾讯云", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
+    dict(name="腾讯云", en="tencentcloud", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
          option={'icon': {'name': 'caise-tengxunyun'}}),
-    dict(name="华为云", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
+    dict(name="华为云", en="huaweicloud", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
          option={'icon': {'name': 'caise-huaweiyun'}}),
-    dict(name="AWS", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
+    dict(name="AWS", en="aws", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
          option={'icon': {'name': 'caise-aws'}}),
 
     dict(name="交换机", type=AutoDiscoveryType.SNMP, is_inner=True, is_plugin=False,
@@ -23,31 +23,47 @@ DEFAULT_HTTP = [
 ]
 
 ClOUD_MAP = {
-    "aliyun": {
+    "aliyun": [{
-        "categories": ["云服务器 ECS"],
+        "category": "计算",
+        "items": ["云服务器 ECS"],
         "map": {
             "云服务器 ECS": "templates/aliyun_ecs.json",
+        },
+        "collect_key_map": {
+            "云服务器 ECS": "ali.ecs",
         }
-    },
+    }],
 
-    "tencentcloud": {
+    "tencentcloud": [{
-        "categories": ["云服务器 CVM"],
+        "category": "计算",
+        "items": ["云服务器 CVM"],
         "map": {
             "云服务器 CVM": "templates/tencent_cvm.json",
+        },
+        "collect_key_map": {
+            "云服务器 CVM": "tencent.cvm",
         }
-    },
+    }],
 
-    "huaweicloud": {
+    "huaweicloud": [{
-        "categories": ["云服务器 ECS"],
+        "category": "计算",
+        "items": ["云服务器 ECS"],
         "map": {
             "云服务器 ECS": "templates/huaweicloud_ecs.json",
+        },
+        "collect_key_map": {
+            "云服务器 ECS": "huawei.ecs",
         }
-    },
+    }],
 
-    "aws": {
+    "aws": [{
-        "categories": ["云服务器 EC2"],
+        "category": "计算",
+        "items": ["云服务器 EC2"],
         "map": {
             "云服务器 EC2": "templates/aws_ec2.json",
+        },
+        "collect_key_map": {
+            "云服务器 EC2": "aws.ec2",
         }
-    },
+    }],
 }

cmdb-api/api/lib/cmdb/cache.py

@@ -2,12 +2,19 @@
 
 from __future__ import unicode_literals
 
+import datetime
+
 from flask import current_app
 
 from api.extensions import cache
 from api.extensions import db
 from api.lib.cmdb.custom_dashboard import CustomDashboardManager
-from api.models.cmdb import Attribute
+from api.models.cmdb import Attribute, AutoDiscoveryExecHistory
+from api.models.cmdb import AutoDiscoveryCI
+from api.models.cmdb import AutoDiscoveryCIType
+from api.models.cmdb import AutoDiscoveryCITypeRelation
+from api.models.cmdb import AutoDiscoveryCounter
+from api.models.cmdb import AutoDiscoveryRuleSyncHistory
 from api.models.cmdb import CI
 from api.models.cmdb import CIType
 from api.models.cmdb import CITypeAttribute
@@ -448,7 +455,67 @@ class CMDBCounterCache(object):
 
         cache.set(cls.KEY2, result, timeout=0)
 
-        return result
+        res = db.session.query(AutoDiscoveryCI.created_at,
+                               AutoDiscoveryCI.updated_at,
+                               AutoDiscoveryCI.adt_id,
+                               AutoDiscoveryCI.type_id,
+                               AutoDiscoveryCI.is_accept).filter(AutoDiscoveryCI.deleted.is_(False))
+
+        today = datetime.datetime.today()
+        this_month = datetime.datetime(today.year, today.month, 1)
+        last_month = this_month - datetime.timedelta(days=1)
+        last_month = datetime.datetime(last_month.year, last_month.month, 1)
+        this_week = today - datetime.timedelta(days=datetime.date.weekday(today))
+        this_week = datetime.datetime(this_week.year, this_week.month, this_week.day)
+        last_week = this_week - datetime.timedelta(days=7)
+        last_week = datetime.datetime(last_week.year, last_week.month, last_week.day)
+        result = dict()
+        for i in res:
+            if i.type_id not in result:
+                result[i.type_id] = dict(instance_count=0, accept_count=0,
+                                         this_month_count=0, this_week_count=0, last_month_count=0, last_week_count=0)
+
+                adts = AutoDiscoveryCIType.get_by(type_id=i.type_id, to_dict=False)
+                result[i.type_id]['rule_count'] = len(adts) + AutoDiscoveryCITypeRelation.get_by(
+                    ad_type_id=i.type_id, only_query=True).count()
+                result[i.type_id]['exec_target_count'] = len(
+                    set([i.oneagent_id for adt in adts for i in db.session.query(
+                        AutoDiscoveryRuleSyncHistory.oneagent_id).filter(
+                        AutoDiscoveryRuleSyncHistory.adt_id == adt.id)]))
+
+            result[i.type_id]['instance_count'] += 1
+            if i.is_accept:
+                result[i.type_id]['accept_count'] += 1
+
+            if last_month <= i.created_at < this_month:
+                result[i.type_id]['last_month_count'] += 1
+            elif i.created_at >= this_month:
+                result[i.type_id]['this_month_count'] += 1
+
+            if last_week <= i.created_at < this_week:
+                result[i.type_id]['last_week_count'] += 1
+            elif i.created_at >= this_week:
+                result[i.type_id]['this_week_count'] += 1
+
+        for type_id in result:
+            existed = AutoDiscoveryCounter.get_by(type_id=type_id, first=True, to_dict=False)
+            if existed is None:
+                AutoDiscoveryCounter.create(type_id=type_id, **result[type_id])
+            else:
+                existed.update(**result[type_id])
+
+        for i in AutoDiscoveryCounter.get_by(to_dict=False):
+            if i.type_id not in result:
+                i.delete()
+
+    @classmethod
+    def clear_ad_exec_history(cls):
+        ci_types = CIType.get_by(to_dict=False)
+        for ci_type in ci_types:
+            for i in AutoDiscoveryExecHistory.get_by(type_id=ci_type.id, only_query=True).order_by(
+                    AutoDiscoveryExecHistory.id.desc()).offset(50000):
+                i.delete(commit=False)
+            db.session.commit()
 
     @classmethod
     def get_adc_counter(cls):

cmdb-api/api/lib/cmdb/ci.py

@@ -223,7 +223,7 @@ class CIManager(object):
     def ci_is_exist(unique_key, unique_value, type_id):
         """
 
-        :param unique_key: is a attribute
+        :param unique_key: is an attribute
         :param unique_value:
         :param type_id:
         :return:
@@ -432,7 +432,7 @@ class CIManager(object):
             for attr_id in password_dict:
                 record_id = cls.save_password(ci.id, attr_id, password_dict[attr_id], record_id, ci_type.id)
 
-        if record_id or has_dynamic:  # has change
+        if record_id or has_dynamic:  # has changed
             ci_cache.apply_async(args=(ci.id, operate_type, record_id), queue=CMDB_QUEUE)
 
         if ref_ci_dict:  # add relations
@@ -504,7 +504,7 @@ class CIManager(object):
             for attr_id in password_dict:
                 record_id = self.save_password(ci.id, attr_id, password_dict[attr_id], record_id, ci.type_id)
 
-        if record_id or has_dynamic:  # has change
+        if record_id or has_dynamic:  # has changed
             if not __sync:
                 ci_cache.apply_async(args=(ci_id, OperateType.UPDATE, record_id), queue=CMDB_QUEUE)
             else:
@@ -737,7 +737,7 @@ class CIManager(object):
                        fields=None, value_tables=None, unique_required=False, excludes=None):
         """
 
-        :param ci_ids: list of CI instance ID, eg. ['1', '2']
+        :param ci_ids: list of CI instance ID, e.g. ['1', '2']
         :param ret_key: name, id or alias
         :param fields:
         :param value_tables:
@@ -1296,7 +1296,7 @@ class CIRelationManager(object):
                     relations = _relations
                 else:
                     relations &= _relations
-            for parent_ci_id, child_ci_id in relations:
+            for parent_ci_id, child_ci_id in (relations or []):
                 CIRelationManager.add(parent_ci_id, child_ci_id, valid=False)
 
         parent_items = CITypeRelation.get_by(child_id=type_id, only_query=True).filter(
@@ -1316,7 +1316,7 @@ class CIRelationManager(object):
                     relations = _relations
                 else:
                     relations &= _relations
-            for parent_ci_id, child_ci_id in relations:
+            for parent_ci_id, child_ci_id in (relations or []):
                 CIRelationManager.add(parent_ci_id, child_ci_id, valid=False)
 
     @classmethod

cmdb-api/api/lib/cmdb/ci_type.py

@@ -229,6 +229,9 @@ class CITypeManager(object):
         if CI.get_by(type_id=type_id, first=True, to_dict=False) is not None:
             return abort(400, ErrFormat.ci_exists_and_cannot_delete_type)
 
+        if CITypeInheritance.get_by(parent_id=type_id, first=True):
+            return abort(400, ErrFormat.ci_type_inheritance_cannot_delete)
+
         relation_views = PreferenceRelationView.get_by(to_dict=False)
         for rv in relation_views:
             for item in (rv.cr_ids or []):
@@ -253,21 +256,21 @@ class CITypeManager(object):
             item.delete(commit=False)
 
         for item in AutoDiscoveryCITypeRelation.get_by(ad_type_id=type_id, to_dict=False):
-            item.delete(commit=False)
+            item.soft_delete(commit=False)
 
         for item in AutoDiscoveryCITypeRelation.get_by(peer_type_id=type_id, to_dict=False):
-            item.delete(commit=False)
+            item.soft_delete(commit=False)
 
         for item in CITypeInheritance.get_by(parent_id=type_id, to_dict=False):
-            item.delete(commit=False)
+            item.soft_delete(commit=False)
 
         for item in CITypeInheritance.get_by(child_id=type_id, to_dict=False):
-            item.delete(commit=False)
+            item.soft_delete(commit=False)
 
         try:
             from api.models.cmdb import CITypeReconciliation
             for item in CITypeReconciliation.get_by(type_id=type_id, to_dict=False):
-                item.delete(commit=False)
+                item.soft_delete(commit=False)
         except Exception:
             pass
 

cmdb-api/api/lib/cmdb/const.py

@@ -55,9 +55,9 @@ class CITypeOperateType(BaseEnum):
     DELETE_UNIQUE_CONSTRAINT = "11"  # 删除联合唯一
     ADD_RELATION = "12"  # 新增关系
     DELETE_RELATION = "13"  # 删除关系
-    ADD_RECONCILIATION = "14"  # 删除关系
+    ADD_RECONCILIATION = "14"  # 新增数据合规
-    UPDATE_RECONCILIATION = "15"  # 删除关系
+    UPDATE_RECONCILIATION = "15"  # 修改数据合规
-    DELETE_RECONCILIATION = "16"  # 删除关系
+    DELETE_RECONCILIATION = "16"  # 删除数据合规
 
 
 class RetKey(BaseEnum):

cmdb-api/api/lib/cmdb/history.py

@@ -13,6 +13,7 @@ from api.lib.cmdb.const import OperateType
 from api.lib.cmdb.perms import CIFilterPermsCRUD
 from api.lib.cmdb.resp_format import ErrFormat
 from api.lib.perm.acl.cache import UserCache
+from api.models.cmdb import CI
 from api.models.cmdb import Attribute
 from api.models.cmdb import AttributeHistory
 from api.models.cmdb import CIRelationHistory
@@ -306,7 +307,7 @@ class CITriggerHistoryManager(object):
     def get(page, page_size, type_id=None, trigger_id=None, operate_type=None):
         query = CITriggerHistory.get_by(only_query=True)
         if type_id:
-            query = query.filter(CITriggerHistory.type_id == type_id)
+            query = query.join(CI, CI.id == CITriggerHistory.ci_id).filter(CI.type_id == type_id)
 
         if trigger_id:
             query = query.filter(CITriggerHistory.trigger_id == trigger_id)

cmdb-api/api/lib/cmdb/resp_format.py

@@ -62,6 +62,7 @@ class ErrFormat(CommonErrFormat):
         "The model cannot be deleted because the CI already exists")  # 因为CI已经存在，不能删除模型
     ci_exists_and_cannot_delete_inheritance = _l(
         "The inheritance cannot be deleted because the CI already exists")  # 因为CI已经存在，不能删除继承关系
+    ci_type_inheritance_cannot_delete = _l("The model is inherited and cannot be deleted")  # 该模型被继承, 不能删除
 
     # 因为关系视图 {} 引用了该模型，不能删除模型
     ci_relation_view_exists_and_cannot_delete_type = _l(

cmdb-api/api/lib/cmdb/topology.py

@@ -16,7 +16,7 @@ from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
 from api.lib.cmdb.const import ResourceTypeEnum
 from api.lib.cmdb.resp_format import ErrFormat
 from api.lib.cmdb.search import SearchError
-from api.lib.cmdb.search.ci import search
+from api.lib.cmdb.search.ci import search as ci_search
 from api.lib.perm.acl.acl import ACLManager
 from api.lib.perm.acl.acl import is_app_admin
 from api.models.cmdb import TopologyView
@@ -178,7 +178,7 @@ class TopologyViewManager(object):
 
         q = (central_node_instances[2:] if central_node_instances.startswith('q=') else
              central_node_instances)
-        s = search(q, fl=['_id', show_key.name], use_id_filter=False, use_ci_filter=False, count=1000000)
+        s = ci_search(q, fl=['_id', show_key.name], use_id_filter=False, use_ci_filter=False, count=1000000)
         try:
             response, _, _, _, _, _ = s.search()
         except SearchError as e:
@@ -238,7 +238,7 @@ class TopologyViewManager(object):
                     type2show[type_id] = attr.name
 
         if id2node:
-            s = search("_id:({})".format(';'.join(id2node.keys())), fl=list(fl),
+            s = ci_search("_id:({})".format(';'.join(id2node.keys())), fl=list(fl),
                        use_id_filter=False, use_ci_filter=False, count=1000000)
             try:
                 response, _, _, _, _, _ = s.search()

cmdb-api/api/lib/mixin.py

@@ -8,6 +8,8 @@ from api.extensions import db
 from api.lib.utils import get_page
 from api.lib.utils import get_page_size
 
+__author__ = 'pycook'
+
 
 class DBMixin(object):
     cls = None
@@ -17,13 +19,18 @@ class DBMixin(object):
         page = get_page(page)
         page_size = get_page_size(page_size)
         if fl is None:
-            query = db.session.query(cls.cls).filter(cls.cls.deleted.is_(False))
+            query = db.session.query(cls.cls)
         else:
-            query = db.session.query(*[getattr(cls.cls, i) for i in fl]).filter(cls.cls.deleted.is_(False))
+            query = db.session.query(*[getattr(cls.cls, i) for i in fl])
 
         _query = None
         if count_query:
-            _query = db.session.query(func.count(cls.cls.id)).filter(cls.cls.deleted.is_(False))
+            _query = db.session.query(func.count(cls.cls.id))
+
+        if hasattr(cls.cls, 'deleted'):
+            query = query.filter(cls.cls.deleted.is_(False))
+            if _query:
+                _query = _query.filter(cls.cls.deleted.is_(False))
 
         for k in kwargs:
             if hasattr(cls.cls, k):

cmdb-api/api/lib/perm/acl/acl.py

@@ -253,9 +253,6 @@ def is_app_admin(app=None):
     if app is None:
         return False
 
-    if hasattr(current_user, 'username') and current_user.username == 'worker':
-        return True
-
     app_id = app.id
     if 'acl_admin' in session.get("acl", {}).get("parentRoles", []):
         return True

cmdb-api/api/lib/resp_format.py

@@ -29,6 +29,6 @@ class CommonErrFormat(object):
 
     role_required = _l("Role {} can only operate!")  # 角色 {} 才能操作!
     user_not_found = _l("User {} does not exist")  # 用户 {} 不存在
-    no_permission = _l("You do not have {} permission for resource: {}!")  # 您没有资源: {} 的{}权限!
+    no_permission = _l("For resource: {}, you do not have {} permission!")  # 您没有资源: {} 的{}权限!
     no_permission2 = _l("You do not have permission to operate!")  # 您没有操作权限!
     no_permission_only_owner = _l("Only the creator or administrator has permission!")  # 只有创建人或者管理员才有权限!

cmdb-api/api/models/cmdb.py

@@ -566,14 +566,14 @@ class AutoDiscoveryCIType(Model):
 
     attributes = db.Column(db.JSON)  # {ad_key: cmdb_key}
 
-    relation = db.Column(db.JSON)  # [{ad_key: {type_id: x, attr_id: x}}]
+    relation = db.Column(db.JSON)  # [{ad_key: {type_id: x, attr_id: x}}], CMDB > 2.4.5: deprecated
 
     auto_accept = db.Column(db.Boolean, default=False)
 
     agent_id = db.Column(db.String(8), index=True)
     query_expr = db.Column(db.Text)
 
-    interval = db.Column(db.Integer)  # seconds
+    interval = db.Column(db.Integer)  # seconds, > 2.4.5: deprecated
     cron = db.Column(db.String(128))
 
     extra_option = db.Column(db.JSON)
@@ -604,6 +604,36 @@ class AutoDiscoveryCI(Model):
     accept_time = db.Column(db.DateTime)
 
 
+class AutoDiscoveryRuleSyncHistory(Model2):
+    __tablename__ = "c_ad_rule_sync_histories"
+
+    adt_id = db.Column(db.Integer, db.ForeignKey('c_ad_ci_types.id'))
+    oneagent_id = db.Column(db.String(8))
+    oneagent_name = db.Column(db.String(64))
+    sync_at = db.Column(db.DateTime, default=datetime.datetime.now())
+
+
+class AutoDiscoveryExecHistory(Model2):
+    __tablename__ = "c_ad_exec_histories"
+
+    type_id = db.Column(db.Integer, index=True)
+    stdout = db.Column(db.Text)
+
+
+class AutoDiscoveryCounter(Model2):
+    __tablename__ = "c_ad_counter"
+
+    type_id = db.Column(db.Integer, index=True)
+    rule_count = db.Column(db.Integer, default=0)
+    exec_target_count = db.Column(db.Integer, default=0)
+    instance_count = db.Column(db.Integer, default=0)
+    accept_count = db.Column(db.Integer, default=0)
+    this_month_count = db.Column(db.Integer, default=0)
+    this_week_count = db.Column(db.Integer, default=0)
+    last_month_count = db.Column(db.Integer, default=0)
+    last_week_count = db.Column(db.Integer, default=0)
+
+
 class CIFilterPerms(Model):
     __tablename__ = "c_ci_filter_perms"
 

cmdb-api/api/tasks/cmdb.py

@@ -2,6 +2,7 @@
 
 
 import json
+import datetime
 
 import redis_lock
 from flask import current_app
@@ -25,6 +26,8 @@ from api.lib.utils import handle_arg_list
 from api.models.cmdb import CI
 from api.models.cmdb import CIRelation
 from api.models.cmdb import CITypeAttribute
+from api.models.cmdb import AutoDiscoveryCI
+from api.models.cmdb import AutoDiscoveryCIType
 
 
 @celery.task(name="cmdb.ci_cache", queue=CMDB_QUEUE)
@@ -87,6 +90,13 @@ def ci_delete(ci_id):
     else:
         rd.delete(ci_id, REDIS_PREFIX_CI)
 
+    instance = AutoDiscoveryCI.get_by(ci_id=ci_id, to_dict=False, first=True)
+    if instance is not None:
+        adt = AutoDiscoveryCIType.get_by_id(instance.adt_id)
+        if adt:
+            adt.update(updated_at=datetime.datetime.now())
+        instance.delete()
+
     current_app.logger.info("{0} delete..........".format(ci_id))
 
 
@@ -249,3 +259,21 @@ def calc_computed_attribute(attr_id, uid):
         cis = CI.get_by(type_id=i.type_id, to_dict=False)
         for ci in cis:
             cim.update(ci.id, {})
+
+
+@celery.task(name="cmdb.write_ad_rule_sync_history", queue=CMDB_QUEUE)
+@reconnect_db
+def write_ad_rule_sync_history(rules, oneagent_id, oneagent_name, sync_at):
+    from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryRuleSyncHistoryCRUD
+
+    for rule in rules:
+        AutoDiscoveryRuleSyncHistoryCRUD().upsert(adt_id=rule['id'],
+                                                  oneagent_id=oneagent_id,
+                                                  oneagent_name=oneagent_name,
+                                                  sync_at=sync_at,
+                                                  commit=False)
+    try:
+        db.session.commit()
+    except Exception as e:
+        current_app.logger.error("write auto discovery rule sync history failed: {}".format(e))
+        db.session.rollback()

cmdb-api/api/translations/zh/LC_MESSAGES/messages.po

@@ -7,7 +7,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2024-05-28 18:05+0800\n"
+"POT-Creation-Date: 2024-06-20 19:12+0800\n"
 "PO-Revision-Date: 2023-12-25 20:21+0800\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language: zh\n"
@@ -81,7 +81,7 @@ msgid "User {} does not exist"
 msgstr "用户 {} 不存在"
 
 #: api/lib/resp_format.py:32
-msgid "You do not have {} permission for resource: {}!"
+msgid "For resource: {}, you do not have {} permission!"
 msgstr "您没有资源: {} 的{}权限!"
 
 #: api/lib/resp_format.py:33
@@ -238,241 +238,245 @@ msgstr "因为CI已经存在，不能删除模型"
 msgid "The inheritance cannot be deleted because the CI already exists"
 msgstr "因为CI已经存在，不能删除继承关系"
 
-#: api/lib/cmdb/resp_format.py:67
+#: api/lib/cmdb/resp_format.py:65
+msgid "The model is inherited and cannot be deleted"
+msgstr "该模型被继承, 不能删除"
+
+#: api/lib/cmdb/resp_format.py:68
 msgid ""
 "The model cannot be deleted because the model is referenced by the "
 "relational view {}"
 msgstr "因为关系视图 {} 引用了该模型，不能删除模型"
 
-#: api/lib/cmdb/resp_format.py:69
+#: api/lib/cmdb/resp_format.py:70
 msgid "Model group {} does not exist"
 msgstr "模型分组 {} 不存在"
 
-#: api/lib/cmdb/resp_format.py:70
+#: api/lib/cmdb/resp_format.py:71
 msgid "Model group {} already exists"
 msgstr "模型分组 {} 已经存在"
 
-#: api/lib/cmdb/resp_format.py:71
+#: api/lib/cmdb/resp_format.py:72
 msgid "Model relationship {} does not exist"
 msgstr "模型关系 {} 不存在"
 
-#: api/lib/cmdb/resp_format.py:72
+#: api/lib/cmdb/resp_format.py:73
 msgid "Attribute group {} already exists"
 msgstr "属性分组 {} 已存在"
 
-#: api/lib/cmdb/resp_format.py:73
+#: api/lib/cmdb/resp_format.py:74
 msgid "Attribute group {} does not exist"
 msgstr "属性分组 {} 不存在"
 
-#: api/lib/cmdb/resp_format.py:75
+#: api/lib/cmdb/resp_format.py:76
 msgid "Attribute group <{0}> - attribute <{1}> does not exist"
 msgstr "属性组<{0}> - 属性<{1}> 不存在"
 
-#: api/lib/cmdb/resp_format.py:76
+#: api/lib/cmdb/resp_format.py:77
 msgid "The unique constraint already exists!"
 msgstr "唯一约束已经存在!"
 
-#: api/lib/cmdb/resp_format.py:78
+#: api/lib/cmdb/resp_format.py:79
 msgid "Uniquely constrained attributes cannot be JSON and multi-valued"
 msgstr "唯一约束的属性不能是 JSON 和 多值"
 
-#: api/lib/cmdb/resp_format.py:79
+#: api/lib/cmdb/resp_format.py:80
 msgid "Duplicated trigger"
 msgstr "重复的触发器"
 
-#: api/lib/cmdb/resp_format.py:80
+#: api/lib/cmdb/resp_format.py:81
 msgid "Trigger {} does not exist"
 msgstr "触发器 {} 不存在"
 
-#: api/lib/cmdb/resp_format.py:81
+#: api/lib/cmdb/resp_format.py:82
 msgid "Duplicated reconciliation rule"
 msgstr ""
 
-#: api/lib/cmdb/resp_format.py:82
+#: api/lib/cmdb/resp_format.py:83
 msgid "Reconciliation rule {} does not exist"
 msgstr "关系类型 {} 不存在"
 
-#: api/lib/cmdb/resp_format.py:84
+#: api/lib/cmdb/resp_format.py:85
 msgid "Operation record {} does not exist"
 msgstr "操作记录 {} 不存在"
 
-#: api/lib/cmdb/resp_format.py:85
+#: api/lib/cmdb/resp_format.py:86
 msgid "Unique identifier cannot be deleted"
 msgstr "不能删除唯一标识"
 
-#: api/lib/cmdb/resp_format.py:86
+#: api/lib/cmdb/resp_format.py:87
 msgid "Cannot delete default sorted attributes"
 msgstr "不能删除默认排序的属性"
 
-#: api/lib/cmdb/resp_format.py:88
+#: api/lib/cmdb/resp_format.py:89
 msgid "No node selected"
 msgstr "没有选择节点"
 
-#: api/lib/cmdb/resp_format.py:89
+#: api/lib/cmdb/resp_format.py:90
 msgid "This search option does not exist!"
 msgstr "该搜索选项不存在!"
 
-#: api/lib/cmdb/resp_format.py:90
+#: api/lib/cmdb/resp_format.py:91
 msgid "This search option has a duplicate name!"
 msgstr "该搜索选项命名重复!"
 
-#: api/lib/cmdb/resp_format.py:92
+#: api/lib/cmdb/resp_format.py:93
 msgid "Relationship type {} already exists"
 msgstr "关系类型 {} 已经存在"
 
-#: api/lib/cmdb/resp_format.py:93
+#: api/lib/cmdb/resp_format.py:94
 msgid "Relationship type {} does not exist"
 msgstr "关系类型 {} 不存在"
 
-#: api/lib/cmdb/resp_format.py:95
+#: api/lib/cmdb/resp_format.py:96
 msgid "Invalid attribute value: {}"
 msgstr "无效的属性值: {}"
 
-#: api/lib/cmdb/resp_format.py:96
+#: api/lib/cmdb/resp_format.py:97
 msgid "{} Invalid value: {}"
 msgstr "{} 无效的值: {}"
 
-#: api/lib/cmdb/resp_format.py:97
+#: api/lib/cmdb/resp_format.py:98
 msgid "{} is not in the predefined values"
 msgstr "{} 不在预定义值里"
 
-#: api/lib/cmdb/resp_format.py:99
+#: api/lib/cmdb/resp_format.py:100
 msgid "The value of attribute {} must be unique, {} already exists"
 msgstr "属性 {} 的值必须是唯一的, 当前值 {} 已存在"
 
-#: api/lib/cmdb/resp_format.py:100
+#: api/lib/cmdb/resp_format.py:101
 msgid "Attribute {} value must exist"
 msgstr "属性 {} 值必须存在"
 
-#: api/lib/cmdb/resp_format.py:101
+#: api/lib/cmdb/resp_format.py:102
 msgid "Out of range value, the maximum value is 2147483647"
 msgstr "超过最大值限制, 最大值是2147483647"
 
-#: api/lib/cmdb/resp_format.py:103
+#: api/lib/cmdb/resp_format.py:104
 msgid "Unknown error when adding or modifying attribute value: {}"
 msgstr "新增或者修改属性值未知错误: {}"
 
-#: api/lib/cmdb/resp_format.py:105
+#: api/lib/cmdb/resp_format.py:106
 msgid "Duplicate custom name"
 msgstr "订制名重复"
 
-#: api/lib/cmdb/resp_format.py:107
+#: api/lib/cmdb/resp_format.py:108
 msgid "Number of models exceeds limit: {}"
 msgstr "模型数超过限制: {}"
 
-#: api/lib/cmdb/resp_format.py:108
+#: api/lib/cmdb/resp_format.py:109
 msgid "The number of CIs exceeds the limit: {}"
 msgstr "CI数超过限制: {}"
 
-#: api/lib/cmdb/resp_format.py:110
+#: api/lib/cmdb/resp_format.py:111
 msgid "Auto-discovery rule: {} already exists!"
 msgstr "自动发现规则: {} 已经存在!"
 
-#: api/lib/cmdb/resp_format.py:111
+#: api/lib/cmdb/resp_format.py:112
 msgid "Auto-discovery rule: {} does not exist!"
 msgstr "自动发现规则: {} 不存在!"
 
-#: api/lib/cmdb/resp_format.py:113
+#: api/lib/cmdb/resp_format.py:114
 msgid "This auto-discovery rule is referenced by the model and cannot be deleted!"
 msgstr "该自动发现规则被模型引用, 不能删除!"
 
-#: api/lib/cmdb/resp_format.py:115
+#: api/lib/cmdb/resp_format.py:116
 msgid "The application of auto-discovery rules cannot be defined repeatedly!"
 msgstr "自动发现规则的应用不能重复定义!"
 
-#: api/lib/cmdb/resp_format.py:116
+#: api/lib/cmdb/resp_format.py:117
 msgid "The auto-discovery you want to modify: {} does not exist!"
 msgstr "您要修改的自动发现: {} 不存在!"
 
-#: api/lib/cmdb/resp_format.py:117
+#: api/lib/cmdb/resp_format.py:118
 msgid "Attribute does not include unique identifier: {}"
 msgstr "属性字段没有包括唯一标识: {}"
 
-#: api/lib/cmdb/resp_format.py:118
+#: api/lib/cmdb/resp_format.py:119
 msgid "The auto-discovery instance does not exist!"
 msgstr "自动发现的实例不存在!"
 
-#: api/lib/cmdb/resp_format.py:119
+#: api/lib/cmdb/resp_format.py:120
 msgid "The model is not associated with this auto-discovery!"
 msgstr "模型并未关联该自动发现!"
 
-#: api/lib/cmdb/resp_format.py:120
+#: api/lib/cmdb/resp_format.py:121
 msgid "Only the creator can modify the Secret!"
 msgstr "只有创建人才能修改Secret!"
 
-#: api/lib/cmdb/resp_format.py:122
+#: api/lib/cmdb/resp_format.py:123
 msgid "This rule already has auto-discovery instances and cannot be deleted!"
 msgstr "该规则已经有自动发现的实例, 不能被删除!"
 
-#: api/lib/cmdb/resp_format.py:124
+#: api/lib/cmdb/resp_format.py:125
 msgid "The default auto-discovery rule is already referenced by model {}!"
 msgstr "该默认的自动发现规则 已经被模型 {} 引用!"
 
-#: api/lib/cmdb/resp_format.py:126
+#: api/lib/cmdb/resp_format.py:127
 msgid "The unique_key method must return a non-empty string!"
 msgstr "unique_key方法必须返回非空字符串!"
 
-#: api/lib/cmdb/resp_format.py:127
+#: api/lib/cmdb/resp_format.py:128
 msgid "The attributes method must return a list"
 msgstr "attributes方法必须返回的是list"
 
-#: api/lib/cmdb/resp_format.py:129
+#: api/lib/cmdb/resp_format.py:130
 msgid "The list returned by the attributes method cannot be empty!"
 msgstr "attributes方法返回的list不能为空!"
 
-#: api/lib/cmdb/resp_format.py:131
+#: api/lib/cmdb/resp_format.py:132
 msgid "Only administrators can define execution targets as: all nodes!"
 msgstr "只有管理员才可以定义执行机器为: 所有节点!"
 
-#: api/lib/cmdb/resp_format.py:132
+#: api/lib/cmdb/resp_format.py:133
 msgid "Execute targets permission check failed: {}"
 msgstr "执行机器权限检查不通过: {}"
 
-#: api/lib/cmdb/resp_format.py:134
+#: api/lib/cmdb/resp_format.py:135
 msgid "CI filter authorization must be named!"
 msgstr "CI过滤授权 必须命名!"
 
-#: api/lib/cmdb/resp_format.py:135
+#: api/lib/cmdb/resp_format.py:136
 msgid "CI filter authorization is currently not supported or query"
 msgstr "CI过滤授权 暂时不支持 或 查询"
 
-#: api/lib/cmdb/resp_format.py:138
+#: api/lib/cmdb/resp_format.py:139
 msgid "You do not have permission to operate attribute {}!"
 msgstr "您没有属性 {} 的操作权限!"
 
-#: api/lib/cmdb/resp_format.py:139
+#: api/lib/cmdb/resp_format.py:140
 msgid "You do not have permission to operate this CI!"
 msgstr "您没有该CI的操作权限!"
 
-#: api/lib/cmdb/resp_format.py:141
+#: api/lib/cmdb/resp_format.py:142
 msgid "Failed to save password: {}"
 msgstr "保存密码失败: {}"
 
-#: api/lib/cmdb/resp_format.py:142
+#: api/lib/cmdb/resp_format.py:143
 msgid "Failed to get password: {}"
 msgstr "获取密码失败: {}"
 
-#: api/lib/cmdb/resp_format.py:144
+#: api/lib/cmdb/resp_format.py:145
 msgid "Scheduling time format error"
 msgstr "{}格式错误，应该为：%Y-%m-%d %H:%M:%S"
 
-#: api/lib/cmdb/resp_format.py:145
+#: api/lib/cmdb/resp_format.py:146
 msgid "CMDB data reconciliation results"
 msgstr ""
 
-#: api/lib/cmdb/resp_format.py:146
+#: api/lib/cmdb/resp_format.py:147
 msgid "Number of {} illegal: {}"
 msgstr ""
 
-#: api/lib/cmdb/resp_format.py:148
+#: api/lib/cmdb/resp_format.py:149
 msgid "Topology view {} already exists"
 msgstr "拓扑视图 {} 已经存在"
 
-#: api/lib/cmdb/resp_format.py:149
+#: api/lib/cmdb/resp_format.py:150
 msgid "Topology group {} already exists"
 msgstr "拓扑视图分组 {} 已经存在"
 
-#: api/lib/cmdb/resp_format.py:151
+#: api/lib/cmdb/resp_format.py:152
 msgid "The group cannot be deleted because the topology view already exists"
 msgstr "因为该分组下定义了拓扑视图，不能删除"
 

cmdb-api/api/views/cmdb/auto_discovery.py

@@ -1,6 +1,7 @@
 # -*- coding:utf-8 -*-
-
+import copy
 import json
+import uuid
 from io import BytesIO
 
 from flask import abort
@@ -10,15 +11,19 @@ from flask_login import current_user
 
 from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCICRUD
 from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeCRUD
+from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeRelationCRUD
+from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCounterCRUD
+from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryExecHistoryCRUD
 from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryHTTPManager
 from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryRuleCRUD
+from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryRuleSyncHistoryCRUD
 from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoverySNMPManager
 from api.lib.cmdb.auto_discovery.const import DEFAULT_HTTP
 from api.lib.cmdb.const import PermEnum
 from api.lib.cmdb.const import ResourceTypeEnum
 from api.lib.cmdb.resp_format import ErrFormat
 from api.lib.cmdb.search import SearchError
-from api.lib.cmdb.search.ci import search
+from api.lib.cmdb.search.ci import search as ci_search
 from api.lib.decorator import args_required
 from api.lib.decorator import args_validate
 from api.lib.perm.acl.acl import has_perm_from_args
@@ -37,14 +42,19 @@ class AutoDiscoveryRuleView(APIView):
 
         rebuild = False
         exists = {i['name'] for i in res}
-        for i in DEFAULT_HTTP:
+        for i in copy.deepcopy(DEFAULT_HTTP):
             if i['name'] not in exists:
+                i.pop('en', None)
                 AutoDiscoveryRuleCRUD().add(**i)
                 rebuild = True
 
         if rebuild:
             _, res = AutoDiscoveryRuleCRUD.search(page=1, page_size=100000, **request.values)
 
+        for i in res:
+            if i['type'] == 'http':
+                i['resources'] = AutoDiscoveryHTTPManager().get_resources(i['name'])
+
         return self.jsonify(res)
 
     @args_required("name", value_required=True)
@@ -98,7 +108,8 @@ class AutoDiscoveryRuleTemplateFileView(APIView):
 
 
 class AutoDiscoveryRuleHTTPView(APIView):
-    url_prefix = ("/adr/http/<string:name>/categories", "/adr/http/<string:name>/attributes",
+    url_prefix = ("/adr/http/<string:name>/categories",
+                  "/adr/http/<string:name>/attributes",
                   "/adr/snmp/<string:name>/attributes")
 
     def get(self, name):
@@ -106,16 +117,21 @@ class AutoDiscoveryRuleHTTPView(APIView):
             return self.jsonify(AutoDiscoverySNMPManager.get_attributes())
 
         if "attributes" in request.url:
-            category = request.values.get('category')
+            resource = request.values.get('resource')
-            return self.jsonify(AutoDiscoveryHTTPManager.get_attributes(name, category))
+            return self.jsonify(AutoDiscoveryHTTPManager.get_attributes(name, resource))
 
         return self.jsonify(AutoDiscoveryHTTPManager.get_categories(name))
 
 
 class AutoDiscoveryCITypeView(APIView):
-    url_prefix = ("/adt/ci_types/<int:type_id>", "/adt/<int:adt_id>")
+    url_prefix = ("/adt/ci_types/<int:type_id>",
+                  "/adt/ci_types/<int:type_id>/attributes",
+                  "/adt/<int:adt_id>")
 
     def get(self, type_id):
+        if "attributes" in request.url:
+            return self.jsonify(AutoDiscoveryCITypeCRUD.get_ad_attributes(type_id))
+
         _, res = AutoDiscoveryCITypeCRUD.search(page=1, page_size=100000, type_id=type_id, **request.values)
         for i in res:
             if isinstance(i.get("extra_option"), dict) and i['extra_option'].get('secret'):
@@ -146,6 +162,27 @@ class AutoDiscoveryCITypeView(APIView):
         return self.jsonify(adt_id=adt_id)
 
 
+class AutoDiscoveryCITypeRelationView(APIView):
+    url_prefix = ("/adt/ci_types/<int:type_id>/relations", "/adt/relations/<int:_id>")
+
+    def get(self, type_id):
+        _, res = AutoDiscoveryCITypeRelationCRUD.search(page=1, page_size=100000, ad_type_id=type_id, **request.values)
+
+        return self.jsonify(res)
+
+    @args_required("relations")
+    def post(self, type_id):
+        return self.jsonify(AutoDiscoveryCITypeRelationCRUD().upsert(type_id, request.values['relations']))
+
+    def put(self):
+        return self.post()
+
+    def delete(self, _id):
+        AutoDiscoveryCITypeRelationCRUD().delete(_id)
+
+        return self.jsonify(id=_id)
+
+
 class AutoDiscoveryCIView(APIView):
     url_prefix = ("/adc", "/adc/<int:adc_id>", "/adc/ci_types/<int:type_id>/attributes", "/adc/ci_types")
 
@@ -220,9 +257,8 @@ class AutoDiscoveryRuleSyncView(APIView):
         oneagent_id = request.values.get('oneagent_id')
         last_update_at = request.values.get('last_update_at')
 
-        query = "{},oneagent_id:{}".format(oneagent_name, oneagent_id)
+        query = "oneagent_id:{}".format(oneagent_id)
-        current_app.logger.info(query)
+        s = ci_search(query)
-        s = search(query)
         try:
             response, _, _, _, _, _ = s.search()
         except SearchError as e:
@@ -230,7 +266,77 @@ class AutoDiscoveryRuleSyncView(APIView):
             current_app.logger.error(traceback.format_exc())
             return abort(400, str(e))
 
-        ci_id = response and response[0]["_id"]
+        for res in response:
-        rules, last_update_at = AutoDiscoveryCITypeCRUD.get(ci_id, oneagent_id, last_update_at)
+            if res.get('{}_name'.format(res['ci_type'])) == oneagent_name or oneagent_name == res.get('oneagent_name'):
+                ci_id = res["_id"]
+                rules, last_update_at = AutoDiscoveryCITypeCRUD.get(ci_id, oneagent_id, oneagent_name, last_update_at)
+
+                return self.jsonify(rules=rules, last_update_at=last_update_at)
+
+        rules, last_update_at = AutoDiscoveryCITypeCRUD.get(None, oneagent_id, oneagent_name, last_update_at)
 
         return self.jsonify(rules=rules, last_update_at=last_update_at)
+
+
+class AutoDiscoveryRuleSyncHistoryView(APIView):
+    url_prefix = ("/adt/<int:adt_id>/sync/histories",)
+
+    def get(self, adt_id):
+        page = get_page(request.values.pop('page', 1))
+        page_size = get_page_size(request.values.pop('page_size', None))
+        numfound, res = AutoDiscoveryRuleSyncHistoryCRUD.search(page=page,
+                                                                page_size=page_size,
+                                                                adt_id=adt_id,
+                                                                **request.values)
+
+        return self.jsonify(page=page,
+                            page_size=page_size,
+                            numfound=numfound,
+                            total=len(res),
+                            result=res)
+
+
+class AutoDiscoveryTestView(APIView):
+    url_prefix = ("/adt/<int:adt_id>/test", "/adt/test/<string:exec_id>/result")
+
+    def get(self, exec_id):
+        return self.jsonify(stdout="1\n2\n3", exec_id=exec_id)
+
+    def post(self, adt_id):
+        return self.jsonify(exec_id=uuid.uuid4().hex)
+
+
+class AutoDiscoveryExecHistoryView(APIView):
+    url_prefix = ("/adc/exec/histories",)
+
+    @args_required('type_id')
+    def get(self):
+        page = get_page(request.values.pop('page', 1))
+        page_size = get_page_size(request.values.pop('page_size', None))
+        numfound, res = AutoDiscoveryExecHistoryCRUD.search(page=page,
+                                                            page_size=page_size,
+                                                            **request.values)
+
+        return self.jsonify(page=page,
+                            page_size=page_size,
+                            numfound=numfound,
+                            total=len(res),
+                            result=res)
+
+    @args_required('type_id')
+    @args_required('stdout')
+    def post(self):
+        AutoDiscoveryExecHistoryCRUD().add(type_id=request.values.get('type_id'),
+                                           stdout=request.values.get('stdout'))
+
+        return self.jsonify(code=200)
+
+
+class AutoDiscoveryCounterView(APIView):
+    url_prefix = ("/adc/counter",)
+
+    @args_required('type_id')
+    def get(self):
+        type_id = request.values.get('type_id')
+
+        return self.jsonify(AutoDiscoveryCounterCRUD().get(type_id))