perf(api): auto discovery (#582)

cmdb-api/api/lib/cmdb/attribute.py

@@ -229,7 +229,7 @@ class AttributeManager(object):
         is_choice = True if choice_value or kwargs.get('choice_web_hook') or kwargs.get('choice_other') else False
 
         name = kwargs.pop("name")
-        if name in BUILTIN_KEYWORDS:
+        if name in BUILTIN_KEYWORDS or kwargs.get('alias') in BUILTIN_KEYWORDS:
             return abort(400, ErrFormat.attribute_name_cannot_be_builtin)
 
         while kwargs.get('choice_other'):

cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py

@@ -18,12 +18,10 @@ from api.lib.cmdb.cache import AutoDiscoveryMappingCache
 from api.lib.cmdb.cache import CITypeAttributeCache
 from api.lib.cmdb.cache import CITypeCache
 from api.lib.cmdb.ci import CIManager
-from api.lib.cmdb.ci import CIRelationManager
 from api.lib.cmdb.ci_type import CITypeGroupManager
 from api.lib.cmdb.const import AutoDiscoveryType
 from api.lib.cmdb.const import CMDB_QUEUE
 from api.lib.cmdb.const import PermEnum
-from api.lib.cmdb.const import RelationSourceEnum
 from api.lib.cmdb.const import ResourceTypeEnum
 from api.lib.cmdb.custom_dashboard import SystemConfigManager
 from api.lib.cmdb.resp_format import ErrFormat
@@ -35,6 +33,7 @@ from api.lib.perm.acl.acl import ACLManager
 from api.lib.perm.acl.acl import is_app_admin
 from api.lib.perm.acl.acl import validate_permission
 from api.lib.utils import AESCrypto
+from api.models.cmdb import AutoDiscoveryAccount
 from api.models.cmdb import AutoDiscoveryCI
 from api.models.cmdb import AutoDiscoveryCIType
 from api.models.cmdb import AutoDiscoveryCITypeRelation
@@ -42,6 +41,7 @@ from api.models.cmdb import AutoDiscoveryCounter
 from api.models.cmdb import AutoDiscoveryExecHistory
 from api.models.cmdb import AutoDiscoveryRule
 from api.models.cmdb import AutoDiscoveryRuleSyncHistory
+from api.tasks.cmdb import build_relations_for_ad_accept
 from api.tasks.cmdb import write_ad_rule_sync_history
 
 PWD = os.path.abspath(os.path.dirname(__file__))
@@ -226,14 +226,14 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
             adr = AutoDiscoveryRuleCRUD.get_by_id(adt.adr_id)
             if not adr:
                 continue
-            if adr.type == "http":
+            if adr.type == AutoDiscoveryType.HTTP:
                 for i in DEFAULT_INNER:
                     if adr.name == i['name']:
                         attrs = AutoDiscoveryHTTPManager.get_attributes(
                             i['en'], (adt.extra_option or {}).get('category')) or []
                         result.extend([i.get('name') for i in attrs])
                         break
-            elif adr.type == "snmp":
+            elif adr.type == AutoDiscoveryType.SNMP:
                 attributes = AutoDiscoverySNMPManager.get_attributes()
                 result.extend([i.get('name') for i in (attributes or [])])
             else:
@@ -243,6 +243,14 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
 
     @classmethod
     def get(cls, ci_id, oneagent_id, oneagent_name, last_update_at=None):
+        """
+        OneAgent sync rules
+        :param ci_id:
+        :param oneagent_id:
+        :param oneagent_name:
+        :param last_update_at:
+        :return:
+        """
         result = []
         rules = cls.cls.get_by(to_dict=True)
 
@@ -250,17 +258,14 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
             if not rule['enabled']:
                 continue
 
-            if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('secret'):
+            if isinstance(rule.get("extra_option"), dict):
-                if not (current_user.username in PRIVILEGED_USERS or current_user.uid == rule['uid']):
+                decrypt_account(rule['extra_option'], rule['uid'])
-                    rule['extra_option'].pop('secret', None)
-                else:
-                    rule['extra_option']['secret'] = AESCrypto.decrypt(rule['extra_option']['secret'])
 
-            if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('password'):
+                if rule['extra_option'].get('_reference'):
-                if not (current_user.username in PRIVILEGED_USERS or current_user.uid == rule['uid']):
                     rule['extra_option'].pop('password', None)
-                else:
+                    rule['extra_option'].pop('secret', None)
-                    rule['extra_option']['password'] = AESCrypto.decrypt(rule['extra_option']['password'])
+                    rule['extra_option'].update(
+                        AutoDiscoveryAccountCRUD().get_config_by_id(rule['extra_option']['_reference']))
 
             if oneagent_id and rule['agent_id'] == oneagent_id:
                 result.append(rule)
@@ -364,7 +369,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
         if kwargs.get('adr_id'):
             adr = AutoDiscoveryRule.get_by_id(kwargs['adr_id']) or abort(
                 404, ErrFormat.adr_not_found.format("id={}".format(kwargs['adr_id'])))
-            if adr.type == "http":
+            if adr.type == AutoDiscoveryType.HTTP:
                 kwargs.setdefault('extra_option', dict())
                 en_name = None
                 for i in DEFAULT_INNER:
@@ -379,13 +384,16 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
                             kwargs["extra_option"]["provider"] = en_name
                             break
 
+            if adr.type == AutoDiscoveryType.COMPONENTS and kwargs.get('extra_option'):
+                for i in DEFAULT_INNER:
+                    if i['name'] == adr.name:
+                        kwargs['extra_option']['collect_key'] = i['option'].get('collect_key')
+                        break
+
         if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
             kwargs = check_plugin_script(**kwargs)
 
-        if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'):
+        encrypt_account(kwargs.get('extra_option'))
-            kwargs['extra_option']['secret'] = AESCrypto.encrypt(kwargs['extra_option']['secret'])
-        if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('password'):
-            kwargs['extra_option']['password'] = AESCrypto.encrypt(kwargs['extra_option']['password'])
 
         ci_type = CITypeCache.get(kwargs['type_id'])
         unique = AttributeCache.get(ci_type.unique_id)
@@ -403,7 +411,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
 
         adr = AutoDiscoveryRule.get_by_id(existed.adr_id) or abort(
             404, ErrFormat.adr_not_found.format("id={}".format(existed.adr_id)))
-        if adr.type == "http":
+        if adr.type == AutoDiscoveryType.HTTP:
             kwargs.setdefault('extra_option', dict())
             en_name = None
             for i in DEFAULT_INNER:
@@ -418,6 +426,12 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
                         kwargs["extra_option"]["provider"] = en_name
                         break
 
+        if adr.type == AutoDiscoveryType.COMPONENTS and kwargs.get('extra_option'):
+            for i in DEFAULT_INNER:
+                if i['name'] == adr.name:
+                    kwargs['extra_option']['collect_key'] = i['option'].get('collect_key')
+                    break
+
         if 'attributes' in kwargs:
             self.__valid_exec_target(kwargs.get('agent_id'), kwargs.get('query_expr'))
 
@@ -441,13 +455,10 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
         if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
             kwargs = check_plugin_script(**kwargs)
 
-        if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'):
+        encrypt_account(kwargs.get('extra_option'))
-            kwargs['extra_option']['secret'] = AESCrypto.encrypt(kwargs['extra_option']['secret'])
-        if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('password'):
-            kwargs['extra_option']['password'] = AESCrypto.encrypt(kwargs['extra_option']['password'])
 
         inst = self._can_update(_id=_id, **kwargs)
-        if len(kwargs) == 1 and 'enabled' in kwargs: # enable or disable
+        if len(kwargs) == 1 and 'enabled' in kwargs:  # enable or disable
             pass
         elif inst.agent_id != kwargs.get('agent_id') or inst.query_expr != kwargs.get('query_expr'):
             for item in AutoDiscoveryRuleSyncHistory.get_by(adt_id=inst.id, to_dict=False):
@@ -688,9 +699,11 @@ class AutoDiscoveryCICRUD(DBMixin):
         adt = AutoDiscoveryCITypeCRUD.get_by_id(adc.adt_id) or abort(404, ErrFormat.adt_not_found)
 
         ci_id = None
-        if adt.attributes:
+
-            ci_dict = {adt.attributes[k]: None if not v and isinstance(v, (list, dict)) else v
+        ad_key2attr = adt.attributes or {}
-                       for k, v in adc.instance.items() if k in adt.attributes}
+        if ad_key2attr:
+            ci_dict = {ad_key2attr[k]: None if not v and isinstance(v, (list, dict)) else v
+                       for k, v in adc.instance.items() if k in ad_key2attr}
             extra_option = adt.extra_option or {}
             mapping, path_mapping = AutoDiscoveryHTTPManager.get_predefined_value_mapping(
                 extra_option.get('provider'), extra_option.get('category'))
@@ -703,37 +716,7 @@ class AutoDiscoveryCICRUD(DBMixin):
             AutoDiscoveryExecHistoryCRUD().add(type_id=adt.type_id,
                                                stdout="accept resource: {}".format(adc.unique_value))
 
-        relation_ads = AutoDiscoveryCITypeRelation.get_by(ad_type_id=adt.type_id, to_dict=False)
+        build_relations_for_ad_accept.apply_async(args=(adc.to_dict(), ci_id, ad_key2attr), queue=CMDB_QUEUE)
-        for r_adt in relation_ads:
-            ad_key = r_adt.ad_key
-            if not adc.instance.get(ad_key):
-                continue
-
-            ad_key_values = [adc.instance.get(ad_key)] if not isinstance(
-                adc.instance.get(ad_key), list) else adc.instance.get(ad_key)
-            for ad_key_value in ad_key_values:
-                query = "_type:{},{}:{}".format(r_adt.peer_type_id, r_adt.peer_attr_id, ad_key_value)
-                s = ci_search(query, use_ci_filter=False, count=1000000)
-                try:
-                    response, _, _, _, _, _ = s.search()
-                except SearchError as e:
-                    current_app.logger.warning(e)
-                    return abort(400, str(e))
-
-                for relation_ci in response:
-                    relation_ci_id = relation_ci['_id']
-                    try:
-                        CIRelationManager.add(ci_id, relation_ci_id,
-                                              valid=False,
-                                              source=RelationSourceEnum.AUTO_DISCOVERY)
-
-                    except:
-                        try:
-                            CIRelationManager.add(relation_ci_id, ci_id,
-                                                  valid=False,
-                                                  source=RelationSourceEnum.AUTO_DISCOVERY)
-                        except:
-                            pass
 
         adc.update(is_accept=True,
                    accept_by=nickname or current_user.nickname,
@@ -893,3 +876,121 @@ class AutoDiscoveryCounterCRUD(DBMixin):
 
     def _can_delete(self, **kwargs):
         pass
+
+
+def encrypt_account(config):
+    if isinstance(config, dict):
+        if config.get('secret'):
+            config['secret'] = AESCrypto.encrypt(config['secret'])
+        if config.get('password'):
+            config['password'] = AESCrypto.encrypt(config['password'])
+
+
+def decrypt_account(config, uid):
+    if isinstance(config, dict):
+        if config.get('password'):
+            if not (current_user.username in PRIVILEGED_USERS or current_user.uid == uid):
+                config.pop('password', None)
+            else:
+                try:
+                    config['password'] = AESCrypto.decrypt(config['password'])
+                except Exception as e:
+                    current_app.logger.error('decrypt account failed: {}'.format(e))
+
+        if config.get('secret'):
+            if not (current_user.username in PRIVILEGED_USERS or current_user.uid == uid):
+                config.pop('secret', None)
+            else:
+                try:
+                    config['secret'] = AESCrypto.decrypt(config['secret'])
+                except Exception as e:
+                    current_app.logger.error('decrypt account failed: {}'.format(e))
+
+
+class AutoDiscoveryAccountCRUD(DBMixin):
+    cls = AutoDiscoveryAccount
+
+    def get(self, adr_id):
+        res = self.cls.get_by(adr_id=adr_id, to_dict=True)
+
+        for i in res:
+            decrypt_account(i.get('config'), i['uid'])
+
+        return res
+
+    def get_config_by_id(self, _id):
+        res = self.cls.get_by_id(_id)
+        if not res:
+            return {}
+
+        config = res.to_dict().get('config') or {}
+
+        decrypt_account(config, res.uid)
+
+        return config
+
+    def _can_add(self, **kwargs):
+        encrypt_account(kwargs.get('config'))
+
+        kwargs['uid'] = current_user.uid
+
+        return kwargs
+
+    def upsert(self, adr_id, accounts):
+        existed_all = self.cls.get_by(adr_id=adr_id, to_dict=False)
+        account_names = {i['name'] for i in accounts}
+
+        name_changed = dict()
+        for account in accounts:
+            existed = None
+            if account.get('id'):
+                existed = self.cls.get_by_id(account.get('id'))
+                if existed is None:
+                    continue
+
+                account.pop('id')
+                name_changed[existed.name] = account.get('name')
+            else:
+                account = self._can_add(**account)
+
+            if existed is not None:
+                if current_user.uid == existed.uid:
+                    config = copy.deepcopy(existed.config) or {}
+                    config.update(account.get('config') or {})
+                    account['config'] = config
+                    existed.update(**account)
+            else:
+                self.cls.create(adr_id=adr_id, **account)
+
+        for item in existed_all:
+            if name_changed.get(item.name, item.name) not in account_names:
+                if current_user.uid == item.uid:
+                    item.soft_delete()
+
+    def _can_update(self, **kwargs):
+        existed = self.cls.get_by_id(kwargs['_id']) or abort(404, ErrFormat.not_found)
+
+        if isinstance(kwargs.get('config'), dict) and kwargs['config'].get('secret'):
+            if current_user.uid != existed.uid:
+                return abort(403, ErrFormat.adt_secret_no_permission)
+        if isinstance(kwargs.get('config'), dict) and kwargs['config'].get('password'):
+            if current_user.uid != existed.uid:
+                return abort(403, ErrFormat.adt_secret_no_permission)
+
+        return existed
+
+    def update(self, _id, **kwargs):
+
+        if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
+            kwargs = check_plugin_script(**kwargs)
+
+        encrypt_account(kwargs.get('config'))
+
+        inst = self._can_update(_id=_id, **kwargs)
+
+        obj = inst.update(_id=_id, filter_none=False, **kwargs)
+
+        return obj
+
+    def _can_delete(self, **kwargs):
+        pass

cmdb-api/api/lib/cmdb/auto_discovery/const.py

@@ -19,10 +19,21 @@ DEFAULT_INNER = [
     dict(name="KVM", en="kvm", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
          option={'icon': {'name': 'ops-KVM'}, "category": "private_cloud", "en": "kvm"}),
 
+
     dict(name="Nginx", en="nginx", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
-         option={'icon': {'name': 'caise-nginx'}, "en": "nginx"}),
+         option={'icon': {'name': 'caise-nginx'}, "en": "nginx", "collect_key": "nginx"}),
+    dict(name="Apache", en="apache", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-apache'}, "en": "apache", "collect_key": "apache"}),
+    dict(name="Tomcat", en="tomcat", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-tomcat'}, "en": "tomcat", "collect_key": "tomcat"}),
+    dict(name="MySQL", en="mysql", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-mySQL'}, "en": "mysql", "collect_key": "mysql"}),
+    dict(name="MSSQL", en="mssql", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+             option={'icon': {'name': 'caise-SQLServer'}, "en": "mssql", "collect_key": "sqlserver"}),
+    dict(name="Oracle", en="oracle", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-oracle'}, "en": "oracle", "collect_key": "oracle"}),
     dict(name="Redis", en="redis", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
-         option={'icon': {'name': 'caise-redis'}, "en": "redis"}),
+         option={'icon': {'name': 'caise-redis'}, "en": "redis", "collect_key": "redis"}),
 
     dict(name="交换机", type=AutoDiscoveryType.SNMP, is_inner=True, is_plugin=False,
          option={'icon': {'name': 'caise-jiaohuanji'}}),
@@ -294,7 +305,7 @@ CLOUD_MAP = {
             "category": "其他",
             "items": ["资源池", "数据中心", "文件夹"],
             "map": {
-                "资源池": "templates/vsphere_datastore.json",
+                "资源池": "templates/vsphere_pool.json",
                 "数据中心": "templates/vsphere_datacenter.json",
                 "文件夹": "templates/vsphere_folder.json",
             },

cmdb-api/api/lib/cmdb/ci.py

@@ -319,8 +319,8 @@ class CIManager(object):
             400, ErrFormat.unique_value_not_found.format("unique_id={}".format(ci_type.unique_id)))
 
         unique_value = None
-        if not (unique_key.default and unique_key.default.get('default') == AttributeDefaultValueEnum.AUTO_INC_ID and
+        # primary key is not auto inc id
-                not ci_dict.get(unique_key.name)):  # primary key is not auto inc id
+        if not (unique_key.default and unique_key.default.get('default') == AttributeDefaultValueEnum.AUTO_INC_ID):
             unique_value = ci_dict.get(unique_key.name) or ci_dict.get(unique_key.alias) or ci_dict.get(unique_key.id)
             unique_value = unique_value or abort(400, ErrFormat.unique_key_required.format(unique_key.name))
 

cmdb-api/api/lib/cmdb/ci_type.py

@@ -1244,17 +1244,16 @@ class CITypeAttributeGroupManager(object):
         if isinstance(_from, int):
             from_group = CITypeAttributeGroup.get_by_id(_from)
         else:
-            from_group = CITypeAttributeGroup.get_by(name=_from, first=True, to_dict=False)
+            from_group = CITypeAttributeGroup.get_by(name=_from, type_id=type_id, first=True, to_dict=False)
         from_group or abort(404, ErrFormat.ci_type_attribute_group_not_found.format("id={}".format(_from)))
 
         if isinstance(_to, int):
             to_group = CITypeAttributeGroup.get_by_id(_to)
         else:
-            to_group = CITypeAttributeGroup.get_by(name=_to, first=True, to_dict=False)
+            to_group = CITypeAttributeGroup.get_by(name=_to, type_id=type_id, first=True, to_dict=False)
         to_group or abort(404, ErrFormat.ci_type_attribute_group_not_found.format("id={}".format(_to)))
 
         from_order, to_order = from_group.order, to_group.order
-
         from_group.update(order=to_order)
         to_group.update(order=from_order)
 
@@ -1541,6 +1540,9 @@ class CITypeTemplateManager(object):
                 if ((i.extra_option or {}).get('alias') or None) == (
                         (rule.get('extra_option') or {}).get('alias') or None):
                     existed = True
+                    rule.pop('extra_option', None)
+                    rule.pop('enabled', None)
+                    rule.pop('cron', None)
                     AutoDiscoveryCITypeCRUD().update(i.id, **rule)
                     break
 
@@ -1698,6 +1700,9 @@ class CITypeTemplateManager(object):
         for r in ad_rules:
             r = r.to_dict()
 
+            if r.get('extra_option') and '_reference' in r['extra_option']:
+                r['extra_option'].pop('_reference')
+
             r['type_name'] = type_id2name.get(r.pop('type_id'))
             if r.get('adr_id'):
                 adr = AutoDiscoveryRuleCRUD.get_by_id(r.pop('adr_id'))

cmdb-api/api/lib/cmdb/const.py

@@ -118,7 +118,7 @@ REDIS_PREFIX_CI = "ONE_CMDB"
 REDIS_PREFIX_CI_RELATION = "CMDB_CI_RELATION"
 REDIS_PREFIX_CI_RELATION2 = "CMDB_CI_RELATION2"
 
-BUILTIN_KEYWORDS = {'id', '_id', 'ci_id', 'type', '_type', 'ci_type'}
+BUILTIN_KEYWORDS = {'id', '_id', 'ci_id', 'type', '_type', 'ci_type', 'ticket_id'}
 
 L_TYPE = None
 L_CI = None

cmdb-api/api/lib/cmdb/resp_format.py

@@ -35,7 +35,7 @@ class ErrFormat(CommonErrFormat):
         "Only creators and administrators are allowed to delete attributes!")  # 目前只允许 属性创建人、管理员 删除属性!
     # 属性字段名不能是内置字段: id, _id, ci_id, type, _type, ci_type
     attribute_name_cannot_be_builtin = _l(
-        "Attribute field names cannot be built-in fields: id, _id, ci_id, type, _type, ci_type")
+        "Attribute field names cannot be built-in fields: id, _id, ci_id, type, _type, ci_type, ticket_id")
     attribute_choice_other_invalid = _l(
         "Predefined value: Other model request parameters are illegal!")  # 预定义值: 其他模型请求参数不合法!
 

cmdb-api/api/lib/mixin.py

@@ -1,21 +1,19 @@
 # -*- coding:utf-8 -*-
 
 
-from flask import abort
 from sqlalchemy import func
 
 from api.extensions import db
 from api.lib.utils import get_page
 from api.lib.utils import get_page_size
 
-__author__ = 'pycook'
-
 
 class DBMixin(object):
     cls = None
 
     @classmethod
-    def search(cls, page, page_size, fl=None, only_query=False, reverse=False, count_query=False, **kwargs):
+    def search(cls, page, page_size, fl=None, only_query=False, reverse=False, count_query=False,
+               last_size=None, **kwargs):
         page = get_page(page)
         page_size = get_page_size(page_size)
         if fl is None:
@@ -47,14 +45,15 @@ class DBMixin(object):
             return _query, query
 
         numfound = query.count()
-        return numfound, [i.to_dict() if fl is None else getattr(i, '_asdict')()
+        if not last_size:
-                          for i in query.offset((page - 1) * page_size).limit(page_size)]
+            return numfound, [i.to_dict() if fl is None else getattr(i, '_asdict')()
-
+                              for i in query.offset((page - 1) * page_size).limit(page_size)]
-    def _must_be_required(self, _id):
+        else:
-        existed = self.cls.get_by_id(_id)
+            offset = numfound - last_size
-        existed or abort(404, "Factor [{}] does not exist".format(_id))
+            if offset < 0:
-
+                offset = 0
-        return existed
+            return numfound, [i.to_dict() if fl is None else getattr(i, '_asdict')()
+                              for i in query.offset(offset).limit(last_size)]
 
     def _can_add(self, **kwargs):
         raise NotImplementedError

cmdb-api/api/models/cmdb.py

@@ -636,6 +636,15 @@ class AutoDiscoveryCounter(Model2):
     last_week_count = db.Column(db.Integer, default=0)
 
 
+class AutoDiscoveryAccount(Model):
+    __tablename__ = "c_ad_accounts"
+
+    uid = db.Column(db.Integer, index=True)
+    name = db.Column(db.String(64))
+    adr_id = db.Column(db.Integer, db.ForeignKey('c_ad_rules.id'))
+    config = db.Column(db.JSON)
+
+
 class CIFilterPerms(Model):
     __tablename__ = "c_ci_filter_perms"
 

cmdb-api/api/tasks/cmdb.py

@@ -1,9 +1,8 @@
 # -*- coding:utf-8 -*- 
 
 
-import json
 import datetime
-
+import json
 import redis_lock
 from flask import current_app
 from flask_login import login_user
@@ -13,21 +12,24 @@ from api.extensions import celery
 from api.extensions import db
 from api.extensions import es
 from api.extensions import rd
+from api.lib.cmdb.cache import AttributeCache
 from api.lib.cmdb.cache import CITypeAttributesCache
 from api.lib.cmdb.const import CMDB_QUEUE
 from api.lib.cmdb.const import REDIS_PREFIX_CI
 from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
 from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION2
+from api.lib.cmdb.const import RelationSourceEnum
 from api.lib.cmdb.perms import CIFilterPermsCRUD
 from api.lib.decorator import flush_db
 from api.lib.decorator import reconnect_db
 from api.lib.perm.acl.cache import UserCache
 from api.lib.utils import handle_arg_list
+from api.models.cmdb import AutoDiscoveryCI
+from api.models.cmdb import AutoDiscoveryCIType
+from api.models.cmdb import AutoDiscoveryCITypeRelation
 from api.models.cmdb import CI
 from api.models.cmdb import CIRelation
 from api.models.cmdb import CITypeAttribute
-from api.models.cmdb import AutoDiscoveryCI
-from api.models.cmdb import AutoDiscoveryCIType
 
 
 @celery.task(name="cmdb.ci_cache", queue=CMDB_QUEUE)
@@ -277,3 +279,75 @@ def write_ad_rule_sync_history(rules, oneagent_id, oneagent_name, sync_at):
     except Exception as e:
         current_app.logger.error("write auto discovery rule sync history failed: {}".format(e))
         db.session.rollback()
+
+
+@celery.task(name="cmdb.build_relations_for_ad_accept", queue=CMDB_QUEUE)
+@reconnect_db
+def build_relations_for_ad_accept(adc, ci_id, ad_key2attr):
+    from api.lib.cmdb.ci import CIRelationManager
+    from api.lib.cmdb.search import SearchError
+    from api.lib.cmdb.search.ci import search as ci_search
+
+    current_app.test_request_context().push()
+    login_user(UserCache.get('worker'))
+
+    relation_ads = AutoDiscoveryCITypeRelation.get_by(ad_type_id=adc['type_id'], to_dict=False)
+    for r_adt in relation_ads:
+        ad_key = r_adt.ad_key
+        if not adc['instance'].get(ad_key):
+            continue
+
+        ad_key_values = [adc['instance'].get(ad_key)] if not isinstance(
+            adc['instance'].get(ad_key), list) else adc['instance'].get(ad_key)
+        for ad_key_value in ad_key_values:
+            query = "_type:{},{}:{}".format(r_adt.peer_type_id, r_adt.peer_attr_id, ad_key_value)
+            s = ci_search(query, use_ci_filter=False, count=1000000)
+            try:
+                response, _, _, _, _, _ = s.search()
+            except SearchError as e:
+                current_app.logger.error("build_relations_for_ad_accept failed: {}".format(e))
+                return
+
+            for relation_ci in response:
+                relation_ci_id = relation_ci['_id']
+                try:
+                    CIRelationManager.add(ci_id, relation_ci_id,
+                                          valid=False,
+                                          source=RelationSourceEnum.AUTO_DISCOVERY)
+
+                except:
+                    try:
+                        CIRelationManager.add(relation_ci_id, ci_id,
+                                              valid=False,
+                                              source=RelationSourceEnum.AUTO_DISCOVERY)
+                    except:
+                        pass
+
+    # build relations in reverse
+    relation_ads = AutoDiscoveryCITypeRelation.get_by(peer_type_id=adc['type_id'], to_dict=False)
+    attr2ad_key = {v: k for k, v in ad_key2attr.items()}
+    for r_adt in relation_ads:
+        attr = AttributeCache.get(r_adt.peer_attr_id)
+        ad_key = attr2ad_key.get(attr and attr.name)
+        if not ad_key:
+            continue
+
+        ad_value = adc['instance'].get(ad_key)
+        peer_ad_key = r_adt.ad_key
+        peer_instances = AutoDiscoveryCI.get_by(type_id=r_adt.ad_type_id, to_dict=False)
+        for peer_instance in peer_instances:
+            peer_ad_values = peer_instance.instance.get(peer_ad_key)
+            peer_ad_values = [peer_ad_values] if not isinstance(peer_ad_values, list) else peer_ad_values
+            if ad_value in peer_ad_values and peer_instance.ci_id:
+                try:
+                    CIRelationManager.add(peer_instance.ci_id, ci_id,
+                                          valid=False,
+                                          source=RelationSourceEnum.AUTO_DISCOVERY)
+
+                except:
+                    try:
+                        CIRelationManager.add(ci_id, peer_instance.ci_id,
+                                              valid=False,
+                                              source=RelationSourceEnum.AUTO_DISCOVERY)
+                    except:
+                        pass

cmdb-api/api/views/cmdb/auto_discovery.py

@@ -8,6 +8,7 @@ from flask import request
 from flask_login import current_user
 from io import BytesIO
 
+from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryAccountCRUD
 from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCICRUD
 from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeCRUD
 from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeRelationCRUD
@@ -20,6 +21,7 @@ from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryRuleSyncHist
 from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoverySNMPManager
 from api.lib.cmdb.auto_discovery.const import DEFAULT_INNER
 from api.lib.cmdb.auto_discovery.const import PRIVILEGED_USERS
+from api.lib.cmdb.cache import AttributeCache
 from api.lib.cmdb.const import PermEnum
 from api.lib.cmdb.const import ResourceTypeEnum
 from api.lib.cmdb.resp_format import ErrFormat
@@ -272,14 +274,16 @@ class AutoDiscoveryRuleSyncView(APIView):
         oneagent_id = request.values.get('oneagent_id')
         last_update_at = request.values.get('last_update_at')
 
-        query = "oneagent_id:{}".format(oneagent_id)
+        response = []
-        s = ci_search(query)
+        if AttributeCache.get('oneagent_id'):
-        try:
+            query = "oneagent_id:{}".format(oneagent_id)
-            response, _, _, _, _, _ = s.search()
+            s = ci_search(query)
-        except SearchError as e:
+            try:
-            import traceback
+                response, _, _, _, _, _ = s.search()
-            current_app.logger.error(traceback.format_exc())
+            except SearchError as e:
-            return abort(400, str(e))
+                import traceback
+                current_app.logger.error(traceback.format_exc())
+                return abort(400, str(e))
 
         for res in response:
             if res.get('{}_name'.format(res['ci_type'])) == oneagent_name or oneagent_name == res.get('oneagent_name'):
@@ -328,8 +332,12 @@ class AutoDiscoveryExecHistoryView(APIView):
     def get(self):
         page = get_page(request.values.pop('page', 1))
         page_size = get_page_size(request.values.pop('page_size', None))
+        last_size = request.values.pop('last_size', None)
+        if last_size and last_size.isdigit():
+            last_size = int(last_size)
         numfound, res = AutoDiscoveryExecHistoryCRUD.search(page=page,
                                                             page_size=page_size,
+                                                            last_size=last_size,
                                                             **request.values)
 
         return self.jsonify(page=page,
@@ -355,3 +363,31 @@ class AutoDiscoveryCounterView(APIView):
         type_id = request.values.get('type_id')
 
         return self.jsonify(AutoDiscoveryCounterCRUD().get(type_id))
+
+
+class AutoDiscoveryAccountView(APIView):
+    url_prefix = ("/adr/accounts", "/adr/accounts/<int:account_id>")
+
+    @args_required('adr_id')
+    def get(self):
+        adr_id = request.values.get('adr_id')
+
+        return self.jsonify(AutoDiscoveryAccountCRUD().get(adr_id))
+
+    @args_required('adr_id')
+    @args_required('accounts', value_required=False)
+    def post(self):
+        AutoDiscoveryAccountCRUD().upsert(**request.values)
+
+        return self.jsonify(code=200)
+
+    @args_required('config')
+    def put(self, account_id):
+        res = AutoDiscoveryAccountCRUD().update(account_id, **request.values)
+
+        return self.jsonify(res.to_dict())
+
+    def delete(self, account_id):
+        AutoDiscoveryAccountCRUD().delete(account_id)
+
+        return self.jsonify(account_id=account_id)