perf(api): auto discovery (#582)

cmdb-api/api/lib/cmdb/attribute.py

@@ -229,7 +229,7 @@ class AttributeManager(object):
         is_choice = True if choice_value or kwargs.get('choice_web_hook') or kwargs.get('choice_other') else False
 
         name = kwargs.pop("name")
-        if name in BUILTIN_KEYWORDS:
+        if name in BUILTIN_KEYWORDS or kwargs.get('alias') in BUILTIN_KEYWORDS:
             return abort(400, ErrFormat.attribute_name_cannot_be_builtin)
 
         while kwargs.get('choice_other'):

cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py

@@ -18,12 +18,10 @@ from api.lib.cmdb.cache import AutoDiscoveryMappingCache
 from api.lib.cmdb.cache import CITypeAttributeCache
 from api.lib.cmdb.cache import CITypeCache
 from api.lib.cmdb.ci import CIManager
-from api.lib.cmdb.ci import CIRelationManager
 from api.lib.cmdb.ci_type import CITypeGroupManager
 from api.lib.cmdb.const import AutoDiscoveryType
 from api.lib.cmdb.const import CMDB_QUEUE
 from api.lib.cmdb.const import PermEnum
-from api.lib.cmdb.const import RelationSourceEnum
 from api.lib.cmdb.const import ResourceTypeEnum
 from api.lib.cmdb.custom_dashboard import SystemConfigManager
 from api.lib.cmdb.resp_format import ErrFormat
@@ -35,6 +33,7 @@ from api.lib.perm.acl.acl import ACLManager
 from api.lib.perm.acl.acl import is_app_admin
 from api.lib.perm.acl.acl import validate_permission
 from api.lib.utils import AESCrypto
+from api.models.cmdb import AutoDiscoveryAccount
 from api.models.cmdb import AutoDiscoveryCI
 from api.models.cmdb import AutoDiscoveryCIType
 from api.models.cmdb import AutoDiscoveryCITypeRelation
@@ -42,6 +41,7 @@ from api.models.cmdb import AutoDiscoveryCounter
 from api.models.cmdb import AutoDiscoveryExecHistory
 from api.models.cmdb import AutoDiscoveryRule
 from api.models.cmdb import AutoDiscoveryRuleSyncHistory
+from api.tasks.cmdb import build_relations_for_ad_accept
 from api.tasks.cmdb import write_ad_rule_sync_history
 
 PWD = os.path.abspath(os.path.dirname(__file__))
@@ -226,14 +226,14 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
             adr = AutoDiscoveryRuleCRUD.get_by_id(adt.adr_id)
             if not adr:
                 continue
-            if adr.type == "http":
+            if adr.type == AutoDiscoveryType.HTTP:
                 for i in DEFAULT_INNER:
                     if adr.name == i['name']:
                         attrs = AutoDiscoveryHTTPManager.get_attributes(
                             i['en'], (adt.extra_option or {}).get('category')) or []
                         result.extend([i.get('name') for i in attrs])
                         break
-            elif adr.type == "snmp":
+            elif adr.type == AutoDiscoveryType.SNMP:
                 attributes = AutoDiscoverySNMPManager.get_attributes()
                 result.extend([i.get('name') for i in (attributes or [])])
             else:
@@ -243,6 +243,14 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
 
     @classmethod
     def get(cls, ci_id, oneagent_id, oneagent_name, last_update_at=None):
+        """
+        OneAgent sync rules
+        :param ci_id:
+        :param oneagent_id:
+        :param oneagent_name:
+        :param last_update_at:
+        :return:
+        """
         result = []
         rules = cls.cls.get_by(to_dict=True)
 
@@ -250,17 +258,14 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
             if not rule['enabled']:
                 continue
 
-            if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('secret'):
-                if not (current_user.username in PRIVILEGED_USERS or current_user.uid == rule['uid']):
-                    rule['extra_option'].pop('secret', None)
-                else:
-                    rule['extra_option']['secret'] = AESCrypto.decrypt(rule['extra_option']['secret'])
+            if isinstance(rule.get("extra_option"), dict):
+                decrypt_account(rule['extra_option'], rule['uid'])
 
-            if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('password'):
-                if not (current_user.username in PRIVILEGED_USERS or current_user.uid == rule['uid']):
+                if rule['extra_option'].get('_reference'):
                     rule['extra_option'].pop('password', None)
-                else:
-                    rule['extra_option']['password'] = AESCrypto.decrypt(rule['extra_option']['password'])
+                    rule['extra_option'].pop('secret', None)
+                    rule['extra_option'].update(
+                        AutoDiscoveryAccountCRUD().get_config_by_id(rule['extra_option']['_reference']))
 
             if oneagent_id and rule['agent_id'] == oneagent_id:
                 result.append(rule)
@@ -364,7 +369,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
         if kwargs.get('adr_id'):
             adr = AutoDiscoveryRule.get_by_id(kwargs['adr_id']) or abort(
                 404, ErrFormat.adr_not_found.format("id={}".format(kwargs['adr_id'])))
-            if adr.type == "http":
+            if adr.type == AutoDiscoveryType.HTTP:
                 kwargs.setdefault('extra_option', dict())
                 en_name = None
                 for i in DEFAULT_INNER:
@@ -379,13 +384,16 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
                             kwargs["extra_option"]["provider"] = en_name
                             break
 
+            if adr.type == AutoDiscoveryType.COMPONENTS and kwargs.get('extra_option'):
+                for i in DEFAULT_INNER:
+                    if i['name'] == adr.name:
+                        kwargs['extra_option']['collect_key'] = i['option'].get('collect_key')
+                        break
+
         if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
             kwargs = check_plugin_script(**kwargs)
 
-        if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'):
-            kwargs['extra_option']['secret'] = AESCrypto.encrypt(kwargs['extra_option']['secret'])
-        if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('password'):
-            kwargs['extra_option']['password'] = AESCrypto.encrypt(kwargs['extra_option']['password'])
+        encrypt_account(kwargs.get('extra_option'))
 
         ci_type = CITypeCache.get(kwargs['type_id'])
         unique = AttributeCache.get(ci_type.unique_id)
@@ -403,7 +411,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
 
         adr = AutoDiscoveryRule.get_by_id(existed.adr_id) or abort(
             404, ErrFormat.adr_not_found.format("id={}".format(existed.adr_id)))
-        if adr.type == "http":
+        if adr.type == AutoDiscoveryType.HTTP:
             kwargs.setdefault('extra_option', dict())
             en_name = None
             for i in DEFAULT_INNER:
@@ -418,6 +426,12 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
                         kwargs["extra_option"]["provider"] = en_name
                         break
 
+        if adr.type == AutoDiscoveryType.COMPONENTS and kwargs.get('extra_option'):
+            for i in DEFAULT_INNER:
+                if i['name'] == adr.name:
+                    kwargs['extra_option']['collect_key'] = i['option'].get('collect_key')
+                    break
+
         if 'attributes' in kwargs:
             self.__valid_exec_target(kwargs.get('agent_id'), kwargs.get('query_expr'))
 
@@ -441,13 +455,10 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
         if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
             kwargs = check_plugin_script(**kwargs)
 
-        if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'):
-            kwargs['extra_option']['secret'] = AESCrypto.encrypt(kwargs['extra_option']['secret'])
-        if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('password'):
-            kwargs['extra_option']['password'] = AESCrypto.encrypt(kwargs['extra_option']['password'])
+        encrypt_account(kwargs.get('extra_option'))
 
         inst = self._can_update(_id=_id, **kwargs)
-        if len(kwargs) == 1 and 'enabled' in kwargs: # enable or disable
+        if len(kwargs) == 1 and 'enabled' in kwargs:  # enable or disable
             pass
         elif inst.agent_id != kwargs.get('agent_id') or inst.query_expr != kwargs.get('query_expr'):
             for item in AutoDiscoveryRuleSyncHistory.get_by(adt_id=inst.id, to_dict=False):
@@ -688,9 +699,11 @@ class AutoDiscoveryCICRUD(DBMixin):
         adt = AutoDiscoveryCITypeCRUD.get_by_id(adc.adt_id) or abort(404, ErrFormat.adt_not_found)
 
         ci_id = None
-        if adt.attributes:
-            ci_dict = {adt.attributes[k]: None if not v and isinstance(v, (list, dict)) else v
-                       for k, v in adc.instance.items() if k in adt.attributes}
+
+        ad_key2attr = adt.attributes or {}
+        if ad_key2attr:
+            ci_dict = {ad_key2attr[k]: None if not v and isinstance(v, (list, dict)) else v
+                       for k, v in adc.instance.items() if k in ad_key2attr}
             extra_option = adt.extra_option or {}
             mapping, path_mapping = AutoDiscoveryHTTPManager.get_predefined_value_mapping(
                 extra_option.get('provider'), extra_option.get('category'))
@@ -703,37 +716,7 @@ class AutoDiscoveryCICRUD(DBMixin):
             AutoDiscoveryExecHistoryCRUD().add(type_id=adt.type_id,
                                                stdout="accept resource: {}".format(adc.unique_value))
 
-        relation_ads = AutoDiscoveryCITypeRelation.get_by(ad_type_id=adt.type_id, to_dict=False)
-        for r_adt in relation_ads:
-            ad_key = r_adt.ad_key
-            if not adc.instance.get(ad_key):
-                continue
-
-            ad_key_values = [adc.instance.get(ad_key)] if not isinstance(
-                adc.instance.get(ad_key), list) else adc.instance.get(ad_key)
-            for ad_key_value in ad_key_values:
-                query = "_type:{},{}:{}".format(r_adt.peer_type_id, r_adt.peer_attr_id, ad_key_value)
-                s = ci_search(query, use_ci_filter=False, count=1000000)
-                try:
-                    response, _, _, _, _, _ = s.search()
-                except SearchError as e:
-                    current_app.logger.warning(e)
-                    return abort(400, str(e))
-
-                for relation_ci in response:
-                    relation_ci_id = relation_ci['_id']
-                    try:
-                        CIRelationManager.add(ci_id, relation_ci_id,
-                                              valid=False,
-                                              source=RelationSourceEnum.AUTO_DISCOVERY)
-
-                    except:
-                        try:
-                            CIRelationManager.add(relation_ci_id, ci_id,
-                                                  valid=False,
-                                                  source=RelationSourceEnum.AUTO_DISCOVERY)
-                        except:
-                            pass
+        build_relations_for_ad_accept.apply_async(args=(adc.to_dict(), ci_id, ad_key2attr), queue=CMDB_QUEUE)
 
         adc.update(is_accept=True,
                    accept_by=nickname or current_user.nickname,
@@ -893,3 +876,121 @@ class AutoDiscoveryCounterCRUD(DBMixin):
 
     def _can_delete(self, **kwargs):
         pass
+
+
+def encrypt_account(config):
+    if isinstance(config, dict):
+        if config.get('secret'):
+            config['secret'] = AESCrypto.encrypt(config['secret'])
+        if config.get('password'):
+            config['password'] = AESCrypto.encrypt(config['password'])
+
+
+def decrypt_account(config, uid):
+    if isinstance(config, dict):
+        if config.get('password'):
+            if not (current_user.username in PRIVILEGED_USERS or current_user.uid == uid):
+                config.pop('password', None)
+            else:
+                try:
+                    config['password'] = AESCrypto.decrypt(config['password'])
+                except Exception as e:
+                    current_app.logger.error('decrypt account failed: {}'.format(e))
+
+        if config.get('secret'):
+            if not (current_user.username in PRIVILEGED_USERS or current_user.uid == uid):
+                config.pop('secret', None)
+            else:
+                try:
+                    config['secret'] = AESCrypto.decrypt(config['secret'])
+                except Exception as e:
+                    current_app.logger.error('decrypt account failed: {}'.format(e))
+
+
+class AutoDiscoveryAccountCRUD(DBMixin):
+    cls = AutoDiscoveryAccount
+
+    def get(self, adr_id):
+        res = self.cls.get_by(adr_id=adr_id, to_dict=True)
+
+        for i in res:
+            decrypt_account(i.get('config'), i['uid'])
+
+        return res
+
+    def get_config_by_id(self, _id):
+        res = self.cls.get_by_id(_id)
+        if not res:
+            return {}
+
+        config = res.to_dict().get('config') or {}
+
+        decrypt_account(config, res.uid)
+
+        return config
+
+    def _can_add(self, **kwargs):
+        encrypt_account(kwargs.get('config'))
+
+        kwargs['uid'] = current_user.uid
+
+        return kwargs
+
+    def upsert(self, adr_id, accounts):
+        existed_all = self.cls.get_by(adr_id=adr_id, to_dict=False)
+        account_names = {i['name'] for i in accounts}
+
+        name_changed = dict()
+        for account in accounts:
+            existed = None
+            if account.get('id'):
+                existed = self.cls.get_by_id(account.get('id'))
+                if existed is None:
+                    continue
+
+                account.pop('id')
+                name_changed[existed.name] = account.get('name')
+            else:
+                account = self._can_add(**account)
+
+            if existed is not None:
+                if current_user.uid == existed.uid:
+                    config = copy.deepcopy(existed.config) or {}
+                    config.update(account.get('config') or {})
+                    account['config'] = config
+                    existed.update(**account)
+            else:
+                self.cls.create(adr_id=adr_id, **account)
+
+        for item in existed_all:
+            if name_changed.get(item.name, item.name) not in account_names:
+                if current_user.uid == item.uid:
+                    item.soft_delete()
+
+    def _can_update(self, **kwargs):
+        existed = self.cls.get_by_id(kwargs['_id']) or abort(404, ErrFormat.not_found)
+
+        if isinstance(kwargs.get('config'), dict) and kwargs['config'].get('secret'):
+            if current_user.uid != existed.uid:
+                return abort(403, ErrFormat.adt_secret_no_permission)
+        if isinstance(kwargs.get('config'), dict) and kwargs['config'].get('password'):
+            if current_user.uid != existed.uid:
+                return abort(403, ErrFormat.adt_secret_no_permission)
+
+        return existed
+
+    def update(self, _id, **kwargs):
+
+        if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
+            kwargs = check_plugin_script(**kwargs)
+
+        encrypt_account(kwargs.get('config'))
+
+        inst = self._can_update(_id=_id, **kwargs)
+
+        obj = inst.update(_id=_id, filter_none=False, **kwargs)
+
+        return obj
+
+    def _can_delete(self, **kwargs):
+        pass

cmdb-api/api/lib/cmdb/auto_discovery/const.py

@@ -19,10 +19,21 @@ DEFAULT_INNER = [
     dict(name="KVM", en="kvm", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
          option={'icon': {'name': 'ops-KVM'}, "category": "private_cloud", "en": "kvm"}),
 
+
     dict(name="Nginx", en="nginx", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
-         option={'icon': {'name': 'caise-nginx'}, "en": "nginx"}),
+         option={'icon': {'name': 'caise-nginx'}, "en": "nginx", "collect_key": "nginx"}),
+    dict(name="Apache", en="apache", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-apache'}, "en": "apache", "collect_key": "apache"}),
+    dict(name="Tomcat", en="tomcat", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-tomcat'}, "en": "tomcat", "collect_key": "tomcat"}),
+    dict(name="MySQL", en="mysql", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-mySQL'}, "en": "mysql", "collect_key": "mysql"}),
+    dict(name="MSSQL", en="mssql", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+             option={'icon': {'name': 'caise-SQLServer'}, "en": "mssql", "collect_key": "sqlserver"}),
+    dict(name="Oracle", en="oracle", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-oracle'}, "en": "oracle", "collect_key": "oracle"}),
     dict(name="Redis", en="redis", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
-         option={'icon': {'name': 'caise-redis'}, "en": "redis"}),
+         option={'icon': {'name': 'caise-redis'}, "en": "redis", "collect_key": "redis"}),
 
     dict(name="交换机", type=AutoDiscoveryType.SNMP, is_inner=True, is_plugin=False,
          option={'icon': {'name': 'caise-jiaohuanji'}}),
@@ -294,7 +305,7 @@ CLOUD_MAP = {
             "category": "其他",
             "items": ["资源池", "数据中心", "文件夹"],
             "map": {
-                "资源池": "templates/vsphere_datastore.json",
+                "资源池": "templates/vsphere_pool.json",
                 "数据中心": "templates/vsphere_datacenter.json",
                 "文件夹": "templates/vsphere_folder.json",
             },

cmdb-api/api/lib/cmdb/ci.py

@@ -319,8 +319,8 @@ class CIManager(object):
             400, ErrFormat.unique_value_not_found.format("unique_id={}".format(ci_type.unique_id)))
 
         unique_value = None
-        if not (unique_key.default and unique_key.default.get('default') == AttributeDefaultValueEnum.AUTO_INC_ID and
-                not ci_dict.get(unique_key.name)):  # primary key is not auto inc id
+        # primary key is not auto inc id
+        if not (unique_key.default and unique_key.default.get('default') == AttributeDefaultValueEnum.AUTO_INC_ID):
             unique_value = ci_dict.get(unique_key.name) or ci_dict.get(unique_key.alias) or ci_dict.get(unique_key.id)
             unique_value = unique_value or abort(400, ErrFormat.unique_key_required.format(unique_key.name))
 

cmdb-api/api/lib/cmdb/ci_type.py

@@ -1244,17 +1244,16 @@ class CITypeAttributeGroupManager(object):
         if isinstance(_from, int):
             from_group = CITypeAttributeGroup.get_by_id(_from)
         else:
-            from_group = CITypeAttributeGroup.get_by(name=_from, first=True, to_dict=False)
+            from_group = CITypeAttributeGroup.get_by(name=_from, type_id=type_id, first=True, to_dict=False)
         from_group or abort(404, ErrFormat.ci_type_attribute_group_not_found.format("id={}".format(_from)))
 
         if isinstance(_to, int):
             to_group = CITypeAttributeGroup.get_by_id(_to)
         else:
-            to_group = CITypeAttributeGroup.get_by(name=_to, first=True, to_dict=False)
+            to_group = CITypeAttributeGroup.get_by(name=_to, type_id=type_id, first=True, to_dict=False)
         to_group or abort(404, ErrFormat.ci_type_attribute_group_not_found.format("id={}".format(_to)))
 
         from_order, to_order = from_group.order, to_group.order
-
         from_group.update(order=to_order)
         to_group.update(order=from_order)
 
@@ -1541,6 +1540,9 @@ class CITypeTemplateManager(object):
                 if ((i.extra_option or {}).get('alias') or None) == (
                         (rule.get('extra_option') or {}).get('alias') or None):
                     existed = True
+                    rule.pop('extra_option', None)
+                    rule.pop('enabled', None)
+                    rule.pop('cron', None)
                     AutoDiscoveryCITypeCRUD().update(i.id, **rule)
                     break
 
@@ -1698,6 +1700,9 @@ class CITypeTemplateManager(object):
         for r in ad_rules:
             r = r.to_dict()
 
+            if r.get('extra_option') and '_reference' in r['extra_option']:
+                r['extra_option'].pop('_reference')
+
             r['type_name'] = type_id2name.get(r.pop('type_id'))
             if r.get('adr_id'):
                 adr = AutoDiscoveryRuleCRUD.get_by_id(r.pop('adr_id'))

cmdb-api/api/lib/cmdb/const.py

@@ -118,7 +118,7 @@ REDIS_PREFIX_CI = "ONE_CMDB"
 REDIS_PREFIX_CI_RELATION = "CMDB_CI_RELATION"
 REDIS_PREFIX_CI_RELATION2 = "CMDB_CI_RELATION2"
 
-BUILTIN_KEYWORDS = {'id', '_id', 'ci_id', 'type', '_type', 'ci_type'}
+BUILTIN_KEYWORDS = {'id', '_id', 'ci_id', 'type', '_type', 'ci_type', 'ticket_id'}
 
 L_TYPE = None
 L_CI = None

cmdb-api/api/lib/cmdb/resp_format.py

@@ -35,7 +35,7 @@ class ErrFormat(CommonErrFormat):
         "Only creators and administrators are allowed to delete attributes!")  # 目前只允许 属性创建人、管理员 删除属性!
     # 属性字段名不能是内置字段: id, _id, ci_id, type, _type, ci_type
     attribute_name_cannot_be_builtin = _l(
-        "Attribute field names cannot be built-in fields: id, _id, ci_id, type, _type, ci_type")
+        "Attribute field names cannot be built-in fields: id, _id, ci_id, type, _type, ci_type, ticket_id")
     attribute_choice_other_invalid = _l(
         "Predefined value: Other model request parameters are illegal!")  # 预定义值: 其他模型请求参数不合法!
 

cmdb-api/api/lib/mixin.py

@@ -1,21 +1,19 @@
 # -*- coding:utf-8 -*-
 
 
-from flask import abort
 from sqlalchemy import func
 
 from api.extensions import db
 from api.lib.utils import get_page
 from api.lib.utils import get_page_size
 
-__author__ = 'pycook'
-
 
 class DBMixin(object):
     cls = None
 
     @classmethod
-    def search(cls, page, page_size, fl=None, only_query=False, reverse=False, count_query=False, **kwargs):
+    def search(cls, page, page_size, fl=None, only_query=False, reverse=False, count_query=False,
+               last_size=None, **kwargs):
         page = get_page(page)
         page_size = get_page_size(page_size)
         if fl is None:
@@ -47,14 +45,15 @@ class DBMixin(object):
             return _query, query
 
         numfound = query.count()
-        return numfound, [i.to_dict() if fl is None else getattr(i, '_asdict')()
-                          for i in query.offset((page - 1) * page_size).limit(page_size)]
-
-    def _must_be_required(self, _id):
-        existed = self.cls.get_by_id(_id)
-        existed or abort(404, "Factor [{}] does not exist".format(_id))
-
-        return existed
+        if not last_size:
+            return numfound, [i.to_dict() if fl is None else getattr(i, '_asdict')()
+                              for i in query.offset((page - 1) * page_size).limit(page_size)]
+        else:
+            offset = numfound - last_size
+            if offset < 0:
+                offset = 0
+            return numfound, [i.to_dict() if fl is None else getattr(i, '_asdict')()
+                              for i in query.offset(offset).limit(last_size)]
 
     def _can_add(self, **kwargs):
         raise NotImplementedError