From 175778a162e550d3cf6cdd7cdea6e866ef160e4a Mon Sep 17 00:00:00 2001 From: pycook Date: Thu, 25 Jul 2024 17:45:26 +0800 Subject: [PATCH] perf(api): auto discovery (#582) --- cmdb-api/api/lib/cmdb/attribute.py | 2 +- .../lib/cmdb/auto_discovery/auto_discovery.py | 217 +++++++++++++----- cmdb-api/api/lib/cmdb/auto_discovery/const.py | 17 +- cmdb-api/api/lib/cmdb/ci.py | 4 +- cmdb-api/api/lib/cmdb/ci_type.py | 11 +- cmdb-api/api/lib/cmdb/const.py | 2 +- cmdb-api/api/lib/cmdb/resp_format.py | 2 +- cmdb-api/api/lib/mixin.py | 23 +- cmdb-api/api/models/cmdb.py | 9 + cmdb-api/api/tasks/cmdb.py | 82 ++++++- cmdb-api/api/views/cmdb/auto_discovery.py | 52 ++++- 11 files changed, 328 insertions(+), 93 deletions(-) diff --git a/cmdb-api/api/lib/cmdb/attribute.py b/cmdb-api/api/lib/cmdb/attribute.py index 37aa31a..f2e3d69 100644 --- a/cmdb-api/api/lib/cmdb/attribute.py +++ b/cmdb-api/api/lib/cmdb/attribute.py @@ -229,7 +229,7 @@ class AttributeManager(object): is_choice = True if choice_value or kwargs.get('choice_web_hook') or kwargs.get('choice_other') else False name = kwargs.pop("name") - if name in BUILTIN_KEYWORDS: + if name in BUILTIN_KEYWORDS or kwargs.get('alias') in BUILTIN_KEYWORDS: return abort(400, ErrFormat.attribute_name_cannot_be_builtin) while kwargs.get('choice_other'): diff --git a/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py b/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py index b909bb7..937177a 100644 --- a/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py +++ b/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py @@ -18,12 +18,10 @@ from api.lib.cmdb.cache import AutoDiscoveryMappingCache from api.lib.cmdb.cache import CITypeAttributeCache from api.lib.cmdb.cache import CITypeCache from api.lib.cmdb.ci import CIManager -from api.lib.cmdb.ci import CIRelationManager from api.lib.cmdb.ci_type import CITypeGroupManager from api.lib.cmdb.const import AutoDiscoveryType from api.lib.cmdb.const import CMDB_QUEUE from api.lib.cmdb.const import PermEnum -from api.lib.cmdb.const import RelationSourceEnum from api.lib.cmdb.const import ResourceTypeEnum from api.lib.cmdb.custom_dashboard import SystemConfigManager from api.lib.cmdb.resp_format import ErrFormat @@ -35,6 +33,7 @@ from api.lib.perm.acl.acl import ACLManager from api.lib.perm.acl.acl import is_app_admin from api.lib.perm.acl.acl import validate_permission from api.lib.utils import AESCrypto +from api.models.cmdb import AutoDiscoveryAccount from api.models.cmdb import AutoDiscoveryCI from api.models.cmdb import AutoDiscoveryCIType from api.models.cmdb import AutoDiscoveryCITypeRelation @@ -42,6 +41,7 @@ from api.models.cmdb import AutoDiscoveryCounter from api.models.cmdb import AutoDiscoveryExecHistory from api.models.cmdb import AutoDiscoveryRule from api.models.cmdb import AutoDiscoveryRuleSyncHistory +from api.tasks.cmdb import build_relations_for_ad_accept from api.tasks.cmdb import write_ad_rule_sync_history PWD = os.path.abspath(os.path.dirname(__file__)) @@ -226,14 +226,14 @@ class AutoDiscoveryCITypeCRUD(DBMixin): adr = AutoDiscoveryRuleCRUD.get_by_id(adt.adr_id) if not adr: continue - if adr.type == "http": + if adr.type == AutoDiscoveryType.HTTP: for i in DEFAULT_INNER: if adr.name == i['name']: attrs = AutoDiscoveryHTTPManager.get_attributes( i['en'], (adt.extra_option or {}).get('category')) or [] result.extend([i.get('name') for i in attrs]) break - elif adr.type == "snmp": + elif adr.type == AutoDiscoveryType.SNMP: attributes = AutoDiscoverySNMPManager.get_attributes() result.extend([i.get('name') for i in (attributes or [])]) else: @@ -243,6 +243,14 @@ class AutoDiscoveryCITypeCRUD(DBMixin): @classmethod def get(cls, ci_id, oneagent_id, oneagent_name, last_update_at=None): + """ + OneAgent sync rules + :param ci_id: + :param oneagent_id: + :param oneagent_name: + :param last_update_at: + :return: + """ result = [] rules = cls.cls.get_by(to_dict=True) @@ -250,17 +258,14 @@ class AutoDiscoveryCITypeCRUD(DBMixin): if not rule['enabled']: continue - if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('secret'): - if not (current_user.username in PRIVILEGED_USERS or current_user.uid == rule['uid']): - rule['extra_option'].pop('secret', None) - else: - rule['extra_option']['secret'] = AESCrypto.decrypt(rule['extra_option']['secret']) + if isinstance(rule.get("extra_option"), dict): + decrypt_account(rule['extra_option'], rule['uid']) - if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('password'): - if not (current_user.username in PRIVILEGED_USERS or current_user.uid == rule['uid']): + if rule['extra_option'].get('_reference'): rule['extra_option'].pop('password', None) - else: - rule['extra_option']['password'] = AESCrypto.decrypt(rule['extra_option']['password']) + rule['extra_option'].pop('secret', None) + rule['extra_option'].update( + AutoDiscoveryAccountCRUD().get_config_by_id(rule['extra_option']['_reference'])) if oneagent_id and rule['agent_id'] == oneagent_id: result.append(rule) @@ -364,7 +369,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin): if kwargs.get('adr_id'): adr = AutoDiscoveryRule.get_by_id(kwargs['adr_id']) or abort( 404, ErrFormat.adr_not_found.format("id={}".format(kwargs['adr_id']))) - if adr.type == "http": + if adr.type == AutoDiscoveryType.HTTP: kwargs.setdefault('extra_option', dict()) en_name = None for i in DEFAULT_INNER: @@ -379,13 +384,16 @@ class AutoDiscoveryCITypeCRUD(DBMixin): kwargs["extra_option"]["provider"] = en_name break + if adr.type == AutoDiscoveryType.COMPONENTS and kwargs.get('extra_option'): + for i in DEFAULT_INNER: + if i['name'] == adr.name: + kwargs['extra_option']['collect_key'] = i['option'].get('collect_key') + break + if kwargs.get('is_plugin') and kwargs.get('plugin_script'): kwargs = check_plugin_script(**kwargs) - if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'): - kwargs['extra_option']['secret'] = AESCrypto.encrypt(kwargs['extra_option']['secret']) - if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('password'): - kwargs['extra_option']['password'] = AESCrypto.encrypt(kwargs['extra_option']['password']) + encrypt_account(kwargs.get('extra_option')) ci_type = CITypeCache.get(kwargs['type_id']) unique = AttributeCache.get(ci_type.unique_id) @@ -403,7 +411,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin): adr = AutoDiscoveryRule.get_by_id(existed.adr_id) or abort( 404, ErrFormat.adr_not_found.format("id={}".format(existed.adr_id))) - if adr.type == "http": + if adr.type == AutoDiscoveryType.HTTP: kwargs.setdefault('extra_option', dict()) en_name = None for i in DEFAULT_INNER: @@ -418,6 +426,12 @@ class AutoDiscoveryCITypeCRUD(DBMixin): kwargs["extra_option"]["provider"] = en_name break + if adr.type == AutoDiscoveryType.COMPONENTS and kwargs.get('extra_option'): + for i in DEFAULT_INNER: + if i['name'] == adr.name: + kwargs['extra_option']['collect_key'] = i['option'].get('collect_key') + break + if 'attributes' in kwargs: self.__valid_exec_target(kwargs.get('agent_id'), kwargs.get('query_expr')) @@ -441,13 +455,10 @@ class AutoDiscoveryCITypeCRUD(DBMixin): if kwargs.get('is_plugin') and kwargs.get('plugin_script'): kwargs = check_plugin_script(**kwargs) - if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'): - kwargs['extra_option']['secret'] = AESCrypto.encrypt(kwargs['extra_option']['secret']) - if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('password'): - kwargs['extra_option']['password'] = AESCrypto.encrypt(kwargs['extra_option']['password']) + encrypt_account(kwargs.get('extra_option')) inst = self._can_update(_id=_id, **kwargs) - if len(kwargs) == 1 and 'enabled' in kwargs: # enable or disable + if len(kwargs) == 1 and 'enabled' in kwargs: # enable or disable pass elif inst.agent_id != kwargs.get('agent_id') or inst.query_expr != kwargs.get('query_expr'): for item in AutoDiscoveryRuleSyncHistory.get_by(adt_id=inst.id, to_dict=False): @@ -688,9 +699,11 @@ class AutoDiscoveryCICRUD(DBMixin): adt = AutoDiscoveryCITypeCRUD.get_by_id(adc.adt_id) or abort(404, ErrFormat.adt_not_found) ci_id = None - if adt.attributes: - ci_dict = {adt.attributes[k]: None if not v and isinstance(v, (list, dict)) else v - for k, v in adc.instance.items() if k in adt.attributes} + + ad_key2attr = adt.attributes or {} + if ad_key2attr: + ci_dict = {ad_key2attr[k]: None if not v and isinstance(v, (list, dict)) else v + for k, v in adc.instance.items() if k in ad_key2attr} extra_option = adt.extra_option or {} mapping, path_mapping = AutoDiscoveryHTTPManager.get_predefined_value_mapping( extra_option.get('provider'), extra_option.get('category')) @@ -703,37 +716,7 @@ class AutoDiscoveryCICRUD(DBMixin): AutoDiscoveryExecHistoryCRUD().add(type_id=adt.type_id, stdout="accept resource: {}".format(adc.unique_value)) - relation_ads = AutoDiscoveryCITypeRelation.get_by(ad_type_id=adt.type_id, to_dict=False) - for r_adt in relation_ads: - ad_key = r_adt.ad_key - if not adc.instance.get(ad_key): - continue - - ad_key_values = [adc.instance.get(ad_key)] if not isinstance( - adc.instance.get(ad_key), list) else adc.instance.get(ad_key) - for ad_key_value in ad_key_values: - query = "_type:{},{}:{}".format(r_adt.peer_type_id, r_adt.peer_attr_id, ad_key_value) - s = ci_search(query, use_ci_filter=False, count=1000000) - try: - response, _, _, _, _, _ = s.search() - except SearchError as e: - current_app.logger.warning(e) - return abort(400, str(e)) - - for relation_ci in response: - relation_ci_id = relation_ci['_id'] - try: - CIRelationManager.add(ci_id, relation_ci_id, - valid=False, - source=RelationSourceEnum.AUTO_DISCOVERY) - - except: - try: - CIRelationManager.add(relation_ci_id, ci_id, - valid=False, - source=RelationSourceEnum.AUTO_DISCOVERY) - except: - pass + build_relations_for_ad_accept.apply_async(args=(adc.to_dict(), ci_id, ad_key2attr), queue=CMDB_QUEUE) adc.update(is_accept=True, accept_by=nickname or current_user.nickname, @@ -893,3 +876,121 @@ class AutoDiscoveryCounterCRUD(DBMixin): def _can_delete(self, **kwargs): pass + + +def encrypt_account(config): + if isinstance(config, dict): + if config.get('secret'): + config['secret'] = AESCrypto.encrypt(config['secret']) + if config.get('password'): + config['password'] = AESCrypto.encrypt(config['password']) + + +def decrypt_account(config, uid): + if isinstance(config, dict): + if config.get('password'): + if not (current_user.username in PRIVILEGED_USERS or current_user.uid == uid): + config.pop('password', None) + else: + try: + config['password'] = AESCrypto.decrypt(config['password']) + except Exception as e: + current_app.logger.error('decrypt account failed: {}'.format(e)) + + if config.get('secret'): + if not (current_user.username in PRIVILEGED_USERS or current_user.uid == uid): + config.pop('secret', None) + else: + try: + config['secret'] = AESCrypto.decrypt(config['secret']) + except Exception as e: + current_app.logger.error('decrypt account failed: {}'.format(e)) + + +class AutoDiscoveryAccountCRUD(DBMixin): + cls = AutoDiscoveryAccount + + def get(self, adr_id): + res = self.cls.get_by(adr_id=adr_id, to_dict=True) + + for i in res: + decrypt_account(i.get('config'), i['uid']) + + return res + + def get_config_by_id(self, _id): + res = self.cls.get_by_id(_id) + if not res: + return {} + + config = res.to_dict().get('config') or {} + + decrypt_account(config, res.uid) + + return config + + def _can_add(self, **kwargs): + encrypt_account(kwargs.get('config')) + + kwargs['uid'] = current_user.uid + + return kwargs + + def upsert(self, adr_id, accounts): + existed_all = self.cls.get_by(adr_id=adr_id, to_dict=False) + account_names = {i['name'] for i in accounts} + + name_changed = dict() + for account in accounts: + existed = None + if account.get('id'): + existed = self.cls.get_by_id(account.get('id')) + if existed is None: + continue + + account.pop('id') + name_changed[existed.name] = account.get('name') + else: + account = self._can_add(**account) + + if existed is not None: + if current_user.uid == existed.uid: + config = copy.deepcopy(existed.config) or {} + config.update(account.get('config') or {}) + account['config'] = config + existed.update(**account) + else: + self.cls.create(adr_id=adr_id, **account) + + for item in existed_all: + if name_changed.get(item.name, item.name) not in account_names: + if current_user.uid == item.uid: + item.soft_delete() + + def _can_update(self, **kwargs): + existed = self.cls.get_by_id(kwargs['_id']) or abort(404, ErrFormat.not_found) + + if isinstance(kwargs.get('config'), dict) and kwargs['config'].get('secret'): + if current_user.uid != existed.uid: + return abort(403, ErrFormat.adt_secret_no_permission) + if isinstance(kwargs.get('config'), dict) and kwargs['config'].get('password'): + if current_user.uid != existed.uid: + return abort(403, ErrFormat.adt_secret_no_permission) + + return existed + + def update(self, _id, **kwargs): + + if kwargs.get('is_plugin') and kwargs.get('plugin_script'): + kwargs = check_plugin_script(**kwargs) + + encrypt_account(kwargs.get('config')) + + inst = self._can_update(_id=_id, **kwargs) + + obj = inst.update(_id=_id, filter_none=False, **kwargs) + + return obj + + def _can_delete(self, **kwargs): + pass diff --git a/cmdb-api/api/lib/cmdb/auto_discovery/const.py b/cmdb-api/api/lib/cmdb/auto_discovery/const.py index eca4b96..4c731c1 100644 --- a/cmdb-api/api/lib/cmdb/auto_discovery/const.py +++ b/cmdb-api/api/lib/cmdb/auto_discovery/const.py @@ -19,10 +19,21 @@ DEFAULT_INNER = [ dict(name="KVM", en="kvm", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False, option={'icon': {'name': 'ops-KVM'}, "category": "private_cloud", "en": "kvm"}), + dict(name="Nginx", en="nginx", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False, - option={'icon': {'name': 'caise-nginx'}, "en": "nginx"}), + option={'icon': {'name': 'caise-nginx'}, "en": "nginx", "collect_key": "nginx"}), + dict(name="Apache", en="apache", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False, + option={'icon': {'name': 'caise-apache'}, "en": "apache", "collect_key": "apache"}), + dict(name="Tomcat", en="tomcat", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False, + option={'icon': {'name': 'caise-tomcat'}, "en": "tomcat", "collect_key": "tomcat"}), + dict(name="MySQL", en="mysql", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False, + option={'icon': {'name': 'caise-mySQL'}, "en": "mysql", "collect_key": "mysql"}), + dict(name="MSSQL", en="mssql", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False, + option={'icon': {'name': 'caise-SQLServer'}, "en": "mssql", "collect_key": "sqlserver"}), + dict(name="Oracle", en="oracle", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False, + option={'icon': {'name': 'caise-oracle'}, "en": "oracle", "collect_key": "oracle"}), dict(name="Redis", en="redis", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False, - option={'icon': {'name': 'caise-redis'}, "en": "redis"}), + option={'icon': {'name': 'caise-redis'}, "en": "redis", "collect_key": "redis"}), dict(name="交换机", type=AutoDiscoveryType.SNMP, is_inner=True, is_plugin=False, option={'icon': {'name': 'caise-jiaohuanji'}}), @@ -294,7 +305,7 @@ CLOUD_MAP = { "category": "其他", "items": ["资源池", "数据中心", "文件夹"], "map": { - "资源池": "templates/vsphere_datastore.json", + "资源池": "templates/vsphere_pool.json", "数据中心": "templates/vsphere_datacenter.json", "文件夹": "templates/vsphere_folder.json", }, diff --git a/cmdb-api/api/lib/cmdb/ci.py b/cmdb-api/api/lib/cmdb/ci.py index 9815d88..1bc6109 100644 --- a/cmdb-api/api/lib/cmdb/ci.py +++ b/cmdb-api/api/lib/cmdb/ci.py @@ -319,8 +319,8 @@ class CIManager(object): 400, ErrFormat.unique_value_not_found.format("unique_id={}".format(ci_type.unique_id))) unique_value = None - if not (unique_key.default and unique_key.default.get('default') == AttributeDefaultValueEnum.AUTO_INC_ID and - not ci_dict.get(unique_key.name)): # primary key is not auto inc id + # primary key is not auto inc id + if not (unique_key.default and unique_key.default.get('default') == AttributeDefaultValueEnum.AUTO_INC_ID): unique_value = ci_dict.get(unique_key.name) or ci_dict.get(unique_key.alias) or ci_dict.get(unique_key.id) unique_value = unique_value or abort(400, ErrFormat.unique_key_required.format(unique_key.name)) diff --git a/cmdb-api/api/lib/cmdb/ci_type.py b/cmdb-api/api/lib/cmdb/ci_type.py index 0164de1..8bffd58 100644 --- a/cmdb-api/api/lib/cmdb/ci_type.py +++ b/cmdb-api/api/lib/cmdb/ci_type.py @@ -1244,17 +1244,16 @@ class CITypeAttributeGroupManager(object): if isinstance(_from, int): from_group = CITypeAttributeGroup.get_by_id(_from) else: - from_group = CITypeAttributeGroup.get_by(name=_from, first=True, to_dict=False) + from_group = CITypeAttributeGroup.get_by(name=_from, type_id=type_id, first=True, to_dict=False) from_group or abort(404, ErrFormat.ci_type_attribute_group_not_found.format("id={}".format(_from))) if isinstance(_to, int): to_group = CITypeAttributeGroup.get_by_id(_to) else: - to_group = CITypeAttributeGroup.get_by(name=_to, first=True, to_dict=False) + to_group = CITypeAttributeGroup.get_by(name=_to, type_id=type_id, first=True, to_dict=False) to_group or abort(404, ErrFormat.ci_type_attribute_group_not_found.format("id={}".format(_to))) from_order, to_order = from_group.order, to_group.order - from_group.update(order=to_order) to_group.update(order=from_order) @@ -1541,6 +1540,9 @@ class CITypeTemplateManager(object): if ((i.extra_option or {}).get('alias') or None) == ( (rule.get('extra_option') or {}).get('alias') or None): existed = True + rule.pop('extra_option', None) + rule.pop('enabled', None) + rule.pop('cron', None) AutoDiscoveryCITypeCRUD().update(i.id, **rule) break @@ -1698,6 +1700,9 @@ class CITypeTemplateManager(object): for r in ad_rules: r = r.to_dict() + if r.get('extra_option') and '_reference' in r['extra_option']: + r['extra_option'].pop('_reference') + r['type_name'] = type_id2name.get(r.pop('type_id')) if r.get('adr_id'): adr = AutoDiscoveryRuleCRUD.get_by_id(r.pop('adr_id')) diff --git a/cmdb-api/api/lib/cmdb/const.py b/cmdb-api/api/lib/cmdb/const.py index ecda9e3..972fa69 100644 --- a/cmdb-api/api/lib/cmdb/const.py +++ b/cmdb-api/api/lib/cmdb/const.py @@ -118,7 +118,7 @@ REDIS_PREFIX_CI = "ONE_CMDB" REDIS_PREFIX_CI_RELATION = "CMDB_CI_RELATION" REDIS_PREFIX_CI_RELATION2 = "CMDB_CI_RELATION2" -BUILTIN_KEYWORDS = {'id', '_id', 'ci_id', 'type', '_type', 'ci_type'} +BUILTIN_KEYWORDS = {'id', '_id', 'ci_id', 'type', '_type', 'ci_type', 'ticket_id'} L_TYPE = None L_CI = None diff --git a/cmdb-api/api/lib/cmdb/resp_format.py b/cmdb-api/api/lib/cmdb/resp_format.py index 245e775..57a5b65 100644 --- a/cmdb-api/api/lib/cmdb/resp_format.py +++ b/cmdb-api/api/lib/cmdb/resp_format.py @@ -35,7 +35,7 @@ class ErrFormat(CommonErrFormat): "Only creators and administrators are allowed to delete attributes!") # 目前只允许 属性创建人、管理员 删除属性! # 属性字段名不能是内置字段: id, _id, ci_id, type, _type, ci_type attribute_name_cannot_be_builtin = _l( - "Attribute field names cannot be built-in fields: id, _id, ci_id, type, _type, ci_type") + "Attribute field names cannot be built-in fields: id, _id, ci_id, type, _type, ci_type, ticket_id") attribute_choice_other_invalid = _l( "Predefined value: Other model request parameters are illegal!") # 预定义值: 其他模型请求参数不合法! diff --git a/cmdb-api/api/lib/mixin.py b/cmdb-api/api/lib/mixin.py index dc57232..d2d4abf 100644 --- a/cmdb-api/api/lib/mixin.py +++ b/cmdb-api/api/lib/mixin.py @@ -1,21 +1,19 @@ # -*- coding:utf-8 -*- -from flask import abort from sqlalchemy import func from api.extensions import db from api.lib.utils import get_page from api.lib.utils import get_page_size -__author__ = 'pycook' - class DBMixin(object): cls = None @classmethod - def search(cls, page, page_size, fl=None, only_query=False, reverse=False, count_query=False, **kwargs): + def search(cls, page, page_size, fl=None, only_query=False, reverse=False, count_query=False, + last_size=None, **kwargs): page = get_page(page) page_size = get_page_size(page_size) if fl is None: @@ -47,14 +45,15 @@ class DBMixin(object): return _query, query numfound = query.count() - return numfound, [i.to_dict() if fl is None else getattr(i, '_asdict')() - for i in query.offset((page - 1) * page_size).limit(page_size)] - - def _must_be_required(self, _id): - existed = self.cls.get_by_id(_id) - existed or abort(404, "Factor [{}] does not exist".format(_id)) - - return existed + if not last_size: + return numfound, [i.to_dict() if fl is None else getattr(i, '_asdict')() + for i in query.offset((page - 1) * page_size).limit(page_size)] + else: + offset = numfound - last_size + if offset < 0: + offset = 0 + return numfound, [i.to_dict() if fl is None else getattr(i, '_asdict')() + for i in query.offset(offset).limit(last_size)] def _can_add(self, **kwargs): raise NotImplementedError diff --git a/cmdb-api/api/models/cmdb.py b/cmdb-api/api/models/cmdb.py index 5ecbee5..5e4f73c 100644 --- a/cmdb-api/api/models/cmdb.py +++ b/cmdb-api/api/models/cmdb.py @@ -636,6 +636,15 @@ class AutoDiscoveryCounter(Model2): last_week_count = db.Column(db.Integer, default=0) +class AutoDiscoveryAccount(Model): + __tablename__ = "c_ad_accounts" + + uid = db.Column(db.Integer, index=True) + name = db.Column(db.String(64)) + adr_id = db.Column(db.Integer, db.ForeignKey('c_ad_rules.id')) + config = db.Column(db.JSON) + + class CIFilterPerms(Model): __tablename__ = "c_ci_filter_perms" diff --git a/cmdb-api/api/tasks/cmdb.py b/cmdb-api/api/tasks/cmdb.py index 1fa1d40..c0bde06 100644 --- a/cmdb-api/api/tasks/cmdb.py +++ b/cmdb-api/api/tasks/cmdb.py @@ -1,9 +1,8 @@ # -*- coding:utf-8 -*- -import json import datetime - +import json import redis_lock from flask import current_app from flask_login import login_user @@ -13,21 +12,24 @@ from api.extensions import celery from api.extensions import db from api.extensions import es from api.extensions import rd +from api.lib.cmdb.cache import AttributeCache from api.lib.cmdb.cache import CITypeAttributesCache from api.lib.cmdb.const import CMDB_QUEUE from api.lib.cmdb.const import REDIS_PREFIX_CI from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION2 +from api.lib.cmdb.const import RelationSourceEnum from api.lib.cmdb.perms import CIFilterPermsCRUD from api.lib.decorator import flush_db from api.lib.decorator import reconnect_db from api.lib.perm.acl.cache import UserCache from api.lib.utils import handle_arg_list +from api.models.cmdb import AutoDiscoveryCI +from api.models.cmdb import AutoDiscoveryCIType +from api.models.cmdb import AutoDiscoveryCITypeRelation from api.models.cmdb import CI from api.models.cmdb import CIRelation from api.models.cmdb import CITypeAttribute -from api.models.cmdb import AutoDiscoveryCI -from api.models.cmdb import AutoDiscoveryCIType @celery.task(name="cmdb.ci_cache", queue=CMDB_QUEUE) @@ -277,3 +279,75 @@ def write_ad_rule_sync_history(rules, oneagent_id, oneagent_name, sync_at): except Exception as e: current_app.logger.error("write auto discovery rule sync history failed: {}".format(e)) db.session.rollback() + + +@celery.task(name="cmdb.build_relations_for_ad_accept", queue=CMDB_QUEUE) +@reconnect_db +def build_relations_for_ad_accept(adc, ci_id, ad_key2attr): + from api.lib.cmdb.ci import CIRelationManager + from api.lib.cmdb.search import SearchError + from api.lib.cmdb.search.ci import search as ci_search + + current_app.test_request_context().push() + login_user(UserCache.get('worker')) + + relation_ads = AutoDiscoveryCITypeRelation.get_by(ad_type_id=adc['type_id'], to_dict=False) + for r_adt in relation_ads: + ad_key = r_adt.ad_key + if not adc['instance'].get(ad_key): + continue + + ad_key_values = [adc['instance'].get(ad_key)] if not isinstance( + adc['instance'].get(ad_key), list) else adc['instance'].get(ad_key) + for ad_key_value in ad_key_values: + query = "_type:{},{}:{}".format(r_adt.peer_type_id, r_adt.peer_attr_id, ad_key_value) + s = ci_search(query, use_ci_filter=False, count=1000000) + try: + response, _, _, _, _, _ = s.search() + except SearchError as e: + current_app.logger.error("build_relations_for_ad_accept failed: {}".format(e)) + return + + for relation_ci in response: + relation_ci_id = relation_ci['_id'] + try: + CIRelationManager.add(ci_id, relation_ci_id, + valid=False, + source=RelationSourceEnum.AUTO_DISCOVERY) + + except: + try: + CIRelationManager.add(relation_ci_id, ci_id, + valid=False, + source=RelationSourceEnum.AUTO_DISCOVERY) + except: + pass + + # build relations in reverse + relation_ads = AutoDiscoveryCITypeRelation.get_by(peer_type_id=adc['type_id'], to_dict=False) + attr2ad_key = {v: k for k, v in ad_key2attr.items()} + for r_adt in relation_ads: + attr = AttributeCache.get(r_adt.peer_attr_id) + ad_key = attr2ad_key.get(attr and attr.name) + if not ad_key: + continue + + ad_value = adc['instance'].get(ad_key) + peer_ad_key = r_adt.ad_key + peer_instances = AutoDiscoveryCI.get_by(type_id=r_adt.ad_type_id, to_dict=False) + for peer_instance in peer_instances: + peer_ad_values = peer_instance.instance.get(peer_ad_key) + peer_ad_values = [peer_ad_values] if not isinstance(peer_ad_values, list) else peer_ad_values + if ad_value in peer_ad_values and peer_instance.ci_id: + try: + CIRelationManager.add(peer_instance.ci_id, ci_id, + valid=False, + source=RelationSourceEnum.AUTO_DISCOVERY) + + except: + try: + CIRelationManager.add(ci_id, peer_instance.ci_id, + valid=False, + source=RelationSourceEnum.AUTO_DISCOVERY) + except: + pass diff --git a/cmdb-api/api/views/cmdb/auto_discovery.py b/cmdb-api/api/views/cmdb/auto_discovery.py index e856243..2015c3c 100644 --- a/cmdb-api/api/views/cmdb/auto_discovery.py +++ b/cmdb-api/api/views/cmdb/auto_discovery.py @@ -8,6 +8,7 @@ from flask import request from flask_login import current_user from io import BytesIO +from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryAccountCRUD from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCICRUD from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeCRUD from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeRelationCRUD @@ -20,6 +21,7 @@ from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryRuleSyncHist from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoverySNMPManager from api.lib.cmdb.auto_discovery.const import DEFAULT_INNER from api.lib.cmdb.auto_discovery.const import PRIVILEGED_USERS +from api.lib.cmdb.cache import AttributeCache from api.lib.cmdb.const import PermEnum from api.lib.cmdb.const import ResourceTypeEnum from api.lib.cmdb.resp_format import ErrFormat @@ -272,14 +274,16 @@ class AutoDiscoveryRuleSyncView(APIView): oneagent_id = request.values.get('oneagent_id') last_update_at = request.values.get('last_update_at') - query = "oneagent_id:{}".format(oneagent_id) - s = ci_search(query) - try: - response, _, _, _, _, _ = s.search() - except SearchError as e: - import traceback - current_app.logger.error(traceback.format_exc()) - return abort(400, str(e)) + response = [] + if AttributeCache.get('oneagent_id'): + query = "oneagent_id:{}".format(oneagent_id) + s = ci_search(query) + try: + response, _, _, _, _, _ = s.search() + except SearchError as e: + import traceback + current_app.logger.error(traceback.format_exc()) + return abort(400, str(e)) for res in response: if res.get('{}_name'.format(res['ci_type'])) == oneagent_name or oneagent_name == res.get('oneagent_name'): @@ -328,8 +332,12 @@ class AutoDiscoveryExecHistoryView(APIView): def get(self): page = get_page(request.values.pop('page', 1)) page_size = get_page_size(request.values.pop('page_size', None)) + last_size = request.values.pop('last_size', None) + if last_size and last_size.isdigit(): + last_size = int(last_size) numfound, res = AutoDiscoveryExecHistoryCRUD.search(page=page, page_size=page_size, + last_size=last_size, **request.values) return self.jsonify(page=page, @@ -355,3 +363,31 @@ class AutoDiscoveryCounterView(APIView): type_id = request.values.get('type_id') return self.jsonify(AutoDiscoveryCounterCRUD().get(type_id)) + + +class AutoDiscoveryAccountView(APIView): + url_prefix = ("/adr/accounts", "/adr/accounts/") + + @args_required('adr_id') + def get(self): + adr_id = request.values.get('adr_id') + + return self.jsonify(AutoDiscoveryAccountCRUD().get(adr_id)) + + @args_required('adr_id') + @args_required('accounts', value_required=False) + def post(self): + AutoDiscoveryAccountCRUD().upsert(**request.values) + + return self.jsonify(code=200) + + @args_required('config') + def put(self, account_id): + res = AutoDiscoveryAccountCRUD().update(account_id, **request.values) + + return self.jsonify(res.to_dict()) + + def delete(self, account_id): + AutoDiscoveryAccountCRUD().delete(account_id) + + return self.jsonify(account_id=account_id)