diff --git a/cmdb-api/api/commands/click_cmdb.py b/cmdb-api/api/commands/click_cmdb.py index 46e3651..ebf8123 100644 --- a/cmdb-api/api/commands/click_cmdb.py +++ b/cmdb-api/api/commands/click_cmdb.py @@ -190,6 +190,7 @@ def cmdb_counter(): login_user(UserCache.get('worker')) i = 0 + today = datetime.date.today() while True: try: db.session.remove() @@ -200,6 +201,10 @@ def cmdb_counter(): CMDBCounterCache.flush_adc_counter() i = 0 + if datetime.date.today() != today: + CMDBCounterCache.clear_ad_exec_history() + today = datetime.date.today() + CMDBCounterCache.flush_sub_counter() i += 1 @@ -493,3 +498,48 @@ def cmdb_agent_init(): click.echo("Key : {}".format(click.style(user.key, bg='red'))) click.echo("Secret: {}".format(click.style(user.secret, bg='red'))) + + +@click.command() +@click.option( + '-v', + '--version', + help='input cmdb version, e.g. 2.4.6', + required=True, +) +@with_appcontext +def cmdb_patch(version): + """ + CMDB upgrade patch + """ + + version = version[1:] if version.lower().startswith("v") else version + + if version >= '2.4.6': + + from api.models.cmdb import CITypeRelation + for cr in CITypeRelation.get_by(to_dict=False): + if hasattr(cr, 'parent_attr_id') and cr.parent_attr_id and not cr.parent_attr_ids: + parent_attr_ids, child_attr_ids = [cr.parent_attr_id], [cr.child_attr_id] + cr.update(parent_attr_ids=parent_attr_ids, child_attr_ids=child_attr_ids, commit=False) + db.session.commit() + + from api.models.cmdb import AutoDiscoveryCIType, AutoDiscoveryCITypeRelation + from api.lib.cmdb.cache import CITypeCache, AttributeCache + for adt in AutoDiscoveryCIType.get_by(to_dict=False): + if adt.relation: + if not AutoDiscoveryCITypeRelation.get_by(ad_type_id=adt.type_id): + peer_type = CITypeCache.get(list(adt.relation.values())['type_name']) + peer_type_id = peer_type and peer_type.id + peer_attr = AttributeCache.get(list(adt.relation.values())['attr_name']) + peer_attr_id = peer_attr and peer_attr.id + if peer_type_id and peer_attr_id: + AutoDiscoveryCITypeRelation.create(ad_type_id=adt.type_id, + ad_key=list(adt.relation.keys())[0], + peer_type_id=peer_type_id, + peer_attr_id=peer_attr_id, + commit=False) + if hasattr(adt, 'interval') and adt.interval and not adt.cron: + adt.cron = "*/{} * * * *".format(adt.interval // 60) + + db.session.commit() diff --git a/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py b/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py index a0323fb..909a05a 100644 --- a/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py +++ b/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py @@ -1,34 +1,47 @@ # -*- coding:utf-8 -*- +import copy import datetime import json import os +from flask import abort +from flask import current_app +from flask_login import current_user +from sqlalchemy import func + from api.extensions import db from api.lib.cmdb.auto_discovery.const import ClOUD_MAP +from api.lib.cmdb.auto_discovery.const import DEFAULT_HTTP +from api.lib.cmdb.cache import AttributeCache from api.lib.cmdb.cache import CITypeAttributeCache from api.lib.cmdb.cache import CITypeCache from api.lib.cmdb.ci import CIManager from api.lib.cmdb.ci import CIRelationManager from api.lib.cmdb.ci_type import CITypeGroupManager from api.lib.cmdb.const import AutoDiscoveryType +from api.lib.cmdb.const import CMDB_QUEUE from api.lib.cmdb.const import PermEnum from api.lib.cmdb.const import ResourceTypeEnum from api.lib.cmdb.resp_format import ErrFormat from api.lib.cmdb.search import SearchError -from api.lib.cmdb.search.ci import search +from api.lib.cmdb.search.ci import search as ci_search +from api.lib.common_setting.role_perm_base import CMDBApp from api.lib.mixin import DBMixin +from api.lib.perm.acl.acl import ACLManager from api.lib.perm.acl.acl import is_app_admin from api.lib.perm.acl.acl import validate_permission from api.lib.utils import AESCrypto from api.models.cmdb import AutoDiscoveryCI from api.models.cmdb import AutoDiscoveryCIType +from api.models.cmdb import AutoDiscoveryCITypeRelation +from api.models.cmdb import AutoDiscoveryCounter +from api.models.cmdb import AutoDiscoveryExecHistory from api.models.cmdb import AutoDiscoveryRule -from flask import abort -from flask import current_app -from flask_login import current_user -from sqlalchemy import func +from api.models.cmdb import AutoDiscoveryRuleSyncHistory +from api.tasks.cmdb import write_ad_rule_sync_history PWD = os.path.abspath(os.path.dirname(__file__)) +app_cli = CMDBApp() def parse_plugin_script(script): @@ -100,6 +113,14 @@ class AutoDiscoveryRuleCRUD(DBMixin): self.cls.get_by(name=kwargs['name']) and abort(400, ErrFormat.adr_duplicate.format(kwargs['name'])) if kwargs.get('is_plugin') and kwargs.get('plugin_script'): kwargs = check_plugin_script(**kwargs) + acl = ACLManager(app_cli.app_name) + if not acl.has_permission(app_cli.op.Auto_Discovery, + app_cli.resource_type_name, + app_cli.op.create_plugin) and not is_app_admin(app_cli.app_name): + return abort(403, ErrFormat.no_permission.format( + app_cli.op.Auto_Discovery, app_cli.op.create_plugin)) + + kwargs['owner'] = current_user.uid return kwargs @@ -115,6 +136,14 @@ class AutoDiscoveryRuleCRUD(DBMixin): if other and other.id != existed.id: return abort(400, ErrFormat.adr_duplicate.format(kwargs['name'])) + if existed.is_plugin: + acl = ACLManager(app_cli.app_name) + if not acl.has_permission(app_cli.op.Auto_Discovery, + app_cli.resource_type_name, + app_cli.op.update_plugin) and not is_app_admin(app_cli.app_name): + return abort(403, ErrFormat.no_permission.format( + app_cli.op.Auto_Discovery, app_cli.op.update_plugin)) + return existed def update(self, _id, **kwargs): @@ -122,13 +151,27 @@ class AutoDiscoveryRuleCRUD(DBMixin): if kwargs.get('is_plugin') and kwargs.get('plugin_script'): kwargs = check_plugin_script(**kwargs) + for item in AutoDiscoveryCIType.get_by(adr_id=_id, to_dict=False): + item.update(updated_at=datetime.datetime.now()) + return super(AutoDiscoveryRuleCRUD, self).update(_id, filter_none=False, **kwargs) def _can_delete(self, **kwargs): if AutoDiscoveryCIType.get_by(adr_id=kwargs['_id'], first=True): return abort(400, ErrFormat.adr_referenced) - return self._can_update(**kwargs) + existed = self.cls.get_by_id(kwargs['_id']) or abort( + 404, ErrFormat.adr_not_found.format("id={}".format(kwargs['_id']))) + + if existed.is_plugin: + acl = ACLManager(app_cli.app_name) + if not acl.has_permission(app_cli.op.Auto_Discovery, + app_cli.resource_type_name, + app_cli.op.delete_plugin) and not is_app_admin(app_cli.app_name): + return abort(403, ErrFormat.no_permission.format( + app_cli.op.Auto_Discovery, app_cli.op.delete_plugin)) + + return existed class AutoDiscoveryCITypeCRUD(DBMixin): @@ -147,14 +190,34 @@ class AutoDiscoveryCITypeCRUD(DBMixin): return cls.cls.get_by(type_id=type_id, to_dict=False) @classmethod - def get(cls, ci_id, oneagent_id, last_update_at=None): + def get_ad_attributes(cls, type_id): + result = [] + adts = cls.get_by_type_id(type_id) + for adt in adts: + adr = AutoDiscoveryRuleCRUD.get_by_id(adt.adr_id) + if not adr: + continue + if adr.type == "http": + for i in DEFAULT_HTTP: + if adr.name == i['name']: + attrs = AutoDiscoveryHTTPManager.get_attributes( + i['en'], (adt.extra_option or {}).get('category')) or [] + result.extend([i.get('name') for i in attrs]) + break + elif adr.type == "snmp": + attributes = AutoDiscoverySNMPManager.get_attributes() + result.extend([i.get('name') for i in (attributes or [])]) + else: + result.extend([i.get('name') for i in (adr.attributes or [])]) + + return sorted(list(set(result))) + + @classmethod + def get(cls, ci_id, oneagent_id, oneagent_name, last_update_at=None): result = [] rules = cls.cls.get_by(to_dict=True) for rule in rules: - if rule.get('relation'): - continue - if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('secret'): if not (current_user.username == "cmdb_agent" or current_user.uid == rule['uid']): rule['extra_option'].pop('secret', None) @@ -165,7 +228,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin): result.append(rule) elif rule['query_expr']: query = rule['query_expr'].lstrip('q').lstrip('=') - s = search(query, fl=['_id'], count=1000000) + s = ci_search(query, fl=['_id'], count=1000000) try: response, _, _, _, _, _ = s.search() except SearchError as e: @@ -182,9 +245,6 @@ class AutoDiscoveryCITypeCRUD(DBMixin): if adr.type in (AutoDiscoveryType.SNMP, AutoDiscoveryType.HTTP): continue - if not rule['updated_at']: - continue - result.append(rule) new_last_update_at = "" @@ -195,6 +255,9 @@ class AutoDiscoveryCITypeCRUD(DBMixin): if new_last_update_at < __last_update_at: new_last_update_at = __last_update_at + write_ad_rule_sync_history.apply_async(args=(result, oneagent_id, oneagent_name, datetime.datetime.now()), + queue=CMDB_QUEUE) + if not last_update_at or new_last_update_at > last_update_at: return result, new_last_update_at else: @@ -213,7 +276,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin): agent_id = agent_id.strip() q = "op_duty:{0},-rd_duty:{0},oneagent_id:{1}" - s = search(q.format(current_user.username, agent_id.strip())) + s = ci_search(q.format(current_user.username, agent_id.strip())) try: response, _, _, _, _, _ = s.search() if response: @@ -222,7 +285,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin): current_app.logger.warning(e) return abort(400, str(e)) - s = search(q.format(current_user.nickname, agent_id.strip())) + s = ci_search(q.format(current_user.nickname, agent_id.strip())) try: response, _, _, _, _, _ = s.search() if response: @@ -236,7 +299,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin): if query_expr.startswith('q='): query_expr = query_expr[2:] - s = search(query_expr, count=1000000) + s = ci_search(query_expr, count=1000000) try: response, _, _, _, _, _ = s.search() for i in response: @@ -254,13 +317,21 @@ class AutoDiscoveryCITypeCRUD(DBMixin): def _can_add(**kwargs): if kwargs.get('adr_id'): - AutoDiscoveryRule.get_by_id(kwargs['adr_id']) or abort( + adr = AutoDiscoveryRule.get_by_id(kwargs['adr_id']) or abort( 404, ErrFormat.adr_not_found.format("id={}".format(kwargs['adr_id']))) - # if not adr.is_plugin: - # other = self.cls.get_by(adr_id=adr.id, first=True, to_dict=False) - # if other: - # ci_type = CITypeCache.get(other.type_id) - # return abort(400, ErrFormat.adr_default_ref_once.format(ci_type.alias)) + if adr.type == "http": + kwargs.setdefault('extra_option', dict) + en_name = None + for i in DEFAULT_HTTP: + if i['name'] == adr.name: + en_name = i['en'] + break + if en_name and kwargs['extra_option'].get('category'): + for item in ClOUD_MAP[en_name]: + if item["collect_key_map"].get(kwargs['extra_option']['category']): + kwargs["extra_option"]["collect_key"] = item["collect_key_map"][ + kwargs['extra_option']['category']] + break if kwargs.get('is_plugin') and kwargs.get('plugin_script'): kwargs = check_plugin_script(**kwargs) @@ -268,6 +339,11 @@ class AutoDiscoveryCITypeCRUD(DBMixin): if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'): kwargs['extra_option']['secret'] = AESCrypto.encrypt(kwargs['extra_option']['secret']) + ci_type = CITypeCache.get(kwargs['type_id']) + unique = AttributeCache.get(ci_type.unique_id) + if unique and unique.name not in (kwargs.get('attributes') or {}).values(): + return abort(400, ErrFormat.ad_not_unique_key.format(unique.name)) + kwargs['uid'] = current_user.uid return kwargs @@ -276,7 +352,29 @@ class AutoDiscoveryCITypeCRUD(DBMixin): existed = self.cls.get_by_id(kwargs['_id']) or abort( 404, ErrFormat.ad_not_found.format("id={}".format(kwargs['_id']))) - self.__valid_exec_target(kwargs.get('agent_id'), kwargs.get('query_expr')) + adr = AutoDiscoveryRule.get_by_id(existed.adr_id) or abort( + 404, ErrFormat.adr_not_found.format("id={}".format(existed.adr_id))) + if adr.type == "http": + kwargs.setdefault('extra_option', dict) + en_name = None + for i in DEFAULT_HTTP: + if i['name'] == adr.name: + en_name = i['en'] + break + if en_name and kwargs['extra_option'].get('category'): + for item in ClOUD_MAP[en_name]: + if item["collect_key_map"].get(kwargs['extra_option']['category']): + kwargs["extra_option"]["collect_key"] = item["collect_key_map"][ + kwargs['extra_option']['category']] + break + + if 'attributes' in kwargs: + self.__valid_exec_target(kwargs.get('agent_id'), kwargs.get('query_expr')) + + ci_type = CITypeCache.get(existed.type_id) + unique = AttributeCache.get(ci_type.unique_id) + if unique and unique.name not in (kwargs.get('attributes') or {}).values(): + return abort(400, ErrFormat.ad_not_unique_key.format(unique.name)) if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'): if current_user.uid != existed.uid: @@ -292,7 +390,15 @@ class AutoDiscoveryCITypeCRUD(DBMixin): if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'): kwargs['extra_option']['secret'] = AESCrypto.encrypt(kwargs['extra_option']['secret']) - return super(AutoDiscoveryCITypeCRUD, self).update(_id, filter_none=False, **kwargs) + inst = self._can_update(_id=_id, **kwargs) + if inst.agent_id != kwargs.get('agent_id') or inst.query_expr != kwargs.get('query_expr'): + for item in AutoDiscoveryRuleSyncHistory.get_by(adt_id=inst.id, to_dict=False): + item.delete(commit=False) + db.session.commit() + + obj = inst.update(_id=_id, filter_none=False, **kwargs) + + return obj def _can_delete(self, **kwargs): if AutoDiscoveryCICRUD.get_by_adt_id(kwargs['_id']): @@ -303,6 +409,56 @@ class AutoDiscoveryCITypeCRUD(DBMixin): return existed + def delete(self, _id): + inst = self._can_delete(_id=_id) + + inst.soft_delete() + + for item in AutoDiscoveryRuleSyncHistory.get_by(adt_id=inst.id, to_dict=False): + item.delete(commit=False) + db.session.commit() + + attributes = self.get_ad_attributes(inst.type_id) + for item in AutoDiscoveryCITypeRelationCRUD.get_by_type_id(inst.type_id): + if item.ad_key not in attributes: + item.soft_delete() + + return inst + + +class AutoDiscoveryCITypeRelationCRUD(DBMixin): + cls = AutoDiscoveryCITypeRelation + + @classmethod + def get_by_type_id(cls, type_id, to_dict=False): + return cls.cls.get_by(ad_type_id=type_id, to_dict=to_dict) + + def upsert(self, ad_type_id, relations): + existed = self.cls.get_by(ad_type_id=ad_type_id, to_dict=False) + existed = {(i.ad_key, i.peer_type_id, i.peer_attr_id): i for i in existed} + + new = [] + for r in relations: + k = (r.get('ad_key'), r.get('peer_type_id'), r.get('peer_attr_id')) + if len(list(filter(lambda x: x, k))) == 3 and k not in existed: + self.cls.create(ad_type_id=ad_type_id, **r) + + new.append(k) + + for deleted in set(existed.keys()) - set(new): + existed[deleted].soft_delete() + + return self.get_by_type_id(ad_type_id, to_dict=True) + + def _can_add(self, **kwargs): + pass + + def _can_update(self, **kwargs): + pass + + def _can_delete(self, **kwargs): + pass + class AutoDiscoveryCICRUD(DBMixin): cls = AutoDiscoveryCI @@ -391,16 +547,24 @@ class AutoDiscoveryCICRUD(DBMixin): changed = False if existed is not None: if existed.instance != kwargs['instance']: + instance = copy.deepcopy(existed.instance) or {} + instance.update(kwargs['instance']) + kwargs['instance'] = instance existed.update(filter_none=False, **kwargs) + AutoDiscoveryExecHistoryCRUD().add(type_id=adt.type_id, + stdout="update resource: {}".format(kwargs.get('unique_value'))) changed = True else: existed = self.cls.create(**kwargs) + AutoDiscoveryExecHistoryCRUD().add(type_id=adt.type_id, + stdout="add resource: {}".format(kwargs.get('unique_value'))) changed = True if adt.auto_accept and changed: try: self.accept(existed) except Exception as e: + current_app.logger.error(e) return abort(400, str(e)) elif changed: existed.update(is_accept=False, accept_time=None, accept_by=None, filter_none=False) @@ -420,6 +584,13 @@ class AutoDiscoveryCICRUD(DBMixin): inst.delete() + adt = AutoDiscoveryCIType.get_by_id(inst.adt_id) + if adt: + adt.update(updated_at=datetime.datetime.now()) + + AutoDiscoveryExecHistoryCRUD().add(type_id=inst.type_id, + stdout="delete resource: {}".format(inst.unique_value)) + self._after_delete(inst) return inst @@ -435,6 +606,13 @@ class AutoDiscoveryCICRUD(DBMixin): not is_app_admin("cmdb") and validate_permission(ci_type.name, ResourceTypeEnum.CI, PermEnum.DELETE, "cmdb") existed.delete() + + adt = AutoDiscoveryCIType.get_by_id(existed.adt_id) + if adt: + adt.update(updated_at=datetime.datetime.now()) + + AutoDiscoveryExecHistoryCRUD().add(type_id=type_id, + stdout="delete resource: {}".format(unique_value)) # TODO: delete ci @classmethod @@ -447,32 +625,34 @@ class AutoDiscoveryCICRUD(DBMixin): ci_id = None if adt.attributes: ci_dict = {adt.attributes[k]: v for k, v in adc.instance.items() if k in adt.attributes} - ci_id = CIManager.add(adc.type_id, is_auto_discovery=True, **ci_dict) + ci_id = CIManager.add(adc.type_id, is_auto_discovery=True, _is_admin=True, **ci_dict) + AutoDiscoveryExecHistoryCRUD().add(type_id=adt.type_id, + stdout="accept resource: {}".format(adc.unique_value)) - relation_adts = AutoDiscoveryCIType.get_by(type_id=adt.type_id, adr_id=None, to_dict=False) - for r_adt in relation_adts: - if not r_adt.relation or ci_id is None: + relation_ads = AutoDiscoveryCITypeRelation.get_by(ad_type_id=adt.type_id, to_dict=False) + for r_adt in relation_ads: + ad_key = r_adt.ad_key + if not adc.instance.get(ad_key): continue - for ad_key in r_adt.relation: - if not adc.instance.get(ad_key): - continue - cmdb_key = r_adt.relation[ad_key] - query = "_type:{},{}:{}".format(cmdb_key.get('type_name'), cmdb_key.get('attr_name'), - adc.instance.get(ad_key)) - s = search(query) + + ad_key_values = [adc.instance.get(ad_key)] if not isinstance( + adc.instance.get(ad_key), list) else adc.instance.get(ad_key) + for ad_key_value in ad_key_values: + query = "_type:{},{}:{}".format(r_adt.peer_type_id, r_adt.peer_attr_id, ad_key_value) + s = ci_search(query, use_ci_filter=False, count=1000000) try: response, _, _, _, _, _ = s.search() except SearchError as e: current_app.logger.warning(e) return abort(400, str(e)) - relation_ci_id = response and response[0]['_id'] - if relation_ci_id: + for relation_ci in response: + relation_ci_id = relation_ci['_id'] try: - CIRelationManager.add(ci_id, relation_ci_id) + CIRelationManager.add(ci_id, relation_ci_id, valid=False) except: try: - CIRelationManager.add(relation_ci_id, ci_id) + CIRelationManager.add(relation_ci_id, ci_id, valid=False) except: pass @@ -485,14 +665,35 @@ class AutoDiscoveryCICRUD(DBMixin): class AutoDiscoveryHTTPManager(object): @staticmethod def get_categories(name): - return (ClOUD_MAP.get(name) or {}).get('categories') or [] + categories = (ClOUD_MAP.get(name) or {}) or [] + for item in copy.deepcopy(categories): + item.pop('map', None) + + return categories + + def get_resources(self, name): + en_name = None + for i in DEFAULT_HTTP: + if i['name'] == name: + en_name = i['en'] + break + + if en_name: + categories = self.get_categories(en_name) + + return [j for i in categories for j in i['items']] + + return [] @staticmethod - def get_attributes(name, category): - tpt = ((ClOUD_MAP.get(name) or {}).get('map') or {}).get(category) - if tpt and os.path.exists(os.path.join(PWD, tpt)): - with open(os.path.join(PWD, tpt)) as f: - return json.loads(f.read()) + def get_attributes(provider, resource): + for item in (ClOUD_MAP.get(provider) or {}): + for _resource in (item.get('map') or {}): + if _resource == resource: + tpt = item['map'][_resource] + if tpt and os.path.exists(os.path.join(PWD, tpt)): + with open(os.path.join(PWD, tpt)) as f: + return json.loads(f.read()) return [] @@ -506,3 +707,62 @@ class AutoDiscoverySNMPManager(object): return json.loads(f.read()) return [] + + +class AutoDiscoveryRuleSyncHistoryCRUD(DBMixin): + cls = AutoDiscoveryRuleSyncHistory + + def _can_add(self, **kwargs): + pass + + def _can_update(self, **kwargs): + pass + + def _can_delete(self, **kwargs): + pass + + def upsert(self, **kwargs): + existed = self.cls.get_by(adt_id=kwargs.get('adt_id'), + oneagent_id=kwargs.get('oneagent_id'), + oneagent_name=kwargs.get('oneagent_name'), + first=True, + to_dict=False) + + if existed is not None: + existed.update(**kwargs) + else: + self.cls.create(**kwargs) + + +class AutoDiscoveryExecHistoryCRUD(DBMixin): + cls = AutoDiscoveryExecHistory + + def _can_add(self, **kwargs): + pass + + def _can_update(self, **kwargs): + pass + + def _can_delete(self, **kwargs): + pass + + +class AutoDiscoveryCounterCRUD(DBMixin): + cls = AutoDiscoveryCounter + + def get(self, type_id): + res = self.cls.get_by(type_id=type_id, first=True, to_dict=True) + if res is None: + return dict(rule_count=0, exec_target_count=0, instance_count=0, accept_count=0, + this_month_count=0, this_week_count=0, last_month_count=0, last_week_count=0) + + return res + + def _can_add(self, **kwargs): + pass + + def _can_update(self, **kwargs): + pass + + def _can_delete(self, **kwargs): + pass diff --git a/cmdb-api/api/lib/cmdb/auto_discovery/const.py b/cmdb-api/api/lib/cmdb/auto_discovery/const.py index 7b0daab..9ca229f 100644 --- a/cmdb-api/api/lib/cmdb/auto_discovery/const.py +++ b/cmdb-api/api/lib/cmdb/auto_discovery/const.py @@ -3,13 +3,13 @@ from api.lib.cmdb.const import AutoDiscoveryType DEFAULT_HTTP = [ - dict(name="阿里云", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False, + dict(name="阿里云", en="aliyun", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False, option={'icon': {'name': 'caise-aliyun'}}), - dict(name="腾讯云", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False, + dict(name="腾讯云", en="tencentcloud", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False, option={'icon': {'name': 'caise-tengxunyun'}}), - dict(name="华为云", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False, + dict(name="华为云", en="huaweicloud", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False, option={'icon': {'name': 'caise-huaweiyun'}}), - dict(name="AWS", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False, + dict(name="AWS", en="aws", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False, option={'icon': {'name': 'caise-aws'}}), dict(name="交换机", type=AutoDiscoveryType.SNMP, is_inner=True, is_plugin=False, @@ -23,31 +23,47 @@ DEFAULT_HTTP = [ ] ClOUD_MAP = { - "aliyun": { - "categories": ["云服务器 ECS"], + "aliyun": [{ + "category": "计算", + "items": ["云服务器 ECS"], "map": { "云服务器 ECS": "templates/aliyun_ecs.json", + }, + "collect_key_map": { + "云服务器 ECS": "ali.ecs", } - }, + }], - "tencentcloud": { - "categories": ["云服务器 CVM"], + "tencentcloud": [{ + "category": "计算", + "items": ["云服务器 CVM"], "map": { "云服务器 CVM": "templates/tencent_cvm.json", + }, + "collect_key_map": { + "云服务器 CVM": "tencent.cvm", } - }, + }], - "huaweicloud": { - "categories": ["云服务器 ECS"], + "huaweicloud": [{ + "category": "计算", + "items": ["云服务器 ECS"], "map": { "云服务器 ECS": "templates/huaweicloud_ecs.json", + }, + "collect_key_map": { + "云服务器 ECS": "huawei.ecs", } - }, + }], - "aws": { - "categories": ["云服务器 EC2"], + "aws": [{ + "category": "计算", + "items": ["云服务器 EC2"], "map": { "云服务器 EC2": "templates/aws_ec2.json", + }, + "collect_key_map": { + "云服务器 EC2": "aws.ec2", } - }, + }], } diff --git a/cmdb-api/api/lib/cmdb/cache.py b/cmdb-api/api/lib/cmdb/cache.py index 1ed6797..f521cd0 100644 --- a/cmdb-api/api/lib/cmdb/cache.py +++ b/cmdb-api/api/lib/cmdb/cache.py @@ -2,12 +2,19 @@ from __future__ import unicode_literals +import datetime + from flask import current_app from api.extensions import cache from api.extensions import db from api.lib.cmdb.custom_dashboard import CustomDashboardManager -from api.models.cmdb import Attribute +from api.models.cmdb import Attribute, AutoDiscoveryExecHistory +from api.models.cmdb import AutoDiscoveryCI +from api.models.cmdb import AutoDiscoveryCIType +from api.models.cmdb import AutoDiscoveryCITypeRelation +from api.models.cmdb import AutoDiscoveryCounter +from api.models.cmdb import AutoDiscoveryRuleSyncHistory from api.models.cmdb import CI from api.models.cmdb import CIType from api.models.cmdb import CITypeAttribute @@ -448,7 +455,67 @@ class CMDBCounterCache(object): cache.set(cls.KEY2, result, timeout=0) - return result + res = db.session.query(AutoDiscoveryCI.created_at, + AutoDiscoveryCI.updated_at, + AutoDiscoveryCI.adt_id, + AutoDiscoveryCI.type_id, + AutoDiscoveryCI.is_accept).filter(AutoDiscoveryCI.deleted.is_(False)) + + today = datetime.datetime.today() + this_month = datetime.datetime(today.year, today.month, 1) + last_month = this_month - datetime.timedelta(days=1) + last_month = datetime.datetime(last_month.year, last_month.month, 1) + this_week = today - datetime.timedelta(days=datetime.date.weekday(today)) + this_week = datetime.datetime(this_week.year, this_week.month, this_week.day) + last_week = this_week - datetime.timedelta(days=7) + last_week = datetime.datetime(last_week.year, last_week.month, last_week.day) + result = dict() + for i in res: + if i.type_id not in result: + result[i.type_id] = dict(instance_count=0, accept_count=0, + this_month_count=0, this_week_count=0, last_month_count=0, last_week_count=0) + + adts = AutoDiscoveryCIType.get_by(type_id=i.type_id, to_dict=False) + result[i.type_id]['rule_count'] = len(adts) + AutoDiscoveryCITypeRelation.get_by( + ad_type_id=i.type_id, only_query=True).count() + result[i.type_id]['exec_target_count'] = len( + set([i.oneagent_id for adt in adts for i in db.session.query( + AutoDiscoveryRuleSyncHistory.oneagent_id).filter( + AutoDiscoveryRuleSyncHistory.adt_id == adt.id)])) + + result[i.type_id]['instance_count'] += 1 + if i.is_accept: + result[i.type_id]['accept_count'] += 1 + + if last_month <= i.created_at < this_month: + result[i.type_id]['last_month_count'] += 1 + elif i.created_at >= this_month: + result[i.type_id]['this_month_count'] += 1 + + if last_week <= i.created_at < this_week: + result[i.type_id]['last_week_count'] += 1 + elif i.created_at >= this_week: + result[i.type_id]['this_week_count'] += 1 + + for type_id in result: + existed = AutoDiscoveryCounter.get_by(type_id=type_id, first=True, to_dict=False) + if existed is None: + AutoDiscoveryCounter.create(type_id=type_id, **result[type_id]) + else: + existed.update(**result[type_id]) + + for i in AutoDiscoveryCounter.get_by(to_dict=False): + if i.type_id not in result: + i.delete() + + @classmethod + def clear_ad_exec_history(cls): + ci_types = CIType.get_by(to_dict=False) + for ci_type in ci_types: + for i in AutoDiscoveryExecHistory.get_by(type_id=ci_type.id, only_query=True).order_by( + AutoDiscoveryExecHistory.id.desc()).offset(50000): + i.delete(commit=False) + db.session.commit() @classmethod def get_adc_counter(cls): diff --git a/cmdb-api/api/lib/cmdb/ci.py b/cmdb-api/api/lib/cmdb/ci.py index 588791e..d3a1047 100644 --- a/cmdb-api/api/lib/cmdb/ci.py +++ b/cmdb-api/api/lib/cmdb/ci.py @@ -223,7 +223,7 @@ class CIManager(object): def ci_is_exist(unique_key, unique_value, type_id): """ - :param unique_key: is a attribute + :param unique_key: is an attribute :param unique_value: :param type_id: :return: @@ -432,7 +432,7 @@ class CIManager(object): for attr_id in password_dict: record_id = cls.save_password(ci.id, attr_id, password_dict[attr_id], record_id, ci_type.id) - if record_id or has_dynamic: # has change + if record_id or has_dynamic: # has changed ci_cache.apply_async(args=(ci.id, operate_type, record_id), queue=CMDB_QUEUE) if ref_ci_dict: # add relations @@ -504,7 +504,7 @@ class CIManager(object): for attr_id in password_dict: record_id = self.save_password(ci.id, attr_id, password_dict[attr_id], record_id, ci.type_id) - if record_id or has_dynamic: # has change + if record_id or has_dynamic: # has changed if not __sync: ci_cache.apply_async(args=(ci_id, OperateType.UPDATE, record_id), queue=CMDB_QUEUE) else: @@ -737,7 +737,7 @@ class CIManager(object): fields=None, value_tables=None, unique_required=False, excludes=None): """ - :param ci_ids: list of CI instance ID, eg. ['1', '2'] + :param ci_ids: list of CI instance ID, e.g. ['1', '2'] :param ret_key: name, id or alias :param fields: :param value_tables: @@ -1296,7 +1296,7 @@ class CIRelationManager(object): relations = _relations else: relations &= _relations - for parent_ci_id, child_ci_id in relations: + for parent_ci_id, child_ci_id in (relations or []): CIRelationManager.add(parent_ci_id, child_ci_id, valid=False) parent_items = CITypeRelation.get_by(child_id=type_id, only_query=True).filter( @@ -1316,7 +1316,7 @@ class CIRelationManager(object): relations = _relations else: relations &= _relations - for parent_ci_id, child_ci_id in relations: + for parent_ci_id, child_ci_id in (relations or []): CIRelationManager.add(parent_ci_id, child_ci_id, valid=False) @classmethod diff --git a/cmdb-api/api/lib/cmdb/ci_type.py b/cmdb-api/api/lib/cmdb/ci_type.py index b05bbda..3b66302 100644 --- a/cmdb-api/api/lib/cmdb/ci_type.py +++ b/cmdb-api/api/lib/cmdb/ci_type.py @@ -229,6 +229,9 @@ class CITypeManager(object): if CI.get_by(type_id=type_id, first=True, to_dict=False) is not None: return abort(400, ErrFormat.ci_exists_and_cannot_delete_type) + if CITypeInheritance.get_by(parent_id=type_id, first=True): + return abort(400, ErrFormat.ci_type_inheritance_cannot_delete) + relation_views = PreferenceRelationView.get_by(to_dict=False) for rv in relation_views: for item in (rv.cr_ids or []): @@ -253,21 +256,21 @@ class CITypeManager(object): item.delete(commit=False) for item in AutoDiscoveryCITypeRelation.get_by(ad_type_id=type_id, to_dict=False): - item.delete(commit=False) + item.soft_delete(commit=False) for item in AutoDiscoveryCITypeRelation.get_by(peer_type_id=type_id, to_dict=False): - item.delete(commit=False) + item.soft_delete(commit=False) for item in CITypeInheritance.get_by(parent_id=type_id, to_dict=False): - item.delete(commit=False) + item.soft_delete(commit=False) for item in CITypeInheritance.get_by(child_id=type_id, to_dict=False): - item.delete(commit=False) + item.soft_delete(commit=False) try: from api.models.cmdb import CITypeReconciliation for item in CITypeReconciliation.get_by(type_id=type_id, to_dict=False): - item.delete(commit=False) + item.soft_delete(commit=False) except Exception: pass diff --git a/cmdb-api/api/lib/cmdb/const.py b/cmdb-api/api/lib/cmdb/const.py index 6e44438..3452129 100644 --- a/cmdb-api/api/lib/cmdb/const.py +++ b/cmdb-api/api/lib/cmdb/const.py @@ -55,9 +55,9 @@ class CITypeOperateType(BaseEnum): DELETE_UNIQUE_CONSTRAINT = "11" # 删除联合唯一 ADD_RELATION = "12" # 新增关系 DELETE_RELATION = "13" # 删除关系 - ADD_RECONCILIATION = "14" # 删除关系 - UPDATE_RECONCILIATION = "15" # 删除关系 - DELETE_RECONCILIATION = "16" # 删除关系 + ADD_RECONCILIATION = "14" # 新增数据合规 + UPDATE_RECONCILIATION = "15" # 修改数据合规 + DELETE_RECONCILIATION = "16" # 删除数据合规 class RetKey(BaseEnum): diff --git a/cmdb-api/api/lib/cmdb/history.py b/cmdb-api/api/lib/cmdb/history.py index b5e388d..6e85038 100644 --- a/cmdb-api/api/lib/cmdb/history.py +++ b/cmdb-api/api/lib/cmdb/history.py @@ -13,6 +13,7 @@ from api.lib.cmdb.const import OperateType from api.lib.cmdb.perms import CIFilterPermsCRUD from api.lib.cmdb.resp_format import ErrFormat from api.lib.perm.acl.cache import UserCache +from api.models.cmdb import CI from api.models.cmdb import Attribute from api.models.cmdb import AttributeHistory from api.models.cmdb import CIRelationHistory @@ -306,7 +307,7 @@ class CITriggerHistoryManager(object): def get(page, page_size, type_id=None, trigger_id=None, operate_type=None): query = CITriggerHistory.get_by(only_query=True) if type_id: - query = query.filter(CITriggerHistory.type_id == type_id) + query = query.join(CI, CI.id == CITriggerHistory.ci_id).filter(CI.type_id == type_id) if trigger_id: query = query.filter(CITriggerHistory.trigger_id == trigger_id) diff --git a/cmdb-api/api/lib/cmdb/resp_format.py b/cmdb-api/api/lib/cmdb/resp_format.py index 39418f5..245e775 100644 --- a/cmdb-api/api/lib/cmdb/resp_format.py +++ b/cmdb-api/api/lib/cmdb/resp_format.py @@ -62,6 +62,7 @@ class ErrFormat(CommonErrFormat): "The model cannot be deleted because the CI already exists") # 因为CI已经存在，不能删除模型 ci_exists_and_cannot_delete_inheritance = _l( "The inheritance cannot be deleted because the CI already exists") # 因为CI已经存在，不能删除继承关系 + ci_type_inheritance_cannot_delete = _l("The model is inherited and cannot be deleted") # 该模型被继承, 不能删除 # 因为关系视图 {} 引用了该模型，不能删除模型 ci_relation_view_exists_and_cannot_delete_type = _l( diff --git a/cmdb-api/api/lib/cmdb/topology.py b/cmdb-api/api/lib/cmdb/topology.py index 103d2ee..bc0c1a6 100644 --- a/cmdb-api/api/lib/cmdb/topology.py +++ b/cmdb-api/api/lib/cmdb/topology.py @@ -16,7 +16,7 @@ from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION from api.lib.cmdb.const import ResourceTypeEnum from api.lib.cmdb.resp_format import ErrFormat from api.lib.cmdb.search import SearchError -from api.lib.cmdb.search.ci import search +from api.lib.cmdb.search.ci import search as ci_search from api.lib.perm.acl.acl import ACLManager from api.lib.perm.acl.acl import is_app_admin from api.models.cmdb import TopologyView @@ -178,7 +178,7 @@ class TopologyViewManager(object): q = (central_node_instances[2:] if central_node_instances.startswith('q=') else central_node_instances) - s = search(q, fl=['_id', show_key.name], use_id_filter=False, use_ci_filter=False, count=1000000) + s = ci_search(q, fl=['_id', show_key.name], use_id_filter=False, use_ci_filter=False, count=1000000) try: response, _, _, _, _, _ = s.search() except SearchError as e: @@ -238,7 +238,7 @@ class TopologyViewManager(object): type2show[type_id] = attr.name if id2node: - s = search("_id:({})".format(';'.join(id2node.keys())), fl=list(fl), + s = ci_search("_id:({})".format(';'.join(id2node.keys())), fl=list(fl), use_id_filter=False, use_ci_filter=False, count=1000000) try: response, _, _, _, _, _ = s.search() diff --git a/cmdb-api/api/lib/mixin.py b/cmdb-api/api/lib/mixin.py index 75123fe..dc57232 100644 --- a/cmdb-api/api/lib/mixin.py +++ b/cmdb-api/api/lib/mixin.py @@ -8,6 +8,8 @@ from api.extensions import db from api.lib.utils import get_page from api.lib.utils import get_page_size +__author__ = 'pycook' + class DBMixin(object): cls = None @@ -17,13 +19,18 @@ class DBMixin(object): page = get_page(page) page_size = get_page_size(page_size) if fl is None: - query = db.session.query(cls.cls).filter(cls.cls.deleted.is_(False)) + query = db.session.query(cls.cls) else: - query = db.session.query(*[getattr(cls.cls, i) for i in fl]).filter(cls.cls.deleted.is_(False)) + query = db.session.query(*[getattr(cls.cls, i) for i in fl]) _query = None if count_query: - _query = db.session.query(func.count(cls.cls.id)).filter(cls.cls.deleted.is_(False)) + _query = db.session.query(func.count(cls.cls.id)) + + if hasattr(cls.cls, 'deleted'): + query = query.filter(cls.cls.deleted.is_(False)) + if _query: + _query = _query.filter(cls.cls.deleted.is_(False)) for k in kwargs: if hasattr(cls.cls, k): diff --git a/cmdb-api/api/lib/perm/acl/acl.py b/cmdb-api/api/lib/perm/acl/acl.py index 8591f49..a829401 100644 --- a/cmdb-api/api/lib/perm/acl/acl.py +++ b/cmdb-api/api/lib/perm/acl/acl.py @@ -253,9 +253,6 @@ def is_app_admin(app=None): if app is None: return False - if hasattr(current_user, 'username') and current_user.username == 'worker': - return True - app_id = app.id if 'acl_admin' in session.get("acl", {}).get("parentRoles", []): return True diff --git a/cmdb-api/api/lib/resp_format.py b/cmdb-api/api/lib/resp_format.py index 48eca6f..18ff0c6 100644 --- a/cmdb-api/api/lib/resp_format.py +++ b/cmdb-api/api/lib/resp_format.py @@ -29,6 +29,6 @@ class CommonErrFormat(object): role_required = _l("Role {} can only operate!") # 角色 {} 才能操作! user_not_found = _l("User {} does not exist") # 用户 {} 不存在 - no_permission = _l("You do not have {} permission for resource: {}!") # 您没有资源: {} 的{}权限! + no_permission = _l("For resource: {}, you do not have {} permission!") # 您没有资源: {} 的{}权限! no_permission2 = _l("You do not have permission to operate!") # 您没有操作权限! no_permission_only_owner = _l("Only the creator or administrator has permission!") # 只有创建人或者管理员才有权限! diff --git a/cmdb-api/api/models/cmdb.py b/cmdb-api/api/models/cmdb.py index 342838d..7a5bbb7 100644 --- a/cmdb-api/api/models/cmdb.py +++ b/cmdb-api/api/models/cmdb.py @@ -566,14 +566,14 @@ class AutoDiscoveryCIType(Model): attributes = db.Column(db.JSON) # {ad_key: cmdb_key} - relation = db.Column(db.JSON) # [{ad_key: {type_id: x, attr_id: x}}] + relation = db.Column(db.JSON) # [{ad_key: {type_id: x, attr_id: x}}], CMDB > 2.4.5: deprecated auto_accept = db.Column(db.Boolean, default=False) agent_id = db.Column(db.String(8), index=True) query_expr = db.Column(db.Text) - interval = db.Column(db.Integer) # seconds + interval = db.Column(db.Integer) # seconds, > 2.4.5: deprecated cron = db.Column(db.String(128)) extra_option = db.Column(db.JSON) @@ -604,6 +604,36 @@ class AutoDiscoveryCI(Model): accept_time = db.Column(db.DateTime) +class AutoDiscoveryRuleSyncHistory(Model2): + __tablename__ = "c_ad_rule_sync_histories" + + adt_id = db.Column(db.Integer, db.ForeignKey('c_ad_ci_types.id')) + oneagent_id = db.Column(db.String(8)) + oneagent_name = db.Column(db.String(64)) + sync_at = db.Column(db.DateTime, default=datetime.datetime.now()) + + +class AutoDiscoveryExecHistory(Model2): + __tablename__ = "c_ad_exec_histories" + + type_id = db.Column(db.Integer, index=True) + stdout = db.Column(db.Text) + + +class AutoDiscoveryCounter(Model2): + __tablename__ = "c_ad_counter" + + type_id = db.Column(db.Integer, index=True) + rule_count = db.Column(db.Integer, default=0) + exec_target_count = db.Column(db.Integer, default=0) + instance_count = db.Column(db.Integer, default=0) + accept_count = db.Column(db.Integer, default=0) + this_month_count = db.Column(db.Integer, default=0) + this_week_count = db.Column(db.Integer, default=0) + last_month_count = db.Column(db.Integer, default=0) + last_week_count = db.Column(db.Integer, default=0) + + class CIFilterPerms(Model): __tablename__ = "c_ci_filter_perms" diff --git a/cmdb-api/api/tasks/cmdb.py b/cmdb-api/api/tasks/cmdb.py index 78c1788..af94423 100644 --- a/cmdb-api/api/tasks/cmdb.py +++ b/cmdb-api/api/tasks/cmdb.py @@ -2,6 +2,7 @@ import json +import datetime import redis_lock from flask import current_app @@ -25,6 +26,8 @@ from api.lib.utils import handle_arg_list from api.models.cmdb import CI from api.models.cmdb import CIRelation from api.models.cmdb import CITypeAttribute +from api.models.cmdb import AutoDiscoveryCI +from api.models.cmdb import AutoDiscoveryCIType @celery.task(name="cmdb.ci_cache", queue=CMDB_QUEUE) @@ -87,6 +90,13 @@ def ci_delete(ci_id): else: rd.delete(ci_id, REDIS_PREFIX_CI) + instance = AutoDiscoveryCI.get_by(ci_id=ci_id, to_dict=False, first=True) + if instance is not None: + adt = AutoDiscoveryCIType.get_by_id(instance.adt_id) + if adt: + adt.update(updated_at=datetime.datetime.now()) + instance.delete() + current_app.logger.info("{0} delete..........".format(ci_id)) @@ -249,3 +259,21 @@ def calc_computed_attribute(attr_id, uid): cis = CI.get_by(type_id=i.type_id, to_dict=False) for ci in cis: cim.update(ci.id, {}) + + +@celery.task(name="cmdb.write_ad_rule_sync_history", queue=CMDB_QUEUE) +@reconnect_db +def write_ad_rule_sync_history(rules, oneagent_id, oneagent_name, sync_at): + from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryRuleSyncHistoryCRUD + + for rule in rules: + AutoDiscoveryRuleSyncHistoryCRUD().upsert(adt_id=rule['id'], + oneagent_id=oneagent_id, + oneagent_name=oneagent_name, + sync_at=sync_at, + commit=False) + try: + db.session.commit() + except Exception as e: + current_app.logger.error("write auto discovery rule sync history failed: {}".format(e)) + db.session.rollback() diff --git a/cmdb-api/api/translations/zh/LC_MESSAGES/messages.mo b/cmdb-api/api/translations/zh/LC_MESSAGES/messages.mo index cb5f170..a62ae1d 100644 Binary files a/cmdb-api/api/translations/zh/LC_MESSAGES/messages.mo and b/cmdb-api/api/translations/zh/LC_MESSAGES/messages.mo differ diff --git a/cmdb-api/api/translations/zh/LC_MESSAGES/messages.po b/cmdb-api/api/translations/zh/LC_MESSAGES/messages.po index 4783c26..38c8949 100644 --- a/cmdb-api/api/translations/zh/LC_MESSAGES/messages.po +++ b/cmdb-api/api/translations/zh/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2024-05-28 18:05+0800\n" +"POT-Creation-Date: 2024-06-20 19:12+0800\n" "PO-Revision-Date: 2023-12-25 20:21+0800\n" "Last-Translator: FULL NAME \n" "Language: zh\n" @@ -81,7 +81,7 @@ msgid "User {} does not exist" msgstr "用户 {} 不存在" #: api/lib/resp_format.py:32 -msgid "You do not have {} permission for resource: {}!" +msgid "For resource: {}, you do not have {} permission!" msgstr "您没有资源: {} 的{}权限!" #: api/lib/resp_format.py:33 @@ -238,241 +238,245 @@ msgstr "因为CI已经存在，不能删除模型" msgid "The inheritance cannot be deleted because the CI already exists" msgstr "因为CI已经存在，不能删除继承关系" -#: api/lib/cmdb/resp_format.py:67 +#: api/lib/cmdb/resp_format.py:65 +msgid "The model is inherited and cannot be deleted" +msgstr "该模型被继承, 不能删除" + +#: api/lib/cmdb/resp_format.py:68 msgid "" "The model cannot be deleted because the model is referenced by the " "relational view {}" msgstr "因为关系视图 {} 引用了该模型，不能删除模型" -#: api/lib/cmdb/resp_format.py:69 +#: api/lib/cmdb/resp_format.py:70 msgid "Model group {} does not exist" msgstr "模型分组 {} 不存在" -#: api/lib/cmdb/resp_format.py:70 +#: api/lib/cmdb/resp_format.py:71 msgid "Model group {} already exists" msgstr "模型分组 {} 已经存在" -#: api/lib/cmdb/resp_format.py:71 +#: api/lib/cmdb/resp_format.py:72 msgid "Model relationship {} does not exist" msgstr "模型关系 {} 不存在" -#: api/lib/cmdb/resp_format.py:72 +#: api/lib/cmdb/resp_format.py:73 msgid "Attribute group {} already exists" msgstr "属性分组 {} 已存在" -#: api/lib/cmdb/resp_format.py:73 +#: api/lib/cmdb/resp_format.py:74 msgid "Attribute group {} does not exist" msgstr "属性分组 {} 不存在" -#: api/lib/cmdb/resp_format.py:75 +#: api/lib/cmdb/resp_format.py:76 msgid "Attribute group <{0}> - attribute <{1}> does not exist" msgstr "属性组<{0}> - 属性<{1}> 不存在" -#: api/lib/cmdb/resp_format.py:76 +#: api/lib/cmdb/resp_format.py:77 msgid "The unique constraint already exists!" msgstr "唯一约束已经存在!" -#: api/lib/cmdb/resp_format.py:78 +#: api/lib/cmdb/resp_format.py:79 msgid "Uniquely constrained attributes cannot be JSON and multi-valued" msgstr "唯一约束的属性不能是 JSON 和 多值" -#: api/lib/cmdb/resp_format.py:79 +#: api/lib/cmdb/resp_format.py:80 msgid "Duplicated trigger" msgstr "重复的触发器" -#: api/lib/cmdb/resp_format.py:80 +#: api/lib/cmdb/resp_format.py:81 msgid "Trigger {} does not exist" msgstr "触发器 {} 不存在" -#: api/lib/cmdb/resp_format.py:81 +#: api/lib/cmdb/resp_format.py:82 msgid "Duplicated reconciliation rule" msgstr "" -#: api/lib/cmdb/resp_format.py:82 +#: api/lib/cmdb/resp_format.py:83 msgid "Reconciliation rule {} does not exist" msgstr "关系类型 {} 不存在" -#: api/lib/cmdb/resp_format.py:84 +#: api/lib/cmdb/resp_format.py:85 msgid "Operation record {} does not exist" msgstr "操作记录 {} 不存在" -#: api/lib/cmdb/resp_format.py:85 +#: api/lib/cmdb/resp_format.py:86 msgid "Unique identifier cannot be deleted" msgstr "不能删除唯一标识" -#: api/lib/cmdb/resp_format.py:86 +#: api/lib/cmdb/resp_format.py:87 msgid "Cannot delete default sorted attributes" msgstr "不能删除默认排序的属性" -#: api/lib/cmdb/resp_format.py:88 +#: api/lib/cmdb/resp_format.py:89 msgid "No node selected" msgstr "没有选择节点" -#: api/lib/cmdb/resp_format.py:89 +#: api/lib/cmdb/resp_format.py:90 msgid "This search option does not exist!" msgstr "该搜索选项不存在!" -#: api/lib/cmdb/resp_format.py:90 +#: api/lib/cmdb/resp_format.py:91 msgid "This search option has a duplicate name!" msgstr "该搜索选项命名重复!" -#: api/lib/cmdb/resp_format.py:92 +#: api/lib/cmdb/resp_format.py:93 msgid "Relationship type {} already exists" msgstr "关系类型 {} 已经存在" -#: api/lib/cmdb/resp_format.py:93 +#: api/lib/cmdb/resp_format.py:94 msgid "Relationship type {} does not exist" msgstr "关系类型 {} 不存在" -#: api/lib/cmdb/resp_format.py:95 +#: api/lib/cmdb/resp_format.py:96 msgid "Invalid attribute value: {}" msgstr "无效的属性值: {}" -#: api/lib/cmdb/resp_format.py:96 +#: api/lib/cmdb/resp_format.py:97 msgid "{} Invalid value: {}" msgstr "{} 无效的值: {}" -#: api/lib/cmdb/resp_format.py:97 +#: api/lib/cmdb/resp_format.py:98 msgid "{} is not in the predefined values" msgstr "{} 不在预定义值里" -#: api/lib/cmdb/resp_format.py:99 +#: api/lib/cmdb/resp_format.py:100 msgid "The value of attribute {} must be unique, {} already exists" msgstr "属性 {} 的值必须是唯一的, 当前值 {} 已存在" -#: api/lib/cmdb/resp_format.py:100 +#: api/lib/cmdb/resp_format.py:101 msgid "Attribute {} value must exist" msgstr "属性 {} 值必须存在" -#: api/lib/cmdb/resp_format.py:101 +#: api/lib/cmdb/resp_format.py:102 msgid "Out of range value, the maximum value is 2147483647" msgstr "超过最大值限制, 最大值是2147483647" -#: api/lib/cmdb/resp_format.py:103 +#: api/lib/cmdb/resp_format.py:104 msgid "Unknown error when adding or modifying attribute value: {}" msgstr "新增或者修改属性值未知错误: {}" -#: api/lib/cmdb/resp_format.py:105 +#: api/lib/cmdb/resp_format.py:106 msgid "Duplicate custom name" msgstr "订制名重复" -#: api/lib/cmdb/resp_format.py:107 +#: api/lib/cmdb/resp_format.py:108 msgid "Number of models exceeds limit: {}" msgstr "模型数超过限制: {}" -#: api/lib/cmdb/resp_format.py:108 +#: api/lib/cmdb/resp_format.py:109 msgid "The number of CIs exceeds the limit: {}" msgstr "CI数超过限制: {}" -#: api/lib/cmdb/resp_format.py:110 +#: api/lib/cmdb/resp_format.py:111 msgid "Auto-discovery rule: {} already exists!" msgstr "自动发现规则: {} 已经存在!" -#: api/lib/cmdb/resp_format.py:111 +#: api/lib/cmdb/resp_format.py:112 msgid "Auto-discovery rule: {} does not exist!" msgstr "自动发现规则: {} 不存在!" -#: api/lib/cmdb/resp_format.py:113 +#: api/lib/cmdb/resp_format.py:114 msgid "This auto-discovery rule is referenced by the model and cannot be deleted!" msgstr "该自动发现规则被模型引用, 不能删除!" -#: api/lib/cmdb/resp_format.py:115 +#: api/lib/cmdb/resp_format.py:116 msgid "The application of auto-discovery rules cannot be defined repeatedly!" msgstr "自动发现规则的应用不能重复定义!" -#: api/lib/cmdb/resp_format.py:116 +#: api/lib/cmdb/resp_format.py:117 msgid "The auto-discovery you want to modify: {} does not exist!" msgstr "您要修改的自动发现: {} 不存在!" -#: api/lib/cmdb/resp_format.py:117 +#: api/lib/cmdb/resp_format.py:118 msgid "Attribute does not include unique identifier: {}" msgstr "属性字段没有包括唯一标识: {}" -#: api/lib/cmdb/resp_format.py:118 +#: api/lib/cmdb/resp_format.py:119 msgid "The auto-discovery instance does not exist!" msgstr "自动发现的实例不存在!" -#: api/lib/cmdb/resp_format.py:119 +#: api/lib/cmdb/resp_format.py:120 msgid "The model is not associated with this auto-discovery!" msgstr "模型并未关联该自动发现!" -#: api/lib/cmdb/resp_format.py:120 +#: api/lib/cmdb/resp_format.py:121 msgid "Only the creator can modify the Secret!" msgstr "只有创建人才能修改Secret!" -#: api/lib/cmdb/resp_format.py:122 +#: api/lib/cmdb/resp_format.py:123 msgid "This rule already has auto-discovery instances and cannot be deleted!" msgstr "该规则已经有自动发现的实例, 不能被删除!" -#: api/lib/cmdb/resp_format.py:124 +#: api/lib/cmdb/resp_format.py:125 msgid "The default auto-discovery rule is already referenced by model {}!" msgstr "该默认的自动发现规则 已经被模型 {} 引用!" -#: api/lib/cmdb/resp_format.py:126 +#: api/lib/cmdb/resp_format.py:127 msgid "The unique_key method must return a non-empty string!" msgstr "unique_key方法必须返回非空字符串!" -#: api/lib/cmdb/resp_format.py:127 +#: api/lib/cmdb/resp_format.py:128 msgid "The attributes method must return a list" msgstr "attributes方法必须返回的是list" -#: api/lib/cmdb/resp_format.py:129 +#: api/lib/cmdb/resp_format.py:130 msgid "The list returned by the attributes method cannot be empty!" msgstr "attributes方法返回的list不能为空!" -#: api/lib/cmdb/resp_format.py:131 +#: api/lib/cmdb/resp_format.py:132 msgid "Only administrators can define execution targets as: all nodes!" msgstr "只有管理员才可以定义执行机器为: 所有节点!" -#: api/lib/cmdb/resp_format.py:132 +#: api/lib/cmdb/resp_format.py:133 msgid "Execute targets permission check failed: {}" msgstr "执行机器权限检查不通过: {}" -#: api/lib/cmdb/resp_format.py:134 +#: api/lib/cmdb/resp_format.py:135 msgid "CI filter authorization must be named!" msgstr "CI过滤授权 必须命名!" -#: api/lib/cmdb/resp_format.py:135 +#: api/lib/cmdb/resp_format.py:136 msgid "CI filter authorization is currently not supported or query" msgstr "CI过滤授权 暂时不支持 或 查询" -#: api/lib/cmdb/resp_format.py:138 +#: api/lib/cmdb/resp_format.py:139 msgid "You do not have permission to operate attribute {}!" msgstr "您没有属性 {} 的操作权限!" -#: api/lib/cmdb/resp_format.py:139 +#: api/lib/cmdb/resp_format.py:140 msgid "You do not have permission to operate this CI!" msgstr "您没有该CI的操作权限!" -#: api/lib/cmdb/resp_format.py:141 +#: api/lib/cmdb/resp_format.py:142 msgid "Failed to save password: {}" msgstr "保存密码失败: {}" -#: api/lib/cmdb/resp_format.py:142 +#: api/lib/cmdb/resp_format.py:143 msgid "Failed to get password: {}" msgstr "获取密码失败: {}" -#: api/lib/cmdb/resp_format.py:144 +#: api/lib/cmdb/resp_format.py:145 msgid "Scheduling time format error" msgstr "{}格式错误，应该为：%Y-%m-%d %H:%M:%S" -#: api/lib/cmdb/resp_format.py:145 +#: api/lib/cmdb/resp_format.py:146 msgid "CMDB data reconciliation results" msgstr "" -#: api/lib/cmdb/resp_format.py:146 +#: api/lib/cmdb/resp_format.py:147 msgid "Number of {} illegal: {}" msgstr "" -#: api/lib/cmdb/resp_format.py:148 +#: api/lib/cmdb/resp_format.py:149 msgid "Topology view {} already exists" msgstr "拓扑视图 {} 已经存在" -#: api/lib/cmdb/resp_format.py:149 +#: api/lib/cmdb/resp_format.py:150 msgid "Topology group {} already exists" msgstr "拓扑视图分组 {} 已经存在" -#: api/lib/cmdb/resp_format.py:151 +#: api/lib/cmdb/resp_format.py:152 msgid "The group cannot be deleted because the topology view already exists" msgstr "因为该分组下定义了拓扑视图，不能删除" diff --git a/cmdb-api/api/views/cmdb/auto_discovery.py b/cmdb-api/api/views/cmdb/auto_discovery.py index 958d831..5e3ead8 100644 --- a/cmdb-api/api/views/cmdb/auto_discovery.py +++ b/cmdb-api/api/views/cmdb/auto_discovery.py @@ -1,6 +1,7 @@ # -*- coding:utf-8 -*- - +import copy import json +import uuid from io import BytesIO from flask import abort @@ -10,15 +11,19 @@ from flask_login import current_user from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCICRUD from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeCRUD +from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeRelationCRUD +from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCounterCRUD +from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryExecHistoryCRUD from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryHTTPManager from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryRuleCRUD +from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryRuleSyncHistoryCRUD from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoverySNMPManager from api.lib.cmdb.auto_discovery.const import DEFAULT_HTTP from api.lib.cmdb.const import PermEnum from api.lib.cmdb.const import ResourceTypeEnum from api.lib.cmdb.resp_format import ErrFormat from api.lib.cmdb.search import SearchError -from api.lib.cmdb.search.ci import search +from api.lib.cmdb.search.ci import search as ci_search from api.lib.decorator import args_required from api.lib.decorator import args_validate from api.lib.perm.acl.acl import has_perm_from_args @@ -37,14 +42,19 @@ class AutoDiscoveryRuleView(APIView): rebuild = False exists = {i['name'] for i in res} - for i in DEFAULT_HTTP: + for i in copy.deepcopy(DEFAULT_HTTP): if i['name'] not in exists: + i.pop('en', None) AutoDiscoveryRuleCRUD().add(**i) rebuild = True if rebuild: _, res = AutoDiscoveryRuleCRUD.search(page=1, page_size=100000, **request.values) + for i in res: + if i['type'] == 'http': + i['resources'] = AutoDiscoveryHTTPManager().get_resources(i['name']) + return self.jsonify(res) @args_required("name", value_required=True) @@ -98,7 +108,8 @@ class AutoDiscoveryRuleTemplateFileView(APIView): class AutoDiscoveryRuleHTTPView(APIView): - url_prefix = ("/adr/http//categories", "/adr/http//attributes", + url_prefix = ("/adr/http//categories", + "/adr/http//attributes", "/adr/snmp//attributes") def get(self, name): @@ -106,16 +117,21 @@ class AutoDiscoveryRuleHTTPView(APIView): return self.jsonify(AutoDiscoverySNMPManager.get_attributes()) if "attributes" in request.url: - category = request.values.get('category') - return self.jsonify(AutoDiscoveryHTTPManager.get_attributes(name, category)) + resource = request.values.get('resource') + return self.jsonify(AutoDiscoveryHTTPManager.get_attributes(name, resource)) return self.jsonify(AutoDiscoveryHTTPManager.get_categories(name)) class AutoDiscoveryCITypeView(APIView): - url_prefix = ("/adt/ci_types/", "/adt/") + url_prefix = ("/adt/ci_types/", + "/adt/ci_types//attributes", + "/adt/") def get(self, type_id): + if "attributes" in request.url: + return self.jsonify(AutoDiscoveryCITypeCRUD.get_ad_attributes(type_id)) + _, res = AutoDiscoveryCITypeCRUD.search(page=1, page_size=100000, type_id=type_id, **request.values) for i in res: if isinstance(i.get("extra_option"), dict) and i['extra_option'].get('secret'): @@ -146,6 +162,27 @@ class AutoDiscoveryCITypeView(APIView): return self.jsonify(adt_id=adt_id) +class AutoDiscoveryCITypeRelationView(APIView): + url_prefix = ("/adt/ci_types//relations", "/adt/relations/") + + def get(self, type_id): + _, res = AutoDiscoveryCITypeRelationCRUD.search(page=1, page_size=100000, ad_type_id=type_id, **request.values) + + return self.jsonify(res) + + @args_required("relations") + def post(self, type_id): + return self.jsonify(AutoDiscoveryCITypeRelationCRUD().upsert(type_id, request.values['relations'])) + + def put(self): + return self.post() + + def delete(self, _id): + AutoDiscoveryCITypeRelationCRUD().delete(_id) + + return self.jsonify(id=_id) + + class AutoDiscoveryCIView(APIView): url_prefix = ("/adc", "/adc/", "/adc/ci_types//attributes", "/adc/ci_types") @@ -220,9 +257,8 @@ class AutoDiscoveryRuleSyncView(APIView): oneagent_id = request.values.get('oneagent_id') last_update_at = request.values.get('last_update_at') - query = "{},oneagent_id:{}".format(oneagent_name, oneagent_id) - current_app.logger.info(query) - s = search(query) + query = "oneagent_id:{}".format(oneagent_id) + s = ci_search(query) try: response, _, _, _, _, _ = s.search() except SearchError as e: @@ -230,7 +266,77 @@ class AutoDiscoveryRuleSyncView(APIView): current_app.logger.error(traceback.format_exc()) return abort(400, str(e)) - ci_id = response and response[0]["_id"] - rules, last_update_at = AutoDiscoveryCITypeCRUD.get(ci_id, oneagent_id, last_update_at) + for res in response: + if res.get('{}_name'.format(res['ci_type'])) == oneagent_name or oneagent_name == res.get('oneagent_name'): + ci_id = res["_id"] + rules, last_update_at = AutoDiscoveryCITypeCRUD.get(ci_id, oneagent_id, oneagent_name, last_update_at) + + return self.jsonify(rules=rules, last_update_at=last_update_at) + + rules, last_update_at = AutoDiscoveryCITypeCRUD.get(None, oneagent_id, oneagent_name, last_update_at) return self.jsonify(rules=rules, last_update_at=last_update_at) + + +class AutoDiscoveryRuleSyncHistoryView(APIView): + url_prefix = ("/adt//sync/histories",) + + def get(self, adt_id): + page = get_page(request.values.pop('page', 1)) + page_size = get_page_size(request.values.pop('page_size', None)) + numfound, res = AutoDiscoveryRuleSyncHistoryCRUD.search(page=page, + page_size=page_size, + adt_id=adt_id, + **request.values) + + return self.jsonify(page=page, + page_size=page_size, + numfound=numfound, + total=len(res), + result=res) + + +class AutoDiscoveryTestView(APIView): + url_prefix = ("/adt//test", "/adt/test//result") + + def get(self, exec_id): + return self.jsonify(stdout="1\n2\n3", exec_id=exec_id) + + def post(self, adt_id): + return self.jsonify(exec_id=uuid.uuid4().hex) + + +class AutoDiscoveryExecHistoryView(APIView): + url_prefix = ("/adc/exec/histories",) + + @args_required('type_id') + def get(self): + page = get_page(request.values.pop('page', 1)) + page_size = get_page_size(request.values.pop('page_size', None)) + numfound, res = AutoDiscoveryExecHistoryCRUD.search(page=page, + page_size=page_size, + **request.values) + + return self.jsonify(page=page, + page_size=page_size, + numfound=numfound, + total=len(res), + result=res) + + @args_required('type_id') + @args_required('stdout') + def post(self): + AutoDiscoveryExecHistoryCRUD().add(type_id=request.values.get('type_id'), + stdout=request.values.get('stdout')) + + return self.jsonify(code=200) + + +class AutoDiscoveryCounterView(APIView): + url_prefix = ("/adc/counter",) + + @args_required('type_id') + def get(self): + type_id = request.values.get('type_id') + + return self.jsonify(AutoDiscoveryCounterCRUD().get(type_id))