diff --git a/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py b/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py index 4fa8968..323cf3a 100644 --- a/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py +++ b/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py @@ -23,6 +23,7 @@ from api.lib.cmdb.ci_type import CITypeGroupManager from api.lib.cmdb.const import AutoDiscoveryType from api.lib.cmdb.const import CMDB_QUEUE from api.lib.cmdb.const import PermEnum +from api.lib.cmdb.const import RelationSourceEnum from api.lib.cmdb.const import ResourceTypeEnum from api.lib.cmdb.custom_dashboard import SystemConfigManager from api.lib.cmdb.resp_format import ErrFormat @@ -246,6 +247,9 @@ class AutoDiscoveryCITypeCRUD(DBMixin): rules = cls.cls.get_by(to_dict=True) for rule in rules: + if not rule['enabled']: + continue + if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('secret'): if not (current_user.username in PRIVILEGED_USERS or current_user.uid == rule['uid']): rule['extra_option'].pop('secret', None) @@ -274,7 +278,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin): break elif not rule['agent_id'] and not rule['query_expr'] and rule['adr_id']: try: - if not int(oneagent_id, 16): # excludes master + if not int(oneagent_id, 16): # excludes master continue except Exception: pass @@ -717,10 +721,15 @@ class AutoDiscoveryCICRUD(DBMixin): for relation_ci in response: relation_ci_id = relation_ci['_id'] try: - CIRelationManager.add(ci_id, relation_ci_id, valid=False) + CIRelationManager.add(ci_id, relation_ci_id, + valid=False, + source=RelationSourceEnum.AUTO_DISCOVERY) + except: try: - CIRelationManager.add(relation_ci_id, ci_id, valid=False) + CIRelationManager.add(relation_ci_id, ci_id, + valid=False, + source=RelationSourceEnum.AUTO_DISCOVERY) except: pass diff --git a/cmdb-api/api/lib/cmdb/auto_discovery/const.py b/cmdb-api/api/lib/cmdb/auto_discovery/const.py index a22aed0..d34c4a8 100644 --- a/cmdb-api/api/lib/cmdb/auto_discovery/const.py +++ b/cmdb-api/api/lib/cmdb/auto_discovery/const.py @@ -41,7 +41,7 @@ CLOUD_MAP = { "items": ["云服务器 ECS", "云服务器 Disk"], "map": { "云服务器 ECS": {"template": "templates/aliyun_ecs.json", "mapping": "ecs"}, - "云服务器 Disk": {"template": "templates/aliyun_ecs_disk2.json", "mapping": "evs"}, + "云服务器 Disk": {"template": "templates/aliyun_ecs_disk.json", "mapping": "evs"}, }, "collect_key_map": { "云服务器 ECS": "ali.ecs", diff --git a/cmdb-api/api/lib/cmdb/ci.py b/cmdb-api/api/lib/cmdb/ci.py index d3a1047..85f8ddd 100644 --- a/cmdb-api/api/lib/cmdb/ci.py +++ b/cmdb-api/api/lib/cmdb/ci.py @@ -4,12 +4,12 @@ import copy import datetime import json -import threading - import redis_lock +import threading from flask import abort from flask import current_app from flask_login import current_user +from sqlalchemy.orm import aliased from werkzeug.exceptions import BadRequest from api.extensions import db @@ -28,6 +28,7 @@ from api.lib.cmdb.const import ExistPolicy from api.lib.cmdb.const import OperateType from api.lib.cmdb.const import PermEnum from api.lib.cmdb.const import REDIS_PREFIX_CI +from api.lib.cmdb.const import RelationSourceEnum from api.lib.cmdb.const import ResourceTypeEnum from api.lib.cmdb.const import RetKey from api.lib.cmdb.const import ValueTypeEnum @@ -1133,7 +1134,14 @@ class CIRelationManager(object): return abort(400, ErrFormat.relation_constraint.format("1-N")) @classmethod - def add(cls, first_ci_id, second_ci_id, more=None, relation_type_id=None, ancestor_ids=None, valid=True): + def add(cls, first_ci_id, second_ci_id, + more=None, + relation_type_id=None, + ancestor_ids=None, + valid=True, + apply_async=True, + source=None, + uid=None): first_ci = CIManager.confirm_ci_existed(first_ci_id) second_ci = CIManager.confirm_ci_existed(second_ci_id) @@ -1145,9 +1153,10 @@ class CIRelationManager(object): first=True) if existed is not None: if existed.relation_type_id != relation_type_id and relation_type_id is not None: - existed.update(relation_type_id=relation_type_id) + source = existed.source or source + existed.update(relation_type_id=relation_type_id, source=source) - CIRelationHistoryManager().add(existed, OperateType.UPDATE) + CIRelationHistoryManager().add(existed, OperateType.UPDATE, uid=uid) else: if relation_type_id is None: type_relation = CITypeRelation.get_by(parent_id=first_ci.type_id, @@ -1177,11 +1186,13 @@ class CIRelationManager(object): existed = CIRelation.create(first_ci_id=first_ci_id, second_ci_id=second_ci_id, relation_type_id=relation_type_id, - ancestor_ids=ancestor_ids) - - CIRelationHistoryManager().add(existed, OperateType.ADD) - - ci_relation_cache.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE) + ancestor_ids=ancestor_ids, + source=source) + CIRelationHistoryManager().add(existed, OperateType.ADD, uid=uid) + if apply_async: + ci_relation_cache.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE) + else: + ci_relation_cache(first_ci_id, second_ci_id, ancestor_ids) if more is not None: existed.upadte(more=more) @@ -1189,7 +1200,7 @@ class CIRelationManager(object): return existed.id @staticmethod - def delete(cr_id): + def delete(cr_id, apply_async=True): cr = CIRelation.get_by_id(cr_id) or abort(404, ErrFormat.relation_not_found.format("id={}".format(cr_id))) if current_app.config.get('USE_ACL') and current_user.username != 'worker': @@ -1205,8 +1216,12 @@ class CIRelationManager(object): his_manager = CIRelationHistoryManager() his_manager.add(cr, operate_type=OperateType.DELETE) - ci_relation_delete.apply_async(args=(cr.first_ci_id, cr.second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE) - delete_id_filter.apply_async(args=(cr.second_ci_id,), queue=CMDB_QUEUE) + if apply_async: + ci_relation_delete.apply_async(args=(cr.first_ci_id, cr.second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE) + delete_id_filter.apply_async(args=(cr.second_ci_id,), queue=CMDB_QUEUE) + else: + ci_relation_delete(cr.first_ci_id, cr.second_ci_id, cr.ancestor_ids) + delete_id_filter(cr.second_ci_id) return cr_id @@ -1221,23 +1236,23 @@ class CIRelationManager(object): if cr is not None: cls.delete(cr.id) - ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE) - delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE) + # ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE) + # delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE) return cr @classmethod - def delete_3(cls, first_ci_id, second_ci_id): + def delete_3(cls, first_ci_id, second_ci_id, apply_async=True): cr = CIRelation.get_by(first_ci_id=first_ci_id, second_ci_id=second_ci_id, to_dict=False, first=True) if cr is not None: - ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE) - delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE) + # ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE) + # delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE) - cls.delete(cr.id) + cls.delete(cr.id, apply_async=apply_async) return cr @@ -1276,6 +1291,27 @@ class CIRelationManager(object): for ci_id in ci_ids: cls.delete_2(parent_id, ci_id, ancestor_ids=ancestor_ids) + @classmethod + def delete_relations_by_source(cls, source, + first_ci_id=None, second_ci_type_id=None, + second_ci_id=None, first_ci_type_id=None, + added=None): + existed = [] + if first_ci_id is not None and second_ci_type_id is not None: + existed = [(i.first_ci_id, i.second_ci_id) for i in CIRelation.get_by( + source=source, first_ci_id=first_ci_id, only_query=True).join( + CI, CIRelation.second_ci_id == CI.id).filter(CI.type_id == second_ci_type_id)] + + if second_ci_id is not None and first_ci_type_id is not None: + existed = [(i.first_ci_id, i.second_ci_id) for i in CIRelation.get_by( + source=source, second_ci_id=second_ci_id, only_query=True).join( + CI, CIRelation.first_ci_id == CI.id).filter(CI.type_id == first_ci_type_id)] + + deleted = set(existed) - set(added or []) + + for first, second in deleted: + cls.delete_3(first, second, apply_async=False) + @classmethod def build_by_attribute(cls, ci_dict): type_id = ci_dict['_type'] @@ -1296,8 +1332,15 @@ class CIRelationManager(object): relations = _relations else: relations &= _relations + + cls.delete_relations_by_source(RelationSourceEnum.ATTRIBUTE_VALUES, + first_ci_id=ci_dict['_id'], + second_ci_type_id=item.child_id, + added=relations) for parent_ci_id, child_ci_id in (relations or []): - CIRelationManager.add(parent_ci_id, child_ci_id, valid=False) + cls.add(parent_ci_id, child_ci_id, + valid=False, + source=RelationSourceEnum.ATTRIBUTE_VALUES) parent_items = CITypeRelation.get_by(child_id=type_id, only_query=True).filter( CITypeRelation.child_attr_ids.isnot(None)) @@ -1316,11 +1359,18 @@ class CIRelationManager(object): relations = _relations else: relations &= _relations + + cls.delete_relations_by_source(RelationSourceEnum.ATTRIBUTE_VALUES, + second_ci_id=ci_dict['_id'], + first_ci_type_id=item.parent_id, + added=relations) for parent_ci_id, child_ci_id in (relations or []): - CIRelationManager.add(parent_ci_id, child_ci_id, valid=False) + cls.add(parent_ci_id, child_ci_id, + valid=False, + source=RelationSourceEnum.ATTRIBUTE_VALUES) @classmethod - def rebuild_all_by_attribute(cls, ci_type_relation): + def rebuild_all_by_attribute(cls, ci_type_relation, uid): relations = None for parent_attr_id, child_attr_id in zip(ci_type_relation['parent_attr_ids'] or [], ci_type_relation['child_attr_ids'] or []): @@ -1352,11 +1402,29 @@ class CIRelationManager(object): else: relations &= _relations + t1 = aliased(CI) + t2 = aliased(CI) + query = db.session.query(CIRelation).join(t1, t1.id == CIRelation.first_ci_id).join( + t2, t2.id == CIRelation.second_ci_id).filter(t1.type_id == ci_type_relation['parent_id']).filter( + t2.type_id == ci_type_relation['child_id']) + for i in query: + db.session.delete(i) + ci_relation_delete(i.first_ci_id, i.second_ci_id, i.ancestor_ids) + try: + db.session.commit() + except Exception as e: + current_app.logger.error(e) + db.session.rollback() + for parent_ci_id, child_ci_id in (relations or []): try: - cls.add(parent_ci_id, child_ci_id, valid=False) - except: - pass + cls.add(parent_ci_id, child_ci_id, + valid=False, + apply_async=False, + source=RelationSourceEnum.ATTRIBUTE_VALUES, + uid=uid) + except Exception as e: + current_app.logger.error(e) class CITriggerManager(object): diff --git a/cmdb-api/api/lib/cmdb/ci_type.py b/cmdb-api/api/lib/cmdb/ci_type.py index 847ef98..0164de1 100644 --- a/cmdb-api/api/lib/cmdb/ci_type.py +++ b/cmdb-api/api/lib/cmdb/ci_type.py @@ -993,7 +993,7 @@ class CITypeRelationManager(object): if ((parent_attr_ids and parent_attr_ids != old_parent_attr_ids) or (child_attr_ids and child_attr_ids != old_child_attr_ids)): from api.tasks.cmdb import rebuild_relation_for_attribute_changed - rebuild_relation_for_attribute_changed.apply_async(args=(existed.to_dict(),)) + rebuild_relation_for_attribute_changed.apply_async(args=(existed.to_dict(), current_user.uid)) CITypeHistoryManager.add(CITypeOperateType.ADD_RELATION, p.id, change=dict(parent=p.to_dict(), child=c.to_dict(), relation_type_id=relation_type_id)) diff --git a/cmdb-api/api/lib/cmdb/const.py b/cmdb-api/api/lib/cmdb/const.py index d0302be..ecda9e3 100644 --- a/cmdb-api/api/lib/cmdb/const.py +++ b/cmdb-api/api/lib/cmdb/const.py @@ -41,23 +41,23 @@ class OperateType(BaseEnum): class CITypeOperateType(BaseEnum): - ADD = "0" # 新增模型 - UPDATE = "1" # 修改模型 - DELETE = "2" # 删除模型 - ADD_ATTRIBUTE = "3" # 新增属性 - UPDATE_ATTRIBUTE = "4" # 修改属性 - DELETE_ATTRIBUTE = "5" # 删除属性 - ADD_TRIGGER = "6" # 新增触发器 - UPDATE_TRIGGER = "7" # 修改触发器 - DELETE_TRIGGER = "8" # 删除触发器 - ADD_UNIQUE_CONSTRAINT = "9" # 新增联合唯一 - UPDATE_UNIQUE_CONSTRAINT = "10" # 修改联合唯一 - DELETE_UNIQUE_CONSTRAINT = "11" # 删除联合唯一 - ADD_RELATION = "12" # 新增关系 - DELETE_RELATION = "13" # 删除关系 - ADD_RECONCILIATION = "14" # 新增数据合规 - UPDATE_RECONCILIATION = "15" # 修改数据合规 - DELETE_RECONCILIATION = "16" # 删除数据合规 + ADD = "0" # add CIType + UPDATE = "1" # update CIType + DELETE = "2" # delete CIType + ADD_ATTRIBUTE = "3" + UPDATE_ATTRIBUTE = "4" + DELETE_ATTRIBUTE = "5" + ADD_TRIGGER = "6" + UPDATE_TRIGGER = "7" + DELETE_TRIGGER = "8" + ADD_UNIQUE_CONSTRAINT = "9" + UPDATE_UNIQUE_CONSTRAINT = "10" + DELETE_UNIQUE_CONSTRAINT = "11" + ADD_RELATION = "12" + DELETE_RELATION = "13" + ADD_RECONCILIATION = "14" + UPDATE_RECONCILIATION = "15" + DELETE_RECONCILIATION = "16" class RetKey(BaseEnum): @@ -93,7 +93,7 @@ class RoleEnum(BaseEnum): class AutoDiscoveryType(BaseEnum): AGENT = "agent" SNMP = "snmp" - HTTP = "http" # cloud + HTTP = "http" # cloud COMPONENTS = "components" @@ -108,6 +108,10 @@ class ExecuteStatusEnum(BaseEnum): FAILED = '1' RUNNING = '2' +class RelationSourceEnum(BaseEnum): + ATTRIBUTE_VALUES = "0" + AUTO_DISCOVERY = "1" + CMDB_QUEUE = "one_cmdb_async" REDIS_PREFIX_CI = "ONE_CMDB" diff --git a/cmdb-api/api/lib/cmdb/history.py b/cmdb-api/api/lib/cmdb/history.py index 6e85038..2c790d2 100644 --- a/cmdb-api/api/lib/cmdb/history.py +++ b/cmdb-api/api/lib/cmdb/history.py @@ -232,8 +232,8 @@ class AttributeHistoryManger(object): class CIRelationHistoryManager(object): @staticmethod - def add(rel_obj, operate_type=OperateType.ADD): - record = OperationRecord.create(uid=current_user.uid) + def add(rel_obj, operate_type=OperateType.ADD, uid=None): + record = OperationRecord.create(uid=uid or current_user.uid) CIRelationHistory.create(relation_id=rel_obj.id, record_id=record.id, diff --git a/cmdb-api/api/lib/cmdb/utils.py b/cmdb-api/api/lib/cmdb/utils.py index 7b0ff2a..ab354ea 100644 --- a/cmdb-api/api/lib/cmdb/utils.py +++ b/cmdb-api/api/lib/cmdb/utils.py @@ -28,10 +28,7 @@ def string2int(x): return v -def str2datetime(x): - - x = x.replace('T', ' ') - x = x.replace('Z', '') +def str2date(x): try: return datetime.datetime.strptime(x, "%Y-%m-%d").date() @@ -43,9 +40,21 @@ def str2datetime(x): except ValueError: pass + +def str2datetime(x): + + x = x.replace('T', ' ') + x = x.replace('Z', '') + + try: + return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S") + except ValueError: + pass + return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M") + class ValueTypeMap(object): deserialize = { ValueTypeEnum.INT: string2int, @@ -53,7 +62,7 @@ class ValueTypeMap(object): ValueTypeEnum.TEXT: lambda x: x, ValueTypeEnum.TIME: lambda x: TIME_RE.findall(x)[0], ValueTypeEnum.DATETIME: str2datetime, - ValueTypeEnum.DATE: str2datetime, + ValueTypeEnum.DATE: str2date, ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x, } diff --git a/cmdb-api/api/models/cmdb.py b/cmdb-api/api/models/cmdb.py index 7a5bbb7..7673a12 100644 --- a/cmdb-api/api/models/cmdb.py +++ b/cmdb-api/api/models/cmdb.py @@ -2,7 +2,6 @@ import datetime - from sqlalchemy.dialects.mysql import DOUBLE from api.extensions import db @@ -11,6 +10,7 @@ from api.lib.cmdb.const import CIStatusEnum from api.lib.cmdb.const import CITypeOperateType from api.lib.cmdb.const import ConstraintEnum from api.lib.cmdb.const import OperateType +from api.lib.cmdb.const import RelationSourceEnum from api.lib.cmdb.const import ValueTypeEnum from api.lib.database import Model from api.lib.database import Model2 @@ -260,6 +260,7 @@ class CIRelation(Model): second_ci_id = db.Column(db.Integer, db.ForeignKey("c_cis.id"), nullable=False) relation_type_id = db.Column(db.Integer, db.ForeignKey("c_relation_types.id"), nullable=False) more = db.Column(db.Integer, db.ForeignKey("c_cis.id")) + source = db.Column(db.Enum(*RelationSourceEnum.all()), name="source") ancestor_ids = db.Column(db.String(128), index=True) diff --git a/cmdb-api/api/tasks/cmdb.py b/cmdb-api/api/tasks/cmdb.py index af94423..1fa1d40 100644 --- a/cmdb-api/api/tasks/cmdb.py +++ b/cmdb-api/api/tasks/cmdb.py @@ -58,10 +58,10 @@ def ci_cache(ci_id, operate_type, record_id): @celery.task(name="cmdb.rebuild_relation_for_attribute_changed", queue=CMDB_QUEUE) @reconnect_db -def rebuild_relation_for_attribute_changed(ci_type_relation): +def rebuild_relation_for_attribute_changed(ci_type_relation, uid): from api.lib.cmdb.ci import CIRelationManager - CIRelationManager.rebuild_all_by_attribute(ci_type_relation) + CIRelationManager.rebuild_all_by_attribute(ci_type_relation, uid) @celery.task(name="cmdb.batch_ci_cache", queue=CMDB_QUEUE)