From 9e62780d50384cd1966a54e27801d3061a360bee Mon Sep 17 00:00:00 2001 From: pycook Date: Wed, 3 Apr 2024 15:13:43 +0800 Subject: [PATCH] feat(api): rebuild relation by attribute (#462) --- cmdb-api/api/lib/cmdb/ci.py | 42 +++++++++++++++++---- cmdb-api/api/lib/cmdb/ci_type.py | 29 +++++++++++--- cmdb-api/api/lib/cmdb/history.py | 5 ++- cmdb-api/api/lib/cmdb/value.py | 8 ++-- cmdb-api/api/tasks/cmdb.py | 8 ++++ cmdb-api/api/views/cmdb/ci.py | 6 ++- cmdb-api/api/views/cmdb/ci_type_relation.py | 4 +- 7 files changed, 81 insertions(+), 21 deletions(-) diff --git a/cmdb-api/api/lib/cmdb/ci.py b/cmdb-api/api/lib/cmdb/ci.py index 8c88f67..3e7c181 100644 --- a/cmdb-api/api/lib/cmdb/ci.py +++ b/cmdb-api/api/lib/cmdb/ci.py @@ -295,6 +295,7 @@ class CIManager(object): _no_attribute_policy=ExistPolicy.IGNORE, is_auto_discovery=False, _is_admin=False, + ticket_id=None, **ci_dict): """ add ci @@ -303,6 +304,7 @@ class CIManager(object): :param _no_attribute_policy: ignore or reject :param is_auto_discovery: default is False :param _is_admin: default is False + :param ticket_id: :param ci_dict: :return: """ @@ -417,7 +419,7 @@ class CIManager(object): operate_type = OperateType.UPDATE if ci is not None else OperateType.ADD try: ci = ci or CI.create(type_id=ci_type.id, is_auto_discovery=is_auto_discovery) - record_id = value_manager.create_or_update_attr_value(ci, ci_dict, key2attr) + record_id = value_manager.create_or_update_attr_value(ci, ci_dict, key2attr, ticket_id=ticket_id) except BadRequest as e: if existed is None: cls.delete(ci.id) @@ -435,7 +437,7 @@ class CIManager(object): return ci.id - def update(self, ci_id, _is_admin=False, **ci_dict): + def update(self, ci_id, _is_admin=False, ticket_id=None, **ci_dict): now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') ci = self.confirm_ci_existed(ci_id) @@ -487,7 +489,7 @@ class CIManager(object): ci_dict.pop(k) try: - record_id = value_manager.create_or_update_attr_value(ci, ci_dict, key2attr) + record_id = value_manager.create_or_update_attr_value(ci, ci_dict, key2attr, ticket_id=ticket_id) except BadRequest as e: raise e @@ -958,7 +960,7 @@ class CIRelationManager(object): return abort(400, ErrFormat.relation_constraint.format("1-N")) @classmethod - def add(cls, first_ci_id, second_ci_id, more=None, relation_type_id=None, ancestor_ids=None): + def add(cls, first_ci_id, second_ci_id, more=None, relation_type_id=None, ancestor_ids=None, valid=True): first_ci = CIManager.confirm_ci_existed(first_ci_id) second_ci = CIManager.confirm_ci_existed(second_ci_id) @@ -983,7 +985,7 @@ class CIRelationManager(object): relation_type_id or abort(404, ErrFormat.relation_not_found.format("{} -> {}".format( first_ci.ci_type.name, second_ci.ci_type.name))) - if current_app.config.get('USE_ACL'): + if current_app.config.get('USE_ACL') and valid: resource_name = CITypeRelationManager.acl_resource_name(first_ci.ci_type.name, second_ci.ci_type.name) if not ACLManager().has_permission( @@ -1098,7 +1100,7 @@ class CIRelationManager(object): value_table = TableMap(attr=child_attr).table for child in value_table.get_by(attr_id=child_attr.id, value=attr_value, only_query=True).join( CI, CI.id == value_table.ci_id).filter(CI.type_id == item.child_id): - CIRelationManager.add(ci_dict['_id'], child.ci_id) + CIRelationManager.add(ci_dict['_id'], child.ci_id, valid=False) parent_items = CITypeRelation.get_by(child_id=type_id, only_query=True).filter( CITypeRelation.child_attr_id.isnot(None)) @@ -1109,7 +1111,33 @@ class CIRelationManager(object): value_table = TableMap(attr=parent_attr).table for parent in value_table.get_by(attr_id=parent_attr.id, value=attr_value, only_query=True).join( CI, CI.id == value_table.ci_id).filter(CI.type_id == item.parent_id): - CIRelationManager.add(parent.ci_id, ci_dict['_id']) + CIRelationManager.add(parent.ci_id, ci_dict['_id'], valid=False) + + @classmethod + def rebuild_all_by_attribute(cls, ci_type_relation): + parent_attr = AttributeCache.get(ci_type_relation['parent_attr_id']) + child_attr = AttributeCache.get(ci_type_relation['child_attr_id']) + if not parent_attr or not child_attr: + return + + parent_value_table = TableMap(attr=parent_attr).table + child_value_table = TableMap(attr=child_attr).table + + parent_values = parent_value_table.get_by(attr_id=parent_attr.id, only_query=True).join( + CI, CI.id == parent_value_table.ci_id).filter(CI.type_id == ci_type_relation['parent_id']) + child_values = child_value_table.get_by(attr_id=child_attr.id, only_query=True).join( + CI, CI.id == child_value_table.ci_id).filter(CI.type_id == ci_type_relation['child_id']) + + child_value2ci_ids = {} + for child in child_values: + child_value2ci_ids.setdefault(child.value, []).append(child.ci_id) + + for parent in parent_values: + for child_ci_id in child_value2ci_ids.get(parent.value, []): + try: + cls.add(parent.ci_id, child_ci_id, valid=False) + except: + pass class CITriggerManager(object): diff --git a/cmdb-api/api/lib/cmdb/ci_type.py b/cmdb-api/api/lib/cmdb/ci_type.py index b43f22e..cee1e7b 100644 --- a/cmdb-api/api/lib/cmdb/ci_type.py +++ b/cmdb-api/api/lib/cmdb/ci_type.py @@ -734,14 +734,19 @@ class CITypeRelationManager(object): @staticmethod def get(): res = CITypeRelation.get_by(to_dict=False) + type2attributes = dict() for idx, item in enumerate(res): _item = item.to_dict() res[idx] = _item res[idx]['parent'] = item.parent.to_dict() + if item.parent_id not in type2attributes: + type2attributes[item.parent_id] = [i[1].to_dict() for i in CITypeAttributesCache.get2(item.parent_id)] res[idx]['child'] = item.child.to_dict() + if item.child_id not in type2attributes: + type2attributes[item.child_id] = [i[1].to_dict() for i in CITypeAttributesCache.get2(item.child_id)] res[idx]['relation_type'] = item.relation_type.to_dict() - return res + return res, type2attributes @staticmethod def get_child_type_ids(type_id, level): @@ -773,6 +778,8 @@ class CITypeRelationManager(object): ci_type_dict["relation_type"] = relation_inst.relation_type.name ci_type_dict["constraint"] = relation_inst.constraint + ci_type_dict["parent_attr_id"] = relation_inst.parent_attr_id + ci_type_dict["child_attr_id"] = relation_inst.child_attr_id return ci_type_dict @@ -833,12 +840,15 @@ class CITypeRelationManager(object): current_app.logger.warning(str(e)) return abort(400, ErrFormat.circular_dependency_error) + old_parent_attr_id = None existed = cls._get(p.id, c.id) if existed is not None: - existed.update(relation_type_id=relation_type_id, - constraint=constraint, - parent_attr_id=parent_attr_id, - child_attr_id=child_attr_id) + old_parent_attr_id = existed.parent_attr_id + existed = existed.update(relation_type_id=relation_type_id, + constraint=constraint, + parent_attr_id=parent_attr_id, + child_attr_id=child_attr_id, + filter_none=False) else: existed = CITypeRelation.create(parent_id=p.id, child_id=c.id, @@ -858,6 +868,11 @@ class CITypeRelationManager(object): current_user.username, ResourceTypeEnum.CI_TYPE_RELATION) + if parent_attr_id and parent_attr_id != old_parent_attr_id: + if parent_attr_id and parent_attr_id != existed.parent_attr_id: + from api.tasks.cmdb import rebuild_relation_for_attribute_changed + rebuild_relation_for_attribute_changed.apply_async(args=(existed.to_dict())) + CITypeHistoryManager.add(CITypeOperateType.ADD_RELATION, p.id, change=dict(parent=p.to_dict(), child=c.to_dict(), relation_type_id=relation_type_id)) @@ -1150,6 +1165,8 @@ class CITypeTemplateManager(object): id2obj_dicts[added_id].get('child_id'), id2obj_dicts[added_id].get('relation_type_id'), id2obj_dicts[added_id].get('constraint'), + id2obj_dicts[added_id].get('parent_attr_id'), + id2obj_dicts[added_id].get('child_attr_id'), ) else: obj = cls.create(flush=True, **id2obj_dicts[added_id]) @@ -1440,7 +1457,7 @@ class CITypeTemplateManager(object): ci_types=CITypeManager.get_ci_types(), ci_type_groups=CITypeGroupManager.get(), relation_types=[i.to_dict() for i in RelationTypeManager.get_all()], - ci_type_relations=CITypeRelationManager.get(), + ci_type_relations=CITypeRelationManager.get()[0], ci_type_auto_discovery_rules=list(), type2attributes=dict(), type2attribute_group=dict(), diff --git a/cmdb-api/api/lib/cmdb/history.py b/cmdb-api/api/lib/cmdb/history.py index adc6d40..2e793f6 100644 --- a/cmdb-api/api/lib/cmdb/history.py +++ b/cmdb-api/api/lib/cmdb/history.py @@ -167,6 +167,7 @@ class AttributeHistoryManger(object): new=hist.new, created_at=record.created_at.strftime('%Y-%m-%d %H:%M:%S'), record_id=record.id, + ticket_id=record.ticket_id, hid=hist.id ) result.append(item) @@ -200,9 +201,9 @@ class AttributeHistoryManger(object): return username, timestamp, attr_dict, rel_dict @staticmethod - def add(record_id, ci_id, history_list, type_id=None, flush=False, commit=True): + def add(record_id, ci_id, history_list, type_id=None, ticket_id=None, flush=False, commit=True): if record_id is None: - record = OperationRecord.create(uid=current_user.uid, type_id=type_id) + record = OperationRecord.create(uid=current_user.uid, type_id=type_id, ticket_id=ticket_id) record_id = record.id for attr_id, operate_type, old, new in history_list or []: diff --git a/cmdb-api/api/lib/cmdb/value.py b/cmdb-api/api/lib/cmdb/value.py index 66594ce..8f98edd 100644 --- a/cmdb-api/api/lib/cmdb/value.py +++ b/cmdb-api/api/lib/cmdb/value.py @@ -150,9 +150,10 @@ class AttributeValueManager(object): return AttributeHistoryManger.add(record_id, ci_id, [(attr_id, operate_type, old, new)], type_id) @staticmethod - def write_change2(changed, record_id=None): + def write_change2(changed, record_id=None, ticket_id=None): for ci_id, attr_id, operate_type, old, new, type_id in changed: record_id = AttributeHistoryManger.add(record_id, ci_id, [(attr_id, operate_type, old, new)], type_id, + ticket_id=ticket_id, commit=False, flush=False) try: db.session.commit() @@ -255,12 +256,13 @@ class AttributeValueManager(object): return key2attr - def create_or_update_attr_value(self, ci, ci_dict, key2attr): + def create_or_update_attr_value(self, ci, ci_dict, key2attr, ticket_id=None): """ add or update attribute value, then write history :param ci: instance object :param ci_dict: attribute dict :param key2attr: attr key to attr + :param ticket_id: :return: """ changed = [] @@ -306,7 +308,7 @@ class AttributeValueManager(object): current_app.logger.warning(str(e)) return abort(400, ErrFormat.attribute_value_unknown_error.format(e.args[0])) - return self.write_change2(changed) + return self.write_change2(changed, ticket_id=ticket_id) @staticmethod def delete_attr_value(attr_id, ci_id, commit=True): diff --git a/cmdb-api/api/tasks/cmdb.py b/cmdb-api/api/tasks/cmdb.py index a5cc9bb..6586909 100644 --- a/cmdb-api/api/tasks/cmdb.py +++ b/cmdb-api/api/tasks/cmdb.py @@ -53,6 +53,14 @@ def ci_cache(ci_id, operate_type, record_id): ci_dict and CIRelationManager.build_by_attribute(ci_dict) +@celery.task(name="cmdb.rebuild_relation_for_attribute_changed", queue=CMDB_QUEUE) +@reconnect_db +def rebuild_relation_for_attribute_changed(ci_type_relation): + from api.lib.cmdb.ci import CIRelationManager + + CIRelationManager.rebuild_all_by_attribute(ci_type_relation) + + @celery.task(name="cmdb.batch_ci_cache", queue=CMDB_QUEUE) @flush_db @reconnect_db diff --git a/cmdb-api/api/views/cmdb/ci.py b/cmdb-api/api/views/cmdb/ci.py index b67cbc2..c568b27 100644 --- a/cmdb-api/api/views/cmdb/ci.py +++ b/cmdb-api/api/views/cmdb/ci.py @@ -76,6 +76,7 @@ class CIView(APIView): @has_perm_for_ci("ci_type", ResourceTypeEnum.CI, PermEnum.ADD, lambda x: CITypeCache.get(x)) def post(self): ci_type = request.values.get("ci_type") + ticket_id = request.values.pop("ticket_id", None) _no_attribute_policy = request.values.get("no_attribute_policy", ExistPolicy.IGNORE) exist_policy = request.values.pop('exist_policy', None) @@ -87,6 +88,7 @@ class CIView(APIView): exist_policy=exist_policy or ExistPolicy.REJECT, _no_attribute_policy=_no_attribute_policy, _is_admin=request.values.pop('__is_admin', None) or False, + ticket_id=ticket_id, **ci_dict) return self.jsonify(ci_id=ci_id) @@ -95,6 +97,7 @@ class CIView(APIView): def put(self, ci_id=None): args = request.values ci_type = args.get("ci_type") + ticket_id = request.values.pop("ticket_id", None) _no_attribute_policy = args.get("no_attribute_policy", ExistPolicy.IGNORE) ci_dict = self._wrap_ci_dict() @@ -102,6 +105,7 @@ class CIView(APIView): if ci_id is not None: manager.update(ci_id, _is_admin=request.values.pop('__is_admin', None) or False, + ticket_id=ticket_id, **ci_dict) else: request.values.pop('exist_policy', None) @@ -109,6 +113,7 @@ class CIView(APIView): exist_policy=ExistPolicy.REPLACE, _no_attribute_policy=_no_attribute_policy, _is_admin=request.values.pop('__is_admin', None) or False, + ticket_id=ticket_id, **ci_dict) return self.jsonify(ci_id=ci_id) @@ -221,7 +226,6 @@ class CIHeartbeatView(APIView): class CIFlushView(APIView): url_prefix = ("/ci/flush", "/ci//flush") - # @auth_abandoned def get(self, ci_id=None): from api.tasks.cmdb import ci_cache from api.lib.cmdb.const import CMDB_QUEUE diff --git a/cmdb-api/api/views/cmdb/ci_type_relation.py b/cmdb-api/api/views/cmdb/ci_type_relation.py index 3f1a72f..410a977 100644 --- a/cmdb-api/api/views/cmdb/ci_type_relation.py +++ b/cmdb-api/api/views/cmdb/ci_type_relation.py @@ -43,9 +43,9 @@ class CITypeRelationView(APIView): @role_required(RoleEnum.CONFIG) def get(self): - res = CITypeRelationManager.get() + res, type2attributes = CITypeRelationManager.get() - return self.jsonify(res) + return self.jsonify(relations=res, type2attributes=type2attributes) @has_perm_from_args("parent_id", ResourceTypeEnum.CI, PermEnum.CONFIG, CITypeManager.get_name_by_id) @args_required("relation_type_id")