feat(api): rebuild relation by attribute (#462)

cmdb-api/api/lib/cmdb/ci.py

@@ -295,6 +295,7 @@ class CIManager(object):
             _no_attribute_policy=ExistPolicy.IGNORE,
             is_auto_discovery=False,
             _is_admin=False,
+            ticket_id=None,
             **ci_dict):
         """
         add ci
@@ -303,6 +304,7 @@ class CIManager(object):
         :param _no_attribute_policy: ignore or reject
         :param is_auto_discovery: default is False
         :param _is_admin: default is False
+        :param ticket_id:
         :param ci_dict:
         :return:
         """
@@ -417,7 +419,7 @@ class CIManager(object):
             operate_type = OperateType.UPDATE if ci is not None else OperateType.ADD
             try:
                 ci = ci or CI.create(type_id=ci_type.id, is_auto_discovery=is_auto_discovery)
-                record_id = value_manager.create_or_update_attr_value(ci, ci_dict, key2attr)
+                record_id = value_manager.create_or_update_attr_value(ci, ci_dict, key2attr, ticket_id=ticket_id)
             except BadRequest as e:
                 if existed is None:
                     cls.delete(ci.id)
@@ -435,7 +437,7 @@ class CIManager(object):
 
         return ci.id
 
-    def update(self, ci_id, _is_admin=False, **ci_dict):
+    def update(self, ci_id, _is_admin=False, ticket_id=None, **ci_dict):
         now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
         ci = self.confirm_ci_existed(ci_id)
 
@@ -487,7 +489,7 @@ class CIManager(object):
                             ci_dict.pop(k)
 
             try:
-                record_id = value_manager.create_or_update_attr_value(ci, ci_dict, key2attr)
+                record_id = value_manager.create_or_update_attr_value(ci, ci_dict, key2attr, ticket_id=ticket_id)
             except BadRequest as e:
                 raise e
 
@@ -958,7 +960,7 @@ class CIRelationManager(object):
                     return abort(400, ErrFormat.relation_constraint.format("1-N"))
 
     @classmethod
-    def add(cls, first_ci_id, second_ci_id, more=None, relation_type_id=None, ancestor_ids=None):
+    def add(cls, first_ci_id, second_ci_id, more=None, relation_type_id=None, ancestor_ids=None, valid=True):
 
         first_ci = CIManager.confirm_ci_existed(first_ci_id)
         second_ci = CIManager.confirm_ci_existed(second_ci_id)
@@ -983,7 +985,7 @@ class CIRelationManager(object):
                 relation_type_id or abort(404, ErrFormat.relation_not_found.format("{} -> {}".format(
                     first_ci.ci_type.name, second_ci.ci_type.name)))
 
-                if current_app.config.get('USE_ACL'):
+                if current_app.config.get('USE_ACL') and valid:
                     resource_name = CITypeRelationManager.acl_resource_name(first_ci.ci_type.name,
                                                                             second_ci.ci_type.name)
                     if not ACLManager().has_permission(
@@ -1098,7 +1100,7 @@ class CIRelationManager(object):
             value_table = TableMap(attr=child_attr).table
             for child in value_table.get_by(attr_id=child_attr.id, value=attr_value, only_query=True).join(
                     CI, CI.id == value_table.ci_id).filter(CI.type_id == item.child_id):
-                CIRelationManager.add(ci_dict['_id'], child.ci_id)
+                CIRelationManager.add(ci_dict['_id'], child.ci_id, valid=False)
 
         parent_items = CITypeRelation.get_by(child_id=type_id, only_query=True).filter(
             CITypeRelation.child_attr_id.isnot(None))
@@ -1109,7 +1111,33 @@ class CIRelationManager(object):
             value_table = TableMap(attr=parent_attr).table
             for parent in value_table.get_by(attr_id=parent_attr.id, value=attr_value, only_query=True).join(
                     CI, CI.id == value_table.ci_id).filter(CI.type_id == item.parent_id):
-                CIRelationManager.add(parent.ci_id, ci_dict['_id'])
+                CIRelationManager.add(parent.ci_id, ci_dict['_id'], valid=False)
+
+    @classmethod
+    def rebuild_all_by_attribute(cls, ci_type_relation):
+        parent_attr = AttributeCache.get(ci_type_relation['parent_attr_id'])
+        child_attr = AttributeCache.get(ci_type_relation['child_attr_id'])
+        if not parent_attr or not child_attr:
+            return
+
+        parent_value_table = TableMap(attr=parent_attr).table
+        child_value_table = TableMap(attr=child_attr).table
+
+        parent_values = parent_value_table.get_by(attr_id=parent_attr.id, only_query=True).join(
+            CI, CI.id == parent_value_table.ci_id).filter(CI.type_id == ci_type_relation['parent_id'])
+        child_values = child_value_table.get_by(attr_id=child_attr.id, only_query=True).join(
+            CI, CI.id == child_value_table.ci_id).filter(CI.type_id == ci_type_relation['child_id'])
+
+        child_value2ci_ids = {}
+        for child in child_values:
+            child_value2ci_ids.setdefault(child.value, []).append(child.ci_id)
+
+        for parent in parent_values:
+            for child_ci_id in child_value2ci_ids.get(parent.value, []):
+                try:
+                    cls.add(parent.ci_id, child_ci_id, valid=False)
+                except:
+                    pass
 
 
 class CITriggerManager(object):

cmdb-api/api/lib/cmdb/ci_type.py

@@ -734,14 +734,19 @@ class CITypeRelationManager(object):
     @staticmethod
     def get():
         res = CITypeRelation.get_by(to_dict=False)
+        type2attributes = dict()
         for idx, item in enumerate(res):
             _item = item.to_dict()
             res[idx] = _item
             res[idx]['parent'] = item.parent.to_dict()
+            if item.parent_id not in type2attributes:
+                type2attributes[item.parent_id] = [i[1].to_dict() for i in CITypeAttributesCache.get2(item.parent_id)]
             res[idx]['child'] = item.child.to_dict()
+            if item.child_id not in type2attributes:
+                type2attributes[item.child_id] = [i[1].to_dict() for i in CITypeAttributesCache.get2(item.child_id)]
             res[idx]['relation_type'] = item.relation_type.to_dict()
 
-        return res
+        return res, type2attributes
 
     @staticmethod
     def get_child_type_ids(type_id, level):
@@ -773,6 +778,8 @@ class CITypeRelationManager(object):
 
         ci_type_dict["relation_type"] = relation_inst.relation_type.name
         ci_type_dict["constraint"] = relation_inst.constraint
+        ci_type_dict["parent_attr_id"] = relation_inst.parent_attr_id
+        ci_type_dict["child_attr_id"] = relation_inst.child_attr_id
 
         return ci_type_dict
 
@@ -833,12 +840,15 @@ class CITypeRelationManager(object):
             current_app.logger.warning(str(e))
             return abort(400, ErrFormat.circular_dependency_error)
 
+        old_parent_attr_id = None
         existed = cls._get(p.id, c.id)
         if existed is not None:
-            existed.update(relation_type_id=relation_type_id,
+            old_parent_attr_id = existed.parent_attr_id
-                           constraint=constraint,
+            existed = existed.update(relation_type_id=relation_type_id,
-                           parent_attr_id=parent_attr_id,
+                                     constraint=constraint,
-                           child_attr_id=child_attr_id)
+                                     parent_attr_id=parent_attr_id,
+                                     child_attr_id=child_attr_id,
+                                     filter_none=False)
         else:
             existed = CITypeRelation.create(parent_id=p.id,
                                             child_id=c.id,
@@ -858,6 +868,11 @@ class CITypeRelationManager(object):
                                                     current_user.username,
                                                     ResourceTypeEnum.CI_TYPE_RELATION)
 
+        if parent_attr_id and parent_attr_id != old_parent_attr_id:
+            if parent_attr_id and parent_attr_id != existed.parent_attr_id:
+                from api.tasks.cmdb import rebuild_relation_for_attribute_changed
+                rebuild_relation_for_attribute_changed.apply_async(args=(existed.to_dict()))
+
         CITypeHistoryManager.add(CITypeOperateType.ADD_RELATION, p.id,
                                  change=dict(parent=p.to_dict(), child=c.to_dict(), relation_type_id=relation_type_id))
 
@@ -1150,6 +1165,8 @@ class CITypeTemplateManager(object):
                                                  id2obj_dicts[added_id].get('child_id'),
                                                  id2obj_dicts[added_id].get('relation_type_id'),
                                                  id2obj_dicts[added_id].get('constraint'),
+                                                 id2obj_dicts[added_id].get('parent_attr_id'),
+                                                 id2obj_dicts[added_id].get('child_attr_id'),
                                                  )
             else:
                 obj = cls.create(flush=True, **id2obj_dicts[added_id])
@@ -1440,7 +1457,7 @@ class CITypeTemplateManager(object):
             ci_types=CITypeManager.get_ci_types(),
             ci_type_groups=CITypeGroupManager.get(),
             relation_types=[i.to_dict() for i in RelationTypeManager.get_all()],
-            ci_type_relations=CITypeRelationManager.get(),
+            ci_type_relations=CITypeRelationManager.get()[0],
             ci_type_auto_discovery_rules=list(),
             type2attributes=dict(),
             type2attribute_group=dict(),

cmdb-api/api/lib/cmdb/history.py

@@ -167,6 +167,7 @@ class AttributeHistoryManger(object):
                         new=hist.new,
                         created_at=record.created_at.strftime('%Y-%m-%d %H:%M:%S'),
                         record_id=record.id,
+                        ticket_id=record.ticket_id,
                         hid=hist.id
                         )
             result.append(item)
@@ -200,9 +201,9 @@ class AttributeHistoryManger(object):
         return username, timestamp, attr_dict, rel_dict
 
     @staticmethod
-    def add(record_id, ci_id, history_list, type_id=None, flush=False, commit=True):
+    def add(record_id, ci_id, history_list, type_id=None, ticket_id=None, flush=False, commit=True):
         if record_id is None:
-            record = OperationRecord.create(uid=current_user.uid, type_id=type_id)
+            record = OperationRecord.create(uid=current_user.uid, type_id=type_id, ticket_id=ticket_id)
             record_id = record.id
 
         for attr_id, operate_type, old, new in history_list or []:

cmdb-api/api/lib/cmdb/value.py

@@ -150,9 +150,10 @@ class AttributeValueManager(object):
         return AttributeHistoryManger.add(record_id, ci_id, [(attr_id, operate_type, old, new)], type_id)
 
     @staticmethod
-    def write_change2(changed, record_id=None):
+    def write_change2(changed, record_id=None, ticket_id=None):
         for ci_id, attr_id, operate_type, old, new, type_id in changed:
             record_id = AttributeHistoryManger.add(record_id, ci_id, [(attr_id, operate_type, old, new)], type_id,
+                                                   ticket_id=ticket_id,
                                                    commit=False, flush=False)
         try:
             db.session.commit()
@@ -255,12 +256,13 @@ class AttributeValueManager(object):
 
         return key2attr
 
-    def create_or_update_attr_value(self, ci, ci_dict, key2attr):
+    def create_or_update_attr_value(self, ci, ci_dict, key2attr, ticket_id=None):
         """
         add or update attribute value, then write history
         :param ci: instance object
         :param ci_dict: attribute dict
         :param key2attr: attr key to attr
+        :param ticket_id:
         :return:
         """
         changed = []
@@ -306,7 +308,7 @@ class AttributeValueManager(object):
             current_app.logger.warning(str(e))
             return abort(400, ErrFormat.attribute_value_unknown_error.format(e.args[0]))
 
-        return self.write_change2(changed)
+        return self.write_change2(changed, ticket_id=ticket_id)
 
     @staticmethod
     def delete_attr_value(attr_id, ci_id, commit=True):

cmdb-api/api/tasks/cmdb.py

@@ -53,6 +53,14 @@ def ci_cache(ci_id, operate_type, record_id):
     ci_dict and CIRelationManager.build_by_attribute(ci_dict)
 
 
+@celery.task(name="cmdb.rebuild_relation_for_attribute_changed", queue=CMDB_QUEUE)
+@reconnect_db
+def rebuild_relation_for_attribute_changed(ci_type_relation):
+    from api.lib.cmdb.ci import CIRelationManager
+
+    CIRelationManager.rebuild_all_by_attribute(ci_type_relation)
+
+
 @celery.task(name="cmdb.batch_ci_cache", queue=CMDB_QUEUE)
 @flush_db
 @reconnect_db

cmdb-api/api/views/cmdb/ci.py

@@ -76,6 +76,7 @@ class CIView(APIView):
     @has_perm_for_ci("ci_type", ResourceTypeEnum.CI, PermEnum.ADD, lambda x: CITypeCache.get(x))
     def post(self):
         ci_type = request.values.get("ci_type")
+        ticket_id = request.values.pop("ticket_id", None)
         _no_attribute_policy = request.values.get("no_attribute_policy", ExistPolicy.IGNORE)
 
         exist_policy = request.values.pop('exist_policy', None)
@@ -87,6 +88,7 @@ class CIView(APIView):
                             exist_policy=exist_policy or ExistPolicy.REJECT,
                             _no_attribute_policy=_no_attribute_policy,
                             _is_admin=request.values.pop('__is_admin', None) or False,
+                            ticket_id=ticket_id,
                             **ci_dict)
 
         return self.jsonify(ci_id=ci_id)
@@ -95,6 +97,7 @@ class CIView(APIView):
     def put(self, ci_id=None):
         args = request.values
         ci_type = args.get("ci_type")
+        ticket_id = request.values.pop("ticket_id", None)
         _no_attribute_policy = args.get("no_attribute_policy", ExistPolicy.IGNORE)
 
         ci_dict = self._wrap_ci_dict()
@@ -102,6 +105,7 @@ class CIView(APIView):
         if ci_id is not None:
             manager.update(ci_id,
                            _is_admin=request.values.pop('__is_admin', None) or False,
+                           ticket_id=ticket_id,
                            **ci_dict)
         else:
             request.values.pop('exist_policy', None)
@@ -109,6 +113,7 @@ class CIView(APIView):
                                 exist_policy=ExistPolicy.REPLACE,
                                 _no_attribute_policy=_no_attribute_policy,
                                 _is_admin=request.values.pop('__is_admin', None) or False,
+                                ticket_id=ticket_id,
                                 **ci_dict)
 
         return self.jsonify(ci_id=ci_id)
@@ -221,7 +226,6 @@ class CIHeartbeatView(APIView):
 class CIFlushView(APIView):
     url_prefix = ("/ci/flush", "/ci/<int:ci_id>/flush")
 
-    # @auth_abandoned
     def get(self, ci_id=None):
         from api.tasks.cmdb import ci_cache
         from api.lib.cmdb.const import CMDB_QUEUE

cmdb-api/api/views/cmdb/ci_type_relation.py

@@ -43,9 +43,9 @@ class CITypeRelationView(APIView):
 
     @role_required(RoleEnum.CONFIG)
     def get(self):
-        res = CITypeRelationManager.get()
+        res, type2attributes = CITypeRelationManager.get()
 
-        return self.jsonify(res)
+        return self.jsonify(relations=res, type2attributes=type2attributes)
 
     @has_perm_from_args("parent_id", ResourceTypeEnum.CI, PermEnum.CONFIG, CITypeManager.get_name_by_id)
     @args_required("relation_type_id")