perf(api): relationships built by attribute values (#572)

cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py

@@ -23,6 +23,7 @@ from api.lib.cmdb.ci_type import CITypeGroupManager
 from api.lib.cmdb.const import AutoDiscoveryType
 from api.lib.cmdb.const import CMDB_QUEUE
 from api.lib.cmdb.const import PermEnum
+from api.lib.cmdb.const import RelationSourceEnum
 from api.lib.cmdb.const import ResourceTypeEnum
 from api.lib.cmdb.custom_dashboard import SystemConfigManager
 from api.lib.cmdb.resp_format import ErrFormat
@@ -246,6 +247,9 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
         rules = cls.cls.get_by(to_dict=True)
 
         for rule in rules:
+            if not rule['enabled']:
+                continue
+
             if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('secret'):
                 if not (current_user.username in PRIVILEGED_USERS or current_user.uid == rule['uid']):
                     rule['extra_option'].pop('secret', None)
@@ -274,7 +278,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
                         break
             elif not rule['agent_id'] and not rule['query_expr'] and rule['adr_id']:
                 try:
-                    if not int(oneagent_id, 16): # excludes master
+                    if not int(oneagent_id, 16):  # excludes master
                         continue
                 except Exception:
                     pass
@@ -717,10 +721,15 @@ class AutoDiscoveryCICRUD(DBMixin):
                 for relation_ci in response:
                     relation_ci_id = relation_ci['_id']
                     try:
-                        CIRelationManager.add(ci_id, relation_ci_id, valid=False)
+                        CIRelationManager.add(ci_id, relation_ci_id,
+                                              valid=False,
+                                              source=RelationSourceEnum.AUTO_DISCOVERY)
+
                     except:
                         try:
-                            CIRelationManager.add(relation_ci_id, ci_id, valid=False)
+                            CIRelationManager.add(relation_ci_id, ci_id,
+                                                  valid=False,
+                                                  source=RelationSourceEnum.AUTO_DISCOVERY)
                         except:
                             pass
 

cmdb-api/api/lib/cmdb/auto_discovery/const.py

@@ -41,7 +41,7 @@ CLOUD_MAP = {
             "items": ["云服务器 ECS", "云服务器 Disk"],
             "map": {
                 "云服务器 ECS": {"template": "templates/aliyun_ecs.json", "mapping": "ecs"},
-                "云服务器 Disk": {"template": "templates/aliyun_ecs_disk2.json", "mapping": "evs"},
+                "云服务器 Disk": {"template": "templates/aliyun_ecs_disk.json", "mapping": "evs"},
             },
             "collect_key_map": {
                 "云服务器 ECS": "ali.ecs",

cmdb-api/api/lib/cmdb/ci.py

@@ -4,12 +4,12 @@
 import copy
 import datetime
 import json
-import threading
-
 import redis_lock
+import threading
 from flask import abort
 from flask import current_app
 from flask_login import current_user
+from sqlalchemy.orm import aliased
 from werkzeug.exceptions import BadRequest
 
 from api.extensions import db
@@ -28,6 +28,7 @@ from api.lib.cmdb.const import ExistPolicy
 from api.lib.cmdb.const import OperateType
 from api.lib.cmdb.const import PermEnum
 from api.lib.cmdb.const import REDIS_PREFIX_CI
+from api.lib.cmdb.const import RelationSourceEnum
 from api.lib.cmdb.const import ResourceTypeEnum
 from api.lib.cmdb.const import RetKey
 from api.lib.cmdb.const import ValueTypeEnum
@@ -1133,7 +1134,14 @@ class CIRelationManager(object):
                     return abort(400, ErrFormat.relation_constraint.format("1-N"))
 
     @classmethod
-    def add(cls, first_ci_id, second_ci_id, more=None, relation_type_id=None, ancestor_ids=None, valid=True):
+    def add(cls, first_ci_id, second_ci_id,
+            more=None,
+            relation_type_id=None,
+            ancestor_ids=None,
+            valid=True,
+            apply_async=True,
+            source=None,
+            uid=None):
 
         first_ci = CIManager.confirm_ci_existed(first_ci_id)
         second_ci = CIManager.confirm_ci_existed(second_ci_id)
@@ -1145,9 +1153,10 @@ class CIRelationManager(object):
                                     first=True)
         if existed is not None:
             if existed.relation_type_id != relation_type_id and relation_type_id is not None:
-                existed.update(relation_type_id=relation_type_id)
+                source = existed.source or source
+                existed.update(relation_type_id=relation_type_id, source=source)
 
-                CIRelationHistoryManager().add(existed, OperateType.UPDATE)
+                CIRelationHistoryManager().add(existed, OperateType.UPDATE, uid=uid)
         else:
             if relation_type_id is None:
                 type_relation = CITypeRelation.get_by(parent_id=first_ci.type_id,
@@ -1177,11 +1186,13 @@ class CIRelationManager(object):
                 existed = CIRelation.create(first_ci_id=first_ci_id,
                                             second_ci_id=second_ci_id,
                                             relation_type_id=relation_type_id,
-                                            ancestor_ids=ancestor_ids)
+                                            ancestor_ids=ancestor_ids,
-
+                                            source=source)
-                CIRelationHistoryManager().add(existed, OperateType.ADD)
+                CIRelationHistoryManager().add(existed, OperateType.ADD, uid=uid)
-
+                if apply_async:
-                ci_relation_cache.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE)
+                    ci_relation_cache.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE)
+                else:
+                    ci_relation_cache(first_ci_id, second_ci_id, ancestor_ids)
 
         if more is not None:
             existed.upadte(more=more)
@@ -1189,7 +1200,7 @@ class CIRelationManager(object):
         return existed.id
 
     @staticmethod
-    def delete(cr_id):
+    def delete(cr_id, apply_async=True):
         cr = CIRelation.get_by_id(cr_id) or abort(404, ErrFormat.relation_not_found.format("id={}".format(cr_id)))
 
         if current_app.config.get('USE_ACL') and current_user.username != 'worker':
@@ -1205,8 +1216,12 @@ class CIRelationManager(object):
         his_manager = CIRelationHistoryManager()
         his_manager.add(cr, operate_type=OperateType.DELETE)
 
-        ci_relation_delete.apply_async(args=(cr.first_ci_id, cr.second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE)
+        if apply_async:
-        delete_id_filter.apply_async(args=(cr.second_ci_id,), queue=CMDB_QUEUE)
+            ci_relation_delete.apply_async(args=(cr.first_ci_id, cr.second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE)
+            delete_id_filter.apply_async(args=(cr.second_ci_id,), queue=CMDB_QUEUE)
+        else:
+            ci_relation_delete(cr.first_ci_id, cr.second_ci_id, cr.ancestor_ids)
+            delete_id_filter(cr.second_ci_id)
 
         return cr_id
 
@@ -1221,23 +1236,23 @@ class CIRelationManager(object):
         if cr is not None:
             cls.delete(cr.id)
 
-        ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE)
+        # ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE)
-        delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE)
+        # delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE)
 
         return cr
 
     @classmethod
-    def delete_3(cls, first_ci_id, second_ci_id):
+    def delete_3(cls, first_ci_id, second_ci_id, apply_async=True):
         cr = CIRelation.get_by(first_ci_id=first_ci_id,
                                second_ci_id=second_ci_id,
                                to_dict=False,
                                first=True)
 
         if cr is not None:
-            ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE)
+            # ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE)
-            delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE)
+            # delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE)
 
-            cls.delete(cr.id)
+            cls.delete(cr.id, apply_async=apply_async)
 
         return cr
 
@@ -1276,6 +1291,27 @@ class CIRelationManager(object):
                 for ci_id in ci_ids:
                     cls.delete_2(parent_id, ci_id, ancestor_ids=ancestor_ids)
 
+    @classmethod
+    def delete_relations_by_source(cls, source,
+                                   first_ci_id=None, second_ci_type_id=None,
+                                   second_ci_id=None, first_ci_type_id=None,
+                                   added=None):
+        existed = []
+        if first_ci_id is not None and second_ci_type_id is not None:
+            existed = [(i.first_ci_id, i.second_ci_id) for i in CIRelation.get_by(
+                source=source, first_ci_id=first_ci_id, only_query=True).join(
+                CI, CIRelation.second_ci_id == CI.id).filter(CI.type_id == second_ci_type_id)]
+
+        if second_ci_id is not None and first_ci_type_id is not None:
+            existed = [(i.first_ci_id, i.second_ci_id) for i in CIRelation.get_by(
+                source=source, second_ci_id=second_ci_id, only_query=True).join(
+                CI, CIRelation.first_ci_id == CI.id).filter(CI.type_id == first_ci_type_id)]
+
+        deleted = set(existed) - set(added or [])
+
+        for first, second in deleted:
+            cls.delete_3(first, second, apply_async=False)
+
     @classmethod
     def build_by_attribute(cls, ci_dict):
         type_id = ci_dict['_type']
@@ -1296,8 +1332,15 @@ class CIRelationManager(object):
                     relations = _relations
                 else:
                     relations &= _relations
+
+            cls.delete_relations_by_source(RelationSourceEnum.ATTRIBUTE_VALUES,
+                                           first_ci_id=ci_dict['_id'],
+                                           second_ci_type_id=item.child_id,
+                                           added=relations)
             for parent_ci_id, child_ci_id in (relations or []):
-                CIRelationManager.add(parent_ci_id, child_ci_id, valid=False)
+                cls.add(parent_ci_id, child_ci_id,
+                        valid=False,
+                        source=RelationSourceEnum.ATTRIBUTE_VALUES)
 
         parent_items = CITypeRelation.get_by(child_id=type_id, only_query=True).filter(
             CITypeRelation.child_attr_ids.isnot(None))
@@ -1316,11 +1359,18 @@ class CIRelationManager(object):
                     relations = _relations
                 else:
                     relations &= _relations
+
+            cls.delete_relations_by_source(RelationSourceEnum.ATTRIBUTE_VALUES,
+                                           second_ci_id=ci_dict['_id'],
+                                           first_ci_type_id=item.parent_id,
+                                           added=relations)
             for parent_ci_id, child_ci_id in (relations or []):
-                CIRelationManager.add(parent_ci_id, child_ci_id, valid=False)
+                cls.add(parent_ci_id, child_ci_id,
+                        valid=False,
+                        source=RelationSourceEnum.ATTRIBUTE_VALUES)
 
     @classmethod
-    def rebuild_all_by_attribute(cls, ci_type_relation):
+    def rebuild_all_by_attribute(cls, ci_type_relation, uid):
         relations = None
         for parent_attr_id, child_attr_id in zip(ci_type_relation['parent_attr_ids'] or [],
                                                  ci_type_relation['child_attr_ids'] or []):
@@ -1352,11 +1402,29 @@ class CIRelationManager(object):
             else:
                 relations &= _relations
 
+        t1 = aliased(CI)
+        t2 = aliased(CI)
+        query = db.session.query(CIRelation).join(t1, t1.id == CIRelation.first_ci_id).join(
+            t2, t2.id == CIRelation.second_ci_id).filter(t1.type_id == ci_type_relation['parent_id']).filter(
+            t2.type_id == ci_type_relation['child_id'])
+        for i in query:
+            db.session.delete(i)
+            ci_relation_delete(i.first_ci_id, i.second_ci_id, i.ancestor_ids)
+        try:
+            db.session.commit()
+        except Exception as e:
+            current_app.logger.error(e)
+            db.session.rollback()
+
         for parent_ci_id, child_ci_id in (relations or []):
             try:
-                cls.add(parent_ci_id, child_ci_id, valid=False)
+                cls.add(parent_ci_id, child_ci_id,
-            except:
+                        valid=False,
-                pass
+                        apply_async=False,
+                        source=RelationSourceEnum.ATTRIBUTE_VALUES,
+                        uid=uid)
+            except Exception as e:
+                current_app.logger.error(e)
 
 
 class CITriggerManager(object):

cmdb-api/api/lib/cmdb/ci_type.py

@@ -993,7 +993,7 @@ class CITypeRelationManager(object):
         if ((parent_attr_ids and parent_attr_ids != old_parent_attr_ids) or
                 (child_attr_ids and child_attr_ids != old_child_attr_ids)):
             from api.tasks.cmdb import rebuild_relation_for_attribute_changed
-            rebuild_relation_for_attribute_changed.apply_async(args=(existed.to_dict(),))
+            rebuild_relation_for_attribute_changed.apply_async(args=(existed.to_dict(), current_user.uid))
 
         CITypeHistoryManager.add(CITypeOperateType.ADD_RELATION, p.id,
                                  change=dict(parent=p.to_dict(), child=c.to_dict(), relation_type_id=relation_type_id))

cmdb-api/api/lib/cmdb/const.py

@@ -41,23 +41,23 @@ class OperateType(BaseEnum):
 
 
 class CITypeOperateType(BaseEnum):
-    ADD = "0"  # 新增模型
+    ADD = "0"  # add CIType
-    UPDATE = "1"  # 修改模型
+    UPDATE = "1"  # update CIType
-    DELETE = "2"  # 删除模型
+    DELETE = "2"  # delete CIType
-    ADD_ATTRIBUTE = "3"  # 新增属性
+    ADD_ATTRIBUTE = "3"
-    UPDATE_ATTRIBUTE = "4"  # 修改属性
+    UPDATE_ATTRIBUTE = "4"
-    DELETE_ATTRIBUTE = "5"  # 删除属性
+    DELETE_ATTRIBUTE = "5"
-    ADD_TRIGGER = "6"  # 新增触发器
+    ADD_TRIGGER = "6"
-    UPDATE_TRIGGER = "7"  # 修改触发器
+    UPDATE_TRIGGER = "7"
-    DELETE_TRIGGER = "8"  # 删除触发器
+    DELETE_TRIGGER = "8"
-    ADD_UNIQUE_CONSTRAINT = "9"  # 新增联合唯一
+    ADD_UNIQUE_CONSTRAINT = "9"
-    UPDATE_UNIQUE_CONSTRAINT = "10"  # 修改联合唯一
+    UPDATE_UNIQUE_CONSTRAINT = "10"
-    DELETE_UNIQUE_CONSTRAINT = "11"  # 删除联合唯一
+    DELETE_UNIQUE_CONSTRAINT = "11"
-    ADD_RELATION = "12"  # 新增关系
+    ADD_RELATION = "12"
-    DELETE_RELATION = "13"  # 删除关系
+    DELETE_RELATION = "13"
-    ADD_RECONCILIATION = "14"  # 新增数据合规
+    ADD_RECONCILIATION = "14"
-    UPDATE_RECONCILIATION = "15"  # 修改数据合规
+    UPDATE_RECONCILIATION = "15"
-    DELETE_RECONCILIATION = "16"  # 删除数据合规
+    DELETE_RECONCILIATION = "16"
 
 
 class RetKey(BaseEnum):
@@ -93,7 +93,7 @@ class RoleEnum(BaseEnum):
 class AutoDiscoveryType(BaseEnum):
     AGENT = "agent"
     SNMP = "snmp"
-    HTTP = "http" # cloud
+    HTTP = "http"  # cloud
     COMPONENTS = "components"
 
 
@@ -108,6 +108,10 @@ class ExecuteStatusEnum(BaseEnum):
     FAILED = '1'
     RUNNING = '2'
 
+class RelationSourceEnum(BaseEnum):
+    ATTRIBUTE_VALUES = "0"
+    AUTO_DISCOVERY = "1"
+
 
 CMDB_QUEUE = "one_cmdb_async"
 REDIS_PREFIX_CI = "ONE_CMDB"

cmdb-api/api/lib/cmdb/history.py

@@ -232,8 +232,8 @@ class AttributeHistoryManger(object):
 
 class CIRelationHistoryManager(object):
     @staticmethod
-    def add(rel_obj, operate_type=OperateType.ADD):
+    def add(rel_obj, operate_type=OperateType.ADD, uid=None):
-        record = OperationRecord.create(uid=current_user.uid)
+        record = OperationRecord.create(uid=uid or current_user.uid)
 
         CIRelationHistory.create(relation_id=rel_obj.id,
                                  record_id=record.id,

cmdb-api/api/lib/cmdb/utils.py

@@ -28,10 +28,7 @@ def string2int(x):
     return v
 
 
-def str2datetime(x):
+def str2date(x):
-
-    x = x.replace('T', ' ')
-    x = x.replace('Z', '')
 
     try:
         return datetime.datetime.strptime(x, "%Y-%m-%d").date()
@@ -43,9 +40,21 @@ def str2datetime(x):
     except ValueError:
         pass
 
+
+def str2datetime(x):
+
+    x = x.replace('T', ' ')
+    x = x.replace('Z', '')
+
+    try:
+        return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")
+    except ValueError:
+        pass
+
     return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M")
 
 
+
 class ValueTypeMap(object):
     deserialize = {
         ValueTypeEnum.INT: string2int,
@@ -53,7 +62,7 @@ class ValueTypeMap(object):
         ValueTypeEnum.TEXT: lambda x: x,
         ValueTypeEnum.TIME: lambda x: TIME_RE.findall(x)[0],
         ValueTypeEnum.DATETIME: str2datetime,
-        ValueTypeEnum.DATE: str2datetime,
+        ValueTypeEnum.DATE: str2date,
         ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x,
     }
 

cmdb-api/api/models/cmdb.py

@@ -2,7 +2,6 @@
 
 
 import datetime
-
 from sqlalchemy.dialects.mysql import DOUBLE
 
 from api.extensions import db
@@ -11,6 +10,7 @@ from api.lib.cmdb.const import CIStatusEnum
 from api.lib.cmdb.const import CITypeOperateType
 from api.lib.cmdb.const import ConstraintEnum
 from api.lib.cmdb.const import OperateType
+from api.lib.cmdb.const import RelationSourceEnum
 from api.lib.cmdb.const import ValueTypeEnum
 from api.lib.database import Model
 from api.lib.database import Model2
@@ -260,6 +260,7 @@ class CIRelation(Model):
     second_ci_id = db.Column(db.Integer, db.ForeignKey("c_cis.id"), nullable=False)
     relation_type_id = db.Column(db.Integer, db.ForeignKey("c_relation_types.id"), nullable=False)
     more = db.Column(db.Integer, db.ForeignKey("c_cis.id"))
+    source = db.Column(db.Enum(*RelationSourceEnum.all()), name="source")
 
     ancestor_ids = db.Column(db.String(128), index=True)
 

cmdb-api/api/tasks/cmdb.py

@@ -58,10 +58,10 @@ def ci_cache(ci_id, operate_type, record_id):
 
 @celery.task(name="cmdb.rebuild_relation_for_attribute_changed", queue=CMDB_QUEUE)
 @reconnect_db
-def rebuild_relation_for_attribute_changed(ci_type_relation):
+def rebuild_relation_for_attribute_changed(ci_type_relation, uid):
     from api.lib.cmdb.ci import CIRelationManager
 
-    CIRelationManager.rebuild_all_by_attribute(ci_type_relation)
+    CIRelationManager.rebuild_all_by_attribute(ci_type_relation, uid)
 
 
 @celery.task(name="cmdb.batch_ci_cache", queue=CMDB_QUEUE)