relation view has been optimised

api/commands/click_cmdb.py

@@ -67,7 +67,7 @@ def init_cache():
     ci_relations = CIRelation.get_by(to_dict=False)
     relations = dict()
     for cr in ci_relations:
-        relations.setdefault(cr.first_ci_id, []).append(cr.second_ci_id)
+        relations.setdefault(cr.first_ci_id, {}).update({cr.second_ci_id: cr.second_ci.type_id})
     for i in relations:
         relations[i] = json.dumps(relations[i])
     if relations:

api/lib/cmdb/ci.py

@@ -566,4 +566,4 @@ class CIRelationManager(object):
 
         ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id), queue=CMDB_QUEUE)
 
-        return cls.delete(cr.cr_id)
+        return cls.delete(cr.id)

api/lib/cmdb/preference.py

@@ -13,6 +13,7 @@ from api.lib.cmdb.cache import AttributeCache
 from api.lib.cmdb.cache import CITypeAttributeCache
 from api.lib.cmdb.cache import CITypeCache
 from api.models.cmdb import CITypeAttribute
+from api.models.cmdb import CITypeRelation
 from api.models.cmdb import PreferenceRelationView
 from api.models.cmdb import PreferenceShowAttributes
 from api.models.cmdb import PreferenceTreeView
@@ -124,17 +125,28 @@ class PreferenceManager(object):
             for i in result[view_name]:
                 id2type[i['parent_id']] = None
                 id2type[i['child_id']] = None
+            topo = {i['child_id']: {i['parent_id']} for i in result[view_name]}
+            leaf = list(set(toposort.toposort_flatten(topo)) - set([j for i in topo.values() for j in i]))
 
-            result[view_name] = toposort.toposort_flatten(
+            leaf2show_types = {i: [t['child_id'] for t in CITypeRelation.get_by(parent_id=i)] for i in leaf}
-                {i['child_id']: {i['parent_id']} for i in result[view_name]})
+
+            result[view_name] = dict(topo=list(map(list, toposort.toposort(topo))),
+                                     topo_flatten=list(toposort.toposort_flatten(topo)),
+                                     leaf=leaf,
+                                     leaf2show_types=leaf2show_types,
+                                     show_types=[CITypeCache.get(j).to_dict()
+                                                 for i in leaf2show_types.values() for j in i])
 
         for type_id in id2type:
             id2type[type_id] = CITypeCache.get(type_id).to_dict()
-
+        print(result)
         return result, id2type, sorted(name2id, key=lambda x: x[1])
 
     @classmethod
     def create_or_update_relation_view(cls, name, cr_ids):
+        if not cr_ids:
+            return abort(400, "Node must be selected")
+
         existed = PreferenceRelationView.get_by(name=name, to_dict=False, first=True)
         if existed is None:
             PreferenceRelationView.create(name=name, cr_ids=json.dumps(cr_ids))
@@ -145,4 +157,5 @@ class PreferenceManager(object):
     def delete_relation_view(name):
         for existed in PreferenceRelationView.get_by(name=name, to_dict=False):
             existed.soft_delete()
+
         return name

api/lib/cmdb/search/ci/es/search.py

@@ -73,6 +73,10 @@ class Search(object):
     def _in_query_handle(self, attr, v):
         terms = v[1:-1].split(";")
         operator = "|"
+        if attr in ('_type', 'ci_type', 'type_id') and terms and terms[0].isdigit():
+            attr = "type_id"
+            terms = map(int, terms)
+        current_app.logger.warning(terms)
         for term in terms:
             self._operator2query(operator).append({
                 "term": {
@@ -164,6 +168,7 @@ class Search(object):
     def _query_build_raw(self):
 
         queries = handle_arg_list(self.orig_query)
+
         current_app.logger.debug(queries)
 
         self.__query_build_by_field(queries)

api/lib/cmdb/search/ci_relation/search.py

@@ -15,7 +15,14 @@ from api.models.cmdb import CI
 
 
 class Search(object):
-    def __init__(self, root_id, level=1, query=None, fl=None, facet_field=None, page=1, count=None, sort=None):
+    def __init__(self, root_id,
+                 level=1,
+                 query=None,
+                 fl=None,
+                 facet_field=None,
+                 page=1,
+                 count=None,
+                 sort=None):
         self.orig_query = query
         self.fl = fl
         self.facet_field = facet_field
@@ -24,22 +31,36 @@ class Search(object):
         self.sort = sort or ("ci_id" if current_app.config.get("USE_ES") else None)
 
         self.root_id = root_id
-        self.level = int(level)
+        self.level = level
 
     def search(self):
-        ci = CI.get_by_id(self.root_id) or abort(404, "CI <{0}> does not exist".format(self.root_id))
         ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
-        for _ in range(0, self.level):
+        cis = [CI.get_by_id(_id) or abort(404, "CI <{0}> does not exist".format(_id)) for _id in ids]
-            print(rd.get(ids, REDIS_PREFIX_CI_RELATION))
+
-            _tmp = list(map(json.loads, filter(lambda x: x is not None, rd.get(ids, REDIS_PREFIX_CI_RELATION) or [])))
+        merge_ids = []
+        for level in self.level:
+            ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
+            for _ in range(0, level):
+                _tmp = list(map(lambda x: list(json.loads(x).keys()),
+                                filter(lambda x: x is not None, rd.get(ids, REDIS_PREFIX_CI_RELATION) or [])))
                 ids = [j for i in _tmp for j in i]
+
+            merge_ids.extend(ids)
+
         if not self.orig_query or ("_type:" not in self.orig_query
                                    and "type_id:" not in self.orig_query
                                    and "ci_type:" not in self.orig_query):
-            type_ids = CITypeRelationManager.get_child_type_ids(ci.type_id, self.level)
+            type_ids = []
+            for level in self.level:
+                for ci in cis:
+                    type_ids.extend(CITypeRelationManager.get_child_type_ids(ci.type_id, level))
+            type_ids = list(set(type_ids))
+            if self.orig_query:
                 self.orig_query = "_type:({0}),{1}".format(";".join(list(map(str, type_ids))), self.orig_query)
+            else:
+                self.orig_query = "_type:({0})".format(";".join(list(map(str, type_ids))))
 
-        if not ids:
+        if not merge_ids:
             # cis, counter, total, self.page, numfound, facet_
             return [], {}, 0, self.page, 0, {}
 
@@ -50,7 +71,7 @@ class Search(object):
                                 page=self.page,
                                 count=self.count,
                                 sort=self.sort,
-                                ci_ids=ids).search()
+                                ci_ids=merge_ids).search()
         else:
             return SearchFromDB(self.orig_query,
                                 fl=self.fl,
@@ -58,17 +79,27 @@ class Search(object):
                                 page=self.page,
                                 count=self.count,
                                 sort=self.sort,
-                                ci_ids=ids).search()
+                                ci_ids=merge_ids).search()
 
-    def statistics(self):
+    def statistics(self, type_ids):
         ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
-        for l in range(0, self.level):
+        for l in range(0, int(self.level)):
             if l == 0:
-                _tmp = list(map(json.loads, [i or '[]' for i in rd.get(ids, REDIS_PREFIX_CI_RELATION) or []]))
+                _tmp = list(map(lambda x: list(json.loads(x).keys()),
+                                [i or '{}' for i in rd.get(ids, REDIS_PREFIX_CI_RELATION) or []]))
             else:
                 for idx, i in enumerate(_tmp):
                     if i:
-                        __tmp = list(map(json.loads, filter(lambda x: x is not None,
+                        if type_ids and l == self.level - 1:
+                            __tmp = list(
+                                map(lambda x: list({_id: 1 for _id, type_id in json.loads(x).items()
+                                                    if type_id in type_ids}.keys()),
+                                    filter(lambda x: x is not None,
+                                           rd.get(i, REDIS_PREFIX_CI_RELATION) or [])))
+                        else:
+
+                            __tmp = list(map(lambda x: list(json.loads(x).keys()),
+                                             filter(lambda x: x is not None,
                                                     rd.get(i, REDIS_PREFIX_CI_RELATION) or [])))
                         _tmp[idx] = [j for i in __tmp for j in i]
                     else:

api/lib/perm/acl/role.py

@@ -90,7 +90,7 @@ class RoleRelationCRUD(object):
 
     @staticmethod
     def delete2(parent_id, child_id):
-        existed = RoleRelation.get_by(parent_id=parent_id, child_id=child_id)
+        existed = RoleRelation.get_by(parent_id=parent_id, child_id=child_id, first=True, to_dict=False)
         existed or abort(400, "RoleRelation < {0} -> {1} > does not exist".format(parent_id, child_id))
 
         existed.soft_delete()

api/tasks/cmdb.py

@@ -14,6 +14,7 @@ from api.extensions import rd
 from api.lib.cmdb.const import CMDB_QUEUE
 from api.lib.cmdb.const import REDIS_PREFIX_CI
 from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
+from api.models.cmdb import CIRelation
 
 
 @celery.task(name="cmdb.ci_cache", queue=CMDB_QUEUE)
@@ -46,10 +47,13 @@ def ci_delete(ci_id):
 @celery.task(name="cmdb.ci_relation_cache", queue=CMDB_QUEUE)
 def ci_relation_cache(parent_id, child_id):
     children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0]
-    children = json.loads(children) if children is not None else []
+    children = json.loads(children) if children is not None else {}
-    children.append(child_id)
 
-    rd.create_or_update({parent_id: json.dumps(list(set(children)))}, REDIS_PREFIX_CI_RELATION)
+    cr = CIRelation.get_by(first_ci_id=parent_id, second_ci_id=child_id, first=True, to_dict=False)
+    if child_id not in children:
+        children[child_id] = cr.second_ci.type_id
+
+    rd.create_or_update({parent_id: json.dumps(children)}, REDIS_PREFIX_CI_RELATION)
 
     current_app.logger.info("ADD ci relation cache: {0} -> {1}".format(parent_id, child_id))
 
@@ -57,10 +61,11 @@ def ci_relation_cache(parent_id, child_id):
 @celery.task(name="cmdb.ci_relation_delete", queue=CMDB_QUEUE)
 def ci_relation_delete(parent_id, child_id):
     children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0]
-    children = json.loads(children) if children is not None else []
+    children = json.loads(children) if children is not None else {}
-    if child_id in children:
-        children.remove(child_id)
 
-    rd.create_or_update({parent_id: json.dumps(list(set(children)))}, REDIS_PREFIX_CI_RELATION)
+    if child_id in children:
+        children.pop(child_id)
+
+    rd.create_or_update({parent_id: json.dumps(children)}, REDIS_PREFIX_CI_RELATION)
 
     current_app.logger.info("DELETE ci relation cache: {0} -> {1}".format(parent_id, child_id))

api/views/cmdb/ci_relation.py

@@ -34,7 +34,7 @@ class CIRelationSearchView(APIView):
         count = get_page_size(request.values.get("count") or request.values.get("page_size"))
 
         root_id = request.values.get('root_id')
-        level = request.values.get('level', 1)
+        level = list(map(int, handle_arg_list(request.values.get('level', '1'))))
 
         query = request.values.get('q', "")
         fl = handle_arg_list(request.values.get('fl', ""))
@@ -64,11 +64,12 @@ class CIRelationStatisticsView(APIView):
     def get(self):
         root_ids = list(map(int, handle_arg_list(request.values.get('root_ids'))))
         level = request.values.get('level', 1)
+        type_ids = set(map(int, handle_arg_list(request.values.get('type_ids', []))))
 
         start = time.time()
         s = Search(root_ids, level)
         try:
-            result = s.statistics()
+            result = s.statistics(type_ids)
         except SearchError as e:
             return abort(400, str(e))
         current_app.logger.debug("search time is :{0}".format(time.time() - start))

api/views/cmdb/preference.py

@@ -77,6 +77,7 @@ class PreferenceRelationApiView(APIView):
 
     @role_required(RoleEnum.CONFIG)
     @args_required("name")
+    @args_required("cr_ids")
     def post(self):
         name = request.values.get("name")
         cr_ids = request.values.get("cr_ids")