Merge pull request #628 from veops/dev_api_relation_path_search

feat(api): relation path search

cmdb-api/api/lib/cmdb/search/ci/db/search.py

@@ -66,6 +66,7 @@ class Search(object):
         self.use_id_filter = use_id_filter
         self.use_ci_filter = use_ci_filter
         self.only_ids = only_ids
+        self.multi_type_has_ci_filter = False
 
         self.valid_type_names = []
         self.type2filter_perms = dict()
@@ -140,9 +141,10 @@ class Search(object):
                                         self.type_id_list.remove(str(ci_type.id))
                                     type_id_list.remove(str(ci_type.id))
                                     sub.extend([i for i in queries[1:] if isinstance(i, six.string_types)])
+
                                     sub.insert(0, "_type:{}".format(ci_type.id))
                                     queries.append(dict(operator="|", queries=sub))
-
+                                    self.multi_type_has_ci_filter = True
                         if self.type2filter_perms[ci_type.id].get('attr_filter'):
                             if type_num == 1:
                                 if not self.fl:
@@ -172,9 +174,9 @@ class Search(object):
         if type_id_list:
             type_ids = ",".join(type_id_list)
             _query_sql = QUERY_CI_BY_TYPE.format(type_ids)
-            if self.only_type_query:
+            if self.only_type_query or self.multi_type_has_ci_filter:
                 return _query_sql
-        elif type_num > 1:
+        elif type_num > 1:  # there must be instance-level access control
             return "select c_cis.id as ci_id from c_cis where c_cis.id=0"
 
         return ""
@@ -253,7 +255,7 @@ class Search(object):
             return ret_sql.format(query_sql, "ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format(
                 (self.page - 1) * self.count, sort_type, self.count))
 
-        elif self.type_id_list:
+        elif self.type_id_list and not self.multi_type_has_ci_filter:
             self.query_sql = "SELECT B.ci_id FROM ({0}) AS B {1}".format(
                 query_sql,
                 "INNER JOIN c_cis on c_cis.id=B.ci_id WHERE c_cis.type_id IN ({0}) ".format(
@@ -278,7 +280,7 @@ class Search(object):
     def __sort_by_type(self, sort_type, query_sql):
         ret_sql = "SELECT SQL_CALC_FOUND_ROWS DISTINCT B.ci_id FROM ({0}) AS B {1}"
 
-        if self.type_id_list:
+        if self.type_id_list and not self.multi_type_has_ci_filter:
             self.query_sql = "SELECT B.ci_id FROM ({0}) AS B {1}".format(
                 query_sql,
                 "INNER JOIN c_cis on c_cis.id=B.ci_id WHERE c_cis.type_id IN ({0}) ".format(
@@ -311,7 +313,7 @@ class Search(object):
                           WHERE {1}.attr_id = {3}""".format("ALIAS", table_name, query_sql, attr_id)
         new_table = _v_query_sql
 
-        if self.only_type_query or not self.type_id_list:
+        if self.only_type_query or not self.type_id_list or self.multi_type_has_ci_filter:
             return ("SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id FROM ({0}) AS C ORDER BY C.value {2} "
                     "LIMIT {1:d}, {3};".format(new_table, (self.page - 1) * self.count, sort_type, self.count))
 
@@ -518,8 +520,8 @@ class Search(object):
             _query_sql = ""
             if isinstance(q, dict):
                 alias, _query_sql, operator = self.__query_build_by_field(q['queries'], True, True, alias, is_sub=True)
-                current_app.logger.info(_query_sql)
+                # current_app.logger.info(_query_sql)
-                current_app.logger.info((operator, is_first, alias))
+                # current_app.logger.info((operator, is_first, alias))
                 operator = q['operator']
 
             elif ":" in q and not q.startswith("*"):
@@ -617,6 +619,7 @@ class Search(object):
                 k, _, _, _ = self._attr_name_proc(f)
                 if k:
                     _fl.append(k)
+
             return _fl
         else:
             return self.fl
@@ -638,6 +641,8 @@ class Search(object):
         if ci_ids:
             response = CIManager.get_cis_by_ids(ci_ids, ret_key=self.ret_key, fields=_fl, excludes=self.excludes)
         for res in response:
+            if not res:
+                continue
             ci_type = res.get("ci_type")
             if ci_type not in counter.keys():
                 counter[ci_type] = 0

cmdb-api/api/lib/cmdb/search/ci_relation/search.py

@@ -29,6 +29,8 @@ from api.lib.cmdb.utils import ValueTypeMap
 from api.lib.perm.acl.acl import ACLManager
 from api.lib.perm.acl.acl import is_app_admin
 from api.models.cmdb import CI
+from api.models.cmdb import CITypeRelation
+from api.models.cmdb import RelationType
 
 
 class Search(object):
@@ -437,7 +439,10 @@ class Search(object):
         if not q.startswith('_type:'):
             q = "_type:({}),{}".format(";".join(map(str, type_ids)), q)
 
-        return SearchFromDB(q, ci_ids=target_ids, use_ci_filter=False, only_ids=True, count=100000).search()
+        ci_ids = SearchFromDB(q, ci_ids=target_ids, use_ci_filter=True, only_ids=True, count=100000).search()
+        cis = CI.get_by(fl=['id', 'type_id'], only_query=True).filter(CI.id.in_(ci_ids))
+
+        return [(str(i.id), i.type_id) for i in cis]
 
     @staticmethod
     def _path2level(src_type_id, target_type_ids, path):
@@ -445,21 +450,31 @@ class Search(object):
             return abort(400, ErrFormat.relation_path_search_src_target_required)
 
         graph = nx.DiGraph()
-        graph.add_edges_from([(int(s), d) for s in path for d in path[s]])
+        graph.add_edges_from([(n, _path[idx + 1]) for _path in path for idx, n in enumerate(_path[:-1])])
-
+        relation_types = defaultdict(dict)
         level2type = defaultdict(set)
-        for target_type_id in target_type_ids:
+        type2show_key = dict()
-            paths = list(nx.all_simple_paths(graph, source=src_type_id, target=target_type_id))
+        for _path in path:
-            for _path in paths:
+            for idx, node in enumerate(_path[1:]):
-                for idx, node in enumerate(_path[1:]):
+                level2type[idx + 1].add(node)
-                    level2type[idx + 1].add(node)
+
+                src = CITypeCache.get(_path[idx])
+                target = CITypeCache.get(node)
+                relation_type = RelationType.get_by(only_query=True).join(
+                    CITypeRelation, CITypeRelation.relation_type_id == RelationType.id).filter(
+                    CITypeRelation.parent_id == src.id).filter(CITypeRelation.child_id == target.id).first()
+                relation_types[src.alias].update({target.alias: relation_type.name})
+
+                if src.id not in type2show_key:
+                    type2show_key[src.id] = AttributeCache.get(src.show_id or src.unique_id).name
+                if target.id not in type2show_key:
+                    type2show_key[target.id] = AttributeCache.get(target.show_id or target.unique_id).name
+
         nodes = graph.nodes()
 
-        del graph
+        return level2type, list(nodes), relation_types, type2show_key
 
-        return level2type, list(nodes)
+    def _build_graph(self, source_ids, source_type_id, level2type, target_type_ids, acl):
-
-    def _build_graph(self, source_ids, level2type, target_type_ids, acl):
         type2filter_perms = dict()
         if not self.is_app_admin:
             res2 = acl.get_resources(ResourceTypeEnum.CI_FILTER)
@@ -469,7 +484,8 @@ class Search(object):
         target_type_ids = set(target_type_ids)
         graph = nx.DiGraph()
         target_ids = []
-        key = list(map(str, source_ids))
+        key = [(str(i), source_type_id) for i in source_ids]
+        graph.add_nodes_from(key)
         for level in level2type:
             filter_type_ids = level2type[level]
             id_filter_limit = dict()
@@ -480,10 +496,11 @@ class Search(object):
 
             has_target = filter_type_ids & target_type_ids
 
-            res = [json.loads(x).items() for x in [i or '{}' for i in rd.get(key, REDIS_PREFIX_CI_RELATION) or []]]
+            res = [json.loads(x).items() for x in [i or '{}' for i in rd.get([i[0] for i in key],
+                                                                             REDIS_PREFIX_CI_RELATION) or []]]
             _key = []
             for idx, _id in enumerate(key):
-                valid_targets = [i[0] for i in res[idx] if i[1] in filter_type_ids and
+                valid_targets = [i for i in res[idx] if i[1] in filter_type_ids and
                                  (not id_filter_limit or int(i[0]) in id_filter_limit)]
                 _key.extend(valid_targets)
                 graph.add_edges_from(zip([_id] * len(valid_targets), valid_targets))
@@ -496,31 +513,41 @@ class Search(object):
         return graph, target_ids
 
     @staticmethod
-    def _find_paths(graph, source_ids, target_ids, max_depth=6):
+    def _find_paths(graph, source_ids, source_type_id, target_ids, valid_path, max_depth=6):
         paths = []
         for source_id in source_ids:
-            _paths = nx.all_simple_paths(graph, source=source_id, target=target_ids, cutoff=max_depth)
+            _paths = nx.all_simple_paths(graph,
-            paths.extend(_paths)
+                                         source=(source_id, source_type_id),
+                                         target=target_ids,
+                                         cutoff=max_depth)
+            for __path in _paths:
+                if tuple([i[1] for i in __path]) in valid_path:
+                    paths.append([i[0] for i in __path])
 
         return paths
 
     @staticmethod
-    def _wrap_path_result(paths, types):
+    def _wrap_path_result(paths, types, valid_path, target_types, type2show_key):
         ci_ids = [j for i in paths for j in i]
 
         response, _, _, _, _, _ = SearchFromDB("_type:({})".format(";".join(map(str, types))),
                                                use_ci_filter=False,
                                                ci_ids=list(map(int, ci_ids)),
                                                count=1000000).search()
-        id2ci = {str(i.get('_id')): i for i in response}
+        id2ci = {str(i.get('_id')): i if i['_type'] in target_types else {
+            type2show_key[i['_type']]: i[type2show_key[i['_type']]],
+            "ci_type_alias": i["ci_type_alias"],
+            "_type": i["_type"],
+        } for i in response}
 
         result = defaultdict(list)
         counter = defaultdict(int)
 
         for path in paths:
             key = "-".join([id2ci.get(i, {}).get('ci_type_alias') or '' for i in path])
-            counter[key] += 1
+            if tuple([id2ci.get(i, {}).get('_type') for i in path]) in valid_path:
-            result[key].append(path)
+                counter[key] += 1
+                result[key].append(path)
 
         return result, counter, id2ci
 
@@ -529,33 +556,38 @@ class Search(object):
 
         :param source: {type_id: id, q: expr}
         :param target: {type_ids: [id], q: expr}
-        :param path: {parent_id: [child_id]}, use type id
+        :param path: [source_type_id, ..., target_type_id], use type id
         :return:
         """
         acl = ACLManager('cmdb')
         if not self.is_app_admin:
             res = {i['name'] for i in acl.get_resources(ResourceTypeEnum.CI_TYPE)}
-            for type_id in (source.get('type_id') or []) + (target.get('type_ids') or []):
+            for type_id in (source.get('type_id') and [source['type_id']] or []) + (target.get('type_ids') or []):
                 _type = CITypeCache.get(type_id)
                 if _type and _type.name not in res:
                     return abort(403, ErrFormat.no_permission.format(_type.alias, PermEnum.READ))
 
-        level2type, types = self._path2level(source.get('type_id'), target.get('type_ids'), path)
+        target['type_ids'] = [i[-1] for i in path]
+        level2type, types, relation_types, type2show_key = self._path2level(
+            source.get('type_id'), target.get('type_ids'), path)
         if not level2type:
-            return [], {}, 0, self.page, 0, {}
+            return [], {}, 0, self.page, 0, {}, {}
 
         source_ids = self._get_src_ids(source)
 
-        graph, target_ids = self._build_graph(source_ids, level2type, target['type_ids'], acl)
+        graph, target_ids = self._build_graph(source_ids, source['type_id'], level2type, target['type_ids'], acl)
-        if target.get('q'):
+        target_ids = self._filter_target_ids(target_ids, target['type_ids'], target.get('q') or '')
-            target_ids = self._filter_target_ids(target_ids, target['type_ids'], target['q'])
+        paths = self._find_paths(graph,
+                                 source_ids,
+                                 source['type_id'],
+                                 set(target_ids),
+                                 {tuple(i): 1 for i in path})
 
-        paths = self._find_paths(graph, source_ids, set(target_ids))
+        numfound = len(paths)
-        del graph
-
-        numfound = len(target_ids)
         paths = paths[(self.page - 1) * self.count:self.page * self.count]
-
+        response, counter, id2ci = self._wrap_path_result(paths,
-        response, counter, id2ci = self._wrap_path_result(paths, types)
+                                                          types,
-
+                                                          {tuple(i): 1 for i in path},
-        return response, counter, len(paths), self.page, numfound, id2ci
+                                                          set(target.get('type_ids') or []),
+                                                          type2show_key)
+        return response, counter, len(paths), self.page, numfound, id2ci, relation_types, type2show_key

cmdb-api/api/views/cmdb/ci_relation.py

@@ -73,7 +73,7 @@ class CIRelationSearchPathView(APIView):
                     page_size | count: page size
                     source: source CIType, e.g. {type_id: 1, q: `search expr`}
                     target: target CIType, e.g. {type_ids: [2], q: `search expr`}
-                    path: Path from the Source CIType to the Target CIType, e.g. {source_id: [target_id]}
+                    path: Path from the Source CIType to the Target CIType, e.g. [1, ..., 2]
         """
 
         page = get_page(request.values.get("page", 1))
@@ -85,7 +85,8 @@ class CIRelationSearchPathView(APIView):
 
         s = Search(page=page, count=count)
         try:
-            response, counter, total, page, numfound, id2ci = s.search_by_path(source, target, path)
+            (response, counter, total, page, numfound, id2ci,
+             relation_types, type2show_key) = s.search_by_path(source, target, path)
         except SearchError as e:
             return abort(400, str(e))
 
@@ -94,7 +95,9 @@ class CIRelationSearchPathView(APIView):
                             page=page,
                             counter=counter,
                             paths=response,
-                            id2ci=id2ci)
+                            id2ci=id2ci,
+                            relation_types=relation_types,
+                            type2show_key=type2show_key)
 
 
 class CIRelationStatisticsView(APIView):