diff --git a/cmdb-api/Pipfile b/cmdb-api/Pipfile index a807215..e26d43f 100644 --- a/cmdb-api/Pipfile +++ b/cmdb-api/Pipfile @@ -68,6 +68,7 @@ pycryptodomex = ">=3.19.0" lz4 = ">=4.3.2" python-magic = "==0.4.27" jsonpath = "==0.82.2" +networkx = ">=3.1" [dev-packages] # Testing diff --git a/cmdb-api/api/lib/cmdb/ci_type.py b/cmdb-api/api/lib/cmdb/ci_type.py index d93dfb6..9ede428 100644 --- a/cmdb-api/api/lib/cmdb/ci_type.py +++ b/cmdb-api/api/lib/cmdb/ci_type.py @@ -3,6 +3,7 @@ from collections import defaultdict import copy +import networkx as nx import toposort from flask import abort from flask import current_app @@ -845,6 +846,29 @@ class CITypeRelationManager(object): return ids + @staticmethod + def find_path(source_type_id, target_type_ids): + source_type_id = int(source_type_id) + target_type_ids = map(int, target_type_ids) + + graph = nx.DiGraph() + + def get_children(_id): + children = CITypeRelation.get_by(parent_id=_id, to_dict=False) + + for i in children: + if i.child_id != _id: + graph.add_edge(i.parent_id, i.child_id) + get_children(i.child_id) + + get_children(source_type_id) + + paths = list(nx.all_simple_paths(graph, source_type_id, target_type_ids)) + + del graph + + return paths + @staticmethod def _wrap_relation_type_dict(type_id, relation_inst): ci_type_dict = CITypeCache.get(type_id).to_dict() diff --git a/cmdb-api/api/lib/cmdb/resp_format.py b/cmdb-api/api/lib/cmdb/resp_format.py index 0a39341..59a26ad 100644 --- a/cmdb-api/api/lib/cmdb/resp_format.py +++ b/cmdb-api/api/lib/cmdb/resp_format.py @@ -154,3 +154,5 @@ class ErrFormat(CommonErrFormat): topology_group_exists = _l("Topology group {} already exists") # 拓扑视图分组 {} 已经存在 # 因为该分组下定义了拓扑视图，不能删除 topo_view_exists_cannot_delete_group = _l("The group cannot be deleted because the topology view already exists") + + relation_path_search_src_target_required = _l("Both the source model and the target model must be selected") \ No newline at end of file diff --git a/cmdb-api/api/lib/cmdb/search/ci_relation/search.py b/cmdb-api/api/lib/cmdb/search/ci_relation/search.py index 1b10c96..a56a41f 100644 --- a/cmdb-api/api/lib/cmdb/search/ci_relation/search.py +++ b/cmdb-api/api/lib/cmdb/search/ci_relation/search.py @@ -1,8 +1,11 @@ # -*- coding:utf-8 -*- -import json -import sys from collections import Counter +from collections import defaultdict +import copy +import json +import networkx as nx +import sys from flask import abort from flask import current_app from flask_login import current_user @@ -13,6 +16,7 @@ from api.lib.cmdb.cache import CITypeCache from api.lib.cmdb.ci import CIRelationManager from api.lib.cmdb.ci_type import CITypeRelationManager from api.lib.cmdb.const import ConstraintEnum +from api.lib.cmdb.const import PermEnum from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION2 from api.lib.cmdb.const import ResourceTypeEnum @@ -28,7 +32,7 @@ from api.models.cmdb import CI class Search(object): - def __init__(self, root_id, + def __init__(self, root_id=None, level=None, query=None, fl=None, @@ -419,3 +423,139 @@ class Search(object): level_ids = _level_ids return result + + @staticmethod + def _get_src_ids(src): + q = src.get('q') or '' + if not q.startswith('_type:'): + q = "_type:{},{}".format(src['type_id'], q) + + return SearchFromDB(q, use_ci_filter=True, only_ids=True, count=100000).search() + + @staticmethod + def _filter_target_ids(target_ids, type_ids, q): + if not q.startswith('_type:'): + q = "_type:({}),{}".format(";".join(map(str, type_ids)), q) + + return SearchFromDB(q, ci_ids=target_ids, use_ci_filter=False, only_ids=True, count=100000).search() + + @staticmethod + def _path2level(src_type_id, target_type_ids, path): + if not src_type_id or not target_type_ids: + return abort(400, ErrFormat.relation_path_search_src_target_required) + + graph = nx.DiGraph() + graph.add_edges_from([(int(s), d) for s in path for d in path[s]]) + + level2type = defaultdict(set) + for target_type_id in target_type_ids: + paths = list(nx.all_simple_paths(graph, source=src_type_id, target=target_type_id)) + for _path in paths: + for idx, node in enumerate(_path[1:]): + level2type[idx + 1].add(node) + nodes = graph.nodes() + + del graph + + return level2type, list(nodes) + + def _build_graph(self, source_ids, level2type, target_type_ids, acl): + type2filter_perms = dict() + if not self.is_app_admin: + res2 = acl.get_resources(ResourceTypeEnum.CI_FILTER) + if res2: + type2filter_perms = CIFilterPermsCRUD().get_by_ids(list(map(int, [i['name'] for i in res2]))) + + target_type_ids = set(target_type_ids) + graph = nx.DiGraph() + target_ids = [] + key = list(map(str, source_ids)) + for level in level2type: + filter_type_ids = level2type[level] + id_filter_limit = dict() + for _type_id in filter_type_ids: + if type2filter_perms.get(_type_id): + _id_filter_limit, _ = self._get_ci_filter(type2filter_perms[_type_id]) + id_filter_limit.update(_id_filter_limit) + + has_target = filter_type_ids & target_type_ids + + res = [json.loads(x).items() for x in [i or '{}' for i in rd.get(key, REDIS_PREFIX_CI_RELATION) or []]] + _key = [] + for idx, _id in enumerate(key): + valid_targets = [i[0] for i in res[idx] if i[1] in filter_type_ids and + (not id_filter_limit or int(i[0]) in id_filter_limit)] + _key.extend(valid_targets) + graph.add_edges_from(zip([_id] * len(valid_targets), valid_targets)) + + if has_target: + target_ids.extend([j[0] for i in res for j in i if j[1] in target_type_ids]) + + key = copy.deepcopy(_key) + + return graph, target_ids + + @staticmethod + def _find_paths(graph, source_ids, target_ids, max_depth=6): + paths = [] + for source_id in source_ids: + _paths = nx.all_simple_paths(graph, source=source_id, target=target_ids, cutoff=max_depth) + paths.extend(_paths) + + return paths + + @staticmethod + def _wrap_path_result(paths, types): + ci_ids = [j for i in paths for j in i] + + response, _, _, _, _, _ = SearchFromDB("_type:({})".format(";".join(map(str, types))), + use_ci_filter=False, + ci_ids=list(map(int, ci_ids)), + count=1000000).search() + id2ci = {str(i.get('_id')): i for i in response} + + result = defaultdict(list) + counter = defaultdict(int) + + for path in paths: + key = "-".join([id2ci.get(i, {}).get('ci_type_alias') or '' for i in path]) + counter[key] += 1 + result[key].append(path) + + return result, counter, id2ci + + def search_by_path(self, source, target, path): + """ + + :param source: {type_id: id, q: expr} + :param target: {type_ids: [id], q: expr} + :param path: {parent_id: [child_id]}, use type id + :return: + """ + acl = ACLManager('cmdb') + if not self.is_app_admin: + res = {i['name'] for i in acl.get_resources(ResourceTypeEnum.CI_TYPE)} + for type_id in (source.get('type_id') or []) + (target.get('type_ids') or []): + _type = CITypeCache.get(type_id) + if _type and _type.name not in res: + return abort(403, ErrFormat.no_permission.format(_type.alias, PermEnum.READ)) + + level2type, types = self._path2level(source.get('type_id'), target.get('type_ids'), path) + if not level2type: + return [], {}, 0, self.page, 0, {} + + source_ids = self._get_src_ids(source) + + graph, target_ids = self._build_graph(source_ids, level2type, target['type_ids'], acl) + if target.get('q'): + target_ids = self._filter_target_ids(target_ids, target['type_ids'], target['q']) + + paths = self._find_paths(graph, source_ids, set(target_ids)) + del graph + + numfound = len(target_ids) + paths = paths[(self.page - 1) * self.count:self.page * self.count] + + response, counter, id2ci = self._wrap_path_result(paths, types) + + return response, counter, len(paths), self.page, numfound, id2ci diff --git a/cmdb-api/api/translations/zh/LC_MESSAGES/messages.mo b/cmdb-api/api/translations/zh/LC_MESSAGES/messages.mo index 8b6e43b..292c0ef 100644 Binary files a/cmdb-api/api/translations/zh/LC_MESSAGES/messages.mo and b/cmdb-api/api/translations/zh/LC_MESSAGES/messages.mo differ diff --git a/cmdb-api/api/translations/zh/LC_MESSAGES/messages.po b/cmdb-api/api/translations/zh/LC_MESSAGES/messages.po index dd3f1bc..6f936de 100644 --- a/cmdb-api/api/translations/zh/LC_MESSAGES/messages.po +++ b/cmdb-api/api/translations/zh/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2024-08-20 13:47+0800\n" +"POT-Creation-Date: 2024-09-26 17:57+0800\n" "PO-Revision-Date: 2023-12-25 20:21+0800\n" "Last-Translator: FULL NAME \n" "Language: zh\n" @@ -492,6 +492,10 @@ msgstr "拓扑视图分组 {} 已经存在" msgid "The group cannot be deleted because the topology view already exists" msgstr "因为该分组下定义了拓扑视图，不能删除" +#: api/lib/cmdb/resp_format.py:158 +msgid "Both the source model and the target model must be selected" +msgstr "源模型和目标模型不能为空!" + #: api/lib/common_setting/resp_format.py:8 msgid "Company info already existed" msgstr "公司信息已存在,无法创建!" diff --git a/cmdb-api/api/views/cmdb/ci_relation.py b/cmdb-api/api/views/cmdb/ci_relation.py index e64621a..ae9bc7d 100644 --- a/cmdb-api/api/views/cmdb/ci_relation.py +++ b/cmdb-api/api/views/cmdb/ci_relation.py @@ -2,7 +2,6 @@ import time - from flask import abort from flask import current_app from flask import request @@ -65,6 +64,39 @@ class CIRelationSearchView(APIView): result=response) +class CIRelationSearchPathView(APIView): + url_prefix = ("/ci_relations/path/s", "/ci_relations/path/search") + + @args_required("source", "target", "path") + def post(self): + """@params: page: page number + page_size | count: page size + source: source CIType, e.g. {type_id: 1, q: `search expr`} + target: target CIType, e.g. {type_ids: [2], q: `search expr`} + path: Path from the Source CIType to the Target CIType, e.g. {source_id: [target_id]} + """ + + page = get_page(request.values.get("page", 1)) + count = get_page_size(request.values.get("count") or request.values.get("page_size")) + + source = request.values.get("source") + target = request.values.get("target") + path = request.values.get("path") + + s = Search(page=page, count=count) + try: + response, counter, total, page, numfound, id2ci = s.search_by_path(source, target, path) + except SearchError as e: + return abort(400, str(e)) + + return self.jsonify(numfound=numfound, + total=total, + page=page, + counter=counter, + paths=response, + id2ci=id2ci) + + class CIRelationStatisticsView(APIView): url_prefix = "/ci_relations/statistics" diff --git a/cmdb-api/api/views/cmdb/ci_type_relation.py b/cmdb-api/api/views/cmdb/ci_type_relation.py index cac6371..060eaaa 100644 --- a/cmdb-api/api/views/cmdb/ci_type_relation.py +++ b/cmdb-api/api/views/cmdb/ci_type_relation.py @@ -8,7 +8,6 @@ from api.lib.cmdb.ci_type import CITypeManager from api.lib.cmdb.ci_type import CITypeRelationManager from api.lib.cmdb.const import PermEnum from api.lib.cmdb.const import ResourceTypeEnum -from api.lib.cmdb.const import RoleEnum from api.lib.cmdb.preference import PreferenceManager from api.lib.cmdb.resp_format import ErrFormat from api.lib.common_setting.decorator import perms_role_required @@ -17,7 +16,7 @@ from api.lib.decorator import args_required from api.lib.perm.acl.acl import ACLManager from api.lib.perm.acl.acl import has_perm_from_args from api.lib.perm.acl.acl import is_app_admin -from api.lib.perm.acl.acl import role_required +from api.lib.utils import handle_arg_list from api.resource import APIView app_cli = CMDBApp() @@ -42,6 +41,19 @@ class GetParentsView(APIView): return self.jsonify(parents=CITypeRelationManager.get_parents(child_id)) +class CITypeRelationPathView(APIView): + url_prefix = ("/ci_type_relations/path",) + + @args_required("source_type_id", "target_type_ids") + def get(self): + source_type_id = request.values.get("source_type_id") + target_type_ids = handle_arg_list(request.values.get("target_type_ids")) + + paths = CITypeRelationManager.find_path(source_type_id, target_type_ids) + + return self.jsonify(paths=paths) + + class CITypeRelationView(APIView): url_prefix = ("/ci_type_relations", "/ci_type_relations//") diff --git a/cmdb-api/requirements.txt b/cmdb-api/requirements.txt index 0ae7166..b05f3e6 100644 --- a/cmdb-api/requirements.txt +++ b/cmdb-api/requirements.txt @@ -56,3 +56,4 @@ colorama>=0.4.6 lz4>=4.3.2 python-magic==0.4.27 jsonpath==0.82.2 +networkx>=3.1