mirror of https://github.com/veops/cmdb.git
Merge pull request #623 from veops/dev_api_relation_path_search
Dev api relation path search
This commit is contained in:
pycook
GitHub
commit
8f7d78c26c
|
|
@ -68,6 +68,7 @@ pycryptodomex = ">=3.19.0"
|
|||
lz4 = ">=4.3.2"
|
||||
python-magic = "==0.4.27"
|
||||
jsonpath = "==0.82.2"
|
||||
networkx = ">=3.1"
|
||||
|
||||
[dev-packages]
|
||||
# Testing
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
from collections import defaultdict
|
||||
|
||||
import copy
|
||||
import networkx as nx
|
||||
import toposort
|
||||
from flask import abort
|
||||
from flask import current_app
|
||||
|
|
@ -845,6 +846,29 @@ class CITypeRelationManager(object):
|
|||
|
||||
return ids
|
||||
|
||||
@staticmethod
|
||||
def find_path(source_type_id, target_type_ids):
|
||||
source_type_id = int(source_type_id)
|
||||
target_type_ids = map(int, target_type_ids)
|
||||
|
||||
graph = nx.DiGraph()
|
||||
|
||||
def get_children(_id):
|
||||
children = CITypeRelation.get_by(parent_id=_id, to_dict=False)
|
||||
|
||||
for i in children:
|
||||
if i.child_id != _id:
|
||||
graph.add_edge(i.parent_id, i.child_id)
|
||||
get_children(i.child_id)
|
||||
|
||||
get_children(source_type_id)
|
||||
|
||||
paths = list(nx.all_simple_paths(graph, source_type_id, target_type_ids))
|
||||
|
||||
del graph
|
||||
|
||||
return paths
|
||||
|
||||
@staticmethod
|
||||
def _wrap_relation_type_dict(type_id, relation_inst):
|
||||
ci_type_dict = CITypeCache.get(type_id).to_dict()
|
||||
|
|
|
|||
|
|
@ -154,3 +154,5 @@ class ErrFormat(CommonErrFormat):
|
|||
topology_group_exists = _l("Topology group {} already exists") # 拓扑视图分组 {} 已经存在
|
||||
# 因为该分组下定义了拓扑视图，不能删除
|
||||
topo_view_exists_cannot_delete_group = _l("The group cannot be deleted because the topology view already exists")
|
||||
|
||||
relation_path_search_src_target_required = _l("Both the source model and the target model must be selected")
|
||||
|
|
@ -1,8 +1,11 @@
|
|||
# -*- coding:utf-8 -*-
|
||||
import json
|
||||
import sys
|
||||
from collections import Counter
|
||||
from collections import defaultdict
|
||||
|
||||
import copy
|
||||
import json
|
||||
import networkx as nx
|
||||
import sys
|
||||
from flask import abort
|
||||
from flask import current_app
|
||||
from flask_login import current_user
|
||||
|
|
@ -13,6 +16,7 @@ from api.lib.cmdb.cache import CITypeCache
|
|||
from api.lib.cmdb.ci import CIRelationManager
|
||||
from api.lib.cmdb.ci_type import CITypeRelationManager
|
||||
from api.lib.cmdb.const import ConstraintEnum
|
||||
from api.lib.cmdb.const import PermEnum
|
||||
from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
|
||||
from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION2
|
||||
from api.lib.cmdb.const import ResourceTypeEnum
|
||||
|
|
@ -28,7 +32,7 @@ from api.models.cmdb import CI
|
|||
|
||||
|
||||
class Search(object):
|
||||
def __init__(self, root_id,
|
||||
def __init__(self, root_id=None,
|
||||
level=None,
|
||||
query=None,
|
||||
fl=None,
|
||||
|
|
@ -419,3 +423,139 @@ class Search(object):
|
|||
level_ids = _level_ids
|
||||
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _get_src_ids(src):
|
||||
q = src.get('q') or ''
|
||||
if not q.startswith('_type:'):
|
||||
q = "_type:{},{}".format(src['type_id'], q)
|
||||
|
||||
return SearchFromDB(q, use_ci_filter=True, only_ids=True, count=100000).search()
|
||||
|
||||
@staticmethod
|
||||
def _filter_target_ids(target_ids, type_ids, q):
|
||||
if not q.startswith('_type:'):
|
||||
q = "_type:({}),{}".format(";".join(map(str, type_ids)), q)
|
||||
|
||||
return SearchFromDB(q, ci_ids=target_ids, use_ci_filter=False, only_ids=True, count=100000).search()
|
||||
|
||||
@staticmethod
|
||||
def _path2level(src_type_id, target_type_ids, path):
|
||||
if not src_type_id or not target_type_ids:
|
||||
return abort(400, ErrFormat.relation_path_search_src_target_required)
|
||||
|
||||
graph = nx.DiGraph()
|
||||
graph.add_edges_from([(int(s), d) for s in path for d in path[s]])
|
||||
|
||||
level2type = defaultdict(set)
|
||||
for target_type_id in target_type_ids:
|
||||
paths = list(nx.all_simple_paths(graph, source=src_type_id, target=target_type_id))
|
||||
for _path in paths:
|
||||
for idx, node in enumerate(_path[1:]):
|
||||
level2type[idx + 1].add(node)
|
||||
nodes = graph.nodes()
|
||||
|
||||
del graph
|
||||
|
||||
return level2type, list(nodes)
|
||||
|
||||
def _build_graph(self, source_ids, level2type, target_type_ids, acl):
|
||||
type2filter_perms = dict()
|
||||
if not self.is_app_admin:
|
||||
res2 = acl.get_resources(ResourceTypeEnum.CI_FILTER)
|
||||
if res2:
|
||||
type2filter_perms = CIFilterPermsCRUD().get_by_ids(list(map(int, [i['name'] for i in res2])))
|
||||
|
||||
target_type_ids = set(target_type_ids)
|
||||
graph = nx.DiGraph()
|
||||
target_ids = []
|
||||
key = list(map(str, source_ids))
|
||||
for level in level2type:
|
||||
filter_type_ids = level2type[level]
|
||||
id_filter_limit = dict()
|
||||
for _type_id in filter_type_ids:
|
||||
if type2filter_perms.get(_type_id):
|
||||
_id_filter_limit, _ = self._get_ci_filter(type2filter_perms[_type_id])
|
||||
id_filter_limit.update(_id_filter_limit)
|
||||
|
||||
has_target = filter_type_ids & target_type_ids
|
||||
|
||||
res = [json.loads(x).items() for x in [i or '{}' for i in rd.get(key, REDIS_PREFIX_CI_RELATION) or []]]
|
||||
_key = []
|
||||
for idx, _id in enumerate(key):
|
||||
valid_targets = [i[0] for i in res[idx] if i[1] in filter_type_ids and
|
||||
(not id_filter_limit or int(i[0]) in id_filter_limit)]
|
||||
_key.extend(valid_targets)
|
||||
graph.add_edges_from(zip([_id] * len(valid_targets), valid_targets))
|
||||
|
||||
if has_target:
|
||||
target_ids.extend([j[0] for i in res for j in i if j[1] in target_type_ids])
|
||||
|
||||
key = copy.deepcopy(_key)
|
||||
|
||||
return graph, target_ids
|
||||
|
||||
@staticmethod
|
||||
def _find_paths(graph, source_ids, target_ids, max_depth=6):
|
||||
paths = []
|
||||
for source_id in source_ids:
|
||||
_paths = nx.all_simple_paths(graph, source=source_id, target=target_ids, cutoff=max_depth)
|
||||
paths.extend(_paths)
|
||||
|
||||
return paths
|
||||
|
||||
@staticmethod
|
||||
def _wrap_path_result(paths, types):
|
||||
ci_ids = [j for i in paths for j in i]
|
||||
|
||||
response, _, _, _, _, _ = SearchFromDB("_type:({})".format(";".join(map(str, types))),
|
||||
use_ci_filter=False,
|
||||
ci_ids=list(map(int, ci_ids)),
|
||||
count=1000000).search()
|
||||
id2ci = {str(i.get('_id')): i for i in response}
|
||||
|
||||
result = defaultdict(list)
|
||||
counter = defaultdict(int)
|
||||
|
||||
for path in paths:
|
||||
key = "-".join([id2ci.get(i, {}).get('ci_type_alias') or '' for i in path])
|
||||
counter[key] += 1
|
||||
result[key].append(path)
|
||||
|
||||
return result, counter, id2ci
|
||||
|
||||
def search_by_path(self, source, target, path):
|
||||
"""
|
||||
|
||||
:param source: {type_id: id, q: expr}
|
||||
:param target: {type_ids: [id], q: expr}
|
||||
:param path: {parent_id: [child_id]}, use type id
|
||||
:return:
|
||||
"""
|
||||
acl = ACLManager('cmdb')
|
||||
if not self.is_app_admin:
|
||||
res = {i['name'] for i in acl.get_resources(ResourceTypeEnum.CI_TYPE)}
|
||||
for type_id in (source.get('type_id') or []) + (target.get('type_ids') or []):
|
||||
_type = CITypeCache.get(type_id)
|
||||
if _type and _type.name not in res:
|
||||
return abort(403, ErrFormat.no_permission.format(_type.alias, PermEnum.READ))
|
||||
|
||||
level2type, types = self._path2level(source.get('type_id'), target.get('type_ids'), path)
|
||||
if not level2type:
|
||||
return [], {}, 0, self.page, 0, {}
|
||||
|
||||
source_ids = self._get_src_ids(source)
|
||||
|
||||
graph, target_ids = self._build_graph(source_ids, level2type, target['type_ids'], acl)
|
||||
if target.get('q'):
|
||||
target_ids = self._filter_target_ids(target_ids, target['type_ids'], target['q'])
|
||||
|
||||
paths = self._find_paths(graph, source_ids, set(target_ids))
|
||||
del graph
|
||||
|
||||
numfound = len(target_ids)
|
||||
paths = paths[(self.page - 1) * self.count:self.page * self.count]
|
||||
|
||||
response, counter, id2ci = self._wrap_path_result(paths, types)
|
||||
|
||||
return response, counter, len(paths), self.page, numfound, id2ci
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -7,7 +7,7 @@ msgid ""
|
|||
msgstr ""
|
||||
"Project-Id-Version: PROJECT VERSION\n"
|
||||
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
|
||||
"POT-Creation-Date: 2024-08-20 13:47+0800\n"
|
||||
"POT-Creation-Date: 2024-09-26 17:57+0800\n"
|
||||
"PO-Revision-Date: 2023-12-25 20:21+0800\n"
|
||||
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
|
||||
"Language: zh\n"
|
||||
|
|
@ -492,6 +492,10 @@ msgstr "拓扑视图分组 {} 已经存在"
|
|||
msgid "The group cannot be deleted because the topology view already exists"
|
||||
msgstr "因为该分组下定义了拓扑视图，不能删除"
|
||||
|
||||
#: api/lib/cmdb/resp_format.py:158
|
||||
msgid "Both the source model and the target model must be selected"
|
||||
msgstr "源模型和目标模型不能为空!"
|
||||
|
||||
#: api/lib/common_setting/resp_format.py:8
|
||||
msgid "Company info already existed"
|
||||
msgstr "公司信息已存在,无法创建!"
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
|
||||
|
||||
import time
|
||||
|
||||
from flask import abort
|
||||
from flask import current_app
|
||||
from flask import request
|
||||
|
|
@ -65,6 +64,39 @@ class CIRelationSearchView(APIView):
|
|||
result=response)
|
||||
|
||||
|
||||
class CIRelationSearchPathView(APIView):
|
||||
url_prefix = ("/ci_relations/path/s", "/ci_relations/path/search")
|
||||
|
||||
@args_required("source", "target", "path")
|
||||
def post(self):
|
||||
"""@params: page: page number
|
||||
page_size | count: page size
|
||||
source: source CIType, e.g. {type_id: 1, q: `search expr`}
|
||||
target: target CIType, e.g. {type_ids: [2], q: `search expr`}
|
||||
path: Path from the Source CIType to the Target CIType, e.g. {source_id: [target_id]}
|
||||
"""
|
||||
|
||||
page = get_page(request.values.get("page", 1))
|
||||
count = get_page_size(request.values.get("count") or request.values.get("page_size"))
|
||||
|
||||
source = request.values.get("source")
|
||||
target = request.values.get("target")
|
||||
path = request.values.get("path")
|
||||
|
||||
s = Search(page=page, count=count)
|
||||
try:
|
||||
response, counter, total, page, numfound, id2ci = s.search_by_path(source, target, path)
|
||||
except SearchError as e:
|
||||
return abort(400, str(e))
|
||||
|
||||
return self.jsonify(numfound=numfound,
|
||||
total=total,
|
||||
page=page,
|
||||
counter=counter,
|
||||
paths=response,
|
||||
id2ci=id2ci)
|
||||
|
||||
|
||||
class CIRelationStatisticsView(APIView):
|
||||
url_prefix = "/ci_relations/statistics"
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ from api.lib.cmdb.ci_type import CITypeManager
|
|||
from api.lib.cmdb.ci_type import CITypeRelationManager
|
||||
from api.lib.cmdb.const import PermEnum
|
||||
from api.lib.cmdb.const import ResourceTypeEnum
|
||||
from api.lib.cmdb.const import RoleEnum
|
||||
from api.lib.cmdb.preference import PreferenceManager
|
||||
from api.lib.cmdb.resp_format import ErrFormat
|
||||
from api.lib.common_setting.decorator import perms_role_required
|
||||
|
|
@ -17,7 +16,7 @@ from api.lib.decorator import args_required
|
|||
from api.lib.perm.acl.acl import ACLManager
|
||||
from api.lib.perm.acl.acl import has_perm_from_args
|
||||
from api.lib.perm.acl.acl import is_app_admin
|
||||
from api.lib.perm.acl.acl import role_required
|
||||
from api.lib.utils import handle_arg_list
|
||||
from api.resource import APIView
|
||||
|
||||
app_cli = CMDBApp()
|
||||
|
|
@ -42,6 +41,19 @@ class GetParentsView(APIView):
|
|||
return self.jsonify(parents=CITypeRelationManager.get_parents(child_id))
|
||||
|
||||
|
||||
class CITypeRelationPathView(APIView):
|
||||
url_prefix = ("/ci_type_relations/path",)
|
||||
|
||||
@args_required("source_type_id", "target_type_ids")
|
||||
def get(self):
|
||||
source_type_id = request.values.get("source_type_id")
|
||||
target_type_ids = handle_arg_list(request.values.get("target_type_ids"))
|
||||
|
||||
paths = CITypeRelationManager.find_path(source_type_id, target_type_ids)
|
||||
|
||||
return self.jsonify(paths=paths)
|
||||
|
||||
|
||||
class CITypeRelationView(APIView):
|
||||
url_prefix = ("/ci_type_relations", "/ci_type_relations/<int:parent_id>/<int:child_id>")
|
||||
|
||||
|
|
|
|||
|
|
@ -56,3 +56,4 @@ colorama>=0.4.6
|
|||
lz4>=4.3.2
|
||||
python-magic==0.4.27
|
||||
jsonpath==0.82.2
|
||||
networkx>=3.1
|
||||
|
|
|
|||
Loading…
Reference in New Issue