mirror of https://github.com/veops/cmdb.git
Merge pull request #628 from veops/dev_api_relation_path_search
feat(api): relation path search
This commit is contained in:
pycook
GitHub
commit
75c31d4256
|
|
@ -66,6 +66,7 @@ class Search(object):
|
|||
self.use_id_filter = use_id_filter
|
||||
self.use_ci_filter = use_ci_filter
|
||||
self.only_ids = only_ids
|
||||
self.multi_type_has_ci_filter = False
|
||||
|
||||
self.valid_type_names = []
|
||||
self.type2filter_perms = dict()
|
||||
|
|
@ -140,9 +141,10 @@ class Search(object):
|
|||
self.type_id_list.remove(str(ci_type.id))
|
||||
type_id_list.remove(str(ci_type.id))
|
||||
sub.extend([i for i in queries[1:] if isinstance(i, six.string_types)])
|
||||
|
||||
sub.insert(0, "_type:{}".format(ci_type.id))
|
||||
queries.append(dict(operator="|", queries=sub))
|
||||
|
||||
self.multi_type_has_ci_filter = True
|
||||
if self.type2filter_perms[ci_type.id].get('attr_filter'):
|
||||
if type_num == 1:
|
||||
if not self.fl:
|
||||
|
|
@ -172,9 +174,9 @@ class Search(object):
|
|||
if type_id_list:
|
||||
type_ids = ",".join(type_id_list)
|
||||
_query_sql = QUERY_CI_BY_TYPE.format(type_ids)
|
||||
if self.only_type_query:
|
||||
if self.only_type_query or self.multi_type_has_ci_filter:
|
||||
return _query_sql
|
||||
elif type_num > 1:
|
||||
elif type_num > 1: # there must be instance-level access control
|
||||
return "select c_cis.id as ci_id from c_cis where c_cis.id=0"
|
||||
|
||||
return ""
|
||||
|
|
@ -253,7 +255,7 @@ class Search(object):
|
|||
return ret_sql.format(query_sql, "ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format(
|
||||
(self.page - 1) * self.count, sort_type, self.count))
|
||||
|
||||
elif self.type_id_list:
|
||||
elif self.type_id_list and not self.multi_type_has_ci_filter:
|
||||
self.query_sql = "SELECT B.ci_id FROM ({0}) AS B {1}".format(
|
||||
query_sql,
|
||||
"INNER JOIN c_cis on c_cis.id=B.ci_id WHERE c_cis.type_id IN ({0}) ".format(
|
||||
|
|
@ -278,7 +280,7 @@ class Search(object):
|
|||
def __sort_by_type(self, sort_type, query_sql):
|
||||
ret_sql = "SELECT SQL_CALC_FOUND_ROWS DISTINCT B.ci_id FROM ({0}) AS B {1}"
|
||||
|
||||
if self.type_id_list:
|
||||
if self.type_id_list and not self.multi_type_has_ci_filter:
|
||||
self.query_sql = "SELECT B.ci_id FROM ({0}) AS B {1}".format(
|
||||
query_sql,
|
||||
"INNER JOIN c_cis on c_cis.id=B.ci_id WHERE c_cis.type_id IN ({0}) ".format(
|
||||
|
|
@ -311,7 +313,7 @@ class Search(object):
|
|||
WHERE {1}.attr_id = {3}""".format("ALIAS", table_name, query_sql, attr_id)
|
||||
new_table = _v_query_sql
|
||||
|
||||
if self.only_type_query or not self.type_id_list:
|
||||
if self.only_type_query or not self.type_id_list or self.multi_type_has_ci_filter:
|
||||
return ("SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id FROM ({0}) AS C ORDER BY C.value {2} "
|
||||
"LIMIT {1:d}, {3};".format(new_table, (self.page - 1) * self.count, sort_type, self.count))
|
||||
|
||||
|
|
@ -518,8 +520,8 @@ class Search(object):
|
|||
_query_sql = ""
|
||||
if isinstance(q, dict):
|
||||
alias, _query_sql, operator = self.__query_build_by_field(q['queries'], True, True, alias, is_sub=True)
|
||||
current_app.logger.info(_query_sql)
|
||||
current_app.logger.info((operator, is_first, alias))
|
||||
# current_app.logger.info(_query_sql)
|
||||
# current_app.logger.info((operator, is_first, alias))
|
||||
operator = q['operator']
|
||||
|
||||
elif ":" in q and not q.startswith("*"):
|
||||
|
|
@ -617,6 +619,7 @@ class Search(object):
|
|||
k, _, _, _ = self._attr_name_proc(f)
|
||||
if k:
|
||||
_fl.append(k)
|
||||
|
||||
return _fl
|
||||
else:
|
||||
return self.fl
|
||||
|
|
@ -638,6 +641,8 @@ class Search(object):
|
|||
if ci_ids:
|
||||
response = CIManager.get_cis_by_ids(ci_ids, ret_key=self.ret_key, fields=_fl, excludes=self.excludes)
|
||||
for res in response:
|
||||
if not res:
|
||||
continue
|
||||
ci_type = res.get("ci_type")
|
||||
if ci_type not in counter.keys():
|
||||
counter[ci_type] = 0
|
||||
|
|
|
|||
|
|
@ -29,6 +29,8 @@ from api.lib.cmdb.utils import ValueTypeMap
|
|||
from api.lib.perm.acl.acl import ACLManager
|
||||
from api.lib.perm.acl.acl import is_app_admin
|
||||
from api.models.cmdb import CI
|
||||
from api.models.cmdb import CITypeRelation
|
||||
from api.models.cmdb import RelationType
|
||||
|
||||
|
||||
class Search(object):
|
||||
|
|
@ -437,7 +439,10 @@ class Search(object):
|
|||
if not q.startswith('_type:'):
|
||||
q = "_type:({}),{}".format(";".join(map(str, type_ids)), q)
|
||||
|
||||
return SearchFromDB(q, ci_ids=target_ids, use_ci_filter=False, only_ids=True, count=100000).search()
|
||||
ci_ids = SearchFromDB(q, ci_ids=target_ids, use_ci_filter=True, only_ids=True, count=100000).search()
|
||||
cis = CI.get_by(fl=['id', 'type_id'], only_query=True).filter(CI.id.in_(ci_ids))
|
||||
|
||||
return [(str(i.id), i.type_id) for i in cis]
|
||||
|
||||
@staticmethod
|
||||
def _path2level(src_type_id, target_type_ids, path):
|
||||
|
|
@ -445,21 +450,31 @@ class Search(object):
|
|||
return abort(400, ErrFormat.relation_path_search_src_target_required)
|
||||
|
||||
graph = nx.DiGraph()
|
||||
graph.add_edges_from([(int(s), d) for s in path for d in path[s]])
|
||||
|
||||
graph.add_edges_from([(n, _path[idx + 1]) for _path in path for idx, n in enumerate(_path[:-1])])
|
||||
relation_types = defaultdict(dict)
|
||||
level2type = defaultdict(set)
|
||||
for target_type_id in target_type_ids:
|
||||
paths = list(nx.all_simple_paths(graph, source=src_type_id, target=target_type_id))
|
||||
for _path in paths:
|
||||
for idx, node in enumerate(_path[1:]):
|
||||
level2type[idx + 1].add(node)
|
||||
type2show_key = dict()
|
||||
for _path in path:
|
||||
for idx, node in enumerate(_path[1:]):
|
||||
level2type[idx + 1].add(node)
|
||||
|
||||
src = CITypeCache.get(_path[idx])
|
||||
target = CITypeCache.get(node)
|
||||
relation_type = RelationType.get_by(only_query=True).join(
|
||||
CITypeRelation, CITypeRelation.relation_type_id == RelationType.id).filter(
|
||||
CITypeRelation.parent_id == src.id).filter(CITypeRelation.child_id == target.id).first()
|
||||
relation_types[src.alias].update({target.alias: relation_type.name})
|
||||
|
||||
if src.id not in type2show_key:
|
||||
type2show_key[src.id] = AttributeCache.get(src.show_id or src.unique_id).name
|
||||
if target.id not in type2show_key:
|
||||
type2show_key[target.id] = AttributeCache.get(target.show_id or target.unique_id).name
|
||||
|
||||
nodes = graph.nodes()
|
||||
|
||||
del graph
|
||||
return level2type, list(nodes), relation_types, type2show_key
|
||||
|
||||
return level2type, list(nodes)
|
||||
|
||||
def _build_graph(self, source_ids, level2type, target_type_ids, acl):
|
||||
def _build_graph(self, source_ids, source_type_id, level2type, target_type_ids, acl):
|
||||
type2filter_perms = dict()
|
||||
if not self.is_app_admin:
|
||||
res2 = acl.get_resources(ResourceTypeEnum.CI_FILTER)
|
||||
|
|
@ -469,7 +484,8 @@ class Search(object):
|
|||
target_type_ids = set(target_type_ids)
|
||||
graph = nx.DiGraph()
|
||||
target_ids = []
|
||||
key = list(map(str, source_ids))
|
||||
key = [(str(i), source_type_id) for i in source_ids]
|
||||
graph.add_nodes_from(key)
|
||||
for level in level2type:
|
||||
filter_type_ids = level2type[level]
|
||||
id_filter_limit = dict()
|
||||
|
|
@ -480,10 +496,11 @@ class Search(object):
|
|||
|
||||
has_target = filter_type_ids & target_type_ids
|
||||
|
||||
res = [json.loads(x).items() for x in [i or '{}' for i in rd.get(key, REDIS_PREFIX_CI_RELATION) or []]]
|
||||
res = [json.loads(x).items() for x in [i or '{}' for i in rd.get([i[0] for i in key],
|
||||
REDIS_PREFIX_CI_RELATION) or []]]
|
||||
_key = []
|
||||
for idx, _id in enumerate(key):
|
||||
valid_targets = [i[0] for i in res[idx] if i[1] in filter_type_ids and
|
||||
valid_targets = [i for i in res[idx] if i[1] in filter_type_ids and
|
||||
(not id_filter_limit or int(i[0]) in id_filter_limit)]
|
||||
_key.extend(valid_targets)
|
||||
graph.add_edges_from(zip([_id] * len(valid_targets), valid_targets))
|
||||
|
|
@ -496,31 +513,41 @@ class Search(object):
|
|||
return graph, target_ids
|
||||
|
||||
@staticmethod
|
||||
def _find_paths(graph, source_ids, target_ids, max_depth=6):
|
||||
def _find_paths(graph, source_ids, source_type_id, target_ids, valid_path, max_depth=6):
|
||||
paths = []
|
||||
for source_id in source_ids:
|
||||
_paths = nx.all_simple_paths(graph, source=source_id, target=target_ids, cutoff=max_depth)
|
||||
paths.extend(_paths)
|
||||
_paths = nx.all_simple_paths(graph,
|
||||
source=(source_id, source_type_id),
|
||||
target=target_ids,
|
||||
cutoff=max_depth)
|
||||
for __path in _paths:
|
||||
if tuple([i[1] for i in __path]) in valid_path:
|
||||
paths.append([i[0] for i in __path])
|
||||
|
||||
return paths
|
||||
|
||||
@staticmethod
|
||||
def _wrap_path_result(paths, types):
|
||||
def _wrap_path_result(paths, types, valid_path, target_types, type2show_key):
|
||||
ci_ids = [j for i in paths for j in i]
|
||||
|
||||
response, _, _, _, _, _ = SearchFromDB("_type:({})".format(";".join(map(str, types))),
|
||||
use_ci_filter=False,
|
||||
ci_ids=list(map(int, ci_ids)),
|
||||
count=1000000).search()
|
||||
id2ci = {str(i.get('_id')): i for i in response}
|
||||
id2ci = {str(i.get('_id')): i if i['_type'] in target_types else {
|
||||
type2show_key[i['_type']]: i[type2show_key[i['_type']]],
|
||||
"ci_type_alias": i["ci_type_alias"],
|
||||
"_type": i["_type"],
|
||||
} for i in response}
|
||||
|
||||
result = defaultdict(list)
|
||||
counter = defaultdict(int)
|
||||
|
||||
for path in paths:
|
||||
key = "-".join([id2ci.get(i, {}).get('ci_type_alias') or '' for i in path])
|
||||
counter[key] += 1
|
||||
result[key].append(path)
|
||||
if tuple([id2ci.get(i, {}).get('_type') for i in path]) in valid_path:
|
||||
counter[key] += 1
|
||||
result[key].append(path)
|
||||
|
||||
return result, counter, id2ci
|
||||
|
||||
|
|
@ -529,33 +556,38 @@ class Search(object):
|
|||
|
||||
:param source: {type_id: id, q: expr}
|
||||
:param target: {type_ids: [id], q: expr}
|
||||
:param path: {parent_id: [child_id]}, use type id
|
||||
:param path: [source_type_id, ..., target_type_id], use type id
|
||||
:return:
|
||||
"""
|
||||
acl = ACLManager('cmdb')
|
||||
if not self.is_app_admin:
|
||||
res = {i['name'] for i in acl.get_resources(ResourceTypeEnum.CI_TYPE)}
|
||||
for type_id in (source.get('type_id') or []) + (target.get('type_ids') or []):
|
||||
for type_id in (source.get('type_id') and [source['type_id']] or []) + (target.get('type_ids') or []):
|
||||
_type = CITypeCache.get(type_id)
|
||||
if _type and _type.name not in res:
|
||||
return abort(403, ErrFormat.no_permission.format(_type.alias, PermEnum.READ))
|
||||
|
||||
level2type, types = self._path2level(source.get('type_id'), target.get('type_ids'), path)
|
||||
target['type_ids'] = [i[-1] for i in path]
|
||||
level2type, types, relation_types, type2show_key = self._path2level(
|
||||
source.get('type_id'), target.get('type_ids'), path)
|
||||
if not level2type:
|
||||
return [], {}, 0, self.page, 0, {}
|
||||
return [], {}, 0, self.page, 0, {}, {}
|
||||
|
||||
source_ids = self._get_src_ids(source)
|
||||
|
||||
graph, target_ids = self._build_graph(source_ids, level2type, target['type_ids'], acl)
|
||||
if target.get('q'):
|
||||
target_ids = self._filter_target_ids(target_ids, target['type_ids'], target['q'])
|
||||
graph, target_ids = self._build_graph(source_ids, source['type_id'], level2type, target['type_ids'], acl)
|
||||
target_ids = self._filter_target_ids(target_ids, target['type_ids'], target.get('q') or '')
|
||||
paths = self._find_paths(graph,
|
||||
source_ids,
|
||||
source['type_id'],
|
||||
set(target_ids),
|
||||
{tuple(i): 1 for i in path})
|
||||
|
||||
paths = self._find_paths(graph, source_ids, set(target_ids))
|
||||
del graph
|
||||
|
||||
numfound = len(target_ids)
|
||||
numfound = len(paths)
|
||||
paths = paths[(self.page - 1) * self.count:self.page * self.count]
|
||||
|
||||
response, counter, id2ci = self._wrap_path_result(paths, types)
|
||||
|
||||
return response, counter, len(paths), self.page, numfound, id2ci
|
||||
response, counter, id2ci = self._wrap_path_result(paths,
|
||||
types,
|
||||
{tuple(i): 1 for i in path},
|
||||
set(target.get('type_ids') or []),
|
||||
type2show_key)
|
||||
return response, counter, len(paths), self.page, numfound, id2ci, relation_types, type2show_key
|
||||
|
|
|
|||
|
|
@ -73,7 +73,7 @@ class CIRelationSearchPathView(APIView):
|
|||
page_size | count: page size
|
||||
source: source CIType, e.g. {type_id: 1, q: `search expr`}
|
||||
target: target CIType, e.g. {type_ids: [2], q: `search expr`}
|
||||
path: Path from the Source CIType to the Target CIType, e.g. {source_id: [target_id]}
|
||||
path: Path from the Source CIType to the Target CIType, e.g. [1, ..., 2]
|
||||
"""
|
||||
|
||||
page = get_page(request.values.get("page", 1))
|
||||
|
|
@ -85,7 +85,8 @@ class CIRelationSearchPathView(APIView):
|
|||
|
||||
s = Search(page=page, count=count)
|
||||
try:
|
||||
response, counter, total, page, numfound, id2ci = s.search_by_path(source, target, path)
|
||||
(response, counter, total, page, numfound, id2ci,
|
||||
relation_types, type2show_key) = s.search_by_path(source, target, path)
|
||||
except SearchError as e:
|
||||
return abort(400, str(e))
|
||||
|
||||
|
|
@ -94,7 +95,9 @@ class CIRelationSearchPathView(APIView):
|
|||
page=page,
|
||||
counter=counter,
|
||||
paths=response,
|
||||
id2ci=id2ci)
|
||||
id2ci=id2ci,
|
||||
relation_types=relation_types,
|
||||
type2show_key=type2show_key)
|
||||
|
||||
|
||||
class CIRelationStatisticsView(APIView):
|
||||
|
|
|
|||
Loading…
Reference in New Issue