mirror of https://github.com/veops/cmdb.git
422 lines
18 KiB
Python
422 lines
18 KiB
Python
# -*- coding:utf-8 -*-
|
|
import json
|
|
import sys
|
|
from collections import Counter
|
|
|
|
from flask import abort
|
|
from flask import current_app
|
|
from flask_login import current_user
|
|
|
|
from api.extensions import rd
|
|
from api.lib.cmdb.cache import AttributeCache
|
|
from api.lib.cmdb.cache import CITypeCache
|
|
from api.lib.cmdb.ci import CIRelationManager
|
|
from api.lib.cmdb.ci_type import CITypeRelationManager
|
|
from api.lib.cmdb.const import ConstraintEnum
|
|
from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
|
|
from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION2
|
|
from api.lib.cmdb.const import ResourceTypeEnum
|
|
from api.lib.cmdb.perms import CIFilterPermsCRUD
|
|
from api.lib.cmdb.resp_format import ErrFormat
|
|
from api.lib.cmdb.search.ci.db.search import Search as SearchFromDB
|
|
from api.lib.cmdb.search.ci.es.search import Search as SearchFromES
|
|
from api.lib.cmdb.utils import TableMap
|
|
from api.lib.cmdb.utils import ValueTypeMap
|
|
from api.lib.perm.acl.acl import ACLManager
|
|
from api.lib.perm.acl.acl import is_app_admin
|
|
from api.models.cmdb import CI
|
|
|
|
|
|
class Search(object):
|
|
def __init__(self, root_id,
|
|
level=None,
|
|
query=None,
|
|
fl=None,
|
|
facet_field=None,
|
|
page=1,
|
|
count=None,
|
|
sort=None,
|
|
reverse=False,
|
|
ancestor_ids=None,
|
|
descendant_ids=None,
|
|
has_m2m=None,
|
|
root_parent_path=None):
|
|
self.orig_query = query
|
|
self.fl = fl
|
|
self.facet_field = facet_field
|
|
self.page = page
|
|
self.count = count or current_app.config.get("DEFAULT_PAGE_COUNT")
|
|
self.sort = sort or ("ci_id" if current_app.config.get("USE_ES") else None)
|
|
|
|
self.root_id = root_id
|
|
self.level = level or 0
|
|
self.reverse = reverse
|
|
|
|
self.level2constraint = CITypeRelationManager.get_level2constraint(
|
|
root_id[0] if root_id and isinstance(root_id, list) else root_id,
|
|
level[0] if isinstance(level, list) and level else level)
|
|
|
|
self.ancestor_ids = ancestor_ids
|
|
self.descendant_ids = descendant_ids
|
|
self.root_parent_path = root_parent_path
|
|
self.has_m2m = has_m2m or False
|
|
if not self.has_m2m:
|
|
if self.ancestor_ids:
|
|
self.has_m2m = True
|
|
else:
|
|
level = level[0] if isinstance(level, list) and level else level
|
|
for _l, c in self.level2constraint.items():
|
|
if _l < int(level) and c == ConstraintEnum.Many2Many:
|
|
self.has_m2m = True
|
|
|
|
self.type2filter_perms = {}
|
|
|
|
self.is_app_admin = is_app_admin('cmdb') or current_user.username == "worker"
|
|
|
|
def _get_ids(self, ids):
|
|
|
|
merge_ids = []
|
|
key = []
|
|
_tmp = []
|
|
for level in range(1, sorted(self.level)[-1] + 1):
|
|
if len(self.descendant_ids or []) >= level and self.type2filter_perms.get(self.descendant_ids[level - 1]):
|
|
id_filter_limit, _ = self._get_ci_filter(self.type2filter_perms[self.descendant_ids[level - 1]])
|
|
else:
|
|
id_filter_limit = {}
|
|
|
|
if not self.has_m2m:
|
|
key, prefix = list(map(str, ids)), REDIS_PREFIX_CI_RELATION
|
|
|
|
else:
|
|
if not self.ancestor_ids:
|
|
if level == 1:
|
|
key, prefix = list(map(str, ids)), REDIS_PREFIX_CI_RELATION
|
|
else:
|
|
key = list(set(["{},{}".format(i, j) for idx, i in enumerate(key) for j in _tmp[idx]]))
|
|
prefix = REDIS_PREFIX_CI_RELATION2
|
|
else:
|
|
if level == 1:
|
|
key, prefix = ["{},{}".format(self.ancestor_ids, i) for i in ids], REDIS_PREFIX_CI_RELATION2
|
|
else:
|
|
key = list(set(["{},{}".format(i, j) for idx, i in enumerate(key) for j in _tmp[idx]]))
|
|
prefix = REDIS_PREFIX_CI_RELATION2
|
|
|
|
if not key or id_filter_limit is None:
|
|
return []
|
|
|
|
res = [json.loads(x).items() for x in [i or '{}' for i in rd.get(key, prefix) or []]]
|
|
_tmp = [[i[0] for i in x if (not id_filter_limit or (
|
|
key[idx] not in id_filter_limit or int(i[0]) in id_filter_limit[key[idx]]) or
|
|
int(i[0]) in id_filter_limit)] for idx, x in enumerate(res)]
|
|
|
|
ids = [j for i in _tmp for j in i]
|
|
|
|
if level in self.level:
|
|
merge_ids.extend(ids)
|
|
|
|
return merge_ids
|
|
|
|
def _get_reverse_ids(self, ids):
|
|
merge_ids = []
|
|
level2ids = {}
|
|
for level in range(1, sorted(self.level)[-1] + 1):
|
|
ids, _level2ids = CIRelationManager.get_ancestor_ids(ids, 1)
|
|
|
|
if _level2ids.get(2):
|
|
level2ids[level + 1] = _level2ids[2]
|
|
|
|
if level in self.level:
|
|
if level in level2ids and level2ids[level]:
|
|
merge_ids.extend(set(ids) & set(level2ids[level]))
|
|
else:
|
|
merge_ids.extend(ids)
|
|
|
|
return merge_ids
|
|
|
|
def _has_read_perm_from_parent_nodes(self):
|
|
self.root_parent_path = list(map(str, self.root_parent_path))
|
|
if str(self.root_id).isdigit() and str(self.root_id) not in self.root_parent_path:
|
|
self.root_parent_path.append(str(self.root_id))
|
|
self.root_parent_path = set(self.root_parent_path)
|
|
|
|
if self.is_app_admin:
|
|
self.type2filter_perms = {}
|
|
return True
|
|
|
|
res = ACLManager().get_resources(ResourceTypeEnum.CI_FILTER) or {}
|
|
self.type2filter_perms = CIFilterPermsCRUD().get_by_ids(list(map(int, [i['name'] for i in res]))) or {}
|
|
for _, filters in self.type2filter_perms.items():
|
|
if set((filters.get('id_filter') or {}).keys()) & self.root_parent_path:
|
|
return True
|
|
|
|
return True
|
|
|
|
def search(self, only_ids=False):
|
|
use_ci_filter = len(self.descendant_ids or []) == self.level[0] - 1
|
|
parent_node_perm_passed = not self.is_app_admin and self._has_read_perm_from_parent_nodes()
|
|
|
|
ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
|
|
cis = [CI.get_by_id(_id) or abort(404, ErrFormat.ci_not_found.format("id={}".format(_id))) for _id in ids]
|
|
|
|
merge_ids = self._get_ids(ids) if not self.reverse else self._get_reverse_ids(ids)
|
|
|
|
if not self.orig_query or ("_type:" not in self.orig_query
|
|
and "type_id:" not in self.orig_query
|
|
and "ci_type:" not in self.orig_query):
|
|
type_ids = []
|
|
for level in self.level:
|
|
for ci in cis:
|
|
if not self.reverse:
|
|
type_ids.extend(CITypeRelationManager.get_child_type_ids(ci.type_id, level))
|
|
else:
|
|
type_ids.extend(CITypeRelationManager.get_parent_type_ids(ci.type_id, level))
|
|
type_ids = set(type_ids)
|
|
if self.orig_query:
|
|
self.orig_query = "_type:({0}),{1}".format(";".join(map(str, type_ids)), self.orig_query)
|
|
else:
|
|
self.orig_query = "_type:({0})".format(";".join(map(str, type_ids)))
|
|
|
|
if not merge_ids:
|
|
# cis, counter, total, self.page, numfound, facet_
|
|
return [], {}, 0, self.page, 0, {}
|
|
|
|
if current_app.config.get("USE_ES"):
|
|
return SearchFromES(self.orig_query,
|
|
fl=self.fl,
|
|
facet_field=self.facet_field,
|
|
page=self.page,
|
|
count=self.count,
|
|
sort=self.sort,
|
|
ci_ids=merge_ids).search()
|
|
else:
|
|
return SearchFromDB(self.orig_query,
|
|
fl=self.fl,
|
|
facet_field=self.facet_field,
|
|
page=self.page,
|
|
count=self.count,
|
|
sort=self.sort,
|
|
ci_ids=merge_ids,
|
|
parent_node_perm_passed=parent_node_perm_passed,
|
|
use_ci_filter=use_ci_filter,
|
|
only_ids=only_ids).search()
|
|
|
|
def _get_ci_filter(self, filter_perms, ci_filters=None):
|
|
ci_filters = ci_filters or []
|
|
if ci_filters:
|
|
result = {}
|
|
for item in ci_filters:
|
|
res = SearchFromDB('_type:{},{}'.format(item['type_id'], item['ci_filter']),
|
|
count=sys.maxsize, parent_node_perm_passed=True).get_ci_ids()
|
|
if res:
|
|
result[item['type_id']] = set(res)
|
|
|
|
return {}, result if result else None
|
|
|
|
result = dict()
|
|
if filter_perms.get('id_filter'):
|
|
for k in filter_perms['id_filter']:
|
|
node_path = k.split(',')
|
|
if len(node_path) == 1:
|
|
result[int(node_path[0])] = 1
|
|
elif not self.has_m2m:
|
|
result.setdefault(node_path[-2], set()).add(int(node_path[-1]))
|
|
else:
|
|
result.setdefault(','.join(node_path[:-1]), set()).add(int(node_path[-1]))
|
|
if result:
|
|
return result, None
|
|
else:
|
|
return None, None
|
|
|
|
return {}, None
|
|
|
|
def statistics(self, type_ids, need_filter=True):
|
|
self.level = int(self.level)
|
|
|
|
acl = ACLManager('cmdb')
|
|
|
|
type2filter_perms = dict()
|
|
if not self.is_app_admin:
|
|
res2 = acl.get_resources(ResourceTypeEnum.CI_FILTER)
|
|
if res2:
|
|
type2filter_perms = CIFilterPermsCRUD().get_by_ids(list(map(int, [i['name'] for i in res2])))
|
|
|
|
ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
|
|
_tmp, tmp_res = [], []
|
|
level2ids = {}
|
|
for lv in range(1, self.level + 1):
|
|
level2ids[lv] = []
|
|
|
|
if need_filter:
|
|
id_filter_limit, ci_filter_limit = None, None
|
|
if len(self.descendant_ids or []) >= lv and type2filter_perms.get(self.descendant_ids[lv - 1]):
|
|
id_filter_limit, _ = self._get_ci_filter(type2filter_perms[self.descendant_ids[lv - 1]])
|
|
elif type_ids and self.level == lv:
|
|
ci_filters = [type2filter_perms[type_id] for type_id in type_ids if type_id in type2filter_perms]
|
|
if ci_filters:
|
|
id_filter_limit, ci_filter_limit = self._get_ci_filter({}, ci_filters=ci_filters)
|
|
else:
|
|
id_filter_limit = {}
|
|
else:
|
|
id_filter_limit = {}
|
|
else:
|
|
id_filter_limit, ci_filter_limit = {}, {}
|
|
|
|
if lv == 1:
|
|
if not self.has_m2m:
|
|
key, prefix = [str(i) for i in ids], REDIS_PREFIX_CI_RELATION
|
|
else:
|
|
key = ["{},{}".format(self.ancestor_ids, _id) for _id in ids]
|
|
if not self.ancestor_ids:
|
|
key, prefix = [str(i) for i in ids], REDIS_PREFIX_CI_RELATION
|
|
else:
|
|
prefix = REDIS_PREFIX_CI_RELATION2
|
|
|
|
level2ids[lv] = [[i] for i in key]
|
|
|
|
if not key or id_filter_limit is None:
|
|
_tmp = [[]] * len(ids)
|
|
continue
|
|
|
|
res = [json.loads(x).items() for x in [i or '{}' for i in rd.get(key, prefix) or []]]
|
|
_tmp = []
|
|
if type_ids and lv == self.level:
|
|
_tmp = [[i for i in x if i[1] in type_ids and
|
|
(not id_filter_limit or (key[idx] not in id_filter_limit or
|
|
int(i[0]) in id_filter_limit[key[idx]]) or
|
|
int(i[0]) in id_filter_limit)] for idx, x in enumerate(res)]
|
|
else:
|
|
_tmp = [[i for i in x if (not id_filter_limit or (key[idx] not in id_filter_limit or
|
|
int(i[0]) in id_filter_limit[key[idx]]) or
|
|
int(i[0]) in id_filter_limit)] for idx, x in enumerate(res)]
|
|
|
|
if ci_filter_limit:
|
|
_tmp = [[j for j in i if j[1] not in ci_filter_limit or int(j[0]) in ci_filter_limit[j[1]]]
|
|
for i in _tmp]
|
|
|
|
else:
|
|
|
|
for idx, item in enumerate(_tmp):
|
|
if item:
|
|
if not self.has_m2m:
|
|
key, prefix = [i[0] for i in item], REDIS_PREFIX_CI_RELATION
|
|
else:
|
|
key = list(set(['{},{}'.format(j, i[0]) for i in item for j in level2ids[lv - 1][idx]]))
|
|
prefix = REDIS_PREFIX_CI_RELATION2
|
|
|
|
level2ids[lv].append(key)
|
|
|
|
if key:
|
|
res = [json.loads(x).items() for x in [i or '{}' for i in rd.get(key, prefix) or []]]
|
|
if type_ids and lv == self.level:
|
|
tmp_res = [[i for i in x if i[1] in type_ids and
|
|
(not id_filter_limit or (
|
|
key[idx] not in id_filter_limit or
|
|
int(i[0]) in id_filter_limit[key[idx]]) or
|
|
int(i[0]) in id_filter_limit)] for idx, x in enumerate(res)]
|
|
else:
|
|
tmp_res = [[i for i in x if (not id_filter_limit or (
|
|
key[idx] not in id_filter_limit or
|
|
int(i[0]) in id_filter_limit[key[idx]]) or
|
|
int(i[0]) in id_filter_limit)] for idx, x in
|
|
enumerate(res)]
|
|
|
|
if ci_filter_limit:
|
|
tmp_res = [[j for j in i if j[1] not in ci_filter_limit or
|
|
int(j[0]) in ci_filter_limit[j[1]]] for i in tmp_res]
|
|
else:
|
|
tmp_res = []
|
|
|
|
if tmp_res:
|
|
_tmp[idx] = [j for i in tmp_res for j in i]
|
|
else:
|
|
_tmp[idx] = []
|
|
level2ids[lv].append([])
|
|
|
|
result = {str(_id): len(_tmp[idx]) for idx, _id in enumerate(ids)}
|
|
|
|
result.update(
|
|
detail={str(_id): dict(Counter([i[1] for i in _tmp[idx]]).items()) for idx, _id in enumerate(ids)})
|
|
|
|
return result
|
|
|
|
def search_full(self, type_ids):
|
|
def _get_id2name(_type_id):
|
|
ci_type = CITypeCache.get(_type_id)
|
|
|
|
attr = AttributeCache.get(ci_type.unique_id)
|
|
value_table = TableMap(attr=attr).table
|
|
serializer = ValueTypeMap.serialize[attr.value_type]
|
|
unique_value = {i.ci_id: serializer(i.value) for i in value_table.get_by(attr_id=attr.id, to_dict=False)}
|
|
|
|
attr = AttributeCache.get(ci_type.show_id)
|
|
if attr:
|
|
value_table = TableMap(attr=attr).table
|
|
serializer = ValueTypeMap.serialize[attr.value_type]
|
|
show_value = {i.ci_id: serializer(i.value) for i in value_table.get_by(attr_id=attr.id, to_dict=False)}
|
|
else:
|
|
show_value = unique_value
|
|
|
|
return show_value, unique_value
|
|
|
|
self.level = int(self.level)
|
|
|
|
acl = ACLManager('cmdb')
|
|
|
|
type2filter_perms = dict()
|
|
if not self.is_app_admin:
|
|
res2 = acl.get_resources(ResourceTypeEnum.CI_FILTER)
|
|
if res2:
|
|
type2filter_perms = CIFilterPermsCRUD().get_by_ids(list(map(int, [i['name'] for i in res2])))
|
|
|
|
ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
|
|
|
|
level_ids = [str(i) for i in ids]
|
|
result = []
|
|
id2children = {}
|
|
id2name = _get_id2name(type_ids[0])
|
|
for i in level_ids:
|
|
item = dict(id=int(i),
|
|
type_id=type_ids[0],
|
|
isLeaf=False,
|
|
title=id2name[0].get(int(i)),
|
|
uniqueValue=id2name[1].get(int(i)),
|
|
children=[])
|
|
result.append(item)
|
|
id2children[str(i)] = item['children']
|
|
|
|
for lv in range(1, self.level):
|
|
|
|
if len(type_ids or []) >= lv and type2filter_perms.get(type_ids[lv]):
|
|
id_filter_limit, _ = self._get_ci_filter(type2filter_perms[type_ids[lv]])
|
|
else:
|
|
id_filter_limit = {}
|
|
|
|
if self.has_m2m and lv != 1:
|
|
key, prefix = [i for i in level_ids], REDIS_PREFIX_CI_RELATION2
|
|
else:
|
|
key, prefix = [i.split(',')[-1] for i in level_ids], REDIS_PREFIX_CI_RELATION
|
|
res = [json.loads(x).items() for x in [i or '{}' for i in rd.get(key, prefix) or []]]
|
|
res = [[i for i in x if (not id_filter_limit or (key[idx] not in id_filter_limit or
|
|
int(i[0]) in id_filter_limit[key[idx]]) or
|
|
int(i[0]) in id_filter_limit)] for idx, x in enumerate(res)]
|
|
_level_ids = []
|
|
type_id = type_ids[lv]
|
|
id2name = _get_id2name(type_id)
|
|
for idx, node_path in enumerate(level_ids):
|
|
for child_id, _ in (res[idx] or []):
|
|
item = dict(id=int(child_id),
|
|
type_id=type_id,
|
|
isLeaf=True if lv == self.level - 1 else False,
|
|
title=id2name[0].get(int(child_id)),
|
|
uniqueValue=id2name[1].get(int(child_id)),
|
|
children=[])
|
|
id2children[node_path].append(item)
|
|
|
|
_node_path = "{},{}".format(node_path, child_id)
|
|
_level_ids.append(_node_path)
|
|
id2children[_node_path] = item['children']
|
|
|
|
level_ids = _level_ids
|
|
|
|
return result
|