7x31
/
cmdb
mirror of https://github.com/veops/cmdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

relation view has been optimised

This commit is contained in:
pycook 2019-12-03 19:10:54 +08:00
parent 4987908368
commit 9b9ae61505
9 changed files with 88 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/commands/click_cmdb.py
View File

@ -67,7 +67,7 @@ def init_cache():
ci_relations = CIRelation.get_by(to_dict=False)
relations = dict()
for cr in ci_relations:
relations.setdefault(cr.first_ci_id, []).append(cr.second_ci_id)
relations.setdefault(cr.first_ci_id, {}).update({cr.second_ci_id: cr.second_ci.type_id})
for i in relations:
relations[i] = json.dumps(relations[i])
if relations:

2
api/lib/cmdb/ci.py
View File

@ -566,4 +566,4 @@ class CIRelationManager(object):
ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id), queue=CMDB_QUEUE)
return cls.delete(cr.cr_id)
return cls.delete(cr.id)

19
api/lib/cmdb/preference.py
View File

@ -13,6 +13,7 @@ from api.lib.cmdb.cache import AttributeCache
from api.lib.cmdb.cache import CITypeAttributeCache
from api.lib.cmdb.cache import CITypeCache
from api.models.cmdb import CITypeAttribute
from api.models.cmdb import CITypeRelation
from api.models.cmdb import PreferenceRelationView
from api.models.cmdb import PreferenceShowAttributes
from api.models.cmdb import PreferenceTreeView
@ -124,17 +125,28 @@ class PreferenceManager(object):
for i in result[view_name]:
id2type[i['parent_id']] = None
id2type[i['child_id']] = None
topo = {i['child_id']: {i['parent_id']} for i in result[view_name]}
leaf = list(set(toposort.toposort_flatten(topo)) - set([j for i in topo.values() for j in i]))
result[view_name] = toposort.toposort_flatten(
{i['child_id']: {i['parent_id']} for i in result[view_name]})
leaf2show_types = {i: [t['child_id'] for t in CITypeRelation.get_by(parent_id=i)] for i in leaf}
result[view_name] = dict(topo=list(map(list, toposort.toposort(topo))),
topo_flatten=list(toposort.toposort_flatten(topo)),
leaf=leaf,
leaf2show_types=leaf2show_types,
show_types=[CITypeCache.get(j).to_dict()
for i in leaf2show_types.values() for j in i])
for type_id in id2type:
id2type[type_id] = CITypeCache.get(type_id).to_dict()
print(result)
return result, id2type, sorted(name2id, key=lambda x: x[1])
@classmethod
def create_or_update_relation_view(cls, name, cr_ids):
if not cr_ids:
return abort(400, "Node must be selected")
existed = PreferenceRelationView.get_by(name=name, to_dict=False, first=True)
if existed is None:
PreferenceRelationView.create(name=name, cr_ids=json.dumps(cr_ids))
@ -145,4 +157,5 @@ class PreferenceManager(object):
def delete_relation_view(name):
for existed in PreferenceRelationView.get_by(name=name, to_dict=False):
existed.soft_delete()
return name

5
api/lib/cmdb/search/ci/es/search.py
View File

@ -73,6 +73,10 @@ class Search(object):
def _in_query_handle(self, attr, v):
terms = v[1:-1].split(";")
operator = "|"
if attr in ('_type', 'ci_type', 'type_id') and terms and terms[0].isdigit():
attr = "type_id"
terms = map(int, terms)
current_app.logger.warning(terms)
for term in terms:
self._operator2query(operator).append({
"term": {
@ -164,6 +168,7 @@ class Search(object):
def _query_build_raw(self):
queries = handle_arg_list(self.orig_query)
current_app.logger.debug(queries)
self.__query_build_by_field(queries)

65
api/lib/cmdb/search/ci_relation/search.py
View File

@ -15,7 +15,14 @@ from api.models.cmdb import CI
class Search(object):
def __init__(self, root_id, level=1, query=None, fl=None, facet_field=None, page=1, count=None, sort=None):
def __init__(self, root_id,
level=1,
query=None,
fl=None,
facet_field=None,
page=1,
count=None,
sort=None):
self.orig_query = query
self.fl = fl
self.facet_field = facet_field
@ -24,22 +31,36 @@ class Search(object):
self.sort = sort or ("ci_id" if current_app.config.get("USE_ES") else None)
self.root_id = root_id
self.level = int(level)
self.level = level
def search(self):
ci = CI.get_by_id(self.root_id) or abort(404, "CI <{0}> does not exist".format(self.root_id))
ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
for _ in range(0, self.level):
print(rd.get(ids, REDIS_PREFIX_CI_RELATION))
_tmp = list(map(json.loads, filter(lambda x: x is not None, rd.get(ids, REDIS_PREFIX_CI_RELATION) or [])))
ids = [j for i in _tmp for j in i]
cis = [CI.get_by_id(_id) or abort(404, "CI <{0}> does not exist".format(_id)) for _id in ids]
merge_ids = []
for level in self.level:
ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
for _ in range(0, level):
_tmp = list(map(lambda x: list(json.loads(x).keys()),
filter(lambda x: x is not None, rd.get(ids, REDIS_PREFIX_CI_RELATION) or [])))
ids = [j for i in _tmp for j in i]
merge_ids.extend(ids)
if not self.orig_query or ("_type:" not in self.orig_query
and "type_id:" not in self.orig_query
and "ci_type:" not in self.orig_query):
type_ids = CITypeRelationManager.get_child_type_ids(ci.type_id, self.level)
self.orig_query = "_type:({0}),{1}".format(";".join(list(map(str, type_ids))), self.orig_query)
type_ids = []
for level in self.level:
for ci in cis:
type_ids.extend(CITypeRelationManager.get_child_type_ids(ci.type_id, level))
type_ids = list(set(type_ids))
if self.orig_query:
self.orig_query = "_type:({0}),{1}".format(";".join(list(map(str, type_ids))), self.orig_query)
else:
self.orig_query = "_type:({0})".format(";".join(list(map(str, type_ids))))
if not ids:
if not merge_ids:
# cis, counter, total, self.page, numfound, facet_
return [], {}, 0, self.page, 0, {}
@ -50,7 +71,7 @@ class Search(object):
page=self.page,
count=self.count,
sort=self.sort,
ci_ids=ids).search()
ci_ids=merge_ids).search()
else:
return SearchFromDB(self.orig_query,
fl=self.fl,
@ -58,18 +79,28 @@ class Search(object):
page=self.page,
count=self.count,
sort=self.sort,
ci_ids=ids).search()
ci_ids=merge_ids).search()
def statistics(self):
def statistics(self, type_ids):
ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
for l in range(0, self.level):
for l in range(0, int(self.level)):
if l == 0:
_tmp = list(map(json.loads, [i or '[]' for i in rd.get(ids, REDIS_PREFIX_CI_RELATION) or []]))
_tmp = list(map(lambda x: list(json.loads(x).keys()),
[i or '{}' for i in rd.get(ids, REDIS_PREFIX_CI_RELATION) or []]))
else:
for idx, i in enumerate(_tmp):
if i:
__tmp = list(map(json.loads, filter(lambda x: x is not None,
rd.get(i, REDIS_PREFIX_CI_RELATION) or [])))
if type_ids and l == self.level - 1:
__tmp = list(
map(lambda x: list({_id: 1 for _id, type_id in json.loads(x).items()
if type_id in type_ids}.keys()),
filter(lambda x: x is not None,
rd.get(i, REDIS_PREFIX_CI_RELATION) or [])))
else:
__tmp = list(map(lambda x: list(json.loads(x).keys()),
filter(lambda x: x is not None,
rd.get(i, REDIS_PREFIX_CI_RELATION) or [])))
_tmp[idx] = [j for i in __tmp for j in i]
else:
_tmp[idx] = []

2
api/lib/perm/acl/role.py
View File

@ -90,7 +90,7 @@ class RoleRelationCRUD(object):
@staticmethod
def delete2(parent_id, child_id):
existed = RoleRelation.get_by(parent_id=parent_id, child_id=child_id)
existed = RoleRelation.get_by(parent_id=parent_id, child_id=child_id, first=True, to_dict=False)
existed or abort(400, "RoleRelation < {0} -> {1} > does not exist".format(parent_id, child_id))
existed.soft_delete()

19
api/tasks/cmdb.py
View File

@ -14,6 +14,7 @@ from api.extensions import rd
from api.lib.cmdb.const import CMDB_QUEUE
from api.lib.cmdb.const import REDIS_PREFIX_CI
from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
from api.models.cmdb import CIRelation
@celery.task(name="cmdb.ci_cache", queue=CMDB_QUEUE)
@ -46,10 +47,13 @@ def ci_delete(ci_id):
@celery.task(name="cmdb.ci_relation_cache", queue=CMDB_QUEUE)
def ci_relation_cache(parent_id, child_id):
children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0]
children = json.loads(children) if children is not None else []
children.append(child_id)
children = json.loads(children) if children is not None else {}
rd.create_or_update({parent_id: json.dumps(list(set(children)))}, REDIS_PREFIX_CI_RELATION)
cr = CIRelation.get_by(first_ci_id=parent_id, second_ci_id=child_id, first=True, to_dict=False)
if child_id not in children:
children[child_id] = cr.second_ci.type_id
rd.create_or_update({parent_id: json.dumps(children)}, REDIS_PREFIX_CI_RELATION)
current_app.logger.info("ADD ci relation cache: {0} -> {1}".format(parent_id, child_id))
@ -57,10 +61,11 @@ def ci_relation_cache(parent_id, child_id):
@celery.task(name="cmdb.ci_relation_delete", queue=CMDB_QUEUE)
def ci_relation_delete(parent_id, child_id):
children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0]
children = json.loads(children) if children is not None else []
if child_id in children:
children.remove(child_id)
children = json.loads(children) if children is not None else {}
rd.create_or_update({parent_id: json.dumps(list(set(children)))}, REDIS_PREFIX_CI_RELATION)
if child_id in children:
children.pop(child_id)
rd.create_or_update({parent_id: json.dumps(children)}, REDIS_PREFIX_CI_RELATION)
current_app.logger.info("DELETE ci relation cache: {0} -> {1}".format(parent_id, child_id))

5
api/views/cmdb/ci_relation.py
View File

@ -34,7 +34,7 @@ class CIRelationSearchView(APIView):
count = get_page_size(request.values.get("count") or request.values.get("page_size"))
root_id = request.values.get('root_id')
level = request.values.get('level', 1)
level = list(map(int, handle_arg_list(request.values.get('level', '1'))))
query = request.values.get('q', "")
fl = handle_arg_list(request.values.get('fl', ""))
@ -64,11 +64,12 @@ class CIRelationStatisticsView(APIView):
def get(self):
root_ids = list(map(int, handle_arg_list(request.values.get('root_ids'))))
level = request.values.get('level', 1)
type_ids = set(map(int, handle_arg_list(request.values.get('type_ids', []))))
start = time.time()
s = Search(root_ids, level)
try:
result = s.statistics()
result = s.statistics(type_ids)
except SearchError as e:
return abort(400, str(e))
current_app.logger.debug("search time is :{0}".format(time.time() - start))

1
api/views/cmdb/preference.py
View File

@ -77,6 +77,7 @@ class PreferenceRelationApiView(APIView):
@role_required(RoleEnum.CONFIG)
@args_required("name")
@args_required("cr_ids")
def post(self):
name = request.values.get("name")
cr_ids = request.values.get("cr_ids")