From 25ad2192fdf206ecc5782ef689a4c7693414a62e Mon Sep 17 00:00:00 2001 From: pycook Date: Fri, 15 Sep 2023 15:26:20 +0800 Subject: [PATCH] enhance dashboard --- cmdb-api/api/commands/click_cmdb.py | 4 + cmdb-api/api/lib/cmdb/cache.py | 162 ++++++++++++++++---- cmdb-api/api/lib/cmdb/custom_dashboard.py | 16 +- cmdb-api/api/lib/cmdb/query_sql.py | 2 +- cmdb-api/api/views/cmdb/ci_type.py | 11 +- cmdb-api/api/views/cmdb/ci_type_relation.py | 7 +- cmdb-api/api/views/cmdb/custom_dashboard.py | 20 ++- 7 files changed, 178 insertions(+), 44 deletions(-) diff --git a/cmdb-api/api/commands/click_cmdb.py b/cmdb-api/api/commands/click_cmdb.py index 1362806..6d41d98 100644 --- a/cmdb-api/api/commands/click_cmdb.py +++ b/cmdb-api/api/commands/click_cmdb.py @@ -9,6 +9,7 @@ import time import click from flask import current_app from flask.cli import with_appcontext +from flask_login import login_user import api.lib.cmdb.ci from api.extensions import db @@ -24,6 +25,7 @@ from api.lib.cmdb.const import ValueTypeEnum from api.lib.exception import AbortException from api.lib.perm.acl.acl import ACLManager from api.lib.perm.acl.cache import AppCache +from api.lib.perm.acl.cache import UserCache from api.lib.perm.acl.resource import ResourceCRUD from api.lib.perm.acl.resource import ResourceTypeCRUD from api.lib.perm.acl.role import RoleCRUD @@ -207,6 +209,8 @@ def cmdb_counter(): """ from api.lib.cmdb.cache import CMDBCounterCache + current_app.test_request_context().push() + login_user(UserCache.get('worker')) while True: try: db.session.remove() diff --git a/cmdb-api/api/lib/cmdb/cache.py b/cmdb-api/api/lib/cmdb/cache.py index d166d42..bbc1c22 100644 --- a/cmdb-api/api/lib/cmdb/cache.py +++ b/cmdb-api/api/lib/cmdb/cache.py @@ -2,14 +2,11 @@ from __future__ import unicode_literals -import requests from flask import current_app from api.extensions import cache -from api.extensions import db from api.lib.cmdb.custom_dashboard import CustomDashboardManager from api.models.cmdb import Attribute -from api.models.cmdb import CI from api.models.cmdb import CIType from api.models.cmdb import CITypeAttribute from api.models.cmdb import RelationType @@ -210,7 +207,6 @@ class CITypeAttributeCache(object): @classmethod def get(cls, type_id, attr_id): - attr = cache.get(cls.PREFIX_ID.format(type_id, attr_id)) attr = attr or cache.get(cls.PREFIX_ID.format(type_id, attr_id)) attr = attr or CITypeAttribute.get_by(type_id=type_id, attr_id=attr_id, first=True, to_dict=False) @@ -251,53 +247,72 @@ class CMDBCounterCache(object): result = {} for custom in customs: if custom['category'] == 0: - result[custom['id']] = cls.summary_counter(custom['type_id']) + res = cls.sum_counter(custom) elif custom['category'] == 1: - result[custom['id']] = cls.attribute_counter(custom['type_id'], custom['attr_id']) - elif custom['category'] == 2: - result[custom['id']] = cls.relation_counter(custom['type_id'], custom['level']) + res = cls.attribute_counter(custom) + else: + res = cls.relation_counter(custom.get('type_id'), + custom.get('level'), + custom.get('options', {}).get('filter', ''), + custom.get('options', {}).get('type_ids', '')) + + if res: + result[custom['id']] = res cls.set(result) return result @classmethod - def update(cls, custom): + def update(cls, custom, flush=True): result = cache.get(cls.KEY) or {} if not result: result = cls.reset() if custom['category'] == 0: - result[custom['id']] = cls.summary_counter(custom['type_id']) + res = cls.sum_counter(custom) elif custom['category'] == 1: - result[custom['id']] = cls.attribute_counter(custom['type_id'], custom['attr_id']) - elif custom['category'] == 2: - result[custom['id']] = cls.relation_counter(custom['type_id'], custom['level']) + res = cls.attribute_counter(custom) + else: + res = cls.relation_counter(custom.get('type_id'), + custom.get('level'), + custom.get('options', {}).get('filter', ''), + custom.get('options', {}).get('type_ids', '')) - cls.set(result) + if res and flush: + result[custom['id']] = res + cls.set(result) + + return res @staticmethod - def summary_counter(type_id): - return db.session.query(CI.id).filter(CI.deleted.is_(False)).filter(CI.type_id == type_id).count() + def relation_counter(type_id, level, other_filer, type_ids): + from api.lib.cmdb.search.ci_relation.search import Search as RelSearch + from api.lib.cmdb.search import SearchError + from api.lib.cmdb.search.ci import search - @staticmethod - def relation_counter(type_id, level): + query = "_type:{}".format(type_id) + s = search(query, count=1000000) + try: + type_names, _, _, _, _, _ = s.search() + except SearchError as e: + current_app.logger.error(e) + return - uri = current_app.config.get('CMDB_API') - - type_names = requests.get("{}/ci/s?q=_type:{}&count=10000".format(uri, type_id)).json().get('result') type_id_names = [(str(i.get('_id')), i.get(i.get('unique'))) for i in type_names] - url = "{}/ci_relations/statistics?root_ids={}&level={}".format( - uri, ','.join([i[0] for i in type_id_names]), level) - stats = requests.get(url).json() + s = RelSearch([i[0] for i in type_id_names], level, other_filer or '') + try: + stats = s.statistics(type_ids) + except SearchError as e: + current_app.logger.error(e) + return id2name = dict(type_id_names) type_ids = set() for i in (stats.get('detail') or []): for j in stats['detail'][i]: type_ids.add(j) - for type_id in type_ids: _type = CITypeCache.get(type_id) id2name[type_id] = _type and _type.alias @@ -317,9 +332,94 @@ class CMDBCounterCache(object): return result @staticmethod - def attribute_counter(type_id, attr_id): - uri = current_app.config.get('CMDB_API') - url = "{}/ci/s?q=_type:{}&fl={}&facet={}".format(uri, type_id, attr_id, attr_id) - res = requests.get(url).json() - if res.get('facet'): - return dict([i[:2] for i in list(res.get('facet').values())[0]]) + def attribute_counter(custom): + from api.lib.cmdb.search import SearchError + from api.lib.cmdb.search.ci import search + + custom.setdefault('options', {}) + type_id = custom.get('type_id') + attr_id = custom.get('attr_id') + type_ids = custom['options'].get('type_ids') or (type_id and [type_id]) + attr_ids = list(map(str, custom['options'].get('attr_ids') or (attr_id and [attr_id]))) + other_filter = custom['options'].get('filter') + other_filter = "({})".format(other_filter) if other_filter else '' + + if custom['options'].get('ret') == 'cis': + query = "_type:({}),{}".format(";".join(map(str, type_ids)), other_filter) + s = search(query, fl=attr_ids, ret_key='alias', count=100) + try: + cis, _, _, _, _, _ = s.search() + except SearchError as e: + current_app.logger.error(e) + return + + return cis + + result = dict() + # level = 1 + query = "_type:({}),{}".format(";".join(map(str, type_ids)), other_filter) + s = search(query, fl=attr_ids, facet=[attr_ids[0]], count=1) + try: + _, _, _, _, _, facet = s.search() + except SearchError as e: + current_app.logger.error(e) + return + for i in (list(facet.values()) or [[]])[0]: + result[i[0]] = i[1] + if len(attr_ids) == 1: + return result + + # level = 2 + for v in result: + query = "_type:({}),{},{}:{}".format(";".join(map(str, type_ids)), other_filter, attr_ids[0], v) + s = search(query, fl=attr_ids, facet=[attr_ids[1]], count=1) + try: + _, _, _, _, _, facet = s.search() + except SearchError as e: + current_app.logger.error(e) + return + result[v] = dict() + for i in (list(facet.values()) or [[]])[0]: + result[v][i[0]] = i[1] + + if len(attr_ids) == 2: + return result + + # level = 3 + for v1 in result: + if not isinstance(result[v1], dict): + continue + for v2 in result[v1]: + query = "_type:({}),{},{}:{},{}:{}".format(";".join(map(str, type_ids)), other_filter, + attr_ids[0], v1, attr_ids[1], v2) + s = search(query, fl=attr_ids, facet=[attr_ids[2]], count=1) + try: + _, _, _, _, _, facet = s.search() + except SearchError as e: + current_app.logger.error(e) + return + result[v1][v2] = dict() + for i in (list(facet.values()) or [[]])[0]: + result[v1][v2][i[0]] = i[1] + + return result + + @staticmethod + def sum_counter(custom): + from api.lib.cmdb.search import SearchError + from api.lib.cmdb.search.ci import search + + custom.setdefault('options', {}) + type_id = custom.get('type_id') + type_ids = custom['options'].get('type_ids') or (type_id and [type_id]) + other_filter = custom['options'].get('filter') or '' + + query = "_type:({}),{}".format(";".join(map(str, type_ids)), other_filter) + s = search(query, count=1) + try: + _, _, _, _, numfound, _ = s.search() + except SearchError as e: + current_app.logger.error(e) + return + + return numfound diff --git a/cmdb-api/api/lib/cmdb/custom_dashboard.py b/cmdb-api/api/lib/cmdb/custom_dashboard.py index 8153eec..133769a 100644 --- a/cmdb-api/api/lib/cmdb/custom_dashboard.py +++ b/cmdb-api/api/lib/cmdb/custom_dashboard.py @@ -14,6 +14,14 @@ class CustomDashboardManager(object): def get(): return sorted(CustomDashboard.get_by(to_dict=True), key=lambda x: (x["category"], x['order'])) + @staticmethod + def preview(**kwargs): + from api.lib.cmdb.cache import CMDBCounterCache + + res = CMDBCounterCache.update(kwargs, flush=False) + + return res + @staticmethod def add(**kwargs): from api.lib.cmdb.cache import CMDBCounterCache @@ -23,9 +31,9 @@ class CustomDashboardManager(object): new = CustomDashboard.create(**kwargs) - CMDBCounterCache.update(new.to_dict()) + res = CMDBCounterCache.update(new.to_dict()) - return new + return new, res @staticmethod def update(_id, **kwargs): @@ -35,9 +43,9 @@ class CustomDashboardManager(object): new = existed.update(**kwargs) - CMDBCounterCache.update(new.to_dict()) + res = CMDBCounterCache.update(new.to_dict()) - return new + return new, res @staticmethod def batch_update(id2options): diff --git a/cmdb-api/api/lib/cmdb/query_sql.py b/cmdb-api/api/lib/cmdb/query_sql.py index c84fd64..f5c598b 100644 --- a/cmdb-api/api/lib/cmdb/query_sql.py +++ b/cmdb-api/api/lib/cmdb/query_sql.py @@ -42,7 +42,7 @@ FACET_QUERY1 = """ FACET_QUERY = """ SELECT {0}.value, - count({0}.ci_id) + count(distinct({0}.ci_id)) FROM {0} INNER JOIN ({1}) AS F ON F.ci_id={0}.ci_id WHERE {0}.attr_id={2:d} diff --git a/cmdb-api/api/views/cmdb/ci_type.py b/cmdb-api/api/views/cmdb/ci_type.py index ba2686f..8bb8f8b 100644 --- a/cmdb-api/api/views/cmdb/ci_type.py +++ b/cmdb-api/api/views/cmdb/ci_type.py @@ -1,4 +1,4 @@ -# -*- coding:utf-8 -*- +# -*- coding:utf-8 -*- import json @@ -154,9 +154,15 @@ class EnableCITypeView(APIView): class CITypeAttributeView(APIView): - url_prefix = ("/ci_types//attributes", "/ci_types//attributes") + url_prefix = ("/ci_types//attributes", "/ci_types//attributes", + "/ci_types/common_attributes") def get(self, type_id=None, type_name=None): + if request.path.endswith("/common_attributes"): + type_ids = handle_arg_list(request.values.get('type_ids')) + + return self.jsonify(attributes=CITypeAttributeManager.get_common_attributes(type_ids)) + t = CITypeCache.get(type_id) or CITypeCache.get(type_name) or abort(404, ErrFormat.ci_type_not_found) type_id = t.id unique_id = t.unique_id @@ -500,3 +506,4 @@ class CITypeFilterPermissionView(APIView): @auth_with_app_token def get(self, type_id): return self.jsonify(CIFilterPermsCRUD().get(type_id)) + diff --git a/cmdb-api/api/views/cmdb/ci_type_relation.py b/cmdb-api/api/views/cmdb/ci_type_relation.py index 55d5d97..20ce323 100644 --- a/cmdb-api/api/views/cmdb/ci_type_relation.py +++ b/cmdb-api/api/views/cmdb/ci_type_relation.py @@ -19,9 +19,14 @@ from api.resource import APIView class GetChildrenView(APIView): - url_prefix = "/ci_type_relations//children" + url_prefix = ("/ci_type_relations//children", + "/ci_type_relations//recursive_level2children", + ) def get(self, parent_id): + if request.url.endswith("recursive_level2children"): + return self.jsonify(CITypeRelationManager.recursive_level2children(parent_id)) + return self.jsonify(children=CITypeRelationManager.get_children(parent_id)) diff --git a/cmdb-api/api/views/cmdb/custom_dashboard.py b/cmdb-api/api/views/cmdb/custom_dashboard.py index 4991ceb..1c20ff9 100644 --- a/cmdb-api/api/views/cmdb/custom_dashboard.py +++ b/cmdb-api/api/views/cmdb/custom_dashboard.py @@ -13,7 +13,8 @@ from api.resource import APIView class CustomDashboardApiView(APIView): - url_prefix = ("/custom_dashboard", "/custom_dashboard/", "/custom_dashboard/batch") + url_prefix = ("/custom_dashboard", "/custom_dashboard/", "/custom_dashboard/batch", + "/custom_dashboard/preview") def get(self): return self.jsonify(CustomDashboardManager.get()) @@ -21,17 +22,26 @@ class CustomDashboardApiView(APIView): @role_required(RoleEnum.CONFIG) @args_validate(CustomDashboardManager.cls) def post(self): - cm = CustomDashboardManager.add(**request.values) + if request.url.endswith("/preview"): + return self.jsonify(counter=CustomDashboardManager.preview(**request.values)) - return self.jsonify(cm.to_dict()) + cm, counter = CustomDashboardManager.add(**request.values) + + res = cm.to_dict() + res.update(counter=counter) + + return self.jsonify(res) @role_required(RoleEnum.CONFIG) @args_validate(CustomDashboardManager.cls) def put(self, _id=None): if _id is not None: - cm = CustomDashboardManager.update(_id, **request.values) + cm, counter = CustomDashboardManager.update(_id, **request.values) - return self.jsonify(cm.to_dict()) + res = cm.to_dict() + res.update(counter=counter) + + return self.jsonify(res) CustomDashboardManager.batch_update(request.values.get("id2options"))