7x31
/
cmdb
mirror of https://github.com/veops/cmdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enhance dashboard

This commit is contained in:
pycook 2023-09-15 15:26:20 +08:00
parent 881b8cc1fa
commit 25ad2192fd
7 changed files with 178 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmdb-api/api/commands/click_cmdb.py
View File

@ -9,6 +9,7 @@ import time
import click
from flask import current_app
from flask.cli import with_appcontext
from flask_login import login_user
import api.lib.cmdb.ci
from api.extensions import db
@ -24,6 +25,7 @@ from api.lib.cmdb.const import ValueTypeEnum
from api.lib.exception import AbortException
from api.lib.perm.acl.acl import ACLManager
from api.lib.perm.acl.cache import AppCache
from api.lib.perm.acl.cache import UserCache
from api.lib.perm.acl.resource import ResourceCRUD
from api.lib.perm.acl.resource import ResourceTypeCRUD
from api.lib.perm.acl.role import RoleCRUD
@ -207,6 +209,8 @@ def cmdb_counter():
"""
from api.lib.cmdb.cache import CMDBCounterCache
current_app.test_request_context().push()
login_user(UserCache.get('worker'))
while True:
try:
db.session.remove()

162
cmdb-api/api/lib/cmdb/cache.py
View File

@ -2,14 +2,11 @@
from __future__ import unicode_literals
import requests
from flask import current_app
from api.extensions import cache
from api.extensions import db
from api.lib.cmdb.custom_dashboard import CustomDashboardManager
from api.models.cmdb import Attribute
from api.models.cmdb import CI
from api.models.cmdb import CIType
from api.models.cmdb import CITypeAttribute
from api.models.cmdb import RelationType
@ -210,7 +207,6 @@ class CITypeAttributeCache(object):
@classmethod
def get(cls, type_id, attr_id):
attr = cache.get(cls.PREFIX_ID.format(type_id, attr_id))
attr = attr or cache.get(cls.PREFIX_ID.format(type_id, attr_id))
attr = attr or CITypeAttribute.get_by(type_id=type_id, attr_id=attr_id, first=True, to_dict=False)
@ -251,53 +247,72 @@ class CMDBCounterCache(object):
result = {}
for custom in customs:
if custom['category'] == 0:
result[custom['id']] = cls.summary_counter(custom['type_id'])
res = cls.sum_counter(custom)
elif custom['category'] == 1:
result[custom['id']] = cls.attribute_counter(custom['type_id'], custom['attr_id'])
elif custom['category'] == 2:
result[custom['id']] = cls.relation_counter(custom['type_id'], custom['level'])
res = cls.attribute_counter(custom)
else:
res = cls.relation_counter(custom.get('type_id'),
custom.get('level'),
custom.get('options', {}).get('filter', ''),
custom.get('options', {}).get('type_ids', ''))
if res:
result[custom['id']] = res
cls.set(result)
return result
@classmethod
def update(cls, custom):
def update(cls, custom, flush=True):
result = cache.get(cls.KEY) or {}
if not result:
result = cls.reset()
if custom['category'] == 0:
result[custom['id']] = cls.summary_counter(custom['type_id'])
res = cls.sum_counter(custom)
elif custom['category'] == 1:
result[custom['id']] = cls.attribute_counter(custom['type_id'], custom['attr_id'])
elif custom['category'] == 2:
result[custom['id']] = cls.relation_counter(custom['type_id'], custom['level'])
res = cls.attribute_counter(custom)
else:
res = cls.relation_counter(custom.get('type_id'),
custom.get('level'),
custom.get('options', {}).get('filter', ''),
custom.get('options', {}).get('type_ids', ''))
cls.set(result)
if res and flush:
result[custom['id']] = res
cls.set(result)
return res
@staticmethod
def summary_counter(type_id):
return db.session.query(CI.id).filter(CI.deleted.is_(False)).filter(CI.type_id == type_id).count()
def relation_counter(type_id, level, other_filer, type_ids):
from api.lib.cmdb.search.ci_relation.search import Search as RelSearch
from api.lib.cmdb.search import SearchError
from api.lib.cmdb.search.ci import search
@staticmethod
def relation_counter(type_id, level):
query = "_type:{}".format(type_id)
s = search(query, count=1000000)
try:
type_names, _, _, _, _, _ = s.search()
except SearchError as e:
current_app.logger.error(e)
return
uri = current_app.config.get('CMDB_API')
type_names = requests.get("{}/ci/s?q=_type:{}&count=10000".format(uri, type_id)).json().get('result')
type_id_names = [(str(i.get('_id')), i.get(i.get('unique'))) for i in type_names]
url = "{}/ci_relations/statistics?root_ids={}&level={}".format(
uri, ','.join([i[0] for i in type_id_names]), level)
stats = requests.get(url).json()
s = RelSearch([i[0] for i in type_id_names], level, other_filer or '')
try:
stats = s.statistics(type_ids)
except SearchError as e:
current_app.logger.error(e)
return
id2name = dict(type_id_names)
type_ids = set()
for i in (stats.get('detail') or []):
for j in stats['detail'][i]:
type_ids.add(j)
for type_id in type_ids:
_type = CITypeCache.get(type_id)
id2name[type_id] = _type and _type.alias
@ -317,9 +332,94 @@ class CMDBCounterCache(object):
return result
@staticmethod
def attribute_counter(type_id, attr_id):
uri = current_app.config.get('CMDB_API')
url = "{}/ci/s?q=_type:{}&fl={}&facet={}".format(uri, type_id, attr_id, attr_id)
res = requests.get(url).json()
if res.get('facet'):
return dict([i[:2] for i in list(res.get('facet').values())[0]])
def attribute_counter(custom):
from api.lib.cmdb.search import SearchError
from api.lib.cmdb.search.ci import search
custom.setdefault('options', {})
type_id = custom.get('type_id')
attr_id = custom.get('attr_id')
type_ids = custom['options'].get('type_ids') or (type_id and [type_id])
attr_ids = list(map(str, custom['options'].get('attr_ids') or (attr_id and [attr_id])))
other_filter = custom['options'].get('filter')
other_filter = "({})".format(other_filter) if other_filter else ''
if custom['options'].get('ret') == 'cis':
query = "_type:({}),{}".format(";".join(map(str, type_ids)), other_filter)
s = search(query, fl=attr_ids, ret_key='alias', count=100)
try:
cis, _, _, _, _, _ = s.search()
except SearchError as e:
current_app.logger.error(e)
return
return cis
result = dict()
# level = 1
query = "_type:({}),{}".format(";".join(map(str, type_ids)), other_filter)
s = search(query, fl=attr_ids, facet=[attr_ids[0]], count=1)
try:
_, _, _, _, _, facet = s.search()
except SearchError as e:
current_app.logger.error(e)
return
for i in (list(facet.values()) or [[]])[0]:
result[i[0]] = i[1]
if len(attr_ids) == 1:
return result
# level = 2
for v in result:
query = "_type:({}),{},{}:{}".format(";".join(map(str, type_ids)), other_filter, attr_ids[0], v)
s = search(query, fl=attr_ids, facet=[attr_ids[1]], count=1)
try:
_, _, _, _, _, facet = s.search()
except SearchError as e:
current_app.logger.error(e)
return
result[v] = dict()
for i in (list(facet.values()) or [[]])[0]:
result[v][i[0]] = i[1]
if len(attr_ids) == 2:
return result
# level = 3
for v1 in result:
if not isinstance(result[v1], dict):
continue
for v2 in result[v1]:
query = "_type:({}),{},{}:{},{}:{}".format(";".join(map(str, type_ids)), other_filter,
attr_ids[0], v1, attr_ids[1], v2)
s = search(query, fl=attr_ids, facet=[attr_ids[2]], count=1)
try:
_, _, _, _, _, facet = s.search()
except SearchError as e:
current_app.logger.error(e)
return
result[v1][v2] = dict()
for i in (list(facet.values()) or [[]])[0]:
result[v1][v2][i[0]] = i[1]
return result
@staticmethod
def sum_counter(custom):
from api.lib.cmdb.search import SearchError
from api.lib.cmdb.search.ci import search
custom.setdefault('options', {})
type_id = custom.get('type_id')
type_ids = custom['options'].get('type_ids') or (type_id and [type_id])
other_filter = custom['options'].get('filter') or ''
query = "_type:({}),{}".format(";".join(map(str, type_ids)), other_filter)
s = search(query, count=1)
try:
_, _, _, _, numfound, _ = s.search()
except SearchError as e:
current_app.logger.error(e)
return
return numfound

16
cmdb-api/api/lib/cmdb/custom_dashboard.py
View File

@ -14,6 +14,14 @@ class CustomDashboardManager(object):
def get():
return sorted(CustomDashboard.get_by(to_dict=True), key=lambda x: (x["category"], x['order']))
@staticmethod
def preview(**kwargs):
from api.lib.cmdb.cache import CMDBCounterCache
res = CMDBCounterCache.update(kwargs, flush=False)
return res
@staticmethod
def add(**kwargs):
from api.lib.cmdb.cache import CMDBCounterCache
@ -23,9 +31,9 @@ class CustomDashboardManager(object):
new = CustomDashboard.create(**kwargs)
CMDBCounterCache.update(new.to_dict())
res = CMDBCounterCache.update(new.to_dict())
return new
return new, res
@staticmethod
def update(_id, **kwargs):
@ -35,9 +43,9 @@ class CustomDashboardManager(object):
new = existed.update(**kwargs)
CMDBCounterCache.update(new.to_dict())
res = CMDBCounterCache.update(new.to_dict())
return new
return new, res
@staticmethod
def batch_update(id2options):

2
cmdb-api/api/lib/cmdb/query_sql.py
View File

@ -42,7 +42,7 @@ FACET_QUERY1 = """
FACET_QUERY = """
SELECT {0}.value,
count({0}.ci_id)
count(distinct({0}.ci_id))
FROM {0}
INNER JOIN ({1}) AS F ON F.ci_id={0}.ci_id
WHERE {0}.attr_id={2:d}

11
cmdb-api/api/views/cmdb/ci_type.py
View File

@ -1,4 +1,4 @@
# -*- coding:utf-8 -*-
# -*- coding:utf-8 -*-
import json
@ -154,9 +154,15 @@ class EnableCITypeView(APIView):
class CITypeAttributeView(APIView):
url_prefix = ("/ci_types/<int:type_id>/attributes", "/ci_types/<string:type_name>/attributes")
url_prefix = ("/ci_types/<int:type_id>/attributes", "/ci_types/<string:type_name>/attributes",
"/ci_types/common_attributes")
def get(self, type_id=None, type_name=None):
if request.path.endswith("/common_attributes"):
type_ids = handle_arg_list(request.values.get('type_ids'))
return self.jsonify(attributes=CITypeAttributeManager.get_common_attributes(type_ids))
t = CITypeCache.get(type_id) or CITypeCache.get(type_name) or abort(404, ErrFormat.ci_type_not_found)
type_id = t.id
unique_id = t.unique_id
@ -500,3 +506,4 @@ class CITypeFilterPermissionView(APIView):
@auth_with_app_token
def get(self, type_id):
return self.jsonify(CIFilterPermsCRUD().get(type_id))

7
cmdb-api/api/views/cmdb/ci_type_relation.py
View File

@ -19,9 +19,14 @@ from api.resource import APIView
class GetChildrenView(APIView):
url_prefix = "/ci_type_relations/<int:parent_id>/children"
url_prefix = ("/ci_type_relations/<int:parent_id>/children",
"/ci_type_relations/<int:parent_id>/recursive_level2children",
)
def get(self, parent_id):
if request.url.endswith("recursive_level2children"):
return self.jsonify(CITypeRelationManager.recursive_level2children(parent_id))
return self.jsonify(children=CITypeRelationManager.get_children(parent_id))

20
cmdb-api/api/views/cmdb/custom_dashboard.py
View File

@ -13,7 +13,8 @@ from api.resource import APIView
class CustomDashboardApiView(APIView):
url_prefix = ("/custom_dashboard", "/custom_dashboard/<int:_id>", "/custom_dashboard/batch")
url_prefix = ("/custom_dashboard", "/custom_dashboard/<int:_id>", "/custom_dashboard/batch",
"/custom_dashboard/preview")
def get(self):
return self.jsonify(CustomDashboardManager.get())
@ -21,17 +22,26 @@ class CustomDashboardApiView(APIView):
@role_required(RoleEnum.CONFIG)
@args_validate(CustomDashboardManager.cls)
def post(self):
cm = CustomDashboardManager.add(**request.values)
if request.url.endswith("/preview"):
return self.jsonify(counter=CustomDashboardManager.preview(**request.values))
return self.jsonify(cm.to_dict())
cm, counter = CustomDashboardManager.add(**request.values)
res = cm.to_dict()
res.update(counter=counter)
return self.jsonify(res)
@role_required(RoleEnum.CONFIG)
@args_validate(CustomDashboardManager.cls)
def put(self, _id=None):
if _id is not None:
cm = CustomDashboardManager.update(_id, **request.values)
cm, counter = CustomDashboardManager.update(_id, **request.values)
return self.jsonify(cm.to_dict())
res = cm.to_dict()
res.update(counter=counter)
return self.jsonify(res)
CustomDashboardManager.batch_update(request.values.get("id2options"))