enhance dashboard

cmdb-api/api/commands/click_cmdb.py

@@ -9,6 +9,7 @@ import time
 import click
 from flask import current_app
 from flask.cli import with_appcontext
+from flask_login import login_user
 
 import api.lib.cmdb.ci
 from api.extensions import db
@@ -24,6 +25,7 @@ from api.lib.cmdb.const import ValueTypeEnum
 from api.lib.exception import AbortException
 from api.lib.perm.acl.acl import ACLManager
 from api.lib.perm.acl.cache import AppCache
+from api.lib.perm.acl.cache import UserCache
 from api.lib.perm.acl.resource import ResourceCRUD
 from api.lib.perm.acl.resource import ResourceTypeCRUD
 from api.lib.perm.acl.role import RoleCRUD
@@ -207,6 +209,8 @@ def cmdb_counter():
     """
     from api.lib.cmdb.cache import CMDBCounterCache
 
+    current_app.test_request_context().push()
+    login_user(UserCache.get('worker'))
     while True:
         try:
             db.session.remove()

cmdb-api/api/lib/cmdb/cache.py

@@ -2,14 +2,11 @@
 
 from __future__ import unicode_literals
 
-import requests
 from flask import current_app
 
 from api.extensions import cache
-from api.extensions import db
 from api.lib.cmdb.custom_dashboard import CustomDashboardManager
 from api.models.cmdb import Attribute
-from api.models.cmdb import CI
 from api.models.cmdb import CIType
 from api.models.cmdb import CITypeAttribute
 from api.models.cmdb import RelationType
@@ -210,7 +207,6 @@ class CITypeAttributeCache(object):
 
     @classmethod
     def get(cls, type_id, attr_id):
-
         attr = cache.get(cls.PREFIX_ID.format(type_id, attr_id))
         attr = attr or cache.get(cls.PREFIX_ID.format(type_id, attr_id))
         attr = attr or CITypeAttribute.get_by(type_id=type_id, attr_id=attr_id, first=True, to_dict=False)
@@ -251,53 +247,72 @@ class CMDBCounterCache(object):
         result = {}
         for custom in customs:
             if custom['category'] == 0:
-                result[custom['id']] = cls.summary_counter(custom['type_id'])
+                res = cls.sum_counter(custom)
             elif custom['category'] == 1:
-                result[custom['id']] = cls.attribute_counter(custom['type_id'], custom['attr_id'])
+                res = cls.attribute_counter(custom)
-            elif custom['category'] == 2:
+            else:
-                result[custom['id']] = cls.relation_counter(custom['type_id'], custom['level'])
+                res = cls.relation_counter(custom.get('type_id'),
+                                           custom.get('level'),
+                                           custom.get('options', {}).get('filter', ''),
+                                           custom.get('options', {}).get('type_ids', ''))
+
+            if res:
+                result[custom['id']] = res
 
         cls.set(result)
 
         return result
 
     @classmethod
-    def update(cls, custom):
+    def update(cls, custom, flush=True):
         result = cache.get(cls.KEY) or {}
         if not result:
             result = cls.reset()
 
         if custom['category'] == 0:
-            result[custom['id']] = cls.summary_counter(custom['type_id'])
+            res = cls.sum_counter(custom)
         elif custom['category'] == 1:
-            result[custom['id']] = cls.attribute_counter(custom['type_id'], custom['attr_id'])
+            res = cls.attribute_counter(custom)
-        elif custom['category'] == 2:
+        else:
-            result[custom['id']] = cls.relation_counter(custom['type_id'], custom['level'])
+            res = cls.relation_counter(custom.get('type_id'),
+                                       custom.get('level'),
+                                       custom.get('options', {}).get('filter', ''),
+                                       custom.get('options', {}).get('type_ids', ''))
 
-        cls.set(result)
+        if res and flush:
+            result[custom['id']] = res
+            cls.set(result)
+
+        return res
 
     @staticmethod
-    def summary_counter(type_id):
+    def relation_counter(type_id, level, other_filer, type_ids):
-        return db.session.query(CI.id).filter(CI.deleted.is_(False)).filter(CI.type_id == type_id).count()
+        from api.lib.cmdb.search.ci_relation.search import Search as RelSearch
+        from api.lib.cmdb.search import SearchError
+        from api.lib.cmdb.search.ci import search
 
-    @staticmethod
+        query = "_type:{}".format(type_id)
-    def relation_counter(type_id, level):
+        s = search(query, count=1000000)
+        try:
+            type_names, _, _, _, _, _ = s.search()
+        except SearchError as e:
+            current_app.logger.error(e)
+            return
 
-        uri = current_app.config.get('CMDB_API')
-
-        type_names = requests.get("{}/ci/s?q=_type:{}&count=10000".format(uri, type_id)).json().get('result')
         type_id_names = [(str(i.get('_id')), i.get(i.get('unique'))) for i in type_names]
 
-        url = "{}/ci_relations/statistics?root_ids={}&level={}".format(
+        s = RelSearch([i[0] for i in type_id_names], level, other_filer or '')
-            uri, ','.join([i[0] for i in type_id_names]), level)
+        try:
-        stats = requests.get(url).json()
+            stats = s.statistics(type_ids)
+        except SearchError as e:
+            current_app.logger.error(e)
+            return
 
         id2name = dict(type_id_names)
         type_ids = set()
         for i in (stats.get('detail') or []):
             for j in stats['detail'][i]:
                 type_ids.add(j)
-
         for type_id in type_ids:
             _type = CITypeCache.get(type_id)
             id2name[type_id] = _type and _type.alias
@@ -317,9 +332,94 @@ class CMDBCounterCache(object):
         return result
 
     @staticmethod
-    def attribute_counter(type_id, attr_id):
+    def attribute_counter(custom):
-        uri = current_app.config.get('CMDB_API')
+        from api.lib.cmdb.search import SearchError
-        url = "{}/ci/s?q=_type:{}&fl={}&facet={}".format(uri, type_id, attr_id, attr_id)
+        from api.lib.cmdb.search.ci import search
-        res = requests.get(url).json()
+
-        if res.get('facet'):
+        custom.setdefault('options', {})
-            return dict([i[:2] for i in list(res.get('facet').values())[0]])
+        type_id = custom.get('type_id')
+        attr_id = custom.get('attr_id')
+        type_ids = custom['options'].get('type_ids') or (type_id and [type_id])
+        attr_ids = list(map(str, custom['options'].get('attr_ids') or (attr_id and [attr_id])))
+        other_filter = custom['options'].get('filter')
+        other_filter = "({})".format(other_filter) if other_filter else ''
+
+        if custom['options'].get('ret') == 'cis':
+            query = "_type:({}),{}".format(";".join(map(str, type_ids)), other_filter)
+            s = search(query, fl=attr_ids, ret_key='alias', count=100)
+            try:
+                cis, _, _, _, _, _ = s.search()
+            except SearchError as e:
+                current_app.logger.error(e)
+                return
+
+            return cis
+
+        result = dict()
+        # level = 1
+        query = "_type:({}),{}".format(";".join(map(str, type_ids)), other_filter)
+        s = search(query, fl=attr_ids, facet=[attr_ids[0]], count=1)
+        try:
+            _, _, _, _, _, facet = s.search()
+        except SearchError as e:
+            current_app.logger.error(e)
+            return
+        for i in (list(facet.values()) or [[]])[0]:
+            result[i[0]] = i[1]
+        if len(attr_ids) == 1:
+            return result
+
+        # level = 2
+        for v in result:
+            query = "_type:({}),{},{}:{}".format(";".join(map(str, type_ids)), other_filter, attr_ids[0], v)
+            s = search(query, fl=attr_ids, facet=[attr_ids[1]], count=1)
+            try:
+                _, _, _, _, _, facet = s.search()
+            except SearchError as e:
+                current_app.logger.error(e)
+                return
+            result[v] = dict()
+            for i in (list(facet.values()) or [[]])[0]:
+                result[v][i[0]] = i[1]
+
+        if len(attr_ids) == 2:
+            return result
+
+        # level = 3
+        for v1 in result:
+            if not isinstance(result[v1], dict):
+                continue
+            for v2 in result[v1]:
+                query = "_type:({}),{},{}:{},{}:{}".format(";".join(map(str, type_ids)), other_filter,
+                                                           attr_ids[0], v1, attr_ids[1], v2)
+                s = search(query, fl=attr_ids, facet=[attr_ids[2]], count=1)
+                try:
+                    _, _, _, _, _, facet = s.search()
+                except SearchError as e:
+                    current_app.logger.error(e)
+                    return
+                result[v1][v2] = dict()
+                for i in (list(facet.values()) or [[]])[0]:
+                    result[v1][v2][i[0]] = i[1]
+
+        return result
+
+    @staticmethod
+    def sum_counter(custom):
+        from api.lib.cmdb.search import SearchError
+        from api.lib.cmdb.search.ci import search
+
+        custom.setdefault('options', {})
+        type_id = custom.get('type_id')
+        type_ids = custom['options'].get('type_ids') or (type_id and [type_id])
+        other_filter = custom['options'].get('filter') or ''
+
+        query = "_type:({}),{}".format(";".join(map(str, type_ids)), other_filter)
+        s = search(query, count=1)
+        try:
+            _, _, _, _, numfound, _ = s.search()
+        except SearchError as e:
+            current_app.logger.error(e)
+            return
+
+        return numfound

cmdb-api/api/lib/cmdb/custom_dashboard.py

@@ -14,6 +14,14 @@ class CustomDashboardManager(object):
     def get():
         return sorted(CustomDashboard.get_by(to_dict=True), key=lambda x: (x["category"], x['order']))
 
+    @staticmethod
+    def preview(**kwargs):
+        from api.lib.cmdb.cache import CMDBCounterCache
+
+        res = CMDBCounterCache.update(kwargs, flush=False)
+
+        return res
+
     @staticmethod
     def add(**kwargs):
         from api.lib.cmdb.cache import CMDBCounterCache
@@ -23,9 +31,9 @@ class CustomDashboardManager(object):
 
         new = CustomDashboard.create(**kwargs)
 
-        CMDBCounterCache.update(new.to_dict())
+        res = CMDBCounterCache.update(new.to_dict())
 
-        return new
+        return new, res
 
     @staticmethod
     def update(_id, **kwargs):
@@ -35,9 +43,9 @@ class CustomDashboardManager(object):
 
         new = existed.update(**kwargs)
 
-        CMDBCounterCache.update(new.to_dict())
+        res = CMDBCounterCache.update(new.to_dict())
 
-        return new
+        return new, res
 
     @staticmethod
     def batch_update(id2options):

cmdb-api/api/lib/cmdb/query_sql.py

@@ -42,7 +42,7 @@ FACET_QUERY1 = """
 
 FACET_QUERY = """
     SELECT {0}.value,
-           count({0}.ci_id)
+           count(distinct({0}.ci_id))
     FROM {0}
     INNER JOIN ({1}) AS F ON F.ci_id={0}.ci_id
     WHERE {0}.attr_id={2:d}

cmdb-api/api/views/cmdb/ci_type.py

@@ -154,9 +154,15 @@ class EnableCITypeView(APIView):
 
 
 class CITypeAttributeView(APIView):
-    url_prefix = ("/ci_types/<int:type_id>/attributes", "/ci_types/<string:type_name>/attributes")
+    url_prefix = ("/ci_types/<int:type_id>/attributes", "/ci_types/<string:type_name>/attributes",
+                  "/ci_types/common_attributes")
 
     def get(self, type_id=None, type_name=None):
+        if request.path.endswith("/common_attributes"):
+            type_ids = handle_arg_list(request.values.get('type_ids'))
+
+            return self.jsonify(attributes=CITypeAttributeManager.get_common_attributes(type_ids))
+
         t = CITypeCache.get(type_id) or CITypeCache.get(type_name) or abort(404, ErrFormat.ci_type_not_found)
         type_id = t.id
         unique_id = t.unique_id
@@ -500,3 +506,4 @@ class CITypeFilterPermissionView(APIView):
     @auth_with_app_token
     def get(self, type_id):
         return self.jsonify(CIFilterPermsCRUD().get(type_id))
+

cmdb-api/api/views/cmdb/ci_type_relation.py

@@ -19,9 +19,14 @@ from api.resource import APIView
 
 
 class GetChildrenView(APIView):
-    url_prefix = "/ci_type_relations/<int:parent_id>/children"
+    url_prefix = ("/ci_type_relations/<int:parent_id>/children",
+                  "/ci_type_relations/<int:parent_id>/recursive_level2children",
+                  )
 
     def get(self, parent_id):
+        if request.url.endswith("recursive_level2children"):
+            return self.jsonify(CITypeRelationManager.recursive_level2children(parent_id))
+
         return self.jsonify(children=CITypeRelationManager.get_children(parent_id))
 
 

cmdb-api/api/views/cmdb/custom_dashboard.py

@@ -13,7 +13,8 @@ from api.resource import APIView
 
 
 class CustomDashboardApiView(APIView):
-    url_prefix = ("/custom_dashboard", "/custom_dashboard/<int:_id>", "/custom_dashboard/batch")
+    url_prefix = ("/custom_dashboard", "/custom_dashboard/<int:_id>", "/custom_dashboard/batch",
+                  "/custom_dashboard/preview")
 
     def get(self):
         return self.jsonify(CustomDashboardManager.get())
@@ -21,17 +22,26 @@ class CustomDashboardApiView(APIView):
     @role_required(RoleEnum.CONFIG)
     @args_validate(CustomDashboardManager.cls)
     def post(self):
-        cm = CustomDashboardManager.add(**request.values)
+        if request.url.endswith("/preview"):
+            return self.jsonify(counter=CustomDashboardManager.preview(**request.values))
 
-        return self.jsonify(cm.to_dict())
+        cm, counter = CustomDashboardManager.add(**request.values)
+
+        res = cm.to_dict()
+        res.update(counter=counter)
+
+        return self.jsonify(res)
 
     @role_required(RoleEnum.CONFIG)
     @args_validate(CustomDashboardManager.cls)
     def put(self, _id=None):
         if _id is not None:
-            cm = CustomDashboardManager.update(_id, **request.values)
+            cm, counter = CustomDashboardManager.update(_id, **request.values)
 
-            return self.jsonify(cm.to_dict())
+            res = cm.to_dict()
+            res.update(counter=counter)
+
+            return self.jsonify(res)
 
         CustomDashboardManager.batch_update(request.values.get("id2options"))