feat(api): topology view (#523)

* feat(api): topology views crud

* feat(api): topology view

* feat(api): topology view api done

* feat(api): topology view is done

cmdb-api/api/lib/cmdb/ci.py

@@ -1090,6 +1090,18 @@ class CIRelationManager(object):
 
         return ci_ids, level2ids
 
+    @classmethod
+    def get_parent_ids(cls, ci_ids):
+        cis = db.session.query(CIRelation.first_ci_id, CIRelation.second_ci_id, CI.type_id).join(
+            CI, CI.id == CIRelation.first_ci_id).filter(
+            CIRelation.second_ci_id.in_(ci_ids)).filter(CIRelation.deleted.is_(False))
+
+        result = {}
+        for ci in cis:
+            result.setdefault(ci.second_ci_id, []).append((ci.first_ci_id, ci.type_id))
+
+        return result
+
     @staticmethod
     def _check_constraint(first_ci_id, first_type_id, second_ci_id, second_type_id, type_relation):
         db.session.commit()

cmdb-api/api/lib/cmdb/ci_type.py

@@ -49,6 +49,7 @@ from api.models.cmdb import CITypeTrigger
 from api.models.cmdb import CITypeUniqueConstraint
 from api.models.cmdb import CustomDashboard
 from api.models.cmdb import PreferenceCITypeOrder
+from api.models.cmdb import TopologyView
 from api.models.cmdb import PreferenceRelationView
 from api.models.cmdb import PreferenceSearchOption
 from api.models.cmdb import PreferenceShowAttributes
@@ -263,6 +264,9 @@ class CITypeManager(object):
         except Exception:
             pass
 
+        for item in TopologyView.get_by(central_node_type=type_id, to_dict=False):
+            item.delete(commit=False)
+
         db.session.commit()
 
         ci_type.soft_delete()
@@ -825,6 +829,70 @@ class CITypeRelationManager(object):
 
         return [cls._wrap_relation_type_dict(parent.parent_id, parent) for parent in parents]
 
+    @staticmethod
+    def get_relations_by_type_id(type_id):
+        nodes, edges = [], []
+        node_ids, edge_tuples = set(), set()
+        ci_type = CITypeCache.get(type_id)
+        if ci_type is None:
+            return nodes, edges
+
+        nodes.append(ci_type.to_dict())
+        node_ids.add(ci_type.id)
+
+        def _find(_id, lv):
+            lv += 1
+            for i in CITypeRelation.get_by(parent_id=_id, to_dict=False):
+                if i.child_id not in node_ids:
+                    node_ids.add(i.child_id)
+                    node = i.child.to_dict()
+                    node.setdefault('level', []).append(lv)
+                    nodes.append(node)
+
+                    edges.append(dict(from_id=i.parent_id, to_id=i.child_id, text=i.relation_type.name, reverse=False))
+                    edge_tuples.add((i.parent_id, i.child_id))
+
+                    _find(i.child_id, lv)
+                    continue
+                elif (i.parent_id, i.child_id) not in edge_tuples:
+                    edges.append(dict(from_id=i.parent_id, to_id=i.child_id, text=i.relation_type.name, reverse=False))
+                    edge_tuples.add((i.parent_id, i.child_id))
+                    _find(i.child_id, lv)
+
+                for _node in nodes:
+                    if _node['id'] == i.child_id and lv not in _node['level']:
+                        _node['level'].append(lv)
+
+        def _reverse_find(_id, lv):
+            lv -= 1
+            for i in CITypeRelation.get_by(child_id=_id, to_dict=False):
+                if i.parent_id not in node_ids:
+                    node_ids.add(i.parent_id)
+                    node = i.parent.to_dict()
+                    node.setdefault('level', []).append(lv)
+                    nodes.append(node)
+
+                    edges.append(dict(from_id=i.parent_id, to_id=i.child_id, text=i.relation_type.name, reverse=True))
+                    edge_tuples.add((i.parent_id, i.child_id))
+
+                    _reverse_find(i.parent_id, lv)
+                    continue
+                elif (i.parent_id, i.child_id) not in edge_tuples:
+                    edges.append(dict(from_id=i.parent_id, to_id=i.child_id, text=i.relation_type.name, reverse=True))
+                    edge_tuples.add((i.parent_id, i.child_id))
+                    _reverse_find(i.parent_id, lv)
+
+                for _node in nodes:
+                    if _node['id'] == i.child_id and lv not in _node['level']:
+                        _node['level'].append(lv)
+
+        level = 0
+        _reverse_find(ci_type.id, level)
+        level = 0
+        _find(ci_type.id, level)
+
+        return nodes, edges
+
     @staticmethod
     def _get(parent_id, child_id):
         return CITypeRelation.get_by(parent_id=parent_id,

cmdb-api/api/lib/cmdb/const.py

@@ -73,6 +73,7 @@ class ResourceTypeEnum(BaseEnum):
     RELATION_VIEW = "RelationView"  # read/update/delete/grant
     CI_FILTER = "CIFilter"  # read
     PAGE = "page"  # read
+    TOPOLOGY_VIEW = "TopologyView"  # read/update/delete/grant
 
 
 class PermEnum(BaseEnum):

cmdb-api/api/lib/cmdb/preference.py

@@ -25,14 +25,15 @@ from api.lib.cmdb.resp_format import ErrFormat
 from api.lib.exception import AbortException
 from api.lib.perm.acl.acl import ACLManager
 from api.models.cmdb import CITypeAttribute
+from api.models.cmdb import CITypeGroup
+from api.models.cmdb import CITypeGroupItem
 from api.models.cmdb import CITypeRelation
 from api.models.cmdb import PreferenceCITypeOrder
 from api.models.cmdb import PreferenceRelationView
 from api.models.cmdb import PreferenceSearchOption
 from api.models.cmdb import PreferenceShowAttributes
 from api.models.cmdb import PreferenceTreeView
-from api.models.cmdb import CITypeGroup
-from api.models.cmdb import CITypeGroupItem
+
 
 class PreferenceManager(object):
     pref_attr_cls = PreferenceShowAttributes
@@ -85,7 +86,6 @@ class PreferenceManager(object):
 
         return dict(group_types=group_types, tree_types=tree_types, type_ids=list(type_ids))
 
-
     @staticmethod
     def get_types2(instance=False, tree=False):
         """

cmdb-api/api/lib/cmdb/resp_format.py

@@ -144,3 +144,8 @@ class ErrFormat(CommonErrFormat):
     cron_time_format_invalid = _l("Scheduling time format error")  # 调度时间格式错误
     reconciliation_title = _l("CMDB data reconciliation results")  # CMDB数据合规检查结果
     reconciliation_body = _l("Number of {} illegal: {}")  # "{} 不合规数: {}"
+
+    topology_exists = _l("Topology view {} already exists")  # 拓扑视图 {} 已经存在
+    topology_group_exists = _l("Topology group {} already exists")  # 拓扑视图分组 {} 已经存在
+    # 因为该分组下定义了拓扑视图，不能删除
+    topo_view_exists_cannot_delete_group = _l("The group cannot be deleted because the topology view already exists")

cmdb-api/api/lib/cmdb/search/ci/__init__.py

@@ -17,10 +17,12 @@ def search(query=None,
            count=1,
            sort=None,
            excludes=None,
-           use_id_filter=False):
+           use_id_filter=False,
+           use_ci_filter=True):
     if current_app.config.get("USE_ES"):
         s = SearchFromES(query, fl, facet, page, ret_key, count, sort)
     else:
-        s = SearchFromDB(query, fl, facet, page, ret_key, count, sort, excludes=excludes, use_id_filter=use_id_filter)
+        s = SearchFromDB(query, fl, facet, page, ret_key, count, sort, excludes=excludes,
+                         use_id_filter=use_id_filter, use_ci_filter=use_ci_filter)
 
     return s

cmdb-api/api/lib/cmdb/search/ci/db/search.py

@@ -70,6 +70,7 @@ class Search(object):
         self.valid_type_names = []
         self.type2filter_perms = dict()
         self.is_app_admin = is_app_admin('cmdb') or current_user.username == "worker"
+        self.is_app_admin = self.is_app_admin or (not self.use_ci_filter and not self.use_id_filter)
 
     @staticmethod
     def _operator_proc(key):

cmdb-api/api/lib/cmdb/topology.py

@@ -0,0 +1,251 @@
+# -*- coding:utf-8 -*-
+
+import json
+
+from flask import abort
+from flask import current_app
+from flask_login import current_user
+from werkzeug.exceptions import BadRequest
+
+from api.extensions import rd
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.cache import CITypeCache
+from api.lib.cmdb.ci import CIRelationManager
+from api.lib.cmdb.ci_type import CITypeRelationManager
+from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
+from api.lib.cmdb.const import ResourceTypeEnum
+from api.lib.cmdb.resp_format import ErrFormat
+from api.lib.cmdb.search import SearchError
+from api.lib.cmdb.search.ci import search
+from api.lib.perm.acl.acl import ACLManager
+from api.lib.perm.acl.acl import is_app_admin
+from api.models.cmdb import TopologyView
+from api.models.cmdb import TopologyViewGroup
+
+
+class TopologyViewManager(object):
+    group_cls = TopologyViewGroup
+    cls = TopologyView
+
+    @classmethod
+    def get_name_by_id(cls, _id):
+        res = cls.cls.get_by_id(_id)
+        return res and res.name
+
+    def get_view_by_id(self, _id):
+        res = self.cls.get_by_id(_id)
+
+        return res and res.to_dict() or {}
+
+    @classmethod
+    def add_group(cls, name, order):
+        if order is None:
+            cur_max_order = cls.group_cls.get_by(only_query=True).order_by(cls.group_cls.order.desc()).first()
+            cur_max_order = cur_max_order and cur_max_order.order or 0
+            order = cur_max_order + 1
+
+        cls.group_cls.get_by(name=name, first=True, to_dict=False) and abort(
+            400, ErrFormat.topology_group_exists.format(name))
+
+        return cls.group_cls.create(name=name, order=order)
+
+    def update_group(self, group_id, name, view_ids):
+        existed = self.group_cls.get_by_id(group_id) or abort(404, ErrFormat.not_found)
+        if name is not None and name != existed.name:
+            existed.update(name=name)
+
+        for idx, view_id in enumerate(view_ids):
+            view = self.cls.get_by_id(view_id)
+            if view is not None:
+                view.update(group_id=group_id, order=idx)
+
+        return existed.to_dict()
+
+    @classmethod
+    def delete_group(cls, _id):
+        existed = cls.group_cls.get_by_id(_id) or abort(404, ErrFormat.not_found)
+
+        if cls.cls.get_by(group_id=_id, first=True):
+            return abort(400, ErrFormat.topo_view_exists_cannot_delete_group)
+
+        existed.soft_delete()
+
+    @classmethod
+    def group_order(cls, group_ids):
+        for idx, group_id in enumerate(group_ids):
+            group = cls.group_cls.get_by_id(group_id)
+            group.update(order=idx + 1)
+
+    @classmethod
+    def add(cls, name, group_id, option, order=None, **kwargs):
+        cls.cls.get_by(name=name, first=True) and abort(400, ErrFormat.topology_exists.format(name))
+        if order is None:
+            cur_max_order = cls.cls.get_by(group_id=group_id, only_query=True).order_by(
+                cls.cls.order.desc()).first()
+            cur_max_order = cur_max_order and cur_max_order.order or 0
+            order = cur_max_order + 1
+
+        inst = cls.cls.create(name=name, group_id=group_id, option=option, order=order, **kwargs).to_dict()
+        if current_app.config.get('USE_ACL'):
+            try:
+                ACLManager().add_resource(name, ResourceTypeEnum.TOPOLOGY_VIEW)
+            except BadRequest:
+                pass
+
+            ACLManager().grant_resource_to_role(name,
+                                                current_user.username,
+                                                ResourceTypeEnum.TOPOLOGY_VIEW)
+
+        return inst
+
+    @classmethod
+    def update(cls, _id, **kwargs):
+        existed = cls.cls.get_by_id(_id) or abort(404, ErrFormat.not_found)
+        existed_name = existed.name
+
+        inst = existed.update(filter_none=False, **kwargs).to_dict()
+        if current_app.config.get('USE_ACL') and existed_name != kwargs.get('name') and kwargs.get('name'):
+            try:
+                ACLManager().update_resource(existed_name, kwargs['name'], ResourceTypeEnum.TOPOLOGY_VIEW)
+            except BadRequest:
+                pass
+
+        return inst
+
+    @classmethod
+    def delete(cls, _id):
+        existed = cls.cls.get_by_id(_id) or abort(404, ErrFormat.not_found)
+
+        existed.soft_delete()
+        if current_app.config.get("USE_ACL"):
+            ACLManager().del_resource(existed.name, ResourceTypeEnum.TOPOLOGY_VIEW)
+
+    @classmethod
+    def group_inner_order(cls, _ids):
+        for idx, _id in enumerate(_ids):
+            topology = cls.cls.get_by_id(_id)
+            topology.update(order=idx + 1)
+
+    @classmethod
+    def get_all(cls):
+        resources = None
+        if current_app.config.get('USE_ACL') and not is_app_admin('cmdb'):
+            resources = set([i.get('name') for i in ACLManager().get_resources(ResourceTypeEnum.TOPOLOGY_VIEW)])
+
+        groups = cls.group_cls.get_by(to_dict=True)
+        groups = sorted(groups, key=lambda x: x['order'])
+        group2pos = {group['id']: idx for idx, group in enumerate(groups)}
+
+        topo_views = sorted(cls.cls.get_by(to_dict=True), key=lambda x: x['order'])
+        other_group = dict(views=[])
+        for view in topo_views:
+            if resources is not None and view['name'] not in resources:
+                continue
+
+            if view['group_id']:
+                groups[group2pos[view['group_id']]].setdefault('views', []).append(view)
+            else:
+                other_group['views'].append(view)
+
+        if other_group['views']:
+            groups.append(other_group)
+
+        return groups
+
+    @staticmethod
+    def relation_from_ci_type(type_id):
+        nodes, edges = CITypeRelationManager.get_relations_by_type_id(type_id)
+
+        return dict(nodes=nodes, edges=edges)
+
+    def topology_view(self, view_id=None, preview=None):
+        if view_id is not None:
+            view = self.cls.get_by_id(view_id) or abort(404, ErrFormat.not_found)
+            central_node_type, central_node_instances, path = (view.central_node_type,
+                                                               view.central_node_instances, view.path)
+        else:
+            central_node_type = preview.get('central_node_type')
+            central_node_instances = preview.get('central_node_instances')
+            path = preview.get('path')
+
+        nodes, links = [], []
+        _type = CITypeCache.get(central_node_type)
+        if not _type:
+            return dict(nodes=nodes, links=links)
+        root_ids = []
+        show_key = AttributeCache.get(_type.show_id or _type.unique_id)
+
+        q = (central_node_instances[2:] if central_node_instances.startswith('q=') else
+             central_node_instances)
+        s = search(q, fl=['_id', show_key.name], use_id_filter=False, use_ci_filter=False, count=1000000)
+        try:
+            response, _, _, _, _, _ = s.search()
+        except SearchError as e:
+            current_app.logger.info(e)
+            return dict(nodes=nodes, links=links)
+        for i in response:
+            root_ids.append(i['_id'])
+            nodes.append(dict(id=str(i['_id']), name=i[show_key.name], type_id=central_node_type))
+        if not root_ids:
+            return dict(nodes=nodes, links=links)
+
+        prefix = REDIS_PREFIX_CI_RELATION
+        key = list(map(str, root_ids))
+        id2node = {}
+        type2meta = {}
+        for level in sorted([i for i in path.keys() if int(i) > 0]):
+            type_ids = {int(i) for i in path[level]}
+
+            res = [json.loads(x).items() for x in [i or '{}' for i in rd.get(key, prefix) or []]]
+            new_key = []
+            for idx, from_id in enumerate(key):
+                for to_id, type_id in res[idx]:
+                    if type_id in type_ids:
+                        links.append({'from': from_id, 'to': to_id})
+                        id2node[to_id] = {'id': to_id, 'type_id': type_id}
+                        new_key.append(to_id)
+                        if type_id not in type2meta:
+                            type2meta[type_id] = CITypeCache.get(type_id).icon
+
+            key = new_key
+
+        ci_ids = list(map(int, root_ids))
+        for level in sorted([i for i in path.keys() if int(i) < 0]):
+            type_ids = {int(i) for i in path[level]}
+            res = CIRelationManager.get_parent_ids(ci_ids)
+            _ci_ids = []
+            for to_id in res:
+                for from_id, type_id in res[to_id]:
+                    if type_id in type_ids:
+                        from_id, to_id = str(from_id), str(to_id)
+                        links.append({'from': from_id, 'to': to_id})
+                        id2node[from_id] = {'id': str(from_id), 'type_id': type_id}
+                        _ci_ids.append(from_id)
+                        if type_id not in type2meta:
+                            type2meta[type_id] = CITypeCache.get(type_id).icon
+
+            ci_ids = _ci_ids
+
+        fl = set()
+        type_ids = {t for lv in path if lv != '0' for t in path[lv]}
+        type2show = {}
+        for type_id in type_ids:
+            ci_type = CITypeCache.get(type_id)
+            if ci_type:
+                attr = AttributeCache.get(ci_type.show_id or ci_type.unique_id)
+                if attr:
+                    fl.add(attr.name)
+                    type2show[type_id] = attr.name
+
+        if id2node:
+            s = search("_id:({})".format(';'.join(id2node.keys())), fl=list(fl),
+                       use_id_filter=False, use_ci_filter=False, count=1000000)
+            try:
+                response, _, _, _, _, _ = s.search()
+            except SearchError:
+                return dict(nodes=nodes, links=links)
+            for i in response:
+                id2node[str(i['_id'])]['name'] = i[type2show[str(i['_type'])]]
+            nodes.extend(id2node.values())
+
+        return dict(nodes=nodes, links=links, type2meta=type2meta)