Dev api 0308 (#424)

* feat(api): grant by node in relation view

* fix(api): When removing attributes, remove the unique constraint

* feat(api): grant by service tree

cmdb-api/api/lib/cmdb/ci.py

@@ -60,6 +60,7 @@ from api.tasks.cmdb import ci_delete_trigger
 from api.tasks.cmdb import ci_relation_add
 from api.tasks.cmdb import ci_relation_cache
 from api.tasks.cmdb import ci_relation_delete
+from api.tasks.cmdb import delete_id_filter
 
 PASSWORD_DEFAULT_SHOW = "******"
 
@@ -558,6 +559,7 @@ class CIManager(object):
             AttributeHistoryManger.add(None, ci_id, [(None, OperateType.DELETE, ci_dict, None)], ci.type_id)
 
         ci_delete.apply_async(args=(ci_id,), queue=CMDB_QUEUE)
+        delete_id_filter.apply_async(args=(ci_id,), queue=CMDB_QUEUE)
 
         return ci_id
 
@@ -864,6 +866,20 @@ class CIRelationManager(object):
 
         return numfound, len(ci_ids), result
 
+    @staticmethod
+    def recursive_children(ci_id):
+        result = []
+
+        def _get_children(_id):
+            children = CIRelation.get_by(first_ci_id=_id, to_dict=False)
+            result.extend([i.second_ci_id for i in children])
+            for child in children:
+                _get_children(child.second_ci_id)
+
+        _get_children(ci_id)
+
+        return result
+
     @staticmethod
     def _sort_handler(sort_by, query_sql):
 
@@ -1015,6 +1031,7 @@ class CIRelationManager(object):
         his_manager.add(cr, operate_type=OperateType.DELETE)
 
         ci_relation_delete.apply_async(args=(cr.first_ci_id, cr.second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE)
+        delete_id_filter.apply_async(args=(cr.second_ci_id,), queue=CMDB_QUEUE)
 
         return cr_id
 
@@ -1026,9 +1043,13 @@ class CIRelationManager(object):
                                to_dict=False,
                                first=True)
 
-        ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE)
+        if cr is not None:
+            cls.delete(cr.id)
 
-        return cr and cls.delete(cr.id)
+        ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE)
+        delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE)
+
+        return cr
 
     @classmethod
     def batch_update(cls, ci_ids, parents, children, ancestor_ids=None):

cmdb-api/api/lib/cmdb/ci_type.py

@@ -76,6 +76,10 @@ class CITypeManager(object):
 
         return CIType.get_by_id(ci_type.id)
 
+    def get_icons(self):
+        return {i.id: i.icon or i.name for i in db.session.query(
+            self.cls.id, self.cls.icon, self.cls.name).filter(self.cls.deleted.is_(False))}
+
     @staticmethod
     def get_ci_types(type_name=None, like=True):
         resources = None
@@ -653,6 +657,21 @@ class CITypeAttributeManager(object):
                 for item in PreferenceShowAttributes.get_by(type_id=type_id, attr_id=attr_id, to_dict=False):
                     item.soft_delete(commit=False)
 
+                child_ids = CITypeInheritanceManager.recursive_children(type_id)
+                for _type_id in [type_id] + child_ids:
+                    for item in CITypeUniqueConstraint.get_by(type_id=_type_id, to_dict=False):
+                        if attr_id in item.attr_ids:
+                            attr_ids = copy.deepcopy(item.attr_ids)
+                            attr_ids.remove(attr_id)
+
+                            if attr_ids:
+                                item.update(attr_ids=attr_ids, commit=False)
+                            else:
+                                item.soft_delete(commit=False)
+
+                    item = CITypeTrigger.get_by(type_id=_type_id, attr_id=attr_id, to_dict=False, first=True)
+                    item and item.soft_delete(commit=False)
+
                 db.session.commit()
 
                 CITypeAttributeCache.clean(type_id, attr_id)

cmdb-api/api/lib/cmdb/perms.py

@@ -1,12 +1,15 @@
 # -*- coding:utf-8 -*-
-
+import copy
 import functools
 
+import redis_lock
 from flask import abort
 from flask import current_app
 from flask import request
 from flask_login import current_user
 
+from api.extensions import db
+from api.extensions import rd
 from api.lib.cmdb.const import ResourceTypeEnum
 from api.lib.cmdb.resp_format import ErrFormat
 from api.lib.mixin import DBMixin
@@ -40,6 +43,11 @@ class CIFilterPermsCRUD(DBMixin):
                         result[i['rid']]['ci_filter'] = ""
                     result[i['rid']]['ci_filter'] += (i['ci_filter'] or "")
 
+                if i['id_filter']:
+                    if not result[i['rid']]['id_filter']:
+                        result[i['rid']]['id_filter'] = {}
+                    result[i['rid']]['id_filter'].update(i['id_filter'] or {})
+
         return result
 
     def get_by_ids(self, _ids, type_id=None):
@@ -70,6 +78,11 @@ class CIFilterPermsCRUD(DBMixin):
                         result[i['type_id']]['ci_filter'] = ""
                     result[i['type_id']]['ci_filter'] += (i['ci_filter'] or "")
 
+                if i['id_filter']:
+                    if not result[i['type_id']]['id_filter']:
+                        result[i['type_id']]['id_filter'] = {}
+                    result[i['type_id']]['id_filter'].update(i['id_filter'] or {})
+
         return result
 
     @classmethod
@@ -82,6 +95,54 @@ class CIFilterPermsCRUD(DBMixin):
             type2filter_perms = cls().get_by_ids(list(map(int, [i['name'] for i in res2])), type_id=type_id)
             return type2filter_perms.get(type_id, {}).get('attr_filter') or []
 
+    def _revoke_children(self, rid, id_filter, rebuild=True):
+        items = self.cls.get_by(rid=rid, ci_filter=None, attr_filter=None, to_dict=False)
+        for item in items:
+            changed, item_id_filter = False, copy.deepcopy(item.id_filter)
+            for prefix in id_filter:
+                for k, v in copy.deepcopy((item.id_filter or {})).items():
+                    if k.startswith(prefix) and k != prefix:
+                        item_id_filter.pop(k)
+                        changed = True
+
+            if not item_id_filter and current_app.config.get('USE_ACL'):
+                item.soft_delete(commit=False)
+                ACLManager().del_resource(str(item.id), ResourceTypeEnum.CI_FILTER, rebuild=rebuild)
+            elif changed:
+                item.update(id_filter=item_id_filter, commit=False)
+
+        db.session.commit()
+
+    def _revoke_parent(self, rid, parent_path, rebuild=True):
+        parent_path = [i for i in parent_path.split(',') if i] or []
+        revoke_nodes = [','.join(parent_path[:i]) for i in range(len(parent_path), 0, -1)]
+        for node_path in revoke_nodes:
+            delete_item, can_deleted = None, True
+            items = self.cls.get_by(rid=rid, ci_filter=None, attr_filter=None, to_dict=False)
+            for item in items:
+                if node_path in item.id_filter:
+                    delete_item = item
+                if any(filter(lambda x: x.startswith(node_path) and x != node_path, item.id_filter.keys())):
+                    can_deleted = False
+                    break
+
+            if can_deleted and delete_item:
+                id_filter = copy.deepcopy(delete_item.id_filter)
+                id_filter.pop(node_path)
+                delete_item = delete_item.update(id_filter=id_filter, filter_none=False)
+
+                if current_app.config.get('USE_ACL') and not id_filter:
+                    ACLManager().del_resource(str(delete_item.id), ResourceTypeEnum.CI_FILTER, rebuild=False)
+                    delete_item.soft_delete()
+                    items.remove(delete_item)
+
+        if rebuild:
+            from api.tasks.acl import role_rebuild
+            from api.lib.perm.acl.const import ACL_QUEUE
+            from api.lib.perm.acl.cache import AppCache
+
+            role_rebuild.apply_async(args=(rid, AppCache.get('cmdb').id), queue=ACL_QUEUE)
+
     def _can_add(self, **kwargs):
         ci_filter = kwargs.get('ci_filter')
         attr_filter = kwargs.get('attr_filter') or ""
@@ -102,36 +163,67 @@ class CIFilterPermsCRUD(DBMixin):
 
     def add(self, **kwargs):
         kwargs = self._can_add(**kwargs) or kwargs
+        with redis_lock.Lock(rd.r, 'CMDB_FILTER_{}_{}'.format(kwargs['type_id'], kwargs['rid'])):
+            request_id_filter = {}
+            if kwargs.get('id_filter'):
+                obj = self.cls.get_by(type_id=kwargs.get('type_id'),
+                                      rid=kwargs.get('rid'),
+                                      ci_filter=None,
+                                      attr_filter=None,
+                                      first=True, to_dict=False)
 
-        obj = self.cls.get_by(type_id=kwargs.get('type_id'),
+                for _id, v in (kwargs.get('id_filter') or {}).items():
-                              rid=kwargs.get('rid'),
+                    key = ",".join(([v['parent_path']] if v.get('parent_path') else []) + [str(_id)])
-                              first=True, to_dict=False)
+                    request_id_filter[key] = v['name']
-        if obj is not None:
-            obj = obj.update(filter_none=False, **kwargs)
-            if not obj.attr_filter and not obj.ci_filter:
-                if current_app.config.get('USE_ACL'):
-                    ACLManager().del_resource(str(obj.id), ResourceTypeEnum.CI_FILTER)
 
-                obj.soft_delete()
+            else:
+                obj = self.cls.get_by(type_id=kwargs.get('type_id'),
+                                      rid=kwargs.get('rid'),
+                                      id_filter=None,
+                                      first=True, to_dict=False)
 
-                return obj
+            is_recursive = kwargs.pop('is_recursive', 0)
+            if obj is not None:
+                if obj.id_filter and isinstance(kwargs.get('id_filter'), dict):
+                    obj_id_filter = copy.deepcopy(obj.id_filter)
+
+                    for k, v in request_id_filter.items():
+                        obj_id_filter[k] = v
+
+                    kwargs['id_filter'] = obj_id_filter
+
+                obj = obj.update(filter_none=False, **kwargs)
+
+                if not obj.attr_filter and not obj.ci_filter and not obj.id_filter:
+                    if current_app.config.get('USE_ACL'):
+                        ACLManager().del_resource(str(obj.id), ResourceTypeEnum.CI_FILTER, rebuild=False)
+
+                    obj.soft_delete()
+
+                if not is_recursive and request_id_filter:
+                    self._revoke_children(obj.rid, request_id_filter, rebuild=False)
 
-        else:
-            if not kwargs.get('ci_filter') and not kwargs.get('attr_filter'):
                 return
 
-            obj = self.cls.create(**kwargs)
+            else:
+                if not kwargs.get('ci_filter') and not kwargs.get('attr_filter') and not kwargs.get('id_filter'):
+                    return
 
-        if current_app.config.get('USE_ACL'):
+                if request_id_filter:
-            try:
+                    kwargs['id_filter'] = request_id_filter
-                ACLManager().add_resource(obj.id, ResourceTypeEnum.CI_FILTER)
-            except:
-                pass
-            ACLManager().grant_resource_to_role_by_rid(obj.id,
-                                                       kwargs.get('rid'),
-                                                       ResourceTypeEnum.CI_FILTER)
 
-        return obj
+                obj = self.cls.create(**kwargs)
+
+                if current_app.config.get('USE_ACL'):  # new resource
+                    try:
+                        ACLManager().add_resource(obj.id, ResourceTypeEnum.CI_FILTER)
+                    except:
+                        pass
+                    ACLManager().grant_resource_to_role_by_rid(obj.id,
+                                                               kwargs.get('rid'),
+                                                               ResourceTypeEnum.CI_FILTER)
+
+                return obj
 
     def _can_update(self, **kwargs):
         pass
@@ -140,19 +232,84 @@ class CIFilterPermsCRUD(DBMixin):
         pass
 
     def delete(self, **kwargs):
-        obj = self.cls.get_by(type_id=kwargs.get('type_id'),
+        with redis_lock.Lock(rd.r, 'CMDB_FILTER_{}_{}'.format(kwargs['type_id'], kwargs['rid'])):
-                              rid=kwargs.get('rid'),
+            obj = self.cls.get_by(type_id=kwargs.get('type_id'),
-                              first=True, to_dict=False)
+                                  rid=kwargs.get('rid'),
+                                  id_filter=None,
+                                  first=True, to_dict=False)
+
+            if obj is not None:
+                resource = None
+                if current_app.config.get('USE_ACL'):
+                    resource = ACLManager().del_resource(str(obj.id), ResourceTypeEnum.CI_FILTER)
+
+                obj.soft_delete()
+
+                return resource
+
+    def delete2(self, **kwargs):
+
+        with redis_lock.Lock(rd.r, 'CMDB_FILTER_{}_{}'.format(kwargs['type_id'], kwargs['rid'])):
+            obj = self.cls.get_by(type_id=kwargs.get('type_id'),
+                                  rid=kwargs.get('rid'),
+                                  ci_filter=None,
+                                  attr_filter=None,
+                                  first=True, to_dict=False)
+
+            request_id_filter = {}
+            for _id, v in (kwargs.get('id_filter') or {}).items():
+                key = ",".join([v['parent_path']] if v.get('parent_path') else [] + [str(_id)])
+                request_id_filter[key] = v['name']
 
-        if obj is not None:
             resource = None
-            if current_app.config.get('USE_ACL'):
+            if obj is not None:
-                resource = ACLManager().del_resource(str(obj.id), ResourceTypeEnum.CI_FILTER)
 
-            obj.soft_delete()
+                id_filter = {}
+                for k, v in copy.deepcopy(obj.id_filter or {}).items():  # important
+                    if k not in request_id_filter:
+                        id_filter[k] = v
+
+                if not id_filter and current_app.config.get('USE_ACL'):
+                    resource = ACLManager().del_resource(str(obj.id), ResourceTypeEnum.CI_FILTER, rebuild=False)
+                    obj.soft_delete()
+                    db.session.commit()
+
+                else:
+                    obj.update(id_filter=id_filter)
+
+                self._revoke_children(kwargs.get('rid'), request_id_filter, rebuild=False)
+                self._revoke_parent(kwargs.get('rid'), kwargs.get('parent_path'))
 
             return resource
 
+    def delete_id_filter_by_ci_id(self, ci_id):
+        items = self.cls.get_by(ci_filter=None, attr_filter=None, to_dict=False)
+
+        rebuild_roles = set()
+        for item in items:
+            id_filter = copy.deepcopy(item.id_filter)
+            changed = False
+            for node_path in item.id_filter:
+                if str(ci_id) in node_path:
+                    id_filter.pop(node_path)
+                    changed = True
+
+            if changed:
+                rebuild_roles.add(item.rid)
+                if not id_filter:
+                    item.soft_delete(commit=False)
+                else:
+                    item.update(id_filter=id_filter, commit=False)
+
+        db.session.commit()
+
+        if rebuild_roles:
+            from api.tasks.acl import role_rebuild
+            from api.lib.perm.acl.const import ACL_QUEUE
+            from api.lib.perm.acl.cache import AppCache
+            for rid in rebuild_roles:
+                role_rebuild.apply_async(args=(rid, AppCache.get('cmdb').id), queue=ACL_QUEUE)
+
 
 def has_perm_for_ci(arg_name, resource_type, perm, callback=None, app=None):
     def decorator_has_perm(func):

cmdb-api/api/lib/cmdb/search/ci/__init__.py

@@ -16,10 +16,11 @@ def search(query=None,
            ret_key=RetKey.NAME,
            count=1,
            sort=None,
-           excludes=None):
+           excludes=None,
+           use_id_filter=True):
     if current_app.config.get("USE_ES"):
         s = SearchFromES(query, fl, facet, page, ret_key, count, sort)
     else:
-        s = SearchFromDB(query, fl, facet, page, ret_key, count, sort, excludes=excludes)
+        s = SearchFromDB(query, fl, facet, page, ret_key, count, sort, excludes=excludes, use_id_filter=use_id_filter)
 
     return s

cmdb-api/api/lib/cmdb/search/ci/db/search.py

@@ -44,7 +44,10 @@ class Search(object):
                  count=1,
                  sort=None,
                  ci_ids=None,
-                 excludes=None):
+                 excludes=None,
+                 parent_node_perm_passed=False,
+                 use_id_filter=False,
+                 use_ci_filter=True):
         self.orig_query = query
         self.fl = fl or []
         self.excludes = excludes or []
@@ -54,9 +57,13 @@ class Search(object):
         self.count = count
         self.sort = sort
         self.ci_ids = ci_ids or []
+        self.raw_ci_ids = copy.deepcopy(self.ci_ids)
         self.query_sql = ""
         self.type_id_list = []
         self.only_type_query = False
+        self.parent_node_perm_passed = parent_node_perm_passed
+        self.use_id_filter = use_id_filter
+        self.use_ci_filter = use_ci_filter
 
         self.valid_type_names = []
         self.type2filter_perms = dict()
@@ -106,7 +113,7 @@ class Search(object):
                     self.type_id_list.append(str(ci_type.id))
                     if ci_type.id in self.type2filter_perms:
                         ci_filter = self.type2filter_perms[ci_type.id].get('ci_filter')
-                        if ci_filter:
+                        if ci_filter and self.use_ci_filter and not self.use_id_filter:
                             sub = []
                             ci_filter = Template(ci_filter).render(user=current_user)
                             for i in ci_filter.split(','):
@@ -122,6 +129,14 @@ class Search(object):
                                 self.fl = set(self.type2filter_perms[ci_type.id]['attr_filter'])
                             else:
                                 self.fl = set(self.fl) & set(self.type2filter_perms[ci_type.id]['attr_filter'])
+
+                        if self.type2filter_perms[ci_type.id].get('id_filter') and self.use_id_filter:
+
+                            if not self.raw_ci_ids:
+                                self.ci_ids = list(self.type2filter_perms[ci_type.id]['id_filter'].keys())
+
+                    if self.use_id_filter and not self.ci_ids and not is_app_admin('cmdb'):
+                        self.raw_ci_ids = [0]
                 else:
                     raise SearchError(ErrFormat.no_permission.format(ci_type.alias, PermEnum.READ))
             else:
@@ -155,6 +170,7 @@ class Search(object):
             "NOT LIKE" if is_not else "LIKE",
             _v.replace("*", "%")) for _v in new_v])
         _query_sql = QUERY_CI_BY_ATTR_NAME.format(table_name, attr.id, in_query)
+
         return _query_sql
 
     @staticmethod
@@ -170,6 +186,7 @@ class Search(object):
             "NOT BETWEEN" if is_not else "BETWEEN",
             start.replace("*", "%"), end.replace("*", "%"))
         _query_sql = QUERY_CI_BY_ATTR_NAME.format(table_name, attr.id, range_query)
+
         return _query_sql
 
     @staticmethod
@@ -186,6 +203,7 @@ class Search(object):
 
             comparison_query = "{0} '{1}'".format(v[0], v[1:].replace("*", "%"))
         _query_sql = QUERY_CI_BY_ATTR_NAME.format(table_name, attr.id, comparison_query)
+
         return _query_sql
 
     @staticmethod
@@ -197,6 +215,7 @@ class Search(object):
         elif field.startswith("-"):
             field = field[1:]
             sort_type = "DESC"
+
         return field, sort_type
 
     def __sort_by_id(self, sort_type, query_sql):
@@ -325,6 +344,11 @@ class Search(object):
 
         return numfound, res
 
+    def __get_type2filter_perms(self):
+        res2 = ACLManager('cmdb').get_resources(ResourceTypeEnum.CI_FILTER)
+        if res2:
+            self.type2filter_perms = CIFilterPermsCRUD().get_by_ids(list(map(int, [i['name'] for i in res2])))
+
     def __get_types_has_read(self):
         """
         :return: _type:(type1;type2)
@@ -334,14 +358,23 @@ class Search(object):
 
         self.valid_type_names = {i['name'] for i in res if PermEnum.READ in i['permissions']}
 
-        res2 = acl.get_resources(ResourceTypeEnum.CI_FILTER)
+        self.__get_type2filter_perms()
-        if res2:
+
-            self.type2filter_perms = CIFilterPermsCRUD().get_by_ids(list(map(int, [i['name'] for i in res2])))
+        for type_id in self.type2filter_perms:
+            ci_type = CITypeCache.get(type_id)
+            if ci_type:
+                if self.type2filter_perms[type_id].get('id_filter'):
+                    if self.use_id_filter:
+                        self.valid_type_names.add(ci_type.name)
+                elif self.type2filter_perms[type_id].get('ci_filter'):
+                    if self.use_ci_filter:
+                        self.valid_type_names.add(ci_type.name)
+                else:
+                    self.valid_type_names.add(ci_type.name)
 
         return "_type:({})".format(";".join(self.valid_type_names))
 
     def __confirm_type_first(self, queries):
-
         has_type = False
 
         result = []
@@ -375,7 +408,10 @@ class Search(object):
                 result.append(q)
 
         _is_app_admin = is_app_admin('cmdb') or current_user.username == "worker"
-        if result and not has_type and not _is_app_admin:
+        if self.parent_node_perm_passed:
+            self.__get_type2filter_perms()
+            self.valid_type_names = "ALL"
+        elif result and not has_type and not _is_app_admin:
             type_q = self.__get_types_has_read()
             if id_query:
                 ci = CIManager.get_by_id(id_query)
@@ -389,8 +425,6 @@ class Search(object):
         else:
             self.__get_types_has_read()
 
-        current_app.logger.warning(result)
-
         return result
 
     def __query_by_attr(self, q, queries, alias):
@@ -482,7 +516,7 @@ class Search(object):
     def _filter_ids(self, query_sql):
         if self.ci_ids:
             return "SELECT * FROM ({0}) AS IN_QUERY WHERE IN_QUERY.ci_id IN ({1})".format(
-                query_sql, ",".join(list(map(str, self.ci_ids))))
+                query_sql, ",".join(list(set(map(str, self.ci_ids)))))
 
         return query_sql
 
@@ -514,6 +548,9 @@ class Search(object):
         s = time.time()
         if query_sql:
             query_sql = self._filter_ids(query_sql)
+            if self.raw_ci_ids and not self.ci_ids:
+                return 0, []
+
             self.query_sql = query_sql
             # current_app.logger.debug(query_sql)
             numfound, res = self._execute_sql(query_sql)
@@ -572,3 +609,8 @@ class Search(object):
         total = len(response)
 
         return response, counter, total, self.page, numfound, facet
+
+    def get_ci_ids(self):
+        _, ci_ids = self._query_build_raw()
+
+        return ci_ids

cmdb-api/api/lib/cmdb/search/ci_relation/search.py

@@ -1,9 +1,11 @@
 # -*- coding:utf-8 -*-
 import json
+import sys
 from collections import Counter
 
 from flask import abort
 from flask import current_app
+from flask_login import current_user
 
 from api.extensions import rd
 from api.lib.cmdb.ci import CIRelationManager
@@ -11,11 +13,14 @@ from api.lib.cmdb.ci_type import CITypeRelationManager
 from api.lib.cmdb.const import ConstraintEnum
 from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
 from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION2
+from api.lib.cmdb.const import ResourceTypeEnum
+from api.lib.cmdb.perms import CIFilterPermsCRUD
 from api.lib.cmdb.resp_format import ErrFormat
 from api.lib.cmdb.search.ci.db.search import Search as SearchFromDB
 from api.lib.cmdb.search.ci.es.search import Search as SearchFromES
+from api.lib.perm.acl.acl import ACLManager
+from api.lib.perm.acl.acl import is_app_admin
 from api.models.cmdb import CI
-from api.models.cmdb import CIRelation
 
 
 class Search(object):
@@ -29,7 +34,9 @@ class Search(object):
                  sort=None,
                  reverse=False,
                  ancestor_ids=None,
-                 has_m2m=None):
+                 descendant_ids=None,
+                 has_m2m=None,
+                 root_parent_path=None):
         self.orig_query = query
         self.fl = fl
         self.facet_field = facet_field
@@ -46,6 +53,8 @@ class Search(object):
             level[0] if isinstance(level, list) and level else level)
 
         self.ancestor_ids = ancestor_ids
+        self.descendant_ids = descendant_ids
+        self.root_parent_path = root_parent_path
         self.has_m2m = has_m2m or False
         if not self.has_m2m:
             if self.ancestor_ids:
@@ -56,27 +65,21 @@ class Search(object):
                     if _l < int(level) and c == ConstraintEnum.Many2Many:
                         self.has_m2m = True
 
+        self.type2filter_perms = None
+
     def _get_ids(self, ids):
-        if self.level[-1] == 1 and len(ids) == 1:
-            if self.ancestor_ids is None:
-                return [i.second_ci_id for i in CIRelation.get_by(first_ci_id=ids[0], to_dict=False)]
-
-            else:
-                seconds = {i.second_ci_id for i in CIRelation.get_by(first_ci_id=ids[0],
-                                                                     ancestor_ids=self.ancestor_ids,
-                                                                     to_dict=False)}
-
-                return list(seconds)
 
         merge_ids = []
         key = []
         _tmp = []
         for level in range(1, sorted(self.level)[-1] + 1):
+            if len(self.descendant_ids) >= level and self.type2filter_perms.get(self.descendant_ids[level - 1]):
+                id_filter_limit, _ = self._get_ci_filter(self.type2filter_perms[self.descendant_ids[level - 1]])
+            else:
+                id_filter_limit = {}
+
             if not self.has_m2m:
-                _tmp = map(lambda x: json.loads(x).keys(),
+                key, prefix = list(map(str, ids)), REDIS_PREFIX_CI_RELATION
-                           filter(lambda x: x is not None, rd.get(ids, REDIS_PREFIX_CI_RELATION) or []))
-                ids = [j for i in _tmp for j in i]
-                key, prefix = ids, REDIS_PREFIX_CI_RELATION
 
             else:
                 if not self.ancestor_ids:
@@ -92,12 +95,16 @@ class Search(object):
                         key = list(set(["{},{}".format(i, j) for idx, i in enumerate(key) for j in _tmp[idx]]))
                         prefix = REDIS_PREFIX_CI_RELATION2
 
-                _tmp = list(map(lambda x: json.loads(x).keys() if x else [], rd.get(key, prefix) or []))
+            if not key or id_filter_limit is None:
-                ids = [j for i in _tmp for j in i]
-
-            if not key:
                 return []
 
+            res = [json.loads(x).items() for x in [i or '{}' for i in rd.get(key, prefix) or []]]
+            _tmp = [[i[0] for i in x if (not id_filter_limit or (
+                    key[idx] not in id_filter_limit or int(i[0]) in id_filter_limit[key[idx]]) or
+                                         int(i[0]) in id_filter_limit)] for idx, x in enumerate(res)]
+
+            ids = [j for i in _tmp for j in i]
+
             if level in self.level:
                 merge_ids.extend(ids)
 
@@ -120,7 +127,28 @@ class Search(object):
 
         return merge_ids
 
+    def _has_read_perm_from_parent_nodes(self):
+        self.root_parent_path = list(map(str, self.root_parent_path))
+        if str(self.root_id).isdigit() and str(self.root_id) not in self.root_parent_path:
+            self.root_parent_path.append(str(self.root_id))
+        self.root_parent_path = set(self.root_parent_path)
+
+        if is_app_admin('cmdb'):
+            self.type2filter_perms = {}
+            return True
+
+        res = ACLManager().get_resources(ResourceTypeEnum.CI_FILTER) or {}
+        self.type2filter_perms = CIFilterPermsCRUD().get_by_ids(list(map(int, [i['name'] for i in res]))) or {}
+        for _, filters in self.type2filter_perms.items():
+            if set((filters.get('id_filter') or {}).keys()) & self.root_parent_path:
+                return True
+
+        return True
+
     def search(self):
+        use_ci_filter = len(self.descendant_ids) == self.level[0] - 1
+        parent_node_perm_passed = self._has_read_perm_from_parent_nodes()
+
         ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
         cis = [CI.get_by_id(_id) or abort(404, ErrFormat.ci_not_found.format("id={}".format(_id))) for _id in ids]
 
@@ -161,42 +189,103 @@ class Search(object):
                                 page=self.page,
                                 count=self.count,
                                 sort=self.sort,
-                                ci_ids=merge_ids).search()
+                                ci_ids=merge_ids,
+                                parent_node_perm_passed=parent_node_perm_passed,
+                                use_ci_filter=use_ci_filter).search()
+
+    def _get_ci_filter(self, filter_perms, ci_filters=None):
+        ci_filters = ci_filters or []
+        if ci_filters:
+            result = {}
+            for item in ci_filters:
+                res = SearchFromDB('_type:{},{}'.format(item['type_id'], item['ci_filter']),
+                                   count=sys.maxsize, parent_node_perm_passed=True).get_ci_ids()
+                if res:
+                    result[item['type_id']] = set(res)
+
+            return {}, result if result else None
+
+        result = dict()
+        if filter_perms.get('id_filter'):
+            for k in filter_perms['id_filter']:
+                node_path = k.split(',')
+                if len(node_path) == 1:
+                    result[int(node_path[0])] = 1
+                elif not self.has_m2m:
+                    result.setdefault(node_path[-2], set()).add(int(node_path[-1]))
+                else:
+                    result.setdefault(','.join(node_path[:-1]), set()).add(int(node_path[-1]))
+            if result:
+                return result, None
+            else:
+                return None, None
+
+        return {}, None
 
     def statistics(self, type_ids):
         self.level = int(self.level)
 
+        acl = ACLManager('cmdb')
+        _is_app_admin = is_app_admin('cmdb') or current_user.username == "worker"
+
+        type2filter_perms = dict()
+        if not _is_app_admin:
+            res2 = acl.get_resources(ResourceTypeEnum.CI_FILTER)
+            if res2:
+                type2filter_perms = CIFilterPermsCRUD().get_by_ids(list(map(int, [i['name'] for i in res2])))
+
         ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
         _tmp = []
         level2ids = {}
         for lv in range(1, self.level + 1):
             level2ids[lv] = []
 
+            id_filter_limit, ci_filter_limit = None, None
+            if len(self.descendant_ids) >= lv and type2filter_perms.get(self.descendant_ids[lv - 1]):
+                id_filter_limit, _ = self._get_ci_filter(type2filter_perms[self.descendant_ids[lv - 1]])
+            elif type_ids and self.level == lv:
+                ci_filters = [type2filter_perms[type_id] for type_id in type_ids if type_id in type2filter_perms]
+                if ci_filters:
+                    id_filter_limit, ci_filter_limit = self._get_ci_filter({}, ci_filters=ci_filters)
+                else:
+                    id_filter_limit = {}
+            else:
+                id_filter_limit = {}
+
             if lv == 1:
                 if not self.has_m2m:
-                    key, prefix = ids, REDIS_PREFIX_CI_RELATION
+                    key, prefix = [str(i) for i in ids], REDIS_PREFIX_CI_RELATION
                 else:
+                    key = ["{},{}".format(self.ancestor_ids, _id) for _id in ids]
                     if not self.ancestor_ids:
-                        key, prefix = ids, REDIS_PREFIX_CI_RELATION
+                        key, prefix = [str(i) for i in ids], REDIS_PREFIX_CI_RELATION
                     else:
-                        key = ["{},{}".format(self.ancestor_ids, _id) for _id in ids]
                         prefix = REDIS_PREFIX_CI_RELATION2
 
                     level2ids[lv] = [[i] for i in key]
 
-                if not key:
+                if not key or id_filter_limit is None:
-                    _tmp = []
+                    _tmp = [[]] * len(ids)
                     continue
 
+                res = [json.loads(x).items() for x in [i or '{}' for i in rd.get(key, prefix) or []]]
+                _tmp = []
                 if type_ids and lv == self.level:
-                    _tmp = list(map(lambda x: [i for i in x if i[1] in type_ids],
+                    _tmp = [[i for i in x if i[1] in type_ids and
-                                    (map(lambda x: list(json.loads(x).items()),
+                             (not id_filter_limit or (key[idx] not in id_filter_limit or
-                                         [i or '{}' for i in rd.get(key, prefix) or []]))))
+                                                      int(i[0]) in id_filter_limit[key[idx]]) or
+                              int(i[0]) in id_filter_limit)] for idx, x in enumerate(res)]
                 else:
-                    _tmp = list(map(lambda x: list(json.loads(x).items()),
+                    _tmp = [[i for i in x if (not id_filter_limit or (key[idx] not in id_filter_limit or
-                                    [i or '{}' for i in rd.get(key, prefix) or []]))
+                                                                      int(i[0]) in id_filter_limit[key[idx]]) or
+                                              int(i[0]) in id_filter_limit)] for idx, x in enumerate(res)]
+
+                if ci_filter_limit:
+                    _tmp = [[j for j in i if j[1] not in ci_filter_limit or int(j[0]) in ci_filter_limit[j[1]]]
+                            for i in _tmp]
 
             else:
+
                 for idx, item in enumerate(_tmp):
                     if item:
                         if not self.has_m2m:
@@ -208,15 +297,22 @@ class Search(object):
                             level2ids[lv].append(key)
 
                         if key:
+                            res = [json.loads(x).items() for x in [i or '{}' for i in rd.get(key, prefix) or []]]
                             if type_ids and lv == self.level:
-                                __tmp = map(lambda x: [(_id, type_id) for _id, type_id in json.loads(x).items()
+                                __tmp = [[i for i in x if i[1] in type_ids and
-                                                       if type_id in type_ids],
+                                          (not id_filter_limit or (
-                                            filter(lambda x: x is not None,
+                                                  key[idx] not in id_filter_limit or
-                                                   rd.get(key, prefix) or []))
+                                                  int(i[0]) in id_filter_limit[key[idx]]) or
+                                           int(i[0]) in id_filter_limit)] for idx, x in enumerate(res)]
                             else:
-                                __tmp = map(lambda x: list(json.loads(x).items()),
+                                __tmp = [[i for i in x if (not id_filter_limit or (
-                                            filter(lambda x: x is not None,
+                                        key[idx] not in id_filter_limit or
-                                                   rd.get(key, prefix) or []))
+                                        int(i[0]) in id_filter_limit[key[idx]]) or
+                                                           int(i[0]) in id_filter_limit)] for idx, x in enumerate(res)]
+
+                            if ci_filter_limit:
+                                __tmp = [[j for j in i if j[1] not in ci_filter_limit or
+                                          int(j[0]) in ci_filter_limit[j[1]]] for i in __tmp]
                         else:
                             __tmp = []
 

cmdb-api/api/models/cmdb.py

@@ -569,6 +569,7 @@ class CIFilterPerms(Model):
     type_id = db.Column(db.Integer, db.ForeignKey('c_ci_types.id'))
     ci_filter = db.Column(db.Text)
     attr_filter = db.Column(db.Text)
+    id_filter = db.Column(db.JSON)  # {node_path: unique_value}
 
     rid = db.Column(db.Integer, index=True)
 

cmdb-api/api/tasks/cmdb.py

@@ -18,6 +18,7 @@ from api.lib.cmdb.const import CMDB_QUEUE
 from api.lib.cmdb.const import REDIS_PREFIX_CI
 from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
 from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION2
+from api.lib.cmdb.perms import CIFilterPermsCRUD
 from api.lib.decorator import flush_db
 from api.lib.decorator import reconnect_db
 from api.lib.perm.acl.cache import UserCache
@@ -83,6 +84,13 @@ def ci_delete(ci_id):
     current_app.logger.info("{0} delete..........".format(ci_id))
 
 
+@celery.task(name="cmdb.delete_id_filter", queue=CMDB_QUEUE)
+@reconnect_db
+def delete_id_filter(ci_id):
+
+    CIFilterPermsCRUD().delete_id_filter_by_ci_id(ci_id)
+
+
 @celery.task(name="cmdb.ci_delete_trigger", queue=CMDB_QUEUE)
 @reconnect_db
 def ci_delete_trigger(trigger, operate_type, ci_dict):

cmdb-api/api/views/cmdb/ci.py

@@ -11,8 +11,7 @@ from api.lib.cmdb.cache import CITypeCache
 from api.lib.cmdb.ci import CIManager
 from api.lib.cmdb.ci import CIRelationManager
 from api.lib.cmdb.const import ExistPolicy
-from api.lib.cmdb.const import PermEnum
+from api.lib.cmdb.const import ResourceTypeEnum, PermEnum
-from api.lib.cmdb.const import ResourceTypeEnum
 from api.lib.cmdb.const import RetKey
 from api.lib.cmdb.perms import has_perm_for_ci
 from api.lib.cmdb.search import SearchError
@@ -152,9 +151,10 @@ class CISearchView(APIView):
             ret_key = RetKey.NAME
         facet = handle_arg_list(request.values.get("facet", ""))
         sort = request.values.get("sort")
+        use_id_filter = request.values.get("use_id_filter", False) in current_app.config.get('BOOL_TRUE')
 
         start = time.time()
-        s = search(query, fl, facet, page, ret_key, count, sort, excludes)
+        s = search(query, fl, facet, page, ret_key, count, sort, excludes, use_id_filter=use_id_filter)
         try:
             response, counter, total, page, numfound, facet = s.search()
         except SearchError as e:

cmdb-api/api/views/cmdb/ci_relation.py

@@ -13,7 +13,6 @@ from api.lib.cmdb.resp_format import ErrFormat
 from api.lib.cmdb.search import SearchError
 from api.lib.cmdb.search.ci_relation.search import Search
 from api.lib.decorator import args_required
-from api.lib.perm.auth import auth_abandoned
 from api.lib.utils import get_page
 from api.lib.utils import get_page_size
 from api.lib.utils import handle_arg_list
@@ -36,6 +35,8 @@ class CIRelationSearchView(APIView):
 
         root_id = request.values.get('root_id')
         ancestor_ids = request.values.get('ancestor_ids') or None  # only for many to many
+        root_parent_path = handle_arg_list(request.values.get('root_parent_path') or '')
+        descendant_ids = list(map(int, handle_arg_list(request.values.get('descendant_ids', []))))
         level = list(map(int, handle_arg_list(request.values.get('level', '1'))))
 
         query = request.values.get('q', "")
@@ -47,7 +48,8 @@ class CIRelationSearchView(APIView):
 
         start = time.time()
         s = Search(root_id, level, query, fl, facet, page, count, sort, reverse,
-                   ancestor_ids=ancestor_ids, has_m2m=has_m2m)
+                   ancestor_ids=ancestor_ids, has_m2m=has_m2m, root_parent_path=root_parent_path,
+                   descendant_ids=descendant_ids)
         try:
             response, counter, total, page, numfound, facet = s.search()
         except SearchError as e:
@@ -65,16 +67,16 @@ class CIRelationSearchView(APIView):
 class CIRelationStatisticsView(APIView):
     url_prefix = "/ci_relations/statistics"
 
-    @auth_abandoned
     def get(self):
         root_ids = list(map(int, handle_arg_list(request.values.get('root_ids'))))
         level = request.values.get('level', 1)
         type_ids = set(map(int, handle_arg_list(request.values.get('type_ids', []))))
         ancestor_ids = request.values.get('ancestor_ids') or None  # only for many to many
+        descendant_ids = list(map(int, handle_arg_list(request.values.get('descendant_ids', []))))
         has_m2m = request.values.get("has_m2m") in current_app.config.get('BOOL_TRUE')
 
         start = time.time()
-        s = Search(root_ids, level, ancestor_ids=ancestor_ids, has_m2m=has_m2m)
+        s = Search(root_ids, level, ancestor_ids=ancestor_ids, descendant_ids=descendant_ids, has_m2m=has_m2m)
         try:
             result = s.statistics(type_ids)
         except SearchError as e:

cmdb-api/api/views/cmdb/ci_type.py

@@ -38,9 +38,13 @@ from api.resource import APIView
 
 
 class CITypeView(APIView):
-    url_prefix = ("/ci_types", "/ci_types/<int:type_id>", "/ci_types/<string:type_name>")
+    url_prefix = ("/ci_types", "/ci_types/<int:type_id>", "/ci_types/<string:type_name>",
+                  "/ci_types/icons")
 
     def get(self, type_id=None, type_name=None):
+        if request.url.endswith("icons"):
+            return self.jsonify(CITypeManager().get_icons())
+
         q = request.args.get("type_name")
 
         if type_id is not None:
@@ -490,13 +494,14 @@ class CITypeGrantView(APIView):
         if not acl.has_permission(type_name, ResourceTypeEnum.CI_TYPE, PermEnum.GRANT) and not is_app_admin('cmdb'):
             return abort(403, ErrFormat.no_permission.format(type_name, PermEnum.GRANT))
 
-        acl.grant_resource_to_role_by_rid(type_name, rid, ResourceTypeEnum.CI_TYPE, perms, rebuild=False)
+        if perms and not request.values.get('id_filter'):
+            acl.grant_resource_to_role_by_rid(type_name, rid, ResourceTypeEnum.CI_TYPE, perms, rebuild=False)
 
-        resource = None
+        new_resource = None
-        if 'ci_filter' in request.values or 'attr_filter' in request.values:
+        if 'ci_filter' in request.values or 'attr_filter' in request.values or 'id_filter' in request.values:
-            resource = CIFilterPermsCRUD().add(type_id=type_id, rid=rid, **request.values)
+            new_resource = CIFilterPermsCRUD().add(type_id=type_id, rid=rid, **request.values)
 
-        if not resource:
+        if not new_resource:
             from api.tasks.acl import role_rebuild
             from api.lib.perm.acl.const import ACL_QUEUE
 
@@ -522,10 +527,18 @@ class CITypeRevokeView(APIView):
         if not acl.has_permission(type_name, ResourceTypeEnum.CI_TYPE, PermEnum.GRANT) and not is_app_admin('cmdb'):
             return abort(403, ErrFormat.no_permission.format(type_name, PermEnum.GRANT))
 
-        acl.revoke_resource_from_role_by_rid(type_name, rid, ResourceTypeEnum.CI_TYPE, perms, rebuild=False)
-
         app_id = AppCache.get('cmdb').id
         resource = None
+
+        if request.values.get('id_filter'):
+            CIFilterPermsCRUD().delete2(
+                type_id=type_id, rid=rid, id_filter=request.values['id_filter'],
+                parent_path=request.values.get('parent_path'))
+
+            return self.jsonify(type_id=type_id, rid=rid)
+
+        acl.revoke_resource_from_role_by_rid(type_name, rid, ResourceTypeEnum.CI_TYPE, perms, rebuild=False)
+
         if PermEnum.READ in perms or not perms:
             resource = CIFilterPermsCRUD().delete(type_id=type_id, rid=rid)