Dev api 240517 (#511)

* fix(api): list values delete

* fix(acl): role rebuild cache

cmdb-api/api/lib/cmdb/ci.py

@@ -263,6 +263,7 @@ class CIManager(object):
             ci_ids = None
             for attr_id in constraint.attr_ids:
                 value_table = TableMap(attr_name=id2name[attr_id]).table
+
                 values = value_table.get_by(attr_id=attr_id,
                                             value=ci_dict.get(id2name[attr_id]) or None,
                                             only_query=True).join(
@@ -438,7 +439,8 @@ class CIManager(object):
 
         return ci.id
 
-    def update(self, ci_id, _is_admin=False, ticket_id=None, **ci_dict):
+    def update(self, ci_id, _is_admin=False, ticket_id=None, __sync=False, **ci_dict):
+        current_app.logger.info((ci_id, ci_dict, __sync))
         now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
         ci = self.confirm_ci_existed(ci_id)
 
@@ -502,11 +504,17 @@ class CIManager(object):
                 record_id = self.save_password(ci.id, attr_id, password_dict[attr_id], record_id, ci.type_id)
 
         if record_id:  # has change
-            ci_cache.apply_async(args=(ci_id, OperateType.UPDATE, record_id), queue=CMDB_QUEUE)
+            if not __sync:
+                ci_cache.apply_async(args=(ci_id, OperateType.UPDATE, record_id), queue=CMDB_QUEUE)
+            else:
+                ci_cache((ci_id, OperateType.UPDATE, record_id))
 
         ref_ci_dict = {k: v for k, v in ci_dict.items() if k.startswith("$") and "." in k}
         if ref_ci_dict:
-            ci_relation_add.apply_async(args=(ref_ci_dict, ci.id), queue=CMDB_QUEUE)
+            if not __sync:
+                ci_relation_add.apply_async(args=(ref_ci_dict, ci.id), queue=CMDB_QUEUE)
+            else:
+                ci_relation_add((ref_ci_dict, ci.id))
 
     @staticmethod
     def update_unique_value(ci_id, unique_name, unique_value):
@@ -830,6 +838,12 @@ class CIManager(object):
             return data.get('v')
 
     def baseline(self, ci_ids, before_date):
+        """
+        return CI changes
+        :param ci_ids:
+        :param before_date:
+        :return:
+        """
         ci_list = self.get_cis_by_ids(ci_ids, ret_key=RetKey.ALIAS)
         if not ci_list:
             return dict()
@@ -913,6 +927,44 @@ class CIManager(object):
 
         return result
 
+    def baseline_cis(self, ci_ids, before_date, fl=None):
+        """
+               return CI changes
+               :param ci_ids:
+               :param before_date:
+               :param fl:
+               :return:
+               """
+        ci_list = self.get_cis_by_ids(ci_ids, fields=fl)
+        if not ci_list:
+            return []
+
+        id2ci = {ci['_id']: ci for ci in ci_list}
+        changed = AttributeHistoryManger.get_records_for_attributes(
+            before_date, None, None, 1, 100000, None, None, ci_ids=ci_ids, more=True)[1]
+        for records in changed:
+            for change in records[1]:
+                if change['is_computed'] or change['is_password']:
+                    continue
+
+                if change.get('default') and change['default'].get('default') == AttributeDefaultValueEnum.UPDATED_AT:
+                    continue
+
+                if change['is_list']:
+                    old, new, value_type, operate_type, ci_id, attr_name = (
+                        change['old'], change['new'], change['value_type'], change['operate_type'],
+                        change['ci_id'], change['attr_name'])
+                    old = ValueTypeMap.deserialize[value_type](old) if old else old
+                    new = ValueTypeMap.deserialize[value_type](new) if new else new
+                    if operate_type == OperateType.ADD and new in (id2ci[ci_id][attr_name] or []):
+                        id2ci[ci_id][attr_name].remove(new)
+                    elif operate_type == OperateType.DELETE and old not in id2ci[ci_id][attr_name]:
+                        id2ci[ci_id][attr_name].append(old)
+                else:
+                    id2ci[change['ci_id']][change['attr_name']] = change['old']
+
+        return list(id2ci.values())
+
     def rollback(self, ci_id, before_date):
         baseline_ci = self.baseline([ci_id], before_date)
 
@@ -1088,7 +1140,7 @@ class CIRelationManager(object):
                 relation_type_id or abort(404, ErrFormat.relation_not_found.format("{} -> {}".format(
                     first_ci.ci_type.name, second_ci.ci_type.name)))
 
-                if current_app.config.get('USE_ACL') and valid:
+                if current_app.config.get('USE_ACL') and valid and current_user.username != 'worker':
                     resource_name = CITypeRelationManager.acl_resource_name(first_ci.ci_type.name,
                                                                             second_ci.ci_type.name)
                     if not ACLManager().has_permission(
@@ -1122,7 +1174,7 @@ class CIRelationManager(object):
     def delete(cr_id):
         cr = CIRelation.get_by_id(cr_id) or abort(404, ErrFormat.relation_not_found.format("id={}".format(cr_id)))
 
-        if current_app.config.get('USE_ACL'):
+        if current_app.config.get('USE_ACL') and current_user.username != 'worker':
             resource_name = CITypeRelationManager.acl_resource_name(cr.first_ci.ci_type.name, cr.second_ci.ci_type.name)
             if not ACLManager().has_permission(
                     resource_name,
@@ -1156,6 +1208,21 @@ class CIRelationManager(object):
 
         return cr
 
+    @classmethod
+    def delete_3(cls, first_ci_id, second_ci_id):
+        cr = CIRelation.get_by(first_ci_id=first_ci_id,
+                               second_ci_id=second_ci_id,
+                               to_dict=False,
+                               first=True)
+
+        if cr is not None:
+            ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE)
+            delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE)
+
+            cls.delete(cr.id)
+
+        return cr
+
     @classmethod
     def batch_update(cls, ci_ids, parents, children, ancestor_ids=None):
         """

cmdb-api/api/lib/cmdb/ci_type.py

@@ -5,7 +5,6 @@ import copy
 import toposort
 from flask import abort
 from flask import current_app
-from flask import session
 from flask_login import current_user
 from toposort import toposort_flatten
 from werkzeug.exceptions import BadRequest
@@ -28,9 +27,11 @@ from api.lib.cmdb.perms import CIFilterPermsCRUD
 from api.lib.cmdb.relation_type import RelationTypeManager
 from api.lib.cmdb.resp_format import ErrFormat
 from api.lib.cmdb.value import AttributeValueManager
+from api.lib.common_setting.role_perm_base import CMDBApp
 from api.lib.decorator import kwargs_required
 from api.lib.perm.acl.acl import ACLManager
 from api.lib.perm.acl.acl import is_app_admin
+from api.lib.perm.acl.acl import validate_permission
 from api.models.cmdb import Attribute
 from api.models.cmdb import AutoDiscoveryCI
 from api.models.cmdb import AutoDiscoveryCIType
@@ -130,7 +131,9 @@ class CITypeManager(object):
     def add(cls, **kwargs):
         if current_app.config.get('USE_ACL') and not is_app_admin('cmdb'):
             if ErrFormat.ci_type_config not in {i['name'] for i in ACLManager().get_resources(ResourceTypeEnum.PAGE)}:
-                return abort(403, ErrFormat.no_permission2)
+                app_cli = CMDBApp()
+                validate_permission(app_cli.op.Model_Configuration, app_cli.resource_type_name,
+                                    app_cli.op.create_CIType, app_cli.app_name)
 
         unique_key = kwargs.pop("unique_key", None) or kwargs.pop("unique_id", None)
         unique_key = AttributeCache.get(unique_key) or abort(404, ErrFormat.unique_key_not_define)

cmdb-api/api/lib/cmdb/resp_format.py

@@ -140,3 +140,7 @@ class ErrFormat(CommonErrFormat):
 
     password_save_failed = _l("Failed to save password: {}")  # 保存密码失败: {}
     password_load_failed = _l("Failed to get password: {}")  # 获取密码失败: {}
+
+    cron_time_format_invalid = _l("Scheduling time format error")  # 调度时间格式错误
+    reconciliation_title = _l("CMDB data reconciliation results")  # CMDB数据合规检查结果
+    reconciliation_body = _l("Number of {} illegal: {}")  # "{} 不合规数: {}"

cmdb-api/api/lib/cmdb/search/ci/db/search.py

@@ -47,7 +47,8 @@ class Search(object):
                  excludes=None,
                  parent_node_perm_passed=False,
                  use_id_filter=False,
-                 use_ci_filter=True):
+                 use_ci_filter=True,
+                 only_ids=False):
         self.orig_query = query
         self.fl = fl or []
         self.excludes = excludes or []
@@ -64,6 +65,7 @@ class Search(object):
         self.parent_node_perm_passed = parent_node_perm_passed
         self.use_id_filter = use_id_filter
         self.use_ci_filter = use_ci_filter
+        self.only_ids = only_ids
 
         self.valid_type_names = []
         self.type2filter_perms = dict()
@@ -590,6 +592,8 @@ class Search(object):
     def search(self):
         numfound, ci_ids = self._query_build_raw()
         ci_ids = list(map(str, ci_ids))
+        if self.only_ids:
+            return ci_ids
 
         _fl = self._fl_build()
 

cmdb-api/api/lib/cmdb/search/ci_relation/search.py

@@ -69,7 +69,7 @@ class Search(object):
                     if _l < int(level) and c == ConstraintEnum.Many2Many:
                         self.has_m2m = True
 
-        self.type2filter_perms = None
+        self.type2filter_perms = {}
 
         self.is_app_admin = is_app_admin('cmdb') or current_user.username == "worker"
 
@@ -79,7 +79,7 @@ class Search(object):
         key = []
         _tmp = []
         for level in range(1, sorted(self.level)[-1] + 1):
-            if len(self.descendant_ids) >= level and self.type2filter_perms.get(self.descendant_ids[level - 1]):
+            if len(self.descendant_ids or []) >= level and self.type2filter_perms.get(self.descendant_ids[level - 1]):
                 id_filter_limit, _ = self._get_ci_filter(self.type2filter_perms[self.descendant_ids[level - 1]])
             else:
                 id_filter_limit = {}
@@ -151,9 +151,9 @@ class Search(object):
 
         return True
 
-    def search(self):
+    def search(self, only_ids=False):
-        use_ci_filter = len(self.descendant_ids) == self.level[0] - 1
+        use_ci_filter = len(self.descendant_ids or []) == self.level[0] - 1
-        parent_node_perm_passed = self._has_read_perm_from_parent_nodes()
+        parent_node_perm_passed = not self.is_app_admin and self._has_read_perm_from_parent_nodes()
 
         ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
         cis = [CI.get_by_id(_id) or abort(404, ErrFormat.ci_not_found.format("id={}".format(_id))) for _id in ids]
@@ -197,7 +197,8 @@ class Search(object):
                                 sort=self.sort,
                                 ci_ids=merge_ids,
                                 parent_node_perm_passed=parent_node_perm_passed,
-                                use_ci_filter=use_ci_filter).search()
+                                use_ci_filter=use_ci_filter,
+                                only_ids=only_ids).search()
 
     def _get_ci_filter(self, filter_perms, ci_filters=None):
         ci_filters = ci_filters or []

cmdb-api/api/lib/cmdb/value.py

@@ -274,7 +274,8 @@ class AttributeValueManager(object):
 
             if attr.is_list:
                 existed_attrs = value_table.get_by(attr_id=attr.id, ci_id=ci.id, to_dict=False)
-                existed_values = [i.value for i in existed_attrs]
+                existed_values = [(ValueTypeMap.serialize[attr.value_type](i.value) if
+                                   i.value or i.value == 0 else i.value) for i in existed_attrs]
 
                 # Comparison array starts from which position changes
                 min_len = min(len(value), len(existed_values))
@@ -283,17 +284,15 @@ class AttributeValueManager(object):
                     if value[index] != existed_values[index]:
                         break
                     index += 1
-                added = value[index:]
-                deleted = existed_values[index:]
 
                 # Delete first and then add to ensure id sorting
-                for v in deleted:
+                for idx in range(index, len(existed_attrs)):
-                    existed_attr = existed_attrs[existed_values.index(v)]
+                    existed_attr = existed_attrs[idx]
                     existed_attr.delete(flush=False, commit=False)
-                    changed.append((ci.id, attr.id, OperateType.DELETE, v, None, ci.type_id))
+                    changed.append((ci.id, attr.id, OperateType.DELETE, existed_values[idx], None, ci.type_id))
-                for v in added:
+                for idx in range(index, len(value)):
-                    value_table.create(ci_id=ci.id, attr_id=attr.id, value=v, flush=False, commit=False)
+                    value_table.create(ci_id=ci.id, attr_id=attr.id, value=value[idx], flush=False, commit=False)
-                    changed.append((ci.id, attr.id, OperateType.ADD, None, v, ci.type_id))
+                    changed.append((ci.id, attr.id, OperateType.ADD, None, value[idx], ci.type_id))
             else:
                 existed_attr = value_table.get_by(attr_id=attr.id, ci_id=ci.id, first=True, to_dict=False)
                 existed_value = existed_attr and existed_attr.value

cmdb-api/api/lib/perm/acl/acl.py

@@ -153,11 +153,11 @@ class ACLManager(object):
         if resource:
             return ResourceCRUD.delete(resource.id, rebuild=rebuild)
 
-    def has_permission(self, resource_name, resource_type, perm, resource_id=None):
+    def has_permission(self, resource_name, resource_type, perm, resource_id=None, rid=None):
         if is_app_admin(self.app_id):
             return True
 
-        role = self._get_role(current_user.username)
+        role = self._get_role(current_user.username) if rid is None else RoleCache.get(rid)
 
         role or abort(404, ErrFormat.role_not_found.format(current_user.username))
 

cmdb-api/api/lib/perm/acl/cache.py

@@ -3,6 +3,7 @@
 
 import msgpack
 import redis_lock
+from flask import current_app
 
 from api.extensions import cache
 from api.extensions import rd
@@ -157,9 +158,9 @@ class RoleRelationCache(object):
     PREFIX_RESOURCES2 = "RoleRelationResources2::id::{0}::AppId::{1}"
 
     @classmethod
-    def get_parent_ids(cls, rid, app_id):
+    def get_parent_ids(cls, rid, app_id, force=False):
         parent_ids = cache.get(cls.PREFIX_PARENT.format(rid, app_id))
-        if not parent_ids:
+        if not parent_ids or force:
             from api.lib.perm.acl.role import RoleRelationCRUD
             parent_ids = RoleRelationCRUD.get_parent_ids(rid, app_id)
             cache.set(cls.PREFIX_PARENT.format(rid, app_id), parent_ids, timeout=0)
@@ -167,9 +168,9 @@ class RoleRelationCache(object):
         return parent_ids
 
     @classmethod
-    def get_child_ids(cls, rid, app_id):
+    def get_child_ids(cls, rid, app_id, force=False):
         child_ids = cache.get(cls.PREFIX_CHILDREN.format(rid, app_id))
-        if not child_ids:
+        if not child_ids or force:
             from api.lib.perm.acl.role import RoleRelationCRUD
             child_ids = RoleRelationCRUD.get_child_ids(rid, app_id)
             cache.set(cls.PREFIX_CHILDREN.format(rid, app_id), child_ids, timeout=0)
@@ -177,14 +178,15 @@ class RoleRelationCache(object):
         return child_ids
 
     @classmethod
-    def get_resources(cls, rid, app_id):
+    def get_resources(cls, rid, app_id, force=False):
         """
         :param rid: 
         :param app_id: 
+        :param force:
         :return: {id2perms: {resource_id: [perm,]}, group2perms: {group_id: [perm, ]}}
         """
         resources = cache.get(cls.PREFIX_RESOURCES.format(rid, app_id))
-        if not resources:
+        if not resources or force:
             from api.lib.perm.acl.role import RoleCRUD
             resources = RoleCRUD.get_resources(rid, app_id)
             if resources['id2perms'] or resources['group2perms']:
@@ -193,9 +195,9 @@ class RoleRelationCache(object):
         return resources or {}
 
     @classmethod
-    def get_resources2(cls, rid, app_id):
+    def get_resources2(cls, rid, app_id, force=False):
         r_g = cache.get(cls.PREFIX_RESOURCES2.format(rid, app_id))
-        if not r_g:
+        if not r_g or force:
             res = cls.get_resources(rid, app_id)
             id2perms = res['id2perms']
             group2perms = res['group2perms']
@@ -232,20 +234,20 @@ class RoleRelationCache(object):
         for _app_id in app_ids:
             cls.clean(rid, _app_id)
 
-            cls.get_parent_ids(rid, _app_id)
+            cls.get_parent_ids(rid, _app_id, force=True)
-            cls.get_child_ids(rid, _app_id)
+            cls.get_child_ids(rid, _app_id, force=True)
-            resources = cls.get_resources(rid, _app_id)
+            resources = cls.get_resources(rid, _app_id, force=True)
             if resources.get('id2perms') or resources.get('group2perms'):
                 HasResourceRoleCache.add(rid, _app_id)
             else:
                 HasResourceRoleCache.remove(rid, _app_id)
-            cls.get_resources2(rid, _app_id)
+            cls.get_resources2(rid, _app_id, force=True)
 
     @classmethod
     @flush_db
     def rebuild2(cls, rid, app_id):
         cache.delete(cls.PREFIX_RESOURCES2.format(rid, app_id))
-        cls.get_resources2(rid, app_id)
+        cls.get_resources2(rid, app_id, force=True)
 
     @classmethod
     def clean(cls, rid, app_id):

cmdb-api/api/lib/perm/acl/role.py

@@ -379,16 +379,16 @@ class RoleCRUD(object):
             resource_type_id = resource_type and resource_type.id
 
         result = dict(resources=dict(), groups=dict())
-        s = time.time()
+        # s = time.time()
         parent_ids = RoleRelationCRUD.recursive_parent_ids(rid, app_id)
-        current_app.logger.info('parent ids {0}: {1}'.format(parent_ids, time.time() - s))
+        # current_app.logger.info('parent ids {0}: {1}'.format(parent_ids, time.time() - s))
         for parent_id in parent_ids:
 
             _resources, _groups = cls._extend_resources(parent_id, resource_type_id, app_id)
-            current_app.logger.info('middle1: {0}'.format(time.time() - s))
+            # current_app.logger.info('middle1: {0}'.format(time.time() - s))
             _merge(result['resources'], _resources)
-            current_app.logger.info('middle2: {0}'.format(time.time() - s))
+            # current_app.logger.info('middle2: {0}'.format(time.time() - s))
-            current_app.logger.info(len(_groups))
+            # current_app.logger.info(len(_groups))
             if not group_flat:
                 _merge(result['groups'], _groups)
             else:
@@ -399,7 +399,7 @@ class RoleCRUD(object):
                             item.setdefault('permissions', [])
                             item['permissions'] = list(set(item['permissions'] + _groups[rg_id]['permissions']))
                             result['resources'][item['id']] = item
-        current_app.logger.info('End: {0}'.format(time.time() - s))
+        # current_app.logger.info('End: {0}'.format(time.time() - s))
 
         result['resources'] = list(result['resources'].values())
         result['groups'] = list(result['groups'].values())