From fe63310c4eb9739edc483d97f4324d3651b5a48f Mon Sep 17 00:00:00 2001 From: pycook Date: Fri, 17 May 2024 11:20:53 +0800 Subject: [PATCH] Dev api 240517 (#511) * fix(api): list values delete * fix(acl): role rebuild cache --- cmdb-api/api/lib/cmdb/ci.py | 77 +++++++++++++++++-- cmdb-api/api/lib/cmdb/ci_type.py | 7 +- cmdb-api/api/lib/cmdb/resp_format.py | 4 + cmdb-api/api/lib/cmdb/search/ci/db/search.py | 6 +- .../api/lib/cmdb/search/ci_relation/search.py | 13 ++-- cmdb-api/api/lib/cmdb/value.py | 17 ++-- cmdb-api/api/lib/perm/acl/acl.py | 4 +- cmdb-api/api/lib/perm/acl/cache.py | 28 +++---- cmdb-api/api/lib/perm/acl/role.py | 12 +-- 9 files changed, 124 insertions(+), 44 deletions(-) diff --git a/cmdb-api/api/lib/cmdb/ci.py b/cmdb-api/api/lib/cmdb/ci.py index c1b56f0..62290cd 100644 --- a/cmdb-api/api/lib/cmdb/ci.py +++ b/cmdb-api/api/lib/cmdb/ci.py @@ -263,6 +263,7 @@ class CIManager(object): ci_ids = None for attr_id in constraint.attr_ids: value_table = TableMap(attr_name=id2name[attr_id]).table + values = value_table.get_by(attr_id=attr_id, value=ci_dict.get(id2name[attr_id]) or None, only_query=True).join( @@ -438,7 +439,8 @@ class CIManager(object): return ci.id - def update(self, ci_id, _is_admin=False, ticket_id=None, **ci_dict): + def update(self, ci_id, _is_admin=False, ticket_id=None, __sync=False, **ci_dict): + current_app.logger.info((ci_id, ci_dict, __sync)) now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') ci = self.confirm_ci_existed(ci_id) @@ -502,11 +504,17 @@ class CIManager(object): record_id = self.save_password(ci.id, attr_id, password_dict[attr_id], record_id, ci.type_id) if record_id: # has change - ci_cache.apply_async(args=(ci_id, OperateType.UPDATE, record_id), queue=CMDB_QUEUE) + if not __sync: + ci_cache.apply_async(args=(ci_id, OperateType.UPDATE, record_id), queue=CMDB_QUEUE) + else: + ci_cache((ci_id, OperateType.UPDATE, record_id)) ref_ci_dict = {k: v for k, v in ci_dict.items() if k.startswith("$") and "." in k} if ref_ci_dict: - ci_relation_add.apply_async(args=(ref_ci_dict, ci.id), queue=CMDB_QUEUE) + if not __sync: + ci_relation_add.apply_async(args=(ref_ci_dict, ci.id), queue=CMDB_QUEUE) + else: + ci_relation_add((ref_ci_dict, ci.id)) @staticmethod def update_unique_value(ci_id, unique_name, unique_value): @@ -830,6 +838,12 @@ class CIManager(object): return data.get('v') def baseline(self, ci_ids, before_date): + """ + return CI changes + :param ci_ids: + :param before_date: + :return: + """ ci_list = self.get_cis_by_ids(ci_ids, ret_key=RetKey.ALIAS) if not ci_list: return dict() @@ -913,6 +927,44 @@ class CIManager(object): return result + def baseline_cis(self, ci_ids, before_date, fl=None): + """ + return CI changes + :param ci_ids: + :param before_date: + :param fl: + :return: + """ + ci_list = self.get_cis_by_ids(ci_ids, fields=fl) + if not ci_list: + return [] + + id2ci = {ci['_id']: ci for ci in ci_list} + changed = AttributeHistoryManger.get_records_for_attributes( + before_date, None, None, 1, 100000, None, None, ci_ids=ci_ids, more=True)[1] + for records in changed: + for change in records[1]: + if change['is_computed'] or change['is_password']: + continue + + if change.get('default') and change['default'].get('default') == AttributeDefaultValueEnum.UPDATED_AT: + continue + + if change['is_list']: + old, new, value_type, operate_type, ci_id, attr_name = ( + change['old'], change['new'], change['value_type'], change['operate_type'], + change['ci_id'], change['attr_name']) + old = ValueTypeMap.deserialize[value_type](old) if old else old + new = ValueTypeMap.deserialize[value_type](new) if new else new + if operate_type == OperateType.ADD and new in (id2ci[ci_id][attr_name] or []): + id2ci[ci_id][attr_name].remove(new) + elif operate_type == OperateType.DELETE and old not in id2ci[ci_id][attr_name]: + id2ci[ci_id][attr_name].append(old) + else: + id2ci[change['ci_id']][change['attr_name']] = change['old'] + + return list(id2ci.values()) + def rollback(self, ci_id, before_date): baseline_ci = self.baseline([ci_id], before_date) @@ -1088,7 +1140,7 @@ class CIRelationManager(object): relation_type_id or abort(404, ErrFormat.relation_not_found.format("{} -> {}".format( first_ci.ci_type.name, second_ci.ci_type.name))) - if current_app.config.get('USE_ACL') and valid: + if current_app.config.get('USE_ACL') and valid and current_user.username != 'worker': resource_name = CITypeRelationManager.acl_resource_name(first_ci.ci_type.name, second_ci.ci_type.name) if not ACLManager().has_permission( @@ -1122,7 +1174,7 @@ class CIRelationManager(object): def delete(cr_id): cr = CIRelation.get_by_id(cr_id) or abort(404, ErrFormat.relation_not_found.format("id={}".format(cr_id))) - if current_app.config.get('USE_ACL'): + if current_app.config.get('USE_ACL') and current_user.username != 'worker': resource_name = CITypeRelationManager.acl_resource_name(cr.first_ci.ci_type.name, cr.second_ci.ci_type.name) if not ACLManager().has_permission( resource_name, @@ -1156,6 +1208,21 @@ class CIRelationManager(object): return cr + @classmethod + def delete_3(cls, first_ci_id, second_ci_id): + cr = CIRelation.get_by(first_ci_id=first_ci_id, + second_ci_id=second_ci_id, + to_dict=False, + first=True) + + if cr is not None: + ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE) + delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE) + + cls.delete(cr.id) + + return cr + @classmethod def batch_update(cls, ci_ids, parents, children, ancestor_ids=None): """ diff --git a/cmdb-api/api/lib/cmdb/ci_type.py b/cmdb-api/api/lib/cmdb/ci_type.py index 35a7396..06d4eb3 100644 --- a/cmdb-api/api/lib/cmdb/ci_type.py +++ b/cmdb-api/api/lib/cmdb/ci_type.py @@ -5,7 +5,6 @@ import copy import toposort from flask import abort from flask import current_app -from flask import session from flask_login import current_user from toposort import toposort_flatten from werkzeug.exceptions import BadRequest @@ -28,9 +27,11 @@ from api.lib.cmdb.perms import CIFilterPermsCRUD from api.lib.cmdb.relation_type import RelationTypeManager from api.lib.cmdb.resp_format import ErrFormat from api.lib.cmdb.value import AttributeValueManager +from api.lib.common_setting.role_perm_base import CMDBApp from api.lib.decorator import kwargs_required from api.lib.perm.acl.acl import ACLManager from api.lib.perm.acl.acl import is_app_admin +from api.lib.perm.acl.acl import validate_permission from api.models.cmdb import Attribute from api.models.cmdb import AutoDiscoveryCI from api.models.cmdb import AutoDiscoveryCIType @@ -130,7 +131,9 @@ class CITypeManager(object): def add(cls, **kwargs): if current_app.config.get('USE_ACL') and not is_app_admin('cmdb'): if ErrFormat.ci_type_config not in {i['name'] for i in ACLManager().get_resources(ResourceTypeEnum.PAGE)}: - return abort(403, ErrFormat.no_permission2) + app_cli = CMDBApp() + validate_permission(app_cli.op.Model_Configuration, app_cli.resource_type_name, + app_cli.op.create_CIType, app_cli.app_name) unique_key = kwargs.pop("unique_key", None) or kwargs.pop("unique_id", None) unique_key = AttributeCache.get(unique_key) or abort(404, ErrFormat.unique_key_not_define) diff --git a/cmdb-api/api/lib/cmdb/resp_format.py b/cmdb-api/api/lib/cmdb/resp_format.py index 1d72ea3..98c11c9 100644 --- a/cmdb-api/api/lib/cmdb/resp_format.py +++ b/cmdb-api/api/lib/cmdb/resp_format.py @@ -140,3 +140,7 @@ class ErrFormat(CommonErrFormat): password_save_failed = _l("Failed to save password: {}") # 保存密码失败: {} password_load_failed = _l("Failed to get password: {}") # 获取密码失败: {} + + cron_time_format_invalid = _l("Scheduling time format error") # 调度时间格式错误 + reconciliation_title = _l("CMDB data reconciliation results") # CMDB数据合规检查结果 + reconciliation_body = _l("Number of {} illegal: {}") # "{} 不合规数: {}" diff --git a/cmdb-api/api/lib/cmdb/search/ci/db/search.py b/cmdb-api/api/lib/cmdb/search/ci/db/search.py index dfc806b..4fb3b50 100644 --- a/cmdb-api/api/lib/cmdb/search/ci/db/search.py +++ b/cmdb-api/api/lib/cmdb/search/ci/db/search.py @@ -47,7 +47,8 @@ class Search(object): excludes=None, parent_node_perm_passed=False, use_id_filter=False, - use_ci_filter=True): + use_ci_filter=True, + only_ids=False): self.orig_query = query self.fl = fl or [] self.excludes = excludes or [] @@ -64,6 +65,7 @@ class Search(object): self.parent_node_perm_passed = parent_node_perm_passed self.use_id_filter = use_id_filter self.use_ci_filter = use_ci_filter + self.only_ids = only_ids self.valid_type_names = [] self.type2filter_perms = dict() @@ -590,6 +592,8 @@ class Search(object): def search(self): numfound, ci_ids = self._query_build_raw() ci_ids = list(map(str, ci_ids)) + if self.only_ids: + return ci_ids _fl = self._fl_build() diff --git a/cmdb-api/api/lib/cmdb/search/ci_relation/search.py b/cmdb-api/api/lib/cmdb/search/ci_relation/search.py index 97f8ff6..1b10c96 100644 --- a/cmdb-api/api/lib/cmdb/search/ci_relation/search.py +++ b/cmdb-api/api/lib/cmdb/search/ci_relation/search.py @@ -69,7 +69,7 @@ class Search(object): if _l < int(level) and c == ConstraintEnum.Many2Many: self.has_m2m = True - self.type2filter_perms = None + self.type2filter_perms = {} self.is_app_admin = is_app_admin('cmdb') or current_user.username == "worker" @@ -79,7 +79,7 @@ class Search(object): key = [] _tmp = [] for level in range(1, sorted(self.level)[-1] + 1): - if len(self.descendant_ids) >= level and self.type2filter_perms.get(self.descendant_ids[level - 1]): + if len(self.descendant_ids or []) >= level and self.type2filter_perms.get(self.descendant_ids[level - 1]): id_filter_limit, _ = self._get_ci_filter(self.type2filter_perms[self.descendant_ids[level - 1]]) else: id_filter_limit = {} @@ -151,9 +151,9 @@ class Search(object): return True - def search(self): - use_ci_filter = len(self.descendant_ids) == self.level[0] - 1 - parent_node_perm_passed = self._has_read_perm_from_parent_nodes() + def search(self, only_ids=False): + use_ci_filter = len(self.descendant_ids or []) == self.level[0] - 1 + parent_node_perm_passed = not self.is_app_admin and self._has_read_perm_from_parent_nodes() ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id cis = [CI.get_by_id(_id) or abort(404, ErrFormat.ci_not_found.format("id={}".format(_id))) for _id in ids] @@ -197,7 +197,8 @@ class Search(object): sort=self.sort, ci_ids=merge_ids, parent_node_perm_passed=parent_node_perm_passed, - use_ci_filter=use_ci_filter).search() + use_ci_filter=use_ci_filter, + only_ids=only_ids).search() def _get_ci_filter(self, filter_perms, ci_filters=None): ci_filters = ci_filters or [] diff --git a/cmdb-api/api/lib/cmdb/value.py b/cmdb-api/api/lib/cmdb/value.py index 4ac709e..96d7c85 100644 --- a/cmdb-api/api/lib/cmdb/value.py +++ b/cmdb-api/api/lib/cmdb/value.py @@ -274,7 +274,8 @@ class AttributeValueManager(object): if attr.is_list: existed_attrs = value_table.get_by(attr_id=attr.id, ci_id=ci.id, to_dict=False) - existed_values = [i.value for i in existed_attrs] + existed_values = [(ValueTypeMap.serialize[attr.value_type](i.value) if + i.value or i.value == 0 else i.value) for i in existed_attrs] # Comparison array starts from which position changes min_len = min(len(value), len(existed_values)) @@ -283,17 +284,15 @@ class AttributeValueManager(object): if value[index] != existed_values[index]: break index += 1 - added = value[index:] - deleted = existed_values[index:] # Delete first and then add to ensure id sorting - for v in deleted: - existed_attr = existed_attrs[existed_values.index(v)] + for idx in range(index, len(existed_attrs)): + existed_attr = existed_attrs[idx] existed_attr.delete(flush=False, commit=False) - changed.append((ci.id, attr.id, OperateType.DELETE, v, None, ci.type_id)) - for v in added: - value_table.create(ci_id=ci.id, attr_id=attr.id, value=v, flush=False, commit=False) - changed.append((ci.id, attr.id, OperateType.ADD, None, v, ci.type_id)) + changed.append((ci.id, attr.id, OperateType.DELETE, existed_values[idx], None, ci.type_id)) + for idx in range(index, len(value)): + value_table.create(ci_id=ci.id, attr_id=attr.id, value=value[idx], flush=False, commit=False) + changed.append((ci.id, attr.id, OperateType.ADD, None, value[idx], ci.type_id)) else: existed_attr = value_table.get_by(attr_id=attr.id, ci_id=ci.id, first=True, to_dict=False) existed_value = existed_attr and existed_attr.value diff --git a/cmdb-api/api/lib/perm/acl/acl.py b/cmdb-api/api/lib/perm/acl/acl.py index 85bf1e9..a829401 100644 --- a/cmdb-api/api/lib/perm/acl/acl.py +++ b/cmdb-api/api/lib/perm/acl/acl.py @@ -153,11 +153,11 @@ class ACLManager(object): if resource: return ResourceCRUD.delete(resource.id, rebuild=rebuild) - def has_permission(self, resource_name, resource_type, perm, resource_id=None): + def has_permission(self, resource_name, resource_type, perm, resource_id=None, rid=None): if is_app_admin(self.app_id): return True - role = self._get_role(current_user.username) + role = self._get_role(current_user.username) if rid is None else RoleCache.get(rid) role or abort(404, ErrFormat.role_not_found.format(current_user.username)) diff --git a/cmdb-api/api/lib/perm/acl/cache.py b/cmdb-api/api/lib/perm/acl/cache.py index dcf9543..8dffa0b 100644 --- a/cmdb-api/api/lib/perm/acl/cache.py +++ b/cmdb-api/api/lib/perm/acl/cache.py @@ -3,6 +3,7 @@ import msgpack import redis_lock +from flask import current_app from api.extensions import cache from api.extensions import rd @@ -157,9 +158,9 @@ class RoleRelationCache(object): PREFIX_RESOURCES2 = "RoleRelationResources2::id::{0}::AppId::{1}" @classmethod - def get_parent_ids(cls, rid, app_id): + def get_parent_ids(cls, rid, app_id, force=False): parent_ids = cache.get(cls.PREFIX_PARENT.format(rid, app_id)) - if not parent_ids: + if not parent_ids or force: from api.lib.perm.acl.role import RoleRelationCRUD parent_ids = RoleRelationCRUD.get_parent_ids(rid, app_id) cache.set(cls.PREFIX_PARENT.format(rid, app_id), parent_ids, timeout=0) @@ -167,9 +168,9 @@ class RoleRelationCache(object): return parent_ids @classmethod - def get_child_ids(cls, rid, app_id): + def get_child_ids(cls, rid, app_id, force=False): child_ids = cache.get(cls.PREFIX_CHILDREN.format(rid, app_id)) - if not child_ids: + if not child_ids or force: from api.lib.perm.acl.role import RoleRelationCRUD child_ids = RoleRelationCRUD.get_child_ids(rid, app_id) cache.set(cls.PREFIX_CHILDREN.format(rid, app_id), child_ids, timeout=0) @@ -177,14 +178,15 @@ class RoleRelationCache(object): return child_ids @classmethod - def get_resources(cls, rid, app_id): + def get_resources(cls, rid, app_id, force=False): """ :param rid: :param app_id: + :param force: :return: {id2perms: {resource_id: [perm,]}, group2perms: {group_id: [perm, ]}} """ resources = cache.get(cls.PREFIX_RESOURCES.format(rid, app_id)) - if not resources: + if not resources or force: from api.lib.perm.acl.role import RoleCRUD resources = RoleCRUD.get_resources(rid, app_id) if resources['id2perms'] or resources['group2perms']: @@ -193,9 +195,9 @@ class RoleRelationCache(object): return resources or {} @classmethod - def get_resources2(cls, rid, app_id): + def get_resources2(cls, rid, app_id, force=False): r_g = cache.get(cls.PREFIX_RESOURCES2.format(rid, app_id)) - if not r_g: + if not r_g or force: res = cls.get_resources(rid, app_id) id2perms = res['id2perms'] group2perms = res['group2perms'] @@ -232,20 +234,20 @@ class RoleRelationCache(object): for _app_id in app_ids: cls.clean(rid, _app_id) - cls.get_parent_ids(rid, _app_id) - cls.get_child_ids(rid, _app_id) - resources = cls.get_resources(rid, _app_id) + cls.get_parent_ids(rid, _app_id, force=True) + cls.get_child_ids(rid, _app_id, force=True) + resources = cls.get_resources(rid, _app_id, force=True) if resources.get('id2perms') or resources.get('group2perms'): HasResourceRoleCache.add(rid, _app_id) else: HasResourceRoleCache.remove(rid, _app_id) - cls.get_resources2(rid, _app_id) + cls.get_resources2(rid, _app_id, force=True) @classmethod @flush_db def rebuild2(cls, rid, app_id): cache.delete(cls.PREFIX_RESOURCES2.format(rid, app_id)) - cls.get_resources2(rid, app_id) + cls.get_resources2(rid, app_id, force=True) @classmethod def clean(cls, rid, app_id): diff --git a/cmdb-api/api/lib/perm/acl/role.py b/cmdb-api/api/lib/perm/acl/role.py index 15569df..b662135 100644 --- a/cmdb-api/api/lib/perm/acl/role.py +++ b/cmdb-api/api/lib/perm/acl/role.py @@ -379,16 +379,16 @@ class RoleCRUD(object): resource_type_id = resource_type and resource_type.id result = dict(resources=dict(), groups=dict()) - s = time.time() + # s = time.time() parent_ids = RoleRelationCRUD.recursive_parent_ids(rid, app_id) - current_app.logger.info('parent ids {0}: {1}'.format(parent_ids, time.time() - s)) + # current_app.logger.info('parent ids {0}: {1}'.format(parent_ids, time.time() - s)) for parent_id in parent_ids: _resources, _groups = cls._extend_resources(parent_id, resource_type_id, app_id) - current_app.logger.info('middle1: {0}'.format(time.time() - s)) + # current_app.logger.info('middle1: {0}'.format(time.time() - s)) _merge(result['resources'], _resources) - current_app.logger.info('middle2: {0}'.format(time.time() - s)) - current_app.logger.info(len(_groups)) + # current_app.logger.info('middle2: {0}'.format(time.time() - s)) + # current_app.logger.info(len(_groups)) if not group_flat: _merge(result['groups'], _groups) else: @@ -399,7 +399,7 @@ class RoleCRUD(object): item.setdefault('permissions', []) item['permissions'] = list(set(item['permissions'] + _groups[rg_id]['permissions'])) result['resources'][item['id']] = item - current_app.logger.info('End: {0}'.format(time.time() - s)) + # current_app.logger.info('End: {0}'.format(time.time() - s)) result['resources'] = list(result['resources'].values()) result['groups'] = list(result['groups'].values())