From 11a289aac98a71cd963a39ff64edbe7a6d0d95f1 Mon Sep 17 00:00:00 2001 From: pycook Date: Mon, 26 Aug 2024 12:14:14 +0800 Subject: [PATCH] feat(api): enum supports --- cmdb-api/api/lib/cmdb/attribute.py | 9 +++++++ cmdb-api/api/lib/cmdb/cache.py | 38 +++++++++++++++++--------- cmdb-api/api/lib/cmdb/ci.py | 21 ++++++++++----- cmdb-api/api/lib/cmdb/ci_type.py | 41 +++++++++++++++++++---------- cmdb-api/api/lib/cmdb/preference.py | 7 ++--- cmdb-api/api/lib/cmdb/value.py | 17 ++++++++---- cmdb-api/api/tasks/cmdb.py | 15 +++++++++-- 7 files changed, 106 insertions(+), 42 deletions(-) diff --git a/cmdb-api/api/lib/cmdb/attribute.py b/cmdb-api/api/lib/cmdb/attribute.py index a7579dc..2049b74 100644 --- a/cmdb-api/api/lib/cmdb/attribute.py +++ b/cmdb-api/api/lib/cmdb/attribute.py @@ -135,6 +135,15 @@ class AttributeManager(object): choice_table and choice_table.get_by(attr_id=_id, only_query=True).delete() db.session.flush() + @classmethod + def get_enum_map(cls, _attr_id, _attr=None): + attr = AttributeCache.get(_attr_id) if _attr_id else _attr + if attr and attr.is_choice: + choice_values = cls.get_choice_values(attr.id, attr.value_type, None, None) + return {i[0]: i[1]['label'] for i in choice_values if i[1].get('label')} + + return {} + @classmethod def search_attributes(cls, name=None, alias=None, page=1, page_size=None): """ diff --git a/cmdb-api/api/lib/cmdb/cache.py b/cmdb-api/api/lib/cmdb/cache.py index d11a028..4c939b1 100644 --- a/cmdb-api/api/lib/cmdb/cache.py +++ b/cmdb-api/api/lib/cmdb/cache.py @@ -2,6 +2,8 @@ from __future__ import unicode_literals +from collections import defaultdict + import datetime import os import yaml @@ -299,11 +301,12 @@ class CMDBCounterCache(object): return res - @staticmethod - def relation_counter(type_id, level, other_filer, type_ids): + @classmethod + def relation_counter(cls, type_id, level, other_filer, type_ids): from api.lib.cmdb.search.ci_relation.search import Search as RelSearch from api.lib.cmdb.search import SearchError from api.lib.cmdb.search.ci import search + from api.lib.cmdb.attribute import AttributeManager query = "_type:{}".format(type_id) if other_filer: @@ -318,8 +321,12 @@ class CMDBCounterCache(object): show_attr_id = root_type and root_type.show_id show_attr = AttributeCache.get(show_attr_id) - type_id_names = [(str(i.get('_id')), i.get(show_attr and show_attr.name) or i.get(i.get('unique'))) - for i in type_names] + type_id_names = [] + for i in type_names: + attr_value = i.get(show_attr and show_attr.name) or i.get(i.get('unique')) + enum_map = AttributeManager.get_enum_map(show_attr_id or i.get('unique')) + + type_id_names.append((str(i.get('_id')), enum_map.get(attr_value, attr_value))) s = RelSearch([i[0] for i in type_id_names], level) try: @@ -351,11 +358,12 @@ class CMDBCounterCache(object): return result - @staticmethod - def attribute_counter(custom): + @classmethod + def attribute_counter(cls, custom): from api.lib.cmdb.search import SearchError from api.lib.cmdb.search.ci import search from api.lib.cmdb.utils import ValueTypeMap + from api.lib.cmdb.attribute import AttributeManager custom.setdefault('options', {}) type_id = custom.get('type_id') @@ -390,12 +398,16 @@ class CMDBCounterCache(object): except SearchError as e: current_app.logger.error(e) return + + enum_map = AttributeManager.get_enum_map(attr_ids[0]) for i in (list(facet.values()) or [[]])[0]: - result[ValueTypeMap.serialize2[attr2value_type[0]](str(i[0]))] = i[1] + k = ValueTypeMap.serialize2[attr2value_type[0]](str(i[0])) + result[enum_map.get(k, k)] = i[1] if len(attr_ids) == 1: return result # level = 2 + enum_map = AttributeManager.get_enum_map(attr_ids[1]) for v in result: query = "_type:({}),{},{}:{}".format(";".join(map(str, type_ids)), other_filter, attr_ids[0], v) s = search(query, fl=attr_ids, facet=[attr_ids[1]], count=1) @@ -406,12 +418,14 @@ class CMDBCounterCache(object): return result[v] = dict() for i in (list(facet.values()) or [[]])[0]: - result[v][ValueTypeMap.serialize2[attr2value_type[1]](str(i[0]))] = i[1] + k = ValueTypeMap.serialize2[attr2value_type[0]](str(i[0])) + result[v][enum_map.get(k, k)] = i[1] if len(attr_ids) == 2: return result # level = 3 + enum_map = AttributeManager.get_enum_map(attr_ids[2]) for v1 in result: if not isinstance(result[v1], dict): continue @@ -426,7 +440,8 @@ class CMDBCounterCache(object): return result[v1][v2] = dict() for i in (list(facet.values()) or [[]])[0]: - result[v1][v2][ValueTypeMap.serialize2[attr2value_type[2]](str(i[0]))] = i[1] + k = ValueTypeMap.serialize2[attr2value_type[2]](str(i[0])) + result[v1][v2][enum_map.get(k, k)] = i[1] return result @@ -530,19 +545,18 @@ class CMDBCounterCache(object): @classmethod def flush_sub_counter(cls): - result = dict(type_id2users=dict()) + result = dict(type_id2users=defaultdict(list)) types = db.session.query(PreferenceShowAttributes.type_id, PreferenceShowAttributes.uid, PreferenceShowAttributes.created_at).filter( PreferenceShowAttributes.deleted.is_(False)).group_by( PreferenceShowAttributes.uid, PreferenceShowAttributes.type_id) for i in types: - result['type_id2users'].setdefault(i.type_id, []).append(i.uid) + result['type_id2users'][i.type_id].append(i.uid) types = PreferenceTreeView.get_by(to_dict=False) for i in types: - result['type_id2users'].setdefault(i.type_id, []) if i.uid not in result['type_id2users'][i.type_id]: result['type_id2users'][i.type_id].append(i.uid) diff --git a/cmdb-api/api/lib/cmdb/ci.py b/cmdb-api/api/lib/cmdb/ci.py index a6d5c76..b5e7084 100644 --- a/cmdb-api/api/lib/cmdb/ci.py +++ b/cmdb-api/api/lib/cmdb/ci.py @@ -161,7 +161,7 @@ class CIManager(object): @classmethod def get_ci_by_id_from_db(cls, ci_id, ret_key=RetKey.NAME, fields=None, need_children=True, use_master=False, - valid=False): + valid=False, enum_use_label=False): """ :param ci_id: @@ -170,6 +170,7 @@ class CIManager(object): :param need_children: :param use_master: whether to use master db :param valid: + :param enum_use_label: :return: """ @@ -187,13 +188,19 @@ class CIManager(object): res["ci_type"] = ci_type.name - fields = CITypeAttributeManager.get_attr_names_by_type_id(ci.type_id) if not fields else fields + enum_map = dict() + if not enum_use_label: + fields = CITypeAttributeManager.get_attr_names_by_type_id(ci.type_id) if not fields else fields + else: + fields, enum_map = CITypeAttributeManager.get_attr_names_label_enum( + ci.type_id) if not fields else (fields, {}) unique_key = AttributeCache.get(ci_type.unique_id) _res = AttributeValueManager().get_attr_values(fields, ci_id, ret_key=ret_key, unique_key=unique_key, - use_master=use_master) + use_master=use_master, + enum_map=enum_map) res.update(_res) res['_type'] = ci_type.id @@ -376,7 +383,7 @@ class CIManager(object): ci_type_attrs_alias = {attr.alias: attr for _, attr in attrs} ci_attr2type_attr = {type_attr.attr_id: type_attr for type_attr, _ in attrs} ci_type_attrs_name_alias = {**ci_type_attrs_name, **ci_type_attrs_alias} - + ci = None record_id = None password_dict = {} @@ -1509,7 +1516,8 @@ class CITriggerManager(object): cls._update_old_attr_value(record_id, ci_dict) if ci_id is not None: - ci_dict = CIManager().get_ci_by_id_from_db(ci_id, need_children=False, use_master=False) + ci_dict = CIManager().get_ci_by_id_from_db( + ci_id, need_children=False, use_master=False, enum_use_label=True) try: response = webhook_request(webhook, ci_dict).text @@ -1536,7 +1544,8 @@ class CITriggerManager(object): with app.app_context(): if ci_id is not None: - ci_dict = CIManager().get_ci_by_id_from_db(ci_id, need_children=False, use_master=False) + ci_dict = CIManager().get_ci_by_id_from_db( + ci_id, need_children=False, use_master=False, enum_use_label=True) if operate_type == OperateType.UPDATE: cls._update_old_attr_value(record_id, ci_dict) diff --git a/cmdb-api/api/lib/cmdb/ci_type.py b/cmdb-api/api/lib/cmdb/ci_type.py index 8b98316..5d5120e 100644 --- a/cmdb-api/api/lib/cmdb/ci_type.py +++ b/cmdb-api/api/lib/cmdb/ci_type.py @@ -1,5 +1,7 @@ # -*- coding:utf-8 -*- +from collections import defaultdict + import copy import toposort from flask import abort @@ -347,9 +349,9 @@ class CITypeInheritanceManager(object): @classmethod def add(cls, parent_ids, child_id): - rels = {} + rels = defaultdict(set) for i in cls.cls.get_by(to_dict=False): - rels.setdefault(i.child_id, set()).add(i.parent_id) + rels[i.child_id].add(i.parent_id) try: toposort_flatten(rels) @@ -363,7 +365,7 @@ class CITypeInheritanceManager(object): existed = cls.cls.get_by(parent_id=parent_id, child_id=child_id, first=True, to_dict=False) if existed is None: - rels.setdefault(child_id, set()).add(parent_id) + rels[child_id].add(parent_id) try: toposort_flatten(rels) except toposort.CircularDependencyError as e: @@ -504,14 +506,13 @@ class CITypeAttributeManager(object): def __init__(self): pass - @staticmethod - def get_attr_name(ci_type_name, key): + @classmethod + def get_attr_name(cls, ci_type_name, key): ci_type = CITypeCache.get(ci_type_name) if ci_type is None: return - for i in CITypeAttributesCache.get(ci_type.id): - attr = AttributeCache.get(i.attr_id) + for _, attr in cls.get_all_attributes(ci_type.id): if attr and (attr.name == key or attr.alias == key): return attr.name @@ -529,6 +530,18 @@ class CITypeAttributeManager(object): def get_attr_names_by_type_id(cls, type_id): return [attr.name for _, attr in cls.get_all_attributes(type_id)] + @classmethod + def get_attr_names_label_enum(cls, type_id): + attr_names, enum_map = list(), defaultdict(dict) + for _, attr in cls.get_all_attributes(type_id): + attr_names.append(attr.name) + if attr.is_choice and not attr.choice_other and not attr.choice_web_hook: + _map = AttributeManager.get_enum_map(attr.id) + if _map: + enum_map[attr.name].update(_map) + + return attr_names, enum_map + @staticmethod def get_attributes_by_type_id(type_id, choice_web_hook_parse=True, choice_other_parse=True): has_config_perm = ACLManager('cmdb').has_permission( @@ -569,10 +582,10 @@ class CITypeAttributeManager(object): CITypeManager.get_name_by_id(type_id), ResourceTypeEnum.CI, PermEnum.CONFIG) result = {type_id: [i for _, i in cls.get_all_attributes(type_id)] for type_id in type_ids} - attr2types = {} + attr2types = defaultdict(list) for type_id in result: for i in result[type_id]: - attr2types.setdefault(i.id, []).append(type_id) + attr2types[i.id].append(type_id) attrs = [] for attr_id in attr2types: @@ -850,12 +863,12 @@ class CITypeRelationManager(object): @classmethod def recursive_level2children(cls, parent_id): - result = dict() + result = defaultdict(list) def get_children(_id, level): children = CITypeRelation.get_by(parent_id=_id, to_dict=False) if children: - result.setdefault(level + 1, []).extend([i.child.to_dict() for i in children]) + result[level + 1].extend([i.child.to_dict() for i in children]) for i in children: if i.child_id != _id: @@ -954,10 +967,10 @@ class CITypeRelationManager(object): p = CITypeManager.check_is_existed(parent) c = CITypeManager.check_is_existed(child) - rels = {} + rels = defaultdict(set) for i in CITypeRelation.get_by(to_dict=False): - rels.setdefault(i.child_id, set()).add(i.parent_id) - rels.setdefault(c.id, set()).add(p.id) + rels[i.child_id].add(i.parent_id) + rels[c.id].add(p.id) try: toposort_flatten(rels) diff --git a/cmdb-api/api/lib/cmdb/preference.py b/cmdb-api/api/lib/cmdb/preference.py index 9d8d0ba..de01a5b 100644 --- a/cmdb-api/api/lib/cmdb/preference.py +++ b/cmdb-api/api/lib/cmdb/preference.py @@ -1,8 +1,9 @@ # -*- coding:utf-8 -*- -import copy +from collections import defaultdict +import copy import six import toposort from flask import abort @@ -263,12 +264,12 @@ class PreferenceManager(object): else: views = _views - view2cr_ids = dict() + view2cr_ids = defaultdict(list) name2view = dict() result = dict() name2id = list() for view in views: - view2cr_ids.setdefault(view['name'], []).extend(view['cr_ids']) + view2cr_ids[view['name']].extend(view['cr_ids']) name2id.append([view['name'], view['id']]) name2view[view['name']] = view diff --git a/cmdb-api/api/lib/cmdb/value.py b/cmdb-api/api/lib/cmdb/value.py index 0edc2b3..4e8f71a 100644 --- a/cmdb-api/api/lib/cmdb/value.py +++ b/cmdb-api/api/lib/cmdb/value.py @@ -3,13 +3,13 @@ from __future__ import unicode_literals -import copy import imp + +import copy +import jinja2 import os import re import tempfile - -import jinja2 from flask import abort from flask import current_app from jinja2schema import infer @@ -47,7 +47,7 @@ class AttributeValueManager(object): """ return AttributeCache.get(key) - def get_attr_values(self, fields, ci_id, ret_key="name", unique_key=None, use_master=False): + def get_attr_values(self, fields, ci_id, ret_key="name", unique_key=None, use_master=False, enum_map=None): """ :param fields: @@ -55,6 +55,7 @@ class AttributeValueManager(object): :param ret_key: It can be name or alias :param unique_key: primary attribute :param use_master: Only for master-slave read-write separation + :param enum_map: :return: """ res = dict() @@ -76,6 +77,12 @@ class AttributeValueManager(object): else: res[field_name] = ValueTypeMap.serialize[attr.value_type](rs[0].value) if rs else None + if field_name in enum_map: + if attr.is_list: + res[field_name] = [enum_map[field_name].get(i, i) for i in res[field_name]] + else: + res[field_name] = enum_map[field_name].get(res[field_name], res[field_name]) + if unique_key is not None and attr.id == unique_key.id and rs: res['unique'] = unique_key.name res['unique_alias'] = unique_key.alias @@ -245,7 +252,7 @@ class AttributeValueManager(object): if value.get('op') == "delete": value['v'] = [ValueTypeMap.serialize[attr.value_type]( self._deserialize_value(attr.alias, attr.value_type, i)) - for i in handle_arg_list(value['v'])] + for i in handle_arg_list(value['v'])] continue _value = value.get('v') or [] else: diff --git a/cmdb-api/api/tasks/cmdb.py b/cmdb-api/api/tasks/cmdb.py index fd00c87..c722be8 100644 --- a/cmdb-api/api/tasks/cmdb.py +++ b/cmdb-api/api/tasks/cmdb.py @@ -40,6 +40,7 @@ from api.models.cmdb import CITypeAttribute def ci_cache(ci_id, operate_type, record_id): from api.lib.cmdb.ci import CITriggerManager from api.lib.cmdb.ci import CIRelationManager + from api.lib.cmdb.ci_type import CITypeAttributeManager m = api.lib.cmdb.ci.CIManager() ci_dict = m.get_ci_by_id_from_db(ci_id, need_children=False, use_master=False) @@ -55,7 +56,17 @@ def ci_cache(ci_id, operate_type, record_id): current_app.test_request_context().push() login_user(UserCache.get('worker')) - CITriggerManager.fire(operate_type, ci_dict, record_id) + _, enum_map = CITypeAttributeManager.get_attr_names_label_enum(ci_dict.get('_type')) + payload = dict() + for k, v in ci_dict.items(): + if k in enum_map: + if isinstance(v, list): + payload[k] = [enum_map[k].get(i, i) for i in v] + else: + payload[k] = enum_map[k].get(v, v) + else: + payload[k] = v + CITriggerManager.fire(operate_type, payload, record_id) ci_dict and CIRelationManager.build_by_attribute(ci_dict) @@ -196,7 +207,7 @@ def ci_relation_add(parent_dict, child_id, uid): for ci in response: try: CIRelationManager.add(ci['_id'], child_id) - ci_relation_cache(ci['_id'], child_id) + ci_relation_cache(ci['_id'], child_id, None) except Exception as e: current_app.logger.warning(e) finally: