From 1660139b27fd98e50eeb0320cba996a808d06ee1 Mon Sep 17 00:00:00 2001 From: pycook Date: Thu, 7 Sep 2023 10:12:42 +0800 Subject: [PATCH] Add CI relationship when creating CI, the text value removes the escape --- cmdb-api/api/lib/cmdb/attribute.py | 20 +++++++-- cmdb-api/api/lib/cmdb/ci.py | 56 +++++++++++++++++------- cmdb-api/api/lib/cmdb/ci_type.py | 11 +++++ cmdb-api/api/lib/cmdb/utils.py | 5 +-- cmdb-api/api/lib/cmdb/value.py | 36 +++++++-------- cmdb-api/api/tasks/cmdb.py | 65 ++++++++++++++++++++++++++++ cmdb-api/api/views/cmdb/attribute.py | 10 ++++- 7 files changed, 160 insertions(+), 43 deletions(-) diff --git a/cmdb-api/api/lib/cmdb/attribute.py b/cmdb-api/api/lib/cmdb/attribute.py index 05e3aa5..be0654c 100644 --- a/cmdb-api/api/lib/cmdb/attribute.py +++ b/cmdb-api/api/lib/cmdb/attribute.py @@ -12,6 +12,7 @@ from api.lib.cmdb.cache import CITypeAttributesCache from api.lib.cmdb.cache import CITypeCache from api.lib.cmdb.const import BUILTIN_KEYWORDS from api.lib.cmdb.const import CITypeOperateType +from api.lib.cmdb.const import CMDB_QUEUE from api.lib.cmdb.const import PermEnum from api.lib.cmdb.const import ResourceTypeEnum from api.lib.cmdb.const import RoleEnum @@ -103,10 +104,10 @@ class AttributeManager(object): @classmethod def search_attributes(cls, name=None, alias=None, page=1, page_size=None): """ - :param name: - :param alias: - :param page: - :param page_size: + :param name: + :param alias: + :param page: + :param page_size: :return: attribute, if name is None, then return all attributes """ if name is not None: @@ -162,6 +163,17 @@ class AttributeManager(object): if RoleEnum.CONFIG not in session.get("acl", {}).get("parentRoles", []) and not is_app_admin('cmdb'): return abort(403, ErrFormat.role_required.format(RoleEnum.CONFIG)) + @staticmethod + def calc_computed_attribute(attr_id): + """ + calculate computed attribute for all ci + :param attr_id: + :return: + """ + from api.tasks.cmdb import calc_computed_attribute + + calc_computed_attribute.apply_async(args=(attr_id, current_user.uid), queue=CMDB_QUEUE) + @classmethod @kwargs_required("name") def add(cls, **kwargs): diff --git a/cmdb-api/api/lib/cmdb/ci.py b/cmdb-api/api/lib/cmdb/ci.py index 943e5b7..9189cd9 100644 --- a/cmdb-api/api/lib/cmdb/ci.py +++ b/cmdb-api/api/lib/cmdb/ci.py @@ -47,6 +47,7 @@ from api.models.cmdb import CITypeAttribute from api.models.cmdb import CITypeRelation from api.tasks.cmdb import ci_cache from api.tasks.cmdb import ci_delete +from api.tasks.cmdb import ci_relation_add from api.tasks.cmdb import ci_relation_cache from api.tasks.cmdb import ci_relation_delete @@ -305,9 +306,7 @@ class CIManager(object): unique_key = AttributeCache.get(ci_type.unique_id) or abort( 400, ErrFormat.unique_value_not_found.format("unique_id={}".format(ci_type.unique_id))) - unique_value = ci_dict.get(unique_key.name) - unique_value = unique_value or ci_dict.get(unique_key.alias) - unique_value = unique_value or ci_dict.get(unique_key.id) + unique_value = ci_dict.get(unique_key.name) or ci_dict.get(unique_key.alias) or ci_dict.get(unique_key.id) unique_value = unique_value or abort(400, ErrFormat.unique_key_required.format(unique_key.name)) attrs = CITypeAttributesCache.get2(ci_type_name) @@ -360,7 +359,12 @@ class CIManager(object): cls._valid_unique_constraint(ci_type.id, ci_dict, ci and ci.id) + ref_ci_dict = dict() for k in ci_dict: + if k.startswith("$") and "." in k: + ref_ci_dict[k] = ci_dict[k] + continue + if k not in ci_type_attrs_name and ( k not in ci_type_attrs_alias and _no_attribute_policy == ExistPolicy.REJECT): return abort(400, ErrFormat.attribute_not_found.format(k)) @@ -385,6 +389,9 @@ class CIManager(object): if record_id: # has change ci_cache.apply_async([ci.id], queue=CMDB_QUEUE) + if ref_ci_dict: # add relations + ci_relation_add.apply_async(args=(ref_ci_dict, ci.id, current_user.uid), queue=CMDB_QUEUE) + return ci.id def update(self, ci_id, _is_admin=False, **ci_dict): @@ -427,6 +434,10 @@ class CIManager(object): if record_id: # has change ci_cache.apply_async([ci_id], queue=CMDB_QUEUE) + ref_ci_dict = {k: v for k, v in ci_dict.items() if k.startswith("$") and "." in k} + if ref_ci_dict: + ci_relation_add.apply_async(args=(ref_ci_dict, ci.id), queue=CMDB_QUEUE) + @staticmethod def update_unique_value(ci_id, unique_name, unique_value): ci = CI.get_by_id(ci_id) or abort(404, ErrFormat.ci_not_found.format("id={}".format(ci_id))) @@ -744,17 +755,28 @@ class CIRelationManager(object): return ci_ids @staticmethod - def _check_constraint(first_ci_id, second_ci_id, type_relation): + def _check_constraint(first_ci_id, first_type_id, second_ci_id, second_type_id, type_relation): + db.session.remove() if type_relation.constraint == ConstraintEnum.Many2Many: return - first_existed = CIRelation.get_by(first_ci_id=first_ci_id, relation_type_id=type_relation.relation_type_id) - second_existed = CIRelation.get_by(second_ci_id=second_ci_id, relation_type_id=type_relation.relation_type_id) - if type_relation.constraint == ConstraintEnum.One2One and (first_existed or second_existed): - return abort(400, ErrFormat.relation_constraint.format("1-1")) + first_existed = CIRelation.get_by(first_ci_id=first_ci_id, + relation_type_id=type_relation.relation_type_id, to_dict=False) + second_existed = CIRelation.get_by(second_ci_id=second_ci_id, + relation_type_id=type_relation.relation_type_id, to_dict=False) + if type_relation.constraint == ConstraintEnum.One2One: + for i in first_existed: + if i.second_ci.type_id == second_type_id: + return abort(400, ErrFormat.relation_constraint.format("1-1")) - if type_relation.constraint == ConstraintEnum.One2Many and second_existed: - return abort(400, ErrFormat.relation_constraint.format("1-N")) + for i in second_existed: + if i.first_ci.type_id == first_type_id: + return abort(400, ErrFormat.relation_constraint.format("1-1")) + + if type_relation.constraint == ConstraintEnum.One2Many: + for i in second_existed: + if i.first_ci.type_id == first_type_id: + return abort(400, ErrFormat.relation_constraint.format("1-N")) @classmethod def add(cls, first_ci_id, second_ci_id, more=None, relation_type_id=None): @@ -793,15 +815,17 @@ class CIRelationManager(object): else: type_relation = CITypeRelation.get_by_id(relation_type_id) - cls._check_constraint(first_ci_id, second_ci_id, type_relation) + with Lock("ci_relation_add_{}_{}".format(first_ci.type_id, second_ci.type_id), need_lock=True): - existed = CIRelation.create(first_ci_id=first_ci_id, - second_ci_id=second_ci_id, - relation_type_id=relation_type_id) + cls._check_constraint(first_ci_id, first_ci.type_id, second_ci_id, second_ci.type_id, type_relation) - CIRelationHistoryManager().add(existed, OperateType.ADD) + existed = CIRelation.create(first_ci_id=first_ci_id, + second_ci_id=second_ci_id, + relation_type_id=relation_type_id) - ci_relation_cache.apply_async(args=(first_ci_id, second_ci_id), queue=CMDB_QUEUE) + CIRelationHistoryManager().add(existed, OperateType.ADD) + + ci_relation_cache.apply_async(args=(first_ci_id, second_ci_id), queue=CMDB_QUEUE) if more is not None: existed.upadte(more=more) diff --git a/cmdb-api/api/lib/cmdb/ci_type.py b/cmdb-api/api/lib/cmdb/ci_type.py index dd45dd5..0f28d97 100644 --- a/cmdb-api/api/lib/cmdb/ci_type.py +++ b/cmdb-api/api/lib/cmdb/ci_type.py @@ -336,6 +336,17 @@ class CITypeAttributeManager(object): def __init__(self): pass + @staticmethod + def get_attr_name(ci_type_name, key): + ci_type = CITypeCache.get(ci_type_name) + if ci_type is None: + return + + for i in CITypeAttributesCache.get(ci_type.id): + attr = AttributeCache.get(i.attr_id) + if attr and (attr.name == key or attr.alias == key): + return attr.name + @staticmethod def get_attr_names_by_type_id(type_id): return [AttributeCache.get(attr.attr_id).name for attr in CITypeAttributesCache.get(type_id)] diff --git a/cmdb-api/api/lib/cmdb/utils.py b/cmdb-api/api/lib/cmdb/utils.py index c12b6b7..c6d4630 100644 --- a/cmdb-api/api/lib/cmdb/utils.py +++ b/cmdb-api/api/lib/cmdb/utils.py @@ -7,7 +7,6 @@ import json import re import six -from markupsafe import escape import api.models.cmdb as model from api.lib.cmdb.cache import AttributeCache @@ -33,8 +32,8 @@ class ValueTypeMap(object): deserialize = { ValueTypeEnum.INT: string2int, ValueTypeEnum.FLOAT: float, - ValueTypeEnum.TEXT: lambda x: escape(x).encode('utf-8').decode('utf-8'), - ValueTypeEnum.TIME: lambda x: TIME_RE.findall(escape(x).encode('utf-8').decode('utf-8'))[0], + ValueTypeEnum.TEXT: lambda x: x, + ValueTypeEnum.TIME: lambda x: TIME_RE.findall(x)[0], ValueTypeEnum.DATETIME: str2datetime, ValueTypeEnum.DATE: str2datetime, ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x, diff --git a/cmdb-api/api/lib/cmdb/value.py b/cmdb-api/api/lib/cmdb/value.py index fd1be1c..aca53a5 100644 --- a/cmdb-api/api/lib/cmdb/value.py +++ b/cmdb-api/api/lib/cmdb/value.py @@ -80,7 +80,7 @@ class AttributeValueManager(object): return res @staticmethod - def __deserialize_value(value_type, value): + def _deserialize_value(value_type, value): if not value: return value @@ -92,13 +92,13 @@ class AttributeValueManager(object): return abort(400, ErrFormat.attribute_value_invalid.format(value)) @staticmethod - def __check_is_choice(attr, value_type, value): + def _check_is_choice(attr, value_type, value): choice_values = AttributeManager.get_choice_values(attr.id, value_type, attr.choice_web_hook) if str(value) not in list(map(str, [i[0] for i in choice_values])): return abort(400, ErrFormat.not_in_choice_values.format(value)) @staticmethod - def __check_is_unique(value_table, attr, ci_id, type_id, value): + def _check_is_unique(value_table, attr, ci_id, type_id, value): existed = db.session.query(value_table.attr_id).join(CI, CI.id == value_table.ci_id).filter( CI.type_id == type_id).filter( value_table.attr_id == attr.id).filter(value_table.deleted.is_(False)).filter( @@ -107,20 +107,20 @@ class AttributeValueManager(object): existed and abort(400, ErrFormat.attribute_value_unique_required.format(attr.alias, value)) @staticmethod - def __check_is_required(type_id, attr, value, type_attr=None): + def _check_is_required(type_id, attr, value, type_attr=None): type_attr = type_attr or CITypeAttributeCache.get(type_id, attr.id) if type_attr and type_attr.is_required and not value and value != 0: return abort(400, ErrFormat.attribute_value_required.format(attr.alias)) def _validate(self, attr, value, value_table, ci=None, type_id=None, ci_id=None, type_attr=None): ci = ci or {} - v = self.__deserialize_value(attr.value_type, value) + v = self._deserialize_value(attr.value_type, value) - attr.is_choice and value and self.__check_is_choice(attr, attr.value_type, v) - attr.is_unique and self.__check_is_unique( + attr.is_choice and value and self._check_is_choice(attr, attr.value_type, v) + attr.is_unique and self._check_is_unique( value_table, attr, ci and ci.id or ci_id, ci and ci.type_id or type_id, v) - self.__check_is_required(ci and ci.type_id or type_id, attr, v, type_attr=type_attr) + self._check_is_required(ci and ci.type_id or type_id, attr, v, type_attr=type_attr) if v == "" and attr.value_type not in (ValueTypeEnum.TEXT,): v = None @@ -145,7 +145,7 @@ class AttributeValueManager(object): return record_id @staticmethod - def __compute_attr_value_from_expr(expr, ci_dict): + def _compute_attr_value_from_expr(expr, ci_dict): t = jinja2.Template(expr).render(ci_dict) try: @@ -155,7 +155,7 @@ class AttributeValueManager(object): return t @staticmethod - def __compute_attr_value_from_script(script, ci_dict): + def _compute_attr_value_from_script(script, ci_dict): script = jinja2.Template(script).render(ci_dict) script_f = tempfile.NamedTemporaryFile(delete=False, suffix=".py") @@ -184,22 +184,22 @@ class AttributeValueManager(object): return [var for var in schema.get("properties")] - def _compute_attr_value(self, attr, payload, ci): + def _compute_attr_value(self, attr, payload, ci_id): attrs = (self._jinja2_parse(attr['compute_expr']) if attr.get('compute_expr') else self._jinja2_parse(attr['compute_script'])) not_existed = [i for i in attrs if i not in payload] - if ci is not None: - payload.update(self.get_attr_values(not_existed, ci.id)) + if ci_id is not None: + payload.update(self.get_attr_values(not_existed, ci_id)) if attr['compute_expr']: - return self.__compute_attr_value_from_expr(attr['compute_expr'], payload) + return self._compute_attr_value_from_expr(attr['compute_expr'], payload) elif attr['compute_script']: - return self.__compute_attr_value_from_script(attr['compute_script'], payload) + return self._compute_attr_value_from_script(attr['compute_script'], payload) def handle_ci_compute_attributes(self, ci_dict, computed_attrs, ci): payload = copy.deepcopy(ci_dict) for attr in computed_attrs: - computed_value = self._compute_attr_value(attr, payload, ci) + computed_value = self._compute_attr_value(attr, payload, ci and ci.id) if computed_value is not None: ci_dict[attr['name']] = computed_value @@ -221,7 +221,7 @@ class AttributeValueManager(object): for i in handle_arg_list(value)] ci_dict[key] = value_list if not value_list: - self.__check_is_required(type_id, attr, '') + self._check_is_required(type_id, attr, '') else: value = self._validate(attr, value, value_table, ci=None, type_id=type_id, ci_id=ci_id, @@ -311,7 +311,7 @@ class AttributeValueManager(object): if attr.is_list: value_list = [self._validate(attr, i, value_table, ci) for i in handle_arg_list(value)] if not value_list: - self.__check_is_required(ci.type_id, attr, '') + self._check_is_required(ci.type_id, attr, '') existed_attrs = value_table.get_by(attr_id=attr.id, ci_id=ci.id, to_dict=False) existed_values = [i.value for i in existed_attrs] diff --git a/cmdb-api/api/tasks/cmdb.py b/cmdb-api/api/tasks/cmdb.py index 4d120ed..491fe1c 100644 --- a/cmdb-api/api/tasks/cmdb.py +++ b/cmdb-api/api/tasks/cmdb.py @@ -7,6 +7,7 @@ import time import jinja2 import requests from flask import current_app +from flask_login import login_user import api.lib.cmdb.ci from api.extensions import celery @@ -18,8 +19,12 @@ from api.lib.cmdb.const import CMDB_QUEUE from api.lib.cmdb.const import REDIS_PREFIX_CI from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION from api.lib.mail import send_mail +from api.lib.perm.acl.cache import UserCache from api.lib.utils import Lock +from api.lib.utils import handle_arg_list +from api.models.cmdb import CI from api.models.cmdb import CIRelation +from api.models.cmdb import CITypeAttribute @celery.task(name="cmdb.ci_cache", queue=CMDB_QUEUE) @@ -84,6 +89,51 @@ def ci_relation_cache(parent_id, child_id): current_app.logger.info("ADD ci relation cache: {0} -> {1}".format(parent_id, child_id)) +@celery.task(name="cmdb.ci_relation_add", queue=CMDB_QUEUE) +def ci_relation_add(parent_dict, child_id, uid): + """ + :param parent_dict: key is '$parent_model.attr_name' + :param child_id: + :param uid: + :return: + """ + from api.lib.cmdb.ci import CIRelationManager + from api.lib.cmdb.ci_type import CITypeAttributeManager + from api.lib.cmdb.search import SearchError + from api.lib.cmdb.search.ci import search + + current_app.test_request_context().push() + login_user(UserCache.get(uid)) + + db.session.remove() + + for parent in parent_dict: + parent_ci_type_name, _attr_name = parent.strip()[1:].split('.', 1) + attr_name = CITypeAttributeManager.get_attr_name(parent_ci_type_name, _attr_name) + if attr_name is None: + current_app.logger.warning("attr name {} does not exist".format(_attr_name)) + continue + + parent_dict[parent] = handle_arg_list(parent_dict[parent]) + for v in parent_dict[parent]: + query = "_type:{},{}:{}".format(parent_ci_type_name, attr_name, v) + s = search(query) + try: + response, _, _, _, _, _ = s.search() + except SearchError as e: + current_app.logger.error('ci relation add failed: {}'.format(e)) + continue + + for ci in response: + try: + CIRelationManager.add(ci['_id'], child_id) + ci_relation_cache(ci['_id'], child_id) + except Exception as e: + current_app.logger.warning(e) + finally: + db.session.remove() + + @celery.task(name="cmdb.ci_relation_delete", queue=CMDB_QUEUE) def ci_relation_delete(parent_id, child_id): with Lock("CIRelation_{}".format(parent_id)): @@ -156,3 +206,18 @@ def trigger_notify(notify, ci_id): for i in notify['mail_to'] if i], subject, body) except Exception as e: current_app.logger.error("Send mail failed: {0}".format(str(e))) + + +@celery.task(name="cmdb.calc_computed_attribute", queue=CMDB_QUEUE) +def calc_computed_attribute(attr_id, uid): + from api.lib.cmdb.ci import CIManager + + db.session.remove() + + current_app.test_request_context().push() + login_user(UserCache.get(uid)) + + for i in CITypeAttribute.get_by(attr_id=attr_id, to_dict=False): + cis = CI.get_by(type_id=i.type_id, to_dict=False) + for ci in cis: + CIManager.update(ci.id, {}) diff --git a/cmdb-api/api/views/cmdb/attribute.py b/cmdb-api/api/views/cmdb/attribute.py index c48de55..2a3edf2 100644 --- a/cmdb-api/api/views/cmdb/attribute.py +++ b/cmdb-api/api/views/cmdb/attribute.py @@ -33,7 +33,8 @@ class AttributeSearchView(APIView): class AttributeView(APIView): - url_prefix = ("/attributes", "/attributes/", "/attributes/") + url_prefix = ("/attributes", "/attributes/", "/attributes/", + "/attributes//calc_computed_attribute") def get(self, attr_name=None, attr_id=None): attr_manager = AttributeManager() @@ -55,7 +56,12 @@ class AttributeView(APIView): @args_required("name") @args_validate(AttributeManager.cls) - def post(self): + def post(self, attr_id=None): + if request.url.endswith("/calc_computed_attribute"): + AttributeManager.calc_computed_attribute(attr_id) + + return self.jsonify(attr_id=attr_id) + choice_value = handle_arg_list(request.values.get("choice_value")) params = request.values params["choice_value"] = choice_value