Add CI relationship when creating CI, the text value removes the escape

cmdb-api/api/lib/cmdb/attribute.py

@@ -12,6 +12,7 @@ from api.lib.cmdb.cache import CITypeAttributesCache
 from api.lib.cmdb.cache import CITypeCache
 from api.lib.cmdb.const import BUILTIN_KEYWORDS
 from api.lib.cmdb.const import CITypeOperateType
+from api.lib.cmdb.const import CMDB_QUEUE
 from api.lib.cmdb.const import PermEnum
 from api.lib.cmdb.const import ResourceTypeEnum
 from api.lib.cmdb.const import RoleEnum
@@ -103,10 +104,10 @@ class AttributeManager(object):
     @classmethod
     def search_attributes(cls, name=None, alias=None, page=1, page_size=None):
         """
-        :param name: 
+        :param name:
-        :param alias: 
+        :param alias:
-        :param page: 
+        :param page:
-        :param page_size: 
+        :param page_size:
         :return: attribute, if name is None, then return all attributes
         """
         if name is not None:
@@ -162,6 +163,17 @@ class AttributeManager(object):
         if RoleEnum.CONFIG not in session.get("acl", {}).get("parentRoles", []) and not is_app_admin('cmdb'):
             return abort(403, ErrFormat.role_required.format(RoleEnum.CONFIG))
 
+    @staticmethod
+    def calc_computed_attribute(attr_id):
+        """
+        calculate computed attribute for all ci
+        :param attr_id:
+        :return:
+        """
+        from api.tasks.cmdb import calc_computed_attribute
+
+        calc_computed_attribute.apply_async(args=(attr_id, current_user.uid), queue=CMDB_QUEUE)
+
     @classmethod
     @kwargs_required("name")
     def add(cls, **kwargs):

cmdb-api/api/lib/cmdb/ci.py

@@ -47,6 +47,7 @@ from api.models.cmdb import CITypeAttribute
 from api.models.cmdb import CITypeRelation
 from api.tasks.cmdb import ci_cache
 from api.tasks.cmdb import ci_delete
+from api.tasks.cmdb import ci_relation_add
 from api.tasks.cmdb import ci_relation_cache
 from api.tasks.cmdb import ci_relation_delete
 
@@ -305,9 +306,7 @@ class CIManager(object):
         unique_key = AttributeCache.get(ci_type.unique_id) or abort(
             400, ErrFormat.unique_value_not_found.format("unique_id={}".format(ci_type.unique_id)))
 
-        unique_value = ci_dict.get(unique_key.name)
+        unique_value = ci_dict.get(unique_key.name) or ci_dict.get(unique_key.alias) or ci_dict.get(unique_key.id)
-        unique_value = unique_value or ci_dict.get(unique_key.alias)
-        unique_value = unique_value or ci_dict.get(unique_key.id)
         unique_value = unique_value or abort(400, ErrFormat.unique_key_required.format(unique_key.name))
 
         attrs = CITypeAttributesCache.get2(ci_type_name)
@@ -360,7 +359,12 @@ class CIManager(object):
 
             cls._valid_unique_constraint(ci_type.id, ci_dict, ci and ci.id)
 
+            ref_ci_dict = dict()
             for k in ci_dict:
+                if k.startswith("$") and "." in k:
+                    ref_ci_dict[k] = ci_dict[k]
+                    continue
+
                 if k not in ci_type_attrs_name and (
                         k not in ci_type_attrs_alias and _no_attribute_policy == ExistPolicy.REJECT):
                     return abort(400, ErrFormat.attribute_not_found.format(k))
@@ -385,6 +389,9 @@ class CIManager(object):
         if record_id:  # has change
             ci_cache.apply_async([ci.id], queue=CMDB_QUEUE)
 
+        if ref_ci_dict:  # add relations
+            ci_relation_add.apply_async(args=(ref_ci_dict, ci.id, current_user.uid), queue=CMDB_QUEUE)
+
         return ci.id
 
     def update(self, ci_id, _is_admin=False, **ci_dict):
@@ -427,6 +434,10 @@ class CIManager(object):
         if record_id:  # has change
             ci_cache.apply_async([ci_id], queue=CMDB_QUEUE)
 
+        ref_ci_dict = {k: v for k, v in ci_dict.items() if k.startswith("$") and "." in k}
+        if ref_ci_dict:
+            ci_relation_add.apply_async(args=(ref_ci_dict, ci.id), queue=CMDB_QUEUE)
+
     @staticmethod
     def update_unique_value(ci_id, unique_name, unique_value):
         ci = CI.get_by_id(ci_id) or abort(404, ErrFormat.ci_not_found.format("id={}".format(ci_id)))
@@ -744,17 +755,28 @@ class CIRelationManager(object):
         return ci_ids
 
     @staticmethod
-    def _check_constraint(first_ci_id, second_ci_id, type_relation):
+    def _check_constraint(first_ci_id, first_type_id, second_ci_id, second_type_id, type_relation):
+        db.session.remove()
         if type_relation.constraint == ConstraintEnum.Many2Many:
             return
 
-        first_existed = CIRelation.get_by(first_ci_id=first_ci_id, relation_type_id=type_relation.relation_type_id)
+        first_existed = CIRelation.get_by(first_ci_id=first_ci_id,
-        second_existed = CIRelation.get_by(second_ci_id=second_ci_id, relation_type_id=type_relation.relation_type_id)
+                                          relation_type_id=type_relation.relation_type_id, to_dict=False)
-        if type_relation.constraint == ConstraintEnum.One2One and (first_existed or second_existed):
+        second_existed = CIRelation.get_by(second_ci_id=second_ci_id,
-            return abort(400, ErrFormat.relation_constraint.format("1-1"))
+                                           relation_type_id=type_relation.relation_type_id, to_dict=False)
+        if type_relation.constraint == ConstraintEnum.One2One:
+            for i in first_existed:
+                if i.second_ci.type_id == second_type_id:
+                    return abort(400, ErrFormat.relation_constraint.format("1-1"))
 
-        if type_relation.constraint == ConstraintEnum.One2Many and second_existed:
+            for i in second_existed:
-            return abort(400, ErrFormat.relation_constraint.format("1-N"))
+                if i.first_ci.type_id == first_type_id:
+                    return abort(400, ErrFormat.relation_constraint.format("1-1"))
+
+        if type_relation.constraint == ConstraintEnum.One2Many:
+            for i in second_existed:
+                if i.first_ci.type_id == first_type_id:
+                    return abort(400, ErrFormat.relation_constraint.format("1-N"))
 
     @classmethod
     def add(cls, first_ci_id, second_ci_id, more=None, relation_type_id=None):
@@ -793,15 +815,17 @@ class CIRelationManager(object):
             else:
                 type_relation = CITypeRelation.get_by_id(relation_type_id)
 
-            cls._check_constraint(first_ci_id, second_ci_id, type_relation)
+            with Lock("ci_relation_add_{}_{}".format(first_ci.type_id, second_ci.type_id), need_lock=True):
 
-            existed = CIRelation.create(first_ci_id=first_ci_id,
+                cls._check_constraint(first_ci_id, first_ci.type_id, second_ci_id, second_ci.type_id, type_relation)
-                                        second_ci_id=second_ci_id,
-                                        relation_type_id=relation_type_id)
 
-            CIRelationHistoryManager().add(existed, OperateType.ADD)
+                existed = CIRelation.create(first_ci_id=first_ci_id,
+                                            second_ci_id=second_ci_id,
+                                            relation_type_id=relation_type_id)
 
-            ci_relation_cache.apply_async(args=(first_ci_id, second_ci_id), queue=CMDB_QUEUE)
+                CIRelationHistoryManager().add(existed, OperateType.ADD)
+
+                ci_relation_cache.apply_async(args=(first_ci_id, second_ci_id), queue=CMDB_QUEUE)
 
         if more is not None:
             existed.upadte(more=more)

cmdb-api/api/lib/cmdb/ci_type.py

@@ -336,6 +336,17 @@ class CITypeAttributeManager(object):
     def __init__(self):
         pass
 
+    @staticmethod
+    def get_attr_name(ci_type_name, key):
+        ci_type = CITypeCache.get(ci_type_name)
+        if ci_type is None:
+            return
+
+        for i in CITypeAttributesCache.get(ci_type.id):
+            attr = AttributeCache.get(i.attr_id)
+            if attr and (attr.name == key or attr.alias == key):
+                return attr.name
+
     @staticmethod
     def get_attr_names_by_type_id(type_id):
         return [AttributeCache.get(attr.attr_id).name for attr in CITypeAttributesCache.get(type_id)]

cmdb-api/api/lib/cmdb/utils.py

@@ -7,7 +7,6 @@ import json
 import re
 
 import six
-from markupsafe import escape
 
 import api.models.cmdb as model
 from api.lib.cmdb.cache import AttributeCache
@@ -33,8 +32,8 @@ class ValueTypeMap(object):
     deserialize = {
         ValueTypeEnum.INT: string2int,
         ValueTypeEnum.FLOAT: float,
-        ValueTypeEnum.TEXT: lambda x: escape(x).encode('utf-8').decode('utf-8'),
+        ValueTypeEnum.TEXT: lambda x: x,
-        ValueTypeEnum.TIME: lambda x: TIME_RE.findall(escape(x).encode('utf-8').decode('utf-8'))[0],
+        ValueTypeEnum.TIME: lambda x: TIME_RE.findall(x)[0],
         ValueTypeEnum.DATETIME: str2datetime,
         ValueTypeEnum.DATE: str2datetime,
         ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x,

cmdb-api/api/lib/cmdb/value.py

@@ -80,7 +80,7 @@ class AttributeValueManager(object):
         return res
 
     @staticmethod
-    def __deserialize_value(value_type, value):
+    def _deserialize_value(value_type, value):
         if not value:
             return value
 
@@ -92,13 +92,13 @@ class AttributeValueManager(object):
             return abort(400, ErrFormat.attribute_value_invalid.format(value))
 
     @staticmethod
-    def __check_is_choice(attr, value_type, value):
+    def _check_is_choice(attr, value_type, value):
         choice_values = AttributeManager.get_choice_values(attr.id, value_type, attr.choice_web_hook)
         if str(value) not in list(map(str, [i[0] for i in choice_values])):
             return abort(400, ErrFormat.not_in_choice_values.format(value))
 
     @staticmethod
-    def __check_is_unique(value_table, attr, ci_id, type_id, value):
+    def _check_is_unique(value_table, attr, ci_id, type_id, value):
         existed = db.session.query(value_table.attr_id).join(CI, CI.id == value_table.ci_id).filter(
             CI.type_id == type_id).filter(
             value_table.attr_id == attr.id).filter(value_table.deleted.is_(False)).filter(
@@ -107,20 +107,20 @@ class AttributeValueManager(object):
         existed and abort(400, ErrFormat.attribute_value_unique_required.format(attr.alias, value))
 
     @staticmethod
-    def __check_is_required(type_id, attr, value, type_attr=None):
+    def _check_is_required(type_id, attr, value, type_attr=None):
         type_attr = type_attr or CITypeAttributeCache.get(type_id, attr.id)
         if type_attr and type_attr.is_required and not value and value != 0:
             return abort(400, ErrFormat.attribute_value_required.format(attr.alias))
 
     def _validate(self, attr, value, value_table, ci=None, type_id=None, ci_id=None, type_attr=None):
         ci = ci or {}
-        v = self.__deserialize_value(attr.value_type, value)
+        v = self._deserialize_value(attr.value_type, value)
 
-        attr.is_choice and value and self.__check_is_choice(attr, attr.value_type, v)
+        attr.is_choice and value and self._check_is_choice(attr, attr.value_type, v)
-        attr.is_unique and self.__check_is_unique(
+        attr.is_unique and self._check_is_unique(
             value_table, attr, ci and ci.id or ci_id, ci and ci.type_id or type_id, v)
 
-        self.__check_is_required(ci and ci.type_id or type_id, attr, v, type_attr=type_attr)
+        self._check_is_required(ci and ci.type_id or type_id, attr, v, type_attr=type_attr)
 
         if v == "" and attr.value_type not in (ValueTypeEnum.TEXT,):
             v = None
@@ -145,7 +145,7 @@ class AttributeValueManager(object):
         return record_id
 
     @staticmethod
-    def __compute_attr_value_from_expr(expr, ci_dict):
+    def _compute_attr_value_from_expr(expr, ci_dict):
         t = jinja2.Template(expr).render(ci_dict)
 
         try:
@@ -155,7 +155,7 @@ class AttributeValueManager(object):
             return t
 
     @staticmethod
-    def __compute_attr_value_from_script(script, ci_dict):
+    def _compute_attr_value_from_script(script, ci_dict):
         script = jinja2.Template(script).render(ci_dict)
 
         script_f = tempfile.NamedTemporaryFile(delete=False, suffix=".py")
@@ -184,22 +184,22 @@ class AttributeValueManager(object):
 
         return [var for var in schema.get("properties")]
 
-    def _compute_attr_value(self, attr, payload, ci):
+    def _compute_attr_value(self, attr, payload, ci_id):
         attrs = (self._jinja2_parse(attr['compute_expr']) if attr.get('compute_expr')
                  else self._jinja2_parse(attr['compute_script']))
         not_existed = [i for i in attrs if i not in payload]
-        if ci is not None:
+        if ci_id is not None:
-            payload.update(self.get_attr_values(not_existed, ci.id))
+            payload.update(self.get_attr_values(not_existed, ci_id))
 
         if attr['compute_expr']:
-            return self.__compute_attr_value_from_expr(attr['compute_expr'], payload)
+            return self._compute_attr_value_from_expr(attr['compute_expr'], payload)
         elif attr['compute_script']:
-            return self.__compute_attr_value_from_script(attr['compute_script'], payload)
+            return self._compute_attr_value_from_script(attr['compute_script'], payload)
 
     def handle_ci_compute_attributes(self, ci_dict, computed_attrs, ci):
         payload = copy.deepcopy(ci_dict)
         for attr in computed_attrs:
-            computed_value = self._compute_attr_value(attr, payload, ci)
+            computed_value = self._compute_attr_value(attr, payload, ci and ci.id)
             if computed_value is not None:
                 ci_dict[attr['name']] = computed_value
 
@@ -221,7 +221,7 @@ class AttributeValueManager(object):
                                   for i in handle_arg_list(value)]
                     ci_dict[key] = value_list
                     if not value_list:
-                        self.__check_is_required(type_id, attr, '')
+                        self._check_is_required(type_id, attr, '')
 
                 else:
                     value = self._validate(attr, value, value_table, ci=None, type_id=type_id, ci_id=ci_id,
@@ -311,7 +311,7 @@ class AttributeValueManager(object):
             if attr.is_list:
                 value_list = [self._validate(attr, i, value_table, ci) for i in handle_arg_list(value)]
                 if not value_list:
-                    self.__check_is_required(ci.type_id, attr, '')
+                    self._check_is_required(ci.type_id, attr, '')
 
                 existed_attrs = value_table.get_by(attr_id=attr.id, ci_id=ci.id, to_dict=False)
                 existed_values = [i.value for i in existed_attrs]

cmdb-api/api/tasks/cmdb.py

@@ -7,6 +7,7 @@ import time
 import jinja2
 import requests
 from flask import current_app
+from flask_login import login_user
 
 import api.lib.cmdb.ci
 from api.extensions import celery
@@ -18,8 +19,12 @@ from api.lib.cmdb.const import CMDB_QUEUE
 from api.lib.cmdb.const import REDIS_PREFIX_CI
 from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
 from api.lib.mail import send_mail
+from api.lib.perm.acl.cache import UserCache
 from api.lib.utils import Lock
+from api.lib.utils import handle_arg_list
+from api.models.cmdb import CI
 from api.models.cmdb import CIRelation
+from api.models.cmdb import CITypeAttribute
 
 
 @celery.task(name="cmdb.ci_cache", queue=CMDB_QUEUE)
@@ -84,6 +89,51 @@ def ci_relation_cache(parent_id, child_id):
     current_app.logger.info("ADD ci relation cache: {0} -> {1}".format(parent_id, child_id))
 
 
+@celery.task(name="cmdb.ci_relation_add", queue=CMDB_QUEUE)
+def ci_relation_add(parent_dict, child_id, uid):
+    """
+    :param parent_dict: key is '$parent_model.attr_name'
+    :param child_id:
+    :param uid:
+    :return:
+    """
+    from api.lib.cmdb.ci import CIRelationManager
+    from api.lib.cmdb.ci_type import CITypeAttributeManager
+    from api.lib.cmdb.search import SearchError
+    from api.lib.cmdb.search.ci import search
+
+    current_app.test_request_context().push()
+    login_user(UserCache.get(uid))
+
+    db.session.remove()
+
+    for parent in parent_dict:
+        parent_ci_type_name, _attr_name = parent.strip()[1:].split('.', 1)
+        attr_name = CITypeAttributeManager.get_attr_name(parent_ci_type_name, _attr_name)
+        if attr_name is None:
+            current_app.logger.warning("attr name {} does not exist".format(_attr_name))
+            continue
+
+        parent_dict[parent] = handle_arg_list(parent_dict[parent])
+        for v in parent_dict[parent]:
+            query = "_type:{},{}:{}".format(parent_ci_type_name, attr_name, v)
+            s = search(query)
+            try:
+                response, _, _, _, _, _ = s.search()
+            except SearchError as e:
+                current_app.logger.error('ci relation add failed: {}'.format(e))
+                continue
+
+            for ci in response:
+                try:
+                    CIRelationManager.add(ci['_id'], child_id)
+                    ci_relation_cache(ci['_id'], child_id)
+                except Exception as e:
+                    current_app.logger.warning(e)
+                finally:
+                    db.session.remove()
+
+
 @celery.task(name="cmdb.ci_relation_delete", queue=CMDB_QUEUE)
 def ci_relation_delete(parent_id, child_id):
     with Lock("CIRelation_{}".format(parent_id)):
@@ -156,3 +206,18 @@ def trigger_notify(notify, ci_id):
                            for i in notify['mail_to'] if i], subject, body)
         except Exception as e:
             current_app.logger.error("Send mail failed: {0}".format(str(e)))
+
+
+@celery.task(name="cmdb.calc_computed_attribute", queue=CMDB_QUEUE)
+def calc_computed_attribute(attr_id, uid):
+    from api.lib.cmdb.ci import CIManager
+
+    db.session.remove()
+
+    current_app.test_request_context().push()
+    login_user(UserCache.get(uid))
+
+    for i in CITypeAttribute.get_by(attr_id=attr_id, to_dict=False):
+        cis = CI.get_by(type_id=i.type_id, to_dict=False)
+        for ci in cis:
+            CIManager.update(ci.id, {})

cmdb-api/api/views/cmdb/attribute.py

@@ -33,7 +33,8 @@ class AttributeSearchView(APIView):
 
 
 class AttributeView(APIView):
-    url_prefix = ("/attributes", "/attributes/<string:attr_name>", "/attributes/<int:attr_id>")
+    url_prefix = ("/attributes", "/attributes/<string:attr_name>", "/attributes/<int:attr_id>",
+                  "/attributes/<int:attr_id>/calc_computed_attribute")
 
     def get(self, attr_name=None, attr_id=None):
         attr_manager = AttributeManager()
@@ -55,7 +56,12 @@ class AttributeView(APIView):
 
     @args_required("name")
     @args_validate(AttributeManager.cls)
-    def post(self):
+    def post(self, attr_id=None):
+        if request.url.endswith("/calc_computed_attribute"):
+            AttributeManager.calc_computed_attribute(attr_id)
+
+            return self.jsonify(attr_id=attr_id)
+
         choice_value = handle_arg_list(request.values.get("choice_value"))
         params = request.values
         params["choice_value"] = choice_value