feat(api): supports bool and reference

cmdb-api/api/lib/cmdb/attribute.py

@@ -167,24 +167,30 @@ class AttributeManager(object):
     def get_attribute_by_name(self, name):
         attr = Attribute.get_by(name=name, first=True)
         if attr.get("is_choice"):
-            attr["choice_value"] = self.get_choice_values(attr["id"], attr["value_type"],
-                                                          attr["choice_web_hook"], attr.get("choice_other"))
+            attr["choice_value"] = self.get_choice_values(attr["id"],
+                                                          attr["value_type"],
+                                                          attr["choice_web_hook"],
+                                                          attr.get("choice_other"))
 
         return attr
 
     def get_attribute_by_alias(self, alias):
         attr = Attribute.get_by(alias=alias, first=True)
         if attr.get("is_choice"):
-            attr["choice_value"] = self.get_choice_values(attr["id"], attr["value_type"],
-                                                          attr["choice_web_hook"], attr.get("choice_other"))
+            attr["choice_value"] = self.get_choice_values(attr["id"],
+                                                          attr["value_type"],
+                                                          attr["choice_web_hook"],
+                                                          attr.get("choice_other"))
 
         return attr
 
     def get_attribute_by_id(self, _id):
         attr = Attribute.get_by_id(_id).to_dict()
         if attr.get("is_choice"):
-            attr["choice_value"] = self.get_choice_values(attr["id"], attr["value_type"],
-                                                          attr["choice_web_hook"], attr.get("choice_other"))
+            attr["choice_value"] = self.get_choice_values(attr["id"],
+                                                          attr["value_type"],
+                                                          attr["choice_web_hook"],
+                                                          attr.get("choice_other"))
 
         return attr
 

cmdb-api/api/lib/cmdb/ci.py

@@ -266,7 +266,7 @@ class CIManager(object):
                 value_table = TableMap(attr_name=id2name[attr_id]).table
 
                 values = value_table.get_by(attr_id=attr_id,
-                                            value=ci_dict.get(id2name[attr_id]) or None,
+                                            value=ci_dict.get(id2name[attr_id]),
                                             only_query=True).join(
                     CI, CI.id == value_table.ci_id).filter(CI.type_id == type_id)
                 _ci_ids = set([i.ci_id for i in values])
@@ -292,6 +292,53 @@ class CIManager(object):
 
         return 1
 
+    @staticmethod
+    def _reference_to_ci_id(attr, payload):
+        def __unique_value2id(_type, _v):
+            value_table = TableMap(attr_name=_type.unique_id).table
+            ci = value_table.get_by(attr_id=attr.id, value=_v)
+            if ci is not None:
+                return ci.ci_id
+
+            return abort(400, ErrFormat.ci_reference_invalid.format(attr.alias, _v))
+
+        def __valid_reference_id_existed(_id, _type_id):
+            ci = CI.get_by_id(_id) or abort(404, ErrFormat.ci_reference_not_found.format(attr.alias, _id))
+
+            if ci.type_id != _type_id:
+                return abort(400, ErrFormat.ci_reference_invalid.format(attr.alias, _id))
+
+        if attr.name in payload:
+            k, reference_value = attr.name, payload[attr.name]
+        elif attr.alias in payload:
+            k, reference_value = attr.alias, payload[attr.alias]
+        else:
+            return
+        if not reference_value:
+            return
+
+        reference_type = None
+        if isinstance(reference_value, list):
+            for idx, v in enumerate(reference_value):
+                if isinstance(v, dict) and v.get('unique'):
+                    if reference_type is None:
+                        reference_type = CITypeCache.get(attr.reference_type_id)
+                    if reference_type is not None:
+                        reference_value[idx] = __unique_value2id(reference_type, v)
+                else:
+                    __valid_reference_id_existed(v, attr.reference_type_id)
+
+        elif isinstance(reference_value, dict) and reference_value.get('unique'):
+            if reference_type is None:
+                reference_type = CITypeCache.get(attr.reference_type_id)
+            if reference_type is not None:
+                reference_value = __unique_value2id(reference_type, reference_value)
+        elif str(reference_value).isdigit():
+            reference_value = int(reference_value)
+            __valid_reference_id_existed(reference_value, attr.reference_type_id)
+
+        payload[k] = reference_value
+
     @classmethod
     def add(cls, ci_type_name,
             exist_policy=ExistPolicy.REPLACE,
@@ -392,6 +439,8 @@ class CIManager(object):
 
                     if attr.re_check and password_dict.get(attr.id):
                         value_manager.check_re(attr.re_check, attr.alias, password_dict[attr.id][0])
+                elif attr.is_reference:
+                    cls._reference_to_ci_id(attr, ci_dict)
 
             cls._valid_unique_constraint(ci_type.id, ci_dict, ci and ci.id)
 
@@ -443,9 +492,10 @@ class CIManager(object):
 
         return ci.id
 
-    def update(self, ci_id, _is_admin=False, ticket_id=None, __sync=False, **ci_dict):
+    def update(self, ci_id, _is_admin=False, ticket_id=None, _sync=False, **ci_dict):
         now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
         ci = self.confirm_ci_existed(ci_id)
+        ci_type = ci.ci_type
 
         attrs = CITypeAttributeManager.get_all_attributes(ci.type_id)
         ci_type_attrs_name = {attr.name: attr for _, attr in attrs}
@@ -475,11 +525,13 @@ class CIManager(object):
 
                 if attr.re_check and password_dict.get(attr.id):
                     value_manager.check_re(attr.re_check, attr.alias, password_dict[attr.id][0])
+            elif attr.is_reference:
+                self._reference_to_ci_id(attr, ci_dict)
 
         limit_attrs = self._valid_ci_for_no_read(ci) if not _is_admin else {}
 
         record_id = None
-        with redis_lock.Lock(rd.r, ci.ci_type.name):
+        with redis_lock.Lock(rd.r, ci_type.name):
             db.session.commit()
 
             self._valid_unique_constraint(ci.type_id, ci_dict, ci_id)
@@ -510,14 +562,14 @@ class CIManager(object):
                 record_id = self.save_password(ci.id, attr_id, password_dict[attr_id], record_id, ci.type_id)
 
         if record_id or has_dynamic:  # has changed
-            if not __sync:
+            if not _sync:
                 ci_cache.apply_async(args=(ci_id, OperateType.UPDATE, record_id), queue=CMDB_QUEUE)
             else:
                 ci_cache(ci_id, OperateType.UPDATE, record_id)
 
         ref_ci_dict = {k: v for k, v in ci_dict.items() if k.startswith("$") and "." in k}
         if ref_ci_dict:
-            if not __sync:
+            if not _sync:
                 ci_relation_add.apply_async(args=(ref_ci_dict, ci.id), queue=CMDB_QUEUE)
             else:
                 ci_relation_add(ref_ci_dict, ci.id)
@@ -578,7 +630,7 @@ class CIManager(object):
         if ci_dict:
             AttributeHistoryManger.add(None, ci_id, [(None, OperateType.DELETE, ci_dict, None)], ci.type_id)
 
-        ci_delete.apply_async(args=(ci_id,), queue=CMDB_QUEUE)
+        ci_delete.apply_async(args=(ci_id, ci.type_id), queue=CMDB_QUEUE)
         delete_id_filter.apply_async(args=(ci_id,), queue=CMDB_QUEUE)
 
         return ci_id
@@ -773,7 +825,7 @@ class CIManager(object):
         value_table = ValueTypeMap.table[ValueTypeEnum.PASSWORD]
         if current_app.config.get('SECRETS_ENGINE') == 'inner':
             if value:
-                encrypt_value, status = InnerCrypt().encrypt(value)
+                encrypt_value, status = InnerCrypt().encrypt(str(value))
                 if not status:
                     current_app.logger.error('save password failed: {}'.format(encrypt_value))
                     return abort(400, ErrFormat.password_save_failed.format(encrypt_value))
@@ -801,7 +853,7 @@ class CIManager(object):
             vault = VaultClient(current_app.config.get('VAULT_URL'), current_app.config.get('VAULT_TOKEN'))
             if value:
                 try:
-                    vault.update("/{}/{}".format(ci_id, attr_id), dict(v=value))
+                    vault.update("/{}/{}".format(ci_id, attr_id), dict(v=str(value)))
                 except Exception as e:
                     current_app.logger.error('save password to vault failed: {}'.format(e))
                     return abort(400, ErrFormat.password_save_failed.format('write vault failed'))

cmdb-api/api/lib/cmdb/ci_type.py

@@ -145,7 +145,7 @@ class CITypeManager(object):
         kwargs["alias"] = kwargs["name"] if not kwargs.get("alias") else kwargs["alias"]
 
         cls._validate_unique(name=kwargs['name'])
-        cls._validate_unique(alias=kwargs['alias'])
+        # cls._validate_unique(alias=kwargs['alias'])
 
         kwargs["unique_id"] = unique_key.id
         kwargs['uid'] = current_user.uid
@@ -184,7 +184,7 @@ class CITypeManager(object):
         ci_type = cls.check_is_existed(type_id)
 
         cls._validate_unique(type_id=type_id, name=kwargs.get('name'))
-        cls._validate_unique(type_id=type_id, alias=kwargs.get('alias') or kwargs.get('name'))
+        # cls._validate_unique(type_id=type_id, alias=kwargs.get('alias') or kwargs.get('name'))
 
         unique_key = kwargs.pop("unique_key", None)
         unique_key = AttributeCache.get(unique_key)
@@ -234,6 +234,10 @@ class CITypeManager(object):
         if CITypeInheritance.get_by(parent_id=type_id, first=True):
             return abort(400, ErrFormat.ci_type_inheritance_cannot_delete)
 
+        reference = Attribute.get_by(reference_type_id=type_id, first=True, to_dict=False)
+        if reference is not None:
+            return abort(400, ErrFormat.ci_type_referenced_cannot_delete.format(reference.alias))
+
         relation_views = PreferenceRelationView.get_by(to_dict=False)
         for rv in relation_views:
             for item in (rv.cr_ids or []):
@@ -1323,6 +1327,7 @@ class CITypeTemplateManager(object):
     def _import_attributes(self, type2attributes):
         attributes = [attr for type_id in type2attributes for attr in type2attributes[type_id]]
         attrs = []
+        references = []
         for i in copy.deepcopy(attributes):
             if i.pop('inherited', None):
                 continue
@@ -1337,6 +1342,10 @@ class CITypeTemplateManager(object):
             if not choice_value:
                 i['is_choice'] = False
 
+            if i.get('reference_type_id'):
+                references.append(copy.deepcopy(i))
+                i.pop('reference_type_id')
+
             attrs.append((i, choice_value))
 
         attr_id_map = self.__import(Attribute, [i[0] for i in copy.deepcopy(attrs)])
@@ -1345,7 +1354,7 @@ class CITypeTemplateManager(object):
             if choice_value and not i.get('choice_web_hook') and not i.get('choice_other'):
                 AttributeManager.add_choice_values(attr_id_map.get(i['id'], i['id']), i['value_type'], choice_value)
 
-        return attr_id_map
+        return attr_id_map, references
 
     def _import_ci_types(self, ci_types, attr_id_map):
         for i in ci_types:
@@ -1359,6 +1368,11 @@ class CITypeTemplateManager(object):
 
         return self.__import(CIType, ci_types)
 
+    def _import_reference_attributes(self, attrs, type_id_map):
+        for attr in attrs:
+            attr['reference_type_id'] = type_id_map.get(attr['reference_type_id'], attr['reference_type_id'])
+        self.__import(Attribute, attrs)
+
     def _import_ci_type_groups(self, ci_type_groups, type_id_map):
         _ci_type_groups = copy.deepcopy(ci_type_groups)
         for i in _ci_type_groups:
@@ -1584,13 +1598,15 @@ class CITypeTemplateManager(object):
 
         import time
         s = time.time()
-        attr_id_map = self._import_attributes(tpt.get('type2attributes') or {})
+        attr_id_map, references = self._import_attributes(tpt.get('type2attributes') or {})
         current_app.logger.info('import attributes cost: {}'.format(time.time() - s))
 
         s = time.time()
         ci_type_id_map = self._import_ci_types(tpt.get('ci_types') or [], attr_id_map)
         current_app.logger.info('import ci_types cost: {}'.format(time.time() - s))
 
+        self._import_reference_attributes(references, ci_type_id_map)
+
         s = time.time()
         self._import_ci_type_groups(tpt.get('ci_type_groups') or [], ci_type_id_map)
         current_app.logger.info('import ci_type_groups cost: {}'.format(time.time() - s))
@@ -1675,6 +1691,16 @@ class CITypeTemplateManager(object):
             type_ids.extend(extend_type_ids)
             ci_types.extend(CITypeManager.get_ci_types(type_ids=extend_type_ids))
 
+        # handle reference type
+        references = Attribute.get_by(only_query=True).join(
+            CITypeAttribute, CITypeAttribute.attr_id == Attribute.id).filter(
+            CITypeAttribute.type_id.in_(type_ids)).filter(CITypeAttribute.deleted.is_(False)).filter(
+            Attribute.reference_type_id.isnot(None))
+        reference_type_ids = list(set([i.reference_type_id for i in references if i.reference_type_id]))
+        if reference_type_ids:
+            type_ids.extend(reference_type_ids)
+            ci_types.extend(CITypeManager.get_ci_types(type_ids=reference_type_ids))
+
         tpt = dict(
             ci_types=ci_types,
             relation_types=[i.to_dict() for i in RelationTypeManager.get_all()],
@@ -1687,6 +1713,7 @@ class CITypeTemplateManager(object):
             icons=dict()
         )
         tpt['ci_type_groups'] = CITypeGroupManager.get(ci_types=tpt['ci_types'], type_ids=type_ids)
+        tpt['ci_type_groups'] = [i for i in tpt['ci_type_groups'] if i.get('ci_types')]
 
         def get_icon_value(icon):
             try:

cmdb-api/api/lib/cmdb/const.py

@@ -14,6 +14,8 @@ class ValueTypeEnum(BaseEnum):
     JSON = "6"
     PASSWORD = TEXT
     LINK = TEXT
+    BOOL = "7"
+    REFERENCE = INT
 
 
 class ConstraintEnum(BaseEnum):

cmdb-api/api/lib/cmdb/resp_format.py

@@ -44,6 +44,8 @@ class ErrFormat(CommonErrFormat):
     unique_value_not_found = _l("The model's primary key {} does not exist!")  # 模型的主键 {} 不存在!
     unique_key_required = _l("Primary key {} is missing")  # 主键字段 {} 缺失
     ci_is_already_existed = _l("CI already exists!")  # CI 已经存在!
+    ci_reference_not_found = _l("{}: CI reference {} does not exist!")  # {}: CI引用 {} 不存在!
+    ci_reference_invalid = _l("{}: CI reference {} is illegal!")  # {}, CI引用 {} 不合法!
     relation_constraint = _l("Relationship constraint: {}, verification failed")  # 关系约束: {}, 校验失败
     # 多对多关系 限制: 模型 {} <-> {} 已经存在多对多关系!
     m2m_relation_constraint = _l(
@@ -63,6 +65,8 @@ class ErrFormat(CommonErrFormat):
     ci_exists_and_cannot_delete_inheritance = _l(
         "The inheritance cannot be deleted because the CI already exists")  # 因为CI已经存在，不能删除继承关系
     ci_type_inheritance_cannot_delete = _l("The model is inherited and cannot be deleted")  # 该模型被继承, 不能删除
+    ci_type_referenced_cannot_delete = _l(
+        "The model is referenced by attribute {} and cannot be deleted")  # 该模型被属性 {} 引用, 不能删除
 
     # 因为关系视图 {} 引用了该模型，不能删除模型
     ci_relation_view_exists_and_cannot_delete_type = _l(

cmdb-api/api/lib/cmdb/search/ci/db/search.py

@@ -451,6 +451,9 @@ class Search(object):
             if field_type == ValueTypeEnum.DATE and len(v) == 10:
                 v = "{} 00:00:00".format(v)
 
+            if field_type == ValueTypeEnum.BOOL and "*" not in str(v):
+                v = str(int(v in current_app.config.get('BOOL_TRUE')))
+
             # in query
             if v.startswith("(") and v.endswith(")"):
                 _query_sql = self._in_query_handler(attr, v, is_not)

cmdb-api/api/lib/cmdb/utils.py

@@ -7,6 +7,7 @@ import json
 import re
 
 import six
+from flask import current_app
 
 import api.models.cmdb as model
 from api.lib.cmdb.cache import AttributeCache
@@ -64,6 +65,7 @@ class ValueTypeMap(object):
         ValueTypeEnum.DATETIME: str2datetime,
         ValueTypeEnum.DATE: str2date,
         ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x,
+        ValueTypeEnum.BOOL: lambda x: x in current_app.config.get('BOOL_TRUE'),
     }
 
     serialize = {
@@ -74,6 +76,7 @@ class ValueTypeMap(object):
         ValueTypeEnum.DATE: lambda x: x.strftime("%Y-%m-%d") if not isinstance(x, six.string_types) else x,
         ValueTypeEnum.DATETIME: lambda x: x.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(x, six.string_types) else x,
         ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x,
+        ValueTypeEnum.BOOL: lambda x: x in current_app.config.get('BOOL_TRUE'),
     }
 
     serialize2 = {
@@ -84,6 +87,7 @@ class ValueTypeMap(object):
         ValueTypeEnum.DATE: lambda x: (x.decode() if not isinstance(x, six.string_types) else x).split()[0],
         ValueTypeEnum.DATETIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
         ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x,
+        ValueTypeEnum.BOOL: lambda x: x in current_app.config.get('BOOL_TRUE'),
     }
 
     choice = {
@@ -105,6 +109,7 @@ class ValueTypeMap(object):
         'index_{0}'.format(ValueTypeEnum.TIME): model.CIIndexValueText,
         'index_{0}'.format(ValueTypeEnum.FLOAT): model.CIIndexValueFloat,
         'index_{0}'.format(ValueTypeEnum.JSON): model.CIValueJson,
+        'index_{0}'.format(ValueTypeEnum.BOOL): model.CIIndexValueInteger,
     }
 
     table_name = {
@@ -117,6 +122,7 @@ class ValueTypeMap(object):
         'index_{0}'.format(ValueTypeEnum.TIME): 'c_value_index_texts',
         'index_{0}'.format(ValueTypeEnum.FLOAT): 'c_value_index_floats',
         'index_{0}'.format(ValueTypeEnum.JSON): 'c_value_json',
+        'index_{0}'.format(ValueTypeEnum.BOOL): 'c_value_index_integers',
     }
 
     es_type = {

cmdb-api/api/lib/cmdb/value.py

@@ -128,14 +128,20 @@ class AttributeValueManager(object):
             return abort(400, ErrFormat.attribute_value_invalid2.format(alias, value))
 
     def _validate(self, attr, value, value_table, ci=None, type_id=None, ci_id=None, type_attr=None):
-        ci = ci or {}
-        v = self._deserialize_value(attr.alias, attr.value_type, value)
+        if not attr.is_reference:
+            ci = ci or {}
+            v = self._deserialize_value(attr.alias, attr.value_type, value)
+
+            attr.is_choice and value and self._check_is_choice(attr, attr.value_type, v)
+
+        else:
+            v = value or None
 
-        attr.is_choice and value and self._check_is_choice(attr, attr.value_type, v)
         attr.is_unique and self._check_is_unique(
             value_table, attr, ci and ci.id or ci_id, ci and ci.type_id or type_id, v)
-
         self._check_is_required(ci and ci.type_id or type_id, attr, v, type_attr=type_attr)
+        if attr.is_reference:
+            return v
 
         if v == "" and attr.value_type not in (ValueTypeEnum.TEXT,):
             v = None

cmdb-api/api/lib/notify.py

@@ -58,7 +58,7 @@ def _request_messenger(subject, body, tos, sender, payload):
 
 def notify_send(subject, body, methods, tos, payload=None):
     payload = payload or {}
-    payload = {k: v or '' for k, v in payload.items()}
+    payload = {k: '' if v is None else v for k, v in payload.items()}
     subject = Template(subject).render(payload)
     body = Template(body).render(payload)