feat(secrets): support vault

cmdb-api/api/lib/cmdb/ci.py

@@ -29,6 +29,7 @@ from api.lib.cmdb.const import PermEnum
 from api.lib.cmdb.const import REDIS_PREFIX_CI
 from api.lib.cmdb.const import ResourceTypeEnum
 from api.lib.cmdb.const import RetKey
+from api.lib.cmdb.const import ValueTypeEnum
 from api.lib.cmdb.history import AttributeHistoryManger
 from api.lib.cmdb.history import CIRelationHistoryManager
 from api.lib.cmdb.history import CITriggerHistoryManager
@@ -42,6 +43,8 @@ from api.lib.notify import notify_send
 from api.lib.perm.acl.acl import ACLManager
 from api.lib.perm.acl.acl import is_app_admin
 from api.lib.perm.acl.acl import validate_permission
+from api.lib.secrets.inner import InnerCrypt
+from api.lib.secrets.vault import VaultClient
 from api.lib.utils import Lock
 from api.lib.utils import handle_arg_list
 from api.lib.webhook import webhook_request
@@ -323,6 +326,8 @@ class CIManager(object):
         ci_attr2type_attr = {type_attr.attr_id: type_attr for type_attr, _ in attrs}
 
         ci = None
+        record_id = None
+        password_dict = {}
         need_lock = current_user.username not in current_app.config.get('PRIVILEGED_USERS', PRIVILEGED_USERS)
         with Lock(ci_type_name, need_lock=need_lock):
             existed = cls.ci_is_exist(unique_key, unique_value, ci_type.id)
@@ -351,14 +356,23 @@ class CIManager(object):
                                 ci_dict.get(attr.name) is None and ci_dict.get(attr.alias) is None)):
                             ci_dict[attr.name] = attr.default.get('default')
 
-                    if type_attr.is_required and (attr.name not in ci_dict and attr.alias not in ci_dict):
+                    if (type_attr.is_required and not attr.is_computed and
+                            (attr.name not in ci_dict and attr.alias not in ci_dict)):
                         return abort(400, ErrFormat.attribute_value_required.format(attr.name))
             else:
                 for type_attr, attr in attrs:
                     if attr.default and attr.default.get('default') == AttributeDefaultValueEnum.UPDATED_AT:
                         ci_dict[attr.name] = now
 
-            computed_attrs = [attr.to_dict() for _, attr in attrs if attr.is_computed] or None
+            computed_attrs = []
+            for _, attr in attrs:
+                if attr.is_computed:
+                    computed_attrs.append(attr.to_dict())
+                elif attr.is_password:
+                    if attr.name in ci_dict:
+                        password_dict[attr.id] = ci_dict.pop(attr.name)
+                    elif attr.alias in ci_dict:
+                        password_dict[attr.id] = ci_dict.pop(attr.alias)
 
             value_manager = AttributeValueManager()
 
@@ -395,6 +409,10 @@ class CIManager(object):
                     cls.delete(ci.id)
                 raise e
 
+        if password_dict:
+            for attr_id in password_dict:
+                record_id = cls.save_password(ci.id, attr_id, password_dict[attr_id], record_id, ci_type.id)
+
         if record_id:  # has change
             ci_cache.apply_async(args=(ci.id, operate_type, record_id), queue=CMDB_QUEUE)
 
@@ -414,7 +432,16 @@ class CIManager(object):
             if attr.default and attr.default.get('default') == AttributeDefaultValueEnum.UPDATED_AT:
                 ci_dict[attr.name] = now
 
-        computed_attrs = [attr.to_dict() for _, attr in attrs if attr.is_computed] or None
+        password_dict = dict()
+        computed_attrs = list()
+        for _, attr in attrs:
+            if attr.is_computed:
+                computed_attrs.append(attr.to_dict())
+            elif attr.is_password:
+                if attr.name in ci_dict:
+                    password_dict[attr.id] = ci_dict.pop(attr.name)
+                elif attr.alias in ci_dict:
+                    password_dict[attr.id] = ci_dict.pop(attr.alias)
 
         value_manager = AttributeValueManager()
 
@@ -423,6 +450,7 @@ class CIManager(object):
 
         limit_attrs = self._valid_ci_for_no_read(ci) if not _is_admin else {}
 
+        record_id = None
         need_lock = current_user.username not in current_app.config.get('PRIVILEGED_USERS', PRIVILEGED_USERS)
         with Lock(ci.ci_type.name, need_lock=need_lock):
             self._valid_unique_constraint(ci.type_id, ci_dict, ci_id)
@@ -440,6 +468,10 @@ class CIManager(object):
             except BadRequest as e:
                 raise e
 
+        if password_dict:
+            for attr_id in password_dict:
+                record_id = self.save_password(ci.id, attr_id, password_dict[attr_id], record_id, ci.type_id)
+
         if record_id:  # has change
             ci_cache.apply_async(args=(ci_id, OperateType.UPDATE, record_id), queue=CMDB_QUEUE)
 
@@ -602,7 +634,7 @@ class CIManager(object):
             _fields = list()
             for field in fields:
                 attr = AttributeCache.get(field)
-                if attr is not None:
+                if attr is not None and not attr.is_password:
                     _fields.append(str(attr.id))
             filter_fields_sql = "WHERE A.attr_id in ({0})".format(",".join(_fields))
 
@@ -620,7 +652,7 @@ class CIManager(object):
         ci_dict = dict()
         unique_id2obj = dict()
         excludes = excludes and set(excludes)
-        for ci_id, type_id, attr_id, attr_name, attr_alias, value, value_type, is_list in cis:
+        for ci_id, type_id, attr_id, attr_name, attr_alias, value, value_type, is_list, is_password in cis:
             if not fields and excludes and (attr_name in excludes or attr_alias in excludes):
                 continue
 
@@ -647,6 +679,9 @@ class CIManager(object):
             else:
                 return abort(400, ErrFormat.argument_invalid.format("ret_key"))
 
+            if is_password and value:
+                ci_dict[attr_key] = '******'
+            else:
                 value = ValueTypeMap.serialize2[value_type](value)
                 if is_list:
                     ci_dict.setdefault(attr_key, []).append(value)
@@ -683,6 +718,75 @@ class CIManager(object):
 
         return cls._get_cis_from_db(ci_ids, ret_key, fields, value_tables, excludes=excludes)
 
+    @classmethod
+    def save_password(cls, ci_id, attr_id, value, record_id, type_id):
+        if not value:
+            return
+
+        changed = None
+
+        value_table = ValueTypeMap.table[ValueTypeEnum.PASSWORD]
+        if current_app.config.get('SECRETS_ENGINE') == 'inner':
+            encrypt_value, status = InnerCrypt().encrypt(value)
+            if not status:
+                current_app.logger.error('save password failed: {}'.format(encrypt_value))
+                return abort(400, ErrFormat.password_save_failed.format(encrypt_value))
+        else:
+            encrypt_value = '******'
+
+        existed = value_table.get_by(ci_id=ci_id, attr_id=attr_id, first=True, to_dict=False)
+        if existed is None:
+            value_table.create(ci_id=ci_id, attr_id=attr_id, value=encrypt_value)
+            changed = [(ci_id, attr_id, OperateType.ADD, '', '******', type_id)]
+        elif existed.value != encrypt_value:
+            existed.update(ci_id=ci_id, attr_id=attr_id, value=encrypt_value)
+            changed = [(ci_id, attr_id, OperateType.UPDATE, '******', '******', type_id)]
+
+        if current_app.config.get('SECRETS_ENGINE') == 'vault':
+            vault = VaultClient(current_app.config.get('VAULT_URL'), current_app.config.get('VAULT_TOKEN'))
+            try:
+                vault.update("/{}/{}".format(ci_id, attr_id), dict(v=value))
+            except Exception as e:
+                current_app.logger.error('save password to vault failed: {}'.format(e))
+                return abort(400, ErrFormat.password_save_failed.format('write vault failed'))
+
+        if changed is not None:
+            AttributeValueManager.write_change2(changed, record_id)
+
+    @classmethod
+    def load_password(cls, ci_id, attr_id):
+        ci = CI.get_by_id(ci_id) or abort(404, ErrFormat.ci_not_found.format(ci_id))
+
+        limit_attrs = cls._valid_ci_for_no_read(ci, ci.ci_type)
+        if limit_attrs:
+            attr = AttributeCache.get(attr_id)
+            if attr and attr.name not in limit_attrs:
+                return abort(403, ErrFormat.no_permission2)
+
+        if current_app.config.get('SECRETS_ENGINE', 'inner') == 'inner':
+            value_table = ValueTypeMap.table[ValueTypeEnum.PASSWORD]
+            v = value_table.get_by(ci_id=ci_id, attr_id=attr_id, first=True, to_dict=False)
+
+            v = v and v.value
+            if not v:
+                return
+
+            decrypt_value, status = InnerCrypt().decrypt(v)
+            if not status:
+                current_app.logger.error('load password failed: {}'.format(decrypt_value))
+                return abort(400, ErrFormat.password_load_failed.format(decrypt_value))
+
+            return decrypt_value
+
+        elif current_app.config.get('SECRETS_ENGINE') == 'vault':
+            vault = VaultClient(current_app.config.get('VAULT_URL'), current_app.config.get('VAULT_TOKEN'))
+            data, status = vault.read("/{}/{}".format(ci_id, attr_id))
+            if not status:
+                current_app.logger.error('read password from vault failed: {}'.format(data))
+                return abort(400, ErrFormat.password_load_failed.format(data))
+
+            return data.get('v')
+
 
 class CIRelationManager(object):
     """

cmdb-api/api/lib/cmdb/const.py

@@ -12,6 +12,8 @@ class ValueTypeEnum(BaseEnum):
     DATE = "4"
     TIME = "5"
     JSON = "6"
+    PASSWORD = TEXT
+    LINK = TEXT
 
 
 class ConstraintEnum(BaseEnum):

cmdb-api/api/lib/cmdb/resp_format.py

@@ -95,3 +95,6 @@ class ErrFormat(CommonErrFormat):
     ci_filter_perm_cannot_or_query = "CI过滤授权 暂时不支持 或 查询"
     ci_filter_perm_attr_no_permission = "您没有属性 {} 的操作权限!"
     ci_filter_perm_ci_no_permission = "您没有该CI的操作权限!"
+
+    password_save_failed = "保存密码失败: {}"
+    password_load_failed = "获取密码失败: {}"

cmdb-api/api/lib/cmdb/utils.py

@@ -37,6 +37,8 @@ class ValueTypeMap(object):
         ValueTypeEnum.DATETIME: str2datetime,
         ValueTypeEnum.DATE: str2datetime,
         ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x,
+        ValueTypeEnum.PASSWORD: lambda x: x,
+        ValueTypeEnum.LINK: lambda x: x,
     }
 
     serialize = {
@@ -47,6 +49,8 @@ class ValueTypeMap(object):
         ValueTypeEnum.DATE: lambda x: x.strftime("%Y-%m-%d") if not isinstance(x, six.string_types) else x,
         ValueTypeEnum.DATETIME: lambda x: x.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(x, six.string_types) else x,
         ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x,
+        ValueTypeEnum.PASSWORD: lambda x: x if isinstance(x, six.string_types) else str(x),
+        ValueTypeEnum.LINK: lambda x: x if isinstance(x, six.string_types) else str(x),
     }
 
     serialize2 = {
@@ -57,6 +61,8 @@ class ValueTypeMap(object):
         ValueTypeEnum.DATE: lambda x: (x.decode() if not isinstance(x, six.string_types) else x).split()[0],
         ValueTypeEnum.DATETIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
         ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x,
+        ValueTypeEnum.PASSWORD: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
+        ValueTypeEnum.LINK: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
     }
 
     choice = {
@@ -71,6 +77,8 @@ class ValueTypeMap(object):
     table = {
         ValueTypeEnum.TEXT: model.CIValueText,
         ValueTypeEnum.JSON: model.CIValueJson,
+        ValueTypeEnum.PASSWORD: model.CIValueText,
+        ValueTypeEnum.LINK: model.CIValueText,
         'index_{0}'.format(ValueTypeEnum.INT): model.CIIndexValueInteger,
         'index_{0}'.format(ValueTypeEnum.TEXT): model.CIIndexValueText,
         'index_{0}'.format(ValueTypeEnum.DATETIME): model.CIIndexValueDateTime,
@@ -83,6 +91,8 @@ class ValueTypeMap(object):
     table_name = {
         ValueTypeEnum.TEXT: 'c_value_texts',
         ValueTypeEnum.JSON: 'c_value_json',
+        ValueTypeEnum.PASSWORD: 'c_value_texts',
+        ValueTypeEnum.LINK: 'c_value_texts',
         'index_{0}'.format(ValueTypeEnum.INT): 'c_value_index_integers',
         'index_{0}'.format(ValueTypeEnum.TEXT): 'c_value_index_texts',
         'index_{0}'.format(ValueTypeEnum.DATETIME): 'c_value_index_datetime',
@@ -100,6 +110,8 @@ class ValueTypeMap(object):
         ValueTypeEnum.TIME: 'text',
         ValueTypeEnum.FLOAT: 'float',
         ValueTypeEnum.JSON: 'object',
+        ValueTypeEnum.PASSWORD: 'text',
+        ValueTypeEnum.LINK: 'text',
     }
 
 
@@ -112,7 +124,7 @@ class TableMap(object):
     @property
     def table(self):
         attr = AttributeCache.get(self.attr_name) if not self.attr else self.attr
-        if attr.value_type != ValueTypeEnum.TEXT and attr.value_type != ValueTypeEnum.JSON:
+        if attr.value_type not in {ValueTypeEnum.TEXT, ValueTypeEnum.JSON, ValueTypeEnum.PASSWORD, ValueTypeEnum.LINK}:
             self.is_index = True
         elif self.is_index is None:
             self.is_index = attr.is_index
@@ -124,7 +136,7 @@ class TableMap(object):
     @property
     def table_name(self):
         attr = AttributeCache.get(self.attr_name) if not self.attr else self.attr
-        if attr.value_type != ValueTypeEnum.TEXT and attr.value_type != ValueTypeEnum.JSON:
+        if attr.value_type not in {ValueTypeEnum.TEXT, ValueTypeEnum.JSON, ValueTypeEnum.PASSWORD, ValueTypeEnum.LINK}:
             self.is_index = True
         elif self.is_index is None:
             self.is_index = attr.is_index

cmdb-api/api/lib/cmdb/value.py

@@ -66,9 +66,10 @@ class AttributeValueManager(object):
                                     use_master=use_master,
                                     to_dict=False)
             field_name = getattr(attr, ret_key)
-
             if attr.is_list:
                 res[field_name] = [ValueTypeMap.serialize[attr.value_type](i.value) for i in rs]
+            elif attr.is_password and rs:
+                res[field_name] = '******'
             else:
                 res[field_name] = ValueTypeMap.serialize[attr.value_type](rs[0].value) if rs else None
 
@@ -131,8 +132,7 @@ class AttributeValueManager(object):
         return AttributeHistoryManger.add(record_id, ci_id, [(attr_id, operate_type, old, new)], type_id)
 
     @staticmethod
-    def _write_change2(changed):
+    def write_change2(changed, record_id=None):
-        record_id = None
         for ci_id, attr_id, operate_type, old, new, type_id in changed:
             record_id = AttributeHistoryManger.add(record_id, ci_id, [(attr_id, operate_type, old, new)], type_id,
                                                    commit=False, flush=False)
@@ -286,7 +286,7 @@ class AttributeValueManager(object):
             current_app.logger.warning(str(e))
             return abort(400, ErrFormat.attribute_value_unknown_error.format(e.args[0]))
 
-        return self._write_change2(changed)
+        return self.write_change2(changed)
 
     @staticmethod
     def delete_attr_value(attr_id, ci_id):

cmdb-api/api/views/cmdb/ci.py

@@ -84,11 +84,10 @@ class CIView(APIView):
         ci_dict = self._wrap_ci_dict()
 
         manager = CIManager()
-        current_app.logger.debug(ci_dict)
         ci_id = manager.add(ci_type,
                             exist_policy=exist_policy or ExistPolicy.REJECT,
                             _no_attribute_policy=_no_attribute_policy,
-                            _is_admin=request.values.pop('__is_admin', False),
+                            _is_admin=request.values.pop('__is_admin', None) or False,
                             **ci_dict)
 
         return self.jsonify(ci_id=ci_id)
@@ -96,7 +95,6 @@ class CIView(APIView):
     @has_perm_for_ci("ci_id", ResourceTypeEnum.CI, PermEnum.UPDATE, CIManager.get_type)
     def put(self, ci_id=None):
         args = request.values
-        current_app.logger.info(args)
         ci_type = args.get("ci_type")
         _no_attribute_policy = args.get("no_attribute_policy", ExistPolicy.IGNORE)
 
@@ -104,14 +102,14 @@ class CIView(APIView):
         manager = CIManager()
         if ci_id is not None:
             manager.update(ci_id,
-                           _is_admin=request.values.pop('__is_admin', False),
+                           _is_admin=request.values.pop('__is_admin', None) or False,
                            **ci_dict)
         else:
             request.values.pop('exist_policy', None)
             ci_id = manager.add(ci_type,
                                 exist_policy=ExistPolicy.REPLACE,
                                 _no_attribute_policy=_no_attribute_policy,
-                                _is_admin=request.values.pop('__is_admin', False),
+                                _is_admin=request.values.pop('__is_admin', None) or False,
                                 **ci_dict)
 
         return self.jsonify(ci_id=ci_id)
@@ -242,3 +240,13 @@ class CIAutoDiscoveryStatisticsView(APIView):
 
     def get(self):
         return self.jsonify(CIManager.get_ad_statistics())
+
+
+class CIPasswordView(APIView):
+    url_prefix = "/ci/<int:ci_id>/attributes/<int:attr_id>/password"
+
+    def get(self, ci_id, attr_id):
+        return self.jsonify(ci_id=ci_id, attr_id=attr_id, value=CIManager.load_password(ci_id, attr_id))
+
+    def post(self, ci_id, attr_id):
+        return self.get(ci_id, attr_id)

cmdb-api/settings.example.py

@@ -97,3 +97,8 @@ BOOL_TRUE = ['true', 'TRUE', 'True', True, '1', 1, "Yes", "YES", "yes", 'Y', 'y'
 
 # # messenger
 USE_MESSENGER = True
+
+# # secrets
+SECRETS_ENGINE = 'inner'  # 'inner' or 'vault'
+VAULT_URL = ''
+VAULT_TOKEN = ''