diff --git a/cmdb-api/api/lib/cmdb/ci.py b/cmdb-api/api/lib/cmdb/ci.py index 3951426..785a2db 100644 --- a/cmdb-api/api/lib/cmdb/ci.py +++ b/cmdb-api/api/lib/cmdb/ci.py @@ -29,6 +29,7 @@ from api.lib.cmdb.const import PermEnum from api.lib.cmdb.const import REDIS_PREFIX_CI from api.lib.cmdb.const import ResourceTypeEnum from api.lib.cmdb.const import RetKey +from api.lib.cmdb.const import ValueTypeEnum from api.lib.cmdb.history import AttributeHistoryManger from api.lib.cmdb.history import CIRelationHistoryManager from api.lib.cmdb.history import CITriggerHistoryManager @@ -42,6 +43,8 @@ from api.lib.notify import notify_send from api.lib.perm.acl.acl import ACLManager from api.lib.perm.acl.acl import is_app_admin from api.lib.perm.acl.acl import validate_permission +from api.lib.secrets.inner import InnerCrypt +from api.lib.secrets.vault import VaultClient from api.lib.utils import Lock from api.lib.utils import handle_arg_list from api.lib.webhook import webhook_request @@ -323,6 +326,8 @@ class CIManager(object): ci_attr2type_attr = {type_attr.attr_id: type_attr for type_attr, _ in attrs} ci = None + record_id = None + password_dict = {} need_lock = current_user.username not in current_app.config.get('PRIVILEGED_USERS', PRIVILEGED_USERS) with Lock(ci_type_name, need_lock=need_lock): existed = cls.ci_is_exist(unique_key, unique_value, ci_type.id) @@ -351,14 +356,23 @@ class CIManager(object): ci_dict.get(attr.name) is None and ci_dict.get(attr.alias) is None)): ci_dict[attr.name] = attr.default.get('default') - if type_attr.is_required and (attr.name not in ci_dict and attr.alias not in ci_dict): + if (type_attr.is_required and not attr.is_computed and + (attr.name not in ci_dict and attr.alias not in ci_dict)): return abort(400, ErrFormat.attribute_value_required.format(attr.name)) else: for type_attr, attr in attrs: if attr.default and attr.default.get('default') == AttributeDefaultValueEnum.UPDATED_AT: ci_dict[attr.name] = now - computed_attrs = [attr.to_dict() for _, attr in attrs if attr.is_computed] or None + computed_attrs = [] + for _, attr in attrs: + if attr.is_computed: + computed_attrs.append(attr.to_dict()) + elif attr.is_password: + if attr.name in ci_dict: + password_dict[attr.id] = ci_dict.pop(attr.name) + elif attr.alias in ci_dict: + password_dict[attr.id] = ci_dict.pop(attr.alias) value_manager = AttributeValueManager() @@ -395,6 +409,10 @@ class CIManager(object): cls.delete(ci.id) raise e + if password_dict: + for attr_id in password_dict: + record_id = cls.save_password(ci.id, attr_id, password_dict[attr_id], record_id, ci_type.id) + if record_id: # has change ci_cache.apply_async(args=(ci.id, operate_type, record_id), queue=CMDB_QUEUE) @@ -414,7 +432,16 @@ class CIManager(object): if attr.default and attr.default.get('default') == AttributeDefaultValueEnum.UPDATED_AT: ci_dict[attr.name] = now - computed_attrs = [attr.to_dict() for _, attr in attrs if attr.is_computed] or None + password_dict = dict() + computed_attrs = list() + for _, attr in attrs: + if attr.is_computed: + computed_attrs.append(attr.to_dict()) + elif attr.is_password: + if attr.name in ci_dict: + password_dict[attr.id] = ci_dict.pop(attr.name) + elif attr.alias in ci_dict: + password_dict[attr.id] = ci_dict.pop(attr.alias) value_manager = AttributeValueManager() @@ -423,6 +450,7 @@ class CIManager(object): limit_attrs = self._valid_ci_for_no_read(ci) if not _is_admin else {} + record_id = None need_lock = current_user.username not in current_app.config.get('PRIVILEGED_USERS', PRIVILEGED_USERS) with Lock(ci.ci_type.name, need_lock=need_lock): self._valid_unique_constraint(ci.type_id, ci_dict, ci_id) @@ -440,6 +468,10 @@ class CIManager(object): except BadRequest as e: raise e + if password_dict: + for attr_id in password_dict: + record_id = self.save_password(ci.id, attr_id, password_dict[attr_id], record_id, ci.type_id) + if record_id: # has change ci_cache.apply_async(args=(ci_id, OperateType.UPDATE, record_id), queue=CMDB_QUEUE) @@ -602,7 +634,7 @@ class CIManager(object): _fields = list() for field in fields: attr = AttributeCache.get(field) - if attr is not None: + if attr is not None and not attr.is_password: _fields.append(str(attr.id)) filter_fields_sql = "WHERE A.attr_id in ({0})".format(",".join(_fields)) @@ -620,7 +652,7 @@ class CIManager(object): ci_dict = dict() unique_id2obj = dict() excludes = excludes and set(excludes) - for ci_id, type_id, attr_id, attr_name, attr_alias, value, value_type, is_list in cis: + for ci_id, type_id, attr_id, attr_name, attr_alias, value, value_type, is_list, is_password in cis: if not fields and excludes and (attr_name in excludes or attr_alias in excludes): continue @@ -647,11 +679,14 @@ class CIManager(object): else: return abort(400, ErrFormat.argument_invalid.format("ret_key")) - value = ValueTypeMap.serialize2[value_type](value) - if is_list: - ci_dict.setdefault(attr_key, []).append(value) + if is_password and value: + ci_dict[attr_key] = '******' else: - ci_dict[attr_key] = value + value = ValueTypeMap.serialize2[value_type](value) + if is_list: + ci_dict.setdefault(attr_key, []).append(value) + else: + ci_dict[attr_key] = value return res @@ -683,6 +718,75 @@ class CIManager(object): return cls._get_cis_from_db(ci_ids, ret_key, fields, value_tables, excludes=excludes) + @classmethod + def save_password(cls, ci_id, attr_id, value, record_id, type_id): + if not value: + return + + changed = None + + value_table = ValueTypeMap.table[ValueTypeEnum.PASSWORD] + if current_app.config.get('SECRETS_ENGINE') == 'inner': + encrypt_value, status = InnerCrypt().encrypt(value) + if not status: + current_app.logger.error('save password failed: {}'.format(encrypt_value)) + return abort(400, ErrFormat.password_save_failed.format(encrypt_value)) + else: + encrypt_value = '******' + + existed = value_table.get_by(ci_id=ci_id, attr_id=attr_id, first=True, to_dict=False) + if existed is None: + value_table.create(ci_id=ci_id, attr_id=attr_id, value=encrypt_value) + changed = [(ci_id, attr_id, OperateType.ADD, '', '******', type_id)] + elif existed.value != encrypt_value: + existed.update(ci_id=ci_id, attr_id=attr_id, value=encrypt_value) + changed = [(ci_id, attr_id, OperateType.UPDATE, '******', '******', type_id)] + + if current_app.config.get('SECRETS_ENGINE') == 'vault': + vault = VaultClient(current_app.config.get('VAULT_URL'), current_app.config.get('VAULT_TOKEN')) + try: + vault.update("/{}/{}".format(ci_id, attr_id), dict(v=value)) + except Exception as e: + current_app.logger.error('save password to vault failed: {}'.format(e)) + return abort(400, ErrFormat.password_save_failed.format('write vault failed')) + + if changed is not None: + AttributeValueManager.write_change2(changed, record_id) + + @classmethod + def load_password(cls, ci_id, attr_id): + ci = CI.get_by_id(ci_id) or abort(404, ErrFormat.ci_not_found.format(ci_id)) + + limit_attrs = cls._valid_ci_for_no_read(ci, ci.ci_type) + if limit_attrs: + attr = AttributeCache.get(attr_id) + if attr and attr.name not in limit_attrs: + return abort(403, ErrFormat.no_permission2) + + if current_app.config.get('SECRETS_ENGINE', 'inner') == 'inner': + value_table = ValueTypeMap.table[ValueTypeEnum.PASSWORD] + v = value_table.get_by(ci_id=ci_id, attr_id=attr_id, first=True, to_dict=False) + + v = v and v.value + if not v: + return + + decrypt_value, status = InnerCrypt().decrypt(v) + if not status: + current_app.logger.error('load password failed: {}'.format(decrypt_value)) + return abort(400, ErrFormat.password_load_failed.format(decrypt_value)) + + return decrypt_value + + elif current_app.config.get('SECRETS_ENGINE') == 'vault': + vault = VaultClient(current_app.config.get('VAULT_URL'), current_app.config.get('VAULT_TOKEN')) + data, status = vault.read("/{}/{}".format(ci_id, attr_id)) + if not status: + current_app.logger.error('read password from vault failed: {}'.format(data)) + return abort(400, ErrFormat.password_load_failed.format(data)) + + return data.get('v') + class CIRelationManager(object): """ diff --git a/cmdb-api/api/lib/cmdb/const.py b/cmdb-api/api/lib/cmdb/const.py index e36bba0..dc9497a 100644 --- a/cmdb-api/api/lib/cmdb/const.py +++ b/cmdb-api/api/lib/cmdb/const.py @@ -12,6 +12,8 @@ class ValueTypeEnum(BaseEnum): DATE = "4" TIME = "5" JSON = "6" + PASSWORD = TEXT + LINK = TEXT class ConstraintEnum(BaseEnum): diff --git a/cmdb-api/api/lib/cmdb/resp_format.py b/cmdb-api/api/lib/cmdb/resp_format.py index 5c64abb..ef040be 100644 --- a/cmdb-api/api/lib/cmdb/resp_format.py +++ b/cmdb-api/api/lib/cmdb/resp_format.py @@ -95,3 +95,6 @@ class ErrFormat(CommonErrFormat): ci_filter_perm_cannot_or_query = "CI过滤授权 暂时不支持 或 查询" ci_filter_perm_attr_no_permission = "您没有属性 {} 的操作权限!" ci_filter_perm_ci_no_permission = "您没有该CI的操作权限!" + + password_save_failed = "保存密码失败: {}" + password_load_failed = "获取密码失败: {}" diff --git a/cmdb-api/api/lib/cmdb/utils.py b/cmdb-api/api/lib/cmdb/utils.py index e529670..cf05466 100644 --- a/cmdb-api/api/lib/cmdb/utils.py +++ b/cmdb-api/api/lib/cmdb/utils.py @@ -37,6 +37,8 @@ class ValueTypeMap(object): ValueTypeEnum.DATETIME: str2datetime, ValueTypeEnum.DATE: str2datetime, ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x, + ValueTypeEnum.PASSWORD: lambda x: x, + ValueTypeEnum.LINK: lambda x: x, } serialize = { @@ -47,6 +49,8 @@ class ValueTypeMap(object): ValueTypeEnum.DATE: lambda x: x.strftime("%Y-%m-%d") if not isinstance(x, six.string_types) else x, ValueTypeEnum.DATETIME: lambda x: x.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(x, six.string_types) else x, ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x, + ValueTypeEnum.PASSWORD: lambda x: x if isinstance(x, six.string_types) else str(x), + ValueTypeEnum.LINK: lambda x: x if isinstance(x, six.string_types) else str(x), } serialize2 = { @@ -57,6 +61,8 @@ class ValueTypeMap(object): ValueTypeEnum.DATE: lambda x: (x.decode() if not isinstance(x, six.string_types) else x).split()[0], ValueTypeEnum.DATETIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x, ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x, + ValueTypeEnum.PASSWORD: lambda x: x.decode() if not isinstance(x, six.string_types) else x, + ValueTypeEnum.LINK: lambda x: x.decode() if not isinstance(x, six.string_types) else x, } choice = { @@ -71,6 +77,8 @@ class ValueTypeMap(object): table = { ValueTypeEnum.TEXT: model.CIValueText, ValueTypeEnum.JSON: model.CIValueJson, + ValueTypeEnum.PASSWORD: model.CIValueText, + ValueTypeEnum.LINK: model.CIValueText, 'index_{0}'.format(ValueTypeEnum.INT): model.CIIndexValueInteger, 'index_{0}'.format(ValueTypeEnum.TEXT): model.CIIndexValueText, 'index_{0}'.format(ValueTypeEnum.DATETIME): model.CIIndexValueDateTime, @@ -83,6 +91,8 @@ class ValueTypeMap(object): table_name = { ValueTypeEnum.TEXT: 'c_value_texts', ValueTypeEnum.JSON: 'c_value_json', + ValueTypeEnum.PASSWORD: 'c_value_texts', + ValueTypeEnum.LINK: 'c_value_texts', 'index_{0}'.format(ValueTypeEnum.INT): 'c_value_index_integers', 'index_{0}'.format(ValueTypeEnum.TEXT): 'c_value_index_texts', 'index_{0}'.format(ValueTypeEnum.DATETIME): 'c_value_index_datetime', @@ -100,6 +110,8 @@ class ValueTypeMap(object): ValueTypeEnum.TIME: 'text', ValueTypeEnum.FLOAT: 'float', ValueTypeEnum.JSON: 'object', + ValueTypeEnum.PASSWORD: 'text', + ValueTypeEnum.LINK: 'text', } @@ -112,7 +124,7 @@ class TableMap(object): @property def table(self): attr = AttributeCache.get(self.attr_name) if not self.attr else self.attr - if attr.value_type != ValueTypeEnum.TEXT and attr.value_type != ValueTypeEnum.JSON: + if attr.value_type not in {ValueTypeEnum.TEXT, ValueTypeEnum.JSON, ValueTypeEnum.PASSWORD, ValueTypeEnum.LINK}: self.is_index = True elif self.is_index is None: self.is_index = attr.is_index @@ -124,7 +136,7 @@ class TableMap(object): @property def table_name(self): attr = AttributeCache.get(self.attr_name) if not self.attr else self.attr - if attr.value_type != ValueTypeEnum.TEXT and attr.value_type != ValueTypeEnum.JSON: + if attr.value_type not in {ValueTypeEnum.TEXT, ValueTypeEnum.JSON, ValueTypeEnum.PASSWORD, ValueTypeEnum.LINK}: self.is_index = True elif self.is_index is None: self.is_index = attr.is_index diff --git a/cmdb-api/api/lib/cmdb/value.py b/cmdb-api/api/lib/cmdb/value.py index 43327f0..3da3fbb 100644 --- a/cmdb-api/api/lib/cmdb/value.py +++ b/cmdb-api/api/lib/cmdb/value.py @@ -66,9 +66,10 @@ class AttributeValueManager(object): use_master=use_master, to_dict=False) field_name = getattr(attr, ret_key) - if attr.is_list: res[field_name] = [ValueTypeMap.serialize[attr.value_type](i.value) for i in rs] + elif attr.is_password and rs: + res[field_name] = '******' else: res[field_name] = ValueTypeMap.serialize[attr.value_type](rs[0].value) if rs else None @@ -131,8 +132,7 @@ class AttributeValueManager(object): return AttributeHistoryManger.add(record_id, ci_id, [(attr_id, operate_type, old, new)], type_id) @staticmethod - def _write_change2(changed): - record_id = None + def write_change2(changed, record_id=None): for ci_id, attr_id, operate_type, old, new, type_id in changed: record_id = AttributeHistoryManger.add(record_id, ci_id, [(attr_id, operate_type, old, new)], type_id, commit=False, flush=False) @@ -286,7 +286,7 @@ class AttributeValueManager(object): current_app.logger.warning(str(e)) return abort(400, ErrFormat.attribute_value_unknown_error.format(e.args[0])) - return self._write_change2(changed) + return self.write_change2(changed) @staticmethod def delete_attr_value(attr_id, ci_id): diff --git a/cmdb-api/api/views/cmdb/ci.py b/cmdb-api/api/views/cmdb/ci.py index d287b86..ce39962 100644 --- a/cmdb-api/api/views/cmdb/ci.py +++ b/cmdb-api/api/views/cmdb/ci.py @@ -84,11 +84,10 @@ class CIView(APIView): ci_dict = self._wrap_ci_dict() manager = CIManager() - current_app.logger.debug(ci_dict) ci_id = manager.add(ci_type, exist_policy=exist_policy or ExistPolicy.REJECT, _no_attribute_policy=_no_attribute_policy, - _is_admin=request.values.pop('__is_admin', False), + _is_admin=request.values.pop('__is_admin', None) or False, **ci_dict) return self.jsonify(ci_id=ci_id) @@ -96,7 +95,6 @@ class CIView(APIView): @has_perm_for_ci("ci_id", ResourceTypeEnum.CI, PermEnum.UPDATE, CIManager.get_type) def put(self, ci_id=None): args = request.values - current_app.logger.info(args) ci_type = args.get("ci_type") _no_attribute_policy = args.get("no_attribute_policy", ExistPolicy.IGNORE) @@ -104,14 +102,14 @@ class CIView(APIView): manager = CIManager() if ci_id is not None: manager.update(ci_id, - _is_admin=request.values.pop('__is_admin', False), + _is_admin=request.values.pop('__is_admin', None) or False, **ci_dict) else: request.values.pop('exist_policy', None) ci_id = manager.add(ci_type, exist_policy=ExistPolicy.REPLACE, _no_attribute_policy=_no_attribute_policy, - _is_admin=request.values.pop('__is_admin', False), + _is_admin=request.values.pop('__is_admin', None) or False, **ci_dict) return self.jsonify(ci_id=ci_id) @@ -242,3 +240,13 @@ class CIAutoDiscoveryStatisticsView(APIView): def get(self): return self.jsonify(CIManager.get_ad_statistics()) + + +class CIPasswordView(APIView): + url_prefix = "/ci//attributes//password" + + def get(self, ci_id, attr_id): + return self.jsonify(ci_id=ci_id, attr_id=attr_id, value=CIManager.load_password(ci_id, attr_id)) + + def post(self, ci_id, attr_id): + return self.get(ci_id, attr_id) diff --git a/cmdb-api/settings.example.py b/cmdb-api/settings.example.py index ed6b6ce..307b929 100644 --- a/cmdb-api/settings.example.py +++ b/cmdb-api/settings.example.py @@ -97,3 +97,8 @@ BOOL_TRUE = ['true', 'TRUE', 'True', True, '1', 1, "Yes", "YES", "yes", 'Y', 'y' # # messenger USE_MESSENGER = True + +# # secrets +SECRETS_ENGINE = 'inner' # 'inner' or 'vault' +VAULT_URL = '' +VAULT_TOKEN = ''