mirror of https://github.com/veops/cmdb.git
feat(secrets): support vault
This commit is contained in:
pycook
parent
5bede8ad73
commit
c808b2cf4b
|
|
@ -29,6 +29,7 @@ from api.lib.cmdb.const import PermEnum
|
|||
from api.lib.cmdb.const import REDIS_PREFIX_CI
|
||||
from api.lib.cmdb.const import ResourceTypeEnum
|
||||
from api.lib.cmdb.const import RetKey
|
||||
from api.lib.cmdb.const import ValueTypeEnum
|
||||
from api.lib.cmdb.history import AttributeHistoryManger
|
||||
from api.lib.cmdb.history import CIRelationHistoryManager
|
||||
from api.lib.cmdb.history import CITriggerHistoryManager
|
||||
|
|
@ -42,6 +43,8 @@ from api.lib.notify import notify_send
|
|||
from api.lib.perm.acl.acl import ACLManager
|
||||
from api.lib.perm.acl.acl import is_app_admin
|
||||
from api.lib.perm.acl.acl import validate_permission
|
||||
from api.lib.secrets.inner import InnerCrypt
|
||||
from api.lib.secrets.vault import VaultClient
|
||||
from api.lib.utils import Lock
|
||||
from api.lib.utils import handle_arg_list
|
||||
from api.lib.webhook import webhook_request
|
||||
|
|
@ -323,6 +326,8 @@ class CIManager(object):
|
|||
ci_attr2type_attr = {type_attr.attr_id: type_attr for type_attr, _ in attrs}
|
||||
|
||||
ci = None
|
||||
record_id = None
|
||||
password_dict = {}
|
||||
need_lock = current_user.username not in current_app.config.get('PRIVILEGED_USERS', PRIVILEGED_USERS)
|
||||
with Lock(ci_type_name, need_lock=need_lock):
|
||||
existed = cls.ci_is_exist(unique_key, unique_value, ci_type.id)
|
||||
|
|
@ -351,14 +356,23 @@ class CIManager(object):
|
|||
ci_dict.get(attr.name) is None and ci_dict.get(attr.alias) is None)):
|
||||
ci_dict[attr.name] = attr.default.get('default')
|
||||
|
||||
if type_attr.is_required and (attr.name not in ci_dict and attr.alias not in ci_dict):
|
||||
if (type_attr.is_required and not attr.is_computed and
|
||||
(attr.name not in ci_dict and attr.alias not in ci_dict)):
|
||||
return abort(400, ErrFormat.attribute_value_required.format(attr.name))
|
||||
else:
|
||||
for type_attr, attr in attrs:
|
||||
if attr.default and attr.default.get('default') == AttributeDefaultValueEnum.UPDATED_AT:
|
||||
ci_dict[attr.name] = now
|
||||
|
||||
computed_attrs = [attr.to_dict() for _, attr in attrs if attr.is_computed] or None
|
||||
computed_attrs = []
|
||||
for _, attr in attrs:
|
||||
if attr.is_computed:
|
||||
computed_attrs.append(attr.to_dict())
|
||||
elif attr.is_password:
|
||||
if attr.name in ci_dict:
|
||||
password_dict[attr.id] = ci_dict.pop(attr.name)
|
||||
elif attr.alias in ci_dict:
|
||||
password_dict[attr.id] = ci_dict.pop(attr.alias)
|
||||
|
||||
value_manager = AttributeValueManager()
|
||||
|
||||
|
|
@ -395,6 +409,10 @@ class CIManager(object):
|
|||
cls.delete(ci.id)
|
||||
raise e
|
||||
|
||||
if password_dict:
|
||||
for attr_id in password_dict:
|
||||
record_id = cls.save_password(ci.id, attr_id, password_dict[attr_id], record_id, ci_type.id)
|
||||
|
||||
if record_id: # has change
|
||||
ci_cache.apply_async(args=(ci.id, operate_type, record_id), queue=CMDB_QUEUE)
|
||||
|
||||
|
|
@ -414,7 +432,16 @@ class CIManager(object):
|
|||
if attr.default and attr.default.get('default') == AttributeDefaultValueEnum.UPDATED_AT:
|
||||
ci_dict[attr.name] = now
|
||||
|
||||
computed_attrs = [attr.to_dict() for _, attr in attrs if attr.is_computed] or None
|
||||
password_dict = dict()
|
||||
computed_attrs = list()
|
||||
for _, attr in attrs:
|
||||
if attr.is_computed:
|
||||
computed_attrs.append(attr.to_dict())
|
||||
elif attr.is_password:
|
||||
if attr.name in ci_dict:
|
||||
password_dict[attr.id] = ci_dict.pop(attr.name)
|
||||
elif attr.alias in ci_dict:
|
||||
password_dict[attr.id] = ci_dict.pop(attr.alias)
|
||||
|
||||
value_manager = AttributeValueManager()
|
||||
|
||||
|
|
@ -423,6 +450,7 @@ class CIManager(object):
|
|||
|
||||
limit_attrs = self._valid_ci_for_no_read(ci) if not _is_admin else {}
|
||||
|
||||
record_id = None
|
||||
need_lock = current_user.username not in current_app.config.get('PRIVILEGED_USERS', PRIVILEGED_USERS)
|
||||
with Lock(ci.ci_type.name, need_lock=need_lock):
|
||||
self._valid_unique_constraint(ci.type_id, ci_dict, ci_id)
|
||||
|
|
@ -440,6 +468,10 @@ class CIManager(object):
|
|||
except BadRequest as e:
|
||||
raise e
|
||||
|
||||
if password_dict:
|
||||
for attr_id in password_dict:
|
||||
record_id = self.save_password(ci.id, attr_id, password_dict[attr_id], record_id, ci.type_id)
|
||||
|
||||
if record_id: # has change
|
||||
ci_cache.apply_async(args=(ci_id, OperateType.UPDATE, record_id), queue=CMDB_QUEUE)
|
||||
|
||||
|
|
@ -602,7 +634,7 @@ class CIManager(object):
|
|||
_fields = list()
|
||||
for field in fields:
|
||||
attr = AttributeCache.get(field)
|
||||
if attr is not None:
|
||||
if attr is not None and not attr.is_password:
|
||||
_fields.append(str(attr.id))
|
||||
filter_fields_sql = "WHERE A.attr_id in ({0})".format(",".join(_fields))
|
||||
|
||||
|
|
@ -620,7 +652,7 @@ class CIManager(object):
|
|||
ci_dict = dict()
|
||||
unique_id2obj = dict()
|
||||
excludes = excludes and set(excludes)
|
||||
for ci_id, type_id, attr_id, attr_name, attr_alias, value, value_type, is_list in cis:
|
||||
for ci_id, type_id, attr_id, attr_name, attr_alias, value, value_type, is_list, is_password in cis:
|
||||
if not fields and excludes and (attr_name in excludes or attr_alias in excludes):
|
||||
continue
|
||||
|
||||
|
|
@ -647,11 +679,14 @@ class CIManager(object):
|
|||
else:
|
||||
return abort(400, ErrFormat.argument_invalid.format("ret_key"))
|
||||
|
||||
value = ValueTypeMap.serialize2[value_type](value)
|
||||
if is_list:
|
||||
ci_dict.setdefault(attr_key, []).append(value)
|
||||
if is_password and value:
|
||||
ci_dict[attr_key] = '******'
|
||||
else:
|
||||
ci_dict[attr_key] = value
|
||||
value = ValueTypeMap.serialize2[value_type](value)
|
||||
if is_list:
|
||||
ci_dict.setdefault(attr_key, []).append(value)
|
||||
else:
|
||||
ci_dict[attr_key] = value
|
||||
|
||||
return res
|
||||
|
||||
|
|
@ -683,6 +718,75 @@ class CIManager(object):
|
|||
|
||||
return cls._get_cis_from_db(ci_ids, ret_key, fields, value_tables, excludes=excludes)
|
||||
|
||||
@classmethod
|
||||
def save_password(cls, ci_id, attr_id, value, record_id, type_id):
|
||||
if not value:
|
||||
return
|
||||
|
||||
changed = None
|
||||
|
||||
value_table = ValueTypeMap.table[ValueTypeEnum.PASSWORD]
|
||||
if current_app.config.get('SECRETS_ENGINE') == 'inner':
|
||||
encrypt_value, status = InnerCrypt().encrypt(value)
|
||||
if not status:
|
||||
current_app.logger.error('save password failed: {}'.format(encrypt_value))
|
||||
return abort(400, ErrFormat.password_save_failed.format(encrypt_value))
|
||||
else:
|
||||
encrypt_value = '******'
|
||||
|
||||
existed = value_table.get_by(ci_id=ci_id, attr_id=attr_id, first=True, to_dict=False)
|
||||
if existed is None:
|
||||
value_table.create(ci_id=ci_id, attr_id=attr_id, value=encrypt_value)
|
||||
changed = [(ci_id, attr_id, OperateType.ADD, '', '******', type_id)]
|
||||
elif existed.value != encrypt_value:
|
||||
existed.update(ci_id=ci_id, attr_id=attr_id, value=encrypt_value)
|
||||
changed = [(ci_id, attr_id, OperateType.UPDATE, '******', '******', type_id)]
|
||||
|
||||
if current_app.config.get('SECRETS_ENGINE') == 'vault':
|
||||
vault = VaultClient(current_app.config.get('VAULT_URL'), current_app.config.get('VAULT_TOKEN'))
|
||||
try:
|
||||
vault.update("/{}/{}".format(ci_id, attr_id), dict(v=value))
|
||||
except Exception as e:
|
||||
current_app.logger.error('save password to vault failed: {}'.format(e))
|
||||
return abort(400, ErrFormat.password_save_failed.format('write vault failed'))
|
||||
|
||||
if changed is not None:
|
||||
AttributeValueManager.write_change2(changed, record_id)
|
||||
|
||||
@classmethod
|
||||
def load_password(cls, ci_id, attr_id):
|
||||
ci = CI.get_by_id(ci_id) or abort(404, ErrFormat.ci_not_found.format(ci_id))
|
||||
|
||||
limit_attrs = cls._valid_ci_for_no_read(ci, ci.ci_type)
|
||||
if limit_attrs:
|
||||
attr = AttributeCache.get(attr_id)
|
||||
if attr and attr.name not in limit_attrs:
|
||||
return abort(403, ErrFormat.no_permission2)
|
||||
|
||||
if current_app.config.get('SECRETS_ENGINE', 'inner') == 'inner':
|
||||
value_table = ValueTypeMap.table[ValueTypeEnum.PASSWORD]
|
||||
v = value_table.get_by(ci_id=ci_id, attr_id=attr_id, first=True, to_dict=False)
|
||||
|
||||
v = v and v.value
|
||||
if not v:
|
||||
return
|
||||
|
||||
decrypt_value, status = InnerCrypt().decrypt(v)
|
||||
if not status:
|
||||
current_app.logger.error('load password failed: {}'.format(decrypt_value))
|
||||
return abort(400, ErrFormat.password_load_failed.format(decrypt_value))
|
||||
|
||||
return decrypt_value
|
||||
|
||||
elif current_app.config.get('SECRETS_ENGINE') == 'vault':
|
||||
vault = VaultClient(current_app.config.get('VAULT_URL'), current_app.config.get('VAULT_TOKEN'))
|
||||
data, status = vault.read("/{}/{}".format(ci_id, attr_id))
|
||||
if not status:
|
||||
current_app.logger.error('read password from vault failed: {}'.format(data))
|
||||
return abort(400, ErrFormat.password_load_failed.format(data))
|
||||
|
||||
return data.get('v')
|
||||
|
||||
|
||||
class CIRelationManager(object):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ class ValueTypeEnum(BaseEnum):
|
|||
DATE = "4"
|
||||
TIME = "5"
|
||||
JSON = "6"
|
||||
PASSWORD = TEXT
|
||||
LINK = TEXT
|
||||
|
||||
|
||||
class ConstraintEnum(BaseEnum):
|
||||
|
|
|
|||
|
|
@ -95,3 +95,6 @@ class ErrFormat(CommonErrFormat):
|
|||
ci_filter_perm_cannot_or_query = "CI过滤授权 暂时不支持 或 查询"
|
||||
ci_filter_perm_attr_no_permission = "您没有属性 {} 的操作权限!"
|
||||
ci_filter_perm_ci_no_permission = "您没有该CI的操作权限!"
|
||||
|
||||
password_save_failed = "保存密码失败: {}"
|
||||
password_load_failed = "获取密码失败: {}"
|
||||
|
|
|
|||
|
|
@ -37,6 +37,8 @@ class ValueTypeMap(object):
|
|||
ValueTypeEnum.DATETIME: str2datetime,
|
||||
ValueTypeEnum.DATE: str2datetime,
|
||||
ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x,
|
||||
ValueTypeEnum.PASSWORD: lambda x: x,
|
||||
ValueTypeEnum.LINK: lambda x: x,
|
||||
}
|
||||
|
||||
serialize = {
|
||||
|
|
@ -47,6 +49,8 @@ class ValueTypeMap(object):
|
|||
ValueTypeEnum.DATE: lambda x: x.strftime("%Y-%m-%d") if not isinstance(x, six.string_types) else x,
|
||||
ValueTypeEnum.DATETIME: lambda x: x.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(x, six.string_types) else x,
|
||||
ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x,
|
||||
ValueTypeEnum.PASSWORD: lambda x: x if isinstance(x, six.string_types) else str(x),
|
||||
ValueTypeEnum.LINK: lambda x: x if isinstance(x, six.string_types) else str(x),
|
||||
}
|
||||
|
||||
serialize2 = {
|
||||
|
|
@ -57,6 +61,8 @@ class ValueTypeMap(object):
|
|||
ValueTypeEnum.DATE: lambda x: (x.decode() if not isinstance(x, six.string_types) else x).split()[0],
|
||||
ValueTypeEnum.DATETIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
|
||||
ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x,
|
||||
ValueTypeEnum.PASSWORD: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
|
||||
ValueTypeEnum.LINK: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
|
||||
}
|
||||
|
||||
choice = {
|
||||
|
|
@ -71,6 +77,8 @@ class ValueTypeMap(object):
|
|||
table = {
|
||||
ValueTypeEnum.TEXT: model.CIValueText,
|
||||
ValueTypeEnum.JSON: model.CIValueJson,
|
||||
ValueTypeEnum.PASSWORD: model.CIValueText,
|
||||
ValueTypeEnum.LINK: model.CIValueText,
|
||||
'index_{0}'.format(ValueTypeEnum.INT): model.CIIndexValueInteger,
|
||||
'index_{0}'.format(ValueTypeEnum.TEXT): model.CIIndexValueText,
|
||||
'index_{0}'.format(ValueTypeEnum.DATETIME): model.CIIndexValueDateTime,
|
||||
|
|
@ -83,6 +91,8 @@ class ValueTypeMap(object):
|
|||
table_name = {
|
||||
ValueTypeEnum.TEXT: 'c_value_texts',
|
||||
ValueTypeEnum.JSON: 'c_value_json',
|
||||
ValueTypeEnum.PASSWORD: 'c_value_texts',
|
||||
ValueTypeEnum.LINK: 'c_value_texts',
|
||||
'index_{0}'.format(ValueTypeEnum.INT): 'c_value_index_integers',
|
||||
'index_{0}'.format(ValueTypeEnum.TEXT): 'c_value_index_texts',
|
||||
'index_{0}'.format(ValueTypeEnum.DATETIME): 'c_value_index_datetime',
|
||||
|
|
@ -100,6 +110,8 @@ class ValueTypeMap(object):
|
|||
ValueTypeEnum.TIME: 'text',
|
||||
ValueTypeEnum.FLOAT: 'float',
|
||||
ValueTypeEnum.JSON: 'object',
|
||||
ValueTypeEnum.PASSWORD: 'text',
|
||||
ValueTypeEnum.LINK: 'text',
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -112,7 +124,7 @@ class TableMap(object):
|
|||
@property
|
||||
def table(self):
|
||||
attr = AttributeCache.get(self.attr_name) if not self.attr else self.attr
|
||||
if attr.value_type != ValueTypeEnum.TEXT and attr.value_type != ValueTypeEnum.JSON:
|
||||
if attr.value_type not in {ValueTypeEnum.TEXT, ValueTypeEnum.JSON, ValueTypeEnum.PASSWORD, ValueTypeEnum.LINK}:
|
||||
self.is_index = True
|
||||
elif self.is_index is None:
|
||||
self.is_index = attr.is_index
|
||||
|
|
@ -124,7 +136,7 @@ class TableMap(object):
|
|||
@property
|
||||
def table_name(self):
|
||||
attr = AttributeCache.get(self.attr_name) if not self.attr else self.attr
|
||||
if attr.value_type != ValueTypeEnum.TEXT and attr.value_type != ValueTypeEnum.JSON:
|
||||
if attr.value_type not in {ValueTypeEnum.TEXT, ValueTypeEnum.JSON, ValueTypeEnum.PASSWORD, ValueTypeEnum.LINK}:
|
||||
self.is_index = True
|
||||
elif self.is_index is None:
|
||||
self.is_index = attr.is_index
|
||||
|
|
|
|||
|
|
@ -66,9 +66,10 @@ class AttributeValueManager(object):
|
|||
use_master=use_master,
|
||||
to_dict=False)
|
||||
field_name = getattr(attr, ret_key)
|
||||
|
||||
if attr.is_list:
|
||||
res[field_name] = [ValueTypeMap.serialize[attr.value_type](i.value) for i in rs]
|
||||
elif attr.is_password and rs:
|
||||
res[field_name] = '******'
|
||||
else:
|
||||
res[field_name] = ValueTypeMap.serialize[attr.value_type](rs[0].value) if rs else None
|
||||
|
||||
|
|
@ -131,8 +132,7 @@ class AttributeValueManager(object):
|
|||
return AttributeHistoryManger.add(record_id, ci_id, [(attr_id, operate_type, old, new)], type_id)
|
||||
|
||||
@staticmethod
|
||||
def _write_change2(changed):
|
||||
record_id = None
|
||||
def write_change2(changed, record_id=None):
|
||||
for ci_id, attr_id, operate_type, old, new, type_id in changed:
|
||||
record_id = AttributeHistoryManger.add(record_id, ci_id, [(attr_id, operate_type, old, new)], type_id,
|
||||
commit=False, flush=False)
|
||||
|
|
@ -286,7 +286,7 @@ class AttributeValueManager(object):
|
|||
current_app.logger.warning(str(e))
|
||||
return abort(400, ErrFormat.attribute_value_unknown_error.format(e.args[0]))
|
||||
|
||||
return self._write_change2(changed)
|
||||
return self.write_change2(changed)
|
||||
|
||||
@staticmethod
|
||||
def delete_attr_value(attr_id, ci_id):
|
||||
|
|
|
|||
|
|
@ -84,11 +84,10 @@ class CIView(APIView):
|
|||
ci_dict = self._wrap_ci_dict()
|
||||
|
||||
manager = CIManager()
|
||||
current_app.logger.debug(ci_dict)
|
||||
ci_id = manager.add(ci_type,
|
||||
exist_policy=exist_policy or ExistPolicy.REJECT,
|
||||
_no_attribute_policy=_no_attribute_policy,
|
||||
_is_admin=request.values.pop('__is_admin', False),
|
||||
_is_admin=request.values.pop('__is_admin', None) or False,
|
||||
**ci_dict)
|
||||
|
||||
return self.jsonify(ci_id=ci_id)
|
||||
|
|
@ -96,7 +95,6 @@ class CIView(APIView):
|
|||
@has_perm_for_ci("ci_id", ResourceTypeEnum.CI, PermEnum.UPDATE, CIManager.get_type)
|
||||
def put(self, ci_id=None):
|
||||
args = request.values
|
||||
current_app.logger.info(args)
|
||||
ci_type = args.get("ci_type")
|
||||
_no_attribute_policy = args.get("no_attribute_policy", ExistPolicy.IGNORE)
|
||||
|
||||
|
|
@ -104,14 +102,14 @@ class CIView(APIView):
|
|||
manager = CIManager()
|
||||
if ci_id is not None:
|
||||
manager.update(ci_id,
|
||||
_is_admin=request.values.pop('__is_admin', False),
|
||||
_is_admin=request.values.pop('__is_admin', None) or False,
|
||||
**ci_dict)
|
||||
else:
|
||||
request.values.pop('exist_policy', None)
|
||||
ci_id = manager.add(ci_type,
|
||||
exist_policy=ExistPolicy.REPLACE,
|
||||
_no_attribute_policy=_no_attribute_policy,
|
||||
_is_admin=request.values.pop('__is_admin', False),
|
||||
_is_admin=request.values.pop('__is_admin', None) or False,
|
||||
**ci_dict)
|
||||
|
||||
return self.jsonify(ci_id=ci_id)
|
||||
|
|
@ -242,3 +240,13 @@ class CIAutoDiscoveryStatisticsView(APIView):
|
|||
|
||||
def get(self):
|
||||
return self.jsonify(CIManager.get_ad_statistics())
|
||||
|
||||
|
||||
class CIPasswordView(APIView):
|
||||
url_prefix = "/ci/<int:ci_id>/attributes/<int:attr_id>/password"
|
||||
|
||||
def get(self, ci_id, attr_id):
|
||||
return self.jsonify(ci_id=ci_id, attr_id=attr_id, value=CIManager.load_password(ci_id, attr_id))
|
||||
|
||||
def post(self, ci_id, attr_id):
|
||||
return self.get(ci_id, attr_id)
|
||||
|
|
|
|||
|
|
@ -97,3 +97,8 @@ BOOL_TRUE = ['true', 'TRUE', 'True', True, '1', 1, "Yes", "YES", "yes", 'Y', 'y'
|
|||
|
||||
# # messenger
|
||||
USE_MESSENGER = True
|
||||
|
||||
# # secrets
|
||||
SECRETS_ENGINE = 'inner' # 'inner' or 'vault'
|
||||
VAULT_URL = ''
|
||||
VAULT_TOKEN = ''
|
||||
|
|
|
|||
Loading…
Reference in New Issue