7x31/cmdb
1
0
Fork 0
mirror of https://github.com/veops/cmdb.git synced 2025-08-08 18:33:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(cmdb-api): CI password data store (#242)

Browse Source 
* add secrets,for test

* feat: vault SDK (#238)

* feat: vault SDK

* docs: i18n

* perf(vault): format code

* feat(secrets): support vault

* feat: add inner password storage

* feat: secrets

* feat: add inner password storage

* feat: add secrets feature

* perf(secrets): review

---------

Co-authored-by: fxiang21 <fxiang21@126.com>
Co-authored-by: Mimo <osatmnzn@gmail.com>
This commit is contained in:
pycook
2023-10-28 16:19:00 +08:00
committed by GitHub
parent c273ecdd29
commit 36a451686e
20 changed files with 950 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

122
cmdb-api/api/lib/cmdb/ci.py
View File

@@ -29,6 +29,7 @@ from api.lib.cmdb.const import PermEnum
from api.lib.cmdb.const import REDIS_PREFIX_CI
from api.lib.cmdb.const import ResourceTypeEnum
from api.lib.cmdb.const import RetKey
from api.lib.cmdb.const import ValueTypeEnum
from api.lib.cmdb.history import AttributeHistoryManger
from api.lib.cmdb.history import CIRelationHistoryManager
from api.lib.cmdb.history import CITriggerHistoryManager
@@ -42,6 +43,8 @@ from api.lib.notify import notify_send
from api.lib.perm.acl.acl import ACLManager
from api.lib.perm.acl.acl import is_app_admin
from api.lib.perm.acl.acl import validate_permission
from api.lib.secrets.inner import InnerCrypt
from api.lib.secrets.vault import VaultClient
from api.lib.utils import Lock
from api.lib.utils import handle_arg_list
from api.lib.webhook import webhook_request
@@ -323,6 +326,8 @@ class CIManager(object):
ci_attr2type_attr = {type_attr.attr_id: type_attr for type_attr, _ in attrs}
ci = None
record_id = None
password_dict = {}
need_lock = current_user.username not in current_app.config.get('PRIVILEGED_USERS', PRIVILEGED_USERS)
with Lock(ci_type_name, need_lock=need_lock):
existed = cls.ci_is_exist(unique_key, unique_value, ci_type.id)
@@ -351,14 +356,23 @@ class CIManager(object):
ci_dict.get(attr.name) is None and ci_dict.get(attr.alias) is None)):
ci_dict[attr.name] = attr.default.get('default')
if type_attr.is_required and (attr.name not in ci_dict and attr.alias not in ci_dict):
if (type_attr.is_required and not attr.is_computed and
(attr.name not in ci_dict and attr.alias not in ci_dict)):
return abort(400, ErrFormat.attribute_value_required.format(attr.name))
else:
for type_attr, attr in attrs:
if attr.default and attr.default.get('default') == AttributeDefaultValueEnum.UPDATED_AT:
ci_dict[attr.name] = now
computed_attrs = [attr.to_dict() for _, attr in attrs if attr.is_computed] or None
computed_attrs = []
for _, attr in attrs:
if attr.is_computed:
computed_attrs.append(attr.to_dict())
elif attr.is_password:
if attr.name in ci_dict:
password_dict[attr.id] = ci_dict.pop(attr.name)
elif attr.alias in ci_dict:
password_dict[attr.id] = ci_dict.pop(attr.alias)
value_manager = AttributeValueManager()
@@ -395,6 +409,10 @@ class CIManager(object):
cls.delete(ci.id)
raise e
if password_dict:
for attr_id in password_dict:
record_id = cls.save_password(ci.id, attr_id, password_dict[attr_id], record_id, ci_type.id)
if record_id: # has change
ci_cache.apply_async(args=(ci.id, operate_type, record_id), queue=CMDB_QUEUE)
@@ -414,7 +432,16 @@ class CIManager(object):
if attr.default and attr.default.get('default') == AttributeDefaultValueEnum.UPDATED_AT:
ci_dict[attr.name] = now
computed_attrs = [attr.to_dict() for _, attr in attrs if attr.is_computed] or None
password_dict = dict()
computed_attrs = list()
for _, attr in attrs:
if attr.is_computed:
computed_attrs.append(attr.to_dict())
elif attr.is_password:
if attr.name in ci_dict:
password_dict[attr.id] = ci_dict.pop(attr.name)
elif attr.alias in ci_dict:
password_dict[attr.id] = ci_dict.pop(attr.alias)
value_manager = AttributeValueManager()
@@ -423,6 +450,7 @@ class CIManager(object):
limit_attrs = self._valid_ci_for_no_read(ci) if not _is_admin else {}
record_id = None
need_lock = current_user.username not in current_app.config.get('PRIVILEGED_USERS', PRIVILEGED_USERS)
with Lock(ci.ci_type.name, need_lock=need_lock):
self._valid_unique_constraint(ci.type_id, ci_dict, ci_id)
@@ -440,6 +468,10 @@ class CIManager(object):
except BadRequest as e:
raise e
if password_dict:
for attr_id in password_dict:
record_id = self.save_password(ci.id, attr_id, password_dict[attr_id], record_id, ci.type_id)
if record_id: # has change
ci_cache.apply_async(args=(ci_id, OperateType.UPDATE, record_id), queue=CMDB_QUEUE)
@@ -602,7 +634,7 @@ class CIManager(object):
_fields = list()
for field in fields:
attr = AttributeCache.get(field)
if attr is not None:
if attr is not None and not attr.is_password:
_fields.append(str(attr.id))
filter_fields_sql = "WHERE A.attr_id in ({0})".format(",".join(_fields))
@@ -620,7 +652,7 @@ class CIManager(object):
ci_dict = dict()
unique_id2obj = dict()
excludes = excludes and set(excludes)
for ci_id, type_id, attr_id, attr_name, attr_alias, value, value_type, is_list in cis:
for ci_id, type_id, attr_id, attr_name, attr_alias, value, value_type, is_list, is_password in cis:
if not fields and excludes and (attr_name in excludes or attr_alias in excludes):
continue
@@ -647,11 +679,14 @@ class CIManager(object):
else:
return abort(400, ErrFormat.argument_invalid.format("ret_key"))
value = ValueTypeMap.serialize2[value_type](value)
if is_list:
ci_dict.setdefault(attr_key, []).append(value)
if is_password and value:
ci_dict[attr_key] = '******'
else:
ci_dict[attr_key] = value
value = ValueTypeMap.serialize2[value_type](value)
if is_list:
ci_dict.setdefault(attr_key, []).append(value)
else:
ci_dict[attr_key] = value
return res
@@ -683,6 +718,75 @@ class CIManager(object):
return cls._get_cis_from_db(ci_ids, ret_key, fields, value_tables, excludes=excludes)
@classmethod
def save_password(cls, ci_id, attr_id, value, record_id, type_id):
if not value:
return
changed = None
value_table = ValueTypeMap.table[ValueTypeEnum.PASSWORD]
if current_app.config.get('SECRETS_ENGINE') == 'inner':
encrypt_value, status = InnerCrypt().encrypt(value)
if not status:
current_app.logger.error('save password failed: {}'.format(encrypt_value))
return abort(400, ErrFormat.password_save_failed.format(encrypt_value))
else:
encrypt_value = '******'
existed = value_table.get_by(ci_id=ci_id, attr_id=attr_id, first=True, to_dict=False)
if existed is None:
value_table.create(ci_id=ci_id, attr_id=attr_id, value=encrypt_value)
changed = [(ci_id, attr_id, OperateType.ADD, '', '******', type_id)]
elif existed.value != encrypt_value:
existed.update(ci_id=ci_id, attr_id=attr_id, value=encrypt_value)
changed = [(ci_id, attr_id, OperateType.UPDATE, '******', '******', type_id)]
if current_app.config.get('SECRETS_ENGINE') == 'vault':
vault = VaultClient(current_app.config.get('VAULT_URL'), current_app.config.get('VAULT_TOKEN'))
try:
vault.update("/{}/{}".format(ci_id, attr_id), dict(v=value))
except Exception as e:
current_app.logger.error('save password to vault failed: {}'.format(e))
return abort(400, ErrFormat.password_save_failed.format('write vault failed'))
if changed is not None:
AttributeValueManager.write_change2(changed, record_id)
@classmethod
def load_password(cls, ci_id, attr_id):
ci = CI.get_by_id(ci_id) or abort(404, ErrFormat.ci_not_found.format(ci_id))
limit_attrs = cls._valid_ci_for_no_read(ci, ci.ci_type)
if limit_attrs:
attr = AttributeCache.get(attr_id)
if attr and attr.name not in limit_attrs:
return abort(403, ErrFormat.no_permission2)
if current_app.config.get('SECRETS_ENGINE', 'inner') == 'inner':
value_table = ValueTypeMap.table[ValueTypeEnum.PASSWORD]
v = value_table.get_by(ci_id=ci_id, attr_id=attr_id, first=True, to_dict=False)
v = v and v.value
if not v:
return
decrypt_value, status = InnerCrypt().decrypt(v)
if not status:
current_app.logger.error('load password failed: {}'.format(decrypt_value))
return abort(400, ErrFormat.password_load_failed.format(decrypt_value))
return decrypt_value
elif current_app.config.get('SECRETS_ENGINE') == 'vault':
vault = VaultClient(current_app.config.get('VAULT_URL'), current_app.config.get('VAULT_TOKEN'))
data, status = vault.read("/{}/{}".format(ci_id, attr_id))
if not status:
current_app.logger.error('read password from vault failed: {}'.format(data))
return abort(400, ErrFormat.password_load_failed.format(data))
return data.get('v')
class CIRelationManager(object):
"""

2
cmdb-api/api/lib/cmdb/const.py
View File

@@ -12,6 +12,8 @@ class ValueTypeEnum(BaseEnum):
DATE = "4"
TIME = "5"
JSON = "6"
PASSWORD = TEXT
LINK = TEXT
class ConstraintEnum(BaseEnum):

3
cmdb-api/api/lib/cmdb/resp_format.py
View File

@@ -95,3 +95,6 @@ class ErrFormat(CommonErrFormat):
ci_filter_perm_cannot_or_query = "CI过滤授权 暂时不支持 或 查询"
ci_filter_perm_attr_no_permission = "您没有属性 {} 的操作权限!"
ci_filter_perm_ci_no_permission = "您没有该CI的操作权限!"
password_save_failed = "保存密码失败: {}"
password_load_failed = "获取密码失败: {}"

6
cmdb-api/api/lib/cmdb/search/ci/db/query_sql.py
View File

@@ -7,6 +7,7 @@ QUERY_CIS_BY_VALUE_TABLE = """
attr.alias AS attr_alias,
attr.value_type,
attr.is_list,
attr.is_password,
c_cis.type_id,
{0}.ci_id,
{0}.attr_id,
@@ -26,7 +27,8 @@ QUERY_CIS_BY_IDS = """
A.attr_alias,
A.value,
A.value_type,
A.is_list
A.is_list,
A.is_password
FROM
({1}) AS A {0}
ORDER BY A.ci_id;
@@ -43,7 +45,7 @@ FACET_QUERY1 = """
FACET_QUERY = """
SELECT {0}.value,
count({0}.ci_id)
count(distinct {0}.ci_id)
FROM {0}
INNER JOIN ({1}) AS F ON F.ci_id={0}.ci_id
WHERE {0}.attr_id={2:d}

16
cmdb-api/api/lib/cmdb/utils.py
View File

@@ -37,6 +37,8 @@ class ValueTypeMap(object):
ValueTypeEnum.DATETIME: str2datetime,
ValueTypeEnum.DATE: str2datetime,
ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x,
ValueTypeEnum.PASSWORD: lambda x: x,
ValueTypeEnum.LINK: lambda x: x,
}
serialize = {
@@ -47,6 +49,8 @@ class ValueTypeMap(object):
ValueTypeEnum.DATE: lambda x: x.strftime("%Y-%m-%d") if not isinstance(x, six.string_types) else x,
ValueTypeEnum.DATETIME: lambda x: x.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(x, six.string_types) else x,
ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x,
ValueTypeEnum.PASSWORD: lambda x: x if isinstance(x, six.string_types) else str(x),
ValueTypeEnum.LINK: lambda x: x if isinstance(x, six.string_types) else str(x),
}
serialize2 = {
@@ -57,6 +61,8 @@ class ValueTypeMap(object):
ValueTypeEnum.DATE: lambda x: (x.decode() if not isinstance(x, six.string_types) else x).split()[0],
ValueTypeEnum.DATETIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x,
ValueTypeEnum.PASSWORD: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
ValueTypeEnum.LINK: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
}
choice = {
@@ -71,6 +77,8 @@ class ValueTypeMap(object):
table = {
ValueTypeEnum.TEXT: model.CIValueText,
ValueTypeEnum.JSON: model.CIValueJson,
ValueTypeEnum.PASSWORD: model.CIValueText,
ValueTypeEnum.LINK: model.CIValueText,
'index_{0}'.format(ValueTypeEnum.INT): model.CIIndexValueInteger,
'index_{0}'.format(ValueTypeEnum.TEXT): model.CIIndexValueText,
'index_{0}'.format(ValueTypeEnum.DATETIME): model.CIIndexValueDateTime,
@@ -83,6 +91,8 @@ class ValueTypeMap(object):
table_name = {
ValueTypeEnum.TEXT: 'c_value_texts',
ValueTypeEnum.JSON: 'c_value_json',
ValueTypeEnum.PASSWORD: 'c_value_texts',
ValueTypeEnum.LINK: 'c_value_texts',
'index_{0}'.format(ValueTypeEnum.INT): 'c_value_index_integers',
'index_{0}'.format(ValueTypeEnum.TEXT): 'c_value_index_texts',
'index_{0}'.format(ValueTypeEnum.DATETIME): 'c_value_index_datetime',
@@ -100,6 +110,8 @@ class ValueTypeMap(object):
ValueTypeEnum.TIME: 'text',
ValueTypeEnum.FLOAT: 'float',
ValueTypeEnum.JSON: 'object',
ValueTypeEnum.PASSWORD: 'text',
ValueTypeEnum.LINK: 'text',
}
@@ -112,7 +124,7 @@ class TableMap(object):
@property
def table(self):
attr = AttributeCache.get(self.attr_name) if not self.attr else self.attr
if attr.value_type != ValueTypeEnum.TEXT and attr.value_type != ValueTypeEnum.JSON:
if attr.value_type not in {ValueTypeEnum.TEXT, ValueTypeEnum.JSON, ValueTypeEnum.PASSWORD, ValueTypeEnum.LINK}:
self.is_index = True
elif self.is_index is None:
self.is_index = attr.is_index
@@ -124,7 +136,7 @@ class TableMap(object):
@property
def table_name(self):
attr = AttributeCache.get(self.attr_name) if not self.attr else self.attr
if attr.value_type != ValueTypeEnum.TEXT and attr.value_type != ValueTypeEnum.JSON:
if attr.value_type not in {ValueTypeEnum.TEXT, ValueTypeEnum.JSON, ValueTypeEnum.PASSWORD, ValueTypeEnum.LINK}:
self.is_index = True
elif self.is_index is None:
self.is_index = attr.is_index

8
cmdb-api/api/lib/cmdb/value.py
View File

@@ -66,9 +66,10 @@ class AttributeValueManager(object):
use_master=use_master,
to_dict=False)
field_name = getattr(attr, ret_key)
if attr.is_list:
res[field_name] = [ValueTypeMap.serialize[attr.value_type](i.value) for i in rs]
elif attr.is_password and rs:
res[field_name] = '******'
else:
res[field_name] = ValueTypeMap.serialize[attr.value_type](rs[0].value) if rs else None
@@ -131,8 +132,7 @@ class AttributeValueManager(object):
return AttributeHistoryManger.add(record_id, ci_id, [(attr_id, operate_type, old, new)], type_id)
@staticmethod
def _write_change2(changed):
record_id = None
def write_change2(changed, record_id=None):
for ci_id, attr_id, operate_type, old, new, type_id in changed:
record_id = AttributeHistoryManger.add(record_id, ci_id, [(attr_id, operate_type, old, new)], type_id,
commit=False, flush=False)
@@ -286,7 +286,7 @@ class AttributeValueManager(object):
current_app.logger.warning(str(e))
return abort(400, ErrFormat.attribute_value_unknown_error.format(e.args[0]))
return self._write_change2(changed)
return self.write_change2(changed)
@staticmethod
def delete_attr_value(attr_id, ci_id):