diff --git a/cmdb-api/Pipfile b/cmdb-api/Pipfile index 20e831a..776410e 100644 --- a/cmdb-api/Pipfile +++ b/cmdb-api/Pipfile @@ -26,17 +26,17 @@ Flask-Bcrypt = "==1.0.1" Flask-Cors = ">=3.0.8" ldap3 = "==2.9.1" pycryptodome = "==3.12.0" -cryptography = "==41.0.2" +cryptography = ">=41.0.2" # Caching Flask-Caching = ">=1.0.0" # Environment variable parsing environs = "==4.2.0" marshmallow = "==2.20.2" # async tasks -celery = "==5.3.1" +celery = ">=5.3.1" celery_once = "==3.0.1" more-itertools = "==5.0.0" -kombu = "==5.3.1" +kombu = ">=5.3.1" # common setting timeout-decorator = "==0.5.0" WTForms = "==3.0.0" @@ -59,6 +59,9 @@ Jinja2 = "==3.1.2" jinja2schema = "==0.1.4" msgpack-python = "==0.5.6" alembic = "==1.7.7" +hvac = "==2.0.0" +colorama = ">=0.4.6" +pycryptodomex = ">=3.19.0" [dev-packages] # Testing @@ -75,4 +78,3 @@ flake8-isort = "==2.7.0" isort = "==4.3.21" pep8-naming = "==0.8.2" pydocstyle = "==3.0.0" - diff --git a/cmdb-api/api/app.py b/cmdb-api/api/app.py index 537abda..6ea299d 100644 --- a/cmdb-api/api/app.py +++ b/cmdb-api/api/app.py @@ -18,7 +18,9 @@ from flask.json.provider import DefaultJSONProvider import api.views.entry from api.extensions import (bcrypt, cache, celery, cors, db, es, login_manager, migrate, rd) +from api.extensions import inner_secrets from api.flask_cas import CAS +from api.lib.secrets.secrets import InnerKVManger from api.models.acl import User HERE = os.path.abspath(os.path.dirname(__file__)) @@ -126,6 +128,10 @@ def register_extensions(app): app.config.update(app.config.get("CELERY")) celery.conf.update(app.config) + if app.config.get('SECRETS_ENGINE') == 'inner': + with app.app_context(): + inner_secrets.init_app(app, InnerKVManger()) + def register_blueprints(app): for item in getmembers(api.views.entry): diff --git a/cmdb-api/api/commands/click_cmdb.py b/cmdb-api/api/commands/click_cmdb.py index a191c00..6c9b774 100644 --- a/cmdb-api/api/commands/click_cmdb.py +++ b/cmdb-api/api/commands/click_cmdb.py @@ -7,6 +7,7 @@ import json import time import click +import requests from flask import current_app from flask.cli import with_appcontext from flask_login import login_user @@ -29,6 +30,9 @@ from api.lib.perm.acl.resource import ResourceCRUD from api.lib.perm.acl.resource import ResourceTypeCRUD from api.lib.perm.acl.role import RoleCRUD from api.lib.perm.acl.user import UserCRUD +from api.lib.secrets.inner import KeyManage +from api.lib.secrets.inner import global_key_threshold +from api.lib.secrets.secrets import InnerKVManger from api.models.acl import App from api.models.acl import ResourceType from api.models.cmdb import Attribute @@ -53,6 +57,7 @@ def cmdb_init_cache(): if relations: rd.create_or_update(relations, REDIS_PREFIX_CI_RELATION) + es = None if current_app.config.get("USE_ES"): from api.extensions import es from api.models.cmdb import Attribute @@ -311,3 +316,128 @@ def cmdb_index_table_upgrade(): CIIndexValueDateTime.create(ci_id=i.ci_id, attr_id=i.attr_id, value=i.value, commit=False) i.delete(commit=False) db.session.commit() + + +@click.command() +@click.option( + '-a', + '--address', + help='inner cmdb api, http://127.0.0.1:8000', +) +@with_appcontext +def cmdb_inner_secrets_init(address): + """ + init inner secrets for password feature + """ + KeyManage(backend=InnerKVManger).init() + + if address and address.startswith("http") and current_app.config.get("INNER_TRIGGER_TOKEN", "") != "": + resp = requests.post("{}/api/v0.1/secrets/auto_seal".format(address.strip("/")), + headers={"Inner-Token": current_app.config.get("INNER_TRIGGER_TOKEN", "")}) + if resp.status_code == 200: + KeyManage.print_response(resp.json()) + else: + KeyManage.print_response({"message": resp.text, "status": "failed"}) + + +@click.command() +@click.option( + '-a', + '--address', + help='inner cmdb api, http://127.0.0.1:8000', + required=True, +) +@with_appcontext +def cmdb_inner_secrets_unseal(address): + """ + unseal the secrets feature + """ + address = "{}/api/v0.1/secrets/unseal".format(address.strip("/")) + if not address.startswith("http"): + KeyManage.print_response({"message": "invalid address, should start with http", "status": "failed"}) + return + for i in range(global_key_threshold): + token = click.prompt(f'Enter unseal token {i + 1}', hide_input=True, confirmation_prompt=False) + assert token is not None + resp = requests.post(address, headers={"Unseal-Token": token}) + if resp.status_code == 200: + KeyManage.print_response(resp.json()) + else: + KeyManage.print_response({"message": resp.text, "status": "failed"}) + return + + +@click.command() +@click.option( + '-a', + '--address', + help='inner cmdb api, http://127.0.0.1:8000', + required=True, +) +@click.option( + '-k', + '--token', + help='root token', + prompt=True, + hide_input=True, +) +@with_appcontext +def cmdb_inner_secrets_seal(address, token): + """ + seal the secrets feature + """ + assert address is not None + assert token is not None + if address.startswith("http"): + address = "{}/api/v0.1/secrets/seal".format(address.strip("/")) + resp = requests.post(address, headers={ + "Inner-Token": token, + }) + if resp.status_code == 200: + KeyManage.print_response(resp.json()) + else: + KeyManage.print_response({"message": resp.text, "status": "failed"}) + + +@click.command() +@with_appcontext +def cmdb_password_data_migrate(): + """ + Migrate CI password data, version >= v2.3.6 + """ + from api.models.cmdb import CIIndexValueText + from api.models.cmdb import CIValueText + from api.lib.secrets.inner import InnerCrypt + from api.lib.secrets.vault import VaultClient + + attrs = Attribute.get_by(to_dict=False) + for attr in attrs: + if attr.is_password: + + value_table = CIIndexValueText if attr.is_index else CIValueText + + for i in value_table.get_by(attr_id=attr.id, to_dict=False): + if current_app.config.get("SECRETS_ENGINE", 'inner') == 'inner': + _, status = InnerCrypt().decrypt(i.value) + if status: + continue + + encrypt_value, status = InnerCrypt().encrypt(i.value) + if status: + CIValueText.create(ci_id=i.ci_id, attr_id=attr.id, value=encrypt_value) + else: + continue + elif current_app.config.get("SECRETS_ENGINE") == 'vault': + if i.value == '******': + continue + + vault = VaultClient(current_app.config.get('VAULT_URL'), current_app.config.get('VAULT_TOKEN')) + try: + vault.update("/{}/{}".format(i.ci_id, i.attr_id), dict(v=i.value)) + except Exception as e: + print('save password to vault failed: {}'.format(e)) + continue + else: + continue + + i.delete() diff --git a/cmdb-api/api/extensions.py b/cmdb-api/api/extensions.py index f540c21..cf68700 100644 --- a/cmdb-api/api/extensions.py +++ b/cmdb-api/api/extensions.py @@ -12,6 +12,9 @@ from flask_sqlalchemy import SQLAlchemy from api.lib.utils import ESHandler from api.lib.utils import RedisHandler +from api.lib.secrets.inner import KeyManage + + bcrypt = Bcrypt() login_manager = LoginManager() db = SQLAlchemy(session_options={"autoflush": False}) @@ -21,3 +24,4 @@ celery = Celery() cors = CORS(supports_credentials=True) rd = RedisHandler() es = ESHandler() +inner_secrets = KeyManage() diff --git a/cmdb-api/api/lib/cmdb/ci.py b/cmdb-api/api/lib/cmdb/ci.py index 3951426..785a2db 100644 --- a/cmdb-api/api/lib/cmdb/ci.py +++ b/cmdb-api/api/lib/cmdb/ci.py @@ -29,6 +29,7 @@ from api.lib.cmdb.const import PermEnum from api.lib.cmdb.const import REDIS_PREFIX_CI from api.lib.cmdb.const import ResourceTypeEnum from api.lib.cmdb.const import RetKey +from api.lib.cmdb.const import ValueTypeEnum from api.lib.cmdb.history import AttributeHistoryManger from api.lib.cmdb.history import CIRelationHistoryManager from api.lib.cmdb.history import CITriggerHistoryManager @@ -42,6 +43,8 @@ from api.lib.notify import notify_send from api.lib.perm.acl.acl import ACLManager from api.lib.perm.acl.acl import is_app_admin from api.lib.perm.acl.acl import validate_permission +from api.lib.secrets.inner import InnerCrypt +from api.lib.secrets.vault import VaultClient from api.lib.utils import Lock from api.lib.utils import handle_arg_list from api.lib.webhook import webhook_request @@ -323,6 +326,8 @@ class CIManager(object): ci_attr2type_attr = {type_attr.attr_id: type_attr for type_attr, _ in attrs} ci = None + record_id = None + password_dict = {} need_lock = current_user.username not in current_app.config.get('PRIVILEGED_USERS', PRIVILEGED_USERS) with Lock(ci_type_name, need_lock=need_lock): existed = cls.ci_is_exist(unique_key, unique_value, ci_type.id) @@ -351,14 +356,23 @@ class CIManager(object): ci_dict.get(attr.name) is None and ci_dict.get(attr.alias) is None)): ci_dict[attr.name] = attr.default.get('default') - if type_attr.is_required and (attr.name not in ci_dict and attr.alias not in ci_dict): + if (type_attr.is_required and not attr.is_computed and + (attr.name not in ci_dict and attr.alias not in ci_dict)): return abort(400, ErrFormat.attribute_value_required.format(attr.name)) else: for type_attr, attr in attrs: if attr.default and attr.default.get('default') == AttributeDefaultValueEnum.UPDATED_AT: ci_dict[attr.name] = now - computed_attrs = [attr.to_dict() for _, attr in attrs if attr.is_computed] or None + computed_attrs = [] + for _, attr in attrs: + if attr.is_computed: + computed_attrs.append(attr.to_dict()) + elif attr.is_password: + if attr.name in ci_dict: + password_dict[attr.id] = ci_dict.pop(attr.name) + elif attr.alias in ci_dict: + password_dict[attr.id] = ci_dict.pop(attr.alias) value_manager = AttributeValueManager() @@ -395,6 +409,10 @@ class CIManager(object): cls.delete(ci.id) raise e + if password_dict: + for attr_id in password_dict: + record_id = cls.save_password(ci.id, attr_id, password_dict[attr_id], record_id, ci_type.id) + if record_id: # has change ci_cache.apply_async(args=(ci.id, operate_type, record_id), queue=CMDB_QUEUE) @@ -414,7 +432,16 @@ class CIManager(object): if attr.default and attr.default.get('default') == AttributeDefaultValueEnum.UPDATED_AT: ci_dict[attr.name] = now - computed_attrs = [attr.to_dict() for _, attr in attrs if attr.is_computed] or None + password_dict = dict() + computed_attrs = list() + for _, attr in attrs: + if attr.is_computed: + computed_attrs.append(attr.to_dict()) + elif attr.is_password: + if attr.name in ci_dict: + password_dict[attr.id] = ci_dict.pop(attr.name) + elif attr.alias in ci_dict: + password_dict[attr.id] = ci_dict.pop(attr.alias) value_manager = AttributeValueManager() @@ -423,6 +450,7 @@ class CIManager(object): limit_attrs = self._valid_ci_for_no_read(ci) if not _is_admin else {} + record_id = None need_lock = current_user.username not in current_app.config.get('PRIVILEGED_USERS', PRIVILEGED_USERS) with Lock(ci.ci_type.name, need_lock=need_lock): self._valid_unique_constraint(ci.type_id, ci_dict, ci_id) @@ -440,6 +468,10 @@ class CIManager(object): except BadRequest as e: raise e + if password_dict: + for attr_id in password_dict: + record_id = self.save_password(ci.id, attr_id, password_dict[attr_id], record_id, ci.type_id) + if record_id: # has change ci_cache.apply_async(args=(ci_id, OperateType.UPDATE, record_id), queue=CMDB_QUEUE) @@ -602,7 +634,7 @@ class CIManager(object): _fields = list() for field in fields: attr = AttributeCache.get(field) - if attr is not None: + if attr is not None and not attr.is_password: _fields.append(str(attr.id)) filter_fields_sql = "WHERE A.attr_id in ({0})".format(",".join(_fields)) @@ -620,7 +652,7 @@ class CIManager(object): ci_dict = dict() unique_id2obj = dict() excludes = excludes and set(excludes) - for ci_id, type_id, attr_id, attr_name, attr_alias, value, value_type, is_list in cis: + for ci_id, type_id, attr_id, attr_name, attr_alias, value, value_type, is_list, is_password in cis: if not fields and excludes and (attr_name in excludes or attr_alias in excludes): continue @@ -647,11 +679,14 @@ class CIManager(object): else: return abort(400, ErrFormat.argument_invalid.format("ret_key")) - value = ValueTypeMap.serialize2[value_type](value) - if is_list: - ci_dict.setdefault(attr_key, []).append(value) + if is_password and value: + ci_dict[attr_key] = '******' else: - ci_dict[attr_key] = value + value = ValueTypeMap.serialize2[value_type](value) + if is_list: + ci_dict.setdefault(attr_key, []).append(value) + else: + ci_dict[attr_key] = value return res @@ -683,6 +718,75 @@ class CIManager(object): return cls._get_cis_from_db(ci_ids, ret_key, fields, value_tables, excludes=excludes) + @classmethod + def save_password(cls, ci_id, attr_id, value, record_id, type_id): + if not value: + return + + changed = None + + value_table = ValueTypeMap.table[ValueTypeEnum.PASSWORD] + if current_app.config.get('SECRETS_ENGINE') == 'inner': + encrypt_value, status = InnerCrypt().encrypt(value) + if not status: + current_app.logger.error('save password failed: {}'.format(encrypt_value)) + return abort(400, ErrFormat.password_save_failed.format(encrypt_value)) + else: + encrypt_value = '******' + + existed = value_table.get_by(ci_id=ci_id, attr_id=attr_id, first=True, to_dict=False) + if existed is None: + value_table.create(ci_id=ci_id, attr_id=attr_id, value=encrypt_value) + changed = [(ci_id, attr_id, OperateType.ADD, '', '******', type_id)] + elif existed.value != encrypt_value: + existed.update(ci_id=ci_id, attr_id=attr_id, value=encrypt_value) + changed = [(ci_id, attr_id, OperateType.UPDATE, '******', '******', type_id)] + + if current_app.config.get('SECRETS_ENGINE') == 'vault': + vault = VaultClient(current_app.config.get('VAULT_URL'), current_app.config.get('VAULT_TOKEN')) + try: + vault.update("/{}/{}".format(ci_id, attr_id), dict(v=value)) + except Exception as e: + current_app.logger.error('save password to vault failed: {}'.format(e)) + return abort(400, ErrFormat.password_save_failed.format('write vault failed')) + + if changed is not None: + AttributeValueManager.write_change2(changed, record_id) + + @classmethod + def load_password(cls, ci_id, attr_id): + ci = CI.get_by_id(ci_id) or abort(404, ErrFormat.ci_not_found.format(ci_id)) + + limit_attrs = cls._valid_ci_for_no_read(ci, ci.ci_type) + if limit_attrs: + attr = AttributeCache.get(attr_id) + if attr and attr.name not in limit_attrs: + return abort(403, ErrFormat.no_permission2) + + if current_app.config.get('SECRETS_ENGINE', 'inner') == 'inner': + value_table = ValueTypeMap.table[ValueTypeEnum.PASSWORD] + v = value_table.get_by(ci_id=ci_id, attr_id=attr_id, first=True, to_dict=False) + + v = v and v.value + if not v: + return + + decrypt_value, status = InnerCrypt().decrypt(v) + if not status: + current_app.logger.error('load password failed: {}'.format(decrypt_value)) + return abort(400, ErrFormat.password_load_failed.format(decrypt_value)) + + return decrypt_value + + elif current_app.config.get('SECRETS_ENGINE') == 'vault': + vault = VaultClient(current_app.config.get('VAULT_URL'), current_app.config.get('VAULT_TOKEN')) + data, status = vault.read("/{}/{}".format(ci_id, attr_id)) + if not status: + current_app.logger.error('read password from vault failed: {}'.format(data)) + return abort(400, ErrFormat.password_load_failed.format(data)) + + return data.get('v') + class CIRelationManager(object): """ diff --git a/cmdb-api/api/lib/cmdb/const.py b/cmdb-api/api/lib/cmdb/const.py index e36bba0..dc9497a 100644 --- a/cmdb-api/api/lib/cmdb/const.py +++ b/cmdb-api/api/lib/cmdb/const.py @@ -12,6 +12,8 @@ class ValueTypeEnum(BaseEnum): DATE = "4" TIME = "5" JSON = "6" + PASSWORD = TEXT + LINK = TEXT class ConstraintEnum(BaseEnum): diff --git a/cmdb-api/api/lib/cmdb/resp_format.py b/cmdb-api/api/lib/cmdb/resp_format.py index 5c64abb..ef040be 100644 --- a/cmdb-api/api/lib/cmdb/resp_format.py +++ b/cmdb-api/api/lib/cmdb/resp_format.py @@ -95,3 +95,6 @@ class ErrFormat(CommonErrFormat): ci_filter_perm_cannot_or_query = "CI过滤授权 暂时不支持 或 查询" ci_filter_perm_attr_no_permission = "您没有属性 {} 的操作权限!" ci_filter_perm_ci_no_permission = "您没有该CI的操作权限!" + + password_save_failed = "保存密码失败: {}" + password_load_failed = "获取密码失败: {}" diff --git a/cmdb-api/api/lib/cmdb/search/ci/db/query_sql.py b/cmdb-api/api/lib/cmdb/search/ci/db/query_sql.py index 78e43e7..24aa0cb 100644 --- a/cmdb-api/api/lib/cmdb/search/ci/db/query_sql.py +++ b/cmdb-api/api/lib/cmdb/search/ci/db/query_sql.py @@ -7,6 +7,7 @@ QUERY_CIS_BY_VALUE_TABLE = """ attr.alias AS attr_alias, attr.value_type, attr.is_list, + attr.is_password, c_cis.type_id, {0}.ci_id, {0}.attr_id, @@ -26,7 +27,8 @@ QUERY_CIS_BY_IDS = """ A.attr_alias, A.value, A.value_type, - A.is_list + A.is_list, + A.is_password FROM ({1}) AS A {0} ORDER BY A.ci_id; @@ -43,7 +45,7 @@ FACET_QUERY1 = """ FACET_QUERY = """ SELECT {0}.value, - count({0}.ci_id) + count(distinct {0}.ci_id) FROM {0} INNER JOIN ({1}) AS F ON F.ci_id={0}.ci_id WHERE {0}.attr_id={2:d} diff --git a/cmdb-api/api/lib/cmdb/utils.py b/cmdb-api/api/lib/cmdb/utils.py index e529670..cf05466 100644 --- a/cmdb-api/api/lib/cmdb/utils.py +++ b/cmdb-api/api/lib/cmdb/utils.py @@ -37,6 +37,8 @@ class ValueTypeMap(object): ValueTypeEnum.DATETIME: str2datetime, ValueTypeEnum.DATE: str2datetime, ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x, + ValueTypeEnum.PASSWORD: lambda x: x, + ValueTypeEnum.LINK: lambda x: x, } serialize = { @@ -47,6 +49,8 @@ class ValueTypeMap(object): ValueTypeEnum.DATE: lambda x: x.strftime("%Y-%m-%d") if not isinstance(x, six.string_types) else x, ValueTypeEnum.DATETIME: lambda x: x.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(x, six.string_types) else x, ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x, + ValueTypeEnum.PASSWORD: lambda x: x if isinstance(x, six.string_types) else str(x), + ValueTypeEnum.LINK: lambda x: x if isinstance(x, six.string_types) else str(x), } serialize2 = { @@ -57,6 +61,8 @@ class ValueTypeMap(object): ValueTypeEnum.DATE: lambda x: (x.decode() if not isinstance(x, six.string_types) else x).split()[0], ValueTypeEnum.DATETIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x, ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x, + ValueTypeEnum.PASSWORD: lambda x: x.decode() if not isinstance(x, six.string_types) else x, + ValueTypeEnum.LINK: lambda x: x.decode() if not isinstance(x, six.string_types) else x, } choice = { @@ -71,6 +77,8 @@ class ValueTypeMap(object): table = { ValueTypeEnum.TEXT: model.CIValueText, ValueTypeEnum.JSON: model.CIValueJson, + ValueTypeEnum.PASSWORD: model.CIValueText, + ValueTypeEnum.LINK: model.CIValueText, 'index_{0}'.format(ValueTypeEnum.INT): model.CIIndexValueInteger, 'index_{0}'.format(ValueTypeEnum.TEXT): model.CIIndexValueText, 'index_{0}'.format(ValueTypeEnum.DATETIME): model.CIIndexValueDateTime, @@ -83,6 +91,8 @@ class ValueTypeMap(object): table_name = { ValueTypeEnum.TEXT: 'c_value_texts', ValueTypeEnum.JSON: 'c_value_json', + ValueTypeEnum.PASSWORD: 'c_value_texts', + ValueTypeEnum.LINK: 'c_value_texts', 'index_{0}'.format(ValueTypeEnum.INT): 'c_value_index_integers', 'index_{0}'.format(ValueTypeEnum.TEXT): 'c_value_index_texts', 'index_{0}'.format(ValueTypeEnum.DATETIME): 'c_value_index_datetime', @@ -100,6 +110,8 @@ class ValueTypeMap(object): ValueTypeEnum.TIME: 'text', ValueTypeEnum.FLOAT: 'float', ValueTypeEnum.JSON: 'object', + ValueTypeEnum.PASSWORD: 'text', + ValueTypeEnum.LINK: 'text', } @@ -112,7 +124,7 @@ class TableMap(object): @property def table(self): attr = AttributeCache.get(self.attr_name) if not self.attr else self.attr - if attr.value_type != ValueTypeEnum.TEXT and attr.value_type != ValueTypeEnum.JSON: + if attr.value_type not in {ValueTypeEnum.TEXT, ValueTypeEnum.JSON, ValueTypeEnum.PASSWORD, ValueTypeEnum.LINK}: self.is_index = True elif self.is_index is None: self.is_index = attr.is_index @@ -124,7 +136,7 @@ class TableMap(object): @property def table_name(self): attr = AttributeCache.get(self.attr_name) if not self.attr else self.attr - if attr.value_type != ValueTypeEnum.TEXT and attr.value_type != ValueTypeEnum.JSON: + if attr.value_type not in {ValueTypeEnum.TEXT, ValueTypeEnum.JSON, ValueTypeEnum.PASSWORD, ValueTypeEnum.LINK}: self.is_index = True elif self.is_index is None: self.is_index = attr.is_index diff --git a/cmdb-api/api/lib/cmdb/value.py b/cmdb-api/api/lib/cmdb/value.py index 43327f0..3da3fbb 100644 --- a/cmdb-api/api/lib/cmdb/value.py +++ b/cmdb-api/api/lib/cmdb/value.py @@ -66,9 +66,10 @@ class AttributeValueManager(object): use_master=use_master, to_dict=False) field_name = getattr(attr, ret_key) - if attr.is_list: res[field_name] = [ValueTypeMap.serialize[attr.value_type](i.value) for i in rs] + elif attr.is_password and rs: + res[field_name] = '******' else: res[field_name] = ValueTypeMap.serialize[attr.value_type](rs[0].value) if rs else None @@ -131,8 +132,7 @@ class AttributeValueManager(object): return AttributeHistoryManger.add(record_id, ci_id, [(attr_id, operate_type, old, new)], type_id) @staticmethod - def _write_change2(changed): - record_id = None + def write_change2(changed, record_id=None): for ci_id, attr_id, operate_type, old, new, type_id in changed: record_id = AttributeHistoryManger.add(record_id, ci_id, [(attr_id, operate_type, old, new)], type_id, commit=False, flush=False) @@ -286,7 +286,7 @@ class AttributeValueManager(object): current_app.logger.warning(str(e)) return abort(400, ErrFormat.attribute_value_unknown_error.format(e.args[0])) - return self._write_change2(changed) + return self.write_change2(changed) @staticmethod def delete_attr_value(attr_id, ci_id): diff --git a/cmdb-api/api/lib/secrets/__init__.py b/cmdb-api/api/lib/secrets/__init__.py new file mode 100644 index 0000000..380474e --- /dev/null +++ b/cmdb-api/api/lib/secrets/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/cmdb-api/api/lib/secrets/inner.py b/cmdb-api/api/lib/secrets/inner.py new file mode 100644 index 0000000..856bc12 --- /dev/null +++ b/cmdb-api/api/lib/secrets/inner.py @@ -0,0 +1,428 @@ +import os +import secrets +import sys +from base64 import b64decode, b64encode + +from colorama import Back +from colorama import Fore +from colorama import init as colorama_init +from colorama import Style +from Cryptodome.Protocol.SecretSharing import Shamir +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives import padding +from cryptography.hazmat.primitives.ciphers import algorithms +from cryptography.hazmat.primitives.ciphers import Cipher +from cryptography.hazmat.primitives.ciphers import modes +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC +from flask import current_app + +global_iv_length = 16 +global_key_shares = 5 # Number of generated key shares +global_key_threshold = 3 # Minimum number of shares required to rebuild the key + +backend_root_key_name = "root_key" +backend_encrypt_key_name = "encrypt_key" +backend_root_key_salt_name = "root_key_salt" +backend_encrypt_key_salt_name = "encrypt_key_salt" +success = "success" +seal_status = True + + +def string_to_bytes(value): + if isinstance(value, bytes): + return value + if sys.version_info.major == 2: + byte_string = value + else: + byte_string = value.encode("utf-8") + + return byte_string + + +class Backend: + def __init__(self, backend=None): + self.backend = backend + + def get(self, key): + return self.backend.get(key) + + def add(self, key, value): + return self.backend.add(key, value) + + +class KeyManage: + + def __init__(self, trigger=None, backend=None): + self.trigger = trigger + self.backend = backend + if backend: + self.backend = Backend(backend) + + def init_app(self, app, backend=None): + self.trigger = app.config.get("INNER_TRIGGER_TOKEN") + if not self.trigger: + return + self.backend = backend + + resp = self.auto_unseal() + self.print_response(resp) + + def hash_root_key(self, value): + algorithm = hashes.SHA256() + salt = self.backend.get(backend_root_key_salt_name) + if not salt: + salt = secrets.token_hex(16) + msg, ok = self.backend.add(backend_root_key_salt_name, salt) + if not ok: + return msg, ok + + kdf = PBKDF2HMAC( + algorithm=algorithm, + length=32, + salt=string_to_bytes(salt), + iterations=100000, + ) + key = kdf.derive(string_to_bytes(value)) + + return b64encode(key).decode('utf-8'), True + + def generate_encrypt_key(self, key): + algorithm = hashes.SHA256() + salt = self.backend.get(backend_encrypt_key_salt_name) + if not salt: + salt = secrets.token_hex(32) + + kdf = PBKDF2HMAC( + algorithm=algorithm, + length=32, + salt=string_to_bytes(salt), + iterations=100000, + backend=default_backend() + ) + key = kdf.derive(string_to_bytes(key)) + msg, ok = self.backend.add(backend_encrypt_key_salt_name, salt) + if ok: + return b64encode(key).decode('utf-8'), ok + else: + return msg, ok + + @classmethod + def generate_keys(cls, secret): + shares = Shamir.split(global_key_threshold, global_key_shares, secret, False) + new_shares = [] + for share in shares: + t = [i for i in share[1]] + [ord(i) for i in "{:0>2}".format(share[0])] + new_shares.append(b64encode(bytes(t))) + + return new_shares + + def auth_root_secret(self, root_key): + root_key_hash, ok = self.hash_root_key(root_key) + if not ok: + return { + "message": root_key_hash, + "status": "failed" + } + + backend_root_key_hash = self.backend.get(backend_root_key_name) + if not backend_root_key_hash: + return { + "message": "should init firstly", + "status": "failed" + } + elif backend_root_key_hash != root_key_hash: + return { + "message": "invalid root key", + "status": "failed" + } + + encrypt_key_aes = self.backend.get(backend_encrypt_key_name) + if not encrypt_key_aes: + return { + "message": "encrypt key is empty", + "status": "failed" + } + + secrets_encrypt_key, ok = InnerCrypt.aes_decrypt(string_to_bytes(root_key), encrypt_key_aes) + if ok: + current_app.config["secrets_encrypt_key"] = secrets_encrypt_key + current_app.config["secrets_root_key"] = root_key + current_app.config["secrets_shares"] = [] + return {"message": success, "status": success} + else: + return { + "message": secrets_encrypt_key, + "status": "failed" + } + + def unseal(self, key): + if not self.is_seal(): + return { + "message": "current status is unseal, skip", + "status": "skip" + } + + try: + t = [i for i in b64decode(key)] + v = (int("".join([chr(i) for i in t[-2:]])), bytes(t[:-2])) + shares = current_app.config.get("secrets_shares", []) + if v not in shares: + shares.append(v) + current_app.config["secrets_shares"] = shares + + if len(shares) >= global_key_threshold: + recovered_secret = Shamir.combine(shares[:global_key_threshold], False) + return self.auth_root_secret(b64encode(recovered_secret)) + else: + return { + "message": "waiting for inputting other unseal key {0}/{1}".format(len(shares), + global_key_threshold), + "status": "waiting" + } + except Exception as e: + return { + "message": "invalid token: " + str(e), + "status": "failed" + } + + def generate_unseal_keys(self): + info = self.backend.get(backend_root_key_name) + if info: + return "already exist", [], False + + secret = AESGCM.generate_key(128) + shares = self.generate_keys(secret) + + return b64encode(secret), shares, True + + def init(self): + """ + init the master key, unseal key and store in backend + :return: + """ + root_key = self.backend.get(backend_root_key_name) + if root_key: + return {"message": "already init, skip"}, False + else: + root_key, shares, status = self.generate_unseal_keys() + if not status: + return {"message": root_key}, False + + # hash root key and store in backend + root_key_hash, ok = self.hash_root_key(root_key) + if not ok: + return {"message": root_key_hash}, False + + msg, ok = self.backend.add(backend_root_key_name, root_key_hash) + if not ok: + return {"message": msg}, False + + # generate encrypt key from root_key and store in backend + encrypt_key, ok = self.generate_encrypt_key(root_key) + if not ok: + return {"message": encrypt_key} + + encrypt_key_aes, status = InnerCrypt.aes_encrypt(root_key, encrypt_key) + if not status: + return {"message": encrypt_key_aes} + + msg, ok = self.backend.add(backend_encrypt_key_name, encrypt_key_aes) + if not ok: + return {"message": msg}, False + + current_app.config["secrets_root_key"] = root_key + current_app.config["secrets_encrypt_key"] = encrypt_key + self.print_token(shares, root_token=root_key) + + return {"message": "OK", + "details": { + "root_token": root_key, + "seal_tokens": shares, + }}, True + + def auto_unseal(self): + if not self.trigger: + return { + "message": "trigger config is empty, skip", + "status": "skip" + } + + if self.trigger.startswith("http"): + return { + "message": "todo in next step, skip", + "status": "skip" + } + # TODO + elif len(self.trigger.strip()) == 24: + res = self.auth_root_secret(self.trigger.encode()) + if res.get("status") == success: + return { + "message": success, + "status": success + } + else: + return { + "message": res.get("message"), + "status": "failed" + } + else: + return { + "message": "trigger config is invalid, skip", + "status": "skip" + } + + def seal(self, root_key): + root_key = root_key.encode() + root_key_hash, ok = self.hash_root_key(root_key) + if not ok: + return { + "message": root_key_hash, + "status": "failed" + } + + backend_root_key_hash = self.backend.get(backend_root_key_name) + if not backend_root_key_hash: + return { + "message": "not init, seal skip", + "status": "skip" + } + elif root_key_hash != backend_root_key_hash: + return { + "message": "invalid root key", + "status": "failed" + } + else: + current_app.config["secrets_root_key"] = '' + current_app.config["secrets_encrypt_key"] = '' + + return { + "message": success, + "status": success + } + + def is_seal(self): + """ + If there is no initialization or the root key is inconsistent, it is considered to be in a sealed state. + :return: + """ + secrets_root_key = current_app.config.get("secrets_root_key") + root_key = self.backend.get(backend_root_key_name) + if root_key == "" or root_key != secrets_root_key: + return "invalid root key", True + + return "", False + + @classmethod + def print_token(cls, shares, root_token): + """ + data: {"message": "OK", + "details": { + "root_token": root_key, + "seal_tokens": shares, + }} + """ + colorama_init() + print(Style.BRIGHT, "Please be sure to store the Unseal Key in a secure location and avoid losing it." + " The Unseal Key is required to unseal the system every time when it restarts." + " Successful unsealing is necessary to enable the password feature." + Style.RESET_ALL) + + for i, v in enumerate(shares): + print( + "unseal token " + str(i + 1) + ": " + Fore.RED + Back.CYAN + v.decode("utf-8") + Style.RESET_ALL) + print() + + print(Fore.GREEN + "root token: " + root_token.decode("utf-8") + Style.RESET_ALL) + + @classmethod + def print_response(cls, data): + status = data.get("status", "") + message = data.get("message", "") + if status == "skip": + print(Style.BRIGHT, message) + elif status == "failed": + print(Fore.RED, message) + elif status == "waiting": + print(Fore.YELLOW, message) + else: + print(Fore.GREEN, message) + + +class InnerCrypt: + def __init__(self): + secrets_encrypt_key = current_app.config.get("secrets_encrypt_key", "") + self.encrypt_key = b64decode(secrets_encrypt_key.encode("utf-8")) + + def encrypt(self, plaintext): + """ + encrypt method contain aes currently + """ + return self.aes_encrypt(self.encrypt_key, plaintext) + + def decrypt(self, ciphertext): + """ + decrypt method contain aes currently + """ + return self.aes_decrypt(self.encrypt_key, ciphertext) + + @classmethod + def aes_encrypt(cls, key, plaintext): + if isinstance(plaintext, str): + plaintext = string_to_bytes(plaintext) + iv = os.urandom(global_iv_length) + try: + cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=default_backend()) + encryptor = cipher.encryptor() + v_padder = padding.PKCS7(algorithms.AES.block_size).padder() + padded_plaintext = v_padder.update(plaintext) + v_padder.finalize() + ciphertext = encryptor.update(padded_plaintext) + encryptor.finalize() + + return b64encode(iv + ciphertext).decode("utf-8"), True + except Exception as e: + return str(e), False + + @classmethod + def aes_decrypt(cls, key, ciphertext): + try: + s = b64decode(ciphertext.encode("utf-8")) + iv = s[:global_iv_length] + ciphertext = s[global_iv_length:] + cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=default_backend()) + decrypter = cipher.decryptor() + decrypted_padded_plaintext = decrypter.update(ciphertext) + decrypter.finalize() + unpadder = padding.PKCS7(algorithms.AES.block_size).unpadder() + plaintext = unpadder.update(decrypted_padded_plaintext) + unpadder.finalize() + + return plaintext.decode('utf-8'), True + except Exception as e: + return str(e), False + + +if __name__ == "__main__": + + km = KeyManage() + # info, shares, status = km.generate_unseal_keys() + # print(info, shares, status) + # print("..................") + # for i in shares: + # print(b64encode(i[1]).decode()) + + res1, ok1 = km.init() + if not ok1: + print(res1) + # for j in res["details"]["seal_tokens"]: + # r = km.unseal(j) + # if r["status"] != "waiting": + # if r["status"] != "success": + # print("r........", r) + # else: + # print(r) + # break + + t_plaintext = b"Hello, World!" # The plaintext to encrypt + c = InnerCrypt() + t_ciphertext, status1 = c.encrypt(t_plaintext) + print("Ciphertext:", t_ciphertext) + decrypted_plaintext, status2 = c.decrypt(t_ciphertext) + print("Decrypted plaintext:", decrypted_plaintext) diff --git a/cmdb-api/api/lib/secrets/secrets.py b/cmdb-api/api/lib/secrets/secrets.py new file mode 100644 index 0000000..bf24c5f --- /dev/null +++ b/cmdb-api/api/lib/secrets/secrets.py @@ -0,0 +1,23 @@ +from api.models.cmdb import InnerKV + + +class InnerKVManger(object): + def __init__(self): + pass + + @classmethod + def add(cls, key, value): + data = {"key": key, "value": value} + res = InnerKV.create(**data) + if res.key == key: + return "success", True + + return "add failed", False + + @classmethod + def get(cls, key): + res = InnerKV.get_by(first=True, to_dict=False, **{"key": key}) + if not res: + return None + + return res.value diff --git a/cmdb-api/api/lib/secrets/vault.py b/cmdb-api/api/lib/secrets/vault.py new file mode 100644 index 0000000..a5746f5 --- /dev/null +++ b/cmdb-api/api/lib/secrets/vault.py @@ -0,0 +1,141 @@ +from base64 import b64decode +from base64 import b64encode + +import hvac + + +class VaultClient: + def __init__(self, base_url, token, mount_path='cmdb'): + self.client = hvac.Client(url=base_url, token=token) + self.mount_path = mount_path + + def create_app_role(self, role_name, policies): + resp = self.client.create_approle(role_name, policies=policies) + + return resp == 200 + + def delete_app_role(self, role_name): + resp = self.client.delete_approle(role_name) + + return resp == 204 + + def update_app_role_policies(self, role_name, policies): + resp = self.client.update_approle_role(role_name, policies=policies) + + return resp == 204 + + def get_app_role(self, role_name): + resp = self.client.get_approle(role_name) + resp.json() + if resp.status_code == 200: + return resp.json + else: + return {} + + def enable_secrets_engine(self): + resp = self.client.sys.enable_secrets_engine('kv', path=self.mount_path) + resp_01 = self.client.sys.enable_secrets_engine('transit') + + if resp.status_code == 200 and resp_01.status_code == 200: + return resp.json + else: + return {} + + def encrypt(self, plaintext): + response = self.client.secrets.transit.encrypt_data(name='transit-key', plaintext=plaintext) + ciphertext = response['data']['ciphertext'] + + return ciphertext + + # decrypt data + def decrypt(self, ciphertext): + response = self.client.secrets.transit.decrypt_data(name='transit-key', ciphertext=ciphertext) + plaintext = response['data']['plaintext'] + + return plaintext + + def write(self, path, data, encrypt=None): + if encrypt: + for k, v in data.items(): + data[k] = self.encrypt(self.encode_base64(v)) + response = self.client.secrets.kv.v2.create_or_update_secret( + path=path, + secret=data, + mount_point=self.mount_path + ) + + return response + + # read data + def read(self, path, decrypt=True): + try: + response = self.client.secrets.kv.v2.read_secret_version( + path=path, raise_on_deleted_version=False, mount_point=self.mount_path + ) + except Exception as e: + return str(e), False + data = response['data']['data'] + if decrypt: + try: + for k, v in data.items(): + data[k] = self.decode_base64(self.decrypt(v)) + except: + return data, True + + return data, True + + # update data + def update(self, path, data, overwrite=True, encrypt=True): + if encrypt: + for k, v in data.items(): + data[k] = self.encrypt(self.encode_base64(v)) + if overwrite: + response = self.client.secrets.kv.v2.create_or_update_secret( + path=path, + secret=data, + mount_point=self.mount_path + ) + else: + response = self.client.secrets.kv.v2.patch(path=path, secret=data, mount_point=self.mount_path) + + return response + + # delete data + def delete(self, path): + response = self.client.secrets.kv.v2.delete_metadata_and_all_versions( + path=path, + mount_point=self.mount_path + ) + + return response + + # Base64 encode + @classmethod + def encode_base64(cls, data): + encoded_bytes = b64encode(data.encode()) + encoded_string = encoded_bytes.decode() + + return encoded_string + + # Base64 decode + @classmethod + def decode_base64(cls, encoded_string): + decoded_bytes = b64decode(encoded_string) + decoded_string = decoded_bytes.decode() + + return decoded_string + + +if __name__ == "__main__": + _base_url = "http://localhost:8200" + _token = "your token" + + _path = "test001" + # Example + sdk = VaultClient(_base_url, _token) + # sdk.enable_secrets_engine() + _data = {"key1": "value1", "key2": "value2", "key3": "value3"} + _data = sdk.update(_path, _data, overwrite=True, encrypt=True) + print(_data) + _data = sdk.read(_path, decrypt=True) + print(_data) diff --git a/cmdb-api/api/models/cmdb.py b/cmdb-api/api/models/cmdb.py index ae15db0..d6b5491 100644 --- a/cmdb-api/api/models/cmdb.py +++ b/cmdb-api/api/models/cmdb.py @@ -504,3 +504,10 @@ class CIFilterPerms(Model): attr_filter = db.Column(db.Text) rid = db.Column(db.Integer, index=True) + + +class InnerKV(Model): + __tablename__ = "c_kv" + + key = db.Column(db.String(128), index=True) + value = db.Column(db.Text) diff --git a/cmdb-api/api/resource.py b/cmdb-api/api/resource.py index 271d4b0..1fc79f5 100644 --- a/cmdb-api/api/resource.py +++ b/cmdb-api/api/resource.py @@ -46,5 +46,4 @@ def register_resources(resource_path, rest_api): resource_cls.url_prefix = ("",) if isinstance(resource_cls.url_prefix, six.string_types): resource_cls.url_prefix = (resource_cls.url_prefix,) - rest_api.add_resource(resource_cls, *resource_cls.url_prefix) diff --git a/cmdb-api/api/views/cmdb/ci.py b/cmdb-api/api/views/cmdb/ci.py index d287b86..ce39962 100644 --- a/cmdb-api/api/views/cmdb/ci.py +++ b/cmdb-api/api/views/cmdb/ci.py @@ -84,11 +84,10 @@ class CIView(APIView): ci_dict = self._wrap_ci_dict() manager = CIManager() - current_app.logger.debug(ci_dict) ci_id = manager.add(ci_type, exist_policy=exist_policy or ExistPolicy.REJECT, _no_attribute_policy=_no_attribute_policy, - _is_admin=request.values.pop('__is_admin', False), + _is_admin=request.values.pop('__is_admin', None) or False, **ci_dict) return self.jsonify(ci_id=ci_id) @@ -96,7 +95,6 @@ class CIView(APIView): @has_perm_for_ci("ci_id", ResourceTypeEnum.CI, PermEnum.UPDATE, CIManager.get_type) def put(self, ci_id=None): args = request.values - current_app.logger.info(args) ci_type = args.get("ci_type") _no_attribute_policy = args.get("no_attribute_policy", ExistPolicy.IGNORE) @@ -104,14 +102,14 @@ class CIView(APIView): manager = CIManager() if ci_id is not None: manager.update(ci_id, - _is_admin=request.values.pop('__is_admin', False), + _is_admin=request.values.pop('__is_admin', None) or False, **ci_dict) else: request.values.pop('exist_policy', None) ci_id = manager.add(ci_type, exist_policy=ExistPolicy.REPLACE, _no_attribute_policy=_no_attribute_policy, - _is_admin=request.values.pop('__is_admin', False), + _is_admin=request.values.pop('__is_admin', None) or False, **ci_dict) return self.jsonify(ci_id=ci_id) @@ -242,3 +240,13 @@ class CIAutoDiscoveryStatisticsView(APIView): def get(self): return self.jsonify(CIManager.get_ad_statistics()) + + +class CIPasswordView(APIView): + url_prefix = "/ci//attributes//password" + + def get(self, ci_id, attr_id): + return self.jsonify(ci_id=ci_id, attr_id=attr_id, value=CIManager.load_password(ci_id, attr_id)) + + def post(self, ci_id, attr_id): + return self.get(ci_id, attr_id) diff --git a/cmdb-api/api/views/cmdb/inner_secrets.py b/cmdb-api/api/views/cmdb/inner_secrets.py new file mode 100644 index 0000000..512f86d --- /dev/null +++ b/cmdb-api/api/views/cmdb/inner_secrets.py @@ -0,0 +1,38 @@ +from api.resource import APIView +from api.lib.secrets.inner import KeyManage +from api.lib.secrets.secrets import InnerKVManger + +from flask import request, abort + + +class InnerSecretUnSealView(APIView): + url_prefix = "/secrets/unseal" + + def post(self): + unseal_key = request.headers.get("Unseal-Token") + res = KeyManage(backend=InnerKVManger()).unseal(unseal_key) + # if res.get("status") == "failed": + # return abort(400, res.get("message")) + return self.jsonify(**res) + + +class InnerSecretSealView(APIView): + url_prefix = "/secrets/seal" + + def post(self): + unseal_key = request.headers.get("Inner-Token") + res = KeyManage(backend=InnerKVManger()).seal(unseal_key) + # if res.get("status") == "failed": + # return abort(400, res.get("message")) + return self.jsonify(**res) + + +class InnerSecretAutoSealView(APIView): + url_prefix = "/secrets/auto_seal" + + def post(self): + unseal_key = request.headers.get("Inner-Token") + res = KeyManage(backend=InnerKVManger()).seal(unseal_key) + # if res.get("status") == "failed": + # return abort(400, res.get("message")) + return self.jsonify(**res) diff --git a/cmdb-api/requirements.txt b/cmdb-api/requirements.txt index 8f92c05..187cc5f 100644 --- a/cmdb-api/requirements.txt +++ b/cmdb-api/requirements.txt @@ -1,7 +1,7 @@ -i https://mirrors.aliyun.com/pypi/simple alembic==1.7.7 bs4==0.0.1 -celery==5.3.1 +celery>=5.3.1 celery-once==3.0.1 click==8.1.3 elasticsearch==7.17.9 @@ -18,18 +18,18 @@ Flask-RESTful==0.3.10 Flask-SQLAlchemy==2.5.0 future==0.18.3 gunicorn==21.0.1 +hvac==2.0.0 itsdangerous==2.1.2 Jinja2==3.1.2 jinja2schema==0.1.4 jsonschema==4.18.0 -kombu==5.3.1 +kombu>=5.3.1 Mako==1.2.4 MarkupSafe==2.1.3 marshmallow==2.20.2 more-itertools==5.0.0 msgpack-python==0.5.6 Pillow==9.3.0 -pycryptodome==3.12.0 cryptography==41.0.2 PyJWT==2.4.0 PyMySQL==1.1.0 @@ -47,3 +47,7 @@ toposort==1.10 treelib==1.6.1 Werkzeug==2.3.6 WTForms==3.0.0 +shamir~=17.12.0 +hvac~=2.0.0 +pycryptodomex>=3.19.0 +colorama>=0.4.6 diff --git a/cmdb-api/settings.example.py b/cmdb-api/settings.example.py index ed6b6ce..373e6b7 100644 --- a/cmdb-api/settings.example.py +++ b/cmdb-api/settings.example.py @@ -97,3 +97,9 @@ BOOL_TRUE = ['true', 'TRUE', 'True', True, '1', 1, "Yes", "YES", "yes", 'Y', 'y' # # messenger USE_MESSENGER = True + +# # secrets +SECRETS_ENGINE = 'inner' # 'inner' or 'vault' +VAULT_URL = '' +VAULT_TOKEN = '' +INNER_TRIGGER_TOKEN = ''