diff --git a/cmdb-api/Pipfile b/cmdb-api/Pipfile index 38ebd94..5f071d1 100644 --- a/cmdb-api/Pipfile +++ b/cmdb-api/Pipfile @@ -26,14 +26,14 @@ Flask-Bcrypt = "==1.0.1" Flask-Cors = ">=3.0.8" ldap3 = "==2.9.1" pycryptodome = "==3.12.0" -cryptography = "==41.0.2" +cryptography = ">=41.0.2" # Caching Flask-Caching = ">=1.0.0" # Environment variable parsing environs = "==4.2.0" marshmallow = "==2.20.2" # async tasks -celery = "==5.3.1" +celery = ">=5.3.1" celery_once = "==3.0.1" more-itertools = "==5.0.0" kombu = "==5.3.1" diff --git a/cmdb-api/api/commands/click_cmdb.py b/cmdb-api/api/commands/click_cmdb.py index a44965e..08c84e8 100644 --- a/cmdb-api/api/commands/click_cmdb.py +++ b/cmdb-api/api/commands/click_cmdb.py @@ -29,10 +29,9 @@ from api.lib.perm.acl.resource import ResourceCRUD from api.lib.perm.acl.resource import ResourceTypeCRUD from api.lib.perm.acl.role import RoleCRUD from api.lib.perm.acl.user import UserCRUD -from api.lib.secrets.inner import KeyMange -from api.lib.secrets.secrets import InnerKVManger +from api.lib.secrets.inner import KeyManage from api.lib.secrets.inner import global_key_threshold - +from api.lib.secrets.secrets import InnerKVManger from api.models.acl import App from api.models.acl import ResourceType from api.models.cmdb import Attribute @@ -57,6 +56,7 @@ def cmdb_init_cache(): if relations: rd.create_or_update(relations, REDIS_PREFIX_CI_RELATION) + es = None if current_app.config.get("USE_ES"): from api.extensions import es from api.models.cmdb import Attribute @@ -323,25 +323,20 @@ def cmdb_inner_secrets_init(): """ init inner secrets for password feature """ - KeyMange(backend=InnerKVManger).init() + KeyManage(backend=InnerKVManger).init() @click.command() -@click.option( - '-k', - '--token', - help='root token', -) @with_appcontext -def cmdb_inner_secrets_unseal(token): +def cmdb_inner_secrets_unseal(): """ unseal the secrets feature """ for i in range(global_key_threshold): - token = click.prompt(f'Enter token {i+1}', hide_input=True, confirmation_prompt=False) + token = click.prompt(f'Enter token {i + 1}', hide_input=True, confirmation_prompt=False) assert token is not None - res = KeyMange(backend=InnerKVManger).unseal(token) - KeyMange.print_response(res) + res = KeyManage(backend=InnerKVManger).unseal(token) + KeyManage.print_response(res) @click.command() @@ -358,8 +353,8 @@ def cmdb_inner_secrets_seal(token): seal the secrets feature """ assert token is not None - res = KeyMange(backend=InnerKVManger()).seal(token) - KeyMange.print_response(res) + res = KeyManage(backend=InnerKVManger()).seal(token) + KeyManage.print_response(res) @click.command() @@ -368,7 +363,49 @@ def cmdb_inner_secrets_auto_seal(): """ auto seal the secrets feature """ - res = KeyMange(current_app.config.get("INNER_TRIGGER_TOKEN"), backend=InnerKVManger()).auto_unseal() - KeyMange.print_response(res) + res = KeyManage(current_app.config.get("INNER_TRIGGER_TOKEN"), backend=InnerKVManger()).auto_unseal() + KeyManage.print_response(res) +@click.command() +@with_appcontext +def cmdb_password_data_migrate(): + """ + Migrate CI password data, version >= v2.3.6 + """ + from api.models.cmdb import CIIndexValueText + from api.models.cmdb import CIValueText + from api.lib.secrets.inner import InnerCrypt + from api.lib.secrets.vault import VaultClient + + attrs = Attribute.get_by(to_dict=False) + for attr in attrs: + if attr.is_password: + + value_table = CIIndexValueText if attr.is_index else CIValueText + + for i in value_table.get_by(attr_id=attr.id, to_dict=False): + if current_app.config.get("SECRETS_ENGINE", 'inner') == 'inner': + _, status = InnerCrypt().decrypt(i.value) + if status: + continue + + encrypt_value, status = InnerCrypt().encrypt(i.value) + if status: + CIValueText.create(ci_id=i.ci_id, attr_id=attr.id, value=encrypt_value) + else: + continue + elif current_app.config.get("SECRETS_ENGINE") == 'vault': + if i.value == '******': + continue + + vault = VaultClient(current_app.config.get('VAULT_URL'), current_app.config.get('VAULT_TOKEN')) + try: + vault.update("/{}/{}".format(i.ci_id, i.attr_id), dict(v=i.value)) + except Exception as e: + print('save password to vault failed: {}'.format(e)) + continue + else: + continue + + i.delete() diff --git a/cmdb-api/api/lib/cmdb/search/ci/db/query_sql.py b/cmdb-api/api/lib/cmdb/search/ci/db/query_sql.py index 78e43e7..24aa0cb 100644 --- a/cmdb-api/api/lib/cmdb/search/ci/db/query_sql.py +++ b/cmdb-api/api/lib/cmdb/search/ci/db/query_sql.py @@ -7,6 +7,7 @@ QUERY_CIS_BY_VALUE_TABLE = """ attr.alias AS attr_alias, attr.value_type, attr.is_list, + attr.is_password, c_cis.type_id, {0}.ci_id, {0}.attr_id, @@ -26,7 +27,8 @@ QUERY_CIS_BY_IDS = """ A.attr_alias, A.value, A.value_type, - A.is_list + A.is_list, + A.is_password FROM ({1}) AS A {0} ORDER BY A.ci_id; @@ -43,7 +45,7 @@ FACET_QUERY1 = """ FACET_QUERY = """ SELECT {0}.value, - count({0}.ci_id) + count(distinct {0}.ci_id) FROM {0} INNER JOIN ({1}) AS F ON F.ci_id={0}.ci_id WHERE {0}.attr_id={2:d} diff --git a/cmdb-api/api/lib/secrets/inner.py b/cmdb-api/api/lib/secrets/inner.py index d5453d6..5caa6cf 100644 --- a/cmdb-api/api/lib/secrets/inner.py +++ b/cmdb-api/api/lib/secrets/inner.py @@ -1,25 +1,24 @@ +import os +import secrets +import sys from base64 import b64encode, b64decode + +from Cryptodome.Protocol.SecretSharing import Shamir from colorama import Back from colorama import Fore from colorama import Style from colorama import init as colorama_init from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives import padding from cryptography.hazmat.primitives.ciphers import Cipher from cryptography.hazmat.primitives.ciphers import algorithms from cryptography.hazmat.primitives.ciphers import modes from cryptography.hazmat.primitives.ciphers.aead import AESGCM -from cryptography.hazmat.primitives import hashes -from cryptography.hazmat.primitives import padding from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC - -import os -import secrets -import sys -from Cryptodome.Protocol.SecretSharing import Shamir +from flask import current_app # global_root_key just for test here -global_root_key = "" -global_encrypt_key = "" global_iv_length = 16 global_key_shares = 5 # Number of generated key shares global_key_threshold = 3 # Minimum number of shares required to rebuild the key @@ -44,7 +43,7 @@ def string_to_bytes(value): return byte_string -class cache_backend: +class CacheBackend: def __init__(self): pass @@ -62,7 +61,7 @@ class cache_backend: class Backend: def __init__(self, backend=None): if not backend: - self.backend = cache_backend + self.backend = CacheBackend else: self.backend = backend @@ -73,14 +72,13 @@ class Backend: return self.backend.add(key, value) -class KeyMange: +class KeyManage: def __init__(self, trigger=None, backend=None): self.trigger = trigger self.backend = backend if backend: self.backend = Backend(backend) - pass def hash_root_key(self, value): algorithm = hashes.SHA256() @@ -136,7 +134,6 @@ class KeyMange: "status": "failed" } backend_root_key_hash = self.backend.get(backend_root_key_name) - print(root_key, root_key_hash, backend_root_key_hash) if not backend_root_key_hash: return { "message": "should init firstly", @@ -153,12 +150,11 @@ class KeyMange: "message": "encrypt key is empty", "status": "failed" } - global global_encrypt_key - global global_root_key - global global_shares - global_encrypt_key = InnerCrypt.aes_decrypt(string_to_bytes(root_key), encrypt_key_aes) - global_root_key = root_key - global_shares = [] + secrets_encrypt_key = InnerCrypt.aes_decrypt(string_to_bytes(root_key), encrypt_key_aes) + setattr(current_app, 'secrets_encrypt_key', secrets_encrypt_key) + setattr(current_app, 'secrets_root_key', root_key) + setattr(current_app, 'secrets_shares', []) + return { "message": success, "status": success @@ -170,7 +166,7 @@ class KeyMange: "message": "current status is unseal, skip", "status": "skip" } - global global_shares, global_root_key, global_encrypt_key + global global_shares try: t = [i for i in b64decode(key)] v = (int("".join([chr(i) for i in t[-2:]])), bytes(t[:-2])) @@ -197,6 +193,7 @@ class KeyMange: return "already exist", [], False secret = AESGCM.generate_key(128) shares = self.generate_keys(secret) + return b64encode(secret), shares, True def init(self): @@ -229,10 +226,11 @@ class KeyMange: if not ok: return {"message": msg}, False # - global global_root_key, global_encrypt_key - global_root_key = root_key - global_encrypt_key = encrypt_key + setattr(current_app, 'secrets_root_key', root_key) + setattr(current_app, 'secrets_encrypt_key', encrypt_key) + self.print_token(shares, root_token=root_key) + return {"message": "OK", "details": { "root_token": root_key, @@ -289,10 +287,9 @@ class KeyMange: "status": "failed" } else: - global global_root_key - global global_encrypt_key - global_root_key = "" - global_encrypt_key = "" + setattr(current_app, 'secrets_root_key', '') + setattr(current_app, 'secrets_encrypt_key', '') + return { "message": success, "status": success @@ -303,9 +300,11 @@ class KeyMange: If there is no initialization or the root key is inconsistent, it is considered to be in a sealed state. :return: """ + secrets_root_key = getattr(current_app, 'secrets_root_key') root_key = self.backend.get(backend_root_key_name) - if root_key == "" or root_key != global_root_key: + if root_key == "" or root_key != secrets_root_key: return "invalid root key", True + return "", False @classmethod @@ -340,9 +339,11 @@ class KeyMange: else: print(Fore.GREEN, message) + class InnerCrypt: - def __init__(self, trigger=None): - self.encrypt_key = b64decode(global_encrypt_key.encode("utf-8")) + def __init__(self): + secrets_encrypt_key = getattr(current_app, 'secrets_encrypt_key', '') + self.encrypt_key = b64decode(secrets_encrypt_key.encode("utf-8")) def encrypt(self, plaintext): """ @@ -389,8 +390,7 @@ class InnerCrypt: if __name__ == "__main__": - print(global_encrypt_key) - km = KeyMange() + km = KeyManage() # info, shares, status = km.generate_unseal_keys() # print(info, shares, status) # print("..................") @@ -415,4 +415,3 @@ if __name__ == "__main__": print("Ciphertext:", t_ciphertext) decrypted_plaintext, status2 = c.decrypt(t_ciphertext) print("Decrypted plaintext:", decrypted_plaintext) - print(global_encrypt_key) diff --git a/cmdb-api/api/lib/secrets/vault.py b/cmdb-api/api/lib/secrets/vault.py index 85bc6db..a5746f5 100644 --- a/cmdb-api/api/lib/secrets/vault.py +++ b/cmdb-api/api/lib/secrets/vault.py @@ -136,6 +136,6 @@ if __name__ == "__main__": # sdk.enable_secrets_engine() _data = {"key1": "value1", "key2": "value2", "key3": "value3"} _data = sdk.update(_path, _data, overwrite=True, encrypt=True) - print(_data.status_code) + print(_data) _data = sdk.read(_path, decrypt=True) print(_data) diff --git a/cmdb-api/api/views/cmdb/secrets.py b/cmdb-api/api/views/cmdb/secrets.py index d2e76fd..b98215f 100644 --- a/cmdb-api/api/views/cmdb/secrets.py +++ b/cmdb-api/api/views/cmdb/secrets.py @@ -1,6 +1,6 @@ from api.resource import APIView from api.models.cmdb import InnerKV -from api.lib.secrets.inner import KeyMange +from api.lib.secrets.inner import KeyManage from flask import request, abort @@ -10,7 +10,7 @@ class InnerSecretUnSealView(APIView): def post(self): unseal_key = request.headers.get("Inner-Token") - res = KeyMange(InnerKV()).unseal(unseal_key) + res = KeyManage(InnerKV()).unseal(unseal_key) if res.get("status") == "failed": return abort(400, res.get("message")) return self.jsonify(**res) @@ -21,7 +21,7 @@ class InnerSecretSealView(APIView): def post(self): unseal_key = request.headers.get("Inner-Token") - res = KeyMange(InnerKV()).seal(unseal_key) + res = KeyManage(InnerKV()).seal(unseal_key) if res.get("status") == "failed": return abort(400, res.get("message")) return self.jsonify(**res) diff --git a/cmdb-api/requirements.txt b/cmdb-api/requirements.txt index ab06a17..19152db 100644 --- a/cmdb-api/requirements.txt +++ b/cmdb-api/requirements.txt @@ -1,7 +1,7 @@ -i https://mirrors.aliyun.com/pypi/simple alembic==1.7.7 bs4==0.0.1 -celery==5.3.1 +celery>=5.3.1 celery-once==3.0.1 click==8.1.3 elasticsearch==7.17.9