feat: secrets

cmdb-api/Pipfile

@@ -26,14 +26,14 @@ Flask-Bcrypt = "==1.0.1"
 Flask-Cors = ">=3.0.8"
 ldap3 = "==2.9.1"
 pycryptodome = "==3.12.0"
-cryptography = "==41.0.2"
+cryptography = ">=41.0.2"
 # Caching
 Flask-Caching = ">=1.0.0"
 # Environment variable parsing
 environs = "==4.2.0"
 marshmallow = "==2.20.2"
 # async tasks
-celery = "==5.3.1"
+celery = ">=5.3.1"
 celery_once = "==3.0.1"
 more-itertools = "==5.0.0"
 kombu = "==5.3.1"

cmdb-api/api/commands/click_cmdb.py

@@ -29,10 +29,9 @@ from api.lib.perm.acl.resource import ResourceCRUD
 from api.lib.perm.acl.resource import ResourceTypeCRUD
 from api.lib.perm.acl.role import RoleCRUD
 from api.lib.perm.acl.user import UserCRUD
-from api.lib.secrets.inner import KeyMange
+from api.lib.secrets.inner import KeyManage
-from api.lib.secrets.secrets import InnerKVManger
 from api.lib.secrets.inner import global_key_threshold
-
+from api.lib.secrets.secrets import InnerKVManger
 from api.models.acl import App
 from api.models.acl import ResourceType
 from api.models.cmdb import Attribute
@@ -57,6 +56,7 @@ def cmdb_init_cache():
     if relations:
         rd.create_or_update(relations, REDIS_PREFIX_CI_RELATION)
 
+    es = None
     if current_app.config.get("USE_ES"):
         from api.extensions import es
         from api.models.cmdb import Attribute
@@ -323,25 +323,20 @@ def cmdb_inner_secrets_init():
     """
     init inner secrets for password feature
     """
-    KeyMange(backend=InnerKVManger).init()
+    KeyManage(backend=InnerKVManger).init()
 
 
 @click.command()
-@click.option(
-    '-k',
-    '--token',
-    help='root token',
-)
 @with_appcontext
-def cmdb_inner_secrets_unseal(token):
+def cmdb_inner_secrets_unseal():
     """
     unseal the secrets feature
     """
     for i in range(global_key_threshold):
         token = click.prompt(f'Enter token {i + 1}', hide_input=True, confirmation_prompt=False)
         assert token is not None
-        res = KeyMange(backend=InnerKVManger).unseal(token)
+        res = KeyManage(backend=InnerKVManger).unseal(token)
-        KeyMange.print_response(res)
+        KeyManage.print_response(res)
 
 
 @click.command()
@@ -358,8 +353,8 @@ def cmdb_inner_secrets_seal(token):
     seal the secrets feature
     """
     assert token is not None
-    res = KeyMange(backend=InnerKVManger()).seal(token)
+    res = KeyManage(backend=InnerKVManger()).seal(token)
-    KeyMange.print_response(res)
+    KeyManage.print_response(res)
 
 
 @click.command()
@@ -368,7 +363,49 @@ def cmdb_inner_secrets_auto_seal():
     """
     auto seal the secrets feature
     """
-    res = KeyMange(current_app.config.get("INNER_TRIGGER_TOKEN"), backend=InnerKVManger()).auto_unseal()
+    res = KeyManage(current_app.config.get("INNER_TRIGGER_TOKEN"), backend=InnerKVManger()).auto_unseal()
-    KeyMange.print_response(res)
+    KeyManage.print_response(res)
 
 
+@click.command()
+@with_appcontext
+def cmdb_password_data_migrate():
+    """
+    Migrate CI password data, version >= v2.3.6
+    """
+    from api.models.cmdb import CIIndexValueText
+    from api.models.cmdb import CIValueText
+    from api.lib.secrets.inner import InnerCrypt
+    from api.lib.secrets.vault import VaultClient
+
+    attrs = Attribute.get_by(to_dict=False)
+    for attr in attrs:
+        if attr.is_password:
+
+            value_table = CIIndexValueText if attr.is_index else CIValueText
+
+            for i in value_table.get_by(attr_id=attr.id, to_dict=False):
+                if current_app.config.get("SECRETS_ENGINE", 'inner') == 'inner':
+                    _, status = InnerCrypt().decrypt(i.value)
+                    if status:
+                        continue
+
+                    encrypt_value, status = InnerCrypt().encrypt(i.value)
+                    if status:
+                        CIValueText.create(ci_id=i.ci_id, attr_id=attr.id, value=encrypt_value)
+                    else:
+                        continue
+                elif current_app.config.get("SECRETS_ENGINE") == 'vault':
+                    if i.value == '******':
+                        continue
+
+                    vault = VaultClient(current_app.config.get('VAULT_URL'), current_app.config.get('VAULT_TOKEN'))
+                    try:
+                        vault.update("/{}/{}".format(i.ci_id, i.attr_id), dict(v=i.value))
+                    except Exception as e:
+                        print('save password to vault failed: {}'.format(e))
+                        continue
+                else:
+                    continue
+
+                i.delete()

cmdb-api/api/lib/cmdb/search/ci/db/query_sql.py

@@ -7,6 +7,7 @@ QUERY_CIS_BY_VALUE_TABLE = """
           attr.alias AS attr_alias,
           attr.value_type,
           attr.is_list,
+          attr.is_password,
           c_cis.type_id,
           {0}.ci_id,
           {0}.attr_id,
@@ -26,7 +27,8 @@ QUERY_CIS_BY_IDS = """
            A.attr_alias,
            A.value,
            A.value_type,
-           A.is_list
+           A.is_list,
+           A.is_password
     FROM
       ({1}) AS A {0}
        ORDER BY A.ci_id;
@@ -43,7 +45,7 @@ FACET_QUERY1 = """
 
 FACET_QUERY = """
     SELECT {0}.value,
-           count({0}.ci_id)
+           count(distinct {0}.ci_id)
     FROM {0}
     INNER JOIN ({1}) AS F ON F.ci_id={0}.ci_id
     WHERE {0}.attr_id={2:d}

cmdb-api/api/lib/secrets/inner.py

@@ -1,25 +1,24 @@
+import os
+import secrets
+import sys
 from base64 import b64encode, b64decode
+
+from Cryptodome.Protocol.SecretSharing import Shamir
 from colorama import Back
 from colorama import Fore
 from colorama import Style
 from colorama import init as colorama_init
 from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives import padding
 from cryptography.hazmat.primitives.ciphers import Cipher
 from cryptography.hazmat.primitives.ciphers import algorithms
 from cryptography.hazmat.primitives.ciphers import modes
 from cryptography.hazmat.primitives.ciphers.aead import AESGCM
-from cryptography.hazmat.primitives import hashes
-from cryptography.hazmat.primitives import padding
 from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
-
+from flask import current_app
-import os
-import secrets
-import sys
-from Cryptodome.Protocol.SecretSharing import Shamir
 
 # global_root_key just for test here
-global_root_key = ""
-global_encrypt_key = ""
 global_iv_length = 16
 global_key_shares = 5  # Number of generated key shares
 global_key_threshold = 3  # Minimum number of shares required to rebuild the key
@@ -44,7 +43,7 @@ def string_to_bytes(value):
     return byte_string
 
 
-class cache_backend:
+class CacheBackend:
     def __init__(self):
         pass
 
@@ -62,7 +61,7 @@ class cache_backend:
 class Backend:
     def __init__(self, backend=None):
         if not backend:
-            self.backend = cache_backend
+            self.backend = CacheBackend
         else:
             self.backend = backend
 
@@ -73,14 +72,13 @@ class Backend:
         return self.backend.add(key, value)
 
 
-class KeyMange:
+class KeyManage:
 
     def __init__(self, trigger=None, backend=None):
         self.trigger = trigger
         self.backend = backend
         if backend:
             self.backend = Backend(backend)
-        pass
 
     def hash_root_key(self, value):
         algorithm = hashes.SHA256()
@@ -136,7 +134,6 @@ class KeyMange:
                 "status": "failed"
             }
         backend_root_key_hash = self.backend.get(backend_root_key_name)
-        print(root_key, root_key_hash, backend_root_key_hash)
         if not backend_root_key_hash:
             return {
                 "message": "should init firstly",
@@ -153,12 +150,11 @@ class KeyMange:
                 "message": "encrypt key is empty",
                 "status": "failed"
             }
-        global global_encrypt_key
+        secrets_encrypt_key = InnerCrypt.aes_decrypt(string_to_bytes(root_key), encrypt_key_aes)
-        global global_root_key
+        setattr(current_app, 'secrets_encrypt_key', secrets_encrypt_key)
-        global global_shares
+        setattr(current_app, 'secrets_root_key', root_key)
-        global_encrypt_key = InnerCrypt.aes_decrypt(string_to_bytes(root_key), encrypt_key_aes)
+        setattr(current_app, 'secrets_shares', [])
-        global_root_key = root_key
+
-        global_shares = []
         return {
             "message": success,
             "status": success
@@ -170,7 +166,7 @@ class KeyMange:
                 "message": "current status is unseal, skip",
                 "status": "skip"
             }
-        global global_shares, global_root_key, global_encrypt_key
+        global global_shares
         try:
             t = [i for i in b64decode(key)]
             v = (int("".join([chr(i) for i in t[-2:]])), bytes(t[:-2]))
@@ -197,6 +193,7 @@ class KeyMange:
             return "already exist", [], False
         secret = AESGCM.generate_key(128)
         shares = self.generate_keys(secret)
+
         return b64encode(secret), shares, True
 
     def init(self):
@@ -229,10 +226,11 @@ class KeyMange:
             if not ok:
                 return {"message": msg}, False
             #
-            global global_root_key, global_encrypt_key
+            setattr(current_app, 'secrets_root_key', root_key)
-            global_root_key = root_key
+            setattr(current_app, 'secrets_encrypt_key', encrypt_key)
-            global_encrypt_key = encrypt_key
+
             self.print_token(shares, root_token=root_key)
+
             return {"message": "OK",
                     "details": {
                         "root_token": root_key,
@@ -289,10 +287,9 @@ class KeyMange:
                 "status": "failed"
             }
         else:
-            global global_root_key
+            setattr(current_app, 'secrets_root_key', '')
-            global global_encrypt_key
+            setattr(current_app, 'secrets_encrypt_key', '')
-            global_root_key = ""
+
-            global_encrypt_key = ""
             return {
                 "message": success,
                 "status": success
@@ -303,9 +300,11 @@ class KeyMange:
         If there is no initialization or the root key is inconsistent, it is considered to be in a sealed state.
         :return:
         """
+        secrets_root_key = getattr(current_app, 'secrets_root_key')
         root_key = self.backend.get(backend_root_key_name)
-        if root_key == "" or root_key != global_root_key:
+        if root_key == "" or root_key != secrets_root_key:
             return "invalid root key", True
+
         return "", False
 
     @classmethod
@@ -340,9 +339,11 @@ class KeyMange:
         else:
             print(Fore.GREEN, message)
 
+
 class InnerCrypt:
-    def __init__(self, trigger=None):
+    def __init__(self):
-        self.encrypt_key = b64decode(global_encrypt_key.encode("utf-8"))
+        secrets_encrypt_key = getattr(current_app, 'secrets_encrypt_key', '')
+        self.encrypt_key = b64decode(secrets_encrypt_key.encode("utf-8"))
 
     def encrypt(self, plaintext):
         """
@@ -389,8 +390,7 @@ class InnerCrypt:
 
 if __name__ == "__main__":
 
-    print(global_encrypt_key)
+    km = KeyManage()
-    km = KeyMange()
     # info, shares, status = km.generate_unseal_keys()
     # print(info, shares, status)
     # print("..................")
@@ -415,4 +415,3 @@ if __name__ == "__main__":
     print("Ciphertext:", t_ciphertext)
     decrypted_plaintext, status2 = c.decrypt(t_ciphertext)
     print("Decrypted plaintext:", decrypted_plaintext)
-    print(global_encrypt_key)

cmdb-api/api/lib/secrets/vault.py

@@ -136,6 +136,6 @@ if __name__ == "__main__":
     # sdk.enable_secrets_engine()
     _data = {"key1": "value1", "key2": "value2", "key3": "value3"}
     _data = sdk.update(_path, _data, overwrite=True, encrypt=True)
-    print(_data.status_code)
+    print(_data)
     _data = sdk.read(_path, decrypt=True)
     print(_data)

cmdb-api/api/views/cmdb/secrets.py

@@ -1,6 +1,6 @@
 from api.resource import APIView
 from api.models.cmdb import InnerKV
-from api.lib.secrets.inner import KeyMange
+from api.lib.secrets.inner import KeyManage
 
 from flask import request, abort
 
@@ -10,7 +10,7 @@ class InnerSecretUnSealView(APIView):
 
     def post(self):
         unseal_key = request.headers.get("Inner-Token")
-        res = KeyMange(InnerKV()).unseal(unseal_key)
+        res = KeyManage(InnerKV()).unseal(unseal_key)
         if res.get("status") == "failed":
             return abort(400, res.get("message"))
         return self.jsonify(**res)
@@ -21,7 +21,7 @@ class InnerSecretSealView(APIView):
 
     def post(self):
         unseal_key = request.headers.get("Inner-Token")
-        res = KeyMange(InnerKV()).seal(unseal_key)
+        res = KeyManage(InnerKV()).seal(unseal_key)
         if res.get("status") == "failed":
             return abort(400, res.get("message"))
         return self.jsonify(**res)

cmdb-api/requirements.txt

@@ -1,7 +1,7 @@
 -i https://mirrors.aliyun.com/pypi/simple
 alembic==1.7.7
 bs4==0.0.1
-celery==5.3.1
+celery>=5.3.1
 celery-once==3.0.1
 click==8.1.3
 elasticsearch==7.17.9