feat(api): upload file save db (#292)

cmdb-api/api/commands/click_common_setting.py

@@ -299,3 +299,10 @@ def common_check_new_columns():
                 except Exception as e:
                     current_app.logger.error(f"add new column [{column.name}] in table [{table_name}] err:")
                     current_app.logger.error(e)
+
+
+@click.command()
+@with_appcontext
+def common_sync_file_to_db():
+    from api.lib.common_setting.upload_file import CommonFileCRUD
+    CommonFileCRUD.sync_file_to_db()

cmdb-api/api/lib/common_setting/resp_format.py

@@ -8,6 +8,9 @@ class ErrFormat(CommonErrFormat):
 
     no_file_part = "没有文件部分"
     file_is_required = "文件是必须的"
+    file_not_found = "文件不存在"
+    file_type_not_allowed = "文件类型不允许"
+    upload_failed = "上传失败: {}"
 
     direct_supervisor_is_not_self = "直属上级不能是自己"
     parent_department_is_not_self = "上级部门不能是自己"

cmdb-api/api/lib/common_setting/upload_file.py

@@ -1,6 +1,13 @@
 import uuid
+import os
+from io import BytesIO
+
+from flask import abort, current_app
+import lz4.frame
 
 from api.lib.common_setting.utils import get_cur_time_str
+from api.models.common_setting import CommonFile
+from api.lib.common_setting.resp_format import ErrFormat
 
 
 def allowed_file(filename, allowed_extensions):
@@ -14,3 +21,48 @@ def generate_new_file_name(name):
     cur_str = get_cur_time_str('_')
 
     return f"{prev_name}_{cur_str}_{uid}.{ext}"
+
+
+class CommonFileCRUD:
+    @staticmethod
+    def add_file(**kwargs):
+        return CommonFile.create(**kwargs)
+
+    @staticmethod
+    def get_file(file_name):
+        existed = CommonFile.get_by(file_name=file_name, first=True, to_dict=False)
+        if not existed:
+            abort(400, ErrFormat.file_not_found)
+
+        uncompressed_data = lz4.frame.decompress(existed.binary)
+
+        return BytesIO(uncompressed_data)
+
+    @staticmethod
+    def sync_file_to_db():
+        for p in ['UPLOAD_DIRECTORY_FULL']:
+            upload_path = current_app.config.get(p, None)
+            if not upload_path:
+                continue
+            for root, dirs, files in os.walk(upload_path):
+                for file in files:
+                    file_path = os.path.join(root, file)
+                    if not os.path.isfile(file_path):
+                        continue
+
+                    existed = CommonFile.get_by(file_name=file, first=True, to_dict=False)
+                    if existed:
+                        continue
+                    with open(file_path, 'rb') as f:
+                        data = f.read()
+                    compressed_data = lz4.frame.compress(data)
+                    try:
+                        CommonFileCRUD.add_file(
+                            origin_name=file,
+                            file_name=file,
+                            binary=compressed_data
+                        )
+
+                        current_app.logger.info(f'sync file {file} to db')
+                    except Exception as e:
+                        current_app.logger.error(f'sync file {file} to db error: {e}')

cmdb-api/api/models/common_setting.py

@@ -96,3 +96,11 @@ class NoticeConfig(Model):
 
     platform = db.Column(db.VARCHAR(255), nullable=False)
     info = db.Column(db.JSON)
+
+
+class CommonFile(Model):
+    __tablename__ = 'common_file'
+
+    file_name = db.Column(db.VARCHAR(512), nullable=False, index=True)
+    origin_name = db.Column(db.VARCHAR(512), nullable=False)
+    binary = db.Column(db.LargeBinary(16777216), nullable=False)

cmdb-api/api/views/common_setting/file_manage.py

@@ -3,9 +3,10 @@ import os
 
 from flask import request, abort, current_app, send_from_directory
 from werkzeug.utils import secure_filename
+import lz4.frame
 
 from api.lib.common_setting.resp_format import ErrFormat
-from api.lib.common_setting.upload_file import allowed_file, generate_new_file_name
+from api.lib.common_setting.upload_file import allowed_file, generate_new_file_name, CommonFileCRUD
 from api.resource import APIView
 
 prefix = '/file'
@@ -28,7 +29,8 @@ class GetFileView(APIView):
     url_prefix = (f'{prefix}/<string:_filename>',)
 
     def get(self, _filename):
-        return send_from_directory(current_app.config['UPLOAD_DIRECTORY_FULL'], _filename, as_attachment=True)
+        file_stream = CommonFileCRUD.get_file(_filename)
+        return self.send_file(file_stream, as_attachment=True, download_name=_filename)
 
 
 class PostFileView(APIView):
@@ -53,11 +55,20 @@ class PostFileView(APIView):
                 filename = file.filename
 
         if allowed_file(filename, current_app.config.get('ALLOWED_EXTENSIONS', ALLOWED_EXTENSIONS)):
-            filename = generate_new_file_name(filename)
+            new_filename = generate_new_file_name(filename)
-            filename = secure_filename(filename)
+            new_filename = secure_filename(new_filename)
-            file.save(os.path.join(
+            file_content = file.read()
-                current_app.config['UPLOAD_DIRECTORY_FULL'], filename))
+            compressed_data = lz4.frame.compress(file_content)
+            try:
+                CommonFileCRUD.add_file(
+                    origin_name=filename,
+                    file_name=new_filename,
+                    binary=compressed_data,
+                )
 
-            return self.jsonify(file_name=filename)
+                return self.jsonify(file_name=new_filename)
+            except Exception as e:
+                current_app.logger.error(e)
+                abort(400, ErrFormat.upload_failed.format(e))
 
-        abort(400, 'Extension not allow')
+        abort(400, ErrFormat.file_type_not_allowed.format(filename))