From 379cabbfca61725766a3ce93c7ad49b91f15b435 Mon Sep 17 00:00:00 2001 From: simontigers <47096077+simontigers@users.noreply.github.com> Date: Mon, 11 Dec 2023 18:22:33 +0800 Subject: [PATCH] feat(api): upload file save db (#292) --- cmdb-api/api/commands/click_common_setting.py | 7 +++ .../api/lib/common_setting/resp_format.py | 3 ++ .../api/lib/common_setting/upload_file.py | 52 +++++++++++++++++++ cmdb-api/api/models/common_setting.py | 8 +++ .../api/views/common_setting/file_manage.py | 27 +++++++--- 5 files changed, 89 insertions(+), 8 deletions(-) diff --git a/cmdb-api/api/commands/click_common_setting.py b/cmdb-api/api/commands/click_common_setting.py index a1f325e..7243345 100644 --- a/cmdb-api/api/commands/click_common_setting.py +++ b/cmdb-api/api/commands/click_common_setting.py @@ -299,3 +299,10 @@ def common_check_new_columns(): except Exception as e: current_app.logger.error(f"add new column [{column.name}] in table [{table_name}] err:") current_app.logger.error(e) + + +@click.command() +@with_appcontext +def common_sync_file_to_db(): + from api.lib.common_setting.upload_file import CommonFileCRUD + CommonFileCRUD.sync_file_to_db() diff --git a/cmdb-api/api/lib/common_setting/resp_format.py b/cmdb-api/api/lib/common_setting/resp_format.py index 4c2d6f7..982a2a7 100644 --- a/cmdb-api/api/lib/common_setting/resp_format.py +++ b/cmdb-api/api/lib/common_setting/resp_format.py @@ -8,6 +8,9 @@ class ErrFormat(CommonErrFormat): no_file_part = "没有文件部分" file_is_required = "文件是必须的" + file_not_found = "文件不存在" + file_type_not_allowed = "文件类型不允许" + upload_failed = "上传失败: {}" direct_supervisor_is_not_self = "直属上级不能是自己" parent_department_is_not_self = "上级部门不能是自己" diff --git a/cmdb-api/api/lib/common_setting/upload_file.py b/cmdb-api/api/lib/common_setting/upload_file.py index 85fe697..7e894ba 100644 --- a/cmdb-api/api/lib/common_setting/upload_file.py +++ b/cmdb-api/api/lib/common_setting/upload_file.py @@ -1,6 +1,13 @@ import uuid +import os +from io import BytesIO + +from flask import abort, current_app +import lz4.frame from api.lib.common_setting.utils import get_cur_time_str +from api.models.common_setting import CommonFile +from api.lib.common_setting.resp_format import ErrFormat def allowed_file(filename, allowed_extensions): @@ -14,3 +21,48 @@ def generate_new_file_name(name): cur_str = get_cur_time_str('_') return f"{prev_name}_{cur_str}_{uid}.{ext}" + + +class CommonFileCRUD: + @staticmethod + def add_file(**kwargs): + return CommonFile.create(**kwargs) + + @staticmethod + def get_file(file_name): + existed = CommonFile.get_by(file_name=file_name, first=True, to_dict=False) + if not existed: + abort(400, ErrFormat.file_not_found) + + uncompressed_data = lz4.frame.decompress(existed.binary) + + return BytesIO(uncompressed_data) + + @staticmethod + def sync_file_to_db(): + for p in ['UPLOAD_DIRECTORY_FULL']: + upload_path = current_app.config.get(p, None) + if not upload_path: + continue + for root, dirs, files in os.walk(upload_path): + for file in files: + file_path = os.path.join(root, file) + if not os.path.isfile(file_path): + continue + + existed = CommonFile.get_by(file_name=file, first=True, to_dict=False) + if existed: + continue + with open(file_path, 'rb') as f: + data = f.read() + compressed_data = lz4.frame.compress(data) + try: + CommonFileCRUD.add_file( + origin_name=file, + file_name=file, + binary=compressed_data + ) + + current_app.logger.info(f'sync file {file} to db') + except Exception as e: + current_app.logger.error(f'sync file {file} to db error: {e}') diff --git a/cmdb-api/api/models/common_setting.py b/cmdb-api/api/models/common_setting.py index a141ee8..f1f5404 100644 --- a/cmdb-api/api/models/common_setting.py +++ b/cmdb-api/api/models/common_setting.py @@ -96,3 +96,11 @@ class NoticeConfig(Model): platform = db.Column(db.VARCHAR(255), nullable=False) info = db.Column(db.JSON) + + +class CommonFile(Model): + __tablename__ = 'common_file' + + file_name = db.Column(db.VARCHAR(512), nullable=False, index=True) + origin_name = db.Column(db.VARCHAR(512), nullable=False) + binary = db.Column(db.LargeBinary(16777216), nullable=False) diff --git a/cmdb-api/api/views/common_setting/file_manage.py b/cmdb-api/api/views/common_setting/file_manage.py index 7150365..7bf3152 100644 --- a/cmdb-api/api/views/common_setting/file_manage.py +++ b/cmdb-api/api/views/common_setting/file_manage.py @@ -3,9 +3,10 @@ import os from flask import request, abort, current_app, send_from_directory from werkzeug.utils import secure_filename +import lz4.frame from api.lib.common_setting.resp_format import ErrFormat -from api.lib.common_setting.upload_file import allowed_file, generate_new_file_name +from api.lib.common_setting.upload_file import allowed_file, generate_new_file_name, CommonFileCRUD from api.resource import APIView prefix = '/file' @@ -28,7 +29,8 @@ class GetFileView(APIView): url_prefix = (f'{prefix}/',) def get(self, _filename): - return send_from_directory(current_app.config['UPLOAD_DIRECTORY_FULL'], _filename, as_attachment=True) + file_stream = CommonFileCRUD.get_file(_filename) + return self.send_file(file_stream, as_attachment=True, download_name=_filename) class PostFileView(APIView): @@ -53,11 +55,20 @@ class PostFileView(APIView): filename = file.filename if allowed_file(filename, current_app.config.get('ALLOWED_EXTENSIONS', ALLOWED_EXTENSIONS)): - filename = generate_new_file_name(filename) - filename = secure_filename(filename) - file.save(os.path.join( - current_app.config['UPLOAD_DIRECTORY_FULL'], filename)) + new_filename = generate_new_file_name(filename) + new_filename = secure_filename(new_filename) + file_content = file.read() + compressed_data = lz4.frame.compress(file_content) + try: + CommonFileCRUD.add_file( + origin_name=filename, + file_name=new_filename, + binary=compressed_data, + ) - return self.jsonify(file_name=filename) + return self.jsonify(file_name=new_filename) + except Exception as e: + current_app.logger.error(e) + abort(400, ErrFormat.upload_failed.format(e)) - abort(400, 'Extension not allow') + abort(400, ErrFormat.file_type_not_allowed.format(filename))