feat(db): support flask migrate (#201)

Co-authored-by: s01249 <songbing@smyfinancial.com>
This commit is contained in:
Evan Sung 2023-10-13 02:55:26 -05:00 committed by GitHub
parent f033af2f58
commit 41d810642b
7 changed files with 558 additions and 1 deletions

View File

@ -7,6 +7,7 @@ import os
import sys import sys
from inspect import getmembers from inspect import getmembers
from logging.handlers import RotatingFileHandler from logging.handlers import RotatingFileHandler
from pathlib import Path
from flask import Flask from flask import Flask
from flask import jsonify from flask import jsonify
@ -22,6 +23,7 @@ from api.models.acl import User
HERE = os.path.abspath(os.path.dirname(__file__)) HERE = os.path.abspath(os.path.dirname(__file__))
PROJECT_ROOT = os.path.join(HERE, os.pardir) PROJECT_ROOT = os.path.join(HERE, os.pardir)
BASE_DIR = Path(__file__).resolve().parent.parent
@login_manager.user_loader @login_manager.user_loader
@ -116,7 +118,7 @@ def register_extensions(app):
db.init_app(app) db.init_app(app)
cors.init_app(app) cors.init_app(app)
login_manager.init_app(app) login_manager.init_app(app)
migrate.init_app(app, db) migrate.init_app(app, db, directory=f"{BASE_DIR}/migrations")
rd.init_app(app) rd.init_app(app)
if app.config.get('USE_ES'): if app.config.get('USE_ES'):
es.init_app(app) es.init_app(app)

View File

@ -0,0 +1 @@
Generic single-database configuration.

View File

@ -0,0 +1,45 @@
# A generic, single database configuration.
[alembic]
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

110
cmdb-api/migrations/env.py Normal file
View File

@ -0,0 +1,110 @@
from __future__ import with_statement
import logging
from logging.config import fileConfig
from sqlalchemy import engine_from_config
from sqlalchemy import pool
from alembic import context
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
logger = logging.getLogger('alembic.env')
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
from flask import current_app
config.set_main_option(
'sqlalchemy.url', current_app.config.get(
'SQLALCHEMY_DATABASE_URI').replace('%', '%%'))
target_metadata = current_app.extensions['migrate'].db.metadata
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
# 添加要屏蔽的table列表
exclude_tables = ["c_cfp"]
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url, target_metadata=target_metadata, literal_binds=True,
include_name=include_name
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
# this callback is used to prevent an auto-migration from being generated
# when there are no changes to the schema
# reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
def process_revision_directives(context, revision, directives):
if getattr(config.cmd_opts, 'autogenerate', False):
script = directives[0]
if script.upgrade_ops.is_empty():
directives[:] = []
logger.info('No changes in schema detected.')
connectable = engine_from_config(
config.get_section(config.config_ini_section),
prefix='sqlalchemy.',
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata,
process_revision_directives=process_revision_directives,
include_name=include_name,
**current_app.extensions['migrate'].configure_args
)
with context.begin_transaction():
context.run_migrations()
def include_name(name, type_, parent_names):
if type_ == "table":
return name not in exclude_tables
elif parent_names.get("table_name") in exclude_tables:
return False
return True
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()

View File

@ -0,0 +1,24 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
branch_labels = ${repr(branch_labels)}
depends_on = ${repr(depends_on)}
def upgrade():
${upgrades if upgrades else "pass"}
def downgrade():
${downgrades if downgrades else "pass"}

View File

@ -0,0 +1,360 @@
"""empty message
Revision ID: 6a4df2623057
Revises:
Create Date: 2023-10-13 15:17:00.066858
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
# revision identifiers, used by Alembic.
revision = '6a4df2623057'
down_revision = None
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('common_data',
sa.Column('deleted_at', sa.DateTime(), nullable=True),
sa.Column('deleted', sa.Boolean(), nullable=True),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
sa.Column('data_type', sa.VARCHAR(length=255), nullable=True),
sa.Column('data', sa.JSON(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_common_data_deleted'), 'common_data', ['deleted'], unique=False)
op.create_table('common_notice_config',
sa.Column('deleted_at', sa.DateTime(), nullable=True),
sa.Column('deleted', sa.Boolean(), nullable=True),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
sa.Column('platform', sa.VARCHAR(length=255), nullable=False),
sa.Column('info', sa.JSON(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_common_notice_config_deleted'), 'common_notice_config', ['deleted'], unique=False)
op.add_column('c_attributes', sa.Column('choice_other', sa.JSON(), nullable=True))
op.drop_index('idx_c_attributes_uid', table_name='c_attributes')
op.create_index(op.f('ix_c_attributes_uid'), 'c_attributes', ['uid'], unique=False)
op.drop_index('ix_c_custom_dashboard_deleted', table_name='c_c_d')
op.create_index(op.f('ix_c_c_d_deleted'), 'c_c_d', ['deleted'], unique=False)
op.drop_index('ix_c_ci_type_triggers_deleted', table_name='c_c_t_t')
op.create_index(op.f('ix_c_c_t_t_deleted'), 'c_c_t_t', ['deleted'], unique=False)
op.drop_index('ix_c_ci_type_unique_constraints_deleted', table_name='c_c_t_u_c')
op.create_index(op.f('ix_c_c_t_u_c_deleted'), 'c_c_t_u_c', ['deleted'], unique=False)
op.drop_index('c_ci_types_uid', table_name='c_ci_types')
op.create_index(op.f('ix_c_ci_types_uid'), 'c_ci_types', ['uid'], unique=False)
op.alter_column('c_prv', 'uid',
existing_type=mysql.INTEGER(),
nullable=False)
op.drop_index('ix_c_preference_relation_views_deleted', table_name='c_prv')
op.drop_index('ix_c_preference_relation_views_name', table_name='c_prv')
op.create_index(op.f('ix_c_prv_deleted'), 'c_prv', ['deleted'], unique=False)
op.create_index(op.f('ix_c_prv_name'), 'c_prv', ['name'], unique=False)
op.create_index(op.f('ix_c_prv_uid'), 'c_prv', ['uid'], unique=False)
op.drop_index('ix_c_preference_show_attributes_deleted', table_name='c_psa')
op.drop_index('ix_c_preference_show_attributes_uid', table_name='c_psa')
op.create_index(op.f('ix_c_psa_deleted'), 'c_psa', ['deleted'], unique=False)
op.create_index(op.f('ix_c_psa_uid'), 'c_psa', ['uid'], unique=False)
op.drop_index('ix_c_preference_tree_views_deleted', table_name='c_ptv')
op.drop_index('ix_c_preference_tree_views_uid', table_name='c_ptv')
op.create_index(op.f('ix_c_ptv_deleted'), 'c_ptv', ['deleted'], unique=False)
op.create_index(op.f('ix_c_ptv_uid'), 'c_ptv', ['uid'], unique=False)
op.alter_column('common_department', 'department_name',
existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255),
comment=None,
existing_comment='部门名称',
existing_nullable=True)
op.alter_column('common_department', 'department_director_id',
existing_type=mysql.INTEGER(),
comment=None,
existing_comment='部门负责人ID',
existing_nullable=True)
op.alter_column('common_department', 'department_parent_id',
existing_type=mysql.INTEGER(),
comment=None,
existing_comment='上级部门ID',
existing_nullable=True)
op.alter_column('common_department', 'sort_value',
existing_type=mysql.INTEGER(),
comment=None,
existing_comment='排序值',
existing_nullable=True)
op.alter_column('common_department', 'acl_rid',
existing_type=mysql.INTEGER(),
comment=None,
existing_comment='ACL中rid',
existing_nullable=True)
op.alter_column('common_employee', 'email',
existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255),
comment=None,
existing_comment='邮箱',
existing_nullable=True)
op.alter_column('common_employee', 'username',
existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255),
comment=None,
existing_comment='用户名',
existing_nullable=True)
op.alter_column('common_employee', 'nickname',
existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255),
comment=None,
existing_comment='姓名',
existing_nullable=True)
op.alter_column('common_employee', 'sex',
existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=64),
comment=None,
existing_comment='性别',
existing_nullable=True)
op.alter_column('common_employee', 'position_name',
existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255),
comment=None,
existing_comment='职位名称',
existing_nullable=True)
op.alter_column('common_employee', 'mobile',
existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255),
comment=None,
existing_comment='电话号码',
existing_nullable=True)
op.alter_column('common_employee', 'avatar',
existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255),
comment=None,
existing_comment='头像',
existing_nullable=True)
op.alter_column('common_employee', 'direct_supervisor_id',
existing_type=mysql.INTEGER(),
comment=None,
existing_comment='直接上级ID',
existing_nullable=True)
op.alter_column('common_employee', 'department_id',
existing_type=mysql.INTEGER(),
comment=None,
existing_comment='部门ID',
existing_nullable=True)
op.alter_column('common_employee', 'acl_uid',
existing_type=mysql.INTEGER(),
comment=None,
existing_comment='ACL中uid',
existing_nullable=True)
op.alter_column('common_employee', 'acl_rid',
existing_type=mysql.INTEGER(),
comment=None,
existing_comment='ACL中rid',
existing_nullable=True)
op.alter_column('common_employee', 'acl_virtual_rid',
existing_type=mysql.INTEGER(),
comment=None,
existing_comment='ACL中虚拟角色rid',
existing_nullable=True)
op.alter_column('common_employee', 'last_login',
existing_type=mysql.TIMESTAMP(),
comment=None,
existing_comment='上次登录时间',
existing_nullable=True)
op.alter_column('common_employee', 'block',
existing_type=mysql.INTEGER(),
comment=None,
existing_comment='锁定状态',
existing_nullable=True)
op.alter_column('common_employee_info', 'info',
existing_type=mysql.JSON(),
comment=None,
existing_comment='员工信息',
existing_nullable=True)
op.alter_column('common_employee_info', 'employee_id',
existing_type=mysql.INTEGER(),
comment=None,
existing_comment='员工ID',
existing_nullable=True)
op.alter_column('common_internal_message', 'title',
existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255),
comment=None,
existing_comment='标题',
existing_nullable=True)
op.alter_column('common_internal_message', 'content',
existing_type=mysql.TEXT(charset='utf8mb3', collation='utf8mb3_unicode_ci'),
comment=None,
existing_comment='内容',
existing_nullable=True)
op.alter_column('common_internal_message', 'path',
existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255),
comment=None,
existing_comment='跳转路径',
existing_nullable=True)
op.alter_column('common_internal_message', 'is_read',
existing_type=mysql.TINYINT(display_width=1),
comment=None,
existing_comment='是否已读',
existing_nullable=True)
op.alter_column('common_internal_message', 'app_name',
existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=128),
comment=None,
existing_comment='应用名称',
existing_nullable=False)
op.alter_column('common_internal_message', 'category',
existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=128),
comment=None,
existing_comment='分类',
existing_nullable=False)
op.alter_column('common_internal_message', 'message_data',
existing_type=mysql.JSON(),
comment=None,
existing_comment='数据',
existing_nullable=True)
op.drop_column('users', 'apps')
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('users', sa.Column('apps', mysql.JSON(), nullable=True))
op.alter_column('common_internal_message', 'message_data',
existing_type=mysql.JSON(),
comment='数据',
existing_nullable=True)
op.alter_column('common_internal_message', 'category',
existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=128),
comment='分类',
existing_nullable=False)
op.alter_column('common_internal_message', 'app_name',
existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=128),
comment='应用名称',
existing_nullable=False)
op.alter_column('common_internal_message', 'is_read',
existing_type=mysql.TINYINT(display_width=1),
comment='是否已读',
existing_nullable=True)
op.alter_column('common_internal_message', 'path',
existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255),
comment='跳转路径',
existing_nullable=True)
op.alter_column('common_internal_message', 'content',
existing_type=mysql.TEXT(charset='utf8mb3', collation='utf8mb3_unicode_ci'),
comment='内容',
existing_nullable=True)
op.alter_column('common_internal_message', 'title',
existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255),
comment='标题',
existing_nullable=True)
op.alter_column('common_employee_info', 'employee_id',
existing_type=mysql.INTEGER(),
comment='员工ID',
existing_nullable=True)
op.alter_column('common_employee_info', 'info',
existing_type=mysql.JSON(),
comment='员工信息',
existing_nullable=True)
op.alter_column('common_employee', 'block',
existing_type=mysql.INTEGER(),
comment='锁定状态',
existing_nullable=True)
op.alter_column('common_employee', 'last_login',
existing_type=mysql.TIMESTAMP(),
comment='上次登录时间',
existing_nullable=True)
op.alter_column('common_employee', 'acl_virtual_rid',
existing_type=mysql.INTEGER(),
comment='ACL中虚拟角色rid',
existing_nullable=True)
op.alter_column('common_employee', 'acl_rid',
existing_type=mysql.INTEGER(),
comment='ACL中rid',
existing_nullable=True)
op.alter_column('common_employee', 'acl_uid',
existing_type=mysql.INTEGER(),
comment='ACL中uid',
existing_nullable=True)
op.alter_column('common_employee', 'department_id',
existing_type=mysql.INTEGER(),
comment='部门ID',
existing_nullable=True)
op.alter_column('common_employee', 'direct_supervisor_id',
existing_type=mysql.INTEGER(),
comment='直接上级ID',
existing_nullable=True)
op.alter_column('common_employee', 'avatar',
existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255),
comment='头像',
existing_nullable=True)
op.alter_column('common_employee', 'mobile',
existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255),
comment='电话号码',
existing_nullable=True)
op.alter_column('common_employee', 'position_name',
existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255),
comment='职位名称',
existing_nullable=True)
op.alter_column('common_employee', 'sex',
existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=64),
comment='性别',
existing_nullable=True)
op.alter_column('common_employee', 'nickname',
existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255),
comment='姓名',
existing_nullable=True)
op.alter_column('common_employee', 'username',
existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255),
comment='用户名',
existing_nullable=True)
op.alter_column('common_employee', 'email',
existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255),
comment='邮箱',
existing_nullable=True)
op.alter_column('common_department', 'acl_rid',
existing_type=mysql.INTEGER(),
comment='ACL中rid',
existing_nullable=True)
op.alter_column('common_department', 'sort_value',
existing_type=mysql.INTEGER(),
comment='排序值',
existing_nullable=True)
op.alter_column('common_department', 'department_parent_id',
existing_type=mysql.INTEGER(),
comment='上级部门ID',
existing_nullable=True)
op.alter_column('common_department', 'department_director_id',
existing_type=mysql.INTEGER(),
comment='部门负责人ID',
existing_nullable=True)
op.alter_column('common_department', 'department_name',
existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255),
comment='部门名称',
existing_nullable=True)
op.drop_index(op.f('ix_c_ptv_uid'), table_name='c_ptv')
op.drop_index(op.f('ix_c_ptv_deleted'), table_name='c_ptv')
op.create_index('ix_c_preference_tree_views_uid', 'c_ptv', ['uid'], unique=False)
op.create_index('ix_c_preference_tree_views_deleted', 'c_ptv', ['deleted'], unique=False)
op.drop_index(op.f('ix_c_psa_uid'), table_name='c_psa')
op.drop_index(op.f('ix_c_psa_deleted'), table_name='c_psa')
op.create_index('ix_c_preference_show_attributes_uid', 'c_psa', ['uid'], unique=False)
op.create_index('ix_c_preference_show_attributes_deleted', 'c_psa', ['deleted'], unique=False)
op.drop_index(op.f('ix_c_prv_uid'), table_name='c_prv')
op.drop_index(op.f('ix_c_prv_name'), table_name='c_prv')
op.drop_index(op.f('ix_c_prv_deleted'), table_name='c_prv')
op.create_index('ix_c_preference_relation_views_name', 'c_prv', ['name'], unique=False)
op.create_index('ix_c_preference_relation_views_deleted', 'c_prv', ['deleted'], unique=False)
op.alter_column('c_prv', 'uid',
existing_type=mysql.INTEGER(),
nullable=True)
op.drop_index(op.f('ix_c_ci_types_uid'), table_name='c_ci_types')
op.create_index('c_ci_types_uid', 'c_ci_types', ['uid'], unique=False)
op.drop_index(op.f('ix_c_c_t_u_c_deleted'), table_name='c_c_t_u_c')
op.create_index('ix_c_ci_type_unique_constraints_deleted', 'c_c_t_u_c', ['deleted'], unique=False)
op.drop_index(op.f('ix_c_c_t_t_deleted'), table_name='c_c_t_t')
op.create_index('ix_c_ci_type_triggers_deleted', 'c_c_t_t', ['deleted'], unique=False)
op.drop_index(op.f('ix_c_c_d_deleted'), table_name='c_c_d')
op.create_index('ix_c_custom_dashboard_deleted', 'c_c_d', ['deleted'], unique=False)
op.drop_index(op.f('ix_c_attributes_uid'), table_name='c_attributes')
op.create_index('idx_c_attributes_uid', 'c_attributes', ['uid'], unique=False)
op.drop_column('c_attributes', 'choice_other')
op.drop_index(op.f('ix_common_notice_config_deleted'), table_name='common_notice_config')
op.drop_table('common_notice_config')
op.drop_index(op.f('ix_common_data_deleted'), table_name='common_data')
op.drop_table('common_data')
# ### end Alembic commands ###

15
docs/flask-migrate.md Normal file
View File

@ -0,0 +1,15 @@
## 使用Flask-Migrate做数据库版本管理
- 首次可以删除cmdb-api/migrations/versions下的所有文件
-
### 进入cmdb-api完成下面步骤操作可能会删除数据库中不被代码管理的表如需保留请看文末中的tips
- 如果是首次使用需要先删除cmdb-api/migrations/versions下的所有文件非首次跳过
- 执行`flask db migrate` 生成对应版本数据库表的升级文件到versions文件夹下需要你的数据库是已经upgrade的
- 执行`flask db upgrade` 数据库表同步更新到mysql
### tips
- cmdb-api/migrations/env.py文件内的exclude_tables列表可以填写不想被flask-migrate管理的数据库表