mirror of https://github.com/veops/cmdb.git
Merge pull request #135 from simontigers/remove_pandas
fix: remove pandas
This commit is contained in:
pycook
GitHub
commit
c3cbdc606f
|
|
@ -8,40 +8,24 @@ from wtforms import StringField
|
|||
from wtforms import validators
|
||||
|
||||
from api.lib.common_setting.resp_format import ErrFormat
|
||||
from api.lib.common_setting.utils import get_df_from_read_sql
|
||||
from api.lib.perm.acl.role import RoleCRUD
|
||||
from api.models.common_setting import Department, Employee
|
||||
|
||||
sub_departments_column_name = 'sub_departments'
|
||||
|
||||
|
||||
def drop_ts_column(df):
|
||||
columns = list(df.columns)
|
||||
remove_columns = []
|
||||
for column in ['created_at', 'updated_at', 'deleted_at', 'last_login']:
|
||||
targets = list(filter(lambda c: c.startswith(column), columns))
|
||||
if targets:
|
||||
remove_columns.extend(targets)
|
||||
|
||||
remove_columns = list(set(remove_columns))
|
||||
|
||||
return df.drop(remove_columns, axis=1) if len(remove_columns) > 0 else df
|
||||
|
||||
|
||||
def get_department_df():
|
||||
def get_all_department_list(to_dict=True):
|
||||
criterion = [
|
||||
Department.deleted == 0,
|
||||
]
|
||||
query = Department.query.filter(
|
||||
*criterion
|
||||
)
|
||||
df = get_df_from_read_sql(query)
|
||||
if df.empty:
|
||||
return
|
||||
return drop_ts_column(df)
|
||||
).order_by(Department.department_id.asc())
|
||||
results = query.all()
|
||||
return [r.to_dict() for r in results] if to_dict else results
|
||||
|
||||
|
||||
def get_all_employee_df(block=0):
|
||||
def get_all_employee_list(block=0, to_dict=True):
|
||||
criterion = [
|
||||
Employee.deleted == 0,
|
||||
]
|
||||
|
|
@ -50,112 +34,110 @@ def get_all_employee_df(block=0):
|
|||
Employee.block == block
|
||||
)
|
||||
|
||||
entities = [getattr(Employee, c) for c in Employee.get_columns(
|
||||
).keys() if c not in ['deleted', 'deleted_at']]
|
||||
query = Employee.query.with_entities(
|
||||
*entities
|
||||
).filter(
|
||||
*criterion
|
||||
)
|
||||
df = get_df_from_read_sql(query)
|
||||
if df.empty:
|
||||
return df
|
||||
return drop_ts_column(df)
|
||||
results = db.session.query(Employee).filter(*criterion).all()
|
||||
|
||||
DepartmentTreeEmployeeColumns = [
|
||||
'acl_rid',
|
||||
'employee_id',
|
||||
'username',
|
||||
'nickname',
|
||||
'email',
|
||||
'mobile',
|
||||
'direct_supervisor_id',
|
||||
'annual_leave',
|
||||
'block',
|
||||
'department_id',
|
||||
]
|
||||
|
||||
def format_columns(e):
|
||||
return {column: getattr(e, column) for column in DepartmentTreeEmployeeColumns}
|
||||
|
||||
return [format_columns(r) for r in results] if to_dict else results
|
||||
|
||||
|
||||
class DepartmentTree(object):
|
||||
def __init__(self, append_employee=False, block=-1):
|
||||
self.append_employee = append_employee
|
||||
self.block = block
|
||||
self.d_df = get_department_df()
|
||||
self.employee_df = get_all_employee_df(
|
||||
self.all_department_list = get_all_department_list()
|
||||
self.all_employee_list = get_all_employee_list(
|
||||
block) if append_employee else None
|
||||
|
||||
def prepare(self):
|
||||
pass
|
||||
|
||||
def get_employees_by_d_id(self, d_id):
|
||||
_df = self.employee_df[
|
||||
self.employee_df['department_id'].eq(d_id)
|
||||
].sort_values(by=['direct_supervisor_id'], ascending=True)
|
||||
if _df.empty:
|
||||
block = self.block
|
||||
|
||||
def filter_department_id(e):
|
||||
if self.block != -1:
|
||||
return e['department_id'] == d_id and e['block'] == block
|
||||
return e.department_id == d_id
|
||||
|
||||
results = list(filter(lambda e: filter_department_id(e), self.all_employee_list))
|
||||
|
||||
return results
|
||||
|
||||
def get_department_by_parent_id(self, parent_id):
|
||||
results = list(filter(lambda d: d['department_parent_id'] == parent_id, self.all_department_list))
|
||||
if not results:
|
||||
return []
|
||||
|
||||
if self.block != -1:
|
||||
_df = _df[
|
||||
_df['block'].eq(self.block)
|
||||
]
|
||||
|
||||
return _df.to_dict('records')
|
||||
return results
|
||||
|
||||
def get_tree_departments(self):
|
||||
# 一级部门
|
||||
top_df = self.d_df[self.d_df['department_parent_id'].eq(-1)]
|
||||
if top_df.empty:
|
||||
top_departments = self.get_department_by_parent_id(-1)
|
||||
if len(top_departments) == 0:
|
||||
return []
|
||||
|
||||
d_list = []
|
||||
|
||||
for index in top_df.index:
|
||||
top_d = top_df.loc[index].to_dict()
|
||||
|
||||
for top_d in top_departments:
|
||||
department_id = top_d['department_id']
|
||||
|
||||
# 检查 department_id 是否作为其他部门的 parent
|
||||
sub_df = self.d_df[
|
||||
self.d_df['department_parent_id'].eq(department_id)
|
||||
].sort_values(by=['sort_value'], ascending=True)
|
||||
|
||||
sub_deps = self.get_department_by_parent_id(department_id)
|
||||
employees = []
|
||||
|
||||
if self.append_employee:
|
||||
# 要包含员工
|
||||
employees = self.get_employees_by_d_id(department_id)
|
||||
|
||||
top_d['employees'] = employees
|
||||
|
||||
if sub_df.empty:
|
||||
if len(sub_deps) == 0:
|
||||
top_d[sub_departments_column_name] = []
|
||||
d_list.append(top_d)
|
||||
continue
|
||||
|
||||
self.parse_sub_department(sub_df, top_d)
|
||||
self.parse_sub_department(sub_deps, top_d)
|
||||
d_list.append(top_d)
|
||||
|
||||
return d_list
|
||||
|
||||
def get_all_departments(self, is_tree=1):
|
||||
if self.d_df.empty:
|
||||
if len(self.all_department_list) == 0:
|
||||
return []
|
||||
|
||||
if is_tree != 1:
|
||||
return self.d_df.to_dict('records')
|
||||
return self.all_department_list
|
||||
|
||||
return self.get_tree_departments()
|
||||
|
||||
def parse_sub_department(self, df, top_d):
|
||||
def parse_sub_department(self, deps, top_d):
|
||||
sub_departments = []
|
||||
for s_index in df.index:
|
||||
d = df.loc[s_index].to_dict()
|
||||
sub_df = self.d_df[
|
||||
self.d_df['department_parent_id'].eq(
|
||||
df.at[s_index, 'department_id'])
|
||||
].sort_values(by=['sort_value'], ascending=True)
|
||||
for d in deps:
|
||||
sub_deps = self.get_department_by_parent_id(d['department_id'])
|
||||
employees = []
|
||||
|
||||
if self.append_employee:
|
||||
# 要包含员工
|
||||
employees = self.get_employees_by_d_id(
|
||||
df.at[s_index, 'department_id'])
|
||||
employees = self.get_employees_by_d_id(d['department_id'])
|
||||
|
||||
d['employees'] = employees
|
||||
|
||||
if sub_df.empty:
|
||||
if len(sub_deps) == 0:
|
||||
d[sub_departments_column_name] = []
|
||||
sub_departments.append(d)
|
||||
continue
|
||||
|
||||
self.parse_sub_department(sub_df, d)
|
||||
self.parse_sub_department(sub_deps, d)
|
||||
sub_departments.append(d)
|
||||
|
||||
top_d[sub_departments_column_name] = sub_departments
|
||||
|
|
@ -321,58 +303,52 @@ class DepartmentCRUD(object):
|
|||
|
||||
@staticmethod
|
||||
def get_department_tree_list():
|
||||
df = get_department_df()
|
||||
if df.empty:
|
||||
all_deps = get_all_department_list()
|
||||
if len(all_deps) == 0:
|
||||
return []
|
||||
|
||||
# 一级部门
|
||||
top_df = df[df['department_parent_id'].eq(-1)]
|
||||
if top_df.empty:
|
||||
top_deps = list(filter(lambda d: d['department_parent_id'] == -1, all_deps))
|
||||
if len(top_deps) == 0:
|
||||
return []
|
||||
|
||||
tree_list = []
|
||||
|
||||
for index in top_df.index:
|
||||
for top_d in top_deps:
|
||||
tree = Tree()
|
||||
identifier_root = top_df.at[index, 'department_id']
|
||||
identifier_root = top_d['department_id']
|
||||
tree.create_node(
|
||||
top_df.at[index, 'department_name'],
|
||||
top_d['department_name'],
|
||||
identifier_root
|
||||
)
|
||||
|
||||
# 检查 department_id 是否作为其他部门的 parent
|
||||
sub_df = df[
|
||||
df['department_parent_id'].eq(identifier_root)
|
||||
]
|
||||
if sub_df.empty:
|
||||
sub_ds = list(filter(lambda d: d['department_parent_id'] == identifier_root, all_deps))
|
||||
if len(sub_ds) == 0:
|
||||
tree_list.append(tree)
|
||||
continue
|
||||
|
||||
DepartmentCRUD.parse_sub_department_node(
|
||||
sub_df, df, tree, identifier_root)
|
||||
sub_ds, all_deps, tree, identifier_root)
|
||||
|
||||
tree_list.append(tree)
|
||||
|
||||
return tree_list
|
||||
|
||||
@staticmethod
|
||||
def parse_sub_department_node(df, all_df, tree, parent_id):
|
||||
for s_index in df.index:
|
||||
def parse_sub_department_node(sub_ds, all_ds, tree, parent_id):
|
||||
for d in sub_ds:
|
||||
tree.create_node(
|
||||
df.at[s_index, 'department_name'],
|
||||
df.at[s_index, 'department_id'],
|
||||
d['department_name'],
|
||||
d['department_id'],
|
||||
parent=parent_id
|
||||
)
|
||||
|
||||
sub_df = all_df[
|
||||
all_df['department_parent_id'].eq(
|
||||
df.at[s_index, 'department_id'])
|
||||
]
|
||||
if sub_df.empty:
|
||||
next_sub_ds = list(filter(lambda item_d: item_d['department_parent_id'] == d['department_id'], all_ds))
|
||||
if len(next_sub_ds) == 0:
|
||||
continue
|
||||
|
||||
DepartmentCRUD.parse_sub_department_node(
|
||||
sub_df, all_df, tree, df.at[s_index, 'department_id'])
|
||||
next_sub_ds, all_ds, tree, d['department_id'])
|
||||
|
||||
@staticmethod
|
||||
def get_departments_and_ids(department_parent_id, block):
|
||||
|
|
@ -380,44 +356,30 @@ class DepartmentCRUD(object):
|
|||
Department.department_parent_id == department_parent_id,
|
||||
Department.deleted == 0,
|
||||
).order_by(Department.sort_value.asc())
|
||||
df = get_df_from_read_sql(query)
|
||||
if df.empty:
|
||||
all_departments = DepartmentCRUD.get_department_by_query(query)
|
||||
if len(all_departments) == 0:
|
||||
return [], []
|
||||
|
||||
tree_list = DepartmentCRUD.get_department_tree_list()
|
||||
employee_df = get_all_employee_df(block)
|
||||
all_employee_list = get_all_employee_list(block)
|
||||
|
||||
department_id_list = list(df['department_id'].values)
|
||||
department_id_list = [d['department_id'] for d in all_departments]
|
||||
query = Department.query.filter(
|
||||
Department.department_parent_id.in_(department_id_list),
|
||||
Department.deleted == 0,
|
||||
).order_by(Department.sort_value.asc()).group_by(Department.department_id)
|
||||
sub_df = get_df_from_read_sql(query)
|
||||
if sub_df.empty:
|
||||
df['has_sub'] = 0
|
||||
sub_deps = DepartmentCRUD.get_department_by_query(query)
|
||||
|
||||
def handle_row_employee_count(row):
|
||||
return len(employee_df[employee_df['department_id'] == row['department_id']])
|
||||
sub_map = {d['department_parent_id']: 1 for d in sub_deps}
|
||||
|
||||
df['employee_count'] = df.apply(
|
||||
lambda row: handle_row_employee_count(row), axis=1)
|
||||
for d in all_departments:
|
||||
d['has_sub'] = sub_map.get(d['department_id'], 0)
|
||||
|
||||
else:
|
||||
sub_map = {d['department_parent_id']: 1 for d in sub_df.to_dict('records')}
|
||||
d_ids = DepartmentCRUD.get_department_id_list_by_root(d['department_id'], tree_list)
|
||||
|
||||
def handle_row(row):
|
||||
d_ids = DepartmentCRUD.get_department_id_list_by_root(
|
||||
row['department_id'], tree_list)
|
||||
row['employee_count'] = len(
|
||||
employee_df[employee_df['department_id'].isin(d_ids)])
|
||||
d['employee_count'] = len(list(filter(lambda e: e['department_id'] in d_ids, all_employee_list)))
|
||||
|
||||
row['has_sub'] = sub_map.get(row['department_id'], 0)
|
||||
|
||||
return row
|
||||
|
||||
df = df.apply(lambda row: handle_row(row), axis=1)
|
||||
|
||||
return df.to_dict('records'), department_id_list
|
||||
return all_departments, department_id_list
|
||||
|
||||
@staticmethod
|
||||
def get_department_id_list_by_root(root_department_id, tree_list=None):
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@
|
|||
import traceback
|
||||
from datetime import datetime
|
||||
|
||||
import pandas as pd
|
||||
from flask import abort
|
||||
from flask_login import current_user
|
||||
from sqlalchemy import or_, literal_column, func, not_, and_
|
||||
|
|
@ -17,7 +16,6 @@ from api.extensions import db
|
|||
from api.lib.common_setting.acl import ACLManager
|
||||
from api.lib.common_setting.const import COMMON_SETTING_QUEUE, OperatorType
|
||||
from api.lib.common_setting.resp_format import ErrFormat
|
||||
from api.lib.common_setting.utils import get_df_from_read_sql
|
||||
from api.models.common_setting import Employee, Department
|
||||
|
||||
|
||||
|
|
@ -214,59 +212,49 @@ class EmployeeCRUD(object):
|
|||
return CreateEmployee().batch_create(employee_list)
|
||||
|
||||
@staticmethod
|
||||
def get_export_employee_df(block_status):
|
||||
criterion = [
|
||||
Employee.deleted == 0
|
||||
]
|
||||
def get_export_employee_list(block_status):
|
||||
if block_status >= 0:
|
||||
criterion.append(
|
||||
Employee.block == block_status
|
||||
)
|
||||
employees = Employee.get_by(block=block_status, to_dict=False)
|
||||
else:
|
||||
employees = Employee.get_by(to_dict=False)
|
||||
|
||||
query = Employee.query.with_entities(
|
||||
Employee.employee_id,
|
||||
Employee.nickname,
|
||||
Employee.email,
|
||||
Employee.sex,
|
||||
Employee.mobile,
|
||||
Employee.position_name,
|
||||
Employee.last_login,
|
||||
Employee.department_id,
|
||||
Employee.direct_supervisor_id,
|
||||
).filter(*criterion)
|
||||
df = get_df_from_read_sql(query)
|
||||
if df.empty:
|
||||
return df
|
||||
keep_cols = EmployeeCRUD.get_current_user_view_columns()
|
||||
|
||||
query = Department.query.filter(
|
||||
*criterion
|
||||
)
|
||||
department_df = get_df_from_read_sql(query)
|
||||
all_departments = Department.get_by(to_dict=False)
|
||||
d_id_map = {d.department_id: d.department_name for d in all_departments}
|
||||
e_id_map = {e.employee_id: e.nickname for e in employees}
|
||||
|
||||
def find_name(row):
|
||||
department_id = row['department_id']
|
||||
_df = department_df[department_df['department_id']
|
||||
== department_id]
|
||||
row['department_name'] = '' if _df.empty else _df.iloc[0]['department_name']
|
||||
export_columns_map = {
|
||||
'username': "用户名",
|
||||
'nickname': '姓名',
|
||||
'email': '邮箱',
|
||||
'department_name': '部门',
|
||||
'sex': '性别',
|
||||
'mobile': '手机号',
|
||||
'position_name': '岗位',
|
||||
'nickname_direct_supervisor': '直属上级',
|
||||
'last_login': '上次登录时间',
|
||||
}
|
||||
|
||||
direct_supervisor_id = row['direct_supervisor_id']
|
||||
_df = df[df['employee_id'] == direct_supervisor_id]
|
||||
row['nickname_direct_supervisor'] = '' if _df.empty else _df.iloc[0]['nickname']
|
||||
data_list = []
|
||||
for e in employees:
|
||||
department_name = d_id_map.get(e.department_id, '')
|
||||
nickname_direct_supervisor = e_id_map.get(e.direct_supervisor_id, '')
|
||||
try:
|
||||
last_login = str(e.last_login) if e.last_login else ''
|
||||
except:
|
||||
last_login = ''
|
||||
data = e.to_dict()
|
||||
data['last_login'] = last_login
|
||||
data['department_name'] = department_name
|
||||
data['nickname_direct_supervisor'] = nickname_direct_supervisor
|
||||
|
||||
if isinstance(row['last_login'], pd.Timestamp):
|
||||
try:
|
||||
row['last_login'] = str(row['last_login'])
|
||||
except:
|
||||
row['last_login'] = ''
|
||||
else:
|
||||
row['last_login'] = ''
|
||||
tmp = {export_columns_map[k]: data[k] for k in export_columns_map.keys() if
|
||||
k in keep_cols or k in sub_columns}
|
||||
|
||||
return row
|
||||
data_list.append(tmp)
|
||||
|
||||
df = df.apply(find_name, axis=1)
|
||||
df.drop(['department_id', 'direct_supervisor_id',
|
||||
'employee_id'], axis=1, inplace=True)
|
||||
return df
|
||||
return data_list
|
||||
|
||||
@staticmethod
|
||||
def batch_employee(column_name, column_value, employee_id_list):
|
||||
|
|
@ -481,7 +469,7 @@ class EmployeeCRUD(object):
|
|||
if value:
|
||||
abort(400, ErrFormat.query_column_none_keep_value_empty.format(column))
|
||||
expr = [attr.is_(None)]
|
||||
if column not in ["entry_date", "leave_date", "dfc_entry_date", "last_login"]:
|
||||
if column not in ["last_login"]:
|
||||
expr += [attr == '']
|
||||
expr = [or_(*expr)]
|
||||
elif operator == OperatorType.IS_NOT_EMPTY:
|
||||
|
|
|
|||
|
|
@ -1,23 +1,6 @@
|
|||
# -*- coding:utf-8 -*-
|
||||
from datetime import datetime
|
||||
|
||||
import pandas as pd
|
||||
from sqlalchemy import text
|
||||
|
||||
from api.extensions import db
|
||||
|
||||
|
||||
def get_df_from_read_sql(query, to_dict=False):
|
||||
bind = query.session.bind
|
||||
query = query.statement.compile(dialect=bind.dialect if bind else None,
|
||||
compile_kwargs={"literal_binds": True}).string
|
||||
a = db.engine
|
||||
df = pd.read_sql(sql=text(query), con=a.connect())
|
||||
|
||||
if to_dict:
|
||||
return df.to_dict('records')
|
||||
return df
|
||||
|
||||
|
||||
def get_cur_time_str(split_flag='-'):
|
||||
f = f"%Y{split_flag}%m{split_flag}%d{split_flag}%H{split_flag}%M{split_flag}%S{split_flag}%f"
|
||||
|
|
|
|||
|
|
@ -150,17 +150,6 @@ class EmployeeViewExportExcel(APIView):
|
|||
url_prefix = (f'{prefix}/export_all',)
|
||||
|
||||
def get(self):
|
||||
col_desc_map = {
|
||||
'nickname': "姓名",
|
||||
'email': '邮箱',
|
||||
'sex': '性别',
|
||||
'mobile': '手机号',
|
||||
'department_name': '部门',
|
||||
'position_name': '岗位',
|
||||
'nickname_direct_supervisor': '直接上级',
|
||||
'last_login': '上次登录时间',
|
||||
}
|
||||
|
||||
# 规定了静态文件的存储位置
|
||||
excel_filename = 'all_employee_info.xlsx'
|
||||
excel_path = current_app.config['UPLOAD_DIRECTORY_FULL']
|
||||
|
|
@ -168,20 +157,19 @@ class EmployeeViewExportExcel(APIView):
|
|||
|
||||
# 根据parameter查表，自连接通过上级id获取上级名字列
|
||||
block_status = int(request.args.get('block_status', -1))
|
||||
df = EmployeeCRUD.get_export_employee_df(block_status)
|
||||
data_list = EmployeeCRUD.get_export_employee_list(block_status)
|
||||
|
||||
# 改变列名为中文head
|
||||
try:
|
||||
df = df.rename(columns=col_desc_map)
|
||||
except Exception as e:
|
||||
abort(500, ErrFormat.rename_columns_failed.format(str(e)))
|
||||
headers = data_list[0].keys()
|
||||
from openpyxl import Workbook
|
||||
|
||||
# 生成静态excel文件
|
||||
try:
|
||||
df.to_excel(excel_path_with_filename,
|
||||
sheet_name='Sheet1', index=False, encoding="utf-8")
|
||||
except Exception as e:
|
||||
current_app.logger.error(e)
|
||||
abort(500, ErrFormat.generate_excel_failed.format(str(e)))
|
||||
wb = Workbook()
|
||||
ws = wb.active
|
||||
# insert header
|
||||
for col_num, col_data in enumerate(headers, start=1):
|
||||
ws.cell(row=1, column=col_num, value=col_data)
|
||||
|
||||
for row_num, row_data in enumerate(data_list, start=2):
|
||||
for col_num, col_data in enumerate(row_data.values(), start=1):
|
||||
ws.cell(row=row_num, column=col_num, value=col_data)
|
||||
wb.save(excel_path_with_filename)
|
||||
return send_from_directory(excel_path, excel_filename, as_attachment=True)
|
||||
|
|
|
|||
Loading…
Reference in New Issue