前后端全面升级

cmdb-api/api/tasks/acl.py

@@ -1,16 +1,200 @@
 # -*- coding:utf-8 -*-
 
+import json
+import re
+
+from celery_once import QueueOnce
 from flask import current_app
+from werkzeug.exceptions import BadRequest, NotFound
 
 from api.extensions import celery
+from api.extensions import db
+from api.lib.perm.acl.cache import AppCache
+from api.lib.perm.acl.cache import RoleCache
 from api.lib.perm.acl.cache import RoleRelationCache
+from api.lib.perm.acl.cache import UserCache
 from api.lib.perm.acl.const import ACL_QUEUE
+from api.lib.perm.acl.record import OperateRecordCRUD
+from api.lib.perm.acl.audit import AuditCRUD, AuditOperateType, AuditOperateSource
+from api.models.acl import Resource
+from api.models.acl import Role
+from api.models.acl import Trigger
 
 
-@celery.task(name="acl.role_rebuild", queue=ACL_QUEUE)
-def role_rebuild(rids):
+@celery.task(base=QueueOnce,
+             name="acl.role_rebuild",
+             queue=ACL_QUEUE,
+             once={"graceful": True, "unlock_before_run": True})
+def role_rebuild(rids, app_id):
     rids = rids if isinstance(rids, list) else [rids]
     for rid in rids:
-        RoleRelationCache.rebuild(rid)
+        RoleRelationCache.rebuild(rid, app_id)
 
-    current_app.logger.info("Role {0} rebuild..........".format(rids))
+    current_app.logger.info("Role {0} App {1} rebuild..........".format(rids, app_id))
+
+
+@celery.task(name="acl.update_resource_to_build_role", queue=ACL_QUEUE)
+def update_resource_to_build_role(resource_id, app_id, group_id=None):
+    rids = [i.id for i in Role.get_by(__func_isnot__key_uid=None, fl='id', to_dict=False)]
+    rids += [i.id for i in Role.get_by(app_id=app_id, fl='id', to_dict=False)]
+    rids += [i.id for i in Role.get_by(__func_is___key_uid=None, __func_is___key_app_id=None, fl='id', to_dict=False)]
+
+    current_app.logger.info(rids)
+    for rid in rids:
+        if resource_id and resource_id in RoleRelationCache.get_resources(rid, app_id).get('id2perms', {}):
+            RoleRelationCache.rebuild2(rid, app_id)
+
+        if group_id and group_id in RoleRelationCache.get_resources(rid, app_id).get('group2perms', {}):
+            RoleRelationCache.rebuild2(rid, app_id)
+
+
+@celery.task(name="acl.apply_trigger", queue=ACL_QUEUE)
+def apply_trigger(_id, resource_id=None, operator_uid=None):
+    db.session.remove()
+
+    from api.lib.perm.acl.permission import PermissionCRUD
+
+    trigger = Trigger.get_by_id(_id)
+    if trigger is None:
+        return
+
+    uid = json.loads(trigger.uid or '[]')
+    if resource_id is None:
+        wildcard = (trigger.wildcard or '')
+
+        if wildcard and uid:
+            query = Resource.get_by(__func_in___key_uid=uid,
+                                    app_id=trigger.app_id,
+                                    resource_type_id=trigger.resource_type_id,
+                                    fl=['id', 'app_id'],
+                                    only_query=True)
+            try:
+                re.compile(wildcard)
+
+                resources = query.filter(Resource.name.op('regexp')(wildcard)).all()
+            except:
+                resources = query.filter(Resource.name.ilike(wildcard.replace('*', '%'))).all()
+        elif wildcard:
+            query = Resource.get_by(app_id=trigger.app_id,
+                                    resource_type_id=trigger.resource_type_id,
+                                    only_query=True)
+            try:
+                re.compile(wildcard)
+
+                resources = query.filter(Resource.name.op('regexp')(wildcard)).all()
+            except:
+                resources = query.filter(Resource.name.ilike(wildcard.replace('*', '%'))).all()
+        elif uid:
+            resources = Resource.get_by(__func_in___key_uid=uid,
+                                        app_id=trigger.app_id,
+                                        resource_type_id=trigger.resource_type_id,
+                                        to_dict=False)
+        else:
+            resources = []
+    else:
+        resources = [Resource.get_by_id(resource_id)]
+
+    perms = json.loads(trigger.permissions)
+    roles = json.loads(trigger.roles)
+    for resource in resources:
+        for rid in roles:
+            try:
+                PermissionCRUD.grant(rid, perms, resource.id, rebuild=False, source=AuditOperateSource.trigger)
+            except (NotFound, BadRequest):
+                pass
+
+    AuditCRUD.add_trigger_log(trigger.app_id, trigger.id, AuditOperateType.trigger_apply, {}, trigger.to_dict(),
+                              {'uid': uid,
+                               'resource_ids': [r.id for r in resources],
+                               'perms': perms,
+                               'rids': roles},
+                              uid=operator_uid, source=AuditOperateSource.trigger)
+
+    if resources:
+        role_rebuild(roles, resources[0].app_id)
+
+
+@celery.task(name="acl.cancel_trigger", queue=ACL_QUEUE)
+def cancel_trigger(_id, resource_id=None, operator_uid=None):
+    db.session.remove()
+
+    from api.lib.perm.acl.permission import PermissionCRUD
+
+    trigger = Trigger.get_by_id(_id)
+    if trigger is None:
+        return
+
+    uid = json.loads(trigger.uid or '[]')
+    if resource_id is None:
+        wildcard = (trigger.wildcard or '')
+
+        if wildcard and uid:
+            query = Resource.get_by(__func_in___key_uid=uid,
+                                    app_id=trigger.app_id,
+                                    resource_type_id=trigger.resource_type_id,
+                                    fl=['id', 'app_id'],
+                                    only_query=True)
+            try:
+                re.compile(wildcard)
+
+                resources = query.filter(Resource.name.op('regexp')(wildcard)).all()
+            except:
+                resources = query.filter(Resource.name.ilike(wildcard.replace('*', '%'))).all()
+        elif wildcard:
+            query = Resource.get_by(app_id=trigger.app_id,
+                                    resource_type_id=trigger.resource_type_id,
+                                    only_query=True)
+            try:
+                re.compile(wildcard)
+
+                resources = query.filter(Resource.name.op('regexp')(wildcard)).all()
+            except:
+                resources = query.filter(Resource.name.ilike(wildcard.replace('*', '%'))).all()
+        elif uid:
+            resources = Resource.get_by(__func_in___key_uid=uid,
+                                        app_id=trigger.app_id,
+                                        resource_type_id=trigger.resource_type_id,
+                                        to_dict=False)
+        else:
+            resources = []
+    else:
+        resources = [Resource.get_by_id(resource_id)]
+
+    perms = json.loads(trigger.permissions)
+    roles = json.loads(trigger.roles)
+    for resource in resources:
+        if not resource:
+            continue
+        for rid in roles:
+            try:
+                PermissionCRUD.revoke(rid, perms, resource.id, rebuild=False, source=AuditOperateSource.trigger)
+            except (NotFound, BadRequest):
+                pass
+
+    AuditCRUD.add_trigger_log(trigger.app_id, trigger.id, AuditOperateType.trigger_cancel, {}, trigger.to_dict(),
+                              {'uid': uid,
+                               'resource_ids': [r.id for r in resources if r],
+                               'perms': perms,
+                               'rids': roles},
+                              uid=operator_uid, source=AuditOperateSource.trigger)
+
+    if resources:
+        role_rebuild(roles, resources[0].app_id)
+
+
+@celery.task(name="acl.op_record", queue=ACL_QUEUE)
+def op_record(app, rolename, operate_type, obj):
+    if isinstance(app, int):
+        app = AppCache.get(app)
+        app = app and app.name
+
+    if isinstance(rolename, int):
+        u = UserCache.get(rolename)
+        if u:
+            rolename = u.username
+        if not u:
+            r = RoleCache.get(rolename)
+            if r:
+                rolename = r.name
+
+    OperateRecordCRUD.add(app, rolename, operate_type, obj)

cmdb-api/api/tasks/cmdb.py

@@ -4,6 +4,8 @@
 import json
 import time
 
+import jinja2
+import requests
 from flask import current_app
 
 import api.lib.cmdb.ci
@@ -15,24 +17,44 @@ from api.lib.cmdb.cache import CITypeAttributesCache
 from api.lib.cmdb.const import CMDB_QUEUE
 from api.lib.cmdb.const import REDIS_PREFIX_CI
 from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
+from api.lib.mail import send_mail
+from api.lib.utils import Lock
 from api.models.cmdb import CIRelation
 
 
 @celery.task(name="cmdb.ci_cache", queue=CMDB_QUEUE)
 def ci_cache(ci_id):
     time.sleep(0.01)
-    db.session.close()
+    db.session.remove()
 
     m = api.lib.cmdb.ci.CIManager()
-    ci = m.get_ci_by_id_from_db(ci_id, need_children=False, use_master=False)
+    ci_dict = m.get_ci_by_id_from_db(ci_id, need_children=False, use_master=False)
+
     if current_app.config.get("USE_ES"):
-        es.create_or_update(ci_id, ci)
+        es.create_or_update(ci_id, ci_dict)
     else:
-        rd.create_or_update({ci_id: json.dumps(ci)}, REDIS_PREFIX_CI)
+        rd.create_or_update({ci_id: json.dumps(ci_dict)}, REDIS_PREFIX_CI)
 
     current_app.logger.info("{0} flush..........".format(ci_id))
 
 
+@celery.task(name="cmdb.batch_ci_cache", queue=CMDB_QUEUE)
+def batch_ci_cache(ci_ids):
+    time.sleep(1)
+    db.session.remove()
+
+    for ci_id in ci_ids:
+        m = api.lib.cmdb.ci.CIManager()
+        ci_dict = m.get_ci_by_id_from_db(ci_id, need_children=False, use_master=False)
+
+        if current_app.config.get("USE_ES"):
+            es.create_or_update(ci_id, ci_dict)
+        else:
+            rd.create_or_update({ci_id: json.dumps(ci_dict)}, REDIS_PREFIX_CI)
+
+        current_app.logger.info("{0} flush..........".format(ci_id))
+
+
 @celery.task(name="cmdb.ci_delete", queue=CMDB_QUEUE)
 def ci_delete(ci_id):
     current_app.logger.info(ci_id)
@@ -47,29 +69,31 @@ def ci_delete(ci_id):
 
 @celery.task(name="cmdb.ci_relation_cache", queue=CMDB_QUEUE)
 def ci_relation_cache(parent_id, child_id):
-    db.session.close()
+    db.session.remove()
 
-    children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0]
-    children = json.loads(children) if children is not None else {}
+    with Lock("CIRelation_{}".format(parent_id)):
+        children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0]
+        children = json.loads(children) if children is not None else {}
 
-    cr = CIRelation.get_by(first_ci_id=parent_id, second_ci_id=child_id, first=True, to_dict=False)
-    if str(child_id) not in children:
-        children[str(child_id)] = cr.second_ci.type_id
+        cr = CIRelation.get_by(first_ci_id=parent_id, second_ci_id=child_id, first=True, to_dict=False)
+        if str(child_id) not in children:
+            children[str(child_id)] = cr.second_ci.type_id
 
-    rd.create_or_update({parent_id: json.dumps(children)}, REDIS_PREFIX_CI_RELATION)
+        rd.create_or_update({parent_id: json.dumps(children)}, REDIS_PREFIX_CI_RELATION)
 
     current_app.logger.info("ADD ci relation cache: {0} -> {1}".format(parent_id, child_id))
 
 
 @celery.task(name="cmdb.ci_relation_delete", queue=CMDB_QUEUE)
 def ci_relation_delete(parent_id, child_id):
-    children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0]
-    children = json.loads(children) if children is not None else {}
+    with Lock("CIRelation_{}".format(parent_id)):
+        children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0]
+        children = json.loads(children) if children is not None else {}
 
-    if str(child_id) in children:
-        children.pop(str(child_id))
+        if str(child_id) in children:
+            children.pop(str(child_id))
 
-    rd.create_or_update({parent_id: json.dumps(children)}, REDIS_PREFIX_CI_RELATION)
+        rd.create_or_update({parent_id: json.dumps(children)}, REDIS_PREFIX_CI_RELATION)
 
     current_app.logger.info("DELETE ci relation cache: {0} -> {1}".format(parent_id, child_id))
 
@@ -92,3 +116,43 @@ def ci_type_attribute_order_rebuild(type_id):
                 id2attr.get(_attr['id']).update(order=order)
 
             order += 1
+
+
+@celery.task(name='cmdb.trigger_notify', queue=CMDB_QUEUE)
+def trigger_notify(notify, ci_id):
+    from api.lib.perm.acl.cache import UserCache
+
+    def _wrap_mail(mail_to):
+        if "@" not in mail_to:
+            user = UserCache.get(mail_to)
+            if user:
+                return user.email
+
+        return mail_to
+
+    db.session.remove()
+
+    m = api.lib.cmdb.ci.CIManager()
+    ci_dict = m.get_ci_by_id_from_db(ci_id, need_children=False, use_master=False)
+
+    subject = jinja2.Template(notify.get('subject') or "").render(ci_dict)
+    body = jinja2.Template(notify.get('body') or "").render(ci_dict)
+
+    if notify.get('wx_to'):
+        to_user = jinja2.Template('|'.join(notify['wx_to'])).render(ci_dict)
+        url = current_app.config.get("WX_URI")
+        data = {"to_user": to_user, "content": subject}
+        try:
+            requests.post(url, data=data)
+        except Exception as e:
+            current_app.logger.error(str(e))
+
+    if notify.get('mail_to'):
+        try:
+            if len(subject) > 700:
+                subject = subject[:600] + "..." + subject[-100:]
+
+            send_mail("", [_wrap_mail(jinja2.Template(i).render(ci_dict))
+                           for i in notify['mail_to'] if i], subject, body)
+        except Exception as e:
+            current_app.logger.error("Send mail failed: {0}".format(str(e)))

cmdb-api/api/tasks/common_setting.py

@@ -0,0 +1,84 @@
+# -*- coding:utf-8 -*-
+import requests
+from flask import current_app
+
+from api.extensions import celery
+from api.extensions import db
+from api.lib.common_setting.acl import ACLManager
+from api.lib.common_setting.const import COMMON_SETTING_QUEUE
+from api.lib.common_setting.resp_format import ErrFormat
+from api.models.common_setting import Department
+
+
+@celery.task(name="common_setting.edit_employee_department_in_acl", queue=COMMON_SETTING_QUEUE)
+def edit_employee_department_in_acl(e_list, new_d_id, op_uid):
+    """
+    在 ACL 员工更换部门
+    :param e_list: 员工列表 {acl_rid: 11, department_id: 22}
+    :param new_d_id: 新部门 ID
+    :param op_uid: 操作人 ID
+
+    在老部门中删除员工
+    在新部门中添加员工
+    """
+    db.session.remove()
+
+    result = []
+    new_department = Department.get_by(
+        first=True, department_id=new_d_id, to_dict=False)
+    if not new_department:
+        result.append(ErrFormat.new_department_is_none)
+        return result
+
+    acl = ACLManager('acl', str(op_uid))
+    role_map = {role['name']: role['id'] for role in acl.get_all_roles()}
+    new_d_rid_in_acl = role_map.get(new_department.department_name, 0)
+    if new_d_rid_in_acl == 0:
+        return
+
+    if new_d_rid_in_acl != new_department.acl_rid:
+        new_department.update(
+            acl_rid=new_d_rid_in_acl
+        )
+    new_department_acl_rid = new_department.acl_rid if new_d_rid_in_acl == new_department.acl_rid else new_d_rid_in_acl
+
+    for employee in e_list:
+        # 根据 部门ID获取部门 acl_rid
+        old_department = Department.get_by(
+            first=True, department_id=employee.get('department_id'), to_dict=False)
+        if not old_department:
+            continue
+        employee_acl_rid = employee.get('e_acl_rid')
+        if employee_acl_rid == 0:
+            result.append(ErrFormat.employee_acl_rid_is_zero)
+            continue
+
+        old_d_rid_in_acl = role_map.get(old_department.department_name, 0)
+        if old_d_rid_in_acl == 0:
+            return
+        if old_d_rid_in_acl != old_department.acl_rid:
+            old_department.update(
+                acl_rid=old_d_rid_in_acl
+            )
+        d_acl_rid = old_department.acl_rid if old_d_rid_in_acl == old_department.acl_rid else old_d_rid_in_acl
+        # 在老部门中删除员工
+        payload = {
+            'app_id': 'acl',
+            'parent_id': d_acl_rid,
+        }
+        try:
+            acl.remove_user_from_role(employee_acl_rid, payload)
+        except Exception as e:
+            result.append(ErrFormat.acl_remove_user_from_role_failed.format(str(e)))
+
+        # 在新部门中添加员工
+        payload = {
+            'app_id': 'acl',
+            'child_ids': [employee_acl_rid],
+        }
+        try:
+            acl.add_user_to_role(new_department_acl_rid, payload)
+        except Exception as e:
+            result.append(ErrFormat.acl_add_user_to_role_failed.format(str(e)))
+
+    return result

cmdb-api/api/tasks/test.py

@@ -1,10 +0,0 @@
-# -*- coding:utf-8 -*-
-
-from flask import current_app
-
-from api.extensions import celery
-
-
-@celery.task(queue="ticket_web")
-def test_task():
-    current_app.logger.info("test task.............................")