mirror of https://github.com/veops/cmdb.git
perf(api): relationships built by attribute values (#572)
This commit is contained in:
pycook
GitHub
parent
ed46a1e1c1
commit
ff2b8ea198
|
|
@ -23,6 +23,7 @@ from api.lib.cmdb.ci_type import CITypeGroupManager
|
|||
from api.lib.cmdb.const import AutoDiscoveryType
|
||||
from api.lib.cmdb.const import CMDB_QUEUE
|
||||
from api.lib.cmdb.const import PermEnum
|
||||
from api.lib.cmdb.const import RelationSourceEnum
|
||||
from api.lib.cmdb.const import ResourceTypeEnum
|
||||
from api.lib.cmdb.custom_dashboard import SystemConfigManager
|
||||
from api.lib.cmdb.resp_format import ErrFormat
|
||||
|
|
@ -246,6 +247,9 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
|
|||
rules = cls.cls.get_by(to_dict=True)
|
||||
|
||||
for rule in rules:
|
||||
if not rule['enabled']:
|
||||
continue
|
||||
|
||||
if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('secret'):
|
||||
if not (current_user.username in PRIVILEGED_USERS or current_user.uid == rule['uid']):
|
||||
rule['extra_option'].pop('secret', None)
|
||||
|
|
@ -717,10 +721,15 @@ class AutoDiscoveryCICRUD(DBMixin):
|
|||
for relation_ci in response:
|
||||
relation_ci_id = relation_ci['_id']
|
||||
try:
|
||||
CIRelationManager.add(ci_id, relation_ci_id, valid=False)
|
||||
CIRelationManager.add(ci_id, relation_ci_id,
|
||||
valid=False,
|
||||
source=RelationSourceEnum.AUTO_DISCOVERY)
|
||||
|
||||
except:
|
||||
try:
|
||||
CIRelationManager.add(relation_ci_id, ci_id, valid=False)
|
||||
CIRelationManager.add(relation_ci_id, ci_id,
|
||||
valid=False,
|
||||
source=RelationSourceEnum.AUTO_DISCOVERY)
|
||||
except:
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ CLOUD_MAP = {
|
|||
"items": ["云服务器 ECS", "云服务器 Disk"],
|
||||
"map": {
|
||||
"云服务器 ECS": {"template": "templates/aliyun_ecs.json", "mapping": "ecs"},
|
||||
"云服务器 Disk": {"template": "templates/aliyun_ecs_disk2.json", "mapping": "evs"},
|
||||
"云服务器 Disk": {"template": "templates/aliyun_ecs_disk.json", "mapping": "evs"},
|
||||
},
|
||||
"collect_key_map": {
|
||||
"云服务器 ECS": "ali.ecs",
|
||||
|
|
|
|||
|
|
@ -4,12 +4,12 @@
|
|||
import copy
|
||||
import datetime
|
||||
import json
|
||||
import threading
|
||||
|
||||
import redis_lock
|
||||
import threading
|
||||
from flask import abort
|
||||
from flask import current_app
|
||||
from flask_login import current_user
|
||||
from sqlalchemy.orm import aliased
|
||||
from werkzeug.exceptions import BadRequest
|
||||
|
||||
from api.extensions import db
|
||||
|
|
@ -28,6 +28,7 @@ from api.lib.cmdb.const import ExistPolicy
|
|||
from api.lib.cmdb.const import OperateType
|
||||
from api.lib.cmdb.const import PermEnum
|
||||
from api.lib.cmdb.const import REDIS_PREFIX_CI
|
||||
from api.lib.cmdb.const import RelationSourceEnum
|
||||
from api.lib.cmdb.const import ResourceTypeEnum
|
||||
from api.lib.cmdb.const import RetKey
|
||||
from api.lib.cmdb.const import ValueTypeEnum
|
||||
|
|
@ -1133,7 +1134,14 @@ class CIRelationManager(object):
|
|||
return abort(400, ErrFormat.relation_constraint.format("1-N"))
|
||||
|
||||
@classmethod
|
||||
def add(cls, first_ci_id, second_ci_id, more=None, relation_type_id=None, ancestor_ids=None, valid=True):
|
||||
def add(cls, first_ci_id, second_ci_id,
|
||||
more=None,
|
||||
relation_type_id=None,
|
||||
ancestor_ids=None,
|
||||
valid=True,
|
||||
apply_async=True,
|
||||
source=None,
|
||||
uid=None):
|
||||
|
||||
first_ci = CIManager.confirm_ci_existed(first_ci_id)
|
||||
second_ci = CIManager.confirm_ci_existed(second_ci_id)
|
||||
|
|
@ -1145,9 +1153,10 @@ class CIRelationManager(object):
|
|||
first=True)
|
||||
if existed is not None:
|
||||
if existed.relation_type_id != relation_type_id and relation_type_id is not None:
|
||||
existed.update(relation_type_id=relation_type_id)
|
||||
source = existed.source or source
|
||||
existed.update(relation_type_id=relation_type_id, source=source)
|
||||
|
||||
CIRelationHistoryManager().add(existed, OperateType.UPDATE)
|
||||
CIRelationHistoryManager().add(existed, OperateType.UPDATE, uid=uid)
|
||||
else:
|
||||
if relation_type_id is None:
|
||||
type_relation = CITypeRelation.get_by(parent_id=first_ci.type_id,
|
||||
|
|
@ -1177,11 +1186,13 @@ class CIRelationManager(object):
|
|||
existed = CIRelation.create(first_ci_id=first_ci_id,
|
||||
second_ci_id=second_ci_id,
|
||||
relation_type_id=relation_type_id,
|
||||
ancestor_ids=ancestor_ids)
|
||||
|
||||
CIRelationHistoryManager().add(existed, OperateType.ADD)
|
||||
|
||||
ancestor_ids=ancestor_ids,
|
||||
source=source)
|
||||
CIRelationHistoryManager().add(existed, OperateType.ADD, uid=uid)
|
||||
if apply_async:
|
||||
ci_relation_cache.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE)
|
||||
else:
|
||||
ci_relation_cache(first_ci_id, second_ci_id, ancestor_ids)
|
||||
|
||||
if more is not None:
|
||||
existed.upadte(more=more)
|
||||
|
|
@ -1189,7 +1200,7 @@ class CIRelationManager(object):
|
|||
return existed.id
|
||||
|
||||
@staticmethod
|
||||
def delete(cr_id):
|
||||
def delete(cr_id, apply_async=True):
|
||||
cr = CIRelation.get_by_id(cr_id) or abort(404, ErrFormat.relation_not_found.format("id={}".format(cr_id)))
|
||||
|
||||
if current_app.config.get('USE_ACL') and current_user.username != 'worker':
|
||||
|
|
@ -1205,8 +1216,12 @@ class CIRelationManager(object):
|
|||
his_manager = CIRelationHistoryManager()
|
||||
his_manager.add(cr, operate_type=OperateType.DELETE)
|
||||
|
||||
if apply_async:
|
||||
ci_relation_delete.apply_async(args=(cr.first_ci_id, cr.second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE)
|
||||
delete_id_filter.apply_async(args=(cr.second_ci_id,), queue=CMDB_QUEUE)
|
||||
else:
|
||||
ci_relation_delete(cr.first_ci_id, cr.second_ci_id, cr.ancestor_ids)
|
||||
delete_id_filter(cr.second_ci_id)
|
||||
|
||||
return cr_id
|
||||
|
||||
|
|
@ -1221,23 +1236,23 @@ class CIRelationManager(object):
|
|||
if cr is not None:
|
||||
cls.delete(cr.id)
|
||||
|
||||
ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE)
|
||||
delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE)
|
||||
# ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE)
|
||||
# delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE)
|
||||
|
||||
return cr
|
||||
|
||||
@classmethod
|
||||
def delete_3(cls, first_ci_id, second_ci_id):
|
||||
def delete_3(cls, first_ci_id, second_ci_id, apply_async=True):
|
||||
cr = CIRelation.get_by(first_ci_id=first_ci_id,
|
||||
second_ci_id=second_ci_id,
|
||||
to_dict=False,
|
||||
first=True)
|
||||
|
||||
if cr is not None:
|
||||
ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE)
|
||||
delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE)
|
||||
# ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE)
|
||||
# delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE)
|
||||
|
||||
cls.delete(cr.id)
|
||||
cls.delete(cr.id, apply_async=apply_async)
|
||||
|
||||
return cr
|
||||
|
||||
|
|
@ -1276,6 +1291,27 @@ class CIRelationManager(object):
|
|||
for ci_id in ci_ids:
|
||||
cls.delete_2(parent_id, ci_id, ancestor_ids=ancestor_ids)
|
||||
|
||||
@classmethod
|
||||
def delete_relations_by_source(cls, source,
|
||||
first_ci_id=None, second_ci_type_id=None,
|
||||
second_ci_id=None, first_ci_type_id=None,
|
||||
added=None):
|
||||
existed = []
|
||||
if first_ci_id is not None and second_ci_type_id is not None:
|
||||
existed = [(i.first_ci_id, i.second_ci_id) for i in CIRelation.get_by(
|
||||
source=source, first_ci_id=first_ci_id, only_query=True).join(
|
||||
CI, CIRelation.second_ci_id == CI.id).filter(CI.type_id == second_ci_type_id)]
|
||||
|
||||
if second_ci_id is not None and first_ci_type_id is not None:
|
||||
existed = [(i.first_ci_id, i.second_ci_id) for i in CIRelation.get_by(
|
||||
source=source, second_ci_id=second_ci_id, only_query=True).join(
|
||||
CI, CIRelation.first_ci_id == CI.id).filter(CI.type_id == first_ci_type_id)]
|
||||
|
||||
deleted = set(existed) - set(added or [])
|
||||
|
||||
for first, second in deleted:
|
||||
cls.delete_3(first, second, apply_async=False)
|
||||
|
||||
@classmethod
|
||||
def build_by_attribute(cls, ci_dict):
|
||||
type_id = ci_dict['_type']
|
||||
|
|
@ -1296,8 +1332,15 @@ class CIRelationManager(object):
|
|||
relations = _relations
|
||||
else:
|
||||
relations &= _relations
|
||||
|
||||
cls.delete_relations_by_source(RelationSourceEnum.ATTRIBUTE_VALUES,
|
||||
first_ci_id=ci_dict['_id'],
|
||||
second_ci_type_id=item.child_id,
|
||||
added=relations)
|
||||
for parent_ci_id, child_ci_id in (relations or []):
|
||||
CIRelationManager.add(parent_ci_id, child_ci_id, valid=False)
|
||||
cls.add(parent_ci_id, child_ci_id,
|
||||
valid=False,
|
||||
source=RelationSourceEnum.ATTRIBUTE_VALUES)
|
||||
|
||||
parent_items = CITypeRelation.get_by(child_id=type_id, only_query=True).filter(
|
||||
CITypeRelation.child_attr_ids.isnot(None))
|
||||
|
|
@ -1316,11 +1359,18 @@ class CIRelationManager(object):
|
|||
relations = _relations
|
||||
else:
|
||||
relations &= _relations
|
||||
|
||||
cls.delete_relations_by_source(RelationSourceEnum.ATTRIBUTE_VALUES,
|
||||
second_ci_id=ci_dict['_id'],
|
||||
first_ci_type_id=item.parent_id,
|
||||
added=relations)
|
||||
for parent_ci_id, child_ci_id in (relations or []):
|
||||
CIRelationManager.add(parent_ci_id, child_ci_id, valid=False)
|
||||
cls.add(parent_ci_id, child_ci_id,
|
||||
valid=False,
|
||||
source=RelationSourceEnum.ATTRIBUTE_VALUES)
|
||||
|
||||
@classmethod
|
||||
def rebuild_all_by_attribute(cls, ci_type_relation):
|
||||
def rebuild_all_by_attribute(cls, ci_type_relation, uid):
|
||||
relations = None
|
||||
for parent_attr_id, child_attr_id in zip(ci_type_relation['parent_attr_ids'] or [],
|
||||
ci_type_relation['child_attr_ids'] or []):
|
||||
|
|
@ -1352,11 +1402,29 @@ class CIRelationManager(object):
|
|||
else:
|
||||
relations &= _relations
|
||||
|
||||
t1 = aliased(CI)
|
||||
t2 = aliased(CI)
|
||||
query = db.session.query(CIRelation).join(t1, t1.id == CIRelation.first_ci_id).join(
|
||||
t2, t2.id == CIRelation.second_ci_id).filter(t1.type_id == ci_type_relation['parent_id']).filter(
|
||||
t2.type_id == ci_type_relation['child_id'])
|
||||
for i in query:
|
||||
db.session.delete(i)
|
||||
ci_relation_delete(i.first_ci_id, i.second_ci_id, i.ancestor_ids)
|
||||
try:
|
||||
db.session.commit()
|
||||
except Exception as e:
|
||||
current_app.logger.error(e)
|
||||
db.session.rollback()
|
||||
|
||||
for parent_ci_id, child_ci_id in (relations or []):
|
||||
try:
|
||||
cls.add(parent_ci_id, child_ci_id, valid=False)
|
||||
except:
|
||||
pass
|
||||
cls.add(parent_ci_id, child_ci_id,
|
||||
valid=False,
|
||||
apply_async=False,
|
||||
source=RelationSourceEnum.ATTRIBUTE_VALUES,
|
||||
uid=uid)
|
||||
except Exception as e:
|
||||
current_app.logger.error(e)
|
||||
|
||||
|
||||
class CITriggerManager(object):
|
||||
|
|
|
|||
|
|
@ -993,7 +993,7 @@ class CITypeRelationManager(object):
|
|||
if ((parent_attr_ids and parent_attr_ids != old_parent_attr_ids) or
|
||||
(child_attr_ids and child_attr_ids != old_child_attr_ids)):
|
||||
from api.tasks.cmdb import rebuild_relation_for_attribute_changed
|
||||
rebuild_relation_for_attribute_changed.apply_async(args=(existed.to_dict(),))
|
||||
rebuild_relation_for_attribute_changed.apply_async(args=(existed.to_dict(), current_user.uid))
|
||||
|
||||
CITypeHistoryManager.add(CITypeOperateType.ADD_RELATION, p.id,
|
||||
change=dict(parent=p.to_dict(), child=c.to_dict(), relation_type_id=relation_type_id))
|
||||
|
|
|
|||
|
|
@ -41,23 +41,23 @@ class OperateType(BaseEnum):
|
|||
|
||||
|
||||
class CITypeOperateType(BaseEnum):
|
||||
ADD = "0" # 新增模型
|
||||
UPDATE = "1" # 修改模型
|
||||
DELETE = "2" # 删除模型
|
||||
ADD_ATTRIBUTE = "3" # 新增属性
|
||||
UPDATE_ATTRIBUTE = "4" # 修改属性
|
||||
DELETE_ATTRIBUTE = "5" # 删除属性
|
||||
ADD_TRIGGER = "6" # 新增触发器
|
||||
UPDATE_TRIGGER = "7" # 修改触发器
|
||||
DELETE_TRIGGER = "8" # 删除触发器
|
||||
ADD_UNIQUE_CONSTRAINT = "9" # 新增联合唯一
|
||||
UPDATE_UNIQUE_CONSTRAINT = "10" # 修改联合唯一
|
||||
DELETE_UNIQUE_CONSTRAINT = "11" # 删除联合唯一
|
||||
ADD_RELATION = "12" # 新增关系
|
||||
DELETE_RELATION = "13" # 删除关系
|
||||
ADD_RECONCILIATION = "14" # 新增数据合规
|
||||
UPDATE_RECONCILIATION = "15" # 修改数据合规
|
||||
DELETE_RECONCILIATION = "16" # 删除数据合规
|
||||
ADD = "0" # add CIType
|
||||
UPDATE = "1" # update CIType
|
||||
DELETE = "2" # delete CIType
|
||||
ADD_ATTRIBUTE = "3"
|
||||
UPDATE_ATTRIBUTE = "4"
|
||||
DELETE_ATTRIBUTE = "5"
|
||||
ADD_TRIGGER = "6"
|
||||
UPDATE_TRIGGER = "7"
|
||||
DELETE_TRIGGER = "8"
|
||||
ADD_UNIQUE_CONSTRAINT = "9"
|
||||
UPDATE_UNIQUE_CONSTRAINT = "10"
|
||||
DELETE_UNIQUE_CONSTRAINT = "11"
|
||||
ADD_RELATION = "12"
|
||||
DELETE_RELATION = "13"
|
||||
ADD_RECONCILIATION = "14"
|
||||
UPDATE_RECONCILIATION = "15"
|
||||
DELETE_RECONCILIATION = "16"
|
||||
|
||||
|
||||
class RetKey(BaseEnum):
|
||||
|
|
@ -108,6 +108,10 @@ class ExecuteStatusEnum(BaseEnum):
|
|||
FAILED = '1'
|
||||
RUNNING = '2'
|
||||
|
||||
class RelationSourceEnum(BaseEnum):
|
||||
ATTRIBUTE_VALUES = "0"
|
||||
AUTO_DISCOVERY = "1"
|
||||
|
||||
|
||||
CMDB_QUEUE = "one_cmdb_async"
|
||||
REDIS_PREFIX_CI = "ONE_CMDB"
|
||||
|
|
|
|||
|
|
@ -232,8 +232,8 @@ class AttributeHistoryManger(object):
|
|||
|
||||
class CIRelationHistoryManager(object):
|
||||
@staticmethod
|
||||
def add(rel_obj, operate_type=OperateType.ADD):
|
||||
record = OperationRecord.create(uid=current_user.uid)
|
||||
def add(rel_obj, operate_type=OperateType.ADD, uid=None):
|
||||
record = OperationRecord.create(uid=uid or current_user.uid)
|
||||
|
||||
CIRelationHistory.create(relation_id=rel_obj.id,
|
||||
record_id=record.id,
|
||||
|
|
|
|||
|
|
@ -28,10 +28,7 @@ def string2int(x):
|
|||
return v
|
||||
|
||||
|
||||
def str2datetime(x):
|
||||
|
||||
x = x.replace('T', ' ')
|
||||
x = x.replace('Z', '')
|
||||
def str2date(x):
|
||||
|
||||
try:
|
||||
return datetime.datetime.strptime(x, "%Y-%m-%d").date()
|
||||
|
|
@ -43,9 +40,21 @@ def str2datetime(x):
|
|||
except ValueError:
|
||||
pass
|
||||
|
||||
|
||||
def str2datetime(x):
|
||||
|
||||
x = x.replace('T', ' ')
|
||||
x = x.replace('Z', '')
|
||||
|
||||
try:
|
||||
return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M")
|
||||
|
||||
|
||||
|
||||
class ValueTypeMap(object):
|
||||
deserialize = {
|
||||
ValueTypeEnum.INT: string2int,
|
||||
|
|
@ -53,7 +62,7 @@ class ValueTypeMap(object):
|
|||
ValueTypeEnum.TEXT: lambda x: x,
|
||||
ValueTypeEnum.TIME: lambda x: TIME_RE.findall(x)[0],
|
||||
ValueTypeEnum.DATETIME: str2datetime,
|
||||
ValueTypeEnum.DATE: str2datetime,
|
||||
ValueTypeEnum.DATE: str2date,
|
||||
ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x,
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
|
||||
|
||||
import datetime
|
||||
|
||||
from sqlalchemy.dialects.mysql import DOUBLE
|
||||
|
||||
from api.extensions import db
|
||||
|
|
@ -11,6 +10,7 @@ from api.lib.cmdb.const import CIStatusEnum
|
|||
from api.lib.cmdb.const import CITypeOperateType
|
||||
from api.lib.cmdb.const import ConstraintEnum
|
||||
from api.lib.cmdb.const import OperateType
|
||||
from api.lib.cmdb.const import RelationSourceEnum
|
||||
from api.lib.cmdb.const import ValueTypeEnum
|
||||
from api.lib.database import Model
|
||||
from api.lib.database import Model2
|
||||
|
|
@ -260,6 +260,7 @@ class CIRelation(Model):
|
|||
second_ci_id = db.Column(db.Integer, db.ForeignKey("c_cis.id"), nullable=False)
|
||||
relation_type_id = db.Column(db.Integer, db.ForeignKey("c_relation_types.id"), nullable=False)
|
||||
more = db.Column(db.Integer, db.ForeignKey("c_cis.id"))
|
||||
source = db.Column(db.Enum(*RelationSourceEnum.all()), name="source")
|
||||
|
||||
ancestor_ids = db.Column(db.String(128), index=True)
|
||||
|
||||
|
|
|
|||
|
|
@ -58,10 +58,10 @@ def ci_cache(ci_id, operate_type, record_id):
|
|||
|
||||
@celery.task(name="cmdb.rebuild_relation_for_attribute_changed", queue=CMDB_QUEUE)
|
||||
@reconnect_db
|
||||
def rebuild_relation_for_attribute_changed(ci_type_relation):
|
||||
def rebuild_relation_for_attribute_changed(ci_type_relation, uid):
|
||||
from api.lib.cmdb.ci import CIRelationManager
|
||||
|
||||
CIRelationManager.rebuild_all_by_attribute(ci_type_relation)
|
||||
CIRelationManager.rebuild_all_by_attribute(ci_type_relation, uid)
|
||||
|
||||
|
||||
@celery.task(name="cmdb.batch_ci_cache", queue=CMDB_QUEUE)
|
||||
|
|
|
|||
Loading…
Reference in New Issue