7x31
/
cmdb
mirror of https://github.com/veops/cmdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

perf(api): relationships built by attribute values (#572)

This commit is contained in:
pycook 2024-07-08 11:42:18 +08:00 committed by GitHub
parent 53dcf5d0d6
commit fff6da943d
9 changed files with 149 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py
View File

@ -23,6 +23,7 @@ from api.lib.cmdb.ci_type import CITypeGroupManager
from api.lib.cmdb.const import AutoDiscoveryType
from api.lib.cmdb.const import CMDB_QUEUE
from api.lib.cmdb.const import PermEnum
from api.lib.cmdb.const import RelationSourceEnum
from api.lib.cmdb.const import ResourceTypeEnum
from api.lib.cmdb.custom_dashboard import SystemConfigManager
from api.lib.cmdb.resp_format import ErrFormat
@ -246,6 +247,9 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
rules = cls.cls.get_by(to_dict=True)
for rule in rules:
if not rule['enabled']:
continue
if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('secret'):
if not (current_user.username in PRIVILEGED_USERS or current_user.uid == rule['uid']):
rule['extra_option'].pop('secret', None)
@ -274,7 +278,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
break
elif not rule['agent_id'] and not rule['query_expr'] and rule['adr_id']:
try:
if not int(oneagent_id, 16): # excludes master
if not int(oneagent_id, 16): # excludes master
continue
except Exception:
pass
@ -717,10 +721,15 @@ class AutoDiscoveryCICRUD(DBMixin):
for relation_ci in response:
relation_ci_id = relation_ci['_id']
try:
CIRelationManager.add(ci_id, relation_ci_id, valid=False)
CIRelationManager.add(ci_id, relation_ci_id,
valid=False,
source=RelationSourceEnum.AUTO_DISCOVERY)
except:
try:
CIRelationManager.add(relation_ci_id, ci_id, valid=False)
CIRelationManager.add(relation_ci_id, ci_id,
valid=False,
source=RelationSourceEnum.AUTO_DISCOVERY)
except:
pass

2
cmdb-api/api/lib/cmdb/auto_discovery/const.py
View File

@ -41,7 +41,7 @@ CLOUD_MAP = {
"items": ["云服务器 ECS", "云服务器 Disk"],
"map": {
"云服务器 ECS": {"template": "templates/aliyun_ecs.json", "mapping": "ecs"},
"云服务器 Disk": {"template": "templates/aliyun_ecs_disk2.json", "mapping": "evs"},
"云服务器 Disk": {"template": "templates/aliyun_ecs_disk.json", "mapping": "evs"},
},
"collect_key_map": {
"云服务器 ECS": "ali.ecs",

118
cmdb-api/api/lib/cmdb/ci.py
View File

@ -4,12 +4,12 @@
import copy
import datetime
import json
import threading
import redis_lock
import threading
from flask import abort
from flask import current_app
from flask_login import current_user
from sqlalchemy.orm import aliased
from werkzeug.exceptions import BadRequest
from api.extensions import db
@ -28,6 +28,7 @@ from api.lib.cmdb.const import ExistPolicy
from api.lib.cmdb.const import OperateType
from api.lib.cmdb.const import PermEnum
from api.lib.cmdb.const import REDIS_PREFIX_CI
from api.lib.cmdb.const import RelationSourceEnum
from api.lib.cmdb.const import ResourceTypeEnum
from api.lib.cmdb.const import RetKey
from api.lib.cmdb.const import ValueTypeEnum
@ -1133,7 +1134,14 @@ class CIRelationManager(object):
return abort(400, ErrFormat.relation_constraint.format("1-N"))
@classmethod
def add(cls, first_ci_id, second_ci_id, more=None, relation_type_id=None, ancestor_ids=None, valid=True):
def add(cls, first_ci_id, second_ci_id,
more=None,
relation_type_id=None,
ancestor_ids=None,
valid=True,
apply_async=True,
source=None,
uid=None):
first_ci = CIManager.confirm_ci_existed(first_ci_id)
second_ci = CIManager.confirm_ci_existed(second_ci_id)
@ -1145,9 +1153,10 @@ class CIRelationManager(object):
first=True)
if existed is not None:
if existed.relation_type_id != relation_type_id and relation_type_id is not None:
existed.update(relation_type_id=relation_type_id)
source = existed.source or source
existed.update(relation_type_id=relation_type_id, source=source)
CIRelationHistoryManager().add(existed, OperateType.UPDATE)
CIRelationHistoryManager().add(existed, OperateType.UPDATE, uid=uid)
else:
if relation_type_id is None:
type_relation = CITypeRelation.get_by(parent_id=first_ci.type_id,
@ -1177,11 +1186,13 @@ class CIRelationManager(object):
existed = CIRelation.create(first_ci_id=first_ci_id,
second_ci_id=second_ci_id,
relation_type_id=relation_type_id,
ancestor_ids=ancestor_ids)
CIRelationHistoryManager().add(existed, OperateType.ADD)
ci_relation_cache.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE)
ancestor_ids=ancestor_ids,
source=source)
CIRelationHistoryManager().add(existed, OperateType.ADD, uid=uid)
if apply_async:
ci_relation_cache.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE)
else:
ci_relation_cache(first_ci_id, second_ci_id, ancestor_ids)
if more is not None:
existed.upadte(more=more)
@ -1189,7 +1200,7 @@ class CIRelationManager(object):
return existed.id
@staticmethod
def delete(cr_id):
def delete(cr_id, apply_async=True):
cr = CIRelation.get_by_id(cr_id) or abort(404, ErrFormat.relation_not_found.format("id={}".format(cr_id)))
if current_app.config.get('USE_ACL') and current_user.username != 'worker':
@ -1205,8 +1216,12 @@ class CIRelationManager(object):
his_manager = CIRelationHistoryManager()
his_manager.add(cr, operate_type=OperateType.DELETE)
ci_relation_delete.apply_async(args=(cr.first_ci_id, cr.second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE)
delete_id_filter.apply_async(args=(cr.second_ci_id,), queue=CMDB_QUEUE)
if apply_async:
ci_relation_delete.apply_async(args=(cr.first_ci_id, cr.second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE)
delete_id_filter.apply_async(args=(cr.second_ci_id,), queue=CMDB_QUEUE)
else:
ci_relation_delete(cr.first_ci_id, cr.second_ci_id, cr.ancestor_ids)
delete_id_filter(cr.second_ci_id)
return cr_id
@ -1221,23 +1236,23 @@ class CIRelationManager(object):
if cr is not None:
cls.delete(cr.id)
ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE)
delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE)
# ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE)
# delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE)
return cr
@classmethod
def delete_3(cls, first_ci_id, second_ci_id):
def delete_3(cls, first_ci_id, second_ci_id, apply_async=True):
cr = CIRelation.get_by(first_ci_id=first_ci_id,
second_ci_id=second_ci_id,
to_dict=False,
first=True)
if cr is not None:
ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE)
delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE)
# ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE)
# delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE)
cls.delete(cr.id)
cls.delete(cr.id, apply_async=apply_async)
return cr
@ -1276,6 +1291,27 @@ class CIRelationManager(object):
for ci_id in ci_ids:
cls.delete_2(parent_id, ci_id, ancestor_ids=ancestor_ids)
@classmethod
def delete_relations_by_source(cls, source,
first_ci_id=None, second_ci_type_id=None,
second_ci_id=None, first_ci_type_id=None,
added=None):
existed = []
if first_ci_id is not None and second_ci_type_id is not None:
existed = [(i.first_ci_id, i.second_ci_id) for i in CIRelation.get_by(
source=source, first_ci_id=first_ci_id, only_query=True).join(
CI, CIRelation.second_ci_id == CI.id).filter(CI.type_id == second_ci_type_id)]
if second_ci_id is not None and first_ci_type_id is not None:
existed = [(i.first_ci_id, i.second_ci_id) for i in CIRelation.get_by(
source=source, second_ci_id=second_ci_id, only_query=True).join(
CI, CIRelation.first_ci_id == CI.id).filter(CI.type_id == first_ci_type_id)]
deleted = set(existed) - set(added or [])
for first, second in deleted:
cls.delete_3(first, second, apply_async=False)
@classmethod
def build_by_attribute(cls, ci_dict):
type_id = ci_dict['_type']
@ -1296,8 +1332,15 @@ class CIRelationManager(object):
relations = _relations
else:
relations &= _relations
cls.delete_relations_by_source(RelationSourceEnum.ATTRIBUTE_VALUES,
first_ci_id=ci_dict['_id'],
second_ci_type_id=item.child_id,
added=relations)
for parent_ci_id, child_ci_id in (relations or []):
CIRelationManager.add(parent_ci_id, child_ci_id, valid=False)
cls.add(parent_ci_id, child_ci_id,
valid=False,
source=RelationSourceEnum.ATTRIBUTE_VALUES)
parent_items = CITypeRelation.get_by(child_id=type_id, only_query=True).filter(
CITypeRelation.child_attr_ids.isnot(None))
@ -1316,11 +1359,18 @@ class CIRelationManager(object):
relations = _relations
else:
relations &= _relations
cls.delete_relations_by_source(RelationSourceEnum.ATTRIBUTE_VALUES,
second_ci_id=ci_dict['_id'],
first_ci_type_id=item.parent_id,
added=relations)
for parent_ci_id, child_ci_id in (relations or []):
CIRelationManager.add(parent_ci_id, child_ci_id, valid=False)
cls.add(parent_ci_id, child_ci_id,
valid=False,
source=RelationSourceEnum.ATTRIBUTE_VALUES)
@classmethod
def rebuild_all_by_attribute(cls, ci_type_relation):
def rebuild_all_by_attribute(cls, ci_type_relation, uid):
relations = None
for parent_attr_id, child_attr_id in zip(ci_type_relation['parent_attr_ids'] or [],
ci_type_relation['child_attr_ids'] or []):
@ -1352,11 +1402,29 @@ class CIRelationManager(object):
else:
relations &= _relations
t1 = aliased(CI)
t2 = aliased(CI)
query = db.session.query(CIRelation).join(t1, t1.id == CIRelation.first_ci_id).join(
t2, t2.id == CIRelation.second_ci_id).filter(t1.type_id == ci_type_relation['parent_id']).filter(
t2.type_id == ci_type_relation['child_id'])
for i in query:
db.session.delete(i)
ci_relation_delete(i.first_ci_id, i.second_ci_id, i.ancestor_ids)
try:
db.session.commit()
except Exception as e:
current_app.logger.error(e)
db.session.rollback()
for parent_ci_id, child_ci_id in (relations or []):
try:
cls.add(parent_ci_id, child_ci_id, valid=False)
except:
pass
cls.add(parent_ci_id, child_ci_id,
valid=False,
apply_async=False,
source=RelationSourceEnum.ATTRIBUTE_VALUES,
uid=uid)
except Exception as e:
current_app.logger.error(e)
class CITriggerManager(object):

2
cmdb-api/api/lib/cmdb/ci_type.py
View File

@ -993,7 +993,7 @@ class CITypeRelationManager(object):
if ((parent_attr_ids and parent_attr_ids != old_parent_attr_ids) or
(child_attr_ids and child_attr_ids != old_child_attr_ids)):
from api.tasks.cmdb import rebuild_relation_for_attribute_changed
rebuild_relation_for_attribute_changed.apply_async(args=(existed.to_dict(),))
rebuild_relation_for_attribute_changed.apply_async(args=(existed.to_dict(), current_user.uid))
CITypeHistoryManager.add(CITypeOperateType.ADD_RELATION, p.id,
change=dict(parent=p.to_dict(), child=c.to_dict(), relation_type_id=relation_type_id))

40
cmdb-api/api/lib/cmdb/const.py
View File

@ -41,23 +41,23 @@ class OperateType(BaseEnum):
class CITypeOperateType(BaseEnum):
ADD = "0" # 新增模型
UPDATE = "1" # 修改模型
DELETE = "2" # 删除模型
ADD_ATTRIBUTE = "3" # 新增属性
UPDATE_ATTRIBUTE = "4" # 修改属性
DELETE_ATTRIBUTE = "5" # 删除属性
ADD_TRIGGER = "6" # 新增触发器
UPDATE_TRIGGER = "7" # 修改触发器
DELETE_TRIGGER = "8" # 删除触发器
ADD_UNIQUE_CONSTRAINT = "9" # 新增联合唯一
UPDATE_UNIQUE_CONSTRAINT = "10" # 修改联合唯一
DELETE_UNIQUE_CONSTRAINT = "11" # 删除联合唯一
ADD_RELATION = "12" # 新增关系
DELETE_RELATION = "13" # 删除关系
ADD_RECONCILIATION = "14" # 新增数据合规
UPDATE_RECONCILIATION = "15" # 修改数据合规
DELETE_RECONCILIATION = "16" # 删除数据合规
ADD = "0" # add CIType
UPDATE = "1" # update CIType
DELETE = "2" # delete CIType
ADD_ATTRIBUTE = "3"
UPDATE_ATTRIBUTE = "4"
DELETE_ATTRIBUTE = "5"
ADD_TRIGGER = "6"
UPDATE_TRIGGER = "7"
DELETE_TRIGGER = "8"
ADD_UNIQUE_CONSTRAINT = "9"
UPDATE_UNIQUE_CONSTRAINT = "10"
DELETE_UNIQUE_CONSTRAINT = "11"
ADD_RELATION = "12"
DELETE_RELATION = "13"
ADD_RECONCILIATION = "14"
UPDATE_RECONCILIATION = "15"
DELETE_RECONCILIATION = "16"
class RetKey(BaseEnum):
@ -93,7 +93,7 @@ class RoleEnum(BaseEnum):
class AutoDiscoveryType(BaseEnum):
AGENT = "agent"
SNMP = "snmp"
HTTP = "http" # cloud
HTTP = "http" # cloud
COMPONENTS = "components"
@ -108,6 +108,10 @@ class ExecuteStatusEnum(BaseEnum):
FAILED = '1'
RUNNING = '2'
class RelationSourceEnum(BaseEnum):
ATTRIBUTE_VALUES = "0"
AUTO_DISCOVERY = "1"
CMDB_QUEUE = "one_cmdb_async"
REDIS_PREFIX_CI = "ONE_CMDB"

4
cmdb-api/api/lib/cmdb/history.py
View File

@ -232,8 +232,8 @@ class AttributeHistoryManger(object):
class CIRelationHistoryManager(object):
@staticmethod
def add(rel_obj, operate_type=OperateType.ADD):
record = OperationRecord.create(uid=current_user.uid)
def add(rel_obj, operate_type=OperateType.ADD, uid=None):
record = OperationRecord.create(uid=uid or current_user.uid)
CIRelationHistory.create(relation_id=rel_obj.id,
record_id=record.id,

19
cmdb-api/api/lib/cmdb/utils.py
View File

@ -28,10 +28,7 @@ def string2int(x):
return v
def str2datetime(x):
x = x.replace('T', ' ')
x = x.replace('Z', '')
def str2date(x):
try:
return datetime.datetime.strptime(x, "%Y-%m-%d").date()
@ -43,9 +40,21 @@ def str2datetime(x):
except ValueError:
pass
def str2datetime(x):
x = x.replace('T', ' ')
x = x.replace('Z', '')
try:
return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")
except ValueError:
pass
return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M")
class ValueTypeMap(object):
deserialize = {
ValueTypeEnum.INT: string2int,
@ -53,7 +62,7 @@ class ValueTypeMap(object):
ValueTypeEnum.TEXT: lambda x: x,
ValueTypeEnum.TIME: lambda x: TIME_RE.findall(x)[0],
ValueTypeEnum.DATETIME: str2datetime,
ValueTypeEnum.DATE: str2datetime,
ValueTypeEnum.DATE: str2date,
ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x,
}

3
cmdb-api/api/models/cmdb.py
View File

@ -2,7 +2,6 @@
import datetime
from sqlalchemy.dialects.mysql import DOUBLE
from api.extensions import db
@ -11,6 +10,7 @@ from api.lib.cmdb.const import CIStatusEnum
from api.lib.cmdb.const import CITypeOperateType
from api.lib.cmdb.const import ConstraintEnum
from api.lib.cmdb.const import OperateType
from api.lib.cmdb.const import RelationSourceEnum
from api.lib.cmdb.const import ValueTypeEnum
from api.lib.database import Model
from api.lib.database import Model2
@ -260,6 +260,7 @@ class CIRelation(Model):
second_ci_id = db.Column(db.Integer, db.ForeignKey("c_cis.id"), nullable=False)
relation_type_id = db.Column(db.Integer, db.ForeignKey("c_relation_types.id"), nullable=False)
more = db.Column(db.Integer, db.ForeignKey("c_cis.id"))
source = db.Column(db.Enum(*RelationSourceEnum.all()), name="source")
ancestor_ids = db.Column(db.String(128), index=True)

4
cmdb-api/api/tasks/cmdb.py
View File

@ -58,10 +58,10 @@ def ci_cache(ci_id, operate_type, record_id):
@celery.task(name="cmdb.rebuild_relation_for_attribute_changed", queue=CMDB_QUEUE)
@reconnect_db
def rebuild_relation_for_attribute_changed(ci_type_relation):
def rebuild_relation_for_attribute_changed(ci_type_relation, uid):
from api.lib.cmdb.ci import CIRelationManager
CIRelationManager.rebuild_all_by_attribute(ci_type_relation)
CIRelationManager.rebuild_all_by_attribute(ci_type_relation, uid)
@celery.task(name="cmdb.batch_ci_cache", queue=CMDB_QUEUE)