7x31/cmdb
1
0
Fork 0
mirror of https://github.com/veops/cmdb.git synced 2025-08-08 09:02:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

perf(api): relationships built by attribute values (#572)

Browse Source
This commit is contained in:
pycook
2024-07-08 11:42:18 +08:00
committed by GitHub
parent 53dcf5d0d6
commit fff6da943d
9 changed files with 149 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py
View File

@@ -23,6 +23,7 @@ from api.lib.cmdb.ci_type import CITypeGroupManager
from api.lib.cmdb.const import AutoDiscoveryType
from api.lib.cmdb.const import CMDB_QUEUE
from api.lib.cmdb.const import PermEnum
from api.lib.cmdb.const import RelationSourceEnum
from api.lib.cmdb.const import ResourceTypeEnum
from api.lib.cmdb.custom_dashboard import SystemConfigManager
from api.lib.cmdb.resp_format import ErrFormat
@@ -246,6 +247,9 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
rules = cls.cls.get_by(to_dict=True)
for rule in rules:
if not rule['enabled']:
continue
if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('secret'):
if not (current_user.username in PRIVILEGED_USERS or current_user.uid == rule['uid']):
rule['extra_option'].pop('secret', None)
@@ -274,7 +278,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
break
elif not rule['agent_id'] and not rule['query_expr'] and rule['adr_id']:
try:
if not int(oneagent_id, 16): # excludes master
if not int(oneagent_id, 16): # excludes master
continue
except Exception:
pass
@@ -717,10 +721,15 @@ class AutoDiscoveryCICRUD(DBMixin):
for relation_ci in response:
relation_ci_id = relation_ci['_id']
try:
CIRelationManager.add(ci_id, relation_ci_id, valid=False)
CIRelationManager.add(ci_id, relation_ci_id,
valid=False,
source=RelationSourceEnum.AUTO_DISCOVERY)
except:
try:
CIRelationManager.add(relation_ci_id, ci_id, valid=False)
CIRelationManager.add(relation_ci_id, ci_id,
valid=False,
source=RelationSourceEnum.AUTO_DISCOVERY)
except:
pass

2
cmdb-api/api/lib/cmdb/auto_discovery/const.py
View File

@@ -41,7 +41,7 @@ CLOUD_MAP = {
"items": ["云服务器 ECS", "云服务器 Disk"],
"map": {
"云服务器 ECS": {"template": "templates/aliyun_ecs.json", "mapping": "ecs"},
"云服务器 Disk": {"template": "templates/aliyun_ecs_disk2.json", "mapping": "evs"},
"云服务器 Disk": {"template": "templates/aliyun_ecs_disk.json", "mapping": "evs"},
},
"collect_key_map": {
"云服务器 ECS": "ali.ecs",

118
cmdb-api/api/lib/cmdb/ci.py
View File

@@ -4,12 +4,12 @@
import copy
import datetime
import json
import threading
import redis_lock
import threading
from flask import abort
from flask import current_app
from flask_login import current_user
from sqlalchemy.orm import aliased
from werkzeug.exceptions import BadRequest
from api.extensions import db
@@ -28,6 +28,7 @@ from api.lib.cmdb.const import ExistPolicy
from api.lib.cmdb.const import OperateType
from api.lib.cmdb.const import PermEnum
from api.lib.cmdb.const import REDIS_PREFIX_CI
from api.lib.cmdb.const import RelationSourceEnum
from api.lib.cmdb.const import ResourceTypeEnum
from api.lib.cmdb.const import RetKey
from api.lib.cmdb.const import ValueTypeEnum
@@ -1133,7 +1134,14 @@ class CIRelationManager(object):
return abort(400, ErrFormat.relation_constraint.format("1-N"))
@classmethod
def add(cls, first_ci_id, second_ci_id, more=None, relation_type_id=None, ancestor_ids=None, valid=True):
def add(cls, first_ci_id, second_ci_id,
more=None,
relation_type_id=None,
ancestor_ids=None,
valid=True,
apply_async=True,
source=None,
uid=None):
first_ci = CIManager.confirm_ci_existed(first_ci_id)
second_ci = CIManager.confirm_ci_existed(second_ci_id)
@@ -1145,9 +1153,10 @@ class CIRelationManager(object):
first=True)
if existed is not None:
if existed.relation_type_id != relation_type_id and relation_type_id is not None:
existed.update(relation_type_id=relation_type_id)
source = existed.source or source
existed.update(relation_type_id=relation_type_id, source=source)
CIRelationHistoryManager().add(existed, OperateType.UPDATE)
CIRelationHistoryManager().add(existed, OperateType.UPDATE, uid=uid)
else:
if relation_type_id is None:
type_relation = CITypeRelation.get_by(parent_id=first_ci.type_id,
@@ -1177,11 +1186,13 @@ class CIRelationManager(object):
existed = CIRelation.create(first_ci_id=first_ci_id,
second_ci_id=second_ci_id,
relation_type_id=relation_type_id,
ancestor_ids=ancestor_ids)
CIRelationHistoryManager().add(existed, OperateType.ADD)
ci_relation_cache.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE)
ancestor_ids=ancestor_ids,
source=source)
CIRelationHistoryManager().add(existed, OperateType.ADD, uid=uid)
if apply_async:
ci_relation_cache.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE)
else:
ci_relation_cache(first_ci_id, second_ci_id, ancestor_ids)
if more is not None:
existed.upadte(more=more)
@@ -1189,7 +1200,7 @@ class CIRelationManager(object):
return existed.id
@staticmethod
def delete(cr_id):
def delete(cr_id, apply_async=True):
cr = CIRelation.get_by_id(cr_id) or abort(404, ErrFormat.relation_not_found.format("id={}".format(cr_id)))
if current_app.config.get('USE_ACL') and current_user.username != 'worker':
@@ -1205,8 +1216,12 @@ class CIRelationManager(object):
his_manager = CIRelationHistoryManager()
his_manager.add(cr, operate_type=OperateType.DELETE)
ci_relation_delete.apply_async(args=(cr.first_ci_id, cr.second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE)
delete_id_filter.apply_async(args=(cr.second_ci_id,), queue=CMDB_QUEUE)
if apply_async:
ci_relation_delete.apply_async(args=(cr.first_ci_id, cr.second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE)
delete_id_filter.apply_async(args=(cr.second_ci_id,), queue=CMDB_QUEUE)
else:
ci_relation_delete(cr.first_ci_id, cr.second_ci_id, cr.ancestor_ids)
delete_id_filter(cr.second_ci_id)
return cr_id
@@ -1221,23 +1236,23 @@ class CIRelationManager(object):
if cr is not None:
cls.delete(cr.id)
ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE)
delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE)
# ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE)
# delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE)
return cr
@classmethod
def delete_3(cls, first_ci_id, second_ci_id):
def delete_3(cls, first_ci_id, second_ci_id, apply_async=True):
cr = CIRelation.get_by(first_ci_id=first_ci_id,
second_ci_id=second_ci_id,
to_dict=False,
first=True)
if cr is not None:
ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE)
delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE)
# ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE)
# delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE)
cls.delete(cr.id)
cls.delete(cr.id, apply_async=apply_async)
return cr
@@ -1276,6 +1291,27 @@ class CIRelationManager(object):
for ci_id in ci_ids:
cls.delete_2(parent_id, ci_id, ancestor_ids=ancestor_ids)
@classmethod
def delete_relations_by_source(cls, source,
first_ci_id=None, second_ci_type_id=None,
second_ci_id=None, first_ci_type_id=None,
added=None):
existed = []
if first_ci_id is not None and second_ci_type_id is not None:
existed = [(i.first_ci_id, i.second_ci_id) for i in CIRelation.get_by(
source=source, first_ci_id=first_ci_id, only_query=True).join(
CI, CIRelation.second_ci_id == CI.id).filter(CI.type_id == second_ci_type_id)]
if second_ci_id is not None and first_ci_type_id is not None:
existed = [(i.first_ci_id, i.second_ci_id) for i in CIRelation.get_by(
source=source, second_ci_id=second_ci_id, only_query=True).join(
CI, CIRelation.first_ci_id == CI.id).filter(CI.type_id == first_ci_type_id)]
deleted = set(existed) - set(added or [])
for first, second in deleted:
cls.delete_3(first, second, apply_async=False)
@classmethod
def build_by_attribute(cls, ci_dict):
type_id = ci_dict['_type']
@@ -1296,8 +1332,15 @@ class CIRelationManager(object):
relations = _relations
else:
relations &= _relations
cls.delete_relations_by_source(RelationSourceEnum.ATTRIBUTE_VALUES,
first_ci_id=ci_dict['_id'],
second_ci_type_id=item.child_id,
added=relations)
for parent_ci_id, child_ci_id in (relations or []):
CIRelationManager.add(parent_ci_id, child_ci_id, valid=False)
cls.add(parent_ci_id, child_ci_id,
valid=False,
source=RelationSourceEnum.ATTRIBUTE_VALUES)
parent_items = CITypeRelation.get_by(child_id=type_id, only_query=True).filter(
CITypeRelation.child_attr_ids.isnot(None))
@@ -1316,11 +1359,18 @@ class CIRelationManager(object):
relations = _relations
else:
relations &= _relations
cls.delete_relations_by_source(RelationSourceEnum.ATTRIBUTE_VALUES,
second_ci_id=ci_dict['_id'],
first_ci_type_id=item.parent_id,
added=relations)
for parent_ci_id, child_ci_id in (relations or []):
CIRelationManager.add(parent_ci_id, child_ci_id, valid=False)
cls.add(parent_ci_id, child_ci_id,
valid=False,
source=RelationSourceEnum.ATTRIBUTE_VALUES)
@classmethod
def rebuild_all_by_attribute(cls, ci_type_relation):
def rebuild_all_by_attribute(cls, ci_type_relation, uid):
relations = None
for parent_attr_id, child_attr_id in zip(ci_type_relation['parent_attr_ids'] or [],
ci_type_relation['child_attr_ids'] or []):
@@ -1352,11 +1402,29 @@ class CIRelationManager(object):
else:
relations &= _relations
t1 = aliased(CI)
t2 = aliased(CI)
query = db.session.query(CIRelation).join(t1, t1.id == CIRelation.first_ci_id).join(
t2, t2.id == CIRelation.second_ci_id).filter(t1.type_id == ci_type_relation['parent_id']).filter(
t2.type_id == ci_type_relation['child_id'])
for i in query:
db.session.delete(i)
ci_relation_delete(i.first_ci_id, i.second_ci_id, i.ancestor_ids)
try:
db.session.commit()
except Exception as e:
current_app.logger.error(e)
db.session.rollback()
for parent_ci_id, child_ci_id in (relations or []):
try:
cls.add(parent_ci_id, child_ci_id, valid=False)
except:
pass
cls.add(parent_ci_id, child_ci_id,
valid=False,
apply_async=False,
source=RelationSourceEnum.ATTRIBUTE_VALUES,
uid=uid)
except Exception as e:
current_app.logger.error(e)
class CITriggerManager(object):

2
cmdb-api/api/lib/cmdb/ci_type.py
View File

@@ -993,7 +993,7 @@ class CITypeRelationManager(object):
if ((parent_attr_ids and parent_attr_ids != old_parent_attr_ids) or
(child_attr_ids and child_attr_ids != old_child_attr_ids)):
from api.tasks.cmdb import rebuild_relation_for_attribute_changed
rebuild_relation_for_attribute_changed.apply_async(args=(existed.to_dict(),))
rebuild_relation_for_attribute_changed.apply_async(args=(existed.to_dict(), current_user.uid))
CITypeHistoryManager.add(CITypeOperateType.ADD_RELATION, p.id,
change=dict(parent=p.to_dict(), child=c.to_dict(), relation_type_id=relation_type_id))

40
cmdb-api/api/lib/cmdb/const.py
View File

@@ -41,23 +41,23 @@ class OperateType(BaseEnum):
class CITypeOperateType(BaseEnum):
ADD = "0" # 新增模型
UPDATE = "1" # 修改模型
DELETE = "2" # 删除模型
ADD_ATTRIBUTE = "3" # 新增属性
UPDATE_ATTRIBUTE = "4" # 修改属性
DELETE_ATTRIBUTE = "5" # 删除属性
ADD_TRIGGER = "6" # 新增触发器
UPDATE_TRIGGER = "7" # 修改触发器
DELETE_TRIGGER = "8" # 删除触发器
ADD_UNIQUE_CONSTRAINT = "9" # 新增联合唯一
UPDATE_UNIQUE_CONSTRAINT = "10" # 修改联合唯一
DELETE_UNIQUE_CONSTRAINT = "11" # 删除联合唯一
ADD_RELATION = "12" # 新增关系
DELETE_RELATION = "13" # 删除关系
ADD_RECONCILIATION = "14" # 新增数据合规
UPDATE_RECONCILIATION = "15" # 修改数据合规
DELETE_RECONCILIATION = "16" # 删除数据合规
ADD = "0" # add CIType
UPDATE = "1" # update CIType
DELETE = "2" # delete CIType
ADD_ATTRIBUTE = "3"
UPDATE_ATTRIBUTE = "4"
DELETE_ATTRIBUTE = "5"
ADD_TRIGGER = "6"
UPDATE_TRIGGER = "7"
DELETE_TRIGGER = "8"
ADD_UNIQUE_CONSTRAINT = "9"
UPDATE_UNIQUE_CONSTRAINT = "10"
DELETE_UNIQUE_CONSTRAINT = "11"
ADD_RELATION = "12"
DELETE_RELATION = "13"
ADD_RECONCILIATION = "14"
UPDATE_RECONCILIATION = "15"
DELETE_RECONCILIATION = "16"
class RetKey(BaseEnum):
@@ -93,7 +93,7 @@ class RoleEnum(BaseEnum):
class AutoDiscoveryType(BaseEnum):
AGENT = "agent"
SNMP = "snmp"
HTTP = "http" # cloud
HTTP = "http" # cloud
COMPONENTS = "components"
@@ -108,6 +108,10 @@ class ExecuteStatusEnum(BaseEnum):
FAILED = '1'
RUNNING = '2'
class RelationSourceEnum(BaseEnum):
ATTRIBUTE_VALUES = "0"
AUTO_DISCOVERY = "1"
CMDB_QUEUE = "one_cmdb_async"
REDIS_PREFIX_CI = "ONE_CMDB"

4
cmdb-api/api/lib/cmdb/history.py
View File

@@ -232,8 +232,8 @@ class AttributeHistoryManger(object):
class CIRelationHistoryManager(object):
@staticmethod
def add(rel_obj, operate_type=OperateType.ADD):
record = OperationRecord.create(uid=current_user.uid)
def add(rel_obj, operate_type=OperateType.ADD, uid=None):
record = OperationRecord.create(uid=uid or current_user.uid)
CIRelationHistory.create(relation_id=rel_obj.id,
record_id=record.id,

19
cmdb-api/api/lib/cmdb/utils.py
View File

@@ -28,10 +28,7 @@ def string2int(x):
return v
def str2datetime(x):
x = x.replace('T', ' ')
x = x.replace('Z', '')
def str2date(x):
try:
return datetime.datetime.strptime(x, "%Y-%m-%d").date()
@@ -43,9 +40,21 @@ def str2datetime(x):
except ValueError:
pass
def str2datetime(x):
x = x.replace('T', ' ')
x = x.replace('Z', '')
try:
return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")
except ValueError:
pass
return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M")
class ValueTypeMap(object):
deserialize = {
ValueTypeEnum.INT: string2int,
@@ -53,7 +62,7 @@ class ValueTypeMap(object):
ValueTypeEnum.TEXT: lambda x: x,
ValueTypeEnum.TIME: lambda x: TIME_RE.findall(x)[0],
ValueTypeEnum.DATETIME: str2datetime,
ValueTypeEnum.DATE: str2datetime,
ValueTypeEnum.DATE: str2date,
ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x,
}