diff --git a/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py b/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py
index 4fa8968..323cf3a 100644
--- a/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py
+++ b/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py
@@ -23,6 +23,7 @@ from api.lib.cmdb.ci_type import CITypeGroupManager
from api.lib.cmdb.const import AutoDiscoveryType
from api.lib.cmdb.const import CMDB_QUEUE
from api.lib.cmdb.const import PermEnum
+from api.lib.cmdb.const import RelationSourceEnum
from api.lib.cmdb.const import ResourceTypeEnum
from api.lib.cmdb.custom_dashboard import SystemConfigManager
from api.lib.cmdb.resp_format import ErrFormat
@@ -246,6 +247,9 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
rules = cls.cls.get_by(to_dict=True)
for rule in rules:
+ if not rule['enabled']:
+ continue
+
if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('secret'):
if not (current_user.username in PRIVILEGED_USERS or current_user.uid == rule['uid']):
rule['extra_option'].pop('secret', None)
@@ -274,7 +278,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
break
elif not rule['agent_id'] and not rule['query_expr'] and rule['adr_id']:
try:
- if not int(oneagent_id, 16): # excludes master
+ if not int(oneagent_id, 16): # excludes master
continue
except Exception:
pass
@@ -717,10 +721,15 @@ class AutoDiscoveryCICRUD(DBMixin):
for relation_ci in response:
relation_ci_id = relation_ci['_id']
try:
- CIRelationManager.add(ci_id, relation_ci_id, valid=False)
+ CIRelationManager.add(ci_id, relation_ci_id,
+ valid=False,
+ source=RelationSourceEnum.AUTO_DISCOVERY)
+
except:
try:
- CIRelationManager.add(relation_ci_id, ci_id, valid=False)
+ CIRelationManager.add(relation_ci_id, ci_id,
+ valid=False,
+ source=RelationSourceEnum.AUTO_DISCOVERY)
except:
pass
diff --git a/cmdb-api/api/lib/cmdb/auto_discovery/const.py b/cmdb-api/api/lib/cmdb/auto_discovery/const.py
index a22aed0..d34c4a8 100644
--- a/cmdb-api/api/lib/cmdb/auto_discovery/const.py
+++ b/cmdb-api/api/lib/cmdb/auto_discovery/const.py
@@ -41,7 +41,7 @@ CLOUD_MAP = {
"items": ["云服务器 ECS", "云服务器 Disk"],
"map": {
"云服务器 ECS": {"template": "templates/aliyun_ecs.json", "mapping": "ecs"},
- "云服务器 Disk": {"template": "templates/aliyun_ecs_disk2.json", "mapping": "evs"},
+ "云服务器 Disk": {"template": "templates/aliyun_ecs_disk.json", "mapping": "evs"},
},
"collect_key_map": {
"云服务器 ECS": "ali.ecs",
diff --git a/cmdb-api/api/lib/cmdb/ci.py b/cmdb-api/api/lib/cmdb/ci.py
index d3a1047..85f8ddd 100644
--- a/cmdb-api/api/lib/cmdb/ci.py
+++ b/cmdb-api/api/lib/cmdb/ci.py
@@ -4,12 +4,12 @@
import copy
import datetime
import json
-import threading
-
import redis_lock
+import threading
from flask import abort
from flask import current_app
from flask_login import current_user
+from sqlalchemy.orm import aliased
from werkzeug.exceptions import BadRequest
from api.extensions import db
@@ -28,6 +28,7 @@ from api.lib.cmdb.const import ExistPolicy
from api.lib.cmdb.const import OperateType
from api.lib.cmdb.const import PermEnum
from api.lib.cmdb.const import REDIS_PREFIX_CI
+from api.lib.cmdb.const import RelationSourceEnum
from api.lib.cmdb.const import ResourceTypeEnum
from api.lib.cmdb.const import RetKey
from api.lib.cmdb.const import ValueTypeEnum
@@ -1133,7 +1134,14 @@ class CIRelationManager(object):
return abort(400, ErrFormat.relation_constraint.format("1-N"))
@classmethod
- def add(cls, first_ci_id, second_ci_id, more=None, relation_type_id=None, ancestor_ids=None, valid=True):
+ def add(cls, first_ci_id, second_ci_id,
+ more=None,
+ relation_type_id=None,
+ ancestor_ids=None,
+ valid=True,
+ apply_async=True,
+ source=None,
+ uid=None):
first_ci = CIManager.confirm_ci_existed(first_ci_id)
second_ci = CIManager.confirm_ci_existed(second_ci_id)
@@ -1145,9 +1153,10 @@ class CIRelationManager(object):
first=True)
if existed is not None:
if existed.relation_type_id != relation_type_id and relation_type_id is not None:
- existed.update(relation_type_id=relation_type_id)
+ source = existed.source or source
+ existed.update(relation_type_id=relation_type_id, source=source)
- CIRelationHistoryManager().add(existed, OperateType.UPDATE)
+ CIRelationHistoryManager().add(existed, OperateType.UPDATE, uid=uid)
else:
if relation_type_id is None:
type_relation = CITypeRelation.get_by(parent_id=first_ci.type_id,
@@ -1177,11 +1186,13 @@ class CIRelationManager(object):
existed = CIRelation.create(first_ci_id=first_ci_id,
second_ci_id=second_ci_id,
relation_type_id=relation_type_id,
- ancestor_ids=ancestor_ids)
-
- CIRelationHistoryManager().add(existed, OperateType.ADD)
-
- ci_relation_cache.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE)
+ ancestor_ids=ancestor_ids,
+ source=source)
+ CIRelationHistoryManager().add(existed, OperateType.ADD, uid=uid)
+ if apply_async:
+ ci_relation_cache.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE)
+ else:
+ ci_relation_cache(first_ci_id, second_ci_id, ancestor_ids)
if more is not None:
existed.upadte(more=more)
@@ -1189,7 +1200,7 @@ class CIRelationManager(object):
return existed.id
@staticmethod
- def delete(cr_id):
+ def delete(cr_id, apply_async=True):
cr = CIRelation.get_by_id(cr_id) or abort(404, ErrFormat.relation_not_found.format("id={}".format(cr_id)))
if current_app.config.get('USE_ACL') and current_user.username != 'worker':
@@ -1205,8 +1216,12 @@ class CIRelationManager(object):
his_manager = CIRelationHistoryManager()
his_manager.add(cr, operate_type=OperateType.DELETE)
- ci_relation_delete.apply_async(args=(cr.first_ci_id, cr.second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE)
- delete_id_filter.apply_async(args=(cr.second_ci_id,), queue=CMDB_QUEUE)
+ if apply_async:
+ ci_relation_delete.apply_async(args=(cr.first_ci_id, cr.second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE)
+ delete_id_filter.apply_async(args=(cr.second_ci_id,), queue=CMDB_QUEUE)
+ else:
+ ci_relation_delete(cr.first_ci_id, cr.second_ci_id, cr.ancestor_ids)
+ delete_id_filter(cr.second_ci_id)
return cr_id
@@ -1221,23 +1236,23 @@ class CIRelationManager(object):
if cr is not None:
cls.delete(cr.id)
- ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE)
- delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE)
+ # ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE)
+ # delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE)
return cr
@classmethod
- def delete_3(cls, first_ci_id, second_ci_id):
+ def delete_3(cls, first_ci_id, second_ci_id, apply_async=True):
cr = CIRelation.get_by(first_ci_id=first_ci_id,
second_ci_id=second_ci_id,
to_dict=False,
first=True)
if cr is not None:
- ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE)
- delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE)
+ # ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE)
+ # delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE)
- cls.delete(cr.id)
+ cls.delete(cr.id, apply_async=apply_async)
return cr
@@ -1276,6 +1291,27 @@ class CIRelationManager(object):
for ci_id in ci_ids:
cls.delete_2(parent_id, ci_id, ancestor_ids=ancestor_ids)
+ @classmethod
+ def delete_relations_by_source(cls, source,
+ first_ci_id=None, second_ci_type_id=None,
+ second_ci_id=None, first_ci_type_id=None,
+ added=None):
+ existed = []
+ if first_ci_id is not None and second_ci_type_id is not None:
+ existed = [(i.first_ci_id, i.second_ci_id) for i in CIRelation.get_by(
+ source=source, first_ci_id=first_ci_id, only_query=True).join(
+ CI, CIRelation.second_ci_id == CI.id).filter(CI.type_id == second_ci_type_id)]
+
+ if second_ci_id is not None and first_ci_type_id is not None:
+ existed = [(i.first_ci_id, i.second_ci_id) for i in CIRelation.get_by(
+ source=source, second_ci_id=second_ci_id, only_query=True).join(
+ CI, CIRelation.first_ci_id == CI.id).filter(CI.type_id == first_ci_type_id)]
+
+ deleted = set(existed) - set(added or [])
+
+ for first, second in deleted:
+ cls.delete_3(first, second, apply_async=False)
+
@classmethod
def build_by_attribute(cls, ci_dict):
type_id = ci_dict['_type']
@@ -1296,8 +1332,15 @@ class CIRelationManager(object):
relations = _relations
else:
relations &= _relations
+
+ cls.delete_relations_by_source(RelationSourceEnum.ATTRIBUTE_VALUES,
+ first_ci_id=ci_dict['_id'],
+ second_ci_type_id=item.child_id,
+ added=relations)
for parent_ci_id, child_ci_id in (relations or []):
- CIRelationManager.add(parent_ci_id, child_ci_id, valid=False)
+ cls.add(parent_ci_id, child_ci_id,
+ valid=False,
+ source=RelationSourceEnum.ATTRIBUTE_VALUES)
parent_items = CITypeRelation.get_by(child_id=type_id, only_query=True).filter(
CITypeRelation.child_attr_ids.isnot(None))
@@ -1316,11 +1359,18 @@ class CIRelationManager(object):
relations = _relations
else:
relations &= _relations
+
+ cls.delete_relations_by_source(RelationSourceEnum.ATTRIBUTE_VALUES,
+ second_ci_id=ci_dict['_id'],
+ first_ci_type_id=item.parent_id,
+ added=relations)
for parent_ci_id, child_ci_id in (relations or []):
- CIRelationManager.add(parent_ci_id, child_ci_id, valid=False)
+ cls.add(parent_ci_id, child_ci_id,
+ valid=False,
+ source=RelationSourceEnum.ATTRIBUTE_VALUES)
@classmethod
- def rebuild_all_by_attribute(cls, ci_type_relation):
+ def rebuild_all_by_attribute(cls, ci_type_relation, uid):
relations = None
for parent_attr_id, child_attr_id in zip(ci_type_relation['parent_attr_ids'] or [],
ci_type_relation['child_attr_ids'] or []):
@@ -1352,11 +1402,29 @@ class CIRelationManager(object):
else:
relations &= _relations
+ t1 = aliased(CI)
+ t2 = aliased(CI)
+ query = db.session.query(CIRelation).join(t1, t1.id == CIRelation.first_ci_id).join(
+ t2, t2.id == CIRelation.second_ci_id).filter(t1.type_id == ci_type_relation['parent_id']).filter(
+ t2.type_id == ci_type_relation['child_id'])
+ for i in query:
+ db.session.delete(i)
+ ci_relation_delete(i.first_ci_id, i.second_ci_id, i.ancestor_ids)
+ try:
+ db.session.commit()
+ except Exception as e:
+ current_app.logger.error(e)
+ db.session.rollback()
+
for parent_ci_id, child_ci_id in (relations or []):
try:
- cls.add(parent_ci_id, child_ci_id, valid=False)
- except:
- pass
+ cls.add(parent_ci_id, child_ci_id,
+ valid=False,
+ apply_async=False,
+ source=RelationSourceEnum.ATTRIBUTE_VALUES,
+ uid=uid)
+ except Exception as e:
+ current_app.logger.error(e)
class CITriggerManager(object):
diff --git a/cmdb-api/api/lib/cmdb/ci_type.py b/cmdb-api/api/lib/cmdb/ci_type.py
index 847ef98..0164de1 100644
--- a/cmdb-api/api/lib/cmdb/ci_type.py
+++ b/cmdb-api/api/lib/cmdb/ci_type.py
@@ -993,7 +993,7 @@ class CITypeRelationManager(object):
if ((parent_attr_ids and parent_attr_ids != old_parent_attr_ids) or
(child_attr_ids and child_attr_ids != old_child_attr_ids)):
from api.tasks.cmdb import rebuild_relation_for_attribute_changed
- rebuild_relation_for_attribute_changed.apply_async(args=(existed.to_dict(),))
+ rebuild_relation_for_attribute_changed.apply_async(args=(existed.to_dict(), current_user.uid))
CITypeHistoryManager.add(CITypeOperateType.ADD_RELATION, p.id,
change=dict(parent=p.to_dict(), child=c.to_dict(), relation_type_id=relation_type_id))
diff --git a/cmdb-api/api/lib/cmdb/const.py b/cmdb-api/api/lib/cmdb/const.py
index d0302be..ecda9e3 100644
--- a/cmdb-api/api/lib/cmdb/const.py
+++ b/cmdb-api/api/lib/cmdb/const.py
@@ -41,23 +41,23 @@ class OperateType(BaseEnum):
class CITypeOperateType(BaseEnum):
- ADD = "0" # 新增模型
- UPDATE = "1" # 修改模型
- DELETE = "2" # 删除模型
- ADD_ATTRIBUTE = "3" # 新增属性
- UPDATE_ATTRIBUTE = "4" # 修改属性
- DELETE_ATTRIBUTE = "5" # 删除属性
- ADD_TRIGGER = "6" # 新增触发器
- UPDATE_TRIGGER = "7" # 修改触发器
- DELETE_TRIGGER = "8" # 删除触发器
- ADD_UNIQUE_CONSTRAINT = "9" # 新增联合唯一
- UPDATE_UNIQUE_CONSTRAINT = "10" # 修改联合唯一
- DELETE_UNIQUE_CONSTRAINT = "11" # 删除联合唯一
- ADD_RELATION = "12" # 新增关系
- DELETE_RELATION = "13" # 删除关系
- ADD_RECONCILIATION = "14" # 新增数据合规
- UPDATE_RECONCILIATION = "15" # 修改数据合规
- DELETE_RECONCILIATION = "16" # 删除数据合规
+ ADD = "0" # add CIType
+ UPDATE = "1" # update CIType
+ DELETE = "2" # delete CIType
+ ADD_ATTRIBUTE = "3"
+ UPDATE_ATTRIBUTE = "4"
+ DELETE_ATTRIBUTE = "5"
+ ADD_TRIGGER = "6"
+ UPDATE_TRIGGER = "7"
+ DELETE_TRIGGER = "8"
+ ADD_UNIQUE_CONSTRAINT = "9"
+ UPDATE_UNIQUE_CONSTRAINT = "10"
+ DELETE_UNIQUE_CONSTRAINT = "11"
+ ADD_RELATION = "12"
+ DELETE_RELATION = "13"
+ ADD_RECONCILIATION = "14"
+ UPDATE_RECONCILIATION = "15"
+ DELETE_RECONCILIATION = "16"
class RetKey(BaseEnum):
@@ -93,7 +93,7 @@ class RoleEnum(BaseEnum):
class AutoDiscoveryType(BaseEnum):
AGENT = "agent"
SNMP = "snmp"
- HTTP = "http" # cloud
+ HTTP = "http" # cloud
COMPONENTS = "components"
@@ -108,6 +108,10 @@ class ExecuteStatusEnum(BaseEnum):
FAILED = '1'
RUNNING = '2'
+class RelationSourceEnum(BaseEnum):
+ ATTRIBUTE_VALUES = "0"
+ AUTO_DISCOVERY = "1"
+
CMDB_QUEUE = "one_cmdb_async"
REDIS_PREFIX_CI = "ONE_CMDB"
diff --git a/cmdb-api/api/lib/cmdb/history.py b/cmdb-api/api/lib/cmdb/history.py
index 6e85038..2c790d2 100644
--- a/cmdb-api/api/lib/cmdb/history.py
+++ b/cmdb-api/api/lib/cmdb/history.py
@@ -232,8 +232,8 @@ class AttributeHistoryManger(object):
class CIRelationHistoryManager(object):
@staticmethod
- def add(rel_obj, operate_type=OperateType.ADD):
- record = OperationRecord.create(uid=current_user.uid)
+ def add(rel_obj, operate_type=OperateType.ADD, uid=None):
+ record = OperationRecord.create(uid=uid or current_user.uid)
CIRelationHistory.create(relation_id=rel_obj.id,
record_id=record.id,
diff --git a/cmdb-api/api/lib/cmdb/utils.py b/cmdb-api/api/lib/cmdb/utils.py
index 7b0ff2a..ab354ea 100644
--- a/cmdb-api/api/lib/cmdb/utils.py
+++ b/cmdb-api/api/lib/cmdb/utils.py
@@ -28,10 +28,7 @@ def string2int(x):
return v
-def str2datetime(x):
-
- x = x.replace('T', ' ')
- x = x.replace('Z', '')
+def str2date(x):
try:
return datetime.datetime.strptime(x, "%Y-%m-%d").date()
@@ -43,9 +40,21 @@ def str2datetime(x):
except ValueError:
pass
+
+def str2datetime(x):
+
+ x = x.replace('T', ' ')
+ x = x.replace('Z', '')
+
+ try:
+ return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")
+ except ValueError:
+ pass
+
return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M")
+
class ValueTypeMap(object):
deserialize = {
ValueTypeEnum.INT: string2int,
@@ -53,7 +62,7 @@ class ValueTypeMap(object):
ValueTypeEnum.TEXT: lambda x: x,
ValueTypeEnum.TIME: lambda x: TIME_RE.findall(x)[0],
ValueTypeEnum.DATETIME: str2datetime,
- ValueTypeEnum.DATE: str2datetime,
+ ValueTypeEnum.DATE: str2date,
ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x,
}
diff --git a/cmdb-api/api/models/cmdb.py b/cmdb-api/api/models/cmdb.py
index 7a5bbb7..7673a12 100644
--- a/cmdb-api/api/models/cmdb.py
+++ b/cmdb-api/api/models/cmdb.py
@@ -2,7 +2,6 @@
import datetime
-
from sqlalchemy.dialects.mysql import DOUBLE
from api.extensions import db
@@ -11,6 +10,7 @@ from api.lib.cmdb.const import CIStatusEnum
from api.lib.cmdb.const import CITypeOperateType
from api.lib.cmdb.const import ConstraintEnum
from api.lib.cmdb.const import OperateType
+from api.lib.cmdb.const import RelationSourceEnum
from api.lib.cmdb.const import ValueTypeEnum
from api.lib.database import Model
from api.lib.database import Model2
@@ -260,6 +260,7 @@ class CIRelation(Model):
second_ci_id = db.Column(db.Integer, db.ForeignKey("c_cis.id"), nullable=False)
relation_type_id = db.Column(db.Integer, db.ForeignKey("c_relation_types.id"), nullable=False)
more = db.Column(db.Integer, db.ForeignKey("c_cis.id"))
+ source = db.Column(db.Enum(*RelationSourceEnum.all()), name="source")
ancestor_ids = db.Column(db.String(128), index=True)
diff --git a/cmdb-api/api/tasks/cmdb.py b/cmdb-api/api/tasks/cmdb.py
index af94423..1fa1d40 100644
--- a/cmdb-api/api/tasks/cmdb.py
+++ b/cmdb-api/api/tasks/cmdb.py
@@ -58,10 +58,10 @@ def ci_cache(ci_id, operate_type, record_id):
@celery.task(name="cmdb.rebuild_relation_for_attribute_changed", queue=CMDB_QUEUE)
@reconnect_db
-def rebuild_relation_for_attribute_changed(ci_type_relation):
+def rebuild_relation_for_attribute_changed(ci_type_relation, uid):
from api.lib.cmdb.ci import CIRelationManager
- CIRelationManager.rebuild_all_by_attribute(ci_type_relation)
+ CIRelationManager.rebuild_all_by_attribute(ci_type_relation, uid)
@celery.task(name="cmdb.batch_ci_cache", queue=CMDB_QUEUE)