7x31
/
cmdb
mirror of https://github.com/veops/cmdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

perf(api): auto discovery (#582)

This commit is contained in:
pycook 2024-07-25 17:45:26 +08:00 committed by GitHub
parent 16c4a08b74
commit 1336a24044
11 changed files with 328 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmdb-api/api/lib/cmdb/attribute.py
View File

@ -229,7 +229,7 @@ class AttributeManager(object):
is_choice = True if choice_value or kwargs.get('choice_web_hook') or kwargs.get('choice_other') else False
name = kwargs.pop("name")
if name in BUILTIN_KEYWORDS:
if name in BUILTIN_KEYWORDS or kwargs.get('alias') in BUILTIN_KEYWORDS:
return abort(400, ErrFormat.attribute_name_cannot_be_builtin)
while kwargs.get('choice_other'):

217
cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py
View File

@ -18,12 +18,10 @@ from api.lib.cmdb.cache import AutoDiscoveryMappingCache
from api.lib.cmdb.cache import CITypeAttributeCache
from api.lib.cmdb.cache import CITypeCache
from api.lib.cmdb.ci import CIManager
from api.lib.cmdb.ci import CIRelationManager
from api.lib.cmdb.ci_type import CITypeGroupManager
from api.lib.cmdb.const import AutoDiscoveryType
from api.lib.cmdb.const import CMDB_QUEUE
from api.lib.cmdb.const import PermEnum
from api.lib.cmdb.const import RelationSourceEnum
from api.lib.cmdb.const import ResourceTypeEnum
from api.lib.cmdb.custom_dashboard import SystemConfigManager
from api.lib.cmdb.resp_format import ErrFormat
@ -35,6 +33,7 @@ from api.lib.perm.acl.acl import ACLManager
from api.lib.perm.acl.acl import is_app_admin
from api.lib.perm.acl.acl import validate_permission
from api.lib.utils import AESCrypto
from api.models.cmdb import AutoDiscoveryAccount
from api.models.cmdb import AutoDiscoveryCI
from api.models.cmdb import AutoDiscoveryCIType
from api.models.cmdb import AutoDiscoveryCITypeRelation
@ -42,6 +41,7 @@ from api.models.cmdb import AutoDiscoveryCounter
from api.models.cmdb import AutoDiscoveryExecHistory
from api.models.cmdb import AutoDiscoveryRule
from api.models.cmdb import AutoDiscoveryRuleSyncHistory
from api.tasks.cmdb import build_relations_for_ad_accept
from api.tasks.cmdb import write_ad_rule_sync_history
PWD = os.path.abspath(os.path.dirname(__file__))
@ -226,14 +226,14 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
adr = AutoDiscoveryRuleCRUD.get_by_id(adt.adr_id)
if not adr:
continue
if adr.type == "http":
if adr.type == AutoDiscoveryType.HTTP:
for i in DEFAULT_INNER:
if adr.name == i['name']:
attrs = AutoDiscoveryHTTPManager.get_attributes(
i['en'], (adt.extra_option or {}).get('category')) or []
result.extend([i.get('name') for i in attrs])
break
elif adr.type == "snmp":
elif adr.type == AutoDiscoveryType.SNMP:
attributes = AutoDiscoverySNMPManager.get_attributes()
result.extend([i.get('name') for i in (attributes or [])])
else:
@ -243,6 +243,14 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
@classmethod
def get(cls, ci_id, oneagent_id, oneagent_name, last_update_at=None):
"""
OneAgent sync rules
:param ci_id:
:param oneagent_id:
:param oneagent_name:
:param last_update_at:
:return:
"""
result = []
rules = cls.cls.get_by(to_dict=True)
@ -250,17 +258,14 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
if not rule['enabled']:
continue
if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('secret'):
if not (current_user.username in PRIVILEGED_USERS or current_user.uid == rule['uid']):
rule['extra_option'].pop('secret', None)
else:
rule['extra_option']['secret'] = AESCrypto.decrypt(rule['extra_option']['secret'])
if isinstance(rule.get("extra_option"), dict):
decrypt_account(rule['extra_option'], rule['uid'])
if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('password'):
if not (current_user.username in PRIVILEGED_USERS or current_user.uid == rule['uid']):
if rule['extra_option'].get('_reference'):
rule['extra_option'].pop('password', None)
else:
rule['extra_option']['password'] = AESCrypto.decrypt(rule['extra_option']['password'])
rule['extra_option'].pop('secret', None)
rule['extra_option'].update(
AutoDiscoveryAccountCRUD().get_config_by_id(rule['extra_option']['_reference']))
if oneagent_id and rule['agent_id'] == oneagent_id:
result.append(rule)
@ -364,7 +369,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
if kwargs.get('adr_id'):
adr = AutoDiscoveryRule.get_by_id(kwargs['adr_id']) or abort(
404, ErrFormat.adr_not_found.format("id={}".format(kwargs['adr_id'])))
if adr.type == "http":
if adr.type == AutoDiscoveryType.HTTP:
kwargs.setdefault('extra_option', dict())
en_name = None
for i in DEFAULT_INNER:
@ -379,13 +384,16 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
kwargs["extra_option"]["provider"] = en_name
break
if adr.type == AutoDiscoveryType.COMPONENTS and kwargs.get('extra_option'):
for i in DEFAULT_INNER:
if i['name'] == adr.name:
kwargs['extra_option']['collect_key'] = i['option'].get('collect_key')
break
if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
kwargs = check_plugin_script(**kwargs)
if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'):
kwargs['extra_option']['secret'] = AESCrypto.encrypt(kwargs['extra_option']['secret'])
if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('password'):
kwargs['extra_option']['password'] = AESCrypto.encrypt(kwargs['extra_option']['password'])
encrypt_account(kwargs.get('extra_option'))
ci_type = CITypeCache.get(kwargs['type_id'])
unique = AttributeCache.get(ci_type.unique_id)
@ -403,7 +411,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
adr = AutoDiscoveryRule.get_by_id(existed.adr_id) or abort(
404, ErrFormat.adr_not_found.format("id={}".format(existed.adr_id)))
if adr.type == "http":
if adr.type == AutoDiscoveryType.HTTP:
kwargs.setdefault('extra_option', dict())
en_name = None
for i in DEFAULT_INNER:
@ -418,6 +426,12 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
kwargs["extra_option"]["provider"] = en_name
break
if adr.type == AutoDiscoveryType.COMPONENTS and kwargs.get('extra_option'):
for i in DEFAULT_INNER:
if i['name'] == adr.name:
kwargs['extra_option']['collect_key'] = i['option'].get('collect_key')
break
if 'attributes' in kwargs:
self.__valid_exec_target(kwargs.get('agent_id'), kwargs.get('query_expr'))
@ -441,13 +455,10 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
kwargs = check_plugin_script(**kwargs)
if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'):
kwargs['extra_option']['secret'] = AESCrypto.encrypt(kwargs['extra_option']['secret'])
if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('password'):
kwargs['extra_option']['password'] = AESCrypto.encrypt(kwargs['extra_option']['password'])
encrypt_account(kwargs.get('extra_option'))
inst = self._can_update(_id=_id, **kwargs)
if len(kwargs) == 1 and 'enabled' in kwargs: # enable or disable
if len(kwargs) == 1 and 'enabled' in kwargs: # enable or disable
pass
elif inst.agent_id != kwargs.get('agent_id') or inst.query_expr != kwargs.get('query_expr'):
for item in AutoDiscoveryRuleSyncHistory.get_by(adt_id=inst.id, to_dict=False):
@ -688,9 +699,11 @@ class AutoDiscoveryCICRUD(DBMixin):
adt = AutoDiscoveryCITypeCRUD.get_by_id(adc.adt_id) or abort(404, ErrFormat.adt_not_found)
ci_id = None
if adt.attributes:
ci_dict = {adt.attributes[k]: None if not v and isinstance(v, (list, dict)) else v
for k, v in adc.instance.items() if k in adt.attributes}
ad_key2attr = adt.attributes or {}
if ad_key2attr:
ci_dict = {ad_key2attr[k]: None if not v and isinstance(v, (list, dict)) else v
for k, v in adc.instance.items() if k in ad_key2attr}
extra_option = adt.extra_option or {}
mapping, path_mapping = AutoDiscoveryHTTPManager.get_predefined_value_mapping(
extra_option.get('provider'), extra_option.get('category'))
@ -703,37 +716,7 @@ class AutoDiscoveryCICRUD(DBMixin):
AutoDiscoveryExecHistoryCRUD().add(type_id=adt.type_id,
stdout="accept resource: {}".format(adc.unique_value))
relation_ads = AutoDiscoveryCITypeRelation.get_by(ad_type_id=adt.type_id, to_dict=False)
for r_adt in relation_ads:
ad_key = r_adt.ad_key
if not adc.instance.get(ad_key):
continue
ad_key_values = [adc.instance.get(ad_key)] if not isinstance(
adc.instance.get(ad_key), list) else adc.instance.get(ad_key)
for ad_key_value in ad_key_values:
query = "_type:{},{}:{}".format(r_adt.peer_type_id, r_adt.peer_attr_id, ad_key_value)
s = ci_search(query, use_ci_filter=False, count=1000000)
try:
response, _, _, _, _, _ = s.search()
except SearchError as e:
current_app.logger.warning(e)
return abort(400, str(e))
for relation_ci in response:
relation_ci_id = relation_ci['_id']
try:
CIRelationManager.add(ci_id, relation_ci_id,
valid=False,
source=RelationSourceEnum.AUTO_DISCOVERY)
except:
try:
CIRelationManager.add(relation_ci_id, ci_id,
valid=False,
source=RelationSourceEnum.AUTO_DISCOVERY)
except:
pass
build_relations_for_ad_accept.apply_async(args=(adc.to_dict(), ci_id, ad_key2attr), queue=CMDB_QUEUE)
adc.update(is_accept=True,
accept_by=nickname or current_user.nickname,
@ -893,3 +876,121 @@ class AutoDiscoveryCounterCRUD(DBMixin):
def _can_delete(self, **kwargs):
pass
def encrypt_account(config):
if isinstance(config, dict):
if config.get('secret'):
config['secret'] = AESCrypto.encrypt(config['secret'])
if config.get('password'):
config['password'] = AESCrypto.encrypt(config['password'])
def decrypt_account(config, uid):
if isinstance(config, dict):
if config.get('password'):
if not (current_user.username in PRIVILEGED_USERS or current_user.uid == uid):
config.pop('password', None)
else:
try:
config['password'] = AESCrypto.decrypt(config['password'])
except Exception as e:
current_app.logger.error('decrypt account failed: {}'.format(e))
if config.get('secret'):
if not (current_user.username in PRIVILEGED_USERS or current_user.uid == uid):
config.pop('secret', None)
else:
try:
config['secret'] = AESCrypto.decrypt(config['secret'])
except Exception as e:
current_app.logger.error('decrypt account failed: {}'.format(e))
class AutoDiscoveryAccountCRUD(DBMixin):
cls = AutoDiscoveryAccount
def get(self, adr_id):
res = self.cls.get_by(adr_id=adr_id, to_dict=True)
for i in res:
decrypt_account(i.get('config'), i['uid'])
return res
def get_config_by_id(self, _id):
res = self.cls.get_by_id(_id)
if not res:
return {}
config = res.to_dict().get('config') or {}
decrypt_account(config, res.uid)
return config
def _can_add(self, **kwargs):
encrypt_account(kwargs.get('config'))
kwargs['uid'] = current_user.uid
return kwargs
def upsert(self, adr_id, accounts):
existed_all = self.cls.get_by(adr_id=adr_id, to_dict=False)
account_names = {i['name'] for i in accounts}
name_changed = dict()
for account in accounts:
existed = None
if account.get('id'):
existed = self.cls.get_by_id(account.get('id'))
if existed is None:
continue
account.pop('id')
name_changed[existed.name] = account.get('name')
else:
account = self._can_add(**account)
if existed is not None:
if current_user.uid == existed.uid:
config = copy.deepcopy(existed.config) or {}
config.update(account.get('config') or {})
account['config'] = config
existed.update(**account)
else:
self.cls.create(adr_id=adr_id, **account)
for item in existed_all:
if name_changed.get(item.name, item.name) not in account_names:
if current_user.uid == item.uid:
item.soft_delete()
def _can_update(self, **kwargs):
existed = self.cls.get_by_id(kwargs['_id']) or abort(404, ErrFormat.not_found)
if isinstance(kwargs.get('config'), dict) and kwargs['config'].get('secret'):
if current_user.uid != existed.uid:
return abort(403, ErrFormat.adt_secret_no_permission)
if isinstance(kwargs.get('config'), dict) and kwargs['config'].get('password'):
if current_user.uid != existed.uid:
return abort(403, ErrFormat.adt_secret_no_permission)
return existed
def update(self, _id, **kwargs):
if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
kwargs = check_plugin_script(**kwargs)
encrypt_account(kwargs.get('config'))
inst = self._can_update(_id=_id, **kwargs)
obj = inst.update(_id=_id, filter_none=False, **kwargs)
return obj
def _can_delete(self, **kwargs):
pass

17
cmdb-api/api/lib/cmdb/auto_discovery/const.py
View File

@ -19,10 +19,21 @@ DEFAULT_INNER = [
dict(name="KVM", en="kvm", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
option={'icon': {'name': 'ops-KVM'}, "category": "private_cloud", "en": "kvm"}),
dict(name="Nginx", en="nginx", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
option={'icon': {'name': 'caise-nginx'}, "en": "nginx"}),
option={'icon': {'name': 'caise-nginx'}, "en": "nginx", "collect_key": "nginx"}),
dict(name="Apache", en="apache", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
option={'icon': {'name': 'caise-apache'}, "en": "apache", "collect_key": "apache"}),
dict(name="Tomcat", en="tomcat", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
option={'icon': {'name': 'caise-tomcat'}, "en": "tomcat", "collect_key": "tomcat"}),
dict(name="MySQL", en="mysql", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
option={'icon': {'name': 'caise-mySQL'}, "en": "mysql", "collect_key": "mysql"}),
dict(name="MSSQL", en="mssql", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
option={'icon': {'name': 'caise-SQLServer'}, "en": "mssql", "collect_key": "sqlserver"}),
dict(name="Oracle", en="oracle", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
option={'icon': {'name': 'caise-oracle'}, "en": "oracle", "collect_key": "oracle"}),
dict(name="Redis", en="redis", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
option={'icon': {'name': 'caise-redis'}, "en": "redis"}),
option={'icon': {'name': 'caise-redis'}, "en": "redis", "collect_key": "redis"}),
dict(name="交换机", type=AutoDiscoveryType.SNMP, is_inner=True, is_plugin=False,
option={'icon': {'name': 'caise-jiaohuanji'}}),
@ -294,7 +305,7 @@ CLOUD_MAP = {
"category": "其他",
"items": ["资源池", "数据中心", "文件夹"],
"map": {
"资源池": "templates/vsphere_datastore.json",
"资源池": "templates/vsphere_pool.json",
"数据中心": "templates/vsphere_datacenter.json",
"文件夹": "templates/vsphere_folder.json",
},

4
cmdb-api/api/lib/cmdb/ci.py
View File

@ -319,8 +319,8 @@ class CIManager(object):
400, ErrFormat.unique_value_not_found.format("unique_id={}".format(ci_type.unique_id)))
unique_value = None
if not (unique_key.default and unique_key.default.get('default') == AttributeDefaultValueEnum.AUTO_INC_ID and
not ci_dict.get(unique_key.name)): # primary key is not auto inc id
# primary key is not auto inc id
if not (unique_key.default and unique_key.default.get('default') == AttributeDefaultValueEnum.AUTO_INC_ID):
unique_value = ci_dict.get(unique_key.name) or ci_dict.get(unique_key.alias) or ci_dict.get(unique_key.id)
unique_value = unique_value or abort(400, ErrFormat.unique_key_required.format(unique_key.name))

11
cmdb-api/api/lib/cmdb/ci_type.py
View File

@ -1244,17 +1244,16 @@ class CITypeAttributeGroupManager(object):
if isinstance(_from, int):
from_group = CITypeAttributeGroup.get_by_id(_from)
else:
from_group = CITypeAttributeGroup.get_by(name=_from, first=True, to_dict=False)
from_group = CITypeAttributeGroup.get_by(name=_from, type_id=type_id, first=True, to_dict=False)
from_group or abort(404, ErrFormat.ci_type_attribute_group_not_found.format("id={}".format(_from)))
if isinstance(_to, int):
to_group = CITypeAttributeGroup.get_by_id(_to)
else:
to_group = CITypeAttributeGroup.get_by(name=_to, first=True, to_dict=False)
to_group = CITypeAttributeGroup.get_by(name=_to, type_id=type_id, first=True, to_dict=False)
to_group or abort(404, ErrFormat.ci_type_attribute_group_not_found.format("id={}".format(_to)))
from_order, to_order = from_group.order, to_group.order
from_group.update(order=to_order)
to_group.update(order=from_order)
@ -1541,6 +1540,9 @@ class CITypeTemplateManager(object):
if ((i.extra_option or {}).get('alias') or None) == (
(rule.get('extra_option') or {}).get('alias') or None):
existed = True
rule.pop('extra_option', None)
rule.pop('enabled', None)
rule.pop('cron', None)
AutoDiscoveryCITypeCRUD().update(i.id, **rule)
break
@ -1698,6 +1700,9 @@ class CITypeTemplateManager(object):
for r in ad_rules:
r = r.to_dict()
if r.get('extra_option') and '_reference' in r['extra_option']:
r['extra_option'].pop('_reference')
r['type_name'] = type_id2name.get(r.pop('type_id'))
if r.get('adr_id'):
adr = AutoDiscoveryRuleCRUD.get_by_id(r.pop('adr_id'))

2
cmdb-api/api/lib/cmdb/const.py
View File

@ -118,7 +118,7 @@ REDIS_PREFIX_CI = "ONE_CMDB"
REDIS_PREFIX_CI_RELATION = "CMDB_CI_RELATION"
REDIS_PREFIX_CI_RELATION2 = "CMDB_CI_RELATION2"
BUILTIN_KEYWORDS = {'id', '_id', 'ci_id', 'type', '_type', 'ci_type'}
BUILTIN_KEYWORDS = {'id', '_id', 'ci_id', 'type', '_type', 'ci_type', 'ticket_id'}
L_TYPE = None
L_CI = None

2
cmdb-api/api/lib/cmdb/resp_format.py
View File

@ -35,7 +35,7 @@ class ErrFormat(CommonErrFormat):
"Only creators and administrators are allowed to delete attributes!") # 目前只允许 属性创建人、管理员 删除属性!
# 属性字段名不能是内置字段: id, _id, ci_id, type, _type, ci_type
attribute_name_cannot_be_builtin = _l(
"Attribute field names cannot be built-in fields: id, _id, ci_id, type, _type, ci_type")
"Attribute field names cannot be built-in fields: id, _id, ci_id, type, _type, ci_type, ticket_id")
attribute_choice_other_invalid = _l(
"Predefined value: Other model request parameters are illegal!") # 预定义值: 其他模型请求参数不合法!

23
cmdb-api/api/lib/mixin.py
View File

@ -1,21 +1,19 @@
# -*- coding:utf-8 -*-
from flask import abort
from sqlalchemy import func
from api.extensions import db
from api.lib.utils import get_page
from api.lib.utils import get_page_size
__author__ = 'pycook'
class DBMixin(object):
cls = None
@classmethod
def search(cls, page, page_size, fl=None, only_query=False, reverse=False, count_query=False, **kwargs):
def search(cls, page, page_size, fl=None, only_query=False, reverse=False, count_query=False,
last_size=None, **kwargs):
page = get_page(page)
page_size = get_page_size(page_size)
if fl is None:
@ -47,14 +45,15 @@ class DBMixin(object):
return _query, query
numfound = query.count()
return numfound, [i.to_dict() if fl is None else getattr(i, '_asdict')()
for i in query.offset((page - 1) * page_size).limit(page_size)]
def _must_be_required(self, _id):
existed = self.cls.get_by_id(_id)
existed or abort(404, "Factor [{}] does not exist".format(_id))
return existed
if not last_size:
return numfound, [i.to_dict() if fl is None else getattr(i, '_asdict')()
for i in query.offset((page - 1) * page_size).limit(page_size)]
else:
offset = numfound - last_size
if offset < 0:
offset = 0
return numfound, [i.to_dict() if fl is None else getattr(i, '_asdict')()
for i in query.offset(offset).limit(last_size)]
def _can_add(self, **kwargs):
raise NotImplementedError

9
cmdb-api/api/models/cmdb.py
View File

@ -636,6 +636,15 @@ class AutoDiscoveryCounter(Model2):
last_week_count = db.Column(db.Integer, default=0)
class AutoDiscoveryAccount(Model):
__tablename__ = "c_ad_accounts"
uid = db.Column(db.Integer, index=True)
name = db.Column(db.String(64))
adr_id = db.Column(db.Integer, db.ForeignKey('c_ad_rules.id'))
config = db.Column(db.JSON)
class CIFilterPerms(Model):
__tablename__ = "c_ci_filter_perms"

82
cmdb-api/api/tasks/cmdb.py
View File

@ -1,9 +1,8 @@
# -*- coding:utf-8 -*-
import json
import datetime
import json
import redis_lock
from flask import current_app
from flask_login import login_user
@ -13,21 +12,24 @@ from api.extensions import celery
from api.extensions import db
from api.extensions import es
from api.extensions import rd
from api.lib.cmdb.cache import AttributeCache
from api.lib.cmdb.cache import CITypeAttributesCache
from api.lib.cmdb.const import CMDB_QUEUE
from api.lib.cmdb.const import REDIS_PREFIX_CI
from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION2
from api.lib.cmdb.const import RelationSourceEnum
from api.lib.cmdb.perms import CIFilterPermsCRUD
from api.lib.decorator import flush_db
from api.lib.decorator import reconnect_db
from api.lib.perm.acl.cache import UserCache
from api.lib.utils import handle_arg_list
from api.models.cmdb import AutoDiscoveryCI
from api.models.cmdb import AutoDiscoveryCIType
from api.models.cmdb import AutoDiscoveryCITypeRelation
from api.models.cmdb import CI
from api.models.cmdb import CIRelation
from api.models.cmdb import CITypeAttribute
from api.models.cmdb import AutoDiscoveryCI
from api.models.cmdb import AutoDiscoveryCIType
@celery.task(name="cmdb.ci_cache", queue=CMDB_QUEUE)
@ -277,3 +279,75 @@ def write_ad_rule_sync_history(rules, oneagent_id, oneagent_name, sync_at):
except Exception as e:
current_app.logger.error("write auto discovery rule sync history failed: {}".format(e))
db.session.rollback()
@celery.task(name="cmdb.build_relations_for_ad_accept", queue=CMDB_QUEUE)
@reconnect_db
def build_relations_for_ad_accept(adc, ci_id, ad_key2attr):
from api.lib.cmdb.ci import CIRelationManager
from api.lib.cmdb.search import SearchError
from api.lib.cmdb.search.ci import search as ci_search
current_app.test_request_context().push()
login_user(UserCache.get('worker'))
relation_ads = AutoDiscoveryCITypeRelation.get_by(ad_type_id=adc['type_id'], to_dict=False)
for r_adt in relation_ads:
ad_key = r_adt.ad_key
if not adc['instance'].get(ad_key):
continue
ad_key_values = [adc['instance'].get(ad_key)] if not isinstance(
adc['instance'].get(ad_key), list) else adc['instance'].get(ad_key)
for ad_key_value in ad_key_values:
query = "_type:{},{}:{}".format(r_adt.peer_type_id, r_adt.peer_attr_id, ad_key_value)
s = ci_search(query, use_ci_filter=False, count=1000000)
try:
response, _, _, _, _, _ = s.search()
except SearchError as e:
current_app.logger.error("build_relations_for_ad_accept failed: {}".format(e))
return
for relation_ci in response:
relation_ci_id = relation_ci['_id']
try:
CIRelationManager.add(ci_id, relation_ci_id,
valid=False,
source=RelationSourceEnum.AUTO_DISCOVERY)
except:
try:
CIRelationManager.add(relation_ci_id, ci_id,
valid=False,
source=RelationSourceEnum.AUTO_DISCOVERY)
except:
pass
# build relations in reverse
relation_ads = AutoDiscoveryCITypeRelation.get_by(peer_type_id=adc['type_id'], to_dict=False)
attr2ad_key = {v: k for k, v in ad_key2attr.items()}
for r_adt in relation_ads:
attr = AttributeCache.get(r_adt.peer_attr_id)
ad_key = attr2ad_key.get(attr and attr.name)
if not ad_key:
continue
ad_value = adc['instance'].get(ad_key)
peer_ad_key = r_adt.ad_key
peer_instances = AutoDiscoveryCI.get_by(type_id=r_adt.ad_type_id, to_dict=False)
for peer_instance in peer_instances:
peer_ad_values = peer_instance.instance.get(peer_ad_key)
peer_ad_values = [peer_ad_values] if not isinstance(peer_ad_values, list) else peer_ad_values
if ad_value in peer_ad_values and peer_instance.ci_id:
try:
CIRelationManager.add(peer_instance.ci_id, ci_id,
valid=False,
source=RelationSourceEnum.AUTO_DISCOVERY)
except:
try:
CIRelationManager.add(ci_id, peer_instance.ci_id,
valid=False,
source=RelationSourceEnum.AUTO_DISCOVERY)
except:
pass

52
cmdb-api/api/views/cmdb/auto_discovery.py
View File

@ -8,6 +8,7 @@ from flask import request
from flask_login import current_user
from io import BytesIO
from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryAccountCRUD
from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCICRUD
from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeCRUD
from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeRelationCRUD
@ -20,6 +21,7 @@ from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryRuleSyncHist
from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoverySNMPManager
from api.lib.cmdb.auto_discovery.const import DEFAULT_INNER
from api.lib.cmdb.auto_discovery.const import PRIVILEGED_USERS
from api.lib.cmdb.cache import AttributeCache
from api.lib.cmdb.const import PermEnum
from api.lib.cmdb.const import ResourceTypeEnum
from api.lib.cmdb.resp_format import ErrFormat
@ -272,14 +274,16 @@ class AutoDiscoveryRuleSyncView(APIView):
oneagent_id = request.values.get('oneagent_id')
last_update_at = request.values.get('last_update_at')
query = "oneagent_id:{}".format(oneagent_id)
s = ci_search(query)
try:
response, _, _, _, _, _ = s.search()
except SearchError as e:
import traceback
current_app.logger.error(traceback.format_exc())
return abort(400, str(e))
response = []
if AttributeCache.get('oneagent_id'):
query = "oneagent_id:{}".format(oneagent_id)
s = ci_search(query)
try:
response, _, _, _, _, _ = s.search()
except SearchError as e:
import traceback
current_app.logger.error(traceback.format_exc())
return abort(400, str(e))
for res in response:
if res.get('{}_name'.format(res['ci_type'])) == oneagent_name or oneagent_name == res.get('oneagent_name'):
@ -328,8 +332,12 @@ class AutoDiscoveryExecHistoryView(APIView):
def get(self):
page = get_page(request.values.pop('page', 1))
page_size = get_page_size(request.values.pop('page_size', None))
last_size = request.values.pop('last_size', None)
if last_size and last_size.isdigit():
last_size = int(last_size)
numfound, res = AutoDiscoveryExecHistoryCRUD.search(page=page,
page_size=page_size,
last_size=last_size,
**request.values)
return self.jsonify(page=page,
@ -355,3 +363,31 @@ class AutoDiscoveryCounterView(APIView):
type_id = request.values.get('type_id')
return self.jsonify(AutoDiscoveryCounterCRUD().get(type_id))
class AutoDiscoveryAccountView(APIView):
url_prefix = ("/adr/accounts", "/adr/accounts/<int:account_id>")
@args_required('adr_id')
def get(self):
adr_id = request.values.get('adr_id')
return self.jsonify(AutoDiscoveryAccountCRUD().get(adr_id))
@args_required('adr_id')
@args_required('accounts', value_required=False)
def post(self):
AutoDiscoveryAccountCRUD().upsert(**request.values)
return self.jsonify(code=200)
@args_required('config')
def put(self, account_id):
res = AutoDiscoveryAccountCRUD().update(account_id, **request.values)
return self.jsonify(res.to_dict())
def delete(self, account_id):
AutoDiscoveryAccountCRUD().delete(account_id)
return self.jsonify(account_id=account_id)