7x31
/
cmdb
mirror of https://github.com/veops/cmdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

perf(api): CIType templates download (#567)

This commit is contained in:
pycook 2024-06-27 20:54:38 +08:00 committed by GitHub
parent 29995b660a
commit fa32758462
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 540 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
cmdb-api/api/commands/click_cmdb.py
View File

@ -540,6 +540,19 @@ def cmdb_patch(version):
peer_attr_id=peer_attr_id,
commit=False)
if hasattr(adt, 'interval') and adt.interval and not adt.cron:
adt.cron = "*/{} * * * *".format(adt.interval // 60)
adt.cron = "*/{} * * * *".format(adt.interval // 60 or 1)
db.session.commit()
if version >= "2.4.7":
from api.lib.cmdb.auto_discovery.const import DEFAULT_INNER
from api.models.cmdb import AutoDiscoveryRule
for i in DEFAULT_INNER:
existed = AutoDiscoveryRule.get_by(name=i['name'], first=True, to_dict=False)
if existed is not None:
if "en" in i['option'] and 'en' not in (existed.option or {}):
option = copy.deepcopy(existed.option)
option['en'] = i['option']['en']
existed.update(option=option, commit=False)
db.session.commit()

68
cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py
View File

@ -3,7 +3,6 @@ import copy
import datetime
import json
import os
from flask import abort
from flask import current_app
from flask_login import current_user
@ -11,7 +10,8 @@ from sqlalchemy import func
from api.extensions import db
from api.lib.cmdb.auto_discovery.const import ClOUD_MAP
from api.lib.cmdb.auto_discovery.const import DEFAULT_HTTP
from api.lib.cmdb.auto_discovery.const import DEFAULT_INNER
from api.lib.cmdb.auto_discovery.const import PRIVILEGED_USERS
from api.lib.cmdb.cache import AttributeCache
from api.lib.cmdb.cache import CITypeAttributeCache
from api.lib.cmdb.cache import CITypeCache
@ -22,6 +22,7 @@ from api.lib.cmdb.const import AutoDiscoveryType
from api.lib.cmdb.const import CMDB_QUEUE
from api.lib.cmdb.const import PermEnum
from api.lib.cmdb.const import ResourceTypeEnum
from api.lib.cmdb.custom_dashboard import SystemConfigManager
from api.lib.cmdb.resp_format import ErrFormat
from api.lib.cmdb.search import SearchError
from api.lib.cmdb.search.ci import search as ci_search
@ -109,9 +110,9 @@ class AutoDiscoveryRuleCRUD(DBMixin):
else:
self.cls.create(**rule)
def _can_add(self, **kwargs):
def _can_add(self, valid=True, **kwargs):
self.cls.get_by(name=kwargs['name']) and abort(400, ErrFormat.adr_duplicate.format(kwargs['name']))
if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
if kwargs.get('is_plugin') and kwargs.get('plugin_script') and valid:
kwargs = check_plugin_script(**kwargs)
acl = ACLManager(app_cli.app_name)
has_perm = True
@ -132,7 +133,7 @@ class AutoDiscoveryRuleCRUD(DBMixin):
return kwargs
def _can_update(self, **kwargs):
def _can_update(self, valid=True, **kwargs):
existed = self.cls.get_by_id(kwargs['_id']) or abort(
404, ErrFormat.adr_not_found.format("id={}".format(kwargs['_id'])))
@ -144,7 +145,7 @@ class AutoDiscoveryRuleCRUD(DBMixin):
if other and other.id != existed.id:
return abort(400, ErrFormat.adr_duplicate.format(kwargs['name']))
if existed.is_plugin:
if existed.is_plugin and valid:
acl = ACLManager(app_cli.app_name)
has_perm = True
try:
@ -202,8 +203,9 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
cls = AutoDiscoveryCIType
@classmethod
def get_all(cls):
return cls.cls.get_by(to_dict=False)
def get_all(cls, type_ids=None):
res = cls.cls.get_by(to_dict=False)
return [i for i in res if type_ids is None or i.type_id in type_ids]
@classmethod
def get_by_id(cls, _id):
@ -222,7 +224,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
if not adr:
continue
if adr.type == "http":
for i in DEFAULT_HTTP:
for i in DEFAULT_INNER:
if adr.name == i['name']:
attrs = AutoDiscoveryHTTPManager.get_attributes(
i['en'], (adt.extra_option or {}).get('category')) or []
@ -243,11 +245,17 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
for rule in rules:
if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('secret'):
if not (current_user.username == "cmdb_agent" or current_user.uid == rule['uid']):
if not (current_user.username in PRIVILEGED_USERS or current_user.uid == rule['uid']):
rule['extra_option'].pop('secret', None)
else:
rule['extra_option']['secret'] = AESCrypto.decrypt(rule['extra_option']['secret'])
if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('password'):
if not (current_user.username in PRIVILEGED_USERS or current_user.uid == rule['uid']):
rule['extra_option'].pop('password', None)
else:
rule['extra_option']['password'] = AESCrypto.decrypt(rule['extra_option']['password'])
if oneagent_id and rule['agent_id'] == oneagent_id:
result.append(rule)
elif rule['query_expr']:
@ -271,17 +279,19 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
result.append(rule)
ad_rules_updated_at = (SystemConfigManager.get('ad_rules_updated_at') or {}).get('option', {}).get('v') or ""
new_last_update_at = ""
for i in result:
i['adr'] = AutoDiscoveryRule.get_by_id(i['adr_id']).to_dict()
i['adr'].pop("attributes", None)
__last_update_at = max([i['updated_at'] or "", i['created_at'] or "",
i['adr']['created_at'] or "", i['adr']['updated_at'] or ""])
i['adr']['created_at'] or "", i['adr']['updated_at'] or "", ad_rules_updated_at])
if new_last_update_at < __last_update_at:
new_last_update_at = __last_update_at
write_ad_rule_sync_history.apply_async(args=(result, oneagent_id, oneagent_name, datetime.datetime.now()),
queue=CMDB_QUEUE)
if not last_update_at or new_last_update_at > last_update_at:
return result, new_last_update_at
else:
@ -346,7 +356,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
if adr.type == "http":
kwargs.setdefault('extra_option', dict)
en_name = None
for i in DEFAULT_HTTP:
for i in DEFAULT_INNER:
if i['name'] == adr.name:
en_name = i['en']
break
@ -362,10 +372,13 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'):
kwargs['extra_option']['secret'] = AESCrypto.encrypt(kwargs['extra_option']['secret'])
if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('password'):
kwargs['extra_option']['password'] = AESCrypto.encrypt(kwargs['extra_option']['password'])
ci_type = CITypeCache.get(kwargs['type_id'])
unique = AttributeCache.get(ci_type.unique_id)
if unique and unique.name not in (kwargs.get('attributes') or {}).values():
current_app.logger.warning((unique.name, kwargs.get('attributes'), ci_type.alias))
return abort(400, ErrFormat.ad_not_unique_key.format(unique.name))
kwargs['uid'] = current_user.uid
@ -381,7 +394,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
if adr.type == "http":
kwargs.setdefault('extra_option', dict)
en_name = None
for i in DEFAULT_HTTP:
for i in DEFAULT_INNER:
if i['name'] == adr.name:
en_name = i['en']
break
@ -398,11 +411,15 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
ci_type = CITypeCache.get(existed.type_id)
unique = AttributeCache.get(ci_type.unique_id)
if unique and unique.name not in (kwargs.get('attributes') or {}).values():
current_app.logger.warning((unique.name, kwargs.get('attributes'), ci_type.alias))
return abort(400, ErrFormat.ad_not_unique_key.format(unique.name))
if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'):
if current_user.uid != existed.uid:
return abort(403, ErrFormat.adt_secret_no_permission)
if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('password'):
if current_user.uid != existed.uid:
return abort(403, ErrFormat.adt_secret_no_permission)
return existed
@ -413,6 +430,8 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'):
kwargs['extra_option']['secret'] = AESCrypto.encrypt(kwargs['extra_option']['secret'])
if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('password'):
kwargs['extra_option']['password'] = AESCrypto.encrypt(kwargs['extra_option']['password'])
inst = self._can_update(_id=_id, **kwargs)
if inst.agent_id != kwargs.get('agent_id') or inst.query_expr != kwargs.get('query_expr'):
@ -420,6 +439,9 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
item.delete(commit=False)
db.session.commit()
SystemConfigManager.create_or_update("ad_rules_updated_at",
dict(v=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
obj = inst.update(_id=_id, filter_none=False, **kwargs)
return obj
@ -453,6 +475,10 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
class AutoDiscoveryCITypeRelationCRUD(DBMixin):
cls = AutoDiscoveryCITypeRelation
@classmethod
def get_all(cls, type_ids=None):
res = cls.cls.get_by(to_dict=False)
return [i for i in res if type_ids is None or i.ad_type_id in type_ids]
@classmethod
def get_by_type_id(cls, type_id, to_dict=False):
return cls.cls.get_by(ad_type_id=type_id, to_dict=to_dict)
@ -692,12 +718,13 @@ class AutoDiscoveryHTTPManager(object):
categories = (ClOUD_MAP.get(name) or {}) or []
for item in copy.deepcopy(categories):
item.pop('map', None)
item.pop('collect_key_map', None)
return categories
def get_resources(self, name):
en_name = None
for i in DEFAULT_HTTP:
for i in DEFAULT_INNER:
if i['name'] == name:
en_name = i['en']
break
@ -733,6 +760,17 @@ class AutoDiscoverySNMPManager(object):
return []
class AutoDiscoveryComponentsManager(object):
@staticmethod
def get_attributes(name):
if os.path.exists(os.path.join(PWD, "templates/{}.json".format(name))):
with open(os.path.join(PWD, "templates/{}.json".format(name))) as f:
return json.loads(f.read())
return []
class AutoDiscoveryRuleSyncHistoryCRUD(DBMixin):
cls = AutoDiscoveryRuleSyncHistory

357
cmdb-api/api/lib/cmdb/auto_discovery/const.py
View File

@ -2,16 +2,28 @@
from api.lib.cmdb.const import AutoDiscoveryType
DEFAULT_HTTP = [
PRIVILEGED_USERS = ("cmdb_agent", "worker", "admin")
DEFAULT_INNER = [
dict(name="阿里云", en="aliyun", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
option={'icon': {'name': 'caise-aliyun'}}),
option={'icon': {'name': 'caise-aliyun'}, "en": "aliyun"}),
dict(name="腾讯云", en="tencentcloud", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
option={'icon': {'name': 'caise-tengxunyun'}}),
option={'icon': {'name': 'caise-tengxunyun'}, "en": "tencentcloud"}),
dict(name="华为云", en="huaweicloud", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
option={'icon': {'name': 'caise-huaweiyun'}}),
option={'icon': {'name': 'caise-huaweiyun'}, "en": "huaweicloud"}),
dict(name="AWS", en="aws", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
option={'icon': {'name': 'caise-aws'}}),
dict(name="VCenter", en="vcenter", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
option={'icon': {'name': 'cmdb-vcenter'}, "category": "private_cloud", "en": "vcenter"}),
dict(name="KVM", en="kvm", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
option={'icon': {'name': 'ops-KVM'}, "category": "private_cloud", "en": "kvm"}),
dict(name="Nginx", en="nginx", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
option={'icon': {'name': 'caise-nginx'}, "en": "nginx"}),
dict(name="Redis", en="redis", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
option={'icon': {'name': 'caise-redis'}, "en": "redis"}),
dict(name="交换机", type=AutoDiscoveryType.SNMP, is_inner=True, is_plugin=False,
option={'icon': {'name': 'caise-jiaohuanji'}}),
dict(name="路由器", type=AutoDiscoveryType.SNMP, is_inner=True, is_plugin=False,
@ -23,47 +35,306 @@ DEFAULT_HTTP = [
]
ClOUD_MAP = {
"aliyun": [{
"category": "计算",
"items": ["云服务器 ECS"],
"map": {
"云服务器 ECS": "templates/aliyun_ecs.json",
"aliyun": [
{
"category": "计算",
"items": ["云服务器 ECS", "云服务器 Disk"],
"map": {
"云服务器 ECS": "templates/aliyun_ecs.json",
"云服务器 Disk": "templates/aliyun_ecs_disk2.json",
},
"collect_key_map": {
"云服务器 ECS": "ali.ecs",
"云服务器 Disk": "ali.ecs_disk",
},
},
"collect_key_map": {
"云服务器 ECS": "ali.ecs",
}
}],
"tencentcloud": [{
"category": "计算",
"items": ["云服务器 CVM"],
"map": {
"云服务器 CVM": "templates/tencent_cvm.json",
{
"category": "网络与CDN",
"items": [
"内容分发CDN",
"负载均衡SLB",
"专有网络VPC",
"交换机Switch",
],
"map": {
"内容分发CDN": "templates/aliyun_cdn.json",
"负载均衡SLB": "templates/aliyun_slb.json",
"专有网络VPC": "templates/aliyun_vpc.json",
"交换机Switch": "templates/aliyun_switch.json",
},
"collect_key_map": {
"内容分发CDN": "ali.cdn",
"负载均衡SLB": "ali.slb",
"专有网络VPC": "ali.vpc",
"交换机Switch": "ali.switch",
},
},
"collect_key_map": {
"云服务器 CVM": "tencent.cvm",
}
}],
"huaweicloud": [{
"category": "计算",
"items": ["云服务器 ECS"],
"map": {
"云服务器 ECS": "templates/huaweicloud_ecs.json",
{
"category": "存储",
"items": ["块存储EBS", "对象存储OSS"],
"map": {
"块存储EBS": "templates/aliyun_ebs.json",
"对象存储OSS": "templates/aliyun_oss.json",
},
"collect_key_map": {
"块存储EBS": "ali.ebs",
"对象存储OSS": "ali.oss",
},
},
"collect_key_map": {
"云服务器 ECS": "huawei.ecs",
}
}],
"aws": [{
"category": "计算",
"items": ["云服务器 EC2"],
"map": {
"云服务器 EC2": "templates/aws_ec2.json",
{
"category": "数据库",
"items": ["云数据库RDS MySQL", "云数据库RDS PostgreSQL", "云数据库 Redis"],
"map": {
"云数据库RDS MySQL": "templates/aliyun_rds_mysql.json",
"云数据库RDS PostgreSQL": "templates/aliyun_rds_postgre.json",
"云数据库 Redis": "templates/aliyun_redis.json",
},
"collect_key_map": {
"云数据库RDS MySQL": "ali.rds_mysql",
"云数据库RDS PostgreSQL": "ali.rds_postgre",
"云数据库 Redis": "ali.redis",
},
},
"collect_key_map": {
"云服务器 EC2": "aws.ec2",
}
}],
],
"tencentcloud": [
{
"category": "计算",
"items": ["云服务器 CVM"],
"map": {
"云服务器 CVM": "templates/tencent_cvm.json",
},
"collect_key_map": {
"云服务器 CVM": "tencent.cvm",
},
},
{
"category": "CDN与边缘",
"items": ["内容分发CDN"],
"map": {
"内容分发CDN": "templates/tencent_cdn.json",
},
"collect_key_map": {
"内容分发CDN": "tencent.cdn",
},
},
{
"category": "网络",
"items": ["负载均衡CLB", "私有网络VPC", "子网"],
"map": {
"负载均衡CLB": "templates/tencent_clb.json",
"私有网络VPC": "templates/tencent_vpc.json",
"子网": "templates/tencent_subnet.json",
},
"collect_key_map": {
"负载均衡CLB": "tencent.clb",
"私有网络VPC": "tencent.vpc",
"子网": "tencent.subnet",
},
},
{
"category": "存储",
"items": ["云硬盘CBS", "对象存储COS"],
"map": {
"云硬盘CBS": "templates/tencent_cbs.json",
"对象存储OSS": "templates/tencent_cos.json",
},
"collect_key_map": {
"云硬盘CBS": "tencent.cbs",
"对象存储OSS": "tencent.cos",
},
},
{
"category": "数据库",
"items": ["云数据库 MySQL", "云数据库 PostgreSQL", "云数据库 Redis"],
"map": {
"云数据库 MySQL": "templates/tencent_rdb.json",
"云数据库 PostgreSQL": "templates/tencent_postgres.json",
"云数据库 Redis": "templates/tencent_redis.json",
},
"collect_key_map": {
"云数据库 MySQL": "tencent.rdb",
"云数据库 PostgreSQL": "tencent.rds_postgres",
"云数据库 Redis": "tencent.redis",
},
},
],
"huaweicloud": [
{
"category": "计算",
"items": ["云服务器 ECS"],
"map": {
"云服务器 ECS": "templates/huaweicloud_ecs.json",
},
"collect_key_map": {
"云服务器 ECS": "huawei.ecs",
},
},
{
"category": "CDN与智能边缘",
"items": ["内容分发网络CDN"],
"map": {
"内容分发网络CDN": "templates/huawei_cdn.json",
},
"collect_key_map": {
"内容分发网络CDN": "huawei.cdn",
},
},
{
"category": "网络",
"items": ["弹性负载均衡ELB", "虚拟私有云VPC", "子网"],
"map": {
"弹性负载均衡ELB": "templates/huawei_elb.json",
"虚拟私有云VPC": "templates/huawei_vpc.json",
"子网": "templates/huawei_subnet.json",
},
"collect_key_map": {
"弹性负载均衡ELB": "huawei.elb",
"虚拟私有云VPC": "huawei.vpc",
"子网": "huawei.subnet",
},
},
{
"category": "存储",
"items": ["云硬盘EVS", "对象存储OBS"],
"map": {
"云硬盘EVS": "templates/huawei_evs.json",
"对象存储OBS": "templates/huawei_obs.json",
},
"collect_key_map": {
"云硬盘EVS": "huawei.evs",
"对象存储OBS": "huawei.obs",
},
},
{
"category": "数据库",
"items": ["云数据库RDS MySQL", "云数据库RDS PostgreSQL"],
"map": {
"云数据库RDS MySQL": "templates/huawei_rds_mysql.json",
"云数据库RDSPostgreSQL": "templates/huaweirds_postgre.json",
},
"collect_key_map": {
"云数据库RDS MySQL": "huawei.rds_mysql",
"云数据库RDS PostgreSQL": "huawei.rds_postgre",
},
},
{
"category": "应用中间件",
"items": ["分布式缓存Redis"],
"map": {
"分布式缓存Redis": "templates/huawei_dcs.json",
},
"collect_key_map": {
"分布式缓存Redis": "huawei.dcs",
},
},
],
"aws": [
{
"category": "计算",
"items": ["云服务器 EC2"],
"map": {
"云服务器 EC2": "templates/aws_ec2.json",
},
"collect_key_map": {
"云服务器 EC2": "aws.ec2",
},
},
{"category": "网络与CDN", "items": [], "map": {}, "collect_key_map": {}},
],
"vcenter": [
{
"category": "计算",
"items": [
"主机",
"虚拟机",
"主机集群"
],
"map": {
"主机": "templates/vsphere_host.json",
"虚拟机": "templates/vsphere_vm.json",
"主机集群": "templates/vsphere_cluster.json",
},
"collect_key_map": {
"主机": "vsphere.host",
"虚拟机": "vsphere.vm",
"主机集群": "vsphere.cluster",
},
},
{
"category": "网络",
"items": [
"网络",
"标准交换机",
"分布式交换机",
],
"map": {
"网络": "templates/vsphere_network.json",
"标准交换机": "templates/vsphere_standard_switch.json",
"分布式交换机": "templates/vsphere_distributed_switch.json",
},
"collect_key_map": {
"网络": "vsphere.network",
"标准交换机": "vsphere.standard_switch",
"分布式交换机": "vsphere.distributed_switch",
},
},
{
"category": "存储",
"items": ["数据存储", "数据存储集群"],
"map": {
"数据存储": "templates/vsphere_datastore.json",
"数据存储集群": "templates/vsphere.storage_pod.json",
},
"collect_key_map": {
"数据存储": "vsphere.datastore",
"数据存储集群": "vsphere.storage_pod",
},
},
{
"category": "其他",
"items": ["资源池", "数据中心", "文件夹"],
"map": {
"资源池": "templates/vsphere_datastore.json",
"数据中心": "templates/vsphere_datacenter.json",
"文件夹": "templates/vsphere_folder.json",
},
"collect_key_map": {
"资源池": "vsphere.pool",
"数据中心": "vsphere.datacenter",
"文件夹": "vsphere.folder",
},
},
],
"kvm": [
{
"category": "计算",
"items": ["虚拟机"],
"map": {
"虚拟机": "templates/kvm_vm.json",
},
"collect_key_map": {
"虚拟机": "kvm.vm",
},
},
{
"category": "存储",
"items": ["存储"],
"map": {
"存储": "templates/kvm_storage.json",
},
"collect_key_map": {
"存储": "kvm.storage",
},
},
{
"category": "network",
"items": ["网络"],
"map": {
"网络": "templates/kvm_network.json",
},
"collect_key_map": {
"网络": "kvm.network",
},
},
],
}

217
cmdb-api/api/lib/cmdb/ci_type.py
View File

@ -1,7 +1,6 @@
# -*- coding:utf-8 -*-
import copy
import toposort
from flask import abort
from flask import current_app
@ -84,7 +83,7 @@ class CITypeManager(object):
self.cls.id, self.cls.icon, self.cls.name).filter(self.cls.deleted.is_(False))}
@staticmethod
def get_ci_types(type_name=None, like=True):
def get_ci_types(type_name=None, like=True, type_ids=None):
resources = None
if current_app.config.get('USE_ACL') and not is_app_admin('cmdb'):
resources = set([i.get('name') for i in ACLManager().get_resources(ResourceTypeEnum.CI_TYPE)])
@ -93,6 +92,9 @@ class CITypeManager(object):
CIType.get_by_like(name=type_name) if like else CIType.get_by(name=type_name))
res = list()
for type_dict in ci_types:
if type_ids is not None and type_dict['id'] not in type_ids:
continue
attr = AttributeCache.get(type_dict["unique_id"])
type_dict["unique_key"] = attr and attr.name
if type_dict.get('show_id'):
@ -292,6 +294,12 @@ class CITypeManager(object):
class CITypeInheritanceManager(object):
cls = CITypeInheritance
@classmethod
def get_all(cls, type_ids=None):
res = cls.cls.get_by(to_dict=True)
return [i for i in res if type_ids is None or (i['parent_id'] in type_ids and i['child_id'] in type_ids)]
@classmethod
def get_parents(cls, type_id):
return [i.parent_id for i in cls.cls.get_by(child_id=type_id, to_dict=False)]
@ -387,7 +395,7 @@ class CITypeGroupManager(object):
cls = CITypeGroup
@staticmethod
def get(need_other=None, config_required=True):
def get(need_other=None, config_required=True, type_ids=None, ci_types=None):
resources = None
if current_app.config.get('USE_ACL'):
resources = ACLManager('cmdb').get_resources(ResourceTypeEnum.CI)
@ -401,6 +409,8 @@ class CITypeGroupManager(object):
for group in groups:
for t in sorted(CITypeGroupItem.get_by(group_id=group['id']), key=lambda x: x['order'] or 0):
ci_type = CITypeCache.get(t['type_id']).to_dict()
if type_ids is not None and ci_type['id'] not in type_ids:
continue
if resources is None or (ci_type and ci_type['name'] in resources):
ci_type['permissions'] = resources[ci_type['name']] if resources is not None else None
ci_type['inherited'] = True if CITypeInheritanceManager.get_parents(ci_type['id']) else False
@ -408,7 +418,7 @@ class CITypeGroupManager(object):
group_types.add(t["type_id"])
if need_other:
ci_types = CITypeManager.get_ci_types()
ci_types = CITypeManager.get_ci_types(type_ids=type_ids) if ci_types is None else ci_types
other_types = dict(ci_types=[])
for ci_type in ci_types:
if ci_type["id"] not in group_types and (resources is None or ci_type['name'] in resources):
@ -529,6 +539,8 @@ class CITypeAttributeManager(object):
attrs = CITypeAttributesCache.get(_type_id)
for attr in sorted(attrs, key=lambda x: (x.order, x.id)):
attr_dict = AttributeManager().get_attribute(attr.attr_id, choice_web_hook_parse, choice_other_parse)
if not attr_dict:
continue
attr_dict["is_required"] = attr.is_required
attr_dict["order"] = attr.order
attr_dict["default_show"] = attr.default_show
@ -537,7 +549,6 @@ class CITypeAttributeManager(object):
if not has_config_perm:
attr_dict.pop('choice_web_hook', None)
attr_dict.pop('choice_other', None)
if attr_dict['id'] not in id2pos:
id2pos[attr_dict['id']] = len(result)
result.append(attr_dict)
@ -602,7 +613,6 @@ class CITypeAttributeManager(object):
if existed is not None:
continue
current_app.logger.debug(attr_id)
CITypeAttribute.create(type_id=type_id, attr_id=attr_id, **kwargs)
attr = AttributeCache.get(attr_id)
@ -769,11 +779,15 @@ class CITypeRelationManager(object):
"""
@staticmethod
def get():
def get(type_ids=None):
res = CITypeRelation.get_by(to_dict=False)
type2attributes = dict()
result = []
for idx, item in enumerate(res):
_item = item.to_dict()
if type_ids is not None and _item['parent_id'] not in type_ids and _item['child_id'] not in type_ids:
continue
res[idx] = _item
res[idx]['parent'] = item.parent.to_dict()
if item.parent_id not in type2attributes:
@ -785,7 +799,9 @@ class CITypeRelationManager(object):
CITypeAttributeManager.get_all_attributes(item.child_id)]
res[idx]['relation_type'] = item.relation_type.to_dict()
return res, type2attributes
result.append(res[idx])
return result, type2attributes
@staticmethod
def get_child_type_ids(type_id, level):
@ -1034,7 +1050,8 @@ class CITypeAttributeGroupManager(object):
parent_ids = CITypeInheritanceManager.base(type_id)
groups = []
id2type = {i: CITypeCache.get(i).alias for i in parent_ids}
id2type = {i: CITypeCache.get(i) for i in parent_ids}
id2type = {k: v.alias for k, v in id2type.items() if v}
for _type_id in parent_ids + [type_id]:
_groups = CITypeAttributeGroup.get_by(type_id=_type_id)
_groups = sorted(_groups, key=lambda x: x["order"] or 0)
@ -1308,13 +1325,14 @@ class CITypeTemplateManager(object):
attributes = [attr for type_id in type2attributes for attr in type2attributes[type_id]]
attrs = []
for i in copy.deepcopy(attributes):
if i.pop('inherited', None):
continue
i.pop('default_show', None)
i.pop('is_required', None)
i.pop('order', None)
i.pop('choice_web_hook', None)
i.pop('choice_other', None)
i.pop('order', None)
i.pop('inherited', None)
i.pop('inherited_from', None)
choice_value = i.pop('choice_value', None)
if not choice_value:
@ -1334,6 +1352,7 @@ class CITypeTemplateManager(object):
for i in ci_types:
i.pop("unique_key", None)
i.pop("show_name", None)
i.pop("parent_ids", None)
i['unique_id'] = attr_id_map.get(i['unique_id'], i['unique_id'])
if i.get('show_id'):
i['show_id'] = attr_id_map.get(i['show_id'], i['show_id'])
@ -1371,7 +1390,7 @@ class CITypeTemplateManager(object):
return self.__import(RelationType, relation_types)
@staticmethod
def _import_ci_type_relations(ci_type_relations, type_id_map, relation_type_id_map):
def _import_ci_type_relations(ci_type_relations, type_id_map, relation_type_id_map, attr_id_map):
for i in ci_type_relations:
i.pop('parent', None)
i.pop('child', None)
@ -1381,15 +1400,32 @@ class CITypeTemplateManager(object):
i['child_id'] = type_id_map.get(i['child_id'], i['child_id'])
i['relation_type_id'] = relation_type_id_map.get(i['relation_type_id'], i['relation_type_id'])
i['parent_attr_ids'] = [attr_id_map.get(attr_id, attr_id) for attr_id in i.get('parent_attr_ids') or []]
i['child_attr_ids'] = [attr_id_map.get(attr_id, attr_id) for attr_id in i.get('child_attr_ids') or []]
try:
CITypeRelationManager.add(i.get('parent_id'),
i.get('child_id'),
i.get('relation_type_id'),
i.get('constraint'),
parent_attr_ids=i.get('parent_attr_ids', []),
child_attr_ids=i.get('child_attr_ids', []),
)
except BadRequest:
except Exception:
pass
@staticmethod
def _import_ci_type_inheritance(ci_type_inheritance, type_id_map):
for i in ci_type_inheritance:
i['parent_id'] = type_id_map.get(i['parent_id'])
i['child_id'] = type_id_map.get(i['child_id'])
if i['parent_id'] and i['child_id']:
try:
CITypeInheritanceManager.add([i.get('parent_id')], i.get('child_id'))
except BadRequest:
pass
@staticmethod
def _import_type_attributes(type2attributes, type_id_map, attr_id_map):
for type_id in type2attributes:
@ -1401,6 +1437,9 @@ class CITypeTemplateManager(object):
handled = set()
for attr in type2attributes[type_id]:
if attr.get('inherited'):
continue
payload = dict(type_id=type_id_map.get(int(type_id), type_id),
attr_id=attr_id_map.get(attr['id'], attr['id']),
default_show=attr['default_show'],
@ -1464,6 +1503,9 @@ class CITypeTemplateManager(object):
for rule in rules:
ci_type = CITypeCache.get(rule.pop('type_name', None))
if ci_type is None:
continue
adr = rule.pop('adr', {}) or {}
if ci_type:
@ -1476,10 +1518,10 @@ class CITypeTemplateManager(object):
if ad_rule:
rule['adr_id'] = ad_rule.id
ad_rule.update(**adr)
ad_rule.update(valid=False, **adr)
elif adr:
ad_rule = AutoDiscoveryRuleCRUD().add(**adr)
ad_rule = AutoDiscoveryRuleCRUD().add(valid=False, **adr)
rule['adr_id'] = ad_rule.id
else:
continue
@ -1508,6 +1550,23 @@ class CITypeTemplateManager(object):
except Exception as e:
current_app.logger.warning("import auto discovery rules failed: {}".format(e))
@staticmethod
def _import_auto_discovery_relation_rules(rules):
from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeRelationCRUD
for rule in rules:
ad_ci_type = CITypeCache.get(rule.pop('ad_type_name', None))
peer_ci_type = CITypeCache.get(rule.pop('peer_type_name', None))
peer_attr = AttributeCache.get(rule.pop('peer_attr_name', None))
if ad_ci_type and peer_attr and peer_ci_type:
if not AutoDiscoveryCITypeRelation.get_by(
ad_type_id=ad_ci_type.id, ad_key=rule.get('ad_key'),
peer_attr_id=peer_attr.id, peer_type_id=peer_ci_type.id):
AutoDiscoveryCITypeRelationCRUD().add(ad_type_id=ad_ci_type.id,
ad_key=rule.get('ad_key'),
peer_attr_id=peer_attr.id,
peer_type_id=peer_ci_type.id)
@staticmethod
def _import_icons(icons):
from api.lib.common_setting.upload_file import CommonFileCRUD
@ -1519,6 +1578,8 @@ class CITypeTemplateManager(object):
current_app.logger.warning("save icon failed: {}".format(e))
def import_template(self, tpt):
db.session.commit()
import time
s = time.time()
attr_id_map = self._import_attributes(tpt.get('type2attributes') or {})
@ -1537,9 +1598,14 @@ class CITypeTemplateManager(object):
current_app.logger.info('import relation_types cost: {}'.format(time.time() - s))
s = time.time()
self._import_ci_type_relations(tpt.get('ci_type_relations') or [], ci_type_id_map, relation_type_id_map)
self._import_ci_type_relations(tpt.get('ci_type_relations') or [],
ci_type_id_map, relation_type_id_map, attr_id_map)
current_app.logger.info('import ci_type_relations cost: {}'.format(time.time() - s))
s = time.time()
self._import_ci_type_inheritance(tpt.get('ci_type_inheritance') or [], ci_type_id_map)
current_app.logger.info('import ci_type_inheritance cost: {}'.format(time.time() - s))
s = time.time()
self._import_type_attributes(tpt.get('type2attributes') or {}, ci_type_id_map, attr_id_map)
current_app.logger.info('import type2attributes cost: {}'.format(time.time() - s))
@ -1552,26 +1618,73 @@ class CITypeTemplateManager(object):
self._import_auto_discovery_rules(tpt.get('ci_type_auto_discovery_rules') or [])
current_app.logger.info('import ci_type_auto_discovery_rules cost: {}'.format(time.time() - s))
s = time.time()
self._import_auto_discovery_relation_rules(tpt.get('ci_type_auto_discovery_relation_rules') or [])
current_app.logger.info('import ci_type_auto_discovery_relation_rules cost: {}'.format(time.time() - s))
s = time.time()
self._import_icons(tpt.get('icons') or {})
current_app.logger.info('import icons cost: {}'.format(time.time() - s))
@staticmethod
def export_template():
def export_template(type_ids=None):
from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeCRUD
from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeRelationCRUD
from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryRuleCRUD
from api.lib.common_setting.upload_file import CommonFileCRUD
ci_types = CITypeManager.get_ci_types(type_ids=type_ids)
extend_type_ids = []
for ci_type in ci_types:
if ci_type.get('parent_ids'):
extend_type_ids.extend(CITypeInheritanceManager.base(ci_type['id']))
extend_type_ids = list(set(extend_type_ids) - set(type_ids))
ci_type_relations = CITypeRelationManager.get(type_ids=type_ids)[0]
for i in ci_type_relations:
if i['parent_id'] not in type_ids:
extend_type_ids.append(i['parent_id'])
if i['child_id'] not in type_ids:
extend_type_ids.append(i['child_id'])
ad_relation_rules = AutoDiscoveryCITypeRelationCRUD.get_all(type_ids=type_ids)
rules = []
for r in ad_relation_rules:
if r.peer_type_id not in type_ids:
extend_type_ids.append(r.peer_type_id)
r = r.to_dict()
r['ad_type_name'] = CITypeCache.get(r.pop('ad_type_id')).name
peer_type_id = r.pop("peer_type_id")
peer_type_name = CITypeCache.get(peer_type_id).name
if not peer_type_name:
peer_type = CITypeCache.get(peer_type_id)
peer_type_name = peer_type and peer_type.name
r['peer_type_name'] = peer_type_name
peer_attr_id = r.pop("peer_attr_id")
peer_attr = AttributeCache.get(peer_attr_id)
r['peer_attr_name'] = peer_attr and peer_attr.name
rules.append(r)
ci_type_auto_discovery_relation_rules = rules
if extend_type_ids:
extend_type_ids = list(set(extend_type_ids))
type_ids.extend(extend_type_ids)
ci_types.extend(CITypeManager.get_ci_types(type_ids=extend_type_ids))
tpt = dict(
ci_types=CITypeManager.get_ci_types(),
ci_type_groups=CITypeGroupManager.get(),
ci_types=ci_types,
relation_types=[i.to_dict() for i in RelationTypeManager.get_all()],
ci_type_relations=CITypeRelationManager.get()[0],
ci_type_relations=ci_type_relations,
ci_type_inheritance=CITypeInheritanceManager.get_all(type_ids=type_ids),
ci_type_auto_discovery_rules=list(),
ci_type_auto_discovery_relation_rules=ci_type_auto_discovery_relation_rules,
type2attributes=dict(),
type2attribute_group=dict(),
icons=dict()
)
tpt['ci_type_groups'] = CITypeGroupManager.get(ci_types=tpt['ci_types'], type_ids=type_ids)
def get_icon_value(icon):
try:
@ -1579,12 +1692,13 @@ class CITypeTemplateManager(object):
except:
return ""
ad_rules = AutoDiscoveryCITypeCRUD.get_all()
type_id2name = {i['id']: i['name'] for i in tpt['ci_types']}
ad_rules = AutoDiscoveryCITypeCRUD.get_all(type_ids=type_ids)
rules = []
for r in ad_rules:
r = r.to_dict()
ci_type = CITypeCache.get(r.pop('type_id'))
r['type_name'] = ci_type and ci_type.name
r['type_name'] = type_id2name.get(r.pop('type_id'))
if r.get('adr_id'):
adr = AutoDiscoveryRuleCRUD.get_by_id(r.pop('adr_id'))
r['adr_name'] = adr and adr.name
@ -1618,65 +1732,6 @@ class CITypeTemplateManager(object):
return tpt
@staticmethod
def export_template_by_type(type_id):
ci_type = CITypeCache.get(type_id) or abort(404, ErrFormat.ci_type_not_found2.format("id={}".format(type_id)))
from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeCRUD
from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryRuleCRUD
from api.lib.common_setting.upload_file import CommonFileCRUD
tpt = dict(
ci_types=CITypeManager.get_ci_types(type_name=ci_type.name, like=False),
ci_type_auto_discovery_rules=list(),
type2attributes=dict(),
type2attribute_group=dict(),
icons=dict()
)
def get_icon_value(icon):
try:
return CommonFileCRUD().get_file_binary_str(icon)
except:
return ""
ad_rules = AutoDiscoveryCITypeCRUD.get_by_type_id(ci_type.id)
rules = []
for r in ad_rules:
r = r.to_dict()
r['type_name'] = ci_type and ci_type.name
if r.get('adr_id'):
adr = AutoDiscoveryRuleCRUD.get_by_id(r.pop('adr_id'))
r['adr_name'] = adr and adr.name
r['adr'] = adr and adr.to_dict() or {}
icon_url = r['adr'].get('option', {}).get('icon', {}).get('url')
if icon_url and icon_url not in tpt['icons']:
tpt['icons'][icon_url] = get_icon_value(icon_url)
rules.append(r)
tpt['ci_type_auto_discovery_rules'] = rules
for ci_type in tpt['ci_types']:
if ci_type['icon'] and len(ci_type['icon'].split('$$')) > 3:
icon_url = ci_type['icon'].split('$$')[3]
if icon_url not in tpt['icons']:
tpt['icons'][icon_url] = get_icon_value(icon_url)
tpt['type2attributes'][ci_type['id']] = CITypeAttributeManager.get_attributes_by_type_id(
ci_type['id'], choice_web_hook_parse=False, choice_other_parse=False)
for attr in tpt['type2attributes'][ci_type['id']]:
for i in (attr.get('choice_value') or []):
if (i[1] or {}).get('icon', {}).get('url') and len(i[1]['icon']['url'].split('$$')) > 3:
icon_url = i[1]['icon']['url'].split('$$')[3]
if icon_url not in tpt['icons']:
tpt['icons'][icon_url] = get_icon_value(icon_url)
tpt['type2attribute_group'][ci_type['id']] = CITypeAttributeGroupManager.get_by_type_id(ci_type['id'])
return tpt
class CITypeUniqueConstraintManager(object):
@staticmethod

3
cmdb-api/api/lib/cmdb/const.py
View File

@ -93,7 +93,8 @@ class RoleEnum(BaseEnum):
class AutoDiscoveryType(BaseEnum):
AGENT = "agent"
SNMP = "snmp"
HTTP = "http"
HTTP = "http" # cloud
COMPONENTS = "components"
class AttributeDefaultValueEnum(BaseEnum):

17
cmdb-api/api/views/cmdb/auto_discovery.py
View File

@ -2,23 +2,24 @@
import copy
import json
import uuid
from io import BytesIO
from flask import abort
from flask import current_app
from flask import request
from flask_login import current_user
from io import BytesIO
from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCICRUD
from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeCRUD
from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeRelationCRUD
from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryComponentsManager
from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCounterCRUD
from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryExecHistoryCRUD
from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryHTTPManager
from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryRuleCRUD
from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryRuleSyncHistoryCRUD
from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoverySNMPManager
from api.lib.cmdb.auto_discovery.const import DEFAULT_HTTP
from api.lib.cmdb.auto_discovery.const import DEFAULT_INNER
from api.lib.cmdb.auto_discovery.const import PRIVILEGED_USERS
from api.lib.cmdb.const import PermEnum
from api.lib.cmdb.const import ResourceTypeEnum
from api.lib.cmdb.resp_format import ErrFormat
@ -42,7 +43,7 @@ class AutoDiscoveryRuleView(APIView):
rebuild = False
exists = {i['name'] for i in res}
for i in copy.deepcopy(DEFAULT_HTTP):
for i in copy.deepcopy(DEFAULT_INNER):
if i['name'] not in exists:
i.pop('en', None)
AutoDiscoveryRuleCRUD().add(**i)
@ -110,12 +111,16 @@ class AutoDiscoveryRuleTemplateFileView(APIView):
class AutoDiscoveryRuleHTTPView(APIView):
url_prefix = ("/adr/http/<string:name>/categories",
"/adr/http/<string:name>/attributes",
"/adr/snmp/<string:name>/attributes")
"/adr/snmp/<string:name>/attributes",
"/adr/components/<string:name>/attributes",)
def get(self, name):
if "snmp" in request.url:
return self.jsonify(AutoDiscoverySNMPManager.get_attributes())
if "components" in request.url:
return self.jsonify(AutoDiscoveryComponentsManager.get_attributes(name))
if "attributes" in request.url:
resource = request.values.get('resource')
return self.jsonify(AutoDiscoveryHTTPManager.get_attributes(name, resource))
@ -250,7 +255,7 @@ class AutoDiscoveryRuleSyncView(APIView):
url_prefix = ("/adt/sync",)
def get(self):
if current_user.username not in ("cmdb_agent", "worker", "admin"):
if current_user.username not in PRIVILEGED_USERS:
return abort(403)
oneagent_name = request.values.get('oneagent_name')

1
cmdb-api/api/views/cmdb/ci.py
View File

@ -268,6 +268,7 @@ class CIBaselineView(APIView):
return self.jsonify(CIManager().baseline(list(map(int, ci_ids)), before_date))
@args_required("before_date")
@has_perm_for_ci("ci_id", ResourceTypeEnum.CI, PermEnum.UPDATE, CIManager.get_type)
def post(self, ci_id):
if 'rollback' in request.url:
before_date = request.values.get('before_date')

16
cmdb-api/api/views/cmdb/ci_type.py
View File

@ -51,7 +51,11 @@ class CITypeView(APIView):
q = request.args.get("type_name")
if type_id is not None:
ci_type = CITypeCache.get(type_id).to_dict()
ci_type = CITypeCache.get(type_id)
if ci_type is None:
return abort(404, ErrFormat.ci_type_not_found)
ci_type = ci_type.to_dict()
ci_type['parent_ids'] = CITypeInheritanceManager.get_parents(type_id)
ci_types = [ci_type]
elif type_name is not None:
@ -357,15 +361,13 @@ class CITypeAttributeGroupView(APIView):
class CITypeTemplateView(APIView):
url_prefix = ("/ci_types/template/import", "/ci_types/template/export", "/ci_types/<int:type_id>/template/export")
url_prefix = ("/ci_types/template/import", "/ci_types/template/export")
@perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Model_Configuration,
app_cli.op.download_CIType, app_cli.admin_name)
def get(self, type_id=None): # export
if type_id is not None:
return self.jsonify(dict(ci_type_template=CITypeTemplateManager.export_template_by_type(type_id)))
return self.jsonify(dict(ci_type_template=CITypeTemplateManager.export_template()))
def get(self): # export
type_ids = list(map(int, handle_arg_list(request.values.get('type_ids')))) or None
return self.jsonify(dict(ci_type_template=CITypeTemplateManager.export_template(type_ids=type_ids)))
@perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Model_Configuration,
app_cli.op.download_CIType, app_cli.admin_name)