relation view has been optimised

This commit is contained in:
pycook 2019-12-03 19:10:54 +08:00
parent 8ee7c6daf8
commit 92dd4c5dfe
13 changed files with 241 additions and 83 deletions

View File

@ -67,7 +67,7 @@ def init_cache():
ci_relations = CIRelation.get_by(to_dict=False)
relations = dict()
for cr in ci_relations:
relations.setdefault(cr.first_ci_id, []).append(cr.second_ci_id)
relations.setdefault(cr.first_ci_id, {}).update({cr.second_ci_id: cr.second_ci.type_id})
for i in relations:
relations[i] = json.dumps(relations[i])
if relations:

View File

@ -566,4 +566,4 @@ class CIRelationManager(object):
ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id), queue=CMDB_QUEUE)
return cls.delete(cr.cr_id)
return cls.delete(cr.id)

View File

@ -13,6 +13,7 @@ from api.lib.cmdb.cache import AttributeCache
from api.lib.cmdb.cache import CITypeAttributeCache
from api.lib.cmdb.cache import CITypeCache
from api.models.cmdb import CITypeAttribute
from api.models.cmdb import CITypeRelation
from api.models.cmdb import PreferenceRelationView
from api.models.cmdb import PreferenceShowAttributes
from api.models.cmdb import PreferenceTreeView
@ -124,17 +125,28 @@ class PreferenceManager(object):
for i in result[view_name]:
id2type[i['parent_id']] = None
id2type[i['child_id']] = None
topo = {i['child_id']: {i['parent_id']} for i in result[view_name]}
leaf = list(set(toposort.toposort_flatten(topo)) - set([j for i in topo.values() for j in i]))
result[view_name] = toposort.toposort_flatten(
{i['child_id']: {i['parent_id']} for i in result[view_name]})
leaf2show_types = {i: [t['child_id'] for t in CITypeRelation.get_by(parent_id=i)] for i in leaf}
result[view_name] = dict(topo=list(map(list, toposort.toposort(topo))),
topo_flatten=list(toposort.toposort_flatten(topo)),
leaf=leaf,
leaf2show_types=leaf2show_types,
show_types=[CITypeCache.get(j).to_dict()
for i in leaf2show_types.values() for j in i])
for type_id in id2type:
id2type[type_id] = CITypeCache.get(type_id).to_dict()
print(result)
return result, id2type, sorted(name2id, key=lambda x: x[1])
@classmethod
def create_or_update_relation_view(cls, name, cr_ids):
if not cr_ids:
return abort(400, "Node must be selected")
existed = PreferenceRelationView.get_by(name=name, to_dict=False, first=True)
if existed is None:
PreferenceRelationView.create(name=name, cr_ids=json.dumps(cr_ids))
@ -145,4 +157,5 @@ class PreferenceManager(object):
def delete_relation_view(name):
for existed in PreferenceRelationView.get_by(name=name, to_dict=False):
existed.soft_delete()
return name

View File

@ -73,6 +73,10 @@ class Search(object):
def _in_query_handle(self, attr, v):
terms = v[1:-1].split(";")
operator = "|"
if attr in ('_type', 'ci_type', 'type_id') and terms and terms[0].isdigit():
attr = "type_id"
terms = map(int, terms)
current_app.logger.warning(terms)
for term in terms:
self._operator2query(operator).append({
"term": {
@ -164,6 +168,7 @@ class Search(object):
def _query_build_raw(self):
queries = handle_arg_list(self.orig_query)
current_app.logger.debug(queries)
self.__query_build_by_field(queries)

View File

@ -15,7 +15,14 @@ from api.models.cmdb import CI
class Search(object):
def __init__(self, root_id, level=1, query=None, fl=None, facet_field=None, page=1, count=None, sort=None):
def __init__(self, root_id,
level=1,
query=None,
fl=None,
facet_field=None,
page=1,
count=None,
sort=None):
self.orig_query = query
self.fl = fl
self.facet_field = facet_field
@ -24,22 +31,36 @@ class Search(object):
self.sort = sort or ("ci_id" if current_app.config.get("USE_ES") else None)
self.root_id = root_id
self.level = int(level)
self.level = level
def search(self):
ci = CI.get_by_id(self.root_id) or abort(404, "CI <{0}> does not exist".format(self.root_id))
ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
for _ in range(0, self.level):
print(rd.get(ids, REDIS_PREFIX_CI_RELATION))
_tmp = list(map(json.loads, filter(lambda x: x is not None, rd.get(ids, REDIS_PREFIX_CI_RELATION) or [])))
cis = [CI.get_by_id(_id) or abort(404, "CI <{0}> does not exist".format(_id)) for _id in ids]
merge_ids = []
for level in self.level:
ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
for _ in range(0, level):
_tmp = list(map(lambda x: list(json.loads(x).keys()),
filter(lambda x: x is not None, rd.get(ids, REDIS_PREFIX_CI_RELATION) or [])))
ids = [j for i in _tmp for j in i]
merge_ids.extend(ids)
if not self.orig_query or ("_type:" not in self.orig_query
and "type_id:" not in self.orig_query
and "ci_type:" not in self.orig_query):
type_ids = CITypeRelationManager.get_child_type_ids(ci.type_id, self.level)
type_ids = []
for level in self.level:
for ci in cis:
type_ids.extend(CITypeRelationManager.get_child_type_ids(ci.type_id, level))
type_ids = list(set(type_ids))
if self.orig_query:
self.orig_query = "_type:({0}),{1}".format(";".join(list(map(str, type_ids))), self.orig_query)
else:
self.orig_query = "_type:({0})".format(";".join(list(map(str, type_ids))))
if not ids:
if not merge_ids:
# cis, counter, total, self.page, numfound, facet_
return [], {}, 0, self.page, 0, {}
@ -50,7 +71,7 @@ class Search(object):
page=self.page,
count=self.count,
sort=self.sort,
ci_ids=ids).search()
ci_ids=merge_ids).search()
else:
return SearchFromDB(self.orig_query,
fl=self.fl,
@ -58,17 +79,27 @@ class Search(object):
page=self.page,
count=self.count,
sort=self.sort,
ci_ids=ids).search()
ci_ids=merge_ids).search()
def statistics(self):
def statistics(self, type_ids):
ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
for l in range(0, self.level):
for l in range(0, int(self.level)):
if l == 0:
_tmp = list(map(json.loads, [i or '[]' for i in rd.get(ids, REDIS_PREFIX_CI_RELATION) or []]))
_tmp = list(map(lambda x: list(json.loads(x).keys()),
[i or '{}' for i in rd.get(ids, REDIS_PREFIX_CI_RELATION) or []]))
else:
for idx, i in enumerate(_tmp):
if i:
__tmp = list(map(json.loads, filter(lambda x: x is not None,
if type_ids and l == self.level - 1:
__tmp = list(
map(lambda x: list({_id: 1 for _id, type_id in json.loads(x).items()
if type_id in type_ids}.keys()),
filter(lambda x: x is not None,
rd.get(i, REDIS_PREFIX_CI_RELATION) or [])))
else:
__tmp = list(map(lambda x: list(json.loads(x).keys()),
filter(lambda x: x is not None,
rd.get(i, REDIS_PREFIX_CI_RELATION) or [])))
_tmp[idx] = [j for i in __tmp for j in i]
else:

View File

@ -90,7 +90,7 @@ class RoleRelationCRUD(object):
@staticmethod
def delete2(parent_id, child_id):
existed = RoleRelation.get_by(parent_id=parent_id, child_id=child_id)
existed = RoleRelation.get_by(parent_id=parent_id, child_id=child_id, first=True, to_dict=False)
existed or abort(400, "RoleRelation < {0} -> {1} > does not exist".format(parent_id, child_id))
existed.soft_delete()

View File

@ -14,6 +14,7 @@ from api.extensions import rd
from api.lib.cmdb.const import CMDB_QUEUE
from api.lib.cmdb.const import REDIS_PREFIX_CI
from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
from api.models.cmdb import CIRelation
@celery.task(name="cmdb.ci_cache", queue=CMDB_QUEUE)
@ -46,10 +47,13 @@ def ci_delete(ci_id):
@celery.task(name="cmdb.ci_relation_cache", queue=CMDB_QUEUE)
def ci_relation_cache(parent_id, child_id):
children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0]
children = json.loads(children) if children is not None else []
children.append(child_id)
children = json.loads(children) if children is not None else {}
rd.create_or_update({parent_id: json.dumps(list(set(children)))}, REDIS_PREFIX_CI_RELATION)
cr = CIRelation.get_by(first_ci_id=parent_id, second_ci_id=child_id, first=True, to_dict=False)
if child_id not in children:
children[child_id] = cr.second_ci.type_id
rd.create_or_update({parent_id: json.dumps(children)}, REDIS_PREFIX_CI_RELATION)
current_app.logger.info("ADD ci relation cache: {0} -> {1}".format(parent_id, child_id))
@ -57,10 +61,11 @@ def ci_relation_cache(parent_id, child_id):
@celery.task(name="cmdb.ci_relation_delete", queue=CMDB_QUEUE)
def ci_relation_delete(parent_id, child_id):
children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0]
children = json.loads(children) if children is not None else []
if child_id in children:
children.remove(child_id)
children = json.loads(children) if children is not None else {}
rd.create_or_update({parent_id: json.dumps(list(set(children)))}, REDIS_PREFIX_CI_RELATION)
if child_id in children:
children.pop(child_id)
rd.create_or_update({parent_id: json.dumps(children)}, REDIS_PREFIX_CI_RELATION)
current_app.logger.info("DELETE ci relation cache: {0} -> {1}".format(parent_id, child_id))

View File

@ -34,7 +34,7 @@ class CIRelationSearchView(APIView):
count = get_page_size(request.values.get("count") or request.values.get("page_size"))
root_id = request.values.get('root_id')
level = request.values.get('level', 1)
level = list(map(int, handle_arg_list(request.values.get('level', '1'))))
query = request.values.get('q', "")
fl = handle_arg_list(request.values.get('fl', ""))
@ -64,11 +64,12 @@ class CIRelationStatisticsView(APIView):
def get(self):
root_ids = list(map(int, handle_arg_list(request.values.get('root_ids'))))
level = request.values.get('level', 1)
type_ids = set(map(int, handle_arg_list(request.values.get('type_ids', []))))
start = time.time()
s = Search(root_ids, level)
try:
result = s.statistics()
result = s.statistics(type_ids)
except SearchError as e:
return abort(400, str(e))
current_app.logger.debug("search time is :{0}".format(time.time() - start))

View File

@ -77,6 +77,7 @@ class PreferenceRelationApiView(APIView):
@role_required(RoleEnum.CONFIG)
@args_required("name")
@args_required("cr_ids")
def post(self):
name = request.values.get("name")
cr_ids = request.values.get("cr_ids")

View File

@ -68,7 +68,7 @@
</a-col>
</template>
<a-col :lg="!advanced && 6 || 24" :md="!advanced && 8 || 24" :sm="24" style="float: right">
<a-col :lg="!advanced && 6 || 24" :md="!advanced && 8 || 24" :sm="24" style="float: right; padding-left: 0">
<span
class="table-page-search-submitButtons"
:style="advanced && { float: 'right', overflow: 'hidden' } || {} "

View File

@ -1,7 +1,13 @@
<template>
<div>
<a-card :bordered="true" title="关系视图定义面板">
<a-card-meta description="点击右键可选择节点"></a-card-meta>
<a-card-meta description="先打开右上角的开关,便可选择树的节点"></a-card-meta>
<a-switch
slot="extra"
@change="toggleSelect"
checkedChildren="on"
unCheckedChildren="off"
/>
<div
id="visualization"
style="height:400px"
@ -9,7 +15,7 @@
@mouseup="mouseUp"
@mousemove="mouseMove">
</div>
<relation-view-form ref="relationViewForm"></relation-view-form>
<relation-view-form @refresh="reload" ref="relationViewForm"></relation-view-form>
</a-card>
<a-row :gutter="0">
@ -23,7 +29,7 @@
v-for="view in Object.keys(relationViews.views)">
<a-card :bordered="true" :title="view">
<a slot="extra"><a-icon type="close" @click="deleteView(view)"></a-icon></a>
<div :id="&quot;view-&quot; + (relationViews.views[view] || []).join(&quot;&quot;)" style="height:200px"></div>
<div :id="&quot;view-&quot; + (relationViews.views[view].topo_flatten || []).join(&quot;&quot;)" style="height:200px"></div>
</a-card>
</a-col>
</a-row>
@ -32,7 +38,6 @@
</template>
<script>
// 按需引入
import { DataSet, Network } from 'vis-network'
import { getCITypeRelations } from '@/api/cmdb/CITypeRelation'
import { getRelationView, deleteRelationView } from '@/api/cmdb/preference'
@ -48,12 +53,15 @@ export default {
relationViews: { views: {} },
relations: [],
network: null,
options: {},
viewData: {},
container: null,
nodes: null,
edges: null,
canvas: null,
ctx: null,
drag: false,
canSelect: false,
rect: {},
drawingSurfaceImageData: null
}
@ -61,6 +69,7 @@ export default {
created () {
this.create()
},
inject: ['reload'],
methods: {
create () {
getRelationView().then(res => {
@ -145,30 +154,54 @@ export default {
// initialize your network!
this.container.oncontextmenu = () => { return false }
this.options = options
this.viewData = data
this.network = new Network(this.container, data, options)
this.canvas = this.network.canvas.frame.canvas
this.ctx = this.canvas.getContext('2d')
})
},
toggleSelect (checked) {
if (checked) {
this.canSelect = true
this.options.autoResize = false
this.options.interaction.hover = false
this.options.interaction.dragView = false
this.options.interaction.dragNodes = false
this.network = new Network(this.container, this.viewData, this.options)
this.canvas = this.network.canvas.frame.canvas
this.ctx = this.canvas.getContext('2d')
} else {
this.canSelect = false
this.options.autoResize = true
this.options.interaction.hover = true
this.options.interaction.dragView = true
this.options.interaction.dragNodes = true
this.network = new Network(this.container, this.viewData, this.options)
this.canvas = this.network.canvas.frame.canvas
this.ctx = this.canvas.getContext('2d')
}
},
createRelationViews () {
Object.keys(this.relationViews.views).forEach(viewName => {
const nodes = []
const edges = []
const len = this.relationViews.views[viewName].length
this.relationViews.views[viewName].slice(0, len - 1).forEach((fromId, idx) => {
const len = this.relationViews.views[viewName].topo_flatten.length
this.relationViews.views[viewName].topo_flatten.slice(0, len - 1).forEach((fromId, idx) => {
nodes.push({
id: fromId,
label: this.relationViews.id2type[fromId].alias
})
edges.push({
from: fromId,
to: this.relationViews.views[viewName][idx + 1]
to: this.relationViews.views[viewName].topo_flatten[idx + 1]
})
})
nodes.push({
id: this.relationViews.views[viewName][len - 1],
label: this.relationViews.id2type[this.relationViews.views[viewName][len - 1]].alias
id: this.relationViews.views[viewName].topo_flatten[len - 1],
label: this.relationViews.id2type[this.relationViews.views[viewName].topo_flatten[len - 1]].alias
})
const _nodes = new DataSet(nodes)
const _edges = new DataSet(edges)
@ -208,9 +241,9 @@ export default {
}
}
setTimeout(() => {
const container = document.querySelector('#view-' + this.relationViews.views[viewName].join(''))
const container = document.querySelector('#view-' + this.relationViews.views[viewName].topo_flatten.join(''))
const n = new Network(container, data, options)
console.log(n)
console.log(n) // TODO
}, 100)
})
},
@ -274,7 +307,7 @@ export default {
},
mouseDown () {
if (event.button === 2) {
if (event.button === 0 && this.canSelect) {
this.saveDrawingSurface()
this.rect.startX = event.offsetX
this.rect.startY = event.offsetY
@ -284,7 +317,7 @@ export default {
},
mouseUp () {
if (event.button === 2) {
if (event.button === 0 && this.canSelect) {
this.restoreDrawingSurface()
this.drag = false

View File

@ -119,6 +119,7 @@ export default {
.then(res => {
this.$message.success(`添加成功`)
this.onClose()
this.$emit('refresh')
})
.catch(err => this.requestFailed(err))
},

View File

@ -1,6 +1,6 @@
<template>
<a-card :bordered="false">
<a-menu v-model="current" mode="horizontal" v-if="relationViews.name2id && relationViews.name2id.length">
<a-card :bordered="false" class="relation-card">
<a-menu v-model="currentView" mode="horizontal" v-if="relationViews.name2id && relationViews.name2id.length">
<a-menu-item :key="item[1]" v-for="item in relationViews.name2id">
<router-link
:to="{name: 'cmdb_relation_views_item', params: { viewId: item[1]} }"
@ -15,9 +15,14 @@
<a-tree showLine :loadData="onLoadData" @select="onSelect" :treeData="treeData"></a-tree>
</a-col>
<a-col :span="19">
<search-form ref="search" @refresh="refreshTable" :preferenceAttrList="preferenceAttrList" />
<a-menu v-model="currentTypeId" mode="horizontal" v-if="showTypeIds && showTypeIds.length > 1">
<a-menu-item :key="item.id" v-for="item in showTypes">
<a @click="changeCIType(item.id)">{{ item.alias || item.name }}</a>
</a-menu-item>
</a-menu>
<search-form style="margin-top: 10px" ref="search" @refresh="refreshTable" :preferenceAttrList="preferenceAttrList" />
<s-table
v-if="levels.length > 1"
v-if="levels.length"
bordered
ref="table"
size="middle"
@ -49,16 +54,22 @@ export default {
},
data () {
return {
parameter: {},
treeData: [],
triggerSelect: false,
treeNode: null,
ciTypes: [],
relationViews: {},
levels: [],
showTypeIds: [],
showTypes: [],
leaf2showTypes: {},
leaf: [],
typeId: null,
viewId: null,
viewName: null,
current: [],
currentView: [],
currentTypeId: [],
instanceList: [],
treeKeys: [],
columns: [],
@ -70,8 +81,9 @@ export default {
loadInstances: parameter => {
console.log(parameter, 'load instances')
this.parameter = parameter
const params = Object.assign(parameter || {}, this.$refs.search.queryParam)
let q = `q=_type:${this.levels[this.levels.length - 1]}`
let q = `q=_type:${this.currentTypeId[0]}`
Object.keys(params).forEach(key => {
if (!['pageNo', 'pageSize', 'sortField', 'sortOrder'].includes(key) && params[key] + '' !== '') {
if (typeof params[key] === 'object' && params[key].length > 1) {
@ -109,15 +121,36 @@ export default {
if (res.numfound !== 0) {
setTimeout(() => {
this.setColumnWidth()
}, 200)
console.log('set column')
}, 300)
}
this.loadRoot()
return result
})
}
q += `&root_id=${this.treeKeys[this.treeKeys.length - 1]}`
q += `&level=${this.levels.length - this.treeKeys.length}`
q += `&root_id=${this.treeKeys[this.treeKeys.length - 1].split('_')[0]}`
const typeId = parseInt(this.treeKeys[this.treeKeys.length - 1].split('_')[1])
let level = []
if (!this.leaf.includes(typeId)) {
let startIdx = 0
this.levels.forEach((item, idx) => {
if (item.includes(typeId)) {
startIdx = idx
}
})
this.leaf.forEach(leafId => {
this.levels.forEach((item, levelIdx) => {
if (item.includes(leafId) && levelIdx - startIdx + 1 > 0) {
level.push(levelIdx - startIdx + 1)
}
})
})
} else {
level = [1]
}
q += `&level=${level.join(',')}`
if (q[0] === '&') {
q = q.slice(1)
}
@ -133,10 +166,10 @@ export default {
if (res.numfound !== 0) {
setTimeout(() => {
this.setColumnWidth()
}, 200)
this.loadNoRoot(this.treeKeys[this.treeKeys.length - 1], this.levels.length - this.treeKeys.length - 1)
console.log('set column')
}, 300)
this.loadNoRoot(this.treeKeys[this.treeKeys.length - 1], level)
}
return result
})
}
@ -161,41 +194,64 @@ export default {
this.$refs.table.refresh(bool)
},
loadRoot () {
searchCI(`q=_type:${this.levels[0]}&count=10000`).then(res => {
changeCIType (typeId) {
this.currentTypeId = [typeId]
this.loadColumns(typeId)
this.$refs.table.renderClear()
setTimeout(() => {
this.refreshTable(true)
}, 100)
},
async loadRoot () {
searchCI(`q=_type:(${this.levels[0].join(';')})&count=10000`).then(async res => {
const facet = []
const ciIds = []
res.result.forEach(item => {
facet.push([item[item.unique], 0, item.ci_id])
facet.push([item[item.unique], 0, item.ci_id, item.type_id])
ciIds.push(item.ci_id)
})
statisticsCIRelation({ root_ids: ciIds.join(','), level: this.levels.length - 1 }).then(num => {
const promises = this.leaf.map(leafId => {
let level = 0
this.levels.forEach((item, idx) => {
if (item.includes(leafId)) {
level = idx + 1
}
})
return statisticsCIRelation({ root_ids: ciIds.join(','), level: level }).then(num => {
facet.forEach((item, idx) => {
item[1] = num[ciIds[idx] + '']
item[1] += num[ciIds[idx] + '']
})
})
})
await Promise.all(promises)
this.wrapTreeData(facet)
})
})
},
loadNoRoot (rootId, level) {
if (level === 0) {
return
}
searchCIRelation(`root_id=${rootId}&level=1&count=10000`).then(res => {
async loadNoRoot (rootIdAndTypeId, level) {
const rootId = rootIdAndTypeId.split('_')[0]
searchCIRelation(`root_id=${rootId}&level=1&count=10000`).then(async res => {
const facet = []
const ciIds = []
res.result.forEach(item => {
facet.push([item[item.unique], 0, item.ci_id])
facet.push([item[item.unique], 0, item.ci_id, item.type_id])
ciIds.push(item.ci_id)
})
statisticsCIRelation({ root_ids: ciIds.join(','), level: level }).then(num => {
const promises = level.map(_level => {
if (_level > 1) {
return statisticsCIRelation({ root_ids: ciIds.join(','), level: _level - 1 }).then(num => {
facet.forEach((item, idx) => {
item[1] = num[ciIds[idx] + '']
item[1] += num[ciIds[idx] + '']
})
})
}
})
await Promise.all(promises)
this.wrapTreeData(facet)
})
})
},
onSelect (keys) {
@ -214,8 +270,8 @@ export default {
facet.forEach(item => {
treeData.push({
title: `${item[0]} (${item[1]})`,
key: this.treeKeys.join('-') + '-' + item[2],
isLeaf: this.levels.length - 2 === this.treeKeys.length
key: this.treeKeys.join('-') + '-' + item[2] + '_' + item[3],
isLeaf: this.leaf.includes(item[3])
})
})
if (this.treeNode === null) {
@ -269,16 +325,25 @@ export default {
this.viewName = item[0]
}
})
this.levels = this.relationViews.views[this.viewName]
this.current = [this.viewId]
this.typeId = this.levels[0]
this.levels = this.relationViews.views[this.viewName].topo
this.showTypes = this.relationViews.views[this.viewName].show_types
const showTypeIds = []
this.showTypes.forEach(item => {
showTypeIds.push(item.id)
})
this.showTypeIds = showTypeIds
this.leaf2showTypes = this.relationViews.views[this.viewName].leaf2show_types
this.leaf = this.relationViews.views[this.viewName].leaf
this.currentView = [this.viewId]
this.currentTypeId = [this.showTypeIds[0]]
this.typeId = this.levels[0][0]
this.loadColumns()
this.$refs.table && this.$refs.table.refresh(true)
}
})
},
loadColumns () {
getSubscribeAttributes(this.levels[this.levels.length - 1]).then(res => {
getSubscribeAttributes(this.currentTypeId[0]).then(res => {
const prefAttrList = res.attributes
this.preferenceAttrList = prefAttrList
@ -309,11 +374,14 @@ export default {
}
</script>
<style scoped>
<style lang='less'>
.ant-menu-horizontal {
border-bottom: 1px solid #ebedf0 !important;
}
.ant-menu-horizontal {
border-bottom: 1px solid #ebedf0 !important;
}
.relation-card > .ant-card-body {
padding-top: 0 !important;
}
</style>