7x31
/
cmdb
mirror of https://github.com/veops/cmdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Realize /api/v0.1/ci_relations/s [done]

This commit is contained in:
pycook 2019-11-21 18:21:03 +08:00
parent bc7b51c544
commit 31a97d7ad0
24 changed files with 451 additions and 229 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
api/commands/click_cmdb.py
View File

@ -10,7 +10,11 @@ from flask.cli import with_appcontext
import api.lib.cmdb.ci
from api.extensions import db
from api.extensions import rd
from api.lib.cmdb.const import REDIS_PREFIX_CI
from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
from api.lib.cmdb.const import ValueTypeEnum
from api.models.cmdb import CI
from api.models.cmdb import CIRelation
@click.command()
@ -21,12 +25,12 @@ def init_cache():
if current_app.config.get("USE_ES"):
from api.extensions import es
from api.models.cmdb import Attribute
from api.lib.cmdb.const import type_map
from api.lib.cmdb.utils import ValueTypeMap
attributes = Attribute.get_by(to_dict=False)
for attr in attributes:
other = dict()
other['index'] = True if attr.is_index else False
if attr.value_type == Attribute.TEXT:
if attr.value_type == ValueTypeEnum.TEXT:
other['analyzer'] = 'ik_max_word'
other['search_analyzer'] = 'ik_smart'
if attr.is_index:
@ -37,7 +41,7 @@ def init_cache():
}
}
try:
es.update_mapping(attr.name, type_map['es_type'][attr.value_type], other)
es.update_mapping(attr.name, ValueTypeMap.es_type[attr.value_type], other)
except Exception as e:
print(e)
@ -48,7 +52,7 @@ def init_cache():
if res:
continue
else:
res = rd.get([ci.id])
res = rd.get([ci.id], REDIS_PREFIX_CI)
if res and list(filter(lambda x: x, res)):
continue
@ -58,7 +62,15 @@ def init_cache():
if current_app.config.get("USE_ES"):
es.create(ci_dict)
else:
rd.delete(ci.id)
rd.add({ci.id: json.dumps(ci_dict)})
rd.create_or_update({ci.id: json.dumps(ci_dict)}, REDIS_PREFIX_CI)
ci_relations = CIRelation.get_by(to_dict=False)
relations = dict()
for cr in ci_relations:
relations.setdefault(cr.first_ci_id, []).append(cr.second_ci_id)
for i in relations:
relations[i] = json.dumps(relations[i])
if relations:
rd.create_or_update(relations, REDIS_PREFIX_CI_RELATION)
db.session.remove()

2
api/extensions.py
View File

@ -19,5 +19,5 @@ migrate = Migrate()
cache = Cache()
celery = Celery()
cors = CORS(supports_credentials=True)
rd = RedisHandler(prefix="CMDB_CI") # TODO
rd = RedisHandler()
es = ESHandler()

13
api/lib/cmdb/attribute.py
View File

@ -5,7 +5,8 @@ from flask import current_app
from api.extensions import db
from api.lib.cmdb.cache import AttributeCache
from api.lib.cmdb.const import type_map
from api.lib.cmdb.const import ValueTypeEnum
from api.lib.cmdb.utils import ValueTypeMap
from api.lib.decorator import kwargs_required
from api.models.cmdb import Attribute
from api.models.cmdb import CITypeAttribute
@ -22,13 +23,13 @@ class AttributeManager(object):
@staticmethod
def get_choice_values(attr_id, value_type):
choice_table = type_map.get("choice").get(value_type)
choice_table = ValueTypeMap.choice.get(value_type)
choice_values = choice_table.get_by(fl=["value"], attr_id=attr_id)
return [choice_value["value"] for choice_value in choice_values]
@staticmethod
def _add_choice_values(_id, value_type, choice_values):
choice_table = type_map.get("choice").get(value_type)
choice_table = ValueTypeMap.choice.get(value_type)
db.session.query(choice_table).filter(choice_table.attr_id == _id).delete()
db.session.flush()
@ -121,7 +122,7 @@ class AttributeManager(object):
from api.extensions import es
other = dict()
other['index'] = True if attr.is_index else False
if attr.value_type == Attribute.TEXT:
if attr.value_type == ValueTypeEnum.TEXT:
other['analyzer'] = 'ik_max_word'
other['search_analyzer'] = 'ik_smart'
if attr.is_index:
@ -131,7 +132,7 @@ class AttributeManager(object):
"ignore_above": 256
}
}
es.update_mapping(name, type_map['es_type'][attr.value_type], other)
es.update_mapping(name, ValueTypeMap.es_type[attr.value_type], other)
return attr.id
@ -172,7 +173,7 @@ class AttributeManager(object):
name = attr.name
if attr.is_choice:
choice_table = type_map["choice"].get(attr.value_type)
choice_table = ValueTypeMap.choice.get(attr.value_type)
db.session.query(choice_table).filter(choice_table.attr_id == _id).delete() # FIXME: session conflict
db.session.flush()

25
api/lib/cmdb/ci.py
View File

@ -18,13 +18,14 @@ from api.lib.cmdb.ci_type import CITypeManager
from api.lib.cmdb.const import CMDB_QUEUE
from api.lib.cmdb.const import ExistPolicy
from api.lib.cmdb.const import OperateType
from api.lib.cmdb.const import REDIS_PREFIX_CI
from api.lib.cmdb.const import RetKey
from api.lib.cmdb.const import TableMap
from api.lib.cmdb.const import type_map
from api.lib.cmdb.history import AttributeHistoryManger
from api.lib.cmdb.history import CIRelationHistoryManager
from api.lib.cmdb.search.db.query_sql import QUERY_CIS_BY_IDS
from api.lib.cmdb.search.db.query_sql import QUERY_CIS_BY_VALUE_TABLE
from api.lib.cmdb.search.ci.db.query_sql import QUERY_CIS_BY_IDS
from api.lib.cmdb.search.ci.db.query_sql import QUERY_CIS_BY_VALUE_TABLE
from api.lib.cmdb.utils import TableMap
from api.lib.cmdb.utils import ValueTypeMap
from api.lib.cmdb.value import AttributeValueManager
from api.lib.decorator import kwargs_required
from api.lib.utils import handle_arg_list
@ -33,6 +34,8 @@ from api.models.cmdb import CIRelation
from api.models.cmdb import CITypeAttribute
from api.tasks.cmdb import ci_cache
from api.tasks.cmdb import ci_delete
from api.tasks.cmdb import ci_relation_cache
from api.tasks.cmdb import ci_relation_delete
class CIManager(object):
@ -302,7 +305,7 @@ class CIManager(object):
@staticmethod
def _get_cis_from_cache(ci_ids, ret_key=RetKey.NAME, fields=None):
res = rd.get(ci_ids)
res = rd.get(ci_ids, REDIS_PREFIX_CI)
if res is not None and None not in res and ret_key == RetKey.NAME:
res = list(map(json.loads, res))
if not fields:
@ -332,7 +335,7 @@ class CIManager(object):
ci_ids = ",".join(ci_ids)
if value_tables is None:
value_tables = type_map["table_name"].values()
value_tables = ValueTypeMap.table_name.values()
value_sql = " UNION ".join([QUERY_CIS_BY_VALUE_TABLE.format(value_table, ci_ids)
for value_table in value_tables])
@ -362,7 +365,7 @@ class CIManager(object):
else:
return abort(400, "invalid ret key")
value = type_map["serialize2"][value_type](value)
value = ValueTypeMap.serialize2[value_type](value)
if is_list:
ci_dict.setdefault(attr_key, []).append(value)
else:
@ -532,6 +535,9 @@ class CIRelationManager(object):
second_ci_id=second_ci_id,
relation_type_id=relation_type_id)
CIRelationHistoryManager().add(existed, OperateType.ADD)
ci_relation_cache.apply_async(args=(first_ci_id, second_ci_id), queue=CMDB_QUEUE)
if more is not None:
existed.upadte(more=more)
@ -545,6 +551,8 @@ class CIRelationManager(object):
his_manager = CIRelationHistoryManager()
his_manager.add(cr, operate_type=OperateType.DELETE)
ci_relation_delete.apply_async(args=(cr.first_ci_id, cr.second_ci_id), queue=CMDB_QUEUE)
return cr_id
@classmethod
@ -553,4 +561,7 @@ class CIRelationManager(object):
second_ci_id=second_ci_id,
to_dict=False,
first=True)
ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id), queue=CMDB_QUEUE)
return cls.delete(cr.cr_id)

23
api/lib/cmdb/ci_type.py
View File

@ -4,6 +4,7 @@
from flask import abort
from flask import current_app
from api.extensions import db
from api.lib.cmdb.attribute import AttributeManager
from api.lib.cmdb.cache import AttributeCache
from api.lib.cmdb.cache import CITypeAttributeCache
@ -263,8 +264,26 @@ class CITypeRelationManager(object):
manage relation between CITypes
"""
def __init__(self):
pass
@staticmethod
def get():
res = CITypeRelation.get_by(to_dict=False)
for idx, item in enumerate(res):
_item = item.to_dict()
res[idx] = _item
res[idx]['parent'] = item.parent.to_dict()
res[idx]['child'] = item.child.to_dict()
res[idx]['relation_type'] = item.relation_type.to_dict()
return res
@staticmethod
def get_child_type_ids(type_id, level):
ids = [type_id]
query = db.session.query(CITypeRelation).filter(CITypeRelation.deleted.is_(False))
for _ in range(0, level):
ids = [i.child_id for i in query.filter(CITypeRelation.parent_id.in_(ids))]
return ids
@staticmethod
def _wrap_relation_type_dict(type_id, relation_inst):

141
api/lib/cmdb/const.py
View File

@ -1,160 +1,57 @@
# -*- coding:utf-8 -*-
from __future__ import unicode_literals
import datetime
import six
from markupsafe import escape
from api.lib.cmdb.cache import AttributeCache
from api.models.cmdb import Attribute
from api.models.cmdb import CIIndexValueDateTime
from api.models.cmdb import CIIndexValueFloat
from api.models.cmdb import CIIndexValueInteger
from api.models.cmdb import CIIndexValueText
from api.models.cmdb import CIValueDateTime
from api.models.cmdb import CIValueFloat
from api.models.cmdb import CIValueInteger
from api.models.cmdb import CIValueText
from api.models.cmdb import FloatChoice
from api.models.cmdb import IntegerChoice
from api.models.cmdb import TextChoice
from api.lib.utils import BaseEnum
def string2int(x):
return int(float(x))
class ValueTypeEnum(BaseEnum):
INT = "0"
FLOAT = "1"
TEXT = "2"
DATETIME = "3"
DATE = "4"
TIME = "5"
def str2datetime(x):
try:
return datetime.datetime.strptime(x, "%Y-%m-%d")
except ValueError:
pass
return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")
class CIStatusEnum(BaseEnum):
REVIEW = "0"
VALIDATE = "1"
type_map = {
'deserialize': {
Attribute.INT: string2int,
Attribute.FLOAT: float,
Attribute.TEXT: lambda x: escape(x).encode('utf-8').decode('utf-8'),
Attribute.TIME: lambda x: escape(x).encode('utf-8').decode('utf-8'),
Attribute.DATETIME: str2datetime,
Attribute.DATE: str2datetime,
},
'serialize': {
Attribute.INT: int,
Attribute.FLOAT: float,
Attribute.TEXT: lambda x: x if isinstance(x, six.text_type) else str(x),
Attribute.TIME: lambda x: x if isinstance(x, six.text_type) else str(x),
Attribute.DATE: lambda x: x.strftime("%Y-%m-%d"),
Attribute.DATETIME: lambda x: x.strftime("%Y-%m-%d %H:%M:%S"),
},
'serialize2': {
Attribute.INT: int,
Attribute.FLOAT: float,
Attribute.TEXT: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
Attribute.TIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
Attribute.DATE: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
Attribute.DATETIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
},
'choice': {
Attribute.INT: IntegerChoice,
Attribute.FLOAT: FloatChoice,
Attribute.TEXT: TextChoice,
},
'table': {
Attribute.INT: CIValueInteger,
Attribute.TEXT: CIValueText,
Attribute.DATETIME: CIValueDateTime,
Attribute.DATE: CIValueDateTime,
Attribute.TIME: CIValueText,
Attribute.FLOAT: CIValueFloat,
'index_{0}'.format(Attribute.INT): CIIndexValueInteger,
'index_{0}'.format(Attribute.TEXT): CIIndexValueText,
'index_{0}'.format(Attribute.DATETIME): CIIndexValueDateTime,
'index_{0}'.format(Attribute.DATE): CIIndexValueDateTime,
'index_{0}'.format(Attribute.TIME): CIIndexValueText,
'index_{0}'.format(Attribute.FLOAT): CIIndexValueFloat,
},
'table_name': {
Attribute.INT: 'c_value_integers',
Attribute.TEXT: 'c_value_texts',
Attribute.DATETIME: 'c_value_datetime',
Attribute.DATE: 'c_value_datetime',
Attribute.TIME: 'c_value_texts',
Attribute.FLOAT: 'c_value_floats',
'index_{0}'.format(Attribute.INT): 'c_value_index_integers',
'index_{0}'.format(Attribute.TEXT): 'c_value_index_texts',
'index_{0}'.format(Attribute.DATETIME): 'c_value_index_datetime',
'index_{0}'.format(Attribute.DATE): 'c_value_index_datetime',
'index_{0}'.format(Attribute.TIME): 'c_value_index_texts',
'index_{0}'.format(Attribute.FLOAT): 'c_value_index_floats',
},
'es_type': {
Attribute.INT: 'long',
Attribute.TEXT: 'text',
Attribute.DATETIME: 'text',
Attribute.DATE: 'text',
Attribute.TIME: 'text',
Attribute.FLOAT: 'float'
}
}
class TableMap(object):
def __init__(self, attr_name=None):
self.attr_name = attr_name
@property
def table(self):
attr = AttributeCache.get(self.attr_name)
i = "index_{0}".format(attr.value_type) if attr.is_index else attr.value_type
return type_map["table"].get(i)
@property
def table_name(self):
attr = AttributeCache.get(self.attr_name)
i = "index_{0}".format(attr.value_type) if attr.is_index else attr.value_type
return type_map["table_name"].get(i)
class ExistPolicy(object):
class ExistPolicy(BaseEnum):
REJECT = "reject"
NEED = "need"
IGNORE = "ignore"
REPLACE = "replace"
class OperateType(object):
class OperateType(BaseEnum):
ADD = "0"
DELETE = "1"
UPDATE = "2"
class RetKey(object):
class RetKey(BaseEnum):
ID = "id"
NAME = "name"
ALIAS = "alias"
class ResourceType(object):
class ResourceType(BaseEnum):
CI = "CIType"
class PermEnum(object):
class PermEnum(BaseEnum):
ADD = "add"
UPDATE = "update"
DELETE = "delete"
READ = "read"
class RoleEnum(object):
class RoleEnum(BaseEnum):
CONFIG = "admin"
CMDB_QUEUE = "cmdb_async"
REDIS_PREFIX = "CMDB_CI"
REDIS_PREFIX_CI = "CMDB_CI"
REDIS_PREFIX_CI_RELATION = "CMDB_CI_RELATION"

3
api/lib/cmdb/history.py
View File

@ -7,6 +7,7 @@ from flask import g
from api.extensions import db
from api.lib.cmdb.cache import AttributeCache
from api.lib.cmdb.cache import RelationTypeCache
from api.lib.cmdb.const import OperateType
from api.lib.perm.acl.cache import UserCache
from api.models.cmdb import Attribute
from api.models.cmdb import AttributeHistory
@ -112,7 +113,7 @@ class AttributeHistoryManger(object):
class CIRelationHistoryManager(object):
@staticmethod
def add(rel_obj, operate_type=CIRelationHistory.ADD):
def add(rel_obj, operate_type=OperateType.ADD):
record = OperationRecord.create(uid=g.user.uid)
CIRelationHistory.create(relation_id=rel_obj.id,

10
api/lib/cmdb/search/__init__.py
View File

@ -1,3 +1,11 @@
# -*- coding:utf-8 -*-
__all__ = ['db', 'es']
__all__ = ['ci', 'ci_relation', 'SearchError']
class SearchError(Exception):
def __init__(self, v):
self.v = v
def __str__(self):
return self.v

3
api/lib/cmdb/search/ci/__init__.py Normal file
View File

@ -0,0 +1,3 @@
# -*- coding:utf-8 -*-
__all__ = ['db', 'es']

0
api/lib/cmdb/search/db/__init__.py → api/lib/cmdb/search/ci/db/__init__.py
View File

0
api/lib/cmdb/search/db/query_sql.py → api/lib/cmdb/search/ci/db/query_sql.py
View File

41
api/lib/cmdb/search/db/search.py → api/lib/cmdb/search/ci/db/search.py
View File

@ -12,25 +12,25 @@ from api.lib.cmdb.cache import AttributeCache
from api.lib.cmdb.cache import CITypeCache
from api.lib.cmdb.ci import CIManager
from api.lib.cmdb.const import RetKey
from api.lib.cmdb.const import TableMap
from api.lib.cmdb.search.db.query_sql import FACET_QUERY
from api.lib.cmdb.search.db.query_sql import QUERY_CI_BY_ATTR_NAME
from api.lib.cmdb.search.db.query_sql import QUERY_CI_BY_TYPE
from api.lib.cmdb.const import ValueTypeEnum
from api.lib.cmdb.search import SearchError
from api.lib.cmdb.search.ci.db.query_sql import FACET_QUERY
from api.lib.cmdb.search.ci.db.query_sql import QUERY_CI_BY_ATTR_NAME
from api.lib.cmdb.search.ci.db.query_sql import QUERY_CI_BY_TYPE
from api.lib.cmdb.utils import TableMap
from api.lib.utils import handle_arg_list
from api.models.cmdb import Attribute
from api.models.cmdb import CI
class SearchError(Exception):
def __init__(self, v):
self.v = v
def __str__(self):
return self.v
class Search(object):
def __init__(self, query=None, fl=None, facet_field=None, page=1, ret_key=RetKey.NAME, count=1, sort=None):
def __init__(self, query=None,
fl=None,
facet_field=None,
page=1,
ret_key=RetKey.NAME,
count=1,
sort=None,
ci_ids=None):
self.orig_query = query
self.fl = fl
self.facet_field = facet_field
@ -38,6 +38,7 @@ class Search(object):
self.ret_key = ret_key
self.count = count
self.sort = sort
self.ci_ids = ci_ids or []
self.query_sql = ""
self.type_id_list = []
self.only_type_query = False
@ -60,10 +61,10 @@ class Search(object):
operator, key = self._operator_proc(key)
if key in ('ci_type', 'type', '_type'):
return '_type', Attribute.TEXT, operator, None
return '_type', ValueTypeEnum.TEXT, operator, None
if key in ('id', 'ci_id', '_id'):
return '_id', Attribute.TEXT, operator, None
return '_id', ValueTypeEnum.TEXT, operator, None
attr = AttributeCache.get(key)
if attr:
@ -286,6 +287,13 @@ class Search(object):
alias += "AA"
return None, query_sql
def _filter_ids(self, query_sql):
if self.ci_ids:
return "SELECT * FROM ({0}) AS IN_QUERY WHERE IN_QUERY.ci_id in ({1})".format(
query_sql, ",".join(list(map(str, self.ci_ids))))
return query_sql
def _query_build_raw(self):
queries = handle_arg_list(self.orig_query)
@ -298,6 +306,7 @@ class Search(object):
s = time.time()
if query_sql:
query_sql = self._filter_ids(query_sql)
self.query_sql = query_sql
current_app.logger.debug(query_sql)
numfound, res = self._execute_sql(query_sql)

0
api/lib/cmdb/search/es/__init__.py → api/lib/cmdb/search/ci/es/__init__.py
View File

35
api/lib/cmdb/search/es/search.py → api/lib/cmdb/search/ci/es/search.py
View File

@ -8,20 +8,20 @@ from flask import current_app
from api.extensions import es
from api.lib.cmdb.cache import AttributeCache
from api.lib.cmdb.const import RetKey
from api.lib.cmdb.const import ValueTypeEnum
from api.lib.cmdb.search import SearchError
from api.lib.utils import handle_arg_list
from api.models.cmdb import Attribute
class SearchError(Exception):
def __init__(self, v):
self.v = v
def __str__(self):
return self.v
class Search(object):
def __init__(self, query=None, fl=None, facet_field=None, page=1, ret_key=RetKey.NAME, count=1, sort=None):
def __init__(self, query=None,
fl=None,
facet_field=None,
page=1,
ret_key=RetKey.NAME,
count=1,
sort=None,
ci_ids=None):
self.orig_query = query
self.fl = fl
self.facet_field = facet_field
@ -29,6 +29,7 @@ class Search(object):
self.ret_key = ret_key
self.count = count or current_app.config.get("DEFAULT_PAGE_COUNT")
self.sort = sort or "ci_id"
self.ci_ids = ci_ids or []
self.query = dict(query=dict(bool=dict(should=[], must=[], must_not=[])))
@ -58,10 +59,10 @@ class Search(object):
operator, key = self._operator_proc(key)
if key in ('ci_type', 'type', '_type'):
return 'ci_type', Attribute.TEXT, operator
return 'ci_type', ValueTypeEnum.TEXT, operator
if key in ('id', 'ci_id', '_id'):
return 'ci_id', Attribute.TEXT, operator
return 'ci_id', ValueTypeEnum.TEXT, operator
attr = AttributeCache.get(key)
if attr:
@ -79,6 +80,10 @@ class Search(object):
}
})
def _filter_ids(self):
if self.ci_ids:
self.query['query']['bool'].update(dict(filter=dict(terms=dict(ci_id=self.ci_ids))))
@staticmethod
def _digit(s):
if s.isdigit():
@ -171,6 +176,8 @@ class Search(object):
self._facet_build()
self._filter_ids()
return es.read(self.query, filter_path=filter_path)
def _facet_build(self):
@ -183,7 +190,7 @@ class Search(object):
field: {
"terms": {
"field": "{0}.keyword".format(field)
if attr.value_type not in (Attribute.INT, Attribute.FLOAT) else field
if attr.value_type not in (ValueTypeEnum.INT, ValueTypeEnum.FLOAT) else field
}
}
})
@ -212,7 +219,7 @@ class Search(object):
raise SearchError("Sort by <{0}> does not exist".format(field))
sort_by = "{0}.keyword".format(field) \
if attr.value_type not in (Attribute.INT, Attribute.FLOAT) else field
if attr.value_type not in (ValueTypeEnum.INT, ValueTypeEnum.FLOAT) else field
sorts.append({sort_by: {"order": sort_type}})
self.query.update(dict(sort=sorts))

1
api/lib/cmdb/search/ci_relation/__init__.py Normal file
View File

@ -0,0 +1 @@
# -*- coding:utf-8 -*-

56
api/lib/cmdb/search/ci_relation/search.py Normal file
View File

@ -0,0 +1,56 @@
# -*- coding:utf-8 -*-
import json
from flask import abort
from flask import current_app
from api.extensions import rd
from api.lib.cmdb.ci_type import CITypeRelationManager
from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
from api.lib.cmdb.search.ci.db.search import Search as SearchFromDB
from api.lib.cmdb.search.ci.es.search import Search as SearchFromES
from api.models.cmdb import CI
class Search(object):
def __init__(self, root_id, level=1, query=None, fl=None, facet_field=None, page=1, count=None, sort=None):
self.orig_query = query
self.fl = fl
self.facet_field = facet_field
self.page = page
self.count = count or current_app.config.get("DEFAULT_PAGE_COUNT")
self.sort = sort or ("ci_id" if current_app.config.get("USE_ES") else None)
self.root_id = root_id
self.level = int(level)
def search(self):
ci = CI.get_by_id(self.root_id) or abort(404, "CI <{0}> does not exist".format(self.root_id))
ids = [self.root_id]
for _ in range(0, self.level):
_tmp = list(map(json.loads, filter(lambda x: x is not None, rd.get(ids, REDIS_PREFIX_CI_RELATION))))
ids = [j for i in _tmp for j in i]
if not self.orig_query or ("_type:" not in self.orig_query
and "type_id:" not in self.orig_query
and "ci_type:" not in self.orig_query):
type_ids = CITypeRelationManager.get_child_type_ids(ci.type_id, self.level)
self.orig_query = "_type:({0}),{1}".format(";".join(list(map(str, type_ids))), self.orig_query)
if current_app.config.get("USE_ES"):
return SearchFromES(self.orig_query,
fl=self.fl,
facet_field=self.facet_field,
page=self.page,
count=self.count,
sort=self.sort,
ci_ids=ids).search()
else:
return SearchFromDB(self.orig_query,
fl=self.fl,
facet_field=self.facet_field,
page=self.page,
count=self.count,
sort=self.sort,
ci_ids=ids).search()

116
api/lib/cmdb/utils.py Normal file
View File

@ -0,0 +1,116 @@
# -*- coding:utf-8 -*-
from __future__ import unicode_literals
import datetime
import six
from markupsafe import escape
import api.models.cmdb as model
from api.lib.cmdb.cache import AttributeCache
from api.lib.cmdb.const import ValueTypeEnum
def string2int(x):
return int(float(x))
def str2datetime(x):
try:
return datetime.datetime.strptime(x, "%Y-%m-%d")
except ValueError:
pass
return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")
class ValueTypeMap(object):
deserialize = {
ValueTypeEnum.INT: string2int,
ValueTypeEnum.FLOAT: float,
ValueTypeEnum.TEXT: lambda x: escape(x).encode('utf-8').decode('utf-8'),
ValueTypeEnum.TIME: lambda x: escape(x).encode('utf-8').decode('utf-8'),
ValueTypeEnum.DATETIME: str2datetime,
ValueTypeEnum.DATE: str2datetime,
}
serialize = {
ValueTypeEnum.INT: int,
ValueTypeEnum.FLOAT: float,
ValueTypeEnum.TEXT: lambda x: x if isinstance(x, six.text_type) else str(x),
ValueTypeEnum.TIME: lambda x: x if isinstance(x, six.text_type) else str(x),
ValueTypeEnum.DATE: lambda x: x.strftime("%Y-%m-%d"),
ValueTypeEnum.DATETIME: lambda x: x.strftime("%Y-%m-%d %H:%M:%S"),
}
serialize2 = {
ValueTypeEnum.INT: int,
ValueTypeEnum.FLOAT: float,
ValueTypeEnum.TEXT: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
ValueTypeEnum.TIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
ValueTypeEnum.DATE: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
ValueTypeEnum.DATETIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
}
choice = {
ValueTypeEnum.INT: model.IntegerChoice,
ValueTypeEnum.FLOAT: model.FloatChoice,
ValueTypeEnum.TEXT: model.TextChoice,
}
table = {
ValueTypeEnum.INT: model.CIValueInteger,
ValueTypeEnum.TEXT: model.CIValueText,
ValueTypeEnum.DATETIME: model.CIValueDateTime,
ValueTypeEnum.DATE: model.CIValueDateTime,
ValueTypeEnum.TIME: model.CIValueText,
ValueTypeEnum.FLOAT: model.CIValueFloat,
'index_{0}'.format(ValueTypeEnum.INT): model.CIIndexValueInteger,
'index_{0}'.format(ValueTypeEnum.TEXT): model.CIIndexValueText,
'index_{0}'.format(ValueTypeEnum.DATETIME): model.CIIndexValueDateTime,
'index_{0}'.format(ValueTypeEnum.DATE): model.CIIndexValueDateTime,
'index_{0}'.format(ValueTypeEnum.TIME): model.CIIndexValueText,
'index_{0}'.format(ValueTypeEnum.FLOAT): model.CIIndexValueFloat,
}
table_name = {
ValueTypeEnum.INT: 'c_value_integers',
ValueTypeEnum.TEXT: 'c_value_texts',
ValueTypeEnum.DATETIME: 'c_value_datetime',
ValueTypeEnum.DATE: 'c_value_datetime',
ValueTypeEnum.TIME: 'c_value_texts',
ValueTypeEnum.FLOAT: 'c_value_floats',
'index_{0}'.format(ValueTypeEnum.INT): 'c_value_index_integers',
'index_{0}'.format(ValueTypeEnum.TEXT): 'c_value_index_texts',
'index_{0}'.format(ValueTypeEnum.DATETIME): 'c_value_index_datetime',
'index_{0}'.format(ValueTypeEnum.DATE): 'c_value_index_datetime',
'index_{0}'.format(ValueTypeEnum.TIME): 'c_value_index_texts',
'index_{0}'.format(ValueTypeEnum.FLOAT): 'c_value_index_floats',
}
es_type = {
ValueTypeEnum.INT: 'long',
ValueTypeEnum.TEXT: 'text',
ValueTypeEnum.DATETIME: 'text',
ValueTypeEnum.DATE: 'text',
ValueTypeEnum.TIME: 'text',
ValueTypeEnum.FLOAT: 'float'
}
class TableMap(object):
def __init__(self, attr_name=None):
self.attr_name = attr_name
@property
def table(self):
attr = AttributeCache.get(self.attr_name)
i = "index_{0}".format(attr.value_type) if attr.is_index else attr.value_type
return ValueTypeMap.table.get(i)
@property
def table_name(self):
attr = AttributeCache.get(self.attr_name)
i = "index_{0}".format(attr.value_type) if attr.is_index else attr.value_type
return ValueTypeMap.table_name.get(i)

15
api/lib/cmdb/value.py
View File

@ -3,7 +3,6 @@
from __future__ import unicode_literals
import markupsafe
from flask import abort
from api.extensions import db
@ -11,11 +10,11 @@ from api.lib.cmdb.attribute import AttributeManager
from api.lib.cmdb.cache import AttributeCache
from api.lib.cmdb.const import ExistPolicy
from api.lib.cmdb.const import OperateType
from api.lib.cmdb.const import TableMap
from api.lib.cmdb.const import type_map
from api.lib.cmdb.const import ValueTypeEnum
from api.lib.cmdb.history import AttributeHistoryManger
from api.lib.cmdb.utils import TableMap
from api.lib.cmdb.utils import ValueTypeMap
from api.lib.utils import handle_arg_list
from api.models.cmdb import Attribute
class AttributeValueManager(object):
@ -58,9 +57,9 @@ class AttributeValueManager(object):
field_name = getattr(attr, ret_key)
if attr.is_list:
res[field_name] = [type_map["serialize"][attr.value_type](i.value) for i in rs]
res[field_name] = [ValueTypeMap.serialize[attr.value_type](i.value) for i in rs]
else:
res[field_name] = type_map["serialize"][attr.value_type](rs[0].value) if rs else None
res[field_name] = ValueTypeMap.serialize[attr.value_type](rs[0].value) if rs else None
if unique_key is not None and attr.id == unique_key.id and rs:
res['unique'] = unique_key.name
@ -71,7 +70,7 @@ class AttributeValueManager(object):
def __deserialize_value(value_type, value):
if not value:
return value
deserialize = type_map["deserialize"][value_type]
deserialize = ValueTypeMap.deserialize[value_type]
try:
v = deserialize(value)
return v
@ -133,7 +132,7 @@ class AttributeValueManager(object):
for v in value_list:
v = self._validate(attr, v, value_table, ci_id)
if not v and attr.value_type != Attribute.TEXT:
if not v and attr.value_type != ValueTypeEnum.TEXT:
v = None
if operate_type == OperateType.ADD:

37
api/lib/utils.py
View File

@ -29,10 +29,27 @@ def handle_arg_list(arg):
return list(filter(lambda x: x != "", arg.strip().split(","))) if isinstance(arg, six.string_types) else arg
class BaseEnum(object):
_ALL_ = set() # type: Set[str]
@classmethod
def is_valid(cls, item):
return item in cls.all()
@classmethod
def all(cls):
if not cls._ALL_:
cls._ALL_ = {
getattr(cls, attr)
for attr in dir(cls)
if not attr.startswith("_") and not callable(getattr(cls, attr))
}
return cls._ALL_
class RedisHandler(object):
def __init__(self, flask_app=None, prefix=None):
def __init__(self, flask_app=None):
self.flask_app = flask_app
self.prefix = prefix
self.r = None
def init_app(self, app):
@ -49,26 +66,26 @@ class RedisHandler(object):
current_app.logger.warning(str(e))
current_app.logger.error("init redis connection failed")
def get(self, key_ids):
def get(self, key_ids, prefix):
try:
value = self.r.hmget(self.prefix, key_ids)
value = self.r.hmget(prefix, key_ids)
except Exception as e:
current_app.logger.error("get redis error, {0}".format(str(e)))
return
return value
def _set(self, obj):
def _set(self, obj, prefix):
try:
self.r.hmset(self.prefix, obj)
self.r.hmset(prefix, obj)
except Exception as e:
current_app.logger.error("set redis error, {0}".format(str(e)))
def add(self, obj):
self._set(obj)
def create_or_update(self, obj, prefix):
self._set(obj, prefix)
def delete(self, key_id):
def delete(self, key_id, prefix):
try:
ret = self.r.hdel(self.prefix, key_id)
ret = self.r.hdel(prefix, key_id)
if not ret:
current_app.logger.warn("[{0}] is not in redis".format(key_id))
except Exception as e:

28
api/models/cmdb.py
View File

@ -4,6 +4,9 @@
import datetime
from api.extensions import db
from api.lib.cmdb.const import CIStatusEnum
from api.lib.cmdb.const import OperateType
from api.lib.cmdb.const import ValueTypeEnum
from api.lib.database import Model
@ -58,16 +61,9 @@ class CITypeRelation(Model):
class Attribute(Model):
__tablename__ = "c_attributes"
INT = "0"
FLOAT = "1"
TEXT = "2"
DATETIME = "3"
DATE = "4"
TIME = "5"
name = db.Column(db.String(32), nullable=False)
alias = db.Column(db.String(32), nullable=False)
value_type = db.Column(db.Enum(INT, FLOAT, TEXT, DATETIME, DATE, TIME), default=TEXT, nullable=False)
value_type = db.Column(db.Enum(*ValueTypeEnum.all()), default=ValueTypeEnum.TEXT, nullable=False)
is_choice = db.Column(db.Boolean, default=False)
is_list = db.Column(db.Boolean, default=False)
@ -111,11 +107,8 @@ class CITypeAttributeGroupItem(Model):
class CI(Model):
__tablename__ = "c_cis"
REVIEW = "0"
VALIDATE = "1"
type_id = db.Column(db.Integer, db.ForeignKey("c_ci_types.id"), nullable=False)
status = db.Column(db.Enum(REVIEW, VALIDATE, name="status"))
status = db.Column(db.Enum(*CIStatusEnum.all(), name="status"))
heartbeat = db.Column(db.DateTime, default=lambda: datetime.datetime.now())
ci_type = db.relationship("CIType", backref="c_cis.type_id")
@ -270,11 +263,7 @@ class OperationRecord(Model):
class AttributeHistory(Model):
__tablename__ = "c_attribute_histories"
ADD = "0"
DELETE = "1"
UPDATE = "2"
operate_type = db.Column(db.Enum(ADD, DELETE, UPDATE, name="operate_type"))
operate_type = db.Column(db.Enum(*OperateType.all(), name="operate_type"))
record_id = db.Column(db.Integer, db.ForeignKey("c_records.id"), nullable=False)
ci_id = db.Column(db.Integer, index=True, nullable=False)
attr_id = db.Column(db.Integer, index=True)
@ -285,10 +274,7 @@ class AttributeHistory(Model):
class CIRelationHistory(Model):
__tablename__ = "c_relation_histories"
ADD = "0"
DELETE = "1"
operate_type = db.Column(db.Enum(ADD, DELETE, name="operate_type"))
operate_type = db.Column(db.Enum(OperateType.ADD, OperateType.DELETE, name="operate_type"))
record_id = db.Column(db.Integer, db.ForeignKey("c_records.id"), nullable=False)
first_ci_id = db.Column(db.Integer)
second_ci_id = db.Column(db.Integer)

36
api/tasks/cmdb.py
View File

@ -9,9 +9,11 @@ from flask import current_app
import api.lib.cmdb.ci
from api.extensions import celery
from api.extensions import db
from api.extensions import rd
from api.extensions import es
from api.extensions import rd
from api.lib.cmdb.const import CMDB_QUEUE
from api.lib.cmdb.const import REDIS_PREFIX_CI
from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
@celery.task(name="cmdb.ci_cache", queue=CMDB_QUEUE)
@ -24,10 +26,9 @@ def ci_cache(ci_id):
if current_app.config.get("USE_ES"):
es.update(ci_id, ci)
else:
rd.delete(ci_id)
rd.add({ci_id: json.dumps(ci)})
rd.create_or_update({ci_id: json.dumps(ci)}, REDIS_PREFIX_CI)
current_app.logger.info("%d flush.........." % ci_id)
current_app.logger.info("{0} flush..........".format(ci_id))
@celery.task(name="cmdb.ci_delete", queue=CMDB_QUEUE)
@ -37,6 +38,29 @@ def ci_delete(ci_id):
if current_app.config.get("USE_ES"):
es.delete(ci_id)
else:
rd.delete(ci_id)
rd.delete(ci_id, REDIS_PREFIX_CI)
current_app.logger.info("%d delete.........." % ci_id)
current_app.logger.info("{0} delete..........".format(ci_id))
@celery.task(name="cmdb.ci_relation_cache", queue=CMDB_QUEUE)
def ci_relation_cache(parent_id, child_id):
children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0]
children = json.loads(children) if children is not None else []
children.append(child_id)
rd.create_or_update({parent_id: json.dumps(list(set(children)))}, REDIS_PREFIX_CI_RELATION)
current_app.logger.info("ADD ci relation cache: {0} -> {1}".format(parent_id, child_id))
@celery.task(name="cmdb.ci_relation_delete", queue=CMDB_QUEUE)
def ci_relation_delete(parent_id, child_id):
children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0]
children = json.loads(children) if children is not None else []
if child_id in children:
children.remove(child_id)
rd.create_or_update({parent_id: json.dumps(list(set(children)))}, REDIS_PREFIX_CI_RELATION)
current_app.logger.info("DELETE ci relation cache: {0} -> {1}".format(parent_id, child_id))

12
api/views/cmdb/ci.py
View File

@ -12,9 +12,9 @@ from api.lib.cmdb.ci import CIManager
from api.lib.cmdb.const import ExistPolicy
from api.lib.cmdb.const import ResourceType, PermEnum
from api.lib.cmdb.const import RetKey
from api.lib.cmdb.search.db.search import Search as SearchFromDB
from api.lib.cmdb.search.es.search import Search as SearchFromES
from api.lib.cmdb.search.db.search import SearchError
from api.lib.cmdb.search import SearchError
from api.lib.cmdb.search.ci.db.search import Search as SearchFromDB
from api.lib.cmdb.search.ci.es.search import Search as SearchFromES
from api.lib.perm.acl.acl import has_perm_from_args
from api.lib.perm.auth import auth_abandoned
from api.lib.utils import get_page
@ -126,13 +126,13 @@ class CISearchView(APIView):
def get(self):
"""@params: q: query statement
fl: filter by column
count: the number of ci
count/page_size: the number of ci
ret_key: id, name, alias
facet: statistic
"""
page = get_page(request.values.get("page", 1))
count = get_page_size(request.values.get("count"))
count = get_page_size(request.values.get("count") or request.values.get("page_size"))
query = request.values.get('q', "")
fl = handle_arg_list(request.values.get('fl', ""))
@ -140,8 +140,6 @@ class CISearchView(APIView):
if ret_key not in (RetKey.NAME, RetKey.ALIAS, RetKey.ID):
ret_key = RetKey.NAME
facet = handle_arg_list(request.values.get("facet", ""))
fl = list(filter(lambda x: x != "", fl))
facet = list(filter(lambda x: x != "", facet))
sort = request.values.get("sort")
start = time.time()

48
api/views/cmdb/ci_relation.py
View File

@ -1,14 +1,62 @@
# -*- coding:utf-8 -*-
import time
from flask import abort
from flask import current_app
from flask import request
from api.lib.cmdb.ci import CIRelationManager
from api.lib.cmdb.search import SearchError
from api.lib.cmdb.search.ci_relation.search import Search
from api.lib.perm.auth import auth_abandoned
from api.lib.utils import get_page
from api.lib.utils import get_page_size
from api.lib.utils import handle_arg_list
from api.resource import APIView
class CIRelationSearchView(APIView):
url_prefix = ("/ci_relations/s", "/ci_relations/search")
@auth_abandoned
def get(self):
"""@params: q: query statement
fl: filter by column
count: the number of ci
root_id: ci id
level: default is 1
facet: statistic
"""
page = get_page(request.values.get("page", 1))
count = get_page_size(request.values.get("count") or request.values.get("page_size"))
root_id = request.values.get('root_id')
level = request.values.get('level', 1)
query = request.values.get('q', "")
fl = handle_arg_list(request.values.get('fl', ""))
facet = handle_arg_list(request.values.get("facet", ""))
sort = request.values.get("sort")
start = time.time()
s = Search(root_id, level, query, fl, facet, page, count, sort)
try:
response, counter, total, page, numfound, facet = s.search()
except SearchError as e:
return abort(400, str(e))
current_app.logger.debug("search time is :{0}".format(time.time() - start))
return self.jsonify(numfound=numfound,
total=total,
page=page,
facet=facet,
counter=counter,
result=response)
class GetSecondCIsView(APIView):
url_prefix = "/ci_relations/<int:first_ci_id>/second_cis"

11
api/views/cmdb/ci_type_relation.py
View File

@ -25,18 +25,26 @@ class GetParentsView(APIView):
class CITypeRelationView(APIView):
url_prefix = "/ci_type_relations/<int:parent_id>/<int:child_id>"
url_prefix = ("/ci_type_relations", "/ci_type_relations/<int:parent_id>/<int:child_id>")
@role_required(RoleEnum.CONFIG)
def get(self):
res = CITypeRelationManager.get()
return self.jsonify(res)
@role_required(RoleEnum.CONFIG)
@args_required("relation_type_id")
def post(self, parent_id, child_id):
relation_type_id = request.values.get("relation_type_id")
ctr_id = CITypeRelationManager.add(parent_id, child_id, relation_type_id)
return self.jsonify(ctr_id=ctr_id)
@role_required(RoleEnum.CONFIG)
def delete(self, parent_id, child_id):
CITypeRelationManager.delete_2(parent_id, child_id)
return self.jsonify(code=200, parent_id=parent_id, child_id=child_id)
@ -46,4 +54,5 @@ class CITypeRelationDelete2View(APIView):
@role_required(RoleEnum.CONFIG)
def delete(self, ctr_id):
CITypeRelationManager.delete(ctr_id)
return self.jsonify(code=200, ctr_id=ctr_id)