diff --git a/api/commands/click_cmdb.py b/api/commands/click_cmdb.py index 0bfab82..f4bcac7 100644 --- a/api/commands/click_cmdb.py +++ b/api/commands/click_cmdb.py @@ -10,7 +10,11 @@ from flask.cli import with_appcontext import api.lib.cmdb.ci from api.extensions import db from api.extensions import rd +from api.lib.cmdb.const import REDIS_PREFIX_CI +from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION +from api.lib.cmdb.const import ValueTypeEnum from api.models.cmdb import CI +from api.models.cmdb import CIRelation @click.command() @@ -21,12 +25,12 @@ def init_cache(): if current_app.config.get("USE_ES"): from api.extensions import es from api.models.cmdb import Attribute - from api.lib.cmdb.const import type_map + from api.lib.cmdb.utils import ValueTypeMap attributes = Attribute.get_by(to_dict=False) for attr in attributes: other = dict() other['index'] = True if attr.is_index else False - if attr.value_type == Attribute.TEXT: + if attr.value_type == ValueTypeEnum.TEXT: other['analyzer'] = 'ik_max_word' other['search_analyzer'] = 'ik_smart' if attr.is_index: @@ -37,7 +41,7 @@ def init_cache(): } } try: - es.update_mapping(attr.name, type_map['es_type'][attr.value_type], other) + es.update_mapping(attr.name, ValueTypeMap.es_type[attr.value_type], other) except Exception as e: print(e) @@ -48,7 +52,7 @@ def init_cache(): if res: continue else: - res = rd.get([ci.id]) + res = rd.get([ci.id], REDIS_PREFIX_CI) if res and list(filter(lambda x: x, res)): continue @@ -58,7 +62,15 @@ def init_cache(): if current_app.config.get("USE_ES"): es.create(ci_dict) else: - rd.delete(ci.id) - rd.add({ci.id: json.dumps(ci_dict)}) + rd.create_or_update({ci.id: json.dumps(ci_dict)}, REDIS_PREFIX_CI) + + ci_relations = CIRelation.get_by(to_dict=False) + relations = dict() + for cr in ci_relations: + relations.setdefault(cr.first_ci_id, []).append(cr.second_ci_id) + for i in relations: + relations[i] = json.dumps(relations[i]) + if relations: + rd.create_or_update(relations, REDIS_PREFIX_CI_RELATION) db.session.remove() diff --git a/api/extensions.py b/api/extensions.py index 45cf662..b23005e 100644 --- a/api/extensions.py +++ b/api/extensions.py @@ -19,5 +19,5 @@ migrate = Migrate() cache = Cache() celery = Celery() cors = CORS(supports_credentials=True) -rd = RedisHandler(prefix="CMDB_CI") # TODO +rd = RedisHandler() es = ESHandler() diff --git a/api/lib/cmdb/attribute.py b/api/lib/cmdb/attribute.py index cfef2b4..06f034c 100644 --- a/api/lib/cmdb/attribute.py +++ b/api/lib/cmdb/attribute.py @@ -5,7 +5,8 @@ from flask import current_app from api.extensions import db from api.lib.cmdb.cache import AttributeCache -from api.lib.cmdb.const import type_map +from api.lib.cmdb.const import ValueTypeEnum +from api.lib.cmdb.utils import ValueTypeMap from api.lib.decorator import kwargs_required from api.models.cmdb import Attribute from api.models.cmdb import CITypeAttribute @@ -22,13 +23,13 @@ class AttributeManager(object): @staticmethod def get_choice_values(attr_id, value_type): - choice_table = type_map.get("choice").get(value_type) + choice_table = ValueTypeMap.choice.get(value_type) choice_values = choice_table.get_by(fl=["value"], attr_id=attr_id) return [choice_value["value"] for choice_value in choice_values] @staticmethod def _add_choice_values(_id, value_type, choice_values): - choice_table = type_map.get("choice").get(value_type) + choice_table = ValueTypeMap.choice.get(value_type) db.session.query(choice_table).filter(choice_table.attr_id == _id).delete() db.session.flush() @@ -121,7 +122,7 @@ class AttributeManager(object): from api.extensions import es other = dict() other['index'] = True if attr.is_index else False - if attr.value_type == Attribute.TEXT: + if attr.value_type == ValueTypeEnum.TEXT: other['analyzer'] = 'ik_max_word' other['search_analyzer'] = 'ik_smart' if attr.is_index: @@ -131,7 +132,7 @@ class AttributeManager(object): "ignore_above": 256 } } - es.update_mapping(name, type_map['es_type'][attr.value_type], other) + es.update_mapping(name, ValueTypeMap.es_type[attr.value_type], other) return attr.id @@ -172,7 +173,7 @@ class AttributeManager(object): name = attr.name if attr.is_choice: - choice_table = type_map["choice"].get(attr.value_type) + choice_table = ValueTypeMap.choice.get(attr.value_type) db.session.query(choice_table).filter(choice_table.attr_id == _id).delete() # FIXME: session conflict db.session.flush() diff --git a/api/lib/cmdb/ci.py b/api/lib/cmdb/ci.py index 1c1ec37..d0bb94e 100644 --- a/api/lib/cmdb/ci.py +++ b/api/lib/cmdb/ci.py @@ -18,13 +18,14 @@ from api.lib.cmdb.ci_type import CITypeManager from api.lib.cmdb.const import CMDB_QUEUE from api.lib.cmdb.const import ExistPolicy from api.lib.cmdb.const import OperateType +from api.lib.cmdb.const import REDIS_PREFIX_CI from api.lib.cmdb.const import RetKey -from api.lib.cmdb.const import TableMap -from api.lib.cmdb.const import type_map from api.lib.cmdb.history import AttributeHistoryManger from api.lib.cmdb.history import CIRelationHistoryManager -from api.lib.cmdb.search.db.query_sql import QUERY_CIS_BY_IDS -from api.lib.cmdb.search.db.query_sql import QUERY_CIS_BY_VALUE_TABLE +from api.lib.cmdb.search.ci.db.query_sql import QUERY_CIS_BY_IDS +from api.lib.cmdb.search.ci.db.query_sql import QUERY_CIS_BY_VALUE_TABLE +from api.lib.cmdb.utils import TableMap +from api.lib.cmdb.utils import ValueTypeMap from api.lib.cmdb.value import AttributeValueManager from api.lib.decorator import kwargs_required from api.lib.utils import handle_arg_list @@ -33,6 +34,8 @@ from api.models.cmdb import CIRelation from api.models.cmdb import CITypeAttribute from api.tasks.cmdb import ci_cache from api.tasks.cmdb import ci_delete +from api.tasks.cmdb import ci_relation_cache +from api.tasks.cmdb import ci_relation_delete class CIManager(object): @@ -302,7 +305,7 @@ class CIManager(object): @staticmethod def _get_cis_from_cache(ci_ids, ret_key=RetKey.NAME, fields=None): - res = rd.get(ci_ids) + res = rd.get(ci_ids, REDIS_PREFIX_CI) if res is not None and None not in res and ret_key == RetKey.NAME: res = list(map(json.loads, res)) if not fields: @@ -332,7 +335,7 @@ class CIManager(object): ci_ids = ",".join(ci_ids) if value_tables is None: - value_tables = type_map["table_name"].values() + value_tables = ValueTypeMap.table_name.values() value_sql = " UNION ".join([QUERY_CIS_BY_VALUE_TABLE.format(value_table, ci_ids) for value_table in value_tables]) @@ -362,7 +365,7 @@ class CIManager(object): else: return abort(400, "invalid ret key") - value = type_map["serialize2"][value_type](value) + value = ValueTypeMap.serialize2[value_type](value) if is_list: ci_dict.setdefault(attr_key, []).append(value) else: @@ -532,6 +535,9 @@ class CIRelationManager(object): second_ci_id=second_ci_id, relation_type_id=relation_type_id) CIRelationHistoryManager().add(existed, OperateType.ADD) + + ci_relation_cache.apply_async(args=(first_ci_id, second_ci_id), queue=CMDB_QUEUE) + if more is not None: existed.upadte(more=more) @@ -545,6 +551,8 @@ class CIRelationManager(object): his_manager = CIRelationHistoryManager() his_manager.add(cr, operate_type=OperateType.DELETE) + ci_relation_delete.apply_async(args=(cr.first_ci_id, cr.second_ci_id), queue=CMDB_QUEUE) + return cr_id @classmethod @@ -553,4 +561,7 @@ class CIRelationManager(object): second_ci_id=second_ci_id, to_dict=False, first=True) + + ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id), queue=CMDB_QUEUE) + return cls.delete(cr.cr_id) diff --git a/api/lib/cmdb/ci_type.py b/api/lib/cmdb/ci_type.py index f0b4e29..7774a60 100644 --- a/api/lib/cmdb/ci_type.py +++ b/api/lib/cmdb/ci_type.py @@ -4,6 +4,7 @@ from flask import abort from flask import current_app +from api.extensions import db from api.lib.cmdb.attribute import AttributeManager from api.lib.cmdb.cache import AttributeCache from api.lib.cmdb.cache import CITypeAttributeCache @@ -263,8 +264,26 @@ class CITypeRelationManager(object): manage relation between CITypes """ - def __init__(self): - pass + @staticmethod + def get(): + res = CITypeRelation.get_by(to_dict=False) + for idx, item in enumerate(res): + _item = item.to_dict() + res[idx] = _item + res[idx]['parent'] = item.parent.to_dict() + res[idx]['child'] = item.child.to_dict() + res[idx]['relation_type'] = item.relation_type.to_dict() + + return res + + @staticmethod + def get_child_type_ids(type_id, level): + ids = [type_id] + query = db.session.query(CITypeRelation).filter(CITypeRelation.deleted.is_(False)) + for _ in range(0, level): + ids = [i.child_id for i in query.filter(CITypeRelation.parent_id.in_(ids))] + + return ids @staticmethod def _wrap_relation_type_dict(type_id, relation_inst): diff --git a/api/lib/cmdb/const.py b/api/lib/cmdb/const.py index 511c599..98263b9 100644 --- a/api/lib/cmdb/const.py +++ b/api/lib/cmdb/const.py @@ -1,160 +1,57 @@ # -*- coding:utf-8 -*- -from __future__ import unicode_literals - -import datetime - -import six -from markupsafe import escape - -from api.lib.cmdb.cache import AttributeCache -from api.models.cmdb import Attribute -from api.models.cmdb import CIIndexValueDateTime -from api.models.cmdb import CIIndexValueFloat -from api.models.cmdb import CIIndexValueInteger -from api.models.cmdb import CIIndexValueText -from api.models.cmdb import CIValueDateTime -from api.models.cmdb import CIValueFloat -from api.models.cmdb import CIValueInteger -from api.models.cmdb import CIValueText -from api.models.cmdb import FloatChoice -from api.models.cmdb import IntegerChoice -from api.models.cmdb import TextChoice +from api.lib.utils import BaseEnum -def string2int(x): - return int(float(x)) +class ValueTypeEnum(BaseEnum): + INT = "0" + FLOAT = "1" + TEXT = "2" + DATETIME = "3" + DATE = "4" + TIME = "5" -def str2datetime(x): - try: - return datetime.datetime.strptime(x, "%Y-%m-%d") - except ValueError: - pass - - return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S") +class CIStatusEnum(BaseEnum): + REVIEW = "0" + VALIDATE = "1" -type_map = { - 'deserialize': { - Attribute.INT: string2int, - Attribute.FLOAT: float, - Attribute.TEXT: lambda x: escape(x).encode('utf-8').decode('utf-8'), - Attribute.TIME: lambda x: escape(x).encode('utf-8').decode('utf-8'), - Attribute.DATETIME: str2datetime, - Attribute.DATE: str2datetime, - }, - 'serialize': { - Attribute.INT: int, - Attribute.FLOAT: float, - Attribute.TEXT: lambda x: x if isinstance(x, six.text_type) else str(x), - Attribute.TIME: lambda x: x if isinstance(x, six.text_type) else str(x), - Attribute.DATE: lambda x: x.strftime("%Y-%m-%d"), - Attribute.DATETIME: lambda x: x.strftime("%Y-%m-%d %H:%M:%S"), - }, - 'serialize2': { - Attribute.INT: int, - Attribute.FLOAT: float, - Attribute.TEXT: lambda x: x.decode() if not isinstance(x, six.string_types) else x, - Attribute.TIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x, - Attribute.DATE: lambda x: x.decode() if not isinstance(x, six.string_types) else x, - Attribute.DATETIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x, - }, - 'choice': { - Attribute.INT: IntegerChoice, - Attribute.FLOAT: FloatChoice, - Attribute.TEXT: TextChoice, - }, - 'table': { - Attribute.INT: CIValueInteger, - Attribute.TEXT: CIValueText, - Attribute.DATETIME: CIValueDateTime, - Attribute.DATE: CIValueDateTime, - Attribute.TIME: CIValueText, - Attribute.FLOAT: CIValueFloat, - 'index_{0}'.format(Attribute.INT): CIIndexValueInteger, - 'index_{0}'.format(Attribute.TEXT): CIIndexValueText, - 'index_{0}'.format(Attribute.DATETIME): CIIndexValueDateTime, - 'index_{0}'.format(Attribute.DATE): CIIndexValueDateTime, - 'index_{0}'.format(Attribute.TIME): CIIndexValueText, - 'index_{0}'.format(Attribute.FLOAT): CIIndexValueFloat, - }, - 'table_name': { - Attribute.INT: 'c_value_integers', - Attribute.TEXT: 'c_value_texts', - Attribute.DATETIME: 'c_value_datetime', - Attribute.DATE: 'c_value_datetime', - Attribute.TIME: 'c_value_texts', - Attribute.FLOAT: 'c_value_floats', - 'index_{0}'.format(Attribute.INT): 'c_value_index_integers', - 'index_{0}'.format(Attribute.TEXT): 'c_value_index_texts', - 'index_{0}'.format(Attribute.DATETIME): 'c_value_index_datetime', - 'index_{0}'.format(Attribute.DATE): 'c_value_index_datetime', - 'index_{0}'.format(Attribute.TIME): 'c_value_index_texts', - 'index_{0}'.format(Attribute.FLOAT): 'c_value_index_floats', - }, - 'es_type': { - Attribute.INT: 'long', - Attribute.TEXT: 'text', - Attribute.DATETIME: 'text', - Attribute.DATE: 'text', - Attribute.TIME: 'text', - Attribute.FLOAT: 'float' - } -} - - -class TableMap(object): - def __init__(self, attr_name=None): - self.attr_name = attr_name - - @property - def table(self): - attr = AttributeCache.get(self.attr_name) - i = "index_{0}".format(attr.value_type) if attr.is_index else attr.value_type - return type_map["table"].get(i) - - @property - def table_name(self): - attr = AttributeCache.get(self.attr_name) - i = "index_{0}".format(attr.value_type) if attr.is_index else attr.value_type - return type_map["table_name"].get(i) - - -class ExistPolicy(object): +class ExistPolicy(BaseEnum): REJECT = "reject" NEED = "need" IGNORE = "ignore" REPLACE = "replace" -class OperateType(object): +class OperateType(BaseEnum): ADD = "0" DELETE = "1" UPDATE = "2" -class RetKey(object): +class RetKey(BaseEnum): ID = "id" NAME = "name" ALIAS = "alias" -class ResourceType(object): +class ResourceType(BaseEnum): CI = "CIType" -class PermEnum(object): +class PermEnum(BaseEnum): ADD = "add" UPDATE = "update" DELETE = "delete" READ = "read" -class RoleEnum(object): +class RoleEnum(BaseEnum): CONFIG = "admin" CMDB_QUEUE = "cmdb_async" -REDIS_PREFIX = "CMDB_CI" +REDIS_PREFIX_CI = "CMDB_CI" +REDIS_PREFIX_CI_RELATION = "CMDB_CI_RELATION" diff --git a/api/lib/cmdb/history.py b/api/lib/cmdb/history.py index e21bc08..de820d0 100644 --- a/api/lib/cmdb/history.py +++ b/api/lib/cmdb/history.py @@ -7,6 +7,7 @@ from flask import g from api.extensions import db from api.lib.cmdb.cache import AttributeCache from api.lib.cmdb.cache import RelationTypeCache +from api.lib.cmdb.const import OperateType from api.lib.perm.acl.cache import UserCache from api.models.cmdb import Attribute from api.models.cmdb import AttributeHistory @@ -112,7 +113,7 @@ class AttributeHistoryManger(object): class CIRelationHistoryManager(object): @staticmethod - def add(rel_obj, operate_type=CIRelationHistory.ADD): + def add(rel_obj, operate_type=OperateType.ADD): record = OperationRecord.create(uid=g.user.uid) CIRelationHistory.create(relation_id=rel_obj.id, diff --git a/api/lib/cmdb/search/__init__.py b/api/lib/cmdb/search/__init__.py index 3c49dd7..5a40ad0 100644 --- a/api/lib/cmdb/search/__init__.py +++ b/api/lib/cmdb/search/__init__.py @@ -1,3 +1,11 @@ # -*- coding:utf-8 -*- -__all__ = ['db', 'es'] +__all__ = ['ci', 'ci_relation', 'SearchError'] + + +class SearchError(Exception): + def __init__(self, v): + self.v = v + + def __str__(self): + return self.v diff --git a/api/lib/cmdb/search/ci/__init__.py b/api/lib/cmdb/search/ci/__init__.py new file mode 100644 index 0000000..3c49dd7 --- /dev/null +++ b/api/lib/cmdb/search/ci/__init__.py @@ -0,0 +1,3 @@ +# -*- coding:utf-8 -*- + +__all__ = ['db', 'es'] diff --git a/api/lib/cmdb/search/db/__init__.py b/api/lib/cmdb/search/ci/db/__init__.py similarity index 100% rename from api/lib/cmdb/search/db/__init__.py rename to api/lib/cmdb/search/ci/db/__init__.py diff --git a/api/lib/cmdb/search/db/query_sql.py b/api/lib/cmdb/search/ci/db/query_sql.py similarity index 100% rename from api/lib/cmdb/search/db/query_sql.py rename to api/lib/cmdb/search/ci/db/query_sql.py diff --git a/api/lib/cmdb/search/db/search.py b/api/lib/cmdb/search/ci/db/search.py similarity index 92% rename from api/lib/cmdb/search/db/search.py rename to api/lib/cmdb/search/ci/db/search.py index eeff10f..b0cd266 100644 --- a/api/lib/cmdb/search/db/search.py +++ b/api/lib/cmdb/search/ci/db/search.py @@ -12,25 +12,25 @@ from api.lib.cmdb.cache import AttributeCache from api.lib.cmdb.cache import CITypeCache from api.lib.cmdb.ci import CIManager from api.lib.cmdb.const import RetKey -from api.lib.cmdb.const import TableMap -from api.lib.cmdb.search.db.query_sql import FACET_QUERY -from api.lib.cmdb.search.db.query_sql import QUERY_CI_BY_ATTR_NAME -from api.lib.cmdb.search.db.query_sql import QUERY_CI_BY_TYPE +from api.lib.cmdb.const import ValueTypeEnum +from api.lib.cmdb.search import SearchError +from api.lib.cmdb.search.ci.db.query_sql import FACET_QUERY +from api.lib.cmdb.search.ci.db.query_sql import QUERY_CI_BY_ATTR_NAME +from api.lib.cmdb.search.ci.db.query_sql import QUERY_CI_BY_TYPE +from api.lib.cmdb.utils import TableMap from api.lib.utils import handle_arg_list -from api.models.cmdb import Attribute from api.models.cmdb import CI -class SearchError(Exception): - def __init__(self, v): - self.v = v - - def __str__(self): - return self.v - - class Search(object): - def __init__(self, query=None, fl=None, facet_field=None, page=1, ret_key=RetKey.NAME, count=1, sort=None): + def __init__(self, query=None, + fl=None, + facet_field=None, + page=1, + ret_key=RetKey.NAME, + count=1, + sort=None, + ci_ids=None): self.orig_query = query self.fl = fl self.facet_field = facet_field @@ -38,6 +38,7 @@ class Search(object): self.ret_key = ret_key self.count = count self.sort = sort + self.ci_ids = ci_ids or [] self.query_sql = "" self.type_id_list = [] self.only_type_query = False @@ -60,10 +61,10 @@ class Search(object): operator, key = self._operator_proc(key) if key in ('ci_type', 'type', '_type'): - return '_type', Attribute.TEXT, operator, None + return '_type', ValueTypeEnum.TEXT, operator, None if key in ('id', 'ci_id', '_id'): - return '_id', Attribute.TEXT, operator, None + return '_id', ValueTypeEnum.TEXT, operator, None attr = AttributeCache.get(key) if attr: @@ -286,6 +287,13 @@ class Search(object): alias += "AA" return None, query_sql + def _filter_ids(self, query_sql): + if self.ci_ids: + return "SELECT * FROM ({0}) AS IN_QUERY WHERE IN_QUERY.ci_id in ({1})".format( + query_sql, ",".join(list(map(str, self.ci_ids)))) + + return query_sql + def _query_build_raw(self): queries = handle_arg_list(self.orig_query) @@ -298,6 +306,7 @@ class Search(object): s = time.time() if query_sql: + query_sql = self._filter_ids(query_sql) self.query_sql = query_sql current_app.logger.debug(query_sql) numfound, res = self._execute_sql(query_sql) diff --git a/api/lib/cmdb/search/es/__init__.py b/api/lib/cmdb/search/ci/es/__init__.py similarity index 100% rename from api/lib/cmdb/search/es/__init__.py rename to api/lib/cmdb/search/ci/es/__init__.py diff --git a/api/lib/cmdb/search/es/search.py b/api/lib/cmdb/search/ci/es/search.py similarity index 89% rename from api/lib/cmdb/search/es/search.py rename to api/lib/cmdb/search/ci/es/search.py index 1650fd0..246b6ec 100644 --- a/api/lib/cmdb/search/es/search.py +++ b/api/lib/cmdb/search/ci/es/search.py @@ -8,20 +8,20 @@ from flask import current_app from api.extensions import es from api.lib.cmdb.cache import AttributeCache from api.lib.cmdb.const import RetKey +from api.lib.cmdb.const import ValueTypeEnum +from api.lib.cmdb.search import SearchError from api.lib.utils import handle_arg_list -from api.models.cmdb import Attribute - - -class SearchError(Exception): - def __init__(self, v): - self.v = v - - def __str__(self): - return self.v class Search(object): - def __init__(self, query=None, fl=None, facet_field=None, page=1, ret_key=RetKey.NAME, count=1, sort=None): + def __init__(self, query=None, + fl=None, + facet_field=None, + page=1, + ret_key=RetKey.NAME, + count=1, + sort=None, + ci_ids=None): self.orig_query = query self.fl = fl self.facet_field = facet_field @@ -29,6 +29,7 @@ class Search(object): self.ret_key = ret_key self.count = count or current_app.config.get("DEFAULT_PAGE_COUNT") self.sort = sort or "ci_id" + self.ci_ids = ci_ids or [] self.query = dict(query=dict(bool=dict(should=[], must=[], must_not=[]))) @@ -58,10 +59,10 @@ class Search(object): operator, key = self._operator_proc(key) if key in ('ci_type', 'type', '_type'): - return 'ci_type', Attribute.TEXT, operator + return 'ci_type', ValueTypeEnum.TEXT, operator if key in ('id', 'ci_id', '_id'): - return 'ci_id', Attribute.TEXT, operator + return 'ci_id', ValueTypeEnum.TEXT, operator attr = AttributeCache.get(key) if attr: @@ -79,6 +80,10 @@ class Search(object): } }) + def _filter_ids(self): + if self.ci_ids: + self.query['query']['bool'].update(dict(filter=dict(terms=dict(ci_id=self.ci_ids)))) + @staticmethod def _digit(s): if s.isdigit(): @@ -171,6 +176,8 @@ class Search(object): self._facet_build() + self._filter_ids() + return es.read(self.query, filter_path=filter_path) def _facet_build(self): @@ -183,7 +190,7 @@ class Search(object): field: { "terms": { "field": "{0}.keyword".format(field) - if attr.value_type not in (Attribute.INT, Attribute.FLOAT) else field + if attr.value_type not in (ValueTypeEnum.INT, ValueTypeEnum.FLOAT) else field } } }) @@ -212,7 +219,7 @@ class Search(object): raise SearchError("Sort by <{0}> does not exist".format(field)) sort_by = "{0}.keyword".format(field) \ - if attr.value_type not in (Attribute.INT, Attribute.FLOAT) else field + if attr.value_type not in (ValueTypeEnum.INT, ValueTypeEnum.FLOAT) else field sorts.append({sort_by: {"order": sort_type}}) self.query.update(dict(sort=sorts)) diff --git a/api/lib/cmdb/search/ci_relation/__init__.py b/api/lib/cmdb/search/ci_relation/__init__.py new file mode 100644 index 0000000..380474e --- /dev/null +++ b/api/lib/cmdb/search/ci_relation/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/api/lib/cmdb/search/ci_relation/search.py b/api/lib/cmdb/search/ci_relation/search.py new file mode 100644 index 0000000..6536a01 --- /dev/null +++ b/api/lib/cmdb/search/ci_relation/search.py @@ -0,0 +1,56 @@ +# -*- coding:utf-8 -*- + + +import json + +from flask import abort +from flask import current_app + +from api.extensions import rd +from api.lib.cmdb.ci_type import CITypeRelationManager +from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION +from api.lib.cmdb.search.ci.db.search import Search as SearchFromDB +from api.lib.cmdb.search.ci.es.search import Search as SearchFromES +from api.models.cmdb import CI + + +class Search(object): + def __init__(self, root_id, level=1, query=None, fl=None, facet_field=None, page=1, count=None, sort=None): + self.orig_query = query + self.fl = fl + self.facet_field = facet_field + self.page = page + self.count = count or current_app.config.get("DEFAULT_PAGE_COUNT") + self.sort = sort or ("ci_id" if current_app.config.get("USE_ES") else None) + + self.root_id = root_id + self.level = int(level) + + def search(self): + ci = CI.get_by_id(self.root_id) or abort(404, "CI <{0}> does not exist".format(self.root_id)) + ids = [self.root_id] + for _ in range(0, self.level): + _tmp = list(map(json.loads, filter(lambda x: x is not None, rd.get(ids, REDIS_PREFIX_CI_RELATION)))) + ids = [j for i in _tmp for j in i] + if not self.orig_query or ("_type:" not in self.orig_query + and "type_id:" not in self.orig_query + and "ci_type:" not in self.orig_query): + type_ids = CITypeRelationManager.get_child_type_ids(ci.type_id, self.level) + self.orig_query = "_type:({0}),{1}".format(";".join(list(map(str, type_ids))), self.orig_query) + + if current_app.config.get("USE_ES"): + return SearchFromES(self.orig_query, + fl=self.fl, + facet_field=self.facet_field, + page=self.page, + count=self.count, + sort=self.sort, + ci_ids=ids).search() + else: + return SearchFromDB(self.orig_query, + fl=self.fl, + facet_field=self.facet_field, + page=self.page, + count=self.count, + sort=self.sort, + ci_ids=ids).search() diff --git a/api/lib/cmdb/utils.py b/api/lib/cmdb/utils.py new file mode 100644 index 0000000..f07ae2d --- /dev/null +++ b/api/lib/cmdb/utils.py @@ -0,0 +1,116 @@ +# -*- coding:utf-8 -*- + +from __future__ import unicode_literals + +import datetime + +import six +from markupsafe import escape + +import api.models.cmdb as model +from api.lib.cmdb.cache import AttributeCache +from api.lib.cmdb.const import ValueTypeEnum + + +def string2int(x): + return int(float(x)) + + +def str2datetime(x): + try: + return datetime.datetime.strptime(x, "%Y-%m-%d") + except ValueError: + pass + + return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S") + + +class ValueTypeMap(object): + deserialize = { + ValueTypeEnum.INT: string2int, + ValueTypeEnum.FLOAT: float, + ValueTypeEnum.TEXT: lambda x: escape(x).encode('utf-8').decode('utf-8'), + ValueTypeEnum.TIME: lambda x: escape(x).encode('utf-8').decode('utf-8'), + ValueTypeEnum.DATETIME: str2datetime, + ValueTypeEnum.DATE: str2datetime, + } + + serialize = { + ValueTypeEnum.INT: int, + ValueTypeEnum.FLOAT: float, + ValueTypeEnum.TEXT: lambda x: x if isinstance(x, six.text_type) else str(x), + ValueTypeEnum.TIME: lambda x: x if isinstance(x, six.text_type) else str(x), + ValueTypeEnum.DATE: lambda x: x.strftime("%Y-%m-%d"), + ValueTypeEnum.DATETIME: lambda x: x.strftime("%Y-%m-%d %H:%M:%S"), + } + + serialize2 = { + ValueTypeEnum.INT: int, + ValueTypeEnum.FLOAT: float, + ValueTypeEnum.TEXT: lambda x: x.decode() if not isinstance(x, six.string_types) else x, + ValueTypeEnum.TIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x, + ValueTypeEnum.DATE: lambda x: x.decode() if not isinstance(x, six.string_types) else x, + ValueTypeEnum.DATETIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x, + } + + choice = { + ValueTypeEnum.INT: model.IntegerChoice, + ValueTypeEnum.FLOAT: model.FloatChoice, + ValueTypeEnum.TEXT: model.TextChoice, + } + + table = { + ValueTypeEnum.INT: model.CIValueInteger, + ValueTypeEnum.TEXT: model.CIValueText, + ValueTypeEnum.DATETIME: model.CIValueDateTime, + ValueTypeEnum.DATE: model.CIValueDateTime, + ValueTypeEnum.TIME: model.CIValueText, + ValueTypeEnum.FLOAT: model.CIValueFloat, + 'index_{0}'.format(ValueTypeEnum.INT): model.CIIndexValueInteger, + 'index_{0}'.format(ValueTypeEnum.TEXT): model.CIIndexValueText, + 'index_{0}'.format(ValueTypeEnum.DATETIME): model.CIIndexValueDateTime, + 'index_{0}'.format(ValueTypeEnum.DATE): model.CIIndexValueDateTime, + 'index_{0}'.format(ValueTypeEnum.TIME): model.CIIndexValueText, + 'index_{0}'.format(ValueTypeEnum.FLOAT): model.CIIndexValueFloat, + } + + table_name = { + ValueTypeEnum.INT: 'c_value_integers', + ValueTypeEnum.TEXT: 'c_value_texts', + ValueTypeEnum.DATETIME: 'c_value_datetime', + ValueTypeEnum.DATE: 'c_value_datetime', + ValueTypeEnum.TIME: 'c_value_texts', + ValueTypeEnum.FLOAT: 'c_value_floats', + 'index_{0}'.format(ValueTypeEnum.INT): 'c_value_index_integers', + 'index_{0}'.format(ValueTypeEnum.TEXT): 'c_value_index_texts', + 'index_{0}'.format(ValueTypeEnum.DATETIME): 'c_value_index_datetime', + 'index_{0}'.format(ValueTypeEnum.DATE): 'c_value_index_datetime', + 'index_{0}'.format(ValueTypeEnum.TIME): 'c_value_index_texts', + 'index_{0}'.format(ValueTypeEnum.FLOAT): 'c_value_index_floats', + } + + es_type = { + ValueTypeEnum.INT: 'long', + ValueTypeEnum.TEXT: 'text', + ValueTypeEnum.DATETIME: 'text', + ValueTypeEnum.DATE: 'text', + ValueTypeEnum.TIME: 'text', + ValueTypeEnum.FLOAT: 'float' + } + + +class TableMap(object): + def __init__(self, attr_name=None): + self.attr_name = attr_name + + @property + def table(self): + attr = AttributeCache.get(self.attr_name) + i = "index_{0}".format(attr.value_type) if attr.is_index else attr.value_type + return ValueTypeMap.table.get(i) + + @property + def table_name(self): + attr = AttributeCache.get(self.attr_name) + i = "index_{0}".format(attr.value_type) if attr.is_index else attr.value_type + return ValueTypeMap.table_name.get(i) diff --git a/api/lib/cmdb/value.py b/api/lib/cmdb/value.py index 75c1c5c..45a5878 100644 --- a/api/lib/cmdb/value.py +++ b/api/lib/cmdb/value.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals -import markupsafe from flask import abort from api.extensions import db @@ -11,11 +10,11 @@ from api.lib.cmdb.attribute import AttributeManager from api.lib.cmdb.cache import AttributeCache from api.lib.cmdb.const import ExistPolicy from api.lib.cmdb.const import OperateType -from api.lib.cmdb.const import TableMap -from api.lib.cmdb.const import type_map +from api.lib.cmdb.const import ValueTypeEnum from api.lib.cmdb.history import AttributeHistoryManger +from api.lib.cmdb.utils import TableMap +from api.lib.cmdb.utils import ValueTypeMap from api.lib.utils import handle_arg_list -from api.models.cmdb import Attribute class AttributeValueManager(object): @@ -58,9 +57,9 @@ class AttributeValueManager(object): field_name = getattr(attr, ret_key) if attr.is_list: - res[field_name] = [type_map["serialize"][attr.value_type](i.value) for i in rs] + res[field_name] = [ValueTypeMap.serialize[attr.value_type](i.value) for i in rs] else: - res[field_name] = type_map["serialize"][attr.value_type](rs[0].value) if rs else None + res[field_name] = ValueTypeMap.serialize[attr.value_type](rs[0].value) if rs else None if unique_key is not None and attr.id == unique_key.id and rs: res['unique'] = unique_key.name @@ -71,7 +70,7 @@ class AttributeValueManager(object): def __deserialize_value(value_type, value): if not value: return value - deserialize = type_map["deserialize"][value_type] + deserialize = ValueTypeMap.deserialize[value_type] try: v = deserialize(value) return v @@ -133,7 +132,7 @@ class AttributeValueManager(object): for v in value_list: v = self._validate(attr, v, value_table, ci_id) - if not v and attr.value_type != Attribute.TEXT: + if not v and attr.value_type != ValueTypeEnum.TEXT: v = None if operate_type == OperateType.ADD: diff --git a/api/lib/utils.py b/api/lib/utils.py index b3bb7cf..9758186 100644 --- a/api/lib/utils.py +++ b/api/lib/utils.py @@ -29,10 +29,27 @@ def handle_arg_list(arg): return list(filter(lambda x: x != "", arg.strip().split(","))) if isinstance(arg, six.string_types) else arg +class BaseEnum(object): + _ALL_ = set() # type: Set[str] + + @classmethod + def is_valid(cls, item): + return item in cls.all() + + @classmethod + def all(cls): + if not cls._ALL_: + cls._ALL_ = { + getattr(cls, attr) + for attr in dir(cls) + if not attr.startswith("_") and not callable(getattr(cls, attr)) + } + return cls._ALL_ + + class RedisHandler(object): - def __init__(self, flask_app=None, prefix=None): + def __init__(self, flask_app=None): self.flask_app = flask_app - self.prefix = prefix self.r = None def init_app(self, app): @@ -49,26 +66,26 @@ class RedisHandler(object): current_app.logger.warning(str(e)) current_app.logger.error("init redis connection failed") - def get(self, key_ids): + def get(self, key_ids, prefix): try: - value = self.r.hmget(self.prefix, key_ids) + value = self.r.hmget(prefix, key_ids) except Exception as e: current_app.logger.error("get redis error, {0}".format(str(e))) return return value - def _set(self, obj): + def _set(self, obj, prefix): try: - self.r.hmset(self.prefix, obj) + self.r.hmset(prefix, obj) except Exception as e: current_app.logger.error("set redis error, {0}".format(str(e))) - def add(self, obj): - self._set(obj) + def create_or_update(self, obj, prefix): + self._set(obj, prefix) - def delete(self, key_id): + def delete(self, key_id, prefix): try: - ret = self.r.hdel(self.prefix, key_id) + ret = self.r.hdel(prefix, key_id) if not ret: current_app.logger.warn("[{0}] is not in redis".format(key_id)) except Exception as e: diff --git a/api/models/cmdb.py b/api/models/cmdb.py index c9d4834..bdf5be1 100644 --- a/api/models/cmdb.py +++ b/api/models/cmdb.py @@ -4,6 +4,9 @@ import datetime from api.extensions import db +from api.lib.cmdb.const import CIStatusEnum +from api.lib.cmdb.const import OperateType +from api.lib.cmdb.const import ValueTypeEnum from api.lib.database import Model @@ -58,16 +61,9 @@ class CITypeRelation(Model): class Attribute(Model): __tablename__ = "c_attributes" - INT = "0" - FLOAT = "1" - TEXT = "2" - DATETIME = "3" - DATE = "4" - TIME = "5" - name = db.Column(db.String(32), nullable=False) alias = db.Column(db.String(32), nullable=False) - value_type = db.Column(db.Enum(INT, FLOAT, TEXT, DATETIME, DATE, TIME), default=TEXT, nullable=False) + value_type = db.Column(db.Enum(*ValueTypeEnum.all()), default=ValueTypeEnum.TEXT, nullable=False) is_choice = db.Column(db.Boolean, default=False) is_list = db.Column(db.Boolean, default=False) @@ -111,11 +107,8 @@ class CITypeAttributeGroupItem(Model): class CI(Model): __tablename__ = "c_cis" - REVIEW = "0" - VALIDATE = "1" - type_id = db.Column(db.Integer, db.ForeignKey("c_ci_types.id"), nullable=False) - status = db.Column(db.Enum(REVIEW, VALIDATE, name="status")) + status = db.Column(db.Enum(*CIStatusEnum.all(), name="status")) heartbeat = db.Column(db.DateTime, default=lambda: datetime.datetime.now()) ci_type = db.relationship("CIType", backref="c_cis.type_id") @@ -270,11 +263,7 @@ class OperationRecord(Model): class AttributeHistory(Model): __tablename__ = "c_attribute_histories" - ADD = "0" - DELETE = "1" - UPDATE = "2" - - operate_type = db.Column(db.Enum(ADD, DELETE, UPDATE, name="operate_type")) + operate_type = db.Column(db.Enum(*OperateType.all(), name="operate_type")) record_id = db.Column(db.Integer, db.ForeignKey("c_records.id"), nullable=False) ci_id = db.Column(db.Integer, index=True, nullable=False) attr_id = db.Column(db.Integer, index=True) @@ -285,10 +274,7 @@ class AttributeHistory(Model): class CIRelationHistory(Model): __tablename__ = "c_relation_histories" - ADD = "0" - DELETE = "1" - - operate_type = db.Column(db.Enum(ADD, DELETE, name="operate_type")) + operate_type = db.Column(db.Enum(OperateType.ADD, OperateType.DELETE, name="operate_type")) record_id = db.Column(db.Integer, db.ForeignKey("c_records.id"), nullable=False) first_ci_id = db.Column(db.Integer) second_ci_id = db.Column(db.Integer) diff --git a/api/tasks/cmdb.py b/api/tasks/cmdb.py index 78fee55..549edfd 100644 --- a/api/tasks/cmdb.py +++ b/api/tasks/cmdb.py @@ -9,9 +9,11 @@ from flask import current_app import api.lib.cmdb.ci from api.extensions import celery from api.extensions import db -from api.extensions import rd from api.extensions import es +from api.extensions import rd from api.lib.cmdb.const import CMDB_QUEUE +from api.lib.cmdb.const import REDIS_PREFIX_CI +from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION @celery.task(name="cmdb.ci_cache", queue=CMDB_QUEUE) @@ -24,10 +26,9 @@ def ci_cache(ci_id): if current_app.config.get("USE_ES"): es.update(ci_id, ci) else: - rd.delete(ci_id) - rd.add({ci_id: json.dumps(ci)}) + rd.create_or_update({ci_id: json.dumps(ci)}, REDIS_PREFIX_CI) - current_app.logger.info("%d flush.........." % ci_id) + current_app.logger.info("{0} flush..........".format(ci_id)) @celery.task(name="cmdb.ci_delete", queue=CMDB_QUEUE) @@ -37,6 +38,29 @@ def ci_delete(ci_id): if current_app.config.get("USE_ES"): es.delete(ci_id) else: - rd.delete(ci_id) + rd.delete(ci_id, REDIS_PREFIX_CI) - current_app.logger.info("%d delete.........." % ci_id) + current_app.logger.info("{0} delete..........".format(ci_id)) + + +@celery.task(name="cmdb.ci_relation_cache", queue=CMDB_QUEUE) +def ci_relation_cache(parent_id, child_id): + children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0] + children = json.loads(children) if children is not None else [] + children.append(child_id) + + rd.create_or_update({parent_id: json.dumps(list(set(children)))}, REDIS_PREFIX_CI_RELATION) + + current_app.logger.info("ADD ci relation cache: {0} -> {1}".format(parent_id, child_id)) + + +@celery.task(name="cmdb.ci_relation_delete", queue=CMDB_QUEUE) +def ci_relation_delete(parent_id, child_id): + children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0] + children = json.loads(children) if children is not None else [] + if child_id in children: + children.remove(child_id) + + rd.create_or_update({parent_id: json.dumps(list(set(children)))}, REDIS_PREFIX_CI_RELATION) + + current_app.logger.info("DELETE ci relation cache: {0} -> {1}".format(parent_id, child_id)) diff --git a/api/views/cmdb/ci.py b/api/views/cmdb/ci.py index c2e839b..a45c4bb 100644 --- a/api/views/cmdb/ci.py +++ b/api/views/cmdb/ci.py @@ -12,9 +12,9 @@ from api.lib.cmdb.ci import CIManager from api.lib.cmdb.const import ExistPolicy from api.lib.cmdb.const import ResourceType, PermEnum from api.lib.cmdb.const import RetKey -from api.lib.cmdb.search.db.search import Search as SearchFromDB -from api.lib.cmdb.search.es.search import Search as SearchFromES -from api.lib.cmdb.search.db.search import SearchError +from api.lib.cmdb.search import SearchError +from api.lib.cmdb.search.ci.db.search import Search as SearchFromDB +from api.lib.cmdb.search.ci.es.search import Search as SearchFromES from api.lib.perm.acl.acl import has_perm_from_args from api.lib.perm.auth import auth_abandoned from api.lib.utils import get_page @@ -126,13 +126,13 @@ class CISearchView(APIView): def get(self): """@params: q: query statement fl: filter by column - count: the number of ci + count/page_size: the number of ci ret_key: id, name, alias facet: statistic """ page = get_page(request.values.get("page", 1)) - count = get_page_size(request.values.get("count")) + count = get_page_size(request.values.get("count") or request.values.get("page_size")) query = request.values.get('q', "") fl = handle_arg_list(request.values.get('fl', "")) @@ -140,8 +140,6 @@ class CISearchView(APIView): if ret_key not in (RetKey.NAME, RetKey.ALIAS, RetKey.ID): ret_key = RetKey.NAME facet = handle_arg_list(request.values.get("facet", "")) - fl = list(filter(lambda x: x != "", fl)) - facet = list(filter(lambda x: x != "", facet)) sort = request.values.get("sort") start = time.time() diff --git a/api/views/cmdb/ci_relation.py b/api/views/cmdb/ci_relation.py index 86639e3..98918ef 100644 --- a/api/views/cmdb/ci_relation.py +++ b/api/views/cmdb/ci_relation.py @@ -1,14 +1,62 @@ # -*- coding:utf-8 -*- +import time + +from flask import abort +from flask import current_app from flask import request from api.lib.cmdb.ci import CIRelationManager +from api.lib.cmdb.search import SearchError +from api.lib.cmdb.search.ci_relation.search import Search +from api.lib.perm.auth import auth_abandoned from api.lib.utils import get_page from api.lib.utils import get_page_size +from api.lib.utils import handle_arg_list from api.resource import APIView +class CIRelationSearchView(APIView): + url_prefix = ("/ci_relations/s", "/ci_relations/search") + + @auth_abandoned + def get(self): + """@params: q: query statement + fl: filter by column + count: the number of ci + root_id: ci id + level: default is 1 + facet: statistic + """ + + page = get_page(request.values.get("page", 1)) + count = get_page_size(request.values.get("count") or request.values.get("page_size")) + + root_id = request.values.get('root_id') + level = request.values.get('level', 1) + + query = request.values.get('q', "") + fl = handle_arg_list(request.values.get('fl', "")) + facet = handle_arg_list(request.values.get("facet", "")) + sort = request.values.get("sort") + + start = time.time() + s = Search(root_id, level, query, fl, facet, page, count, sort) + try: + response, counter, total, page, numfound, facet = s.search() + except SearchError as e: + return abort(400, str(e)) + current_app.logger.debug("search time is :{0}".format(time.time() - start)) + + return self.jsonify(numfound=numfound, + total=total, + page=page, + facet=facet, + counter=counter, + result=response) + + class GetSecondCIsView(APIView): url_prefix = "/ci_relations//second_cis" diff --git a/api/views/cmdb/ci_type_relation.py b/api/views/cmdb/ci_type_relation.py index ead8ed2..e609cae 100644 --- a/api/views/cmdb/ci_type_relation.py +++ b/api/views/cmdb/ci_type_relation.py @@ -25,18 +25,26 @@ class GetParentsView(APIView): class CITypeRelationView(APIView): - url_prefix = "/ci_type_relations//" + url_prefix = ("/ci_type_relations", "/ci_type_relations//") + + @role_required(RoleEnum.CONFIG) + def get(self): + res = CITypeRelationManager.get() + + return self.jsonify(res) @role_required(RoleEnum.CONFIG) @args_required("relation_type_id") def post(self, parent_id, child_id): relation_type_id = request.values.get("relation_type_id") ctr_id = CITypeRelationManager.add(parent_id, child_id, relation_type_id) + return self.jsonify(ctr_id=ctr_id) @role_required(RoleEnum.CONFIG) def delete(self, parent_id, child_id): CITypeRelationManager.delete_2(parent_id, child_id) + return self.jsonify(code=200, parent_id=parent_id, child_id=child_id) @@ -46,4 +54,5 @@ class CITypeRelationDelete2View(APIView): @role_required(RoleEnum.CONFIG) def delete(self, ctr_id): CITypeRelationManager.delete(ctr_id) + return self.jsonify(code=200, ctr_id=ctr_id)