Realize /api/v0.1/ci_relations/s [done]

api/commands/click_cmdb.py

@@ -10,7 +10,11 @@ from flask.cli import with_appcontext
 import api.lib.cmdb.ci
 from api.extensions import db
 from api.extensions import rd
+from api.lib.cmdb.const import REDIS_PREFIX_CI
+from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
+from api.lib.cmdb.const import ValueTypeEnum
 from api.models.cmdb import CI
+from api.models.cmdb import CIRelation
 
 
 @click.command()
@@ -21,12 +25,12 @@ def init_cache():
     if current_app.config.get("USE_ES"):
         from api.extensions import es
         from api.models.cmdb import Attribute
-        from api.lib.cmdb.const import type_map
+        from api.lib.cmdb.utils import ValueTypeMap
         attributes = Attribute.get_by(to_dict=False)
         for attr in attributes:
             other = dict()
             other['index'] = True if attr.is_index else False
-            if attr.value_type == Attribute.TEXT:
+            if attr.value_type == ValueTypeEnum.TEXT:
                 other['analyzer'] = 'ik_max_word'
                 other['search_analyzer'] = 'ik_smart'
                 if attr.is_index:
@@ -37,7 +41,7 @@ def init_cache():
                         }
                     }
             try:
-                es.update_mapping(attr.name, type_map['es_type'][attr.value_type], other)
+                es.update_mapping(attr.name, ValueTypeMap.es_type[attr.value_type], other)
             except Exception as e:
                 print(e)
 
@@ -48,7 +52,7 @@ def init_cache():
             if res:
                 continue
         else:
-            res = rd.get([ci.id])
+            res = rd.get([ci.id], REDIS_PREFIX_CI)
             if res and list(filter(lambda x: x, res)):
                 continue
 
@@ -58,7 +62,15 @@ def init_cache():
         if current_app.config.get("USE_ES"):
             es.create(ci_dict)
         else:
-            rd.delete(ci.id)
+            rd.create_or_update({ci.id: json.dumps(ci_dict)}, REDIS_PREFIX_CI)
-            rd.add({ci.id: json.dumps(ci_dict)})
+
+    ci_relations = CIRelation.get_by(to_dict=False)
+    relations = dict()
+    for cr in ci_relations:
+        relations.setdefault(cr.first_ci_id, []).append(cr.second_ci_id)
+    for i in relations:
+        relations[i] = json.dumps(relations[i])
+    if relations:
+        rd.create_or_update(relations, REDIS_PREFIX_CI_RELATION)
 
     db.session.remove()

api/extensions.py

@@ -19,5 +19,5 @@ migrate = Migrate()
 cache = Cache()
 celery = Celery()
 cors = CORS(supports_credentials=True)
-rd = RedisHandler(prefix="CMDB_CI")  # TODO
+rd = RedisHandler()
 es = ESHandler()

api/lib/cmdb/attribute.py

@@ -5,7 +5,8 @@ from flask import current_app
 
 from api.extensions import db
 from api.lib.cmdb.cache import AttributeCache
-from api.lib.cmdb.const import type_map
+from api.lib.cmdb.const import ValueTypeEnum
+from api.lib.cmdb.utils import ValueTypeMap
 from api.lib.decorator import kwargs_required
 from api.models.cmdb import Attribute
 from api.models.cmdb import CITypeAttribute
@@ -22,13 +23,13 @@ class AttributeManager(object):
 
     @staticmethod
     def get_choice_values(attr_id, value_type):
-        choice_table = type_map.get("choice").get(value_type)
+        choice_table = ValueTypeMap.choice.get(value_type)
         choice_values = choice_table.get_by(fl=["value"], attr_id=attr_id)
         return [choice_value["value"] for choice_value in choice_values]
 
     @staticmethod
     def _add_choice_values(_id, value_type, choice_values):
-        choice_table = type_map.get("choice").get(value_type)
+        choice_table = ValueTypeMap.choice.get(value_type)
 
         db.session.query(choice_table).filter(choice_table.attr_id == _id).delete()
         db.session.flush()
@@ -121,7 +122,7 @@ class AttributeManager(object):
             from api.extensions import es
             other = dict()
             other['index'] = True if attr.is_index else False
-            if attr.value_type == Attribute.TEXT:
+            if attr.value_type == ValueTypeEnum.TEXT:
                 other['analyzer'] = 'ik_max_word'
                 other['search_analyzer'] = 'ik_smart'
                 if attr.is_index:
@@ -131,7 +132,7 @@ class AttributeManager(object):
                             "ignore_above": 256
                         }
                     }
-            es.update_mapping(name, type_map['es_type'][attr.value_type], other)
+            es.update_mapping(name, ValueTypeMap.es_type[attr.value_type], other)
 
         return attr.id
 
@@ -172,7 +173,7 @@ class AttributeManager(object):
         name = attr.name
 
         if attr.is_choice:
-            choice_table = type_map["choice"].get(attr.value_type)
+            choice_table = ValueTypeMap.choice.get(attr.value_type)
             db.session.query(choice_table).filter(choice_table.attr_id == _id).delete()  # FIXME: session conflict
             db.session.flush()
 

api/lib/cmdb/ci.py

@@ -18,13 +18,14 @@ from api.lib.cmdb.ci_type import CITypeManager
 from api.lib.cmdb.const import CMDB_QUEUE
 from api.lib.cmdb.const import ExistPolicy
 from api.lib.cmdb.const import OperateType
+from api.lib.cmdb.const import REDIS_PREFIX_CI
 from api.lib.cmdb.const import RetKey
-from api.lib.cmdb.const import TableMap
-from api.lib.cmdb.const import type_map
 from api.lib.cmdb.history import AttributeHistoryManger
 from api.lib.cmdb.history import CIRelationHistoryManager
-from api.lib.cmdb.search.db.query_sql import QUERY_CIS_BY_IDS
+from api.lib.cmdb.search.ci.db.query_sql import QUERY_CIS_BY_IDS
-from api.lib.cmdb.search.db.query_sql import QUERY_CIS_BY_VALUE_TABLE
+from api.lib.cmdb.search.ci.db.query_sql import QUERY_CIS_BY_VALUE_TABLE
+from api.lib.cmdb.utils import TableMap
+from api.lib.cmdb.utils import ValueTypeMap
 from api.lib.cmdb.value import AttributeValueManager
 from api.lib.decorator import kwargs_required
 from api.lib.utils import handle_arg_list
@@ -33,6 +34,8 @@ from api.models.cmdb import CIRelation
 from api.models.cmdb import CITypeAttribute
 from api.tasks.cmdb import ci_cache
 from api.tasks.cmdb import ci_delete
+from api.tasks.cmdb import ci_relation_cache
+from api.tasks.cmdb import ci_relation_delete
 
 
 class CIManager(object):
@@ -302,7 +305,7 @@ class CIManager(object):
 
     @staticmethod
     def _get_cis_from_cache(ci_ids, ret_key=RetKey.NAME, fields=None):
-        res = rd.get(ci_ids)
+        res = rd.get(ci_ids, REDIS_PREFIX_CI)
         if res is not None and None not in res and ret_key == RetKey.NAME:
             res = list(map(json.loads, res))
             if not fields:
@@ -332,7 +335,7 @@ class CIManager(object):
 
         ci_ids = ",".join(ci_ids)
         if value_tables is None:
-            value_tables = type_map["table_name"].values()
+            value_tables = ValueTypeMap.table_name.values()
 
         value_sql = " UNION ".join([QUERY_CIS_BY_VALUE_TABLE.format(value_table, ci_ids)
                                     for value_table in value_tables])
@@ -362,7 +365,7 @@ class CIManager(object):
             else:
                 return abort(400, "invalid ret key")
 
-            value = type_map["serialize2"][value_type](value)
+            value = ValueTypeMap.serialize2[value_type](value)
             if is_list:
                 ci_dict.setdefault(attr_key, []).append(value)
             else:
@@ -532,6 +535,9 @@ class CIRelationManager(object):
                                         second_ci_id=second_ci_id,
                                         relation_type_id=relation_type_id)
             CIRelationHistoryManager().add(existed, OperateType.ADD)
+
+            ci_relation_cache.apply_async(args=(first_ci_id, second_ci_id), queue=CMDB_QUEUE)
+
         if more is not None:
             existed.upadte(more=more)
 
@@ -545,6 +551,8 @@ class CIRelationManager(object):
         his_manager = CIRelationHistoryManager()
         his_manager.add(cr, operate_type=OperateType.DELETE)
 
+        ci_relation_delete.apply_async(args=(cr.first_ci_id, cr.second_ci_id), queue=CMDB_QUEUE)
+
         return cr_id
 
     @classmethod
@@ -553,4 +561,7 @@ class CIRelationManager(object):
                                second_ci_id=second_ci_id,
                                to_dict=False,
                                first=True)
+
+        ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id), queue=CMDB_QUEUE)
+
         return cls.delete(cr.cr_id)

api/lib/cmdb/ci_type.py

@@ -4,6 +4,7 @@
 from flask import abort
 from flask import current_app
 
+from api.extensions import db
 from api.lib.cmdb.attribute import AttributeManager
 from api.lib.cmdb.cache import AttributeCache
 from api.lib.cmdb.cache import CITypeAttributeCache
@@ -263,8 +264,26 @@ class CITypeRelationManager(object):
     manage relation between CITypes
     """
 
-    def __init__(self):
+    @staticmethod
-        pass
+    def get():
+        res = CITypeRelation.get_by(to_dict=False)
+        for idx, item in enumerate(res):
+            _item = item.to_dict()
+            res[idx] = _item
+            res[idx]['parent'] = item.parent.to_dict()
+            res[idx]['child'] = item.child.to_dict()
+            res[idx]['relation_type'] = item.relation_type.to_dict()
+
+        return res
+
+    @staticmethod
+    def get_child_type_ids(type_id, level):
+        ids = [type_id]
+        query = db.session.query(CITypeRelation).filter(CITypeRelation.deleted.is_(False))
+        for _ in range(0, level):
+            ids = [i.child_id for i in query.filter(CITypeRelation.parent_id.in_(ids))]
+
+        return ids
 
     @staticmethod
     def _wrap_relation_type_dict(type_id, relation_inst):

api/lib/cmdb/const.py

@@ -1,160 +1,57 @@
 # -*- coding:utf-8 -*- 
 
 
-from __future__ import unicode_literals
+from api.lib.utils import BaseEnum
-
-import datetime
-
-import six
-from markupsafe import escape
-
-from api.lib.cmdb.cache import AttributeCache
-from api.models.cmdb import Attribute
-from api.models.cmdb import CIIndexValueDateTime
-from api.models.cmdb import CIIndexValueFloat
-from api.models.cmdb import CIIndexValueInteger
-from api.models.cmdb import CIIndexValueText
-from api.models.cmdb import CIValueDateTime
-from api.models.cmdb import CIValueFloat
-from api.models.cmdb import CIValueInteger
-from api.models.cmdb import CIValueText
-from api.models.cmdb import FloatChoice
-from api.models.cmdb import IntegerChoice
-from api.models.cmdb import TextChoice
 
 
-def string2int(x):
+class ValueTypeEnum(BaseEnum):
-    return int(float(x))
+    INT = "0"
+    FLOAT = "1"
+    TEXT = "2"
+    DATETIME = "3"
+    DATE = "4"
+    TIME = "5"
 
 
-def str2datetime(x):
+class CIStatusEnum(BaseEnum):
-    try:
+    REVIEW = "0"
-        return datetime.datetime.strptime(x, "%Y-%m-%d")
+    VALIDATE = "1"
-    except ValueError:
-        pass
-
-    return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")
 
 
-type_map = {
+class ExistPolicy(BaseEnum):
-    'deserialize': {
-        Attribute.INT: string2int,
-        Attribute.FLOAT: float,
-        Attribute.TEXT: lambda x: escape(x).encode('utf-8').decode('utf-8'),
-        Attribute.TIME: lambda x: escape(x).encode('utf-8').decode('utf-8'),
-        Attribute.DATETIME: str2datetime,
-        Attribute.DATE: str2datetime,
-    },
-    'serialize': {
-        Attribute.INT: int,
-        Attribute.FLOAT: float,
-        Attribute.TEXT: lambda x: x if isinstance(x, six.text_type) else str(x),
-        Attribute.TIME: lambda x: x if isinstance(x, six.text_type) else str(x),
-        Attribute.DATE: lambda x: x.strftime("%Y-%m-%d"),
-        Attribute.DATETIME: lambda x: x.strftime("%Y-%m-%d %H:%M:%S"),
-    },
-    'serialize2': {
-        Attribute.INT: int,
-        Attribute.FLOAT: float,
-        Attribute.TEXT: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
-        Attribute.TIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
-        Attribute.DATE: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
-        Attribute.DATETIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
-    },
-    'choice': {
-        Attribute.INT: IntegerChoice,
-        Attribute.FLOAT: FloatChoice,
-        Attribute.TEXT: TextChoice,
-    },
-    'table': {
-        Attribute.INT: CIValueInteger,
-        Attribute.TEXT: CIValueText,
-        Attribute.DATETIME: CIValueDateTime,
-        Attribute.DATE: CIValueDateTime,
-        Attribute.TIME: CIValueText,
-        Attribute.FLOAT: CIValueFloat,
-        'index_{0}'.format(Attribute.INT): CIIndexValueInteger,
-        'index_{0}'.format(Attribute.TEXT): CIIndexValueText,
-        'index_{0}'.format(Attribute.DATETIME): CIIndexValueDateTime,
-        'index_{0}'.format(Attribute.DATE): CIIndexValueDateTime,
-        'index_{0}'.format(Attribute.TIME): CIIndexValueText,
-        'index_{0}'.format(Attribute.FLOAT): CIIndexValueFloat,
-    },
-    'table_name': {
-        Attribute.INT: 'c_value_integers',
-        Attribute.TEXT: 'c_value_texts',
-        Attribute.DATETIME: 'c_value_datetime',
-        Attribute.DATE: 'c_value_datetime',
-        Attribute.TIME: 'c_value_texts',
-        Attribute.FLOAT: 'c_value_floats',
-        'index_{0}'.format(Attribute.INT): 'c_value_index_integers',
-        'index_{0}'.format(Attribute.TEXT): 'c_value_index_texts',
-        'index_{0}'.format(Attribute.DATETIME): 'c_value_index_datetime',
-        'index_{0}'.format(Attribute.DATE): 'c_value_index_datetime',
-        'index_{0}'.format(Attribute.TIME): 'c_value_index_texts',
-        'index_{0}'.format(Attribute.FLOAT): 'c_value_index_floats',
-    },
-    'es_type': {
-        Attribute.INT: 'long',
-        Attribute.TEXT: 'text',
-        Attribute.DATETIME: 'text',
-        Attribute.DATE: 'text',
-        Attribute.TIME: 'text',
-        Attribute.FLOAT: 'float'
-    }
-}
-
-
-class TableMap(object):
-    def __init__(self, attr_name=None):
-        self.attr_name = attr_name
-
-    @property
-    def table(self):
-        attr = AttributeCache.get(self.attr_name)
-        i = "index_{0}".format(attr.value_type) if attr.is_index else attr.value_type
-        return type_map["table"].get(i)
-
-    @property
-    def table_name(self):
-        attr = AttributeCache.get(self.attr_name)
-        i = "index_{0}".format(attr.value_type) if attr.is_index else attr.value_type
-        return type_map["table_name"].get(i)
-
-
-class ExistPolicy(object):
     REJECT = "reject"
     NEED = "need"
     IGNORE = "ignore"
     REPLACE = "replace"
 
 
-class OperateType(object):
+class OperateType(BaseEnum):
     ADD = "0"
     DELETE = "1"
     UPDATE = "2"
 
 
-class RetKey(object):
+class RetKey(BaseEnum):
     ID = "id"
     NAME = "name"
     ALIAS = "alias"
 
 
-class ResourceType(object):
+class ResourceType(BaseEnum):
     CI = "CIType"
 
 
-class PermEnum(object):
+class PermEnum(BaseEnum):
     ADD = "add"
     UPDATE = "update"
     DELETE = "delete"
     READ = "read"
 
 
-class RoleEnum(object):
+class RoleEnum(BaseEnum):
     CONFIG = "admin"
 
 
 CMDB_QUEUE = "cmdb_async"
-REDIS_PREFIX = "CMDB_CI"
+REDIS_PREFIX_CI = "CMDB_CI"
+REDIS_PREFIX_CI_RELATION = "CMDB_CI_RELATION"

api/lib/cmdb/history.py

@@ -7,6 +7,7 @@ from flask import g
 from api.extensions import db
 from api.lib.cmdb.cache import AttributeCache
 from api.lib.cmdb.cache import RelationTypeCache
+from api.lib.cmdb.const import OperateType
 from api.lib.perm.acl.cache import UserCache
 from api.models.cmdb import Attribute
 from api.models.cmdb import AttributeHistory
@@ -112,7 +113,7 @@ class AttributeHistoryManger(object):
 
 class CIRelationHistoryManager(object):
     @staticmethod
-    def add(rel_obj, operate_type=CIRelationHistory.ADD):
+    def add(rel_obj, operate_type=OperateType.ADD):
         record = OperationRecord.create(uid=g.user.uid)
 
         CIRelationHistory.create(relation_id=rel_obj.id,

api/lib/cmdb/search/__init__.py

@@ -1,3 +1,11 @@
 # -*- coding:utf-8 -*-
 
-__all__ = ['db', 'es']
+__all__ = ['ci', 'ci_relation', 'SearchError']
+
+
+class SearchError(Exception):
+    def __init__(self, v):
+        self.v = v
+
+    def __str__(self):
+        return self.v

api/lib/cmdb/search/ci/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding:utf-8 -*-
+
+__all__ = ['db', 'es']

api/lib/cmdb/search/ci/db/search.py

@@ -12,25 +12,25 @@ from api.lib.cmdb.cache import AttributeCache
 from api.lib.cmdb.cache import CITypeCache
 from api.lib.cmdb.ci import CIManager
 from api.lib.cmdb.const import RetKey
-from api.lib.cmdb.const import TableMap
+from api.lib.cmdb.const import ValueTypeEnum
-from api.lib.cmdb.search.db.query_sql import FACET_QUERY
+from api.lib.cmdb.search import SearchError
-from api.lib.cmdb.search.db.query_sql import QUERY_CI_BY_ATTR_NAME
+from api.lib.cmdb.search.ci.db.query_sql import FACET_QUERY
-from api.lib.cmdb.search.db.query_sql import QUERY_CI_BY_TYPE
+from api.lib.cmdb.search.ci.db.query_sql import QUERY_CI_BY_ATTR_NAME
+from api.lib.cmdb.search.ci.db.query_sql import QUERY_CI_BY_TYPE
+from api.lib.cmdb.utils import TableMap
 from api.lib.utils import handle_arg_list
-from api.models.cmdb import Attribute
 from api.models.cmdb import CI
 
 
-class SearchError(Exception):
-    def __init__(self, v):
-        self.v = v
-
-    def __str__(self):
-        return self.v
-
-
 class Search(object):
-    def __init__(self, query=None, fl=None, facet_field=None, page=1, ret_key=RetKey.NAME, count=1, sort=None):
+    def __init__(self, query=None,
+                 fl=None,
+                 facet_field=None,
+                 page=1,
+                 ret_key=RetKey.NAME,
+                 count=1,
+                 sort=None,
+                 ci_ids=None):
         self.orig_query = query
         self.fl = fl
         self.facet_field = facet_field
@@ -38,6 +38,7 @@ class Search(object):
         self.ret_key = ret_key
         self.count = count
         self.sort = sort
+        self.ci_ids = ci_ids or []
         self.query_sql = ""
         self.type_id_list = []
         self.only_type_query = False
@@ -60,10 +61,10 @@ class Search(object):
         operator, key = self._operator_proc(key)
 
         if key in ('ci_type', 'type', '_type'):
-            return '_type', Attribute.TEXT, operator, None
+            return '_type', ValueTypeEnum.TEXT, operator, None
 
         if key in ('id', 'ci_id', '_id'):
-            return '_id', Attribute.TEXT, operator, None
+            return '_id', ValueTypeEnum.TEXT, operator, None
 
         attr = AttributeCache.get(key)
         if attr:
@@ -286,6 +287,13 @@ class Search(object):
                 alias += "AA"
         return None, query_sql
 
+    def _filter_ids(self, query_sql):
+        if self.ci_ids:
+            return "SELECT * FROM ({0}) AS IN_QUERY WHERE IN_QUERY.ci_id in ({1})".format(
+                query_sql, ",".join(list(map(str, self.ci_ids))))
+
+        return query_sql
+
     def _query_build_raw(self):
 
         queries = handle_arg_list(self.orig_query)
@@ -298,6 +306,7 @@ class Search(object):
 
         s = time.time()
         if query_sql:
+            query_sql = self._filter_ids(query_sql)
             self.query_sql = query_sql
             current_app.logger.debug(query_sql)
             numfound, res = self._execute_sql(query_sql)

api/lib/cmdb/search/ci/es/search.py

@@ -8,20 +8,20 @@ from flask import current_app
 from api.extensions import es
 from api.lib.cmdb.cache import AttributeCache
 from api.lib.cmdb.const import RetKey
+from api.lib.cmdb.const import ValueTypeEnum
+from api.lib.cmdb.search import SearchError
 from api.lib.utils import handle_arg_list
-from api.models.cmdb import Attribute
-
-
-class SearchError(Exception):
-    def __init__(self, v):
-        self.v = v
-
-    def __str__(self):
-        return self.v
 
 
 class Search(object):
-    def __init__(self, query=None, fl=None, facet_field=None, page=1, ret_key=RetKey.NAME, count=1, sort=None):
+    def __init__(self, query=None,
+                 fl=None,
+                 facet_field=None,
+                 page=1,
+                 ret_key=RetKey.NAME,
+                 count=1,
+                 sort=None,
+                 ci_ids=None):
         self.orig_query = query
         self.fl = fl
         self.facet_field = facet_field
@@ -29,6 +29,7 @@ class Search(object):
         self.ret_key = ret_key
         self.count = count or current_app.config.get("DEFAULT_PAGE_COUNT")
         self.sort = sort or "ci_id"
+        self.ci_ids = ci_ids or []
 
         self.query = dict(query=dict(bool=dict(should=[], must=[], must_not=[])))
 
@@ -58,10 +59,10 @@ class Search(object):
         operator, key = self._operator_proc(key)
 
         if key in ('ci_type', 'type', '_type'):
-            return 'ci_type', Attribute.TEXT, operator
+            return 'ci_type', ValueTypeEnum.TEXT, operator
 
         if key in ('id', 'ci_id', '_id'):
-            return 'ci_id', Attribute.TEXT, operator
+            return 'ci_id', ValueTypeEnum.TEXT, operator
 
         attr = AttributeCache.get(key)
         if attr:
@@ -79,6 +80,10 @@ class Search(object):
                 }
             })
 
+    def _filter_ids(self):
+        if self.ci_ids:
+            self.query['query']['bool'].update(dict(filter=dict(terms=dict(ci_id=self.ci_ids))))
+
     @staticmethod
     def _digit(s):
         if s.isdigit():
@@ -171,6 +176,8 @@ class Search(object):
 
         self._facet_build()
 
+        self._filter_ids()
+
         return es.read(self.query, filter_path=filter_path)
 
     def _facet_build(self):
@@ -183,7 +190,7 @@ class Search(object):
                 field: {
                     "terms": {
                         "field": "{0}.keyword".format(field)
-                        if attr.value_type not in (Attribute.INT, Attribute.FLOAT) else field
+                        if attr.value_type not in (ValueTypeEnum.INT, ValueTypeEnum.FLOAT) else field
                     }
                 }
             })
@@ -212,7 +219,7 @@ class Search(object):
                 raise SearchError("Sort by <{0}> does not exist".format(field))
 
             sort_by = "{0}.keyword".format(field) \
-                if attr.value_type not in (Attribute.INT, Attribute.FLOAT) else field
+                if attr.value_type not in (ValueTypeEnum.INT, ValueTypeEnum.FLOAT) else field
             sorts.append({sort_by: {"order": sort_type}})
 
         self.query.update(dict(sort=sorts))

api/lib/cmdb/search/ci_relation/__init__.py

@@ -0,0 +1 @@
+# -*- coding:utf-8 -*-

api/lib/cmdb/search/ci_relation/search.py

@@ -0,0 +1,56 @@
+# -*- coding:utf-8 -*-
+
+
+import json
+
+from flask import abort
+from flask import current_app
+
+from api.extensions import rd
+from api.lib.cmdb.ci_type import CITypeRelationManager
+from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
+from api.lib.cmdb.search.ci.db.search import Search as SearchFromDB
+from api.lib.cmdb.search.ci.es.search import Search as SearchFromES
+from api.models.cmdb import CI
+
+
+class Search(object):
+    def __init__(self, root_id, level=1, query=None, fl=None, facet_field=None, page=1, count=None, sort=None):
+        self.orig_query = query
+        self.fl = fl
+        self.facet_field = facet_field
+        self.page = page
+        self.count = count or current_app.config.get("DEFAULT_PAGE_COUNT")
+        self.sort = sort or ("ci_id" if current_app.config.get("USE_ES") else None)
+
+        self.root_id = root_id
+        self.level = int(level)
+
+    def search(self):
+        ci = CI.get_by_id(self.root_id) or abort(404, "CI <{0}> does not exist".format(self.root_id))
+        ids = [self.root_id]
+        for _ in range(0, self.level):
+            _tmp = list(map(json.loads, filter(lambda x: x is not None, rd.get(ids, REDIS_PREFIX_CI_RELATION))))
+            ids = [j for i in _tmp for j in i]
+        if not self.orig_query or ("_type:" not in self.orig_query
+                                   and "type_id:" not in self.orig_query
+                                   and "ci_type:" not in self.orig_query):
+            type_ids = CITypeRelationManager.get_child_type_ids(ci.type_id, self.level)
+            self.orig_query = "_type:({0}),{1}".format(";".join(list(map(str, type_ids))), self.orig_query)
+
+        if current_app.config.get("USE_ES"):
+            return SearchFromES(self.orig_query,
+                                fl=self.fl,
+                                facet_field=self.facet_field,
+                                page=self.page,
+                                count=self.count,
+                                sort=self.sort,
+                                ci_ids=ids).search()
+        else:
+            return SearchFromDB(self.orig_query,
+                                fl=self.fl,
+                                facet_field=self.facet_field,
+                                page=self.page,
+                                count=self.count,
+                                sort=self.sort,
+                                ci_ids=ids).search()

api/lib/cmdb/utils.py

@@ -0,0 +1,116 @@
+# -*- coding:utf-8 -*-
+
+from __future__ import unicode_literals
+
+import datetime
+
+import six
+from markupsafe import escape
+
+import api.models.cmdb as model
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.const import ValueTypeEnum
+
+
+def string2int(x):
+    return int(float(x))
+
+
+def str2datetime(x):
+    try:
+        return datetime.datetime.strptime(x, "%Y-%m-%d")
+    except ValueError:
+        pass
+
+    return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")
+
+
+class ValueTypeMap(object):
+    deserialize = {
+        ValueTypeEnum.INT: string2int,
+        ValueTypeEnum.FLOAT: float,
+        ValueTypeEnum.TEXT: lambda x: escape(x).encode('utf-8').decode('utf-8'),
+        ValueTypeEnum.TIME: lambda x: escape(x).encode('utf-8').decode('utf-8'),
+        ValueTypeEnum.DATETIME: str2datetime,
+        ValueTypeEnum.DATE: str2datetime,
+    }
+
+    serialize = {
+        ValueTypeEnum.INT: int,
+        ValueTypeEnum.FLOAT: float,
+        ValueTypeEnum.TEXT: lambda x: x if isinstance(x, six.text_type) else str(x),
+        ValueTypeEnum.TIME: lambda x: x if isinstance(x, six.text_type) else str(x),
+        ValueTypeEnum.DATE: lambda x: x.strftime("%Y-%m-%d"),
+        ValueTypeEnum.DATETIME: lambda x: x.strftime("%Y-%m-%d %H:%M:%S"),
+    }
+
+    serialize2 = {
+        ValueTypeEnum.INT: int,
+        ValueTypeEnum.FLOAT: float,
+        ValueTypeEnum.TEXT: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
+        ValueTypeEnum.TIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
+        ValueTypeEnum.DATE: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
+        ValueTypeEnum.DATETIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
+    }
+
+    choice = {
+        ValueTypeEnum.INT: model.IntegerChoice,
+        ValueTypeEnum.FLOAT: model.FloatChoice,
+        ValueTypeEnum.TEXT: model.TextChoice,
+    }
+
+    table = {
+        ValueTypeEnum.INT: model.CIValueInteger,
+        ValueTypeEnum.TEXT: model.CIValueText,
+        ValueTypeEnum.DATETIME: model.CIValueDateTime,
+        ValueTypeEnum.DATE: model.CIValueDateTime,
+        ValueTypeEnum.TIME: model.CIValueText,
+        ValueTypeEnum.FLOAT: model.CIValueFloat,
+        'index_{0}'.format(ValueTypeEnum.INT): model.CIIndexValueInteger,
+        'index_{0}'.format(ValueTypeEnum.TEXT): model.CIIndexValueText,
+        'index_{0}'.format(ValueTypeEnum.DATETIME): model.CIIndexValueDateTime,
+        'index_{0}'.format(ValueTypeEnum.DATE): model.CIIndexValueDateTime,
+        'index_{0}'.format(ValueTypeEnum.TIME): model.CIIndexValueText,
+        'index_{0}'.format(ValueTypeEnum.FLOAT): model.CIIndexValueFloat,
+    }
+
+    table_name = {
+        ValueTypeEnum.INT: 'c_value_integers',
+        ValueTypeEnum.TEXT: 'c_value_texts',
+        ValueTypeEnum.DATETIME: 'c_value_datetime',
+        ValueTypeEnum.DATE: 'c_value_datetime',
+        ValueTypeEnum.TIME: 'c_value_texts',
+        ValueTypeEnum.FLOAT: 'c_value_floats',
+        'index_{0}'.format(ValueTypeEnum.INT): 'c_value_index_integers',
+        'index_{0}'.format(ValueTypeEnum.TEXT): 'c_value_index_texts',
+        'index_{0}'.format(ValueTypeEnum.DATETIME): 'c_value_index_datetime',
+        'index_{0}'.format(ValueTypeEnum.DATE): 'c_value_index_datetime',
+        'index_{0}'.format(ValueTypeEnum.TIME): 'c_value_index_texts',
+        'index_{0}'.format(ValueTypeEnum.FLOAT): 'c_value_index_floats',
+    }
+
+    es_type = {
+        ValueTypeEnum.INT: 'long',
+        ValueTypeEnum.TEXT: 'text',
+        ValueTypeEnum.DATETIME: 'text',
+        ValueTypeEnum.DATE: 'text',
+        ValueTypeEnum.TIME: 'text',
+        ValueTypeEnum.FLOAT: 'float'
+    }
+
+
+class TableMap(object):
+    def __init__(self, attr_name=None):
+        self.attr_name = attr_name
+
+    @property
+    def table(self):
+        attr = AttributeCache.get(self.attr_name)
+        i = "index_{0}".format(attr.value_type) if attr.is_index else attr.value_type
+        return ValueTypeMap.table.get(i)
+
+    @property
+    def table_name(self):
+        attr = AttributeCache.get(self.attr_name)
+        i = "index_{0}".format(attr.value_type) if attr.is_index else attr.value_type
+        return ValueTypeMap.table_name.get(i)

api/lib/cmdb/value.py

@@ -3,7 +3,6 @@
 
 from __future__ import unicode_literals
 
-import markupsafe
 from flask import abort
 
 from api.extensions import db
@@ -11,11 +10,11 @@ from api.lib.cmdb.attribute import AttributeManager
 from api.lib.cmdb.cache import AttributeCache
 from api.lib.cmdb.const import ExistPolicy
 from api.lib.cmdb.const import OperateType
-from api.lib.cmdb.const import TableMap
+from api.lib.cmdb.const import ValueTypeEnum
-from api.lib.cmdb.const import type_map
 from api.lib.cmdb.history import AttributeHistoryManger
+from api.lib.cmdb.utils import TableMap
+from api.lib.cmdb.utils import ValueTypeMap
 from api.lib.utils import handle_arg_list
-from api.models.cmdb import Attribute
 
 
 class AttributeValueManager(object):
@@ -58,9 +57,9 @@ class AttributeValueManager(object):
             field_name = getattr(attr, ret_key)
 
             if attr.is_list:
-                res[field_name] = [type_map["serialize"][attr.value_type](i.value) for i in rs]
+                res[field_name] = [ValueTypeMap.serialize[attr.value_type](i.value) for i in rs]
             else:
-                res[field_name] = type_map["serialize"][attr.value_type](rs[0].value) if rs else None
+                res[field_name] = ValueTypeMap.serialize[attr.value_type](rs[0].value) if rs else None
 
             if unique_key is not None and attr.id == unique_key.id and rs:
                 res['unique'] = unique_key.name
@@ -71,7 +70,7 @@ class AttributeValueManager(object):
     def __deserialize_value(value_type, value):
         if not value:
             return value
-        deserialize = type_map["deserialize"][value_type]
+        deserialize = ValueTypeMap.deserialize[value_type]
         try:
             v = deserialize(value)
             return v
@@ -133,7 +132,7 @@ class AttributeValueManager(object):
 
         for v in value_list:
             v = self._validate(attr, v, value_table, ci_id)
-            if not v and attr.value_type != Attribute.TEXT:
+            if not v and attr.value_type != ValueTypeEnum.TEXT:
                 v = None
 
             if operate_type == OperateType.ADD:

api/lib/utils.py

@@ -29,10 +29,27 @@ def handle_arg_list(arg):
     return list(filter(lambda x: x != "", arg.strip().split(","))) if isinstance(arg, six.string_types) else arg
 
 
+class BaseEnum(object):
+    _ALL_ = set()  # type: Set[str]
+
+    @classmethod
+    def is_valid(cls, item):
+        return item in cls.all()
+
+    @classmethod
+    def all(cls):
+        if not cls._ALL_:
+            cls._ALL_ = {
+                getattr(cls, attr)
+                for attr in dir(cls)
+                if not attr.startswith("_") and not callable(getattr(cls, attr))
+            }
+        return cls._ALL_
+
+
 class RedisHandler(object):
-    def __init__(self, flask_app=None, prefix=None):
+    def __init__(self, flask_app=None):
         self.flask_app = flask_app
-        self.prefix = prefix
         self.r = None
 
     def init_app(self, app):
@@ -49,26 +66,26 @@ class RedisHandler(object):
             current_app.logger.warning(str(e))
             current_app.logger.error("init redis connection failed")
 
-    def get(self, key_ids):
+    def get(self, key_ids, prefix):
         try:
-            value = self.r.hmget(self.prefix, key_ids)
+            value = self.r.hmget(prefix, key_ids)
         except Exception as e:
             current_app.logger.error("get redis error, {0}".format(str(e)))
             return
         return value
 
-    def _set(self, obj):
+    def _set(self, obj, prefix):
         try:
-            self.r.hmset(self.prefix, obj)
+            self.r.hmset(prefix, obj)
         except Exception as e:
             current_app.logger.error("set redis error, {0}".format(str(e)))
 
-    def add(self, obj):
+    def create_or_update(self, obj, prefix):
-        self._set(obj)
+        self._set(obj, prefix)
 
-    def delete(self, key_id):
+    def delete(self, key_id, prefix):
         try:
-            ret = self.r.hdel(self.prefix, key_id)
+            ret = self.r.hdel(prefix, key_id)
             if not ret:
                 current_app.logger.warn("[{0}] is not in redis".format(key_id))
         except Exception as e:

api/models/cmdb.py

@@ -4,6 +4,9 @@
 import datetime
 
 from api.extensions import db
+from api.lib.cmdb.const import CIStatusEnum
+from api.lib.cmdb.const import OperateType
+from api.lib.cmdb.const import ValueTypeEnum
 from api.lib.database import Model
 
 
@@ -58,16 +61,9 @@ class CITypeRelation(Model):
 class Attribute(Model):
     __tablename__ = "c_attributes"
 
-    INT = "0"
-    FLOAT = "1"
-    TEXT = "2"
-    DATETIME = "3"
-    DATE = "4"
-    TIME = "5"
-
     name = db.Column(db.String(32), nullable=False)
     alias = db.Column(db.String(32), nullable=False)
-    value_type = db.Column(db.Enum(INT, FLOAT, TEXT, DATETIME, DATE, TIME), default=TEXT, nullable=False)
+    value_type = db.Column(db.Enum(*ValueTypeEnum.all()), default=ValueTypeEnum.TEXT, nullable=False)
 
     is_choice = db.Column(db.Boolean, default=False)
     is_list = db.Column(db.Boolean, default=False)
@@ -111,11 +107,8 @@ class CITypeAttributeGroupItem(Model):
 class CI(Model):
     __tablename__ = "c_cis"
 
-    REVIEW = "0"
-    VALIDATE = "1"
-
     type_id = db.Column(db.Integer, db.ForeignKey("c_ci_types.id"), nullable=False)
-    status = db.Column(db.Enum(REVIEW, VALIDATE, name="status"))
+    status = db.Column(db.Enum(*CIStatusEnum.all(), name="status"))
     heartbeat = db.Column(db.DateTime, default=lambda: datetime.datetime.now())
 
     ci_type = db.relationship("CIType", backref="c_cis.type_id")
@@ -270,11 +263,7 @@ class OperationRecord(Model):
 class AttributeHistory(Model):
     __tablename__ = "c_attribute_histories"
 
-    ADD = "0"
+    operate_type = db.Column(db.Enum(*OperateType.all(), name="operate_type"))
-    DELETE = "1"
-    UPDATE = "2"
-
-    operate_type = db.Column(db.Enum(ADD, DELETE, UPDATE, name="operate_type"))
     record_id = db.Column(db.Integer, db.ForeignKey("c_records.id"), nullable=False)
     ci_id = db.Column(db.Integer, index=True, nullable=False)
     attr_id = db.Column(db.Integer, index=True)
@@ -285,10 +274,7 @@ class AttributeHistory(Model):
 class CIRelationHistory(Model):
     __tablename__ = "c_relation_histories"
 
-    ADD = "0"
+    operate_type = db.Column(db.Enum(OperateType.ADD, OperateType.DELETE, name="operate_type"))
-    DELETE = "1"
-
-    operate_type = db.Column(db.Enum(ADD, DELETE, name="operate_type"))
     record_id = db.Column(db.Integer, db.ForeignKey("c_records.id"), nullable=False)
     first_ci_id = db.Column(db.Integer)
     second_ci_id = db.Column(db.Integer)

api/tasks/cmdb.py

@@ -9,9 +9,11 @@ from flask import current_app
 import api.lib.cmdb.ci
 from api.extensions import celery
 from api.extensions import db
-from api.extensions import rd
 from api.extensions import es
+from api.extensions import rd
 from api.lib.cmdb.const import CMDB_QUEUE
+from api.lib.cmdb.const import REDIS_PREFIX_CI
+from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
 
 
 @celery.task(name="cmdb.ci_cache", queue=CMDB_QUEUE)
@@ -24,10 +26,9 @@ def ci_cache(ci_id):
     if current_app.config.get("USE_ES"):
         es.update(ci_id, ci)
     else:
-        rd.delete(ci_id)
+        rd.create_or_update({ci_id: json.dumps(ci)}, REDIS_PREFIX_CI)
-        rd.add({ci_id: json.dumps(ci)})
 
-    current_app.logger.info("%d flush.........." % ci_id)
+    current_app.logger.info("{0} flush..........".format(ci_id))
 
 
 @celery.task(name="cmdb.ci_delete", queue=CMDB_QUEUE)
@@ -37,6 +38,29 @@ def ci_delete(ci_id):
     if current_app.config.get("USE_ES"):
         es.delete(ci_id)
     else:
-        rd.delete(ci_id)
+        rd.delete(ci_id, REDIS_PREFIX_CI)
 
-    current_app.logger.info("%d delete.........." % ci_id)
+    current_app.logger.info("{0} delete..........".format(ci_id))
+
+
+@celery.task(name="cmdb.ci_relation_cache", queue=CMDB_QUEUE)
+def ci_relation_cache(parent_id, child_id):
+    children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0]
+    children = json.loads(children) if children is not None else []
+    children.append(child_id)
+
+    rd.create_or_update({parent_id: json.dumps(list(set(children)))}, REDIS_PREFIX_CI_RELATION)
+
+    current_app.logger.info("ADD ci relation cache: {0} -> {1}".format(parent_id, child_id))
+
+
+@celery.task(name="cmdb.ci_relation_delete", queue=CMDB_QUEUE)
+def ci_relation_delete(parent_id, child_id):
+    children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0]
+    children = json.loads(children) if children is not None else []
+    if child_id in children:
+        children.remove(child_id)
+
+    rd.create_or_update({parent_id: json.dumps(list(set(children)))}, REDIS_PREFIX_CI_RELATION)
+
+    current_app.logger.info("DELETE ci relation cache: {0} -> {1}".format(parent_id, child_id))

api/views/cmdb/ci.py

@@ -12,9 +12,9 @@ from api.lib.cmdb.ci import CIManager
 from api.lib.cmdb.const import ExistPolicy
 from api.lib.cmdb.const import ResourceType, PermEnum
 from api.lib.cmdb.const import RetKey
-from api.lib.cmdb.search.db.search import Search as SearchFromDB
+from api.lib.cmdb.search import SearchError
-from api.lib.cmdb.search.es.search import Search as SearchFromES
+from api.lib.cmdb.search.ci.db.search import Search as SearchFromDB
-from api.lib.cmdb.search.db.search import SearchError
+from api.lib.cmdb.search.ci.es.search import Search as SearchFromES
 from api.lib.perm.acl.acl import has_perm_from_args
 from api.lib.perm.auth import auth_abandoned
 from api.lib.utils import get_page
@@ -126,13 +126,13 @@ class CISearchView(APIView):
     def get(self):
         """@params: q: query statement
                     fl: filter by column
-                    count: the number of ci
+                    count/page_size: the number of ci
                     ret_key: id, name, alias
                     facet: statistic
         """
 
         page = get_page(request.values.get("page", 1))
-        count = get_page_size(request.values.get("count"))
+        count = get_page_size(request.values.get("count") or request.values.get("page_size"))
 
         query = request.values.get('q', "")
         fl = handle_arg_list(request.values.get('fl', ""))
@@ -140,8 +140,6 @@ class CISearchView(APIView):
         if ret_key not in (RetKey.NAME, RetKey.ALIAS, RetKey.ID):
             ret_key = RetKey.NAME
         facet = handle_arg_list(request.values.get("facet", ""))
-        fl = list(filter(lambda x: x != "", fl))
-        facet = list(filter(lambda x: x != "", facet))
         sort = request.values.get("sort")
 
         start = time.time()

api/views/cmdb/ci_relation.py

@@ -1,14 +1,62 @@
 # -*- coding:utf-8 -*- 
 
 
+import time
+
+from flask import abort
+from flask import current_app
 from flask import request
 
 from api.lib.cmdb.ci import CIRelationManager
+from api.lib.cmdb.search import SearchError
+from api.lib.cmdb.search.ci_relation.search import Search
+from api.lib.perm.auth import auth_abandoned
 from api.lib.utils import get_page
 from api.lib.utils import get_page_size
+from api.lib.utils import handle_arg_list
 from api.resource import APIView
 
 
+class CIRelationSearchView(APIView):
+    url_prefix = ("/ci_relations/s", "/ci_relations/search")
+
+    @auth_abandoned
+    def get(self):
+        """@params: q: query statement
+                    fl: filter by column
+                    count: the number of ci
+                    root_id: ci id
+                    level: default is 1
+                    facet: statistic
+        """
+
+        page = get_page(request.values.get("page", 1))
+        count = get_page_size(request.values.get("count") or request.values.get("page_size"))
+
+        root_id = request.values.get('root_id')
+        level = request.values.get('level', 1)
+
+        query = request.values.get('q', "")
+        fl = handle_arg_list(request.values.get('fl', ""))
+        facet = handle_arg_list(request.values.get("facet", ""))
+        sort = request.values.get("sort")
+
+        start = time.time()
+        s = Search(root_id, level, query, fl, facet, page, count, sort)
+        try:
+            response, counter, total, page, numfound, facet = s.search()
+        except SearchError as e:
+            return abort(400, str(e))
+        current_app.logger.debug("search time is :{0}".format(time.time() - start))
+
+        return self.jsonify(numfound=numfound,
+                            total=total,
+                            page=page,
+                            facet=facet,
+                            counter=counter,
+                            result=response)
+
+
 class GetSecondCIsView(APIView):
     url_prefix = "/ci_relations/<int:first_ci_id>/second_cis"
 

api/views/cmdb/ci_type_relation.py

@@ -25,18 +25,26 @@ class GetParentsView(APIView):
 
 
 class CITypeRelationView(APIView):
-    url_prefix = "/ci_type_relations/<int:parent_id>/<int:child_id>"
+    url_prefix = ("/ci_type_relations", "/ci_type_relations/<int:parent_id>/<int:child_id>")
+
+    @role_required(RoleEnum.CONFIG)
+    def get(self):
+        res = CITypeRelationManager.get()
+
+        return self.jsonify(res)
 
     @role_required(RoleEnum.CONFIG)
     @args_required("relation_type_id")
     def post(self, parent_id, child_id):
         relation_type_id = request.values.get("relation_type_id")
         ctr_id = CITypeRelationManager.add(parent_id, child_id, relation_type_id)
+
         return self.jsonify(ctr_id=ctr_id)
 
     @role_required(RoleEnum.CONFIG)
     def delete(self, parent_id, child_id):
         CITypeRelationManager.delete_2(parent_id, child_id)
+
         return self.jsonify(code=200, parent_id=parent_id, child_id=child_id)
 
 
@@ -46,4 +54,5 @@ class CITypeRelationDelete2View(APIView):
     @role_required(RoleEnum.CONFIG)
     def delete(self, ctr_id):
         CITypeRelationManager.delete(ctr_id)
+
         return self.jsonify(code=200, ctr_id=ctr_id)