Modify code organization

cmdb-api/api/lib/cmdb/__init__.py

@@ -0,0 +1 @@
+# -*- coding:utf-8 -*-

cmdb-api/api/lib/cmdb/attribute.py

@@ -0,0 +1,190 @@
+# -*- coding:utf-8 -*- 
+
+from flask import abort
+from flask import current_app
+
+from api.extensions import db
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.const import ValueTypeEnum
+from api.lib.cmdb.utils import ValueTypeMap
+from api.lib.decorator import kwargs_required
+from api.models.cmdb import Attribute
+from api.models.cmdb import CITypeAttribute
+from api.models.cmdb import PreferenceShowAttributes
+
+
+class AttributeManager(object):
+    """
+    CI attributes manager
+    """
+
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def get_choice_values(attr_id, value_type):
+        choice_table = ValueTypeMap.choice.get(value_type)
+        choice_values = choice_table.get_by(fl=["value"], attr_id=attr_id)
+        return [choice_value["value"] for choice_value in choice_values]
+
+    @staticmethod
+    def _add_choice_values(_id, value_type, choice_values):
+        choice_table = ValueTypeMap.choice.get(value_type)
+
+        db.session.query(choice_table).filter(choice_table.attr_id == _id).delete()
+        db.session.flush()
+        choice_values = choice_values
+        for v in choice_values:
+            table = choice_table(attr_id=_id, value=v)
+            db.session.add(table)
+        db.session.flush()
+
+    @classmethod
+    def search_attributes(cls, name=None, alias=None, page=1, page_size=None):
+        """
+        :param name: 
+        :param alias: 
+        :param page: 
+        :param page_size: 
+        :return: attribute, if name is None, then return all attributes
+        """
+        if name is not None:
+            attrs = Attribute.get_by_like(name=name)
+        elif alias is not None:
+            attrs = Attribute.get_by_like(alias=alias)
+        else:
+            attrs = Attribute.get_by()
+
+        numfound = len(attrs)
+        attrs = attrs[(page - 1) * page_size:][:page_size]
+        res = list()
+        for attr in attrs:
+            attr["is_choice"] and attr.update(dict(choice_value=cls.get_choice_values(attr["id"], attr["value_type"])))
+            res.append(attr)
+
+        return numfound, res
+
+    def get_attribute_by_name(self, name):
+        attr = Attribute.get_by(name=name, first=True)
+        if attr and attr["is_choice"]:
+            attr.update(dict(choice_value=self.get_choice_values(attr["id"], attr["value_type"])))
+        return attr
+
+    def get_attribute_by_alias(self, alias):
+        attr = Attribute.get_by(alias=alias, first=True)
+        if attr and attr["is_choice"]:
+            attr.update(dict(choice_value=self.get_choice_values(attr["id"], attr["value_type"])))
+        return attr
+
+    def get_attribute_by_id(self, _id):
+        attr = Attribute.get_by_id(_id).to_dict()
+        if attr and attr["is_choice"]:
+            attr.update(dict(choice_value=self.get_choice_values(attr["id"], attr["value_type"])))
+        return attr
+
+    def get_attribute(self, key):
+        attr = AttributeCache.get(key).to_dict()
+        if attr and attr["is_choice"]:
+            attr.update(dict(choice_value=self.get_choice_values(attr["id"], attr["value_type"])))
+        return attr
+
+    @classmethod
+    @kwargs_required("name")
+    def add(cls, **kwargs):
+        choice_value = kwargs.pop("choice_value", [])
+        kwargs.pop("is_choice", None)
+        is_choice = True if choice_value else False
+        name = kwargs.pop("name")
+        alias = kwargs.pop("alias", "")
+        alias = name if not alias else alias
+        Attribute.get_by(name=name, first=True) and abort(400, "attribute name <{0}> is duplicated".format(name))
+        Attribute.get_by(alias=alias, first=True) and abort(400, "attribute alias <{0}> is duplicated".format(name))
+
+        attr = Attribute.create(flush=True,
+                                name=name,
+                                alias=alias,
+                                is_choice=is_choice,
+                                **kwargs)
+
+        if choice_value:
+            cls._add_choice_values(attr.id, attr.value_type, choice_value)
+
+        try:
+            db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error("add attribute error, {0}".format(str(e)))
+            return abort(400, "add attribute <{0}> failed".format(name))
+
+        AttributeCache.clean(attr)
+
+        if current_app.config.get("USE_ES"):
+            from api.extensions import es
+            other = dict()
+            other['index'] = True if attr.is_index else False
+            if attr.value_type == ValueTypeEnum.TEXT:
+                other['analyzer'] = 'ik_max_word'
+                other['search_analyzer'] = 'ik_smart'
+                if attr.is_index:
+                    other["fields"] = {
+                        "keyword": {
+                            "type": "keyword",
+                            "ignore_above": 256
+                        }
+                    }
+            es.update_mapping(name, ValueTypeMap.es_type[attr.value_type], other)
+
+        return attr.id
+
+    def update(self, _id, **kwargs):
+        attr = Attribute.get_by_id(_id) or abort(404, "Attribute <{0}> does not exist".format(_id))
+
+        if kwargs.get("name"):
+            other = Attribute.get_by(name=kwargs['name'], first=True, to_dict=False)
+            if other and other.id != attr.id:
+                return abort(400, "Attribute name <{0}> cannot be duplicate!".format(kwargs['name']))
+        if kwargs.get("alias"):
+            other = Attribute.get_by(alias=kwargs['alias'], first=True, to_dict=False)
+            if other and other.id != attr.id:
+                return abort(400, "Attribute alias <{0}> cannot be duplicate!".format(kwargs['alias']))
+
+        choice_value = kwargs.pop("choice_value", False)
+        is_choice = True if choice_value else False
+
+        attr.update(flush=True, **kwargs)
+
+        if is_choice:
+            self._add_choice_values(attr.id, attr.value_type, choice_value)
+
+        try:
+            db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error("update attribute error, {0}".format(str(e)))
+            return abort(400, "update attribute <{0}> failed".format(_id))
+
+        AttributeCache.clean(attr)
+
+        return attr.id
+
+    @staticmethod
+    def delete(_id):
+        attr = Attribute.get_by_id(_id) or abort(404, "Attribute <{0}> does not exist".format(_id))
+        name = attr.name
+
+        if attr.is_choice:
+            choice_table = ValueTypeMap.choice.get(attr.value_type)
+            db.session.query(choice_table).filter(choice_table.attr_id == _id).delete()  # FIXME: session conflict
+            db.session.flush()
+
+        AttributeCache.clean(attr)
+
+        attr.soft_delete()
+
+        for i in CITypeAttribute.get_by(attr_id=_id, to_dict=False):
+            i.soft_delete()
+
+        for i in PreferenceShowAttributes.get_by(attr_id=_id, to_dict=False):
+            i.soft_delete()
+
+        return name

cmdb-api/api/lib/cmdb/cache.py

@@ -0,0 +1,148 @@
+# -*- coding:utf-8 -*-
+
+from __future__ import unicode_literals
+
+from api.extensions import cache
+from api.models.cmdb import Attribute
+from api.models.cmdb import CIType
+from api.models.cmdb import CITypeAttribute
+from api.models.cmdb import RelationType
+
+
+class AttributeCache(object):
+    PREFIX_ID = 'Field::ID::{0}'
+    PREFIX_NAME = 'Field::Name::{0}'
+    PREFIX_ALIAS = 'Field::Alias::{0}'
+
+    @classmethod
+    def get(cls, key):
+        if key is None:
+            return
+        attr = cache.get(cls.PREFIX_NAME.format(key))
+        attr = attr or cache.get(cls.PREFIX_ID.format(key))
+        attr = attr or cache.get(cls.PREFIX_ALIAS.format(key))
+
+        if attr is None:
+            attr = Attribute.get_by(name=key, first=True, to_dict=False)
+            attr = attr or Attribute.get_by_id(key)
+            attr = attr or Attribute.get_by(alias=key, first=True, to_dict=False)
+            if attr is not None:
+                cls.set(attr)
+        return attr
+
+    @classmethod
+    def set(cls, attr):
+        cache.set(cls.PREFIX_ID.format(attr.id), attr)
+        cache.set(cls.PREFIX_NAME.format(attr.name), attr)
+        cache.set(cls.PREFIX_ALIAS.format(attr.alias), attr)
+
+    @classmethod
+    def clean(cls, attr):
+        cache.delete(cls.PREFIX_ID.format(attr.id))
+        cache.delete(cls.PREFIX_NAME.format(attr.name))
+        cache.delete(cls.PREFIX_ALIAS.format(attr.alias))
+
+
+class CITypeCache(object):
+    PREFIX_ID = "CIType::ID::{0}"
+    PREFIX_NAME = "CIType::Name::{0}"
+    PREFIX_ALIAS = "CIType::Alias::{0}"
+
+    @classmethod
+    def get(cls, key):
+        if key is None:
+            return
+        ct = cache.get(cls.PREFIX_NAME.format(key))
+        ct = ct or cache.get(cls.PREFIX_ID.format(key))
+        ct = ct or cache.get(cls.PREFIX_ALIAS.format(key))
+        if ct is None:
+            ct = CIType.get_by(name=key, first=True, to_dict=False)
+            ct = ct or CIType.get_by_id(key)
+            ct = ct or CIType.get_by(alias=key, first=True, to_dict=False)
+            if ct is not None:
+                cls.set(ct)
+        return ct
+
+    @classmethod
+    def set(cls, ct):
+        cache.set(cls.PREFIX_NAME.format(ct.name), ct)
+        cache.set(cls.PREFIX_ID.format(ct.id), ct)
+        cache.set(cls.PREFIX_ALIAS.format(ct.alias), ct)
+
+    @classmethod
+    def clean(cls, key):
+        ct = cls.get(key)
+        if ct is not None:
+            cache.delete(cls.PREFIX_NAME.format(ct.name))
+            cache.delete(cls.PREFIX_ID.format(ct.id))
+            cache.delete(cls.PREFIX_ALIAS.format(ct.alias))
+
+
+class RelationTypeCache(object):
+    PREFIX_ID = "RelationType::ID::{0}"
+    PREFIX_NAME = "RelationType::Name::{0}"
+
+    @classmethod
+    def get(cls, key):
+        if key is None:
+            return
+        ct = cache.get(cls.PREFIX_NAME.format(key))
+        ct = ct or cache.get(cls.PREFIX_ID.format(key))
+        if ct is None:
+            ct = RelationType.get_by(name=key, first=True, to_dict=False) or RelationType.get_by_id(key)
+            if ct is not None:
+                cls.set(ct)
+        return ct
+
+    @classmethod
+    def set(cls, ct):
+        cache.set(cls.PREFIX_NAME.format(ct.name), ct)
+        cache.set(cls.PREFIX_ID.format(ct.id), ct)
+
+    @classmethod
+    def clean(cls, key):
+        ct = cls.get(key)
+        if ct is not None:
+            cache.delete(cls.PREFIX_NAME.format(ct.name))
+            cache.delete(cls.PREFIX_ID.format(ct.id))
+
+
+class CITypeAttributeCache(object):
+    """
+    key is type_id or type_name
+    """
+
+    PREFIX_ID = "CITypeAttribute::ID::{0}"
+    PREFIX_NAME = "CITypeAttribute::Name::{0}"
+
+    @classmethod
+    def get(cls, key):
+        if key is None:
+            return
+
+        attrs = cache.get(cls.PREFIX_NAME.format(key))
+        attrs = attrs or cache.get(cls.PREFIX_ID.format(key))
+        if not attrs:
+            attrs = CITypeAttribute.get_by(type_id=key, to_dict=False)
+            if not attrs:
+                ci_type = CIType.get_by(name=key, first=True, to_dict=False)
+                if ci_type is not None:
+                    attrs = CITypeAttribute.get_by(type_id=ci_type.id, to_dict=False)
+            if attrs is not None:
+                cls.set(key, attrs)
+        return attrs
+
+    @classmethod
+    def set(cls, key, values):
+        ci_type = CITypeCache.get(key)
+        if ci_type is not None:
+            cache.set(cls.PREFIX_ID.format(ci_type.id), values)
+            cache.set(cls.PREFIX_NAME.format(ci_type.name), values)
+
+    @classmethod
+    def clean(cls, key):
+        ci_type = CITypeCache.get(key)
+        attrs = cls.get(key)
+        if attrs is not None and ci_type:
+            cache.delete(cls.PREFIX_ID.format(ci_type.id))
+            cache.delete(cls.PREFIX_NAME.format(ci_type.name))

cmdb-api/api/lib/cmdb/ci.py

@@ -0,0 +1,545 @@
+# -*- coding:utf-8 -*- 
+
+
+import datetime
+import json
+
+from flask import abort
+from flask import current_app
+from werkzeug.exceptions import BadRequest
+
+from api.extensions import db
+from api.extensions import rd
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.cache import CITypeCache
+from api.lib.cmdb.ci_type import CITypeAttributeManager
+from api.lib.cmdb.ci_type import CITypeManager
+from api.lib.cmdb.const import CMDB_QUEUE
+from api.lib.cmdb.const import ExistPolicy
+from api.lib.cmdb.const import OperateType
+from api.lib.cmdb.const import REDIS_PREFIX_CI
+from api.lib.cmdb.const import RetKey
+from api.lib.cmdb.history import AttributeHistoryManger
+from api.lib.cmdb.history import CIRelationHistoryManager
+from api.lib.cmdb.search.ci.db.query_sql import QUERY_CIS_BY_IDS
+from api.lib.cmdb.search.ci.db.query_sql import QUERY_CIS_BY_VALUE_TABLE
+from api.lib.cmdb.utils import TableMap
+from api.lib.cmdb.utils import ValueTypeMap
+from api.lib.cmdb.value import AttributeValueManager
+from api.lib.decorator import kwargs_required
+from api.lib.utils import handle_arg_list
+from api.models.cmdb import CI
+from api.models.cmdb import CIRelation
+from api.models.cmdb import CITypeAttribute
+from api.models.cmdb import CITypeRelation
+from api.tasks.cmdb import ci_cache
+from api.tasks.cmdb import ci_delete
+from api.tasks.cmdb import ci_relation_cache
+from api.tasks.cmdb import ci_relation_delete
+
+
+class CIManager(object):
+    """ manage CI interface
+    """
+
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def get_type_name(ci_id):
+        ci = CI.get_by_id(ci_id) or abort(404, "CI <{0}> is not existed".format(ci_id))
+        return CITypeCache.get(ci.type_id).name
+
+    @staticmethod
+    def confirm_ci_existed(ci_id):
+        return CI.get_by_id(ci_id) or abort(404, "CI <{0}> is not existed".format(ci_id))
+
+    @classmethod
+    def get_ci_by_id(cls, ci_id, ret_key=RetKey.NAME, fields=None, need_children=True):
+        """
+        
+        :param ci_id: 
+        :param ret_key: name, id, or alias
+        :param fields:  attribute list
+        :param need_children: 
+        :return: 
+        """
+
+        ci = CI.get_by_id(ci_id) or abort(404, "CI <{0}> is not existed".format(ci_id))
+
+        res = dict()
+
+        if need_children:
+            children = CIRelationManager.get_children(ci_id, ret_key=ret_key)  # one floor
+            res.update(children)
+
+        ci_type = CITypeCache.get(ci.type_id)
+        res["ci_type"] = ci_type.name
+
+        res.update(cls.get_cis_by_ids([str(ci_id)], fields=fields, ret_key=ret_key))
+
+        res['_type'] = ci_type.id
+        res['_id'] = ci_id
+
+        return res
+
+    @staticmethod
+    def get_ci_by_id_from_db(ci_id, ret_key=RetKey.NAME, fields=None, need_children=True, use_master=False):
+        """
+        
+        :param ci_id: 
+        :param ret_key: name, id or alias
+        :param fields: list
+        :param need_children: 
+        :param use_master: whether to use master db
+        :return: 
+        """
+
+        ci = CI.get_by_id(ci_id) or abort(404, "CI <{0}> is not existed".format(ci_id))
+
+        res = dict()
+
+        if need_children:
+            children = CIRelationManager.get_children(ci_id, ret_key=ret_key)  # one floor
+            res.update(children)
+
+        ci_type = CITypeCache.get(ci.type_id)
+        res["ci_type"] = ci_type.name
+
+        fields = CITypeAttributeManager.get_attr_names_by_type_id(ci.type_id) if not fields else fields
+        unique_key = AttributeCache.get(ci_type.unique_id)
+        _res = AttributeValueManager().get_attr_values(fields,
+                                                       ci_id,
+                                                       ret_key=ret_key,
+                                                       unique_key=unique_key,
+                                                       use_master=use_master)
+        res.update(_res)
+
+        res['type_id'] = ci_type.id
+        res['ci_id'] = ci_id
+
+        return res
+
+    def get_ci_by_ids(self, ci_id_list, ret_key=RetKey.NAME, fields=None):
+        return [self.get_ci_by_id(ci_id, ret_key=ret_key, fields=fields) for ci_id in ci_id_list]
+
+    @classmethod
+    def get_cis_by_type(cls, type_id, ret_key=RetKey.NAME, fields="", page=1, per_page=None):
+        cis = db.session.query(CI.id).filter(CI.type_id == type_id).filter(CI.deleted.is_(False))
+        numfound = cis.count()
+
+        cis = cis.offset((page - 1) * per_page).limit(per_page)
+        ci_ids = [str(ci.id) for ci in cis]
+        res = cls.get_cis_by_ids(ci_ids, ret_key, fields)
+
+        return numfound, page, res
+
+    @staticmethod
+    def ci_is_exist(unique_key, unique_value):
+        """
+        
+        :param unique_key: is a attribute
+        :param unique_value: 
+        :return: 
+        """
+        value_table = TableMap(attr_name=unique_key.name).table
+        unique = value_table.get_by(attr_id=unique_key.id,
+                                    value=unique_value,
+                                    to_dict=False,
+                                    first=True)
+        if unique:
+            return CI.get_by_id(unique.ci_id)
+
+    @staticmethod
+    def _delete_ci_by_id(ci_id):
+        ci = CI.get_by_id(ci_id)
+        ci.delete()  # TODO: soft delete
+
+    @classmethod
+    def add(cls, ci_type_name, exist_policy=ExistPolicy.REPLACE, _no_attribute_policy=ExistPolicy.IGNORE, **ci_dict):
+        """
+        
+        :param ci_type_name: 
+        :param exist_policy: replace or reject or need
+        :param _no_attribute_policy: ignore or reject
+        :param ci_dict: 
+        :return: 
+        """
+
+        ci_type = CITypeManager.check_is_existed(ci_type_name)
+
+        unique_key = AttributeCache.get(ci_type.unique_id) or abort(400, 'illegality unique attribute')
+
+        unique_value = ci_dict.get(unique_key.name)
+        unique_value = unique_value or ci_dict.get(unique_key.alias)
+        unique_value = unique_value or ci_dict.get(unique_key.id)
+        unique_value = unique_value or abort(400, '{0} missing'.format(unique_key.name))
+
+        existed = cls.ci_is_exist(unique_key, unique_value)
+        if existed is not None:
+            if exist_policy == ExistPolicy.REJECT:
+                return abort(400, 'CI is already existed')
+            if existed.type_id != ci_type.id:
+                existed.update(type_id=ci_type.id)
+            ci = existed
+        else:
+            if exist_policy == ExistPolicy.NEED:
+                return abort(404, 'CI <{0}> does not exist'.format(unique_value))
+            ci = CI.create(type_id=ci_type.id)
+
+        value_manager = AttributeValueManager()
+        for p, v in ci_dict.items():
+            try:
+                value_manager.create_or_update_attr_value(p, v, ci.id, _no_attribute_policy)
+            except BadRequest as e:
+                if existed is None:
+                    cls.delete(ci.id)
+                raise e
+
+        ci_cache.apply_async([ci.id], queue=CMDB_QUEUE)
+
+        return ci.id
+
+    def update(self, ci_id, **ci_dict):
+        self.confirm_ci_existed(ci_id)
+        value_manager = AttributeValueManager()
+        for p, v in ci_dict.items():
+            try:
+                value_manager.create_or_update_attr_value(p, v, ci_id)
+            except BadRequest as e:
+                raise e
+
+        ci_cache.apply_async([ci_id], queue=CMDB_QUEUE)
+
+    @staticmethod
+    def update_unique_value(ci_id, unique_name, unique_value):
+        CI.get_by_id(ci_id) or abort(404, "CI <{0}> is not found".format(ci_id))
+
+        AttributeValueManager().create_or_update_attr_value(unique_name, unique_value, ci_id)
+
+        ci_cache.apply_async([ci_id], queue=CMDB_QUEUE)
+
+    @staticmethod
+    def delete(ci_id):
+        ci = CI.get_by_id(ci_id) or abort(404, "CI <{0}> is not found".format(ci_id))
+
+        attrs = CITypeAttribute.get_by(type_id=ci.type_id, to_dict=False)
+        attr_names = set([AttributeCache.get(attr.attr_id).name for attr in attrs])
+        for attr_name in attr_names:
+            value_table = TableMap(attr_name=attr_name).table
+            for item in value_table.get_by(ci_id=ci_id, to_dict=False):
+                item.delete()
+
+        for item in CIRelation.get_by(first_ci_id=ci_id, to_dict=False):
+            ci_relation_delete.apply_async(args=(item.first_ci_id, item.second_ci_id), queue=CMDB_QUEUE)
+            item.delete()
+
+        for item in CIRelation.get_by(second_ci_id=ci_id, to_dict=False):
+            ci_relation_delete.apply_async(args=(item.first_ci_id, item.second_ci_id), queue=CMDB_QUEUE)
+            item.delete()
+
+        ci.delete()  # TODO: soft delete
+
+        AttributeHistoryManger.add(ci_id, [(None, OperateType.DELETE, None, None)])
+
+        ci_delete.apply_async([ci.id], queue=CMDB_QUEUE)
+
+        return ci_id
+
+    @staticmethod
+    def add_heartbeat(ci_type, unique_value):
+        ci_type = CITypeManager().check_is_existed(ci_type)
+
+        unique_key = AttributeCache.get(ci_type.unique_id)
+        value_table = TableMap(attr_name=unique_key.name).table
+
+        v = value_table.get_by(attr_id=unique_key.id,
+                               value=unique_value,
+                               to_dict=False,
+                               first=True) \
+            or abort(404, "not found")
+
+        ci = CI.get_by_id(v.ci_id) or abort(404, "CI <{0}> is not found".format(v.ci_id))
+
+        ci.update(heartbeat=datetime.datetime.now())
+
+    @classmethod
+    @kwargs_required("type_id", "page")
+    def get_heartbeat(cls, **kwargs):
+        query = db.session.query(CI.id, CI.heartbeat).filter(CI.deleted.is_(False))
+
+        expire = datetime.datetime.now() - datetime.timedelta(minutes=72)
+        type_ids = handle_arg_list(kwargs["type_id"])
+
+        query = query.filter(CI.type_id.in_(type_ids))
+
+        page = kwargs.get("page")
+        agent_status = kwargs.get("agent_status")
+        if agent_status == -1:
+            query = query.filter(CI.heartbeat.is_(None))
+        elif agent_status == 0:
+            query = query.filter(CI.heartbeat <= expire)
+        elif agent_status == 1:
+            query = query.filter(CI.heartbeat > expire)
+
+        numfound = query.count()
+        per_page_count = current_app.config.get("DEFAULT_PAGE_COUNT")
+        cis = query.offset((page - 1) * per_page_count).limit(per_page_count).all()
+        ci_ids = [ci.id for ci in cis]
+        heartbeat_dict = {}
+        for ci in cis:
+            if agent_status is not None:
+                heartbeat_dict[ci.id] = agent_status
+            else:
+                if ci.heartbeat is None:
+                    heartbeat_dict[ci.id] = -1
+                elif ci.heartbeat <= expire:
+                    heartbeat_dict[ci.id] = 0
+                else:
+                    heartbeat_dict[ci.id] = 1
+        current_app.logger.debug(heartbeat_dict)
+        ci_ids = list(map(str, ci_ids))
+        res = cls.get_cis_by_ids(ci_ids, fields=["hostname", "private_ip"])
+        result = [(i.get("hostname"), i.get("private_ip")[0], i.get("ci_type"),
+                   heartbeat_dict.get(i.get("_id"))) for i in res
+                  if i.get("private_ip")]
+        return numfound, result
+
+    @staticmethod
+    def _get_cis_from_cache(ci_ids, ret_key=RetKey.NAME, fields=None):
+        res = rd.get(ci_ids, REDIS_PREFIX_CI)
+        if res is not None and None not in res and ret_key == RetKey.NAME:
+            res = list(map(json.loads, res))
+            if not fields:
+                return res
+            else:
+                _res = []
+                for d in res:
+                    _d = dict()
+                    _d["_id"], _d["_type"] = d.get("_id"), d.get("_type")
+                    _d["ci_type"] = d.get("ci_type")
+                    for field in fields:
+                        _d[field] = d.get(field)
+                    _res.append(_d)
+                return _res
+
+    @staticmethod
+    def _get_cis_from_db(ci_ids, ret_key=RetKey.NAME, fields=None, value_tables=None):
+        if not fields:
+            filter_fields_sql = ""
+        else:
+            _fields = list()
+            for field in fields:
+                attr = AttributeCache.get(field)
+                if attr is not None:
+                    _fields.append(str(attr.id))
+            filter_fields_sql = "WHERE A.attr_id in ({0})".format(",".join(_fields))
+
+        ci_ids = ",".join(ci_ids)
+        if value_tables is None:
+            value_tables = ValueTypeMap.table_name.values()
+
+        value_sql = " UNION ".join([QUERY_CIS_BY_VALUE_TABLE.format(value_table, ci_ids)
+                                    for value_table in value_tables])
+        query_sql = QUERY_CIS_BY_IDS.format(filter_fields_sql, value_sql)
+        # current_app.logger.debug(query_sql)
+        cis = db.session.execute(query_sql).fetchall()
+        ci_set = set()
+        res = list()
+        ci_dict = dict()
+        for ci_id, type_id, attr_id, attr_name, attr_alias, value, value_type, is_list in cis:
+            if ci_id not in ci_set:
+                ci_dict = dict()
+                ci_type = CITypeCache.get(type_id)
+                ci_dict["ci_id"] = ci_id
+                ci_dict["ci_type"] = type_id
+                ci_dict["ci_type"] = ci_type.name
+                ci_dict["ci_type_alias"] = ci_type.alias
+                ci_set.add(ci_id)
+                res.append(ci_dict)
+
+            if ret_key == RetKey.NAME:
+                attr_key = attr_name
+            elif ret_key == RetKey.ALIAS:
+                attr_key = attr_alias
+            elif ret_key == RetKey.ID:
+                attr_key = attr_id
+            else:
+                return abort(400, "invalid ret key")
+
+            value = ValueTypeMap.serialize2[value_type](value)
+            if is_list:
+                ci_dict.setdefault(attr_key, []).append(value)
+            else:
+                ci_dict[attr_key] = value
+
+        return res
+
+    @classmethod
+    def get_cis_by_ids(cls, ci_ids, ret_key=RetKey.NAME, fields=None, value_tables=None):
+        """
+
+        :param ci_ids: list of CI instance ID, eg. ['1', '2']
+        :param ret_key: name, id or alias
+        :param fields: 
+        :param value_tables: 
+        :return: 
+        """
+
+        if not ci_ids:
+            return []
+
+        fields = [] if fields is None or not isinstance(fields, list) else fields
+
+        ci_id_tuple = tuple(map(int, ci_ids))
+        res = cls._get_cis_from_cache(ci_id_tuple, ret_key, fields)
+        if res is not None:
+            return res
+
+        current_app.logger.warning("cache not hit...............")
+        return cls._get_cis_from_db(ci_ids, ret_key, fields, value_tables)
+
+
+class CIRelationManager(object):
+    """
+    Manage relation between CIs
+    """
+
+    def __init__(self):
+        pass
+
+    @classmethod
+    def get_children(cls, ci_id, ret_key=RetKey.NAME):
+        second_cis = CIRelation.get_by(first_ci_id=ci_id, to_dict=False)
+        second_ci_ids = (second_ci.second_ci_id for second_ci in second_cis)
+        ci_type2ci_ids = dict()
+        for ci_id in second_ci_ids:
+            type_id = CI.get_by_id(ci_id).type_id
+            ci_type2ci_ids.setdefault(type_id, []).append(ci_id)
+
+        res = {}
+        for type_id in ci_type2ci_ids:
+            ci_type = CITypeCache.get(type_id)
+            children = CIManager.get_cis_by_ids(list(map(str, ci_type2ci_ids[type_id])), ret_key=ret_key)
+            res[ci_type.name] = children
+        return res
+
+    @staticmethod
+    def get_second_cis(first_ci_id, relation_type_id=None, page=1, per_page=None):
+        second_cis = db.session.query(CI.id).filter(CI.deleted.is_(False)).join(
+            CIRelation, CIRelation.second_ci_id == CI.id).filter(
+            CIRelation.first_ci_id == first_ci_id).filter(CIRelation.deleted.is_(False))
+
+        if relation_type_id is not None:
+            second_cis = second_cis.filter(CIRelation.relation_type_id == relation_type_id)
+
+        numfound = second_cis.count()
+        if per_page != "all":
+            second_cis = second_cis.offset((page - 1) * per_page).limit(per_page).all()
+        ci_ids = [str(son.id) for son in second_cis]
+        result = CIManager.get_cis_by_ids(ci_ids)
+
+        return numfound, len(ci_ids), result
+
+    @staticmethod
+    def _sort_handler(sort_by, query_sql):
+
+        if sort_by.startswith("+"):
+            sort_type = "asc"
+            sort_by = sort_by[1:]
+        elif sort_by.startswith("-"):
+            sort_type = "desc"
+            sort_by = sort_by[1:]
+        else:
+            sort_type = "asc"
+        attr = AttributeCache.get(sort_by)
+        if attr is None:
+            return query_sql
+
+        attr_id = attr.id
+        value_table = TableMap(attr_name=sort_by).table
+
+        ci_table = query_sql.subquery()
+        query_sql = db.session.query(ci_table.c.id, value_table.value).join(
+            value_table, value_table.ci_id == ci_table.c.id).filter(
+            value_table.attr_id == attr_id).filter(ci_table.deleted.is_(False)).order_by(
+            getattr(value_table.value, sort_type)())
+
+        return query_sql
+
+    @classmethod
+    def get_first_cis(cls, second_ci, relation_type_id=None, page=1, per_page=None):
+        first_cis = db.session.query(CIRelation.first_ci_id).filter(
+            CIRelation.second_ci_id == second_ci).filter(CIRelation.deleted.is_(False))
+        if relation_type_id is not None:
+            first_cis = first_cis.filter(CIRelation.relation_type_id == relation_type_id)
+
+        numfound = first_cis.count()
+        if per_page != "all":
+            first_cis = first_cis.offset((page - 1) * per_page).limit(per_page).all()
+
+        first_ci_ids = [str(first_ci.first_ci_id) for first_ci in first_cis]
+        result = CIManager.get_cis_by_ids(first_ci_ids)
+
+        return numfound, len(first_ci_ids), result
+
+    @classmethod
+    def add(cls, first_ci_id, second_ci_id, more=None, relation_type_id=None):
+
+        first_ci = CIManager.confirm_ci_existed(first_ci_id)
+        second_ci = CIManager.confirm_ci_existed(second_ci_id)
+
+        existed = CIRelation.get_by(first_ci_id=first_ci_id,
+                                    second_ci_id=second_ci_id,
+                                    to_dict=False,
+                                    first=True)
+        if existed is not None:
+            if existed.relation_type_id != relation_type_id:
+                existed.update(relation_type_id=relation_type_id)
+
+                CIRelationHistoryManager().add(existed, OperateType.UPDATE)
+        else:
+            if relation_type_id is None:
+                type_relation = CITypeRelation.get_by(parent_id=first_ci.type_id,
+                                                      child_id=second_ci.type_id,
+                                                      first=True,
+                                                      to_dict=False)
+                relation_type_id = type_relation and type_relation.relation_type_id
+                relation_type_id or abort(404, "Relation {0} <-> {1} is not found".format(
+                    first_ci.ci_type.name, second_ci.ci_type.name))
+
+            existed = CIRelation.create(first_ci_id=first_ci_id,
+                                        second_ci_id=second_ci_id,
+                                        relation_type_id=relation_type_id)
+
+            CIRelationHistoryManager().add(existed, OperateType.ADD)
+
+            ci_relation_cache.apply_async(args=(first_ci_id, second_ci_id), queue=CMDB_QUEUE)
+
+        if more is not None:
+            existed.upadte(more=more)
+
+        return existed.id
+
+    @staticmethod
+    def delete(cr_id):
+        cr = CIRelation.get_by_id(cr_id) or abort(404, "CIRelation <{0}> is not existed".format(cr_id))
+        cr.delete()
+
+        his_manager = CIRelationHistoryManager()
+        his_manager.add(cr, operate_type=OperateType.DELETE)
+
+        ci_relation_delete.apply_async(args=(cr.first_ci_id, cr.second_ci_id), queue=CMDB_QUEUE)
+
+        return cr_id
+
+    @classmethod
+    def delete_2(cls, first_ci_id, second_ci_id):
+        cr = CIRelation.get_by(first_ci_id=first_ci_id,
+                               second_ci_id=second_ci_id,
+                               to_dict=False,
+                               first=True)
+
+        ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id), queue=CMDB_QUEUE)
+
+        return cls.delete(cr.id)

cmdb-api/api/lib/cmdb/ci_type.py

@@ -0,0 +1,443 @@
+# -*- coding:utf-8 -*- 
+
+
+from flask import abort
+from flask import current_app
+
+from api.extensions import db
+from api.lib.cmdb.attribute import AttributeManager
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.cache import CITypeAttributeCache
+from api.lib.cmdb.cache import CITypeCache
+from api.lib.decorator import kwargs_required
+from api.models.cmdb import CI
+from api.models.cmdb import CIType
+from api.models.cmdb import CITypeAttribute
+from api.models.cmdb import CITypeAttributeGroup
+from api.models.cmdb import CITypeAttributeGroupItem
+from api.models.cmdb import CITypeGroup
+from api.models.cmdb import CITypeGroupItem
+from api.models.cmdb import CITypeRelation
+from api.models.cmdb import PreferenceTreeView
+from api.models.cmdb import PreferenceShowAttributes
+
+
+class CITypeManager(object):
+    """
+    manage CIType
+    """
+
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def get_name_by_id(type_id):
+        return CITypeCache.get(type_id).name
+
+    @staticmethod
+    def check_is_existed(key):
+        return CITypeCache.get(key) or abort(404, "CIType <{0}> is not existed".format(key))
+
+    @staticmethod
+    def get_ci_types(type_name=None):
+        ci_types = CIType.get_by() if type_name is None else CIType.get_by_like(name=type_name)
+        res = list()
+        for type_dict in ci_types:
+            type_dict["unique_key"] = AttributeCache.get(type_dict["unique_id"]).name
+            res.append(type_dict)
+        return res
+
+    @staticmethod
+    def query(_type):
+        ci_type = CITypeCache.get(_type) or abort(404, "CIType <{0}> is not found".format(_type))
+        return ci_type.to_dict()
+
+    @classmethod
+    @kwargs_required("name")
+    def add(cls, **kwargs):
+        unique_key = kwargs.pop("unique_key", None)
+        unique_key = AttributeCache.get(unique_key) or abort(404, "Unique key is not defined")
+
+        CIType.get_by(name=kwargs['name']) and abort(404, "CIType <{0}> is already existed".format(kwargs.get("name")))
+
+        kwargs["alias"] = kwargs["name"] if not kwargs.get("alias") else kwargs["alias"]
+
+        kwargs["unique_id"] = unique_key.id
+        ci_type = CIType.create(**kwargs)
+
+        CITypeAttributeManager.add(ci_type.id, [unique_key.id], is_required=True)
+
+        CITypeCache.clean(ci_type.name)
+
+        if current_app.config.get("USE_ACL"):
+            from api.lib.perm.acl.acl import ACLManager
+            from api.lib.cmdb.const import ResourceTypeEnum, RoleEnum, PermEnum
+            ACLManager().add_resource(ci_type.name, ResourceTypeEnum.CI)
+            ACLManager().grant_resource_to_role(ci_type.name,
+                                                RoleEnum.CMDB_READ_ALL,
+                                                ResourceTypeEnum.CI,
+                                                permissions=[PermEnum.READ])
+
+        return ci_type.id
+
+    @classmethod
+    def update(cls, type_id, **kwargs):
+
+        ci_type = cls.check_is_existed(type_id)
+
+        unique_key = kwargs.pop("unique_key", None)
+        unique_key = AttributeCache.get(unique_key)
+        if unique_key is not None:
+            kwargs["unique_id"] = unique_key.id
+            type_attr = CITypeAttribute.get_by(type_id=type_id,
+                                               attr_id=unique_key.id,
+                                               first=True,
+                                               to_dict=False)
+            if type_attr is None:
+                CITypeAttributeManager.add(type_id, [unique_key.id], is_required=True)
+
+        ci_type.update(**kwargs)
+
+        CITypeCache.clean(type_id)
+
+        return type_id
+
+    @classmethod
+    def set_enabled(cls, type_id, enabled=True):
+        ci_type = cls.check_is_existed(type_id)
+        ci_type.update(enabled=enabled)
+        return type_id
+
+    @classmethod
+    def delete(cls, type_id):
+        ci_type = cls.check_is_existed(type_id)
+
+        if CI.get_by(type_id=type_id, first=True, to_dict=False) is not None:
+            return abort(400, "cannot delete, because CI instance exists")
+
+        for item in CITypeRelation.get_by(parent_id=type_id, to_dict=False):
+            item.soft_delete()
+
+        for item in CITypeRelation.get_by(child_id=type_id, to_dict=False):
+            item.soft_delete()
+
+        for item in PreferenceTreeView.get_by(type_id=type_id, to_dict=False):
+            item.soft_delete()
+
+        for item in PreferenceShowAttributes.get_by(type_id=type_id, to_dict=False):
+            item.soft_delete()
+
+        ci_type.soft_delete()
+
+        CITypeCache.clean(type_id)
+
+        if current_app.config.get("USE_ACL"):
+            from api.lib.perm.acl.acl import ACLManager
+            from api.lib.cmdb.const import ResourceTypeEnum, RoleEnum, PermEnum
+            ACLManager().del_resource(ci_type.name, ResourceTypeEnum.CI)
+
+
+class CITypeGroupManager(object):
+    @staticmethod
+    def get(need_other=None):
+        groups = CITypeGroup.get_by()
+        group_types = set()
+        for group in groups:
+            for t in sorted(CITypeGroupItem.get_by(group_id=group['id']), key=lambda x: x['order']):
+                group.setdefault("ci_types", []).append(CITypeCache.get(t['type_id']).to_dict())
+                group_types.add(t["type_id"])
+
+        if need_other:
+            ci_types = CITypeManager.get_ci_types()
+            other_types = dict(ci_types=[ci_type for ci_type in ci_types if ci_type["id"] not in group_types])
+            groups.append(other_types)
+
+        return groups
+
+    @staticmethod
+    def add(name):
+        CITypeGroup.get_by(name=name, first=True) and abort(400, "Group {0} does exist".format(name))
+        return CITypeGroup.create(name=name)
+
+    @staticmethod
+    def update(gid, name, type_ids):
+        """
+        update all
+        :param gid: 
+        :param name: 
+        :param type_ids: 
+        :return: 
+        """
+        existed = CITypeGroup.get_by_id(gid) or abort(404, "Group <{0}> does not exist".format(gid))
+        if name is not None:
+            existed.update(name=name)
+
+        for idx, type_id in enumerate(type_ids):
+
+            item = CITypeGroupItem.get_by(group_id=gid, type_id=type_id, first=True, to_dict=False)
+            if item is not None:
+                item.update(order=idx)
+            else:
+                CITypeGroupItem.create(group_id=gid, type_id=type_id, order=idx)
+
+    @staticmethod
+    def delete(gid):
+        existed = CITypeGroup.get_by_id(gid) or abort(404, "Group <{0}> does not exist".format(gid))
+
+        items = CITypeGroupItem.get_by(group_id=gid, to_dict=False)
+        for item in items:
+            item.soft_delete()
+
+        existed.soft_delete()
+
+
+class CITypeAttributeManager(object):
+    """
+    manage CIType's attributes, include query, add, update, delete
+    """
+
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def get_attr_names_by_type_id(type_id):
+        return [AttributeCache.get(attr.attr_id).name for attr in CITypeAttributeCache.get(type_id)]
+
+    @staticmethod
+    def get_attributes_by_type_id(type_id):
+        attrs = CITypeAttributeCache.get(type_id)
+        result = list()
+        for attr in sorted(attrs, key=lambda x: (x.order, x.id)):
+            attr_dict = AttributeManager().get_attribute(attr.attr_id)
+            attr_dict["is_required"] = attr.is_required
+            attr_dict["order"] = attr.order
+            attr_dict["default_show"] = attr.default_show
+            result.append(attr_dict)
+        return result
+
+    @staticmethod
+    def _check(type_id, attr_ids):
+        CITypeManager.check_is_existed(type_id)
+
+        if not attr_ids or not isinstance(attr_ids, list):
+            return abort(400, "Attributes are required")
+
+        for attr_id in attr_ids:
+            AttributeCache.get(attr_id) or abort(404, "Attribute <{0}> is not existed".format(attr_id))
+
+    @classmethod
+    def add(cls, type_id, attr_ids=None, **kwargs):
+        """
+        add attributes to CIType
+        :param type_id: 
+        :param attr_ids: list
+        :param kwargs: 
+        :return: 
+        """
+        cls._check(type_id, attr_ids)
+
+        for attr_id in attr_ids:
+            existed = CITypeAttribute.get_by(type_id=type_id,
+                                             attr_id=attr_id,
+                                             first=True,
+                                             to_dict=False)
+            if existed is not None:
+                continue
+
+            current_app.logger.debug(attr_id)
+            CITypeAttribute.create(type_id=type_id, attr_id=attr_id, **kwargs)
+
+        CITypeAttributeCache.clean(type_id)
+
+    @classmethod
+    def update(cls, type_id, attributes):
+        """
+        update attributes to CIType
+        :param type_id: 
+        :param attributes: list
+        :return: 
+        """
+        cls._check(type_id, [i.get('attr_id') for i in attributes])
+
+        for attr in attributes:
+            existed = CITypeAttribute.get_by(type_id=type_id,
+                                             attr_id=attr.get("attr_id"),
+                                             first=True,
+                                             to_dict=False)
+            if existed is None:
+                continue
+
+            existed.update(**attr)
+
+        CITypeAttributeCache.clean(type_id)
+
+    @classmethod
+    def delete(cls, type_id, attr_ids=None):
+        """
+        delete attributes from CIType
+        :param type_id: 
+        :param attr_ids: list
+        :return: 
+        """
+        cls._check(type_id, attr_ids)
+
+        for attr_id in attr_ids:
+            existed = CITypeAttribute.get_by(type_id=type_id,
+                                             attr_id=attr_id,
+                                             first=True,
+                                             to_dict=False)
+            if existed is not None:
+                existed.soft_delete()
+
+        CITypeAttributeCache.clean(type_id)
+
+
+class CITypeRelationManager(object):
+    """
+    manage relation between CITypes
+    """
+
+    @staticmethod
+    def get():
+        res = CITypeRelation.get_by(to_dict=False)
+        for idx, item in enumerate(res):
+            _item = item.to_dict()
+            res[idx] = _item
+            res[idx]['parent'] = item.parent.to_dict()
+            res[idx]['child'] = item.child.to_dict()
+            res[idx]['relation_type'] = item.relation_type.to_dict()
+
+        return res
+
+    @staticmethod
+    def get_child_type_ids(type_id, level):
+        ids = [type_id]
+        query = db.session.query(CITypeRelation).filter(CITypeRelation.deleted.is_(False))
+        for _ in range(0, level):
+            ids = [i.child_id for i in query.filter(CITypeRelation.parent_id.in_(ids))]
+
+        return ids
+
+    @staticmethod
+    def _wrap_relation_type_dict(type_id, relation_inst):
+        ci_type_dict = CITypeCache.get(type_id).to_dict()
+        ci_type_dict["ctr_id"] = relation_inst.id
+        ci_type_dict["attributes"] = CITypeAttributeManager.get_attributes_by_type_id(ci_type_dict["id"])
+        ci_type_dict["relation_type"] = relation_inst.relation_type.name
+        return ci_type_dict
+
+    @classmethod
+    def get_children(cls, parent_id):
+        children = CITypeRelation.get_by(parent_id=parent_id, to_dict=False)
+
+        return [cls._wrap_relation_type_dict(child.child_id, child) for child in children]
+
+    @classmethod
+    def get_parents(cls, child_id):
+        parents = CITypeRelation.get_by(child_id=child_id, to_dict=False)
+
+        return [cls._wrap_relation_type_dict(parent.parent_id, parent) for parent in parents]
+
+    @staticmethod
+    def _get(parent_id, child_id):
+        return CITypeRelation.get_by(parent_id=parent_id,
+                                     child_id=child_id,
+                                     to_dict=False,
+                                     first=True)
+
+    @classmethod
+    def add(cls, parent, child, relation_type_id):
+        p = CITypeManager.check_is_existed(parent)
+        c = CITypeManager.check_is_existed(child)
+
+        existed = cls._get(p.id, c.id)
+        if existed is not None:
+            existed.update(relation_type_id=relation_type_id)
+        else:
+            existed = CITypeRelation.create(parent_id=p.id,
+                                            child_id=c.id,
+                                            relation_type_id=relation_type_id)
+        return existed.id
+
+    @staticmethod
+    def delete(_id):
+        ctr = CITypeRelation.get_by_id(_id) or abort(404, "Type relation <{0}> is not found".format(_id))
+        ctr.soft_delete()
+
+    @classmethod
+    def delete_2(cls, parent, child):
+        ctr = cls._get(parent, child)
+        return cls.delete(ctr.id)
+
+
+class CITypeAttributeGroupManager(object):
+    @staticmethod
+    def get_by_type_id(type_id, need_other=None):
+        groups = CITypeAttributeGroup.get_by(type_id=type_id)
+        groups = sorted(groups, key=lambda x: x["order"])
+        grouped = list()
+        for group in groups:
+            items = CITypeAttributeGroupItem.get_by(group_id=group["id"], to_dict=False)
+            items = sorted(items, key=lambda x: x.order)
+            group["attributes"] = [AttributeCache.get(i.attr_id).to_dict() for i in items]
+            grouped.extend([i.attr_id for i in items])
+
+        if need_other is not None:
+            grouped = set(grouped)
+            attributes = CITypeAttributeManager.get_attributes_by_type_id(type_id)
+            other_attributes = [attr for attr in attributes if attr["id"] not in grouped]
+            groups.append(dict(attributes=other_attributes))
+
+        return groups
+
+    @staticmethod
+    def create_or_update(type_id, name, attr_order, group_order=0):
+        """
+        create or update
+        :param type_id:
+        :param name:
+        :param group_order: group order
+        :param attr_order:
+        :return:
+        """
+        existed = CITypeAttributeGroup.get_by(type_id=type_id, name=name, first=True, to_dict=False)
+        existed = existed or CITypeAttributeGroup.create(type_id=type_id, name=name, order=group_order)
+        existed.update(order=group_order)
+        attr_order = dict(attr_order)
+        current_app.logger.info(attr_order)
+        existed_items = CITypeAttributeGroupItem.get_by(group_id=existed.id, to_dict=False)
+        for item in existed_items:
+            if item.attr_id not in attr_order:
+                item.soft_delete()
+            else:
+                item.update(order=attr_order[item.attr_id])
+
+        existed_items = {item.attr_id: 1 for item in existed_items}
+        for attr_id, order in attr_order.items():
+            if attr_id not in existed_items:
+                CITypeAttributeGroupItem.create(group_id=existed.id, attr_id=attr_id, order=order)
+
+        return existed
+
+    @classmethod
+    def update(cls, group_id, name, attr_order, group_order=0):
+        group = CITypeAttributeGroup.get_by_id(group_id) or abort(404, "Group <{0}> does not exist".format(group_id))
+        other = CITypeAttributeGroup.get_by(type_id=group.type_id, name=name, first=True, to_dict=False)
+        if other is not None and other.id != group.id:
+            return abort(400, "Group <{0}> duplicate".format(name))
+        if name is not None:
+            group.update(name=name)
+
+        cls.create_or_update(group.type_id, name, attr_order, group_order)
+
+    @staticmethod
+    def delete(group_id):
+        group = CITypeAttributeGroup.get_by_id(group_id) \
+                or abort(404, "AttributeGroup <{0}> does not exist".format(group_id))
+        group.soft_delete()
+
+        items = CITypeAttributeGroupItem.get_by(group_id=group_id, to_dict=False)
+        for item in items:
+            item.soft_delete()
+
+        return group_id

cmdb-api/api/lib/cmdb/const.py

@@ -0,0 +1,59 @@
+# -*- coding:utf-8 -*- 
+
+
+from api.lib.utils import BaseEnum
+
+
+class ValueTypeEnum(BaseEnum):
+    INT = "0"
+    FLOAT = "1"
+    TEXT = "2"
+    DATETIME = "3"
+    DATE = "4"
+    TIME = "5"
+
+
+class CIStatusEnum(BaseEnum):
+    REVIEW = "0"
+    VALIDATE = "1"
+
+
+class ExistPolicy(BaseEnum):
+    REJECT = "reject"
+    NEED = "need"
+    IGNORE = "ignore"
+    REPLACE = "replace"
+
+
+class OperateType(BaseEnum):
+    ADD = "0"
+    DELETE = "1"
+    UPDATE = "2"
+
+
+class RetKey(BaseEnum):
+    ID = "id"
+    NAME = "name"
+    ALIAS = "alias"
+
+
+class ResourceTypeEnum(BaseEnum):
+    CI = "CIType"
+    RELATION_VIEW = "RelationView"
+
+
+class PermEnum(BaseEnum):
+    ADD = "add"
+    UPDATE = "update"
+    DELETE = "delete"
+    READ = "read"
+
+
+class RoleEnum(BaseEnum):
+    CONFIG = "admin"
+    CMDB_READ_ALL = "CMDB_READ_ALL"
+
+
+CMDB_QUEUE = "cmdb_async"
+REDIS_PREFIX_CI = "CMDB_CI"
+REDIS_PREFIX_CI_RELATION = "CMDB_CI_RELATION"

cmdb-api/api/lib/cmdb/history.py

@@ -0,0 +1,124 @@
+# -*- coding:utf-8 -*- 
+
+
+from flask import abort
+from flask import g
+
+from api.extensions import db
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.cache import RelationTypeCache
+from api.lib.cmdb.const import OperateType
+from api.lib.perm.acl.cache import UserCache
+from api.models.cmdb import Attribute
+from api.models.cmdb import AttributeHistory
+from api.models.cmdb import CIRelationHistory
+from api.models.cmdb import OperationRecord
+
+
+class AttributeHistoryManger(object):
+    @staticmethod
+    def get_records(start, end, username, page, page_size):
+        records = db.session.query(OperationRecord).filter(OperationRecord.deleted.is_(False))
+        numfound = db.session.query(db.func.count(OperationRecord.id)).filter(OperationRecord.deleted.is_(False))
+        if start:
+            records = records.filter(OperationRecord.created_at >= start)
+            numfound = numfound.filter(OperationRecord.created_at >= start)
+        if end:
+            records = records.filter(OperationRecord.created_at <= end)
+            numfound = records.filter(OperationRecord.created_at <= end)
+        if username:
+            user = UserCache.get(username)
+            if user:
+                records = records.filter(OperationRecord.uid == user.uid)
+            else:
+                return abort(404, "User <{0}> is not found".format(username))
+
+        records = records.order_by(-OperationRecord.id).offset(page_size * (page - 1)).limit(page_size).all()
+        total = len(records)
+        numfound = numfound.first()[0]
+        res = []
+        for record in records:
+            _res = record.to_dict()
+            _res["user"] = UserCache.get(_res.get("uid")).nickname or UserCache.get(_res.get("uid")).username
+
+            attr_history = AttributeHistory.get_by(record_id=_res.get("id"), to_dict=False)
+            _res["attr_history"] = [AttributeCache.get(h.attr_id).attr_alias for h in attr_history]
+
+            rel_history = CIRelationHistory.get_by(record_id=_res.get("id"), to_dict=False)
+            rel_statis = {}
+            for rel in rel_history:
+                if rel.operate_type not in rel_statis:
+                    rel_statis[rel.operate_type] = 1
+                else:
+                    rel_statis[rel.operate_type] += 1
+            _res["rel_history"] = rel_statis
+            res.append(_res)
+
+        return numfound, total, res
+
+    @staticmethod
+    def get_by_ci_id(ci_id):
+        res = db.session.query(AttributeHistory, Attribute, OperationRecord).join(
+            Attribute, Attribute.id == AttributeHistory.attr_id).join(
+            OperationRecord, OperationRecord.id == AttributeHistory.record_id).filter(
+            AttributeHistory.ci_id == ci_id).order_by(OperationRecord.id.desc())
+        return [dict(attr_name=i.Attribute.name,
+                     attr_alias=i.Attribute.alias,
+                     operate_type=i.AttributeHistory.operate_type,
+                     username=UserCache.get(i.OperationRecord.uid).nickname,
+                     old=i.AttributeHistory.old,
+                     new=i.AttributeHistory.new,
+                     created_at=i.OperationRecord.created_at.strftime('%Y-%m-%d %H:%M:%S'),
+                     record_id=i.OperationRecord.id,
+                     hid=i.AttributeHistory.id
+                     ) for i in res]
+
+    @staticmethod
+    def get_record_detail(record_id):
+        from api.lib.cmdb.ci import CIManager
+
+        record = OperationRecord.get_by_id(record_id) or abort(404, "Record <{0}> is not found".format(record_id))
+
+        username = UserCache.get(record.uid).nickname or UserCache.get(record.uid).username
+        timestamp = record.created_at.strftime("%Y-%m-%d %H:%M:%S")
+        attr_history = AttributeHistory.get_By(record_id=record_id, to_dict=False)
+        rel_history = CIRelationHistory.get_by(record_id=record_id, to_dict=False)
+
+        attr_dict, rel_dict = dict(), {"add": [], "delete": []}
+        for attr_h in attr_history:
+            attr_dict[AttributeCache.get(attr_h.attr_id).alias] = dict(
+                old=attr_h.old,
+                new=attr_h.new,
+                operate_type=attr_h.operate_type)
+
+        for rel_h in rel_history:
+            first = CIManager.get_ci_by_id(rel_h.first_ci_id)
+            second = CIManager.get_ci_by_id(rel_h.second_ci_id)
+            rel_dict[rel_h.operate_type].append((first, RelationTypeCache.get(rel_h.relation_type_id).name, second))
+
+        return username, timestamp, attr_dict, rel_dict
+
+    @staticmethod
+    def add(ci_id, history_list):
+        record = OperationRecord.create(uid=g.user.uid)
+
+        for attr_id, operate_type, old, new in history_list or []:
+            AttributeHistory.create(attr_id=attr_id,
+                                    operate_type=operate_type,
+                                    old=old,
+                                    new=new,
+                                    ci_id=ci_id,
+                                    record_id=record.id)
+
+
+class CIRelationHistoryManager(object):
+    @staticmethod
+    def add(rel_obj, operate_type=OperateType.ADD):
+        record = OperationRecord.create(uid=g.user.uid)
+
+        CIRelationHistory.create(relation_id=rel_obj.id,
+                                 record_id=record.id,
+                                 operate_type=operate_type,
+                                 first_ci_id=rel_obj.first_ci_id,
+                                 second_ci_id=rel_obj.second_ci_id,
+                                 relation_type_id=rel_obj.relation_type_id)

cmdb-api/api/lib/cmdb/preference.py

@@ -0,0 +1,208 @@
+# -*- coding:utf-8 -*-
+
+
+import copy
+import json
+
+import six
+import toposort
+from flask import abort
+from flask import current_app
+from flask import g
+
+from api.extensions import db
+from api.lib.cmdb.attribute import AttributeManager
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.cache import CITypeAttributeCache
+from api.lib.cmdb.cache import CITypeCache
+from api.lib.cmdb.const import ResourceTypeEnum, RoleEnum, PermEnum
+from api.lib.exception import AbortException
+from api.lib.perm.acl.acl import ACLManager
+from api.models.cmdb import CITypeAttribute
+from api.models.cmdb import CITypeRelation
+from api.models.cmdb import PreferenceRelationView
+from api.models.cmdb import PreferenceShowAttributes
+from api.models.cmdb import PreferenceTreeView
+
+
+class PreferenceManager(object):
+    @staticmethod
+    def get_types(instance=False, tree=False):
+        types = db.session.query(PreferenceShowAttributes.type_id).filter(
+            PreferenceShowAttributes.uid == g.user.uid).filter(
+            PreferenceShowAttributes.deleted.is_(False)).group_by(PreferenceShowAttributes.type_id).all() \
+            if instance else []
+        tree_types = PreferenceTreeView.get_by(uid=g.user.uid, to_dict=False) if tree else []
+        type_ids = list(set([i.type_id for i in types + tree_types]))
+        return [CITypeCache.get(type_id).to_dict() for type_id in type_ids]
+
+    @staticmethod
+    def get_show_attributes(type_id):
+        if not isinstance(type_id, six.integer_types):
+            type_id = CITypeCache.get(type_id).id
+
+        attrs = db.session.query(PreferenceShowAttributes, CITypeAttribute.order).join(
+            CITypeAttribute, CITypeAttribute.attr_id == PreferenceShowAttributes.attr_id).filter(
+            PreferenceShowAttributes.uid == g.user.uid).filter(
+            PreferenceShowAttributes.type_id == type_id).filter(
+            PreferenceShowAttributes.deleted.is_(False)).filter(CITypeAttribute.deleted.is_(False)).filter(
+            CITypeAttribute.type_id == type_id).order_by(
+            CITypeAttribute.order).all()
+        result = [i.PreferenceShowAttributes.attr.to_dict() for i in attrs]
+        is_subscribed = True
+        if not attrs:
+            attrs = db.session.query(CITypeAttribute).filter(
+                CITypeAttribute.type_id == type_id).filter(
+                CITypeAttribute.deleted.is_(False)).filter(
+                CITypeAttribute.default_show.is_(True)).order_by(CITypeAttribute.order)
+            result = [i.attr.to_dict() for i in attrs]
+            is_subscribed = False
+
+        for i in result:
+            if i["is_choice"]:
+                i.update(dict(choice_value=AttributeManager.get_choice_values(i["id"], i["value_type"])))
+
+        return is_subscribed, result
+
+    @classmethod
+    def create_or_update_show_attributes(cls, type_id, attr_order):
+        existed_all = PreferenceShowAttributes.get_by(type_id=type_id, uid=g.user.uid, to_dict=False)
+        for _attr, order in attr_order:
+            attr = AttributeCache.get(_attr) or abort(404, "Attribute <{0}> does not exist".format(_attr))
+            existed = PreferenceShowAttributes.get_by(type_id=type_id,
+                                                      uid=g.user.uid,
+                                                      attr_id=attr.id,
+                                                      first=True,
+                                                      to_dict=False)
+            if existed is None:
+                PreferenceShowAttributes.create(type_id=type_id,
+                                                uid=g.user.uid,
+                                                attr_id=attr.id,
+                                                order=order)
+            else:
+                existed.update(order=order)
+
+        attr_dict = {int(i): j for i, j in attr_order}
+        for i in existed_all:
+            if i.attr_id not in attr_dict:
+                i.soft_delete()
+
+    @staticmethod
+    def get_tree_view():
+        res = PreferenceTreeView.get_by(uid=g.user.uid, to_dict=True)
+        for item in res:
+            if item["levels"]:
+                item.update(CITypeCache.get(item['type_id']).to_dict())
+                item.update(dict(levels=[AttributeCache.get(l).to_dict()
+                                         for l in item["levels"].split(",") if AttributeCache.get(l)]))
+
+        return res
+
+    @staticmethod
+    def create_or_update_tree_view(type_id, levels):
+        attrs = CITypeAttributeCache.get(type_id)
+        for idx, i in enumerate(levels):
+            for attr in attrs:
+                attr = AttributeCache.get(attr.attr_id)
+                if i == attr.id or i == attr.name or i == attr.alias:
+                    levels[idx] = str(attr.id)
+        levels = ",".join(levels)
+
+        existed = PreferenceTreeView.get_by(uid=g.user.uid, type_id=type_id, to_dict=False, first=True)
+        if existed is not None:
+            if not levels:
+                existed.soft_delete()
+                return existed
+            return existed.update(levels=levels)
+        elif levels:
+            return PreferenceTreeView.create(levels=levels, type_id=type_id, uid=g.user.uid)
+
+    @staticmethod
+    def get_relation_view():
+        _views = PreferenceRelationView.get_by(to_dict=True)
+        views = []
+        if current_app.config.get("USE_ACL"):
+            for i in _views:
+                try:
+                    if ACLManager().has_permission(i.get('name'),
+                                                   ResourceTypeEnum.RELATION_VIEW,
+                                                   PermEnum.READ):
+                        views.append(i)
+                except AbortException:
+                    pass
+        else:
+            views = _views
+
+        view2cr_ids = dict()
+        result = dict()
+        name2id = list()
+        for view in views:
+            view2cr_ids.setdefault(view['name'], []).extend(json.loads(view['cr_ids']))
+            name2id.append([view['name'], view['id']])
+
+        id2type = dict()
+        for view_name in view2cr_ids:
+            for i in view2cr_ids[view_name]:
+                id2type[i['parent_id']] = None
+                id2type[i['child_id']] = None
+            topo = {i['child_id']: {i['parent_id']} for i in view2cr_ids[view_name]}
+            leaf = list(set(toposort.toposort_flatten(topo)) - set([j for i in topo.values() for j in i]))
+
+            leaf2show_types = {i: [t['child_id'] for t in CITypeRelation.get_by(parent_id=i)] for i in leaf}
+            node2show_types = copy.deepcopy(leaf2show_types)
+
+            def _find_parent(_node_id):
+                parents = topo.get(_node_id, {})
+                for parent in parents:
+                    node2show_types.setdefault(parent, []).extend(node2show_types.get(_node_id, []))
+                    _find_parent(parent)
+                if not parents:
+                    return
+
+            for l in leaf:
+                _find_parent(l)
+
+            for node_id in node2show_types:
+                node2show_types[node_id] = [CITypeCache.get(i).to_dict() for i in set(node2show_types[node_id])]
+
+            result[view_name] = dict(topo=list(map(list, toposort.toposort(topo))),
+                                     topo_flatten=list(toposort.toposort_flatten(topo)),
+                                     leaf=leaf,
+                                     leaf2show_types=leaf2show_types,
+                                     node2show_types=node2show_types,
+                                     show_types=[CITypeCache.get(j).to_dict()
+                                                 for i in leaf2show_types.values() for j in i])
+
+        for type_id in id2type:
+            id2type[type_id] = CITypeCache.get(type_id).to_dict()
+
+        return result, id2type, sorted(name2id, key=lambda x: x[1])
+
+    @classmethod
+    def create_or_update_relation_view(cls, name, cr_ids):
+        if not cr_ids:
+            return abort(400, "Node must be selected")
+
+        existed = PreferenceRelationView.get_by(name=name, to_dict=False, first=True)
+        current_app.logger.debug(existed)
+        if existed is None:
+            PreferenceRelationView.create(name=name, cr_ids=json.dumps(cr_ids))
+
+            if current_app.config.get("USE_ACL"):
+                ACLManager().add_resource(name, ResourceTypeEnum.RELATION_VIEW)
+                ACLManager().grant_resource_to_role(name,
+                                                    RoleEnum.CMDB_READ_ALL,
+                                                    ResourceTypeEnum.RELATION_VIEW,
+                                                    permissions=[PermEnum.READ])
+
+        return cls.get_relation_view()
+
+    @staticmethod
+    def delete_relation_view(name):
+        for existed in PreferenceRelationView.get_by(name=name, to_dict=False):
+            existed.soft_delete()
+
+        if current_app.config.get("USE_ACL"):
+            ACLManager().del_resource(name, ResourceTypeEnum.RELATION_VIEW)
+
+        return name

cmdb-api/api/lib/cmdb/relation_type.py

@@ -0,0 +1,37 @@
+# -*- coding:utf-8 -*-
+
+
+from flask import abort
+
+from api.models.cmdb import RelationType
+
+
+class RelationTypeManager(object):
+    @staticmethod
+    def get_all():
+        return RelationType.get_by(to_dict=False)
+
+    @classmethod
+    def get_names(cls):
+        return [i.name for i in cls.get_all()]
+
+    @classmethod
+    def get_pairs(cls):
+        return [(i.id, i.name) for i in cls.get_all()]
+
+    @staticmethod
+    def add(name):
+        RelationType.get_by(name=name, first=True, to_dict=False) and abort(400, "It's already existed")
+        return RelationType.create(name=name)
+
+    @staticmethod
+    def update(rel_id, name):
+        existed = RelationType.get_by_id(rel_id) or abort(404, "RelationType <{0}> does not exist".format(rel_id))
+
+        return existed.update(name=name)
+
+    @staticmethod
+    def delete(rel_id):
+        existed = RelationType.get_by_id(rel_id) or abort(404, "RelationType <{0}> does not exist".format(rel_id))
+
+        existed.soft_delete()

cmdb-api/api/lib/cmdb/search/__init__.py

@@ -0,0 +1,11 @@
+# -*- coding:utf-8 -*-
+
+__all__ = ['ci', 'ci_relation', 'SearchError']
+
+
+class SearchError(Exception):
+    def __init__(self, v):
+        self.v = v
+
+    def __str__(self):
+        return self.v

cmdb-api/api/lib/cmdb/search/ci/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding:utf-8 -*-
+
+__all__ = ['db', 'es']

cmdb-api/api/lib/cmdb/search/ci/db/__init__.py

@@ -0,0 +1 @@
+# -*- coding:utf-8 -*-

cmdb-api/api/lib/cmdb/search/ci/db/query_sql.py

@@ -0,0 +1,66 @@
+# -*- coding:utf-8 -*- 
+
+from __future__ import unicode_literals
+
+
+QUERY_CIS_BY_VALUE_TABLE = """
+    SELECT attr.name AS attr_name,
+          attr.alias AS attr_alias,
+          attr.value_type,
+          attr.is_list,
+          c_cis.type_id,
+          {0}.ci_id,
+          {0}.attr_id,
+          {0}.value
+   FROM {0}
+   INNER JOIN c_cis ON {0}.ci_id=c_cis.id
+   AND {0}.`ci_id` IN ({1})
+   INNER JOIN c_attributes as attr ON attr.id = {0}.attr_id
+"""
+
+#  {2}: value_table
+QUERY_CIS_BY_IDS = """
+    SELECT A.ci_id,
+           A.type_id,
+           A.attr_id,
+           A.attr_name,
+           A.attr_alias,
+           A.value,
+           A.value_type,
+           A.is_list
+    FROM
+      ({1}) AS A {0}
+       ORDER BY A.ci_id;
+"""
+
+FACET_QUERY1 = """
+    SELECT {0}.value,
+           count({0}.ci_id)
+    FROM {0}
+    INNER JOIN c_attributes AS attr ON attr.id={0}.attr_id
+    WHERE attr.name="{1}"
+    GROUP BY {0}.ci_id;
+"""
+
+FACET_QUERY = """
+    SELECT {0}.value,
+           count({0}.ci_id)
+    FROM {0}
+    INNER JOIN ({1}) AS F ON F.ci_id={0}.ci_id
+    WHERE {0}.attr_id={2:d}
+    GROUP BY {0}.value
+"""
+
+QUERY_CI_BY_ATTR_NAME = """
+    SELECT {0}.ci_id
+    FROM {0}
+    WHERE {0}.attr_id={1:d}
+      AND {0}.value {2}
+"""
+
+QUERY_CI_BY_TYPE = """
+    SELECT c_cis.id AS ci_id
+    FROM c_cis
+    WHERE c_cis.type_id in ({0})
+"""
+

cmdb-api/api/lib/cmdb/search/ci/db/search.py

@@ -0,0 +1,367 @@
+# -*- coding:utf-8 -*- 
+
+
+from __future__ import unicode_literals
+
+import time
+
+from flask import current_app
+
+from api.extensions import db
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.cache import CITypeCache
+from api.lib.cmdb.ci import CIManager
+from api.lib.cmdb.const import RetKey
+from api.lib.cmdb.const import ValueTypeEnum
+from api.lib.cmdb.search import SearchError
+from api.lib.cmdb.search.ci.db.query_sql import FACET_QUERY
+from api.lib.cmdb.search.ci.db.query_sql import QUERY_CI_BY_ATTR_NAME
+from api.lib.cmdb.search.ci.db.query_sql import QUERY_CI_BY_TYPE
+from api.lib.cmdb.utils import TableMap
+from api.lib.utils import handle_arg_list
+from api.models.cmdb import CI
+
+
+class Search(object):
+    def __init__(self, query=None,
+                 fl=None,
+                 facet_field=None,
+                 page=1,
+                 ret_key=RetKey.NAME,
+                 count=1,
+                 sort=None,
+                 ci_ids=None):
+        self.orig_query = query
+        self.fl = fl
+        self.facet_field = facet_field
+        self.page = page
+        self.ret_key = ret_key
+        self.count = count
+        self.sort = sort
+        self.ci_ids = ci_ids or []
+        self.query_sql = ""
+        self.type_id_list = []
+        self.only_type_query = False
+
+    @staticmethod
+    def _operator_proc(key):
+        operator = "&"
+        if key.startswith("+"):
+            key = key[1:].strip()
+        elif key.startswith("-"):
+            operator = "|"
+            key = key[1:].strip()
+        elif key.startswith("~"):
+            operator = "~"
+            key = key[1:].strip()
+
+        return operator, key
+
+    def _attr_name_proc(self, key):
+        operator, key = self._operator_proc(key)
+
+        if key in ('ci_type', 'type', '_type'):
+            return '_type', ValueTypeEnum.TEXT, operator, None
+
+        if key in ('id', 'ci_id', '_id'):
+            return '_id', ValueTypeEnum.TEXT, operator, None
+
+        attr = AttributeCache.get(key)
+        if attr:
+            return attr.name, attr.value_type, operator, attr
+        else:
+            raise SearchError("{0} is not existed".format(key))
+
+    def _type_query_handler(self, v):
+        new_v = v[1:-1].split(";") if v.startswith("(") and v.endswith(")") else [v]
+        for _v in new_v:
+            ci_type = CITypeCache.get(_v)
+            if ci_type is not None:
+                self.type_id_list.append(str(ci_type.id))
+
+        if self.type_id_list:
+            type_ids = ",".join(self.type_id_list)
+            _query_sql = QUERY_CI_BY_TYPE.format(type_ids)
+            if self.only_type_query:
+                return _query_sql
+            else:
+                return ""
+        return ""
+
+    @staticmethod
+    def _in_query_handler(attr, v):
+        new_v = v[1:-1].split(";")
+        table_name = TableMap(attr_name=attr.name).table_name
+        in_query = " OR {0}.value ".format(table_name).join(['LIKE "{0}"'.format(_v.replace("*", "%")) for _v in new_v])
+        _query_sql = QUERY_CI_BY_ATTR_NAME.format(table_name, attr.id, in_query)
+        return _query_sql
+
+    @staticmethod
+    def _range_query_handler(attr, v):
+        start, end = [x.strip() for x in v[1:-1].split("_TO_")]
+        table_name = TableMap(attr_name=attr.name).table_name
+        range_query = "BETWEEN '{0}' AND '{1}'".format(start.replace("*", "%"), end.replace("*", "%"))
+        _query_sql = QUERY_CI_BY_ATTR_NAME.format(table_name, attr.id, range_query)
+        return _query_sql
+
+    @staticmethod
+    def _comparison_query_handler(attr, v):
+        table_name = TableMap(attr_name=attr.name).table_name
+        if v.startswith(">=") or v.startswith("<="):
+            comparison_query = "{0} '{1}'".format(v[:2], v[2:].replace("*", "%"))
+        else:
+            comparison_query = "{0} '{1}'".format(v[0], v[1:].replace("*", "%"))
+        _query_sql = QUERY_CI_BY_ATTR_NAME.format(table_name, attr.id, comparison_query)
+        return _query_sql
+
+    @staticmethod
+    def __sort_by(field):
+        field = field or ""
+        sort_type = "ASC"
+        if field.startswith("+"):
+            field = field[1:]
+        elif field.startswith("-"):
+            field = field[1:]
+            sort_type = "DESC"
+        return field, sort_type
+
+    def __sort_by_id(self, sort_type, query_sql):
+        ret_sql = "SELECT SQL_CALC_FOUND_ROWS DISTINCT B.ci_id FROM ({0}) AS B {1}"
+
+        if self.only_type_query:
+            return ret_sql.format(query_sql, "ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format(
+                (self.page - 1) * self.count, sort_type, self.count))
+
+        elif self.type_id_list:
+            self.query_sql = "SELECT B.ci_id FROM ({0}) AS B {1}".format(
+                query_sql,
+                "INNER JOIN c_cis on c_cis.id=B.ci_id WHERE c_cis.type_id IN ({0}) ".format(
+                    ",".join(self.type_id_list)))
+
+            return ret_sql.format(
+                query_sql,
+                "INNER JOIN c_cis on c_cis.id=B.ci_id WHERE c_cis.type_id IN ({3}) "
+                "ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format(
+                    (self.page - 1) * self.count, sort_type, self.count, ",".join(self.type_id_list)))
+
+        else:
+            self.query_sql = "SELECT B.ci_id FROM ({0}) AS B {1}".format(
+                query_sql,
+                "INNER JOIN c_cis on c_cis.id=B.ci_id ")
+
+            return ret_sql.format(
+                query_sql,
+                "INNER JOIN c_cis on c_cis.id=B.ci_id "
+                "ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format((self.page - 1) * self.count, sort_type, self.count))
+
+    def __sort_by_field(self, field, sort_type, query_sql):
+        attr = AttributeCache.get(field)
+        attr_id = attr.id
+
+        table_name = TableMap(attr_name=attr.name).table_name
+        _v_query_sql = """SELECT {0}.ci_id, {1}.value 
+                          FROM ({2}) AS {0} INNER JOIN {1} ON {1}.ci_id = {0}.ci_id
+                          WHERE {1}.attr_id = {3}""".format("ALIAS", table_name, query_sql, attr_id)
+        new_table = _v_query_sql
+
+        if self.only_type_query or not self.type_id_list:
+            return "SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id, C.value " \
+                   "FROM ({0}) AS C " \
+                   "ORDER BY C.value {2} " \
+                   "LIMIT {1:d}, {3};".format(new_table, (self.page - 1) * self.count, sort_type, self.count)
+
+        elif self.type_id_list:
+            self.query_sql = """SELECT C.ci_id
+                                FROM ({0}) AS C
+                                INNER JOIN c_cis on c_cis.id=C.ci_id
+                                WHERE c_cis.type_id IN ({1})""".format(new_table, ",".join(self.type_id_list))
+
+            return """SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id, C.value
+                      FROM ({0}) AS C
+                      INNER JOIN c_cis on c_cis.id=C.ci_id
+                      WHERE c_cis.type_id IN ({4})
+                      ORDER BY C.value {2}
+                      LIMIT {1:d}, {3};""".format(new_table,
+                                                  (self.page - 1) * self.count,
+                                                  sort_type, self.count,
+                                                  ",".join(self.type_id_list))
+
+    def _sort_query_handler(self, field, query_sql):
+
+        field, sort_type = self.__sort_by(field)
+
+        if field in ("_id", "ci_id") or not field:
+            return self.__sort_by_id(sort_type, query_sql)
+        else:
+            return self.__sort_by_field(field, sort_type, query_sql)
+
+    @staticmethod
+    def _wrap_sql(operator, alias, _query_sql, query_sql):
+        if operator == "&":
+            query_sql = """SELECT * FROM ({0}) as {1}
+                           INNER JOIN ({2}) as {3} USING(ci_id)""".format(query_sql, alias, _query_sql, alias + "A")
+
+        elif operator == "|":
+            query_sql = "SELECT * FROM ({0}) as {1} UNION ALL ({2})".format(query_sql, alias, _query_sql)
+
+        elif operator == "~":
+            query_sql = """SELECT * FROM ({0}) as {1} LEFT JOIN ({2}) as {3} USING(ci_id) 
+                           WHERE {3}.ci_id is NULL""".format(query_sql, alias, _query_sql, alias + "A")
+
+        return query_sql
+
+    def _execute_sql(self, query_sql):
+        v_query_sql = self._sort_query_handler(self.sort, query_sql)
+
+        start = time.time()
+        execute = db.session.execute
+        current_app.logger.debug(v_query_sql)
+        res = execute(v_query_sql).fetchall()
+        end_time = time.time()
+        current_app.logger.debug("query ci ids time is: {0}".format(end_time - start))
+
+        numfound = execute("SELECT FOUND_ROWS();").fetchall()[0][0]
+        current_app.logger.debug("statistics ci ids time is: {0}".format(time.time() - end_time))
+
+        return numfound, res
+
+    def __confirm_type_first(self, queries):
+        for q in queries:
+            if q.startswith("_type"):
+                queries.remove(q)
+                queries.insert(0, q)
+                if len(queries) == 1 or queries[1].startswith("-") or queries[1].startswith("~"):
+                    self.only_type_query = True
+        return queries
+
+    def __query_build_by_field(self, queries):
+        query_sql, alias, operator = "", "A", "&"
+        is_first, only_type_query_special = True, True
+
+        for q in queries:
+            _query_sql = ""
+            if ":" in q:
+                k = q.split(":")[0].strip()
+                v = ":".join(q.split(":")[1:]).strip()
+                current_app.logger.debug(v)
+                field, field_type, operator, attr = self._attr_name_proc(k)
+                if field == "_type":
+                    _query_sql = self._type_query_handler(v)
+                    current_app.logger.debug(_query_sql)
+                elif field == "_id":  # exclude all others
+                    ci = CI.get_by_id(v)
+                    if ci is not None:
+                        return 1, [str(v)]
+                elif field:
+                    if attr is None:
+                        raise SearchError("{0} is not found".format(field))
+
+                    # in query
+                    if v.startswith("(") and v.endswith(")"):
+                        _query_sql = self._in_query_handler(attr, v)
+                    # range query
+                    elif v.startswith("[") and v.endswith("]") and "_TO_" in v:
+                        _query_sql = self._range_query_handler(attr, v)
+                    # comparison query
+                    elif v.startswith(">=") or v.startswith("<=") or v.startswith(">") or v.startswith("<"):
+                        _query_sql = self._comparison_query_handler(attr, v)
+                    else:
+                        table_name = TableMap(attr_name=attr.name).table_name
+                        _query_sql = QUERY_CI_BY_ATTR_NAME.format(
+                            table_name, attr.id, 'LIKE "{0}"'.format(v.replace("*", "%")))
+                else:
+                    raise SearchError("argument q format invalid: {0}".format(q))
+            elif q:
+                raise SearchError("argument q format invalid: {0}".format(q))
+
+            if is_first and _query_sql and not self.only_type_query:
+                query_sql = "SELECT * FROM ({0}) AS {1}".format(_query_sql, alias)
+                is_first = False
+                alias += "A"
+            elif self.only_type_query and only_type_query_special:
+                is_first = False
+                only_type_query_special = False
+                query_sql = _query_sql
+            elif _query_sql:
+                query_sql = self._wrap_sql(operator, alias, _query_sql, query_sql)
+                alias += "AA"
+        return None, query_sql
+
+    def _filter_ids(self, query_sql):
+        if self.ci_ids:
+            return "SELECT * FROM ({0}) AS IN_QUERY WHERE IN_QUERY.ci_id IN ({1})".format(
+                query_sql, ",".join(list(map(str, self.ci_ids))))
+
+        return query_sql
+
+    def _query_build_raw(self):
+
+        queries = handle_arg_list(self.orig_query)
+        queries = self.__confirm_type_first(queries)
+        current_app.logger.debug(queries)
+
+        ret, query_sql = self.__query_build_by_field(queries)
+        if ret is not None:
+            return ret, query_sql
+
+        s = time.time()
+        if query_sql:
+            query_sql = self._filter_ids(query_sql)
+            self.query_sql = query_sql
+            current_app.logger.debug(query_sql)
+            numfound, res = self._execute_sql(query_sql)
+            current_app.logger.debug("query ci ids is: {0}".format(time.time() - s))
+            return numfound, [_res[0] for _res in res]
+
+        return 0, []
+
+    def _facet_build(self):
+        facet = {}
+        for f in self.facet_field:
+            k, field_type, _, attr = self._attr_name_proc(f)
+            if k:
+                table_name = TableMap(attr_name=k).table_name
+                query_sql = FACET_QUERY.format(table_name, self.query_sql, attr.id)
+                current_app.logger.debug(query_sql)
+                result = db.session.execute(query_sql).fetchall()
+                facet[k] = result
+
+        facet_result = dict()
+        for k, v in facet.items():
+            if not k.startswith('_'):
+                a = getattr(AttributeCache.get(k), self.ret_key)
+                facet_result[a] = [(f[0], f[1], a) for f in v]
+
+        return facet_result
+
+    def _fl_build(self):
+        _fl = list()
+        for f in self.fl:
+            k, _, _, _ = self._attr_name_proc(f)
+            if k:
+                _fl.append(k)
+
+        return _fl
+
+    def search(self):
+        numfound, ci_ids = self._query_build_raw()
+        ci_ids = list(map(str, ci_ids))
+
+        _fl = self._fl_build()
+
+        if self.facet_field and numfound:
+            facet = self._facet_build()
+        else:
+            facet = dict()
+
+        response, counter = [], {}
+        if ci_ids:
+            response = CIManager.get_cis_by_ids(ci_ids, ret_key=self.ret_key, fields=_fl)
+        for res in response:
+            ci_type = res.get("ci_type")
+            if ci_type not in counter.keys():
+                counter[ci_type] = 0
+            counter[ci_type] += 1
+        total = len(response)
+
+        return response, counter, total, self.page, numfound, facet

cmdb-api/api/lib/cmdb/search/ci/es/__init__.py

@@ -0,0 +1 @@
+# -*- coding:utf-8 -*-

cmdb-api/api/lib/cmdb/search/ci/es/search.py

@@ -0,0 +1,259 @@
+# -*- coding:utf-8 -*-
+
+
+from __future__ import unicode_literals
+
+from flask import current_app
+
+from api.extensions import es
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.const import RetKey
+from api.lib.cmdb.const import ValueTypeEnum
+from api.lib.cmdb.search import SearchError
+from api.lib.utils import handle_arg_list
+
+
+class Search(object):
+    def __init__(self, query=None,
+                 fl=None,
+                 facet_field=None,
+                 page=1,
+                 ret_key=RetKey.NAME,
+                 count=1,
+                 sort=None,
+                 ci_ids=None):
+        self.orig_query = query
+        self.fl = fl
+        self.facet_field = facet_field
+        self.page = page
+        self.ret_key = ret_key
+        self.count = count or current_app.config.get("DEFAULT_PAGE_COUNT")
+        self.sort = sort or "ci_id"
+        self.ci_ids = ci_ids or []
+
+        self.query = dict(query=dict(bool=dict(should=[], must=[], must_not=[])))
+
+    @staticmethod
+    def _operator_proc(key):
+        operator = "&"
+        if key.startswith("+"):
+            key = key[1:].strip()
+        elif key.startswith("-"):
+            operator = "|"
+            key = key[1:].strip()
+        elif key.startswith("~"):
+            operator = "~"
+            key = key[1:].strip()
+
+        return operator, key
+
+    def _operator2query(self, operator):
+        if operator == "&":
+            return self.query['query']['bool']['must']
+        elif operator == "|":
+            return self.query['query']['bool']['should']
+        else:
+            return self.query['query']['bool']['must_not']
+
+    def _attr_name_proc(self, key):
+        operator, key = self._operator_proc(key)
+
+        if key in ('ci_type', 'type', '_type'):
+            return 'ci_type', ValueTypeEnum.TEXT, operator
+
+        if key in ('id', 'ci_id', '_id'):
+            return 'ci_id', ValueTypeEnum.TEXT, operator
+
+        attr = AttributeCache.get(key)
+        if attr:
+            return attr.name, attr.value_type, operator
+        else:
+            raise SearchError("{0} is not existed".format(key))
+
+    def _in_query_handle(self, attr, v):
+        terms = v[1:-1].split(";")
+        operator = "|"
+        if attr in ('_type', 'ci_type', 'type_id') and terms and terms[0].isdigit():
+            attr = "type_id"
+            terms = map(int, terms)
+        current_app.logger.warning(terms)
+        for term in terms:
+            self._operator2query(operator).append({
+                "term": {
+                    attr: term
+                }
+            })
+
+    def _filter_ids(self):
+        if self.ci_ids:
+            self.query['query']['bool'].update(dict(filter=dict(terms=dict(ci_id=self.ci_ids))))
+
+    @staticmethod
+    def _digit(s):
+        if s.isdigit():
+            return int(float(s))
+        return s
+
+    def _range_query_handle(self, attr, v, operator):
+        left, right = v.split("_TO_")
+        left, right = left.strip()[1:], right.strip()[:-1]
+        self._operator2query(operator).append({
+            "range": {
+                attr: {
+                    "lte": self._digit(right),
+                    "gte": self._digit(left),
+                    "boost": 2.0
+                }
+            }
+        })
+
+    def _comparison_query_handle(self, attr, v, operator):
+        if v.startswith(">="):
+            _query = dict(gte=self._digit(v[2:]), boost=2.0)
+        elif v.startswith("<="):
+            _query = dict(lte=self._digit(v[2:]), boost=2.0)
+        elif v.startswith(">"):
+            _query = dict(gt=self._digit(v[1:]), boost=2.0)
+        elif v.startswith("<"):
+            _query = dict(lt=self._digit(v[1:]), boost=2.0)
+        else:
+            return
+
+        self._operator2query(operator).append({
+            "range": {
+                attr: _query
+            }
+        })
+
+    def _match_query_handle(self, attr, v, operator):
+        if "*" in v:
+            self._operator2query(operator).append({
+                "wildcard": {
+                    attr: v
+                }
+            })
+        else:
+            if attr == "ci_type" and v.isdigit():
+                attr = "type_id"
+            self._operator2query(operator).append({
+                "term": {
+                    attr: v
+                }
+            })
+
+    def __query_build_by_field(self, queries):
+
+        for q in queries:
+            if ":" in q:
+                k = q.split(":")[0].strip()
+                v = ":".join(q.split(":")[1:]).strip()
+                field_name, field_type, operator = self._attr_name_proc(k)
+                if field_name:
+                    # in query
+                    if v.startswith("(") and v.endswith(")"):
+                        self._in_query_handle(field_name, v)
+                    # range query
+                    elif v.startswith("[") and v.endswith("]") and "_TO_" in v:
+                        self._range_query_handle(field_name, v, operator)
+                    # comparison query
+                    elif v.startswith(">=") or v.startswith("<=") or v.startswith(">") or v.startswith("<"):
+                        self._comparison_query_handle(field_name, v, operator)
+                    else:
+                        self._match_query_handle(field_name, v, operator)
+                else:
+                    raise SearchError("argument q format invalid: {0}".format(q))
+            elif q:
+                raise SearchError("argument q format invalid: {0}".format(q))
+
+    def _query_build_raw(self):
+
+        queries = handle_arg_list(self.orig_query)
+
+        current_app.logger.debug(queries)
+
+        self.__query_build_by_field(queries)
+
+        self._paginate_build()
+
+        filter_path = self._fl_build()
+
+        self._sort_build()
+
+        self._facet_build()
+
+        self._filter_ids()
+
+        return es.read(self.query, filter_path=filter_path)
+
+    def _facet_build(self):
+        aggregations = dict(aggs={})
+        for field in self.facet_field:
+            attr = AttributeCache.get(field)
+            if not attr:
+                raise SearchError("Facet by <{0}> does not exist".format(field))
+            aggregations['aggs'].update({
+                field: {
+                    "terms": {
+                        "field": "{0}.keyword".format(field)
+                        if attr.value_type not in (ValueTypeEnum.INT, ValueTypeEnum.FLOAT) else field
+                    }
+                }
+            })
+
+        if aggregations['aggs']:
+            self.query.update(aggregations)
+
+    def _sort_build(self):
+        fields = list(filter(lambda x: x != "", (self.sort or "").split(",")))
+        sorts = []
+        for field in fields:
+            sort_type = "asc"
+            if field.startswith("+"):
+                field = field[1:]
+            elif field.startswith("-"):
+                field = field[1:]
+                sort_type = "desc"
+            else:
+                field = field
+            if field == "ci_id":
+                sorts.append({field: {"order": sort_type}})
+                continue
+
+            attr = AttributeCache.get(field)
+            if not attr:
+                raise SearchError("Sort by <{0}> does not exist".format(field))
+
+            sort_by = "{0}.keyword".format(field) \
+                if attr.value_type not in (ValueTypeEnum.INT, ValueTypeEnum.FLOAT) else field
+            sorts.append({sort_by: {"order": sort_type}})
+
+        self.query.update(dict(sort=sorts))
+
+    def _paginate_build(self):
+        self.query.update({"from": (self.page - 1) * self.count,
+                           "size": self.count})
+
+    def _fl_build(self):
+        return ['hits.hits._source.{0}'.format(i) for i in self.fl]
+
+    def search(self):
+        try:
+            numfound, cis, facet = self._query_build_raw()
+        except Exception as e:
+            current_app.logger.error(str(e))
+            raise SearchError("unknown search error")
+
+        total = len(cis)
+
+        counter = dict()
+        for ci in cis:
+            ci_type = ci.get("ci_type")
+            if ci_type not in counter.keys():
+                counter[ci_type] = 0
+            counter[ci_type] += 1
+
+        facet_ = dict()
+        for k in facet:
+            facet_[k] = [[i['key'], i['doc_count'], k] for i in facet[k]["buckets"]]
+
+        return cis, counter, total, self.page, numfound, facet_

cmdb-api/api/lib/cmdb/search/ci_relation/__init__.py

@@ -0,0 +1 @@
+# -*- coding:utf-8 -*-

cmdb-api/api/lib/cmdb/search/ci_relation/search.py

@@ -0,0 +1,108 @@
+# -*- coding:utf-8 -*-
+
+
+import json
+
+from flask import abort
+from flask import current_app
+
+from api.extensions import rd
+from api.lib.cmdb.ci_type import CITypeRelationManager
+from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
+from api.lib.cmdb.search.ci.db.search import Search as SearchFromDB
+from api.lib.cmdb.search.ci.es.search import Search as SearchFromES
+from api.models.cmdb import CI
+
+
+class Search(object):
+    def __init__(self, root_id,
+                 level=1,
+                 query=None,
+                 fl=None,
+                 facet_field=None,
+                 page=1,
+                 count=None,
+                 sort=None):
+        self.orig_query = query
+        self.fl = fl
+        self.facet_field = facet_field
+        self.page = page
+        self.count = count or current_app.config.get("DEFAULT_PAGE_COUNT")
+        self.sort = sort or ("ci_id" if current_app.config.get("USE_ES") else None)
+
+        self.root_id = root_id
+        self.level = level
+
+    def search(self):
+        ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
+        cis = [CI.get_by_id(_id) or abort(404, "CI <{0}> does not exist".format(_id)) for _id in ids]
+
+        merge_ids = []
+        for level in self.level:
+            ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
+            for _ in range(0, level):
+                _tmp = list(map(lambda x: list(json.loads(x).keys()),
+                                filter(lambda x: x is not None, rd.get(ids, REDIS_PREFIX_CI_RELATION) or [])))
+                ids = [j for i in _tmp for j in i]
+
+            merge_ids.extend(ids)
+
+        if not self.orig_query or ("_type:" not in self.orig_query
+                                   and "type_id:" not in self.orig_query
+                                   and "ci_type:" not in self.orig_query):
+            type_ids = []
+            for level in self.level:
+                for ci in cis:
+                    type_ids.extend(CITypeRelationManager.get_child_type_ids(ci.type_id, level))
+            type_ids = list(set(type_ids))
+            if self.orig_query:
+                self.orig_query = "_type:({0}),{1}".format(";".join(list(map(str, type_ids))), self.orig_query)
+            else:
+                self.orig_query = "_type:({0})".format(";".join(list(map(str, type_ids))))
+
+        if not merge_ids:
+            # cis, counter, total, self.page, numfound, facet_
+            return [], {}, 0, self.page, 0, {}
+
+        if current_app.config.get("USE_ES"):
+            return SearchFromES(self.orig_query,
+                                fl=self.fl,
+                                facet_field=self.facet_field,
+                                page=self.page,
+                                count=self.count,
+                                sort=self.sort,
+                                ci_ids=merge_ids).search()
+        else:
+            return SearchFromDB(self.orig_query,
+                                fl=self.fl,
+                                facet_field=self.facet_field,
+                                page=self.page,
+                                count=self.count,
+                                sort=self.sort,
+                                ci_ids=merge_ids).search()
+
+    def statistics(self, type_ids):
+        ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
+        for l in range(0, int(self.level)):
+            if l == 0:
+                _tmp = list(map(lambda x: list(json.loads(x).keys()),
+                                [i or '{}' for i in rd.get(ids, REDIS_PREFIX_CI_RELATION) or []]))
+            else:
+                for idx, i in enumerate(_tmp):
+                    if i:
+                        if type_ids and l == self.level - 1:
+                            __tmp = list(
+                                map(lambda x: list({_id: 1 for _id, type_id in json.loads(x).items()
+                                                    if type_id in type_ids}.keys()),
+                                    filter(lambda x: x is not None,
+                                           rd.get(i, REDIS_PREFIX_CI_RELATION) or [])))
+                        else:
+
+                            __tmp = list(map(lambda x: list(json.loads(x).keys()),
+                                             filter(lambda x: x is not None,
+                                                    rd.get(i, REDIS_PREFIX_CI_RELATION) or [])))
+                        _tmp[idx] = [j for i in __tmp for j in i]
+                    else:
+                        _tmp[idx] = []
+
+        return {_id: len(_tmp[idx]) for idx, _id in enumerate(ids)}

cmdb-api/api/lib/cmdb/utils.py

@@ -0,0 +1,116 @@
+# -*- coding:utf-8 -*-
+
+from __future__ import unicode_literals
+
+import datetime
+
+import six
+from markupsafe import escape
+
+import api.models.cmdb as model
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.const import ValueTypeEnum
+
+
+def string2int(x):
+    return int(float(x))
+
+
+def str2datetime(x):
+    try:
+        return datetime.datetime.strptime(x, "%Y-%m-%d")
+    except ValueError:
+        pass
+
+    return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")
+
+
+class ValueTypeMap(object):
+    deserialize = {
+        ValueTypeEnum.INT: string2int,
+        ValueTypeEnum.FLOAT: float,
+        ValueTypeEnum.TEXT: lambda x: escape(x).encode('utf-8').decode('utf-8'),
+        ValueTypeEnum.TIME: lambda x: escape(x).encode('utf-8').decode('utf-8'),
+        ValueTypeEnum.DATETIME: str2datetime,
+        ValueTypeEnum.DATE: str2datetime,
+    }
+
+    serialize = {
+        ValueTypeEnum.INT: int,
+        ValueTypeEnum.FLOAT: float,
+        ValueTypeEnum.TEXT: lambda x: x if isinstance(x, six.text_type) else str(x),
+        ValueTypeEnum.TIME: lambda x: x if isinstance(x, six.text_type) else str(x),
+        ValueTypeEnum.DATE: lambda x: x.strftime("%Y-%m-%d"),
+        ValueTypeEnum.DATETIME: lambda x: x.strftime("%Y-%m-%d %H:%M:%S"),
+    }
+
+    serialize2 = {
+        ValueTypeEnum.INT: int,
+        ValueTypeEnum.FLOAT: float,
+        ValueTypeEnum.TEXT: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
+        ValueTypeEnum.TIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
+        ValueTypeEnum.DATE: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
+        ValueTypeEnum.DATETIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
+    }
+
+    choice = {
+        ValueTypeEnum.INT: model.IntegerChoice,
+        ValueTypeEnum.FLOAT: model.FloatChoice,
+        ValueTypeEnum.TEXT: model.TextChoice,
+    }
+
+    table = {
+        ValueTypeEnum.INT: model.CIValueInteger,
+        ValueTypeEnum.TEXT: model.CIValueText,
+        ValueTypeEnum.DATETIME: model.CIValueDateTime,
+        ValueTypeEnum.DATE: model.CIValueDateTime,
+        ValueTypeEnum.TIME: model.CIValueText,
+        ValueTypeEnum.FLOAT: model.CIValueFloat,
+        'index_{0}'.format(ValueTypeEnum.INT): model.CIIndexValueInteger,
+        'index_{0}'.format(ValueTypeEnum.TEXT): model.CIIndexValueText,
+        'index_{0}'.format(ValueTypeEnum.DATETIME): model.CIIndexValueDateTime,
+        'index_{0}'.format(ValueTypeEnum.DATE): model.CIIndexValueDateTime,
+        'index_{0}'.format(ValueTypeEnum.TIME): model.CIIndexValueText,
+        'index_{0}'.format(ValueTypeEnum.FLOAT): model.CIIndexValueFloat,
+    }
+
+    table_name = {
+        ValueTypeEnum.INT: 'c_value_integers',
+        ValueTypeEnum.TEXT: 'c_value_texts',
+        ValueTypeEnum.DATETIME: 'c_value_datetime',
+        ValueTypeEnum.DATE: 'c_value_datetime',
+        ValueTypeEnum.TIME: 'c_value_texts',
+        ValueTypeEnum.FLOAT: 'c_value_floats',
+        'index_{0}'.format(ValueTypeEnum.INT): 'c_value_index_integers',
+        'index_{0}'.format(ValueTypeEnum.TEXT): 'c_value_index_texts',
+        'index_{0}'.format(ValueTypeEnum.DATETIME): 'c_value_index_datetime',
+        'index_{0}'.format(ValueTypeEnum.DATE): 'c_value_index_datetime',
+        'index_{0}'.format(ValueTypeEnum.TIME): 'c_value_index_texts',
+        'index_{0}'.format(ValueTypeEnum.FLOAT): 'c_value_index_floats',
+    }
+
+    es_type = {
+        ValueTypeEnum.INT: 'long',
+        ValueTypeEnum.TEXT: 'text',
+        ValueTypeEnum.DATETIME: 'text',
+        ValueTypeEnum.DATE: 'text',
+        ValueTypeEnum.TIME: 'text',
+        ValueTypeEnum.FLOAT: 'float'
+    }
+
+
+class TableMap(object):
+    def __init__(self, attr_name=None):
+        self.attr_name = attr_name
+
+    @property
+    def table(self):
+        attr = AttributeCache.get(self.attr_name)
+        i = "index_{0}".format(attr.value_type) if attr.is_index else attr.value_type
+        return ValueTypeMap.table.get(i)
+
+    @property
+    def table_name(self):
+        attr = AttributeCache.get(self.attr_name)
+        i = "index_{0}".format(attr.value_type) if attr.is_index else attr.value_type
+        return ValueTypeMap.table_name.get(i)

cmdb-api/api/lib/cmdb/value.py

@@ -0,0 +1,150 @@
+# -*- coding:utf-8 -*- 
+
+
+from __future__ import unicode_literals
+
+from flask import abort
+
+from api.extensions import db
+from api.lib.cmdb.attribute import AttributeManager
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.const import ExistPolicy
+from api.lib.cmdb.const import OperateType
+from api.lib.cmdb.history import AttributeHistoryManger
+from api.lib.cmdb.utils import TableMap
+from api.lib.cmdb.utils import ValueTypeMap
+from api.lib.utils import handle_arg_list
+
+
+class AttributeValueManager(object):
+    """
+    manage CI attribute values
+    """
+
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def _get_attr(key):
+        """
+        :param key: id, name or alias
+        :return: attribute instance
+        """
+        return AttributeCache.get(key)
+
+    def get_attr_values(self, fields, ci_id, ret_key="name", unique_key=None, use_master=False):
+        """
+
+        :param fields:
+        :param ci_id:
+        :param ret_key: It can be name or alias
+        :param unique_key: primary attribute
+        :param use_master: Only for master-slave read-write separation
+        :return:
+        """
+        res = dict()
+        for field in fields:
+            attr = self._get_attr(field)
+            if not attr:
+                continue
+
+            value_table = TableMap(attr_name=attr.name).table
+            rs = value_table.get_by(ci_id=ci_id,
+                                    attr_id=attr.id,
+                                    use_master=use_master,
+                                    to_dict=False)
+            field_name = getattr(attr, ret_key)
+
+            if attr.is_list:
+                res[field_name] = [ValueTypeMap.serialize[attr.value_type](i.value) for i in rs]
+            else:
+                res[field_name] = ValueTypeMap.serialize[attr.value_type](rs[0].value) if rs else None
+
+            if unique_key is not None and attr.id == unique_key.id and rs:
+                res['unique'] = unique_key.name
+
+        return res
+
+    @staticmethod
+    def __deserialize_value(value_type, value):
+        if not value:
+            return value
+        deserialize = ValueTypeMap.deserialize[value_type]
+        try:
+            v = deserialize(value)
+            return v
+        except ValueError:
+            return abort(400, "attribute value <{0}> is invalid".format(value))
+
+    @staticmethod
+    def __check_is_choice(attr_id, value_type, value):
+        choice_values = AttributeManager.get_choice_values(attr_id, value_type)
+        if value not in choice_values:
+            return abort(400, "{0} does not existed in choice values".format(value))
+
+    @staticmethod
+    def __check_is_unique(value_table, attr_id, ci_id, value):
+        existed = db.session.query(value_table.attr_id).filter(
+            value_table.attr_id == attr_id).filter(value_table.deleted.is_(False)).filter(
+            value_table.value == value).filter(value_table.ci_id != ci_id).first()
+        existed and abort(400, "attribute <{0}> value {1} must be unique".format(attr_id, value))
+
+    def _validate(self, attr, value, value_table, ci_id):
+        v = self.__deserialize_value(attr.value_type, value)
+
+        attr.is_choice and value and self.__check_is_choice(attr.id, attr.value_type, v)
+        attr.is_unique and self.__check_is_unique(value_table, attr.id, ci_id, v)
+
+        return v
+
+    @staticmethod
+    def _write_change(ci_id, attr_id, operate_type, old, new):
+        AttributeHistoryManger.add(ci_id, [(attr_id, operate_type, old, new)])
+
+    def create_or_update_attr_value(self, key, value, ci_id, _no_attribute_policy=ExistPolicy.IGNORE):
+        """
+        add or update attribute value, then write history
+        :param key: id, name or alias
+        :param value:
+        :param ci_id:
+        :param _no_attribute_policy: ignore or reject
+        :return:
+        """
+        attr = self._get_attr(key)
+        if attr is None:
+            if _no_attribute_policy == ExistPolicy.IGNORE:
+                return
+            if _no_attribute_policy == ExistPolicy.REJECT:
+                return abort(400, 'attribute {0} does not exist'.format(key))
+
+        value_table = TableMap(attr_name=attr.name).table
+
+        if attr.is_list:
+            value_list = [self._validate(attr, i, value_table, ci_id) for i in handle_arg_list(value)]
+            existed_attrs = value_table.get_by(attr_id=attr.id,
+                                               ci_id=ci_id,
+                                               to_dict=False)
+            existed_values = [i.value for i in existed_attrs]
+            added = set(value_list) - set(existed_values)
+            deleted = set(existed_values) - set(value_list)
+            for v in added:
+                value_table.create(ci_id=ci_id, attr_id=attr.id, value=v)
+                self._write_change(ci_id, attr.id, OperateType.ADD, None, v)
+
+            for v in deleted:
+                existed_attr = existed_attrs[existed_values.index(v)]
+                existed_attr.delete()
+                self._write_change(ci_id, attr.id, OperateType.DELETE, v, None)
+        else:
+            value = self._validate(attr, value, value_table, ci_id)
+            existed_attr = value_table.get_by(attr_id=attr.id,
+                                              ci_id=ci_id,
+                                              first=True,
+                                              to_dict=False)
+            existed_value = existed_attr and existed_attr.value
+            if existed_value is None:
+                value_table.create(ci_id=ci_id, attr_id=attr.id, value=value)
+                self._write_change(ci_id, attr.id, OperateType.ADD, None, value)
+            else:
+                existed_attr.update(value=value)
+                self._write_change(ci_id, attr.id, OperateType.UPDATE, existed_value, value)