v1 vsersion

lib/__init__.py

@@ -0,0 +1,4 @@
+# -*- coding:utf-8 -*- 
+
+
+__all__ = []

lib/account.py

@@ -0,0 +1,145 @@
+# -*- coding:utf-8 -*- 
+
+
+import uuid
+import random
+import string
+import datetime
+
+from flask import current_app
+from flask import abort
+
+from extensions import db
+from models.account import UserCache
+from models.account import User
+from models.account import UserRole
+
+
+class AccountManager(object):
+    def __init__(self):
+        pass
+
+    def get_user_by_uid(self, uid):
+        user = UserCache.get(uid)
+        return user
+
+    def _generate_key(self):
+        key = uuid.uuid4().hex
+        secret = ''.join(random.sample(string.ascii_letters +
+                                       string.digits + '~!@#$%^&*?', 32))
+        return key, secret
+
+    def validate(self, username, password):
+        user, authenticated = User.query.authenticate(username, password)
+        return user, authenticated
+
+    def create_user(self, **kwargs):
+        username = kwargs.get("username")
+        if username:
+            user = UserCache.get(username)
+            if user is not None:
+                user, authenticated = self.validate(
+                    username, kwargs.get("password"))
+                if authenticated:
+                    return user
+                else:
+                    return abort(401, "authenticate validate failed")
+        else:
+            return abort(400, "argument username is required")
+        user = User()
+        email = kwargs.get("email", "")
+        if not email:
+            return abort(400, "argument email is required")
+        user.email = email
+        user.password = kwargs.get("password")
+        user.username = kwargs.get("username", "")
+        user.nickname = kwargs.get("nickname") if kwargs.get("nickname") \
+            else kwargs.get("username", "")
+        key, secret = self._generate_key()
+        user.key = key
+        user.secret = secret
+        user.date_joined = datetime.datetime.now()
+        user.block = 0
+
+        db.session.add(user)
+        try:
+            db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error("create user is error {0}".format(str(e)))
+            return abort(500, "create user is error, {0}".format(str(e)))
+        return user
+
+    def update_user(self, uid, **kwargs):
+        user = UserCache.get(uid)
+        if user is None:
+            return abort(400, "the user[{0}] is not existed".format(uid))
+        user.username = kwargs.get("username", "") \
+            if kwargs.get("username") else user.username
+        user.nickname = kwargs.get("nickname") \
+            if kwargs.get("nickname") else user.nickname
+        user.department = kwargs.get("department") \
+            if kwargs.get("department") else user.department
+        user.catalog = kwargs.get("catalog") \
+            if kwargs.get("catalog") else user.catalog
+        user.email = kwargs.get("email") \
+            if kwargs.get("email") else user.email
+        user.mobile = kwargs.get("mobile") \
+            if kwargs.get("mobile") else user.mobile
+        db.session.add(user)
+        try:
+            db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error("create user is error {0}".format(str(e)))
+            return abort(500, "create user is error, {0}".format(str(e)))
+        return True, user
+
+    def delete_user(self, uid):
+        user = UserCache.get(uid)
+        if user is None:
+            return abort(400, "the user[{0}] is not existed".format(uid))
+        db.session.query(UserRole).filter(UserRole.uid == uid).delete()
+        db.session.delete(user)
+        try:
+            db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error("delete user error, {0}".format(str(e)))
+            return abort(500, "delete user error, {0}".format(str(e)))
+        return True, uid
+
+    def update_password(self, uid, old, new, confirm):
+        user = User.query.get(uid)
+        if not user:
+            return abort(400, "user is not existed")
+        if not user.check_password(old):
+            return abort(400, "invalidate old password")
+        if not (new and confirm and new == confirm):
+            return abort(400, """Password cannot be empty,
+            two inputs must be the same""")
+        user.password = new
+        db.session.add(user)
+        try:
+            db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error("set password error, %s" % str(e))
+            return abort(500, "set password errors, {0:s}".format(str(e)))
+        return True, user
+
+    def reset_key(self, uid):
+        user = UserCache.get(uid)
+        if user is None:
+            return abort(400, "the user[{0}] is not existed".format(uid))
+        key, secret = self._generate_key()
+        user.key = key
+        user.secret = secret
+        db.session.add(user)
+        try:
+            db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error("reset key is error, {0}".format(str(e)))
+            return abort(500, "reset key is error, {0}".format(str(e)))
+        return True, user

lib/attribute.py

@@ -0,0 +1,167 @@
+# -*- coding:utf-8 -*- 
+
+from flask import current_app
+from flask import abort
+
+from extensions import db
+from models.attribute import CIAttribute
+from models.attribute import CIAttributeCache
+from models import row2dict
+from lib.const import type_map
+
+
+class AttributeManager(object):
+    """
+    CI attributes manager
+    """
+
+    def __init__(self):
+        pass
+
+    def _get_choice_value(self, attr_id, value_type):
+        _table = type_map.get("choice").get(value_type)
+        choice_values = db.session.query(_table.value).filter(
+            _table.attr_id == attr_id).all()
+        return [choice_value.value for choice_value in choice_values]
+
+    def _add_choice_value(self, choice_value, attr_id, value_type):
+        _table = type_map.get("choice").get(value_type)
+        db.session.query(_table).filter(_table.attr_id == attr_id).delete()
+        db.session.flush()
+        for v in choice_value.strip().split(","):
+            table = _table()
+            table.attr_id = attr_id
+            table.value = v
+            db.session.add(table)
+        db.session.flush()
+
+    def get_attributes(self, name=None):
+        """
+        return attribute by name,
+        if name is None, then return all attributes
+        """
+        attrs = db.session.query(CIAttribute).filter(
+            CIAttribute.attr_name.ilike("%{0}%".format(name))).all() \
+            if name is not None else db.session.query(CIAttribute).all()
+        res = list()
+        for attr in attrs:
+            attr_dict = row2dict(attr)
+            if attr.is_choice:
+                attr_dict["choice_value"] = self._get_choice_value(
+                    attr.attr_id, attr.value_type)
+            res.append(attr_dict)
+        return res
+
+    def get_attribute_by_name(self, attr_name):
+        attr = db.session.query(CIAttribute).filter(
+            CIAttribute.attr_name == attr_name).first()
+        if attr:
+            attr_dict = row2dict(attr)
+            if attr.is_choice:
+                attr_dict["choice_value"] = self._get_choice_value(
+                    attr.attr_id, attr.value_type)
+            return attr_dict
+
+    def get_attribute_by_alias(self, attr_alias):
+        attr = db.session.query(CIAttribute).filter(
+            CIAttribute.attr_alias == attr_alias).first()
+        if attr:
+            attr_dict = row2dict(attr)
+            if attr.is_choice:
+                attr_dict["choice_value"] = self._get_choice_value(
+                    attr.attr_id, attr.value_type)
+            return attr_dict
+
+    def get_attribute_by_id(self, attr_id):
+        attr = db.session.query(CIAttribute).filter(
+            CIAttribute.attr_id == attr_id).first()
+        if attr:
+            attr_dict = row2dict(attr)
+            if attr.is_choice:
+                attr_dict["choice_value"] = self._get_choice_value(
+                    attr.attr_id, attr.value_type)
+            return attr_dict
+
+    def add(self, attr_name, attr_alias, **kwargs):
+        choice_value = kwargs.get("choice_value", False)
+        attr = CIAttributeCache.get(attr_name)
+        if attr is not None:
+            return False, "attribute {0} is already existed".format(attr_name)
+        is_choice = False
+        if choice_value:
+            is_choice = True
+        if not attr_alias:
+            attr_alias = attr_name
+        attr = CIAttribute()
+        attr.attr_name = attr_name
+        attr.attr_alias = attr_alias
+        attr.is_choice = is_choice
+        attr.is_multivalue = kwargs.get("is_multivalue", False)
+        attr.is_uniq = kwargs.get("is_uniq", False)
+        attr.is_index = kwargs.get("is_index", False)
+        attr.value_type = kwargs.get("value_type", "text")
+        db.session.add(attr)
+        db.session.flush()
+
+        if choice_value:
+            self._add_choice_value(choice_value, attr.attr_id, attr.value_type)
+        try:
+            db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error("add attribute error, {0}".format(str(e)))
+            return False, str(e)
+        CIAttributeCache.clean(attr)
+        return True, attr.attr_id
+
+    def update(self, attr_id, *args, **kwargs):
+        attr = db.session.query(CIAttribute).filter_by(attr_id=attr_id).first()
+        if not attr:
+            return False, "CI attribute you want to update is not existed"
+        choice_value = kwargs.get("choice_value", False)
+        is_choice = False
+        if choice_value:
+            is_choice = True
+        attr.attr_name = args[0]
+        attr.attr_alias = args[1]
+        if not args[1]:
+            attr.attr_alias = args[0]
+        attr.is_choice = is_choice
+        attr.is_multivalue = kwargs.get("is_multivalue", False)
+        attr.is_uniq = kwargs.get("is_uniq", False)
+        attr.value_type = kwargs.get("value_type", "text")
+        db.session.add(attr)
+        db.session.flush()
+        if is_choice:
+            self._add_choice_value(choice_value, attr.attr_id, attr.value_type)
+        try:
+            db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error("update attribute error, {0}".format(
+                str(e)))
+            return False, str(e)
+        CIAttributeCache.clean(attr)
+        return True, attr.attr_id
+
+    def delete(self, attr_id):
+        attr, name = db.session.query(CIAttribute).filter_by(
+            attr_id=attr_id).first(), None
+        if attr:
+            if attr.is_choice:
+                choice_table = type_map["choice"].get(attr.value_type)
+                db.session.query(choice_table).filter(
+                    choice_table.attr_id == attr_id).delete()
+            name = attr.attr_name
+            CIAttributeCache.clean(attr)
+            db.session.delete(attr)
+            try:
+                db.session.commit()
+            except Exception as e:
+                db.session.rollback()
+                current_app.logger.error("delete attribute error, {0}".format(
+                    str(e)))
+                return abort(500, str(e))
+        else:
+            return abort(404, "attribute you want to delete is not existed")
+        return name

lib/audit.py

@@ -0,0 +1,168 @@
+# -*- coding:utf-8 -*- 
+
+__author__ = 'pycook'
+
+import datetime
+
+from flask import current_app
+
+from models.cmdb import CIAudit
+from models.cmdb import CIAttributeAudit
+from models.cmdb import CIAttributeCache
+from models.cmdb import CITypeCache
+from models.cmdb import CI
+from models import row2dict
+from extensions import db
+from lib.const import TableMap
+from tasks.cmdb import ci_cache
+
+
+class CIAuditManager(object):
+    def __init__(self):
+        pass
+
+    def get_cis_for_audits(self, page, type_ids, per_page=25):
+        audit_cis = db.session.query(CIAudit)
+        if type_ids:
+            audit_cis = audit_cis.join(CI, CI.ci_id == CIAudit.ci_id).filter(
+                CI.type_id.in_(type_ids))
+
+        audit_cis = audit_cis.filter(CIAudit.is_audit == 0).order_by(
+            CIAudit.created_at)
+        numfound = audit_cis.count()
+        audit_cis = audit_cis.offset((page - 1) * per_page).limit(per_page)
+        total = audit_cis.count()
+        audit_cis = audit_cis.all()
+        result = list()
+        for audit_ci in audit_cis:
+            audit_dict = row2dict(audit_ci)
+            audit_attrs = db.session.query(CIAttributeAudit).filter(
+                CIAttributeAudit.audit_id == audit_ci.audit_id).all()
+            audit_dict["values"] = list()
+            for audit_attr in audit_attrs:
+                audit_attr_dict = row2dict(audit_attr)
+                audit_attr_dict["attr_name"] = CIAttributeCache.get(
+                    audit_attr.attr_id).attr_name
+                audit_dict['values'].append(audit_attr_dict)
+            result.append(audit_dict)
+        return numfound, total, result
+
+    def create_ci_audits(self, type_name, attr_pairs):
+        ci_type = CITypeCache.get(type_name)
+        uniq_key = CIAttributeCache.get(ci_type.uniq_id)
+        table = TableMap(attr_name=uniq_key.attr_name).table
+        value = db.session.query(table.ci_id).filter(
+            table.attr_id == uniq_key.attr_id).filter(
+                table.value == attr_pairs.get(uniq_key.attr_name)).first()
+        del attr_pairs[uniq_key.attr_name]
+        if value and attr_pairs:
+            ci_audit = db.session.query(CIAudit).filter(
+                CIAudit.ci_id == value.ci_id).filter(
+                    CIAudit.is_audit == 0).first()
+            if ci_audit is None:
+                ci_audit = CIAudit()
+                ci_audit.is_notified = False
+                ci_audit.created_at = datetime.datetime.now()
+            ci_audit.ci_id = value.ci_id
+            ci_audit.updated_at = datetime.datetime.now()
+            ci_audit.origin = 1   # TODO
+            db.session.add(ci_audit)
+            db.session.commit()
+            for attr, attr_value in attr_pairs.items():
+                attr_id = CIAttributeCache.get(attr).attr_id
+                ci_attr_audit = CIAttributeAudit()
+                ci_attr_audit.attr_id = attr_id
+                all_values = attr_value.strip().split("###")
+                ci_attr_audit.cur_value = all_values[0]
+                ci_attr_audit.new_value = all_values[1]
+                ci_attr_audit.audit_id = ci_audit.audit_id
+                db.session.add(ci_attr_audit)
+                db.session.flush()
+            try:
+                db.session.commit()
+            except Exception as e:
+                db.session.rollback()
+                current_app.logger.error("create ci audits error, %s" % str(e))
+                return False, "create ci audits error, %s" % str(e)
+        return True, None
+
+    def _update_cmdb(self, ci_id, attr_id, value):
+        try:
+            attr_name = CIAttributeCache.get(attr_id).attr_name
+            table = TableMap(attr_name=attr_name).table
+            attr_value = db.session.query(table).filter(
+                table.attr_id == attr_id).filter(
+                    table.ci_id == ci_id).first()
+            attr_value.value = value
+            db.session.add(attr_value)
+
+        except Exception as e:
+            return False, "audit failed, %s" % str(e)
+        return True, ci_id
+
+    def audit_by_attr(self, audit_id, attr_ids, value=None):
+        ci_audit = CIAudit.query.get(audit_id)
+        ci_id = ci_audit.ci_id
+        for attr_id in attr_ids:
+            attr_audit = db.session.query(CIAttributeAudit).filter(
+                CIAttributeAudit.audit_id == audit_id).filter(
+                    CIAttributeAudit.attr_id == attr_id).first()
+            if attr_audit:
+                attr_audit.is_audit = True
+                attr_audit.auditor = 1  # TODO
+                attr_audit.audited_at = datetime.datetime.now()
+                if value is not None:
+                    attr_audit.audit_value = value
+                else:
+                    attr_audit.audit_value = attr_audit.new_value
+                if attr_audit.cur_value != value:  # update cmdb
+                    ret, res = self._update_cmdb(ci_id, attr_id, value)
+                    if not ret:
+                        return False, res
+                    attr_audit.is_updated = True
+                db.session.add(attr_audit)
+                db.session.flush()
+            if db.session.query(CIAttributeAudit).filter_by(
+                    audit_id=audit_id).filter_by(is_audit=0).first() is None:
+                ci_audit.is_audit = True
+                ci_audit.updated_at = datetime.datetime.now()
+                db.session.add(ci_audit)
+        ci_cache.apply_async([ci_id], queue="cmdb_async")
+        try:
+            db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error(
+                "audit by attribute is error, {0}".format(str(e)))
+            return False, "audit by attribute is error, %s" % str(e)
+        return True, None
+
+    def audit_by_cis(self, ci_ids):
+        for ci_id in ci_ids:
+            ci_audit = db.session.query(CIAudit).filter_by(ci_id=ci_id).first()
+            attr_audits = db.session.query(CIAttributeAudit).filter_by(
+                audit_id=ci_audit.audit_id).all()
+            for attr_audit in attr_audits:
+                attr_audit.is_audit = True
+                attr_audit.auditor = 1  # TODO
+                attr_audit.audited_at = datetime.datetime.now()
+                attr_audit.audit_value = attr_audit.new_value
+                ret, res = self._update_cmdb(
+                    ci_id, attr_audit.attr_id,
+                    attr_audit.new_value)
+                if not ret:
+                    return False, res
+                attr_audit.is_updated = True
+                db.session.add(attr_audit)
+            ci_audit.is_audit = True
+            ci_audit.updated_at = datetime.datetime.now()
+            db.session.add(ci_audit)
+            ci_cache.apply_async([ci_id], queue="cmdb_async")
+        try:
+            db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error(
+                "audit by attribute error, {0}".format(str(e)))
+            return False, "audit by cis is error, %s" % str(e)
+        return True, None

lib/auth.py

@@ -0,0 +1,68 @@
+# -*- coding:utf-8 -*- 
+
+import urllib
+from functools import wraps
+
+from flask import current_app
+from flask import g
+from flask import request
+from flask import abort
+from flask.ext.principal import identity_changed
+from flask.ext.principal import Identity
+from flask.ext.principal import AnonymousIdentity
+
+from models.account import User
+from models.account import UserCache
+
+
+def auth_with_key(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        if isinstance(getattr(g, 'user', None), User):
+            identity_changed.send(current_app._get_current_object(),
+                                  identity=Identity(g.user.uid))
+            return func(*args, **kwargs)
+        ip = request.remote_addr
+        if request.data:
+            request_args = dict()
+            _args = request.data.split("&")
+            for arg in _args:
+                if arg:
+                    request_args[arg.split("=")[0]] = \
+                        urllib.unquote(arg.split("=")[1])
+        else:
+            request_args = request.values
+
+        key = request_args.get('_key')
+        secret = request_args.get('_secret')
+        if not key and not secret and \
+                ip.strip() in current_app.config.get("WHITE_LIST"):
+            ip = ip.strip()
+            user = UserCache.get(ip)
+            if user:
+                identity_changed.send(current_app._get_current_object(),
+                                      identity=Identity(user.uid))
+                return func(*args, **kwargs)
+            else:
+                identity_changed.send(current_app._get_current_object(),
+                                      identity=AnonymousIdentity())
+                return abort(400, "invalid _key and _secret")
+
+        path = request.path
+
+        keys = sorted(request_args.keys())
+        req_args = [request_args[k] for k in keys
+                    if str(k) not in ("_key", "_secret")]
+        current_app.logger.debug('args is %s' % req_args)
+        user, authenticated = User.query.authenticate_with_key(
+            key, secret, req_args, path)
+        if user and authenticated:
+            identity_changed.send(current_app._get_current_object(),
+                                  identity=Identity(user.get("uid")))
+            return func(*args, **kwargs)
+        else:
+            identity_changed.send(current_app._get_current_object(),
+                                  identity=AnonymousIdentity())
+            return abort(400, "invalid _key and _secret")
+
+    return wrapper

lib/ci.py

@@ -0,0 +1,677 @@
+# -*- coding:utf-8 -*- 
+
+
+import uuid
+import time
+import datetime
+import json
+
+from flask import current_app
+from flask import abort
+from sqlalchemy import or_
+
+from extensions import db
+from models.ci import CI
+from models.ci_relation import CIRelation
+from models.ci_type import CITypeAttribute
+from models.ci_type import CITypeCache
+from models.ci_type import CITypeSpecCache
+from models.history import CIAttributeHistory
+from models.attribute import CIAttributeCache
+from lib.const import TableMap
+from lib.const import type_map
+from lib.value import AttributeValueManager
+from lib.history import CIAttributeHistoryManger
+from lib.history import CIRelationHistoryManager
+from lib.query_sql import QUERY_HOSTS_NUM_BY_PRODUCT
+from lib.query_sql import QUERY_HOSTS_NUM_BY_BU
+from lib.query_sql import QUERY_HOSTS_NUM_BY_PROJECT
+from lib.query_sql import QUERY_CIS_BY_IDS
+from lib.query_sql import QUERY_CIS_BY_VALUE_TABLE
+from lib.utils import rd
+from tasks.cmdb import ci_cache
+from tasks.cmdb import ci_delete
+
+
+class CIManager(object):
+    """ manage CI interface
+    """
+
+    def __init__(self):
+        pass
+
+    def get_ci_by_id(self, ci_id, ret_key="name",
+                     fields=None, need_children=True, use_master=False):
+        """@params: `ret_key` is one of 'name', 'id', 'alias'
+                    `fields` is list of attribute name/alias/id
+        """
+        ci = CI.query.get(ci_id) or \
+            abort(404, "CI {0} is not existed".format(ci_id))
+
+        res = dict()
+
+        if need_children:
+            children = self.get_children(ci_id, ret_key=ret_key)  # one floor
+            res.update(children)
+        ci_type = CITypeCache.get(ci.type_id)
+        res["ci_type"] = ci_type.type_name
+        uniq_key = CIAttributeCache.get(ci_type.uniq_id)
+        if not fields:   # fields are all attributes
+            attr_ids = db.session.query(CITypeAttribute.attr_id).filter_by(
+                type_id=ci.type_id)
+            fields = [CIAttributeCache.get(_.attr_id).attr_name
+                      for _ in attr_ids]
+
+        if uniq_key.attr_name not in fields:
+            fields.append(uniq_key.attr_name)
+        if fields:
+            value_manager = AttributeValueManager()
+            _res = value_manager._get_attr_values(
+                fields, ci_id,
+                ret_key=ret_key, uniq_key=uniq_key, use_master=use_master)
+            res.update(_res)
+            res['_type'] = ci_type.type_id
+            res['_id'] = ci_id
+        return res
+
+    def get_ci_by_ids(self, ci_id_list, ret_key="name", fields=None):
+        result = list()
+        for ci_id in ci_id_list:
+            res = self.get_ci_by_id(ci_id, ret_key=ret_key, fields=fields)
+            result.append(res)
+        return result
+
+    def get_children(self, ci_id, ret_key='name', relation_type="contain"):
+        second_cis = db.session.query(CIRelation.second_ci_id).filter(
+            CIRelation.first_ci_id == ci_id).filter(or_(
+                CIRelation.relation_type == relation_type,
+                CIRelation.relation_type == "deploy"))
+        second_ci_ids = (second_ci.second_ci_id for second_ci in second_cis)
+        ci_types = {}
+        for ci_id in second_ci_ids:
+            type_id = db.session.query(CI.type_id).filter(
+                CI.ci_id == ci_id).first().type_id
+            if type_id not in ci_types:
+                ci_types[type_id] = [ci_id]
+            else:
+                ci_types[type_id].append(ci_id)
+        res = {}
+        for type_id in ci_types:
+            ci_type = CITypeCache.get(type_id)
+            children = get_cis_by_ids(map(str, ci_types.get(type_id)),
+                                      ret_key=ret_key)
+            res[ci_type.type_name] = children
+        return res
+
+    def get_cis_by_type(self, type_id, ret_key="name", fields="",
+                        page=1, per_page=None):
+        if per_page is None:
+            per_page = current_app.config.get("DEFAULT_PAGE_COUNT")
+        cis = db.session.query(CI.ci_id).filter(CI.type_id == type_id)
+        numfound = cis.count()
+        cis = cis.offset((page - 1) * per_page).limit(per_page)
+        res = list()
+        ci_ids = [str(ci.ci_id) for ci in cis]
+        if ci_ids:
+            res = get_cis_by_ids(ci_ids, ret_key, fields)
+        return numfound, page, res
+
+    def ci_is_exist(self, ci_type, unique_key, unique):
+        table = TableMap(attr_name=unique_key.attr_name).table
+        unique = db.session.query(table).filter(
+            table.attr_id == unique_key.attr_id).filter(
+                table.value == unique).first()
+        if unique:
+            return db.session.query(CI).filter(
+                CI.ci_id == unique.ci_id).first()
+
+    def _delete_ci_by_id(self, ci_id):
+        db.session.query(CI.ci_id).filter(CI.ci_id == ci_id).delete()
+        try:
+            db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error("delete ci is error, {0}".format(str(e)))
+
+    def add(self, ci_type_name, exist_policy="replace",
+            _no_attribute_policy="ignore", **ci_dict):
+        ci_existed = False
+        ci_type = CITypeCache.get(ci_type_name) or \
+            abort(404, "CIType {0} is not existed".format(ci_type_name))
+
+        unique_key = CIAttributeCache.get(ci_type.uniq_id) \
+            or abort(500, 'illegality unique attribute')
+
+        unique = ci_dict.get(unique_key.attr_name) \
+            or abort(500, '{0} missing'.format(unique_key.attr_name))
+
+        old_ci = self.ci_is_exist(ci_type, unique_key, unique)
+        if old_ci is not None:
+            ci_existed = True
+            if exist_policy == 'reject':
+                return abort(500, 'CI is existed')
+            if old_ci.type_id != ci_type.type_id:  # update ci_type
+                old_ci.type_id = ci_type.type_id
+                db.session.add(old_ci)
+                db.session.flush()
+            ci = old_ci
+        else:
+            if exist_policy == 'need':
+                return abort(404, 'CI {0} not exist'.format(unique))
+            ci = CI()
+            ci.type_id = ci_type.type_id
+            _uuid = uuid.uuid4().hex
+            ci.uuid = _uuid
+            ci.created_time = datetime.datetime.now()
+            db.session.add(ci)
+            try:
+                db.session.commit()
+            except Exception as e:
+                db.session.rollback()
+                current_app.logger.error('add CI error: {0}'.format(str(e)))
+                return abort(500, 'add CI error')
+        value_manager = AttributeValueManager()
+        histories = list()
+        for p, v in ci_dict.items():
+            ret, res = value_manager.add_attr_value(
+                p, v, ci.ci_id, ci_type,
+                _no_attribute_policy=_no_attribute_policy,
+                ci_existed=ci_existed)
+            if not ret:
+                db.session.rollback()
+                if not ci_existed:
+                    self.delete(ci.ci_id)
+                current_app.logger.info(res)
+                return abort(500, res)
+            if res is not None:
+                histories.append(res)
+        try:
+            db.session.commit()
+        except Exception as e:
+            current_app.logger.error(str(e))
+            db.session.rollback()
+            if not ci_existed:  # only add
+                self.delete(ci.ci_id)
+            return abort(500, "add CI error")
+        his_manager = CIAttributeHistoryManger()
+        his_manager.add(ci.ci_id, histories)
+        ci_cache.apply_async([ci.ci_id], queue="cmdb_async")
+        return ci.ci_id
+
+    def delete(self, ci_id):
+        ci = db.session.query(CI).filter(CI.ci_id == ci_id).first()
+        if ci is not None:
+            attrs = db.session.query(CITypeAttribute.attr_id).filter(
+                CITypeAttribute.type_id == ci.type_id).all()
+            attr_names = []
+            for attr in attrs:
+                attr_names.append(CIAttributeCache.get(attr.attr_id).attr_name)
+            attr_names = set(attr_names)
+            for attr_name in attr_names:
+                Table = TableMap(attr_name=attr_name).table
+                db.session.query(Table).filter(Table.ci_id == ci_id).delete()
+            db.session.query(CIRelation).filter(
+                CIRelation.first_ci_id == ci_id).delete()
+            db.session.query(CIRelation).filter(
+                CIRelation.second_ci_id == ci_id).delete()
+            db.session.query(CIAttributeHistory).filter(
+                CIAttributeHistory.ci_id == ci_id).delete()
+
+            db.session.flush()
+            db.session.delete(ci)
+            try:
+                db.session.commit()
+            except Exception as e:
+                db.session.rollback()
+                current_app.logger.error("delete CI error, {0}".format(str(e)))
+                return abort(500, "delete CI error, {0}".format(str(e)))
+            # TODO: write history
+            ci_delete.apply_async([ci.ci_id], queue="cmdb_async")
+            return ci_id
+        return abort(404, "CI {0} not found".format(ci_id))
+
+    def add_heartbeat(self, ci_type, unique):
+        ci_type = CITypeCache.get(ci_type)
+        if not ci_type:
+            return 'error'
+        uniq_key = CIAttributeCache.get(ci_type.uniq_id)
+        Table = TableMap(attr_name=uniq_key.attr_name).table
+        ci_id = db.session.query(Table.ci_id).filter(
+            Table.attr_id == uniq_key.attr_id).filter(
+                Table.value == unique).first()
+        if ci_id is None:
+            return 'error'
+        ci = db.session.query(CI).filter(CI.ci_id == ci_id.ci_id).first()
+        if ci is None:
+            return 'error'
+
+        ci.heartbeat = datetime.datetime.now()
+
+        db.session.add(ci)
+        db.session.commit()
+        return "ok"
+
+    def get_heartbeat(self, page, type_id, agent_status=None):
+        query = db.session.query(CI.ci_id, CI.heartbeat)
+        expire = datetime.datetime.now() - datetime.timedelta(minutes=72)
+        if type_id:
+            query = query.filter(CI.type_id == type_id)
+        else:
+            query = query.filter(db.or_(CI.type_id == 7, CI.type_id == 8))
+        if agent_status == -1:
+            query = query.filter(CI.heartbeat == None)
+        elif agent_status == 0:
+            query = query.filter(CI.heartbeat <= expire)
+        elif agent_status == 1:
+            query = query.filter(CI.heartbeat > expire)
+        numfound = query.count()
+        per_page_count = current_app.config.get("DEFAULT_PAGE_COUNT")
+        cis = query.offset((page - 1) * per_page_count).limit(
+            per_page_count).all()
+        ci_ids = [ci.ci_id for ci in cis]
+        heartbeat_dict = {}
+        for ci in cis:
+            if agent_status is not None:
+                heartbeat_dict[ci.ci_id] = agent_status
+            else:
+                if ci.heartbeat is None:
+                    heartbeat_dict[ci.ci_id] = -1
+                elif ci.heartbeat <= expire:
+                    heartbeat_dict[ci.ci_id] = 0
+                else:
+                    heartbeat_dict[ci.ci_id] = 1
+        current_app.logger.debug(heartbeat_dict)
+        ci_ids = map(str, ci_ids)
+        res = get_cis_by_ids(ci_ids, fields=["hostname", "private_ip"])
+        result = [(i.get("hostname"), i.get("private_ip")[0], i.get("ci_type"),
+                   heartbeat_dict.get(i.get("_id"))) for i in res
+                  if i.get("private_ip")]
+        return numfound, result
+
+
+class CIRelationManager(object):
+    """
+    manage relation between CIs
+    """
+
+    def __init__(self):
+        pass
+
+    @property
+    def relation_types(self):
+        """ all CIType relation types
+        """
+        from lib.const import CI_RELATION_TYPES
+
+        return CI_RELATION_TYPES
+
+    def get_second_cis(self, first_ci, relation_type="contain",
+                       page=1, per_page=None, **kwargs):
+        if per_page is None:
+            per_page = current_app.config.get("DEFAULT_PAGE_COUNT")
+        second_cis = db.session.query(
+            CI.ci_id).join(
+                CIRelation, CIRelation.second_ci_id == CI.ci_id).filter(
+                    CIRelation.first_ci_id == first_ci).filter(
+                        CIRelation.relation_type == relation_type)
+        if kwargs:  # special for devices
+            second_cis = self._query_wrap_for_device(second_cis, **kwargs)
+        numfound = second_cis.count()
+        second_cis = second_cis.offset(
+            (page - 1) * per_page).limit(per_page).all()
+        ci_ids = [str(son.ci_id) for son in second_cis]
+        total = len(ci_ids)
+        result = get_cis_by_ids(ci_ids)
+        return numfound, total, result
+
+    def get_grandsons(self, ci_id, page=1, per_page=None, **kwargs):
+        if per_page is None:
+            per_page = current_app.config.get("DEFAULT_PAGE_COUNT")
+        children = db.session.query(CIRelation.second_ci_id).filter(
+            CIRelation.first_ci_id == ci_id).subquery()
+        grandsons = db.session.query(CIRelation.second_ci_id).join(
+            children,
+            children.c.second_ci_id == CIRelation.first_ci_id).subquery()
+        grandsons = db.session.query(CI.ci_id).join(
+            grandsons, grandsons.c.second_ci_id == CI.ci_id)
+        if kwargs:
+            grandsons = self._query_wrap_for_device(grandsons, **kwargs)
+
+        numfound = grandsons.count()
+        grandsons = grandsons.offset(
+            (page - 1) * per_page).limit(per_page).all()
+        if not grandsons:
+            return 0, 0, []
+        ci_ids = [str(son.ci_id) for son in grandsons]
+        total = len(ci_ids)
+        result = get_cis_by_ids(ci_ids)
+
+        return numfound, total, result
+
+    def _sort_handler(self, sort_by, query_sql):
+
+        if sort_by.startswith("+"):
+            sort_type = "asc"
+            sort_by = sort_by[1:]
+        elif sort_by.startswith("-"):
+            sort_type = "desc"
+            sort_by = sort_by[1:]
+        else:
+            sort_type = "asc"
+        attr = CIAttributeCache.get(sort_by)
+        if attr is None:
+            return query_sql
+
+        attr_id = attr.attr_id
+        Table = TableMap(attr_name=sort_by).table
+
+        CI_table = query_sql.subquery()
+        query_sql = db.session.query(CI_table.c.ci_id, Table.value).join(
+            Table, Table.ci_id == CI_table.c.ci_id).filter(
+                Table.attr_id == attr_id).order_by(
+                    getattr(Table.value, sort_type)())
+
+        return query_sql
+
+    def _query_wrap_for_device(self, query_sql, **kwargs):
+        _type = kwargs.pop("_type", False) or kwargs.pop("type", False) \
+            or kwargs.pop("ci_type", False)
+        if _type:
+            ci_type = CITypeCache.get(_type)
+            if ci_type is None:
+                return
+            query_sql = query_sql.filter(CI.type_id == ci_type.type_id)
+
+        for k, v in kwargs.iteritems():
+            attr = CIAttributeCache.get(k)
+            if attr is None:
+                continue
+            Table = TableMap(attr_name=k).table
+            CI_table = query_sql.subquery()
+            query_sql = db.session.query(CI_table.c.ci_id).join(
+                Table, Table.ci_id == CI_table.c.ci_id).filter(
+                    Table.attr_id == attr.attr_id).filter(
+                        Table.value.ilike(v.replace("*", "%")))
+
+        current_app.logger.debug(query_sql)
+        sort_by = kwargs.pop("sort", False)
+        if sort_by:
+            query_sql = self._sort_handler(sort_by, query_sql)
+        return query_sql
+
+    def get_great_grandsons(self, ci_id, page=1, per_page=None, **kwargs):
+        if per_page is None:
+            per_page = current_app.config.get("DEFAULT_PAGE_COUNT")
+
+        children = db.session.query(CIRelation.second_ci_id).filter(
+            CIRelation.first_ci_id == ci_id).subquery()
+        grandsons = db.session.query(CIRelation.second_ci_id).join(
+            children,
+            children.c.second_ci_id == CIRelation.first_ci_id).subquery()
+        great_grandsons = db.session.query(CIRelation.second_ci_id).join(
+            grandsons,
+            grandsons.c.second_ci_id == CIRelation.first_ci_id).subquery()
+        great_grandsons = db.session.query(CI.ci_id).join(
+            great_grandsons, great_grandsons.c.second_ci_id == CI.ci_id)
+        if kwargs:
+            great_grandsons = self._query_wrap_for_device(
+                great_grandsons, **kwargs)
+        if great_grandsons is None:
+            return 0, 0, []
+        numfound = great_grandsons.count()
+        great_grandsons = great_grandsons.offset(
+            (page - 1) * per_page).limit(per_page).all()
+        ci_ids = [str(son.ci_id) for son in great_grandsons]
+        total = len(ci_ids)
+        result = get_cis_by_ids(ci_ids)
+
+        return numfound, total, result
+
+    def get_first_cis(self, second_ci, relation_type="contain",
+                      page=1, per_page=None):
+        """only for CI Type
+        """
+        if per_page is None:
+            per_page = current_app.config.get("DEFAULT_PAGE_COUNT")
+        first_cis = db.session.query(CIRelation.first_ci_id).filter(
+            CIRelation.second_ci_id == second_ci).filter(
+                CIRelation.relation_type == relation_type)
+        numfound = first_cis.count()
+        first_cis = first_cis.offset(
+            (page - 1) * per_page).limit(per_page).all()
+        result = []
+        first_ci_ids = [str(first_ci.first_ci_id) for first_ci in first_cis]
+        total = len(first_ci_ids)
+        if first_ci_ids:
+            result = get_cis_by_ids(first_ci_ids)
+        return numfound, total, result
+
+    def get_grandfather(self, ci_id, relation_type="contain"):
+        """only for CI Type
+        """
+        grandfather = db.session.query(CIRelation.first_ci_id).filter(
+            CIRelation.second_ci_id.in_(db.session.query(
+                CIRelation.first_ci_id).filter(
+                    CIRelation.second_ci_id == ci_id).filter(
+                        CIRelation.relation_type == relation_type))).filter(
+                            CIRelation.relation_type == relation_type).first()
+        if grandfather:
+            return CIManager().get_ci_by_id(grandfather.first_ci_id,
+                                            need_children=False)
+
+    def add(self, first_ci, second_ci, more=None, relation_type="contain"):
+        ci = db.session.query(CI.ci_id).filter(CI.ci_id == first_ci).first()
+        if ci is None:
+            return abort(404, "first_ci {0} is not existed".format(first_ci))
+        c = db.session.query(CI.ci_id).filter(CI.ci_id == second_ci).first()
+        if c is None:
+            return abort(404, "second_ci {0} is not existed".format(
+                second_ci))
+        existed = db.session.query(CIRelation.cr_id).filter(
+            CIRelation.first_ci_id == first_ci).filter(
+                CIRelation.second_ci_id == second_ci).first()
+        if existed is not None:
+            return existed.cr_id
+        cr = CIRelation()
+        cr.first_ci_id = first_ci
+        cr.second_ci_id = second_ci
+        if more is not None:
+            cr.more = more
+        cr.relation_type = relation_type
+        db.session.add(cr)
+        try:
+            db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error("add CIRelation is error, {0}".format(
+                str(e)))
+            return abort(500, "add CIRelation is error, {0}".format(str(e)))
+            # write history
+        his_manager = CIRelationHistoryManager()
+        his_manager.add(cr.cr_id, cr.first_ci_id, cr.second_ci_id,
+                        relation_type, operate_type="add")
+        return cr.cr_id
+
+    def delete(self, cr_id):
+        cr = db.session.query(CIRelation).filter(
+            CIRelation.cr_id == cr_id).first()
+        cr_id = cr.cr_id
+        first_ci = cr.first_ci_id
+        second_ci = cr.second_ci_id
+        if cr is not None:
+            db.session.delete(cr)
+            try:
+                db.session.commit()
+            except Exception as e:
+                db.session.rollback()
+                current_app.logger.error(
+                    "delete CIRelation is error, {0}".format(str(e)))
+                return abort(
+                    500, "delete CIRelation is error, {0}".format(str(e)))
+            his_manager = CIRelationHistoryManager()
+            his_manager.add(cr_id, first_ci, second_ci,
+                            cr.relation_type, operate_type="delete")
+            return True
+        return abort(404, "CI relation is not existed")
+
+    def delete_2(self, first_ci, second_ci):
+        cr = db.session.query(CIRelation).filter(
+            CIRelation.first_ci_id == first_ci).filter(
+                CIRelation.second_ci_id == second_ci).first()
+        return self.delete(cr.cr_id)
+
+
+class HostNumStatis(object):
+    def __init__(self):
+        pass
+
+    def get_hosts_by_project(self, project_id_list=None):
+        res = {}
+        if not project_id_list:
+            project = CITypeCache.get("project")
+            projects = db.session.query(CI.ci_id).filter(
+                CI.type_id == project.type_id).all()
+            project_id_list = (project.ci_id for project in projects)
+        project_id_list = map(str, project_id_list)
+        project_ids = ",".join(project_id_list)
+        nums = db.session.execute(QUERY_HOSTS_NUM_BY_PROJECT.format(
+            "".join(["(", project_ids, ")"]))).fetchall()
+        if nums:
+            for ci_id in project_id_list:
+                res[int(ci_id)] = 0
+            for ci_id, num in nums:
+                res[ci_id] = num
+        return res
+
+    def get_hosts_by_product(self, product_id_list=None):
+        res = {}
+        if not product_id_list:
+            product = CITypeCache.get("product")
+            products = db.session.query(CI.ci_id).filter(
+                CI.type_id == product.type_id).all()
+            product_id_list = (product.ci_id for product in products)
+        product_id_list = map(str, product_id_list)
+        product_ids = ",".join(product_id_list)
+        nums = db.session.execute(QUERY_HOSTS_NUM_BY_PRODUCT.format(
+            "".join(["(", product_ids, ")"]))).fetchall()
+        if nums:
+            for ci_id in product_id_list:
+                res[int(ci_id)] = 0
+            for ci_id, num in nums:
+                res[ci_id] = num
+        return res
+
+    def get_hosts_by_bu(self, bu_id_list=None):
+        res = {}
+        if not bu_id_list:
+            bu = CITypeCache.get("bu")
+            bus = db.session.query(CI.ci_id).filter(
+                CI.type_id == bu.type_id).all()
+            bu_id_list = (bu.ci_id for bu in bus)
+        bu_id_list = map(str, bu_id_list)
+        bu_ids = ",".join(bu_id_list)
+        current_app.logger.debug(QUERY_HOSTS_NUM_BY_BU.format(
+            "".join(["(", bu_ids, ")"])))
+        if not bu_ids:
+            return res
+        nums = db.session.execute(
+            QUERY_HOSTS_NUM_BY_BU.format(
+                "".join(["(", bu_ids, ")"]))).fetchall()
+        if nums:
+            for ci_id in bu_id_list:
+                res[int(ci_id)] = 0
+            for ci_id, num in nums:
+                res[ci_id] = num
+        return res
+
+
+def get_cis_by_ids(ci_ids, ret_key="name", fields="", value_tables=None):
+    """ argument ci_ids are string list of CI instance ID, eg. ['1', '2']
+    """
+    if not ci_ids:
+        return []
+    start = time.time()
+    ci_id_tuple = tuple(map(int, ci_ids))
+    res = rd.get(ci_id_tuple)
+    if res is not None and None not in res and ret_key == "name":
+        res = map(json.loads, res)
+        if not fields:
+            return res
+        else:
+            _res = []
+            for d in res:
+                _d = dict()
+                _d["_id"], _d["_type"] = d.get("_id"), d.get("_type")
+                _d["ci_type"] = d.get("ci_type")
+                for field in fields:
+                    _d[field] = d.get(field)
+                _res.append(_d)
+            current_app.logger.debug("filter time: %s" % (time.time() - start))
+            return _res
+    current_app.logger.warning("cache not hit...............")
+    if not fields:
+        _fields = ""
+    else:
+        _fields = list()
+        for field in fields:
+            attr = CIAttributeCache.get(field)
+            if attr is not None:
+                _fields.append(str(attr.attr_id))
+        _fields = "WHERE A.attr_id in ({0})".format(",".join(_fields))
+    ci_ids = ",".join(ci_ids)
+    if value_tables is None:
+        value_tables = type_map["table_name"].values()
+    current_app.logger.debug(value_tables)
+    value_sql = " UNION ".join([QUERY_CIS_BY_VALUE_TABLE.format(value_table,
+                                                                ci_ids)
+                                for value_table in value_tables])
+    query_sql = QUERY_CIS_BY_IDS.format(ci_ids, _fields, value_sql)
+    current_app.logger.debug(query_sql)
+    start = time.time()
+    hosts = db.session.execute(query_sql).fetchall()
+    current_app.logger.info("get cis time is: {0}".format(
+        time.time() - start))
+
+    ci_list = set()
+    res = list()
+    ci_dict = dict()
+    start = time.time()
+    for ci_id, type_id, attr_id, attr_name, \
+            attr_alias, value, value_type, is_multivalue in hosts:
+        if ci_id not in ci_list:
+            ci_dict = dict()
+            ci_type = CITypeSpecCache.get(type_id)
+            ci_dict["_id"] = ci_id
+            ci_dict["_type"] = type_id
+            ci_dict["ci_type"] = ci_type.type_name
+            ci_dict["ci_type_alias"] = ci_type.type_alias
+            ci_list.add(ci_id)
+            res.append(ci_dict)
+        if ret_key == "name":
+            if is_multivalue:
+                if isinstance(ci_dict.get(attr_name), list):
+                    ci_dict[attr_name].append(value)
+                else:
+                    ci_dict[attr_name] = [value]
+            else:
+                ci_dict[attr_name] = value
+        elif ret_key == "alias":
+            if is_multivalue:
+                if isinstance(ci_dict.get(attr_alias), list):
+                    ci_dict[attr_alias].append(value)
+                else:
+                    ci_dict[attr_alias] = [value]
+            else:
+                ci_dict[attr_alias] = value
+        elif ret_key == "id":
+            if is_multivalue:
+                if isinstance(ci_dict.get(attr_id), list):
+                    ci_dict[attr_id].append(value)
+                else:
+                    ci_dict[attr_id] = [value]
+            else:
+                ci_dict[attr_id] = value
+
+    current_app.logger.debug("result parser time is: {0}".format(
+        time.time() - start))
+    return res

lib/ci_type.py

@@ -0,0 +1,315 @@
+# -*- coding:utf-8 -*- 
+
+
+from flask import current_app
+from flask import abort
+
+from extensions import db
+from models import row2dict
+from models.ci_type import CITypeAttribute
+from models.ci_type import CIType
+from models.ci_type import CITypeAttributeCache
+from models.ci_type import CITypeCache
+from models.ci_type_relation import CITypeRelation
+from models.attribute import CIAttributeCache
+from lib.attribute import AttributeManager
+
+
+class CITypeAttributeManager(object):
+    """
+    manage CIType's attributes, include query, add, update, delete
+    """
+
+    def __init__(self):
+        pass
+
+    def get_attributes_by_type_id(self, type_id):
+        attrs = CITypeAttributeCache.get(type_id)
+        attr_manager = AttributeManager()
+        result = list()
+        for attr in attrs:
+            attr_dict = attr_manager.get_attribute_by_id(attr.attr_id)
+            attr_dict["is_required"] = attr.is_required
+            result.append(attr_dict)
+        return result
+
+    def add(self, type_id, attr_ids=None, is_required=False):
+        """
+        add attributes to CIType, attr_ids are list
+        """
+        if not attr_ids or not isinstance(attr_ids, list):
+            return abort(500, "attr_ids must be required")
+        ci_type = CITypeCache.get(type_id)
+        if ci_type is None:
+            return abort(404, "CIType ID({0}) is not existed".format(type_id))
+        for attr_id in attr_ids:
+            attr = CIAttributeCache.get(attr_id)
+            if attr is None:
+                return abort(404,
+                             "attribute id {0} is not existed".format(attr_id))
+            existed = db.session.query(CITypeAttribute.attr_id).filter_by(
+                type_id=type_id).filter_by(attr_id=attr_id).first()
+            if existed is not None:
+                continue
+            current_app.logger.debug(attr_id)
+            db.session.add(CITypeAttribute(
+                type_id=type_id, attr_id=attr_id, is_required=is_required))
+        try:
+            db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error(
+                "add attribute to CIType is error, {0}".format(str(e)))
+            return abort(
+                500, "add attribute to CIType is error, maybe duplicate entry")
+
+        CITypeAttributeCache.clean(type_id)
+        return True
+
+    def delete(self, type_id, attr_ids=None):
+        """
+        delete attributes at CIType, attr_ids are list
+        """
+        if not attr_ids or not isinstance(attr_ids, list):
+            return abort(
+                500, "delete attribute of CIType, attr_ids must be required")
+        ci_type = CITypeCache.get(type_id)
+        if ci_type is None:
+            return abort(
+                404, "CIType ID({0}) is not existed".format(type_id))
+        for attr_id in attr_ids:
+            attr = CIAttributeCache.get(attr_id)
+            if attr is None:
+                return abort(
+                    404, "attribute id {0} is not existed".format(attr_id))
+            db.session.query(CITypeAttribute).filter_by(
+                type_id=type_id).filter_by(attr_id=attr_id).delete()
+        try:
+            db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error(
+                "delete attributes of CIType is error, {0}".format(str(e)))
+            return abort(500, "delete attributes of CIType is error")
+        CITypeAttributeCache.clean(type_id)
+        return True
+
+
+class CITypeManager(object):
+    """
+    manage CIType
+    """
+
+    def __init__(self):
+        pass
+
+    def get_citypes(self, type_name=None):
+        ci_types = db.session.query(CIType).all() if type_name is None else \
+            db.session.query(CIType).filter(
+                CIType.type_name.ilike("%{0}%".format(type_name))).all()
+        res = list()
+        for ci_type in ci_types:
+            type_dict = row2dict(ci_type)
+            type_dict["uniq_key"] = CIAttributeCache.get(
+                type_dict["uniq_id"]).attr_name
+            res.append(type_dict)
+        return res
+
+    def query(self, _type):
+        citype = CITypeCache.get(_type)
+        if citype:
+            return row2dict(citype)
+        return abort(404, "citype is not found")
+
+    def add(self, type_name, type_alias, _id=None, unique=None,
+            icon_url="", enabled=True):
+        uniq_key = CIAttributeCache.get(_id) or CIAttributeCache.get(unique)
+        if uniq_key is None:
+            return False, "uniq_key is not existed"
+        citype = CITypeCache.get(type_name)
+        if citype:
+            return False, "this CIType {0} is existed".format(type_name)
+        _citype = CIType()
+        _citype.type_name = type_name
+        _citype.type_alias = type_alias
+        _citype.uniq_id = uniq_key.attr_id
+        _citype.enabled = enabled
+        _citype.icon_url = icon_url
+        db.session.add(_citype)
+        db.session.flush()
+        _citype_attr = CITypeAttribute()
+        _citype_attr.attr_id = uniq_key.attr_id
+        _citype_attr.type_id = _citype.type_id
+        _citype_attr.is_required = True
+        db.session.add(_citype_attr)
+        try:
+            db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error("add CIType is error, {0}".format(str(e)))
+            return False, str(e)
+        CITypeCache.clean(type_name)
+        return True, _citype.type_id
+
+    def update(self, type_id, type_name, type_alias, _id=None, unique=None,
+               icon_url="", enabled=None):
+        citype = CITypeCache.get(type_id)
+        if citype is None:
+            return False, "CIType {0} is not existed".format(type_name)
+        uniq_key = CIAttributeCache.get(_id) or CIAttributeCache.get(unique)
+        if uniq_key is not None:
+            citype.uniq_id = uniq_key.attr_id
+            citype_attr = db.session.query(CITypeAttribute).filter(
+                CITypeAttribute.type_id == type_id).filter(
+                    CITypeAttribute.attr_id == uniq_key.attr_id).first()
+            if citype_attr is None:
+                citype_attr = CITypeAttribute()
+                citype_attr.attr_id = uniq_key.attr_id
+                citype_attr.type_id = type_id
+            citype_attr.is_required = True
+            db.session.add(citype_attr)
+        if type_name:
+            citype.type_name = type_name
+        if type_alias:
+            citype.type_alias = type_alias
+        if icon_url:
+            citype.icon_url = icon_url
+        if enabled is not None:
+            citype.enabled = enabled
+        db.session.add(citype)
+        try:
+            db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error("add CIType is error, {0}".format(str(e)))
+            return False, str(e)
+        CITypeCache.clean(type_id)
+        return True, type_id
+
+    def set_enabled(self, type_id, enabled=True):
+        citype = CITypeCache.get(type_id)
+        if citype is None:
+            return abort(404, "CIType[{0}] is not existed".format(type_id))
+        citype.enabled = enabled
+        db.session.add(citype)
+        try:
+            db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error(
+                "set CIType enabled is error, {0}".format(str(e)))
+            return abort(500, str(e))
+        return type_id
+
+    def delete(self, type_id):
+        citype = db.session.query(CIType).filter_by(type_id=type_id).first()
+        type_name = citype.type_name
+        if citype:
+            db.session.delete(citype)
+            try:
+                db.session.commit()
+            except Exception as e:
+                db.session.rollback()
+                current_app.logger.error(
+                    "delete CIType is error, {0}".format(str(e)))
+                return abort(500, str(e))
+            CITypeCache.clean(type_id)
+            return "CIType {0} deleted".format(type_name)
+        return abort(404, "CIType is not existed")
+
+
+class CITypeRelationManager(object):
+    """
+    manage relation between CITypes
+    """
+
+    def __init__(self):
+        pass
+
+    @property
+    def relation_types(self):
+        """ all CIType relation types
+        """
+        from lib.const import CITYPE_RELATION_TYPES
+
+        return CITYPE_RELATION_TYPES
+
+    def get_children(self, parent_id):
+        children = db.session.query(CITypeRelation).filter(
+            CITypeRelation.parent_id == parent_id).all()
+        result = []
+        for child in children:
+            ctr_id = child.ctr_id
+            citype = CITypeCache.get(child.child_id)
+            citype_dict = row2dict(citype)
+            citype_dict["ctr_id"] = ctr_id
+            manager = CITypeAttributeManager()
+            citype_dict["attributes"] = manager.get_attributes_by_type_id(
+                citype.type_id)
+            citype_dict["relation_type"] = child.relation_type
+            result.append(citype_dict)
+        return result
+
+    def get_parents(self, child_id):
+        parents = db.session.query(CITypeRelation).filter(
+            CITypeRelation.child_id == child_id).all()
+        result = []
+        for parent in parents:
+            ctr_id = parent.ctr_id
+            citype = CITypeCache.get(parent.parent_id)
+            citype_dict = row2dict(citype)
+            citype_dict["ctr_id"] = ctr_id
+            manager = CITypeAttributeManager()
+            citype_dict["attributes"] = manager.get_attributes_by_type_id(
+                citype.type_id)
+            citype_dict["relation_type"] = parent.relation_type
+            result.append(citype_dict)
+        return result
+
+    def add(self, parent, child, relation_type="contain"):
+        p = CITypeCache.get(parent)
+        if p is None:
+            return abort(404, "parent {0} is not existed".format(parent))
+        c = CITypeCache.get(child)
+        if c is None:
+            return abort(404, "child {0} is not existed".format(child))
+        existed = db.session.query(CITypeRelation.ctr_id).filter_by(
+            parent_id=parent).filter_by(child_id=child).first()
+        if existed is not None:
+            return True, existed.ctr_id
+        ctr = CITypeRelation()
+        ctr.parent_id = parent
+        ctr.child_id = child
+        ctr.relation_type = relation_type
+        db.session.add(ctr)
+        try:
+            db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error(
+                "add CITypeRelation is error, {0}".format(str(e)))
+            return abort(
+                500, "add CITypeRelation is error, {0}".format(str(e)))
+        return ctr.ctr_id
+
+    def delete(self, ctr_id):
+        ctr = db.session.query(CITypeRelation).filter(
+            CITypeRelation.ctr_id == ctr_id).first()
+        if ctr:
+            db.session.delete(ctr)
+            try:
+                db.session.commit()
+            except Exception as e:
+                db.session.rollback()
+                current_app.logger.error(
+                    "delete CITypeRelation is error, {0}".format(str(e)))
+                return abort(
+                    500, "delete CITypeRelation is error, {0}".format(str(e)))
+            return True
+        return abort(404, "CIType relation is not existed")
+
+    def delete_2(self, parent, child):
+        ctr = db.session.query(CITypeRelation).filter(
+            CITypeRelation.parent_id == parent).filter(
+                CITypeRelation.child_id == child).first()
+        return self.delete(ctr.ctr_id)

lib/const.py

@@ -0,0 +1,99 @@
+# -*- coding:utf-8 -*- 
+
+
+import datetime
+
+from models.attribute import TextChoice
+from models.attribute import FloatChoice
+from models.attribute import IntegerChoice
+from models.attribute import CIAttributeCache
+from models.ci_value import CIValueText
+from models.ci_value import CIValueInteger
+from models.ci_value import CIValueFloat
+from models.ci_value import CIValueDateTime
+from models.ci_value import CIIndexValueDateTime
+from models.ci_value import CIIndexValueFloat
+from models.ci_value import CIIndexValueInteger
+from models.ci_value import CIIndexValueText
+
+
+def string2int(x):
+    return int(float(x))
+
+
+def str2datetime(x):
+    try:
+        v = datetime.datetime.strptime(x, "%Y-%m-%d")
+        return v
+    except ValueError:
+        pass
+    try:
+        v = datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")
+        return v
+    except ValueError:
+        pass
+    raise ValueError
+
+
+type_map = {
+    'converter': {
+        'int': string2int,
+        'float': float,
+        'text': unicode,
+        'datetime': str2datetime,
+    },
+    'choice': {
+        'int': IntegerChoice,
+        'float': FloatChoice,
+        'text': TextChoice,
+    },
+    'table': {
+        'int': CIValueInteger,
+        'text': CIValueText,
+        'datetime': CIValueDateTime,
+        'float': CIValueFloat,
+        'index_int': CIIndexValueInteger,
+        'index_text': CIIndexValueText,
+        'index_datetime': CIIndexValueDateTime,
+        'index_float': CIIndexValueFloat,
+    },
+    'table_name': {
+        'int': 'integers',
+        'text': 'texts',
+        'datetime': 'datetime',
+        'float': 'floats',
+        'index_int': 'index_integers',
+        'index_text': 'index_texts',
+        'index_datetime': 'index_datetime',
+        'index_float': 'index_floats',
+    }
+}
+
+
+class TableMap():
+    def __init__(self, attr_name=None):
+        self.attr_name = attr_name
+
+    @property
+    def table(self):
+        if self.attr_name is not None:
+            attr = CIAttributeCache.get(self.attr_name)
+            if attr.is_index:
+                i = "index_{0}".format(attr.value_type)
+            else:
+                i = attr.value_type
+            return type_map["table"].get(i)
+
+    @property
+    def table_name(self):
+        if self.attr_name is not None:
+            attr = CIAttributeCache.get(self.attr_name)
+            if attr.is_index:
+                i = "index_{0}".format(attr.value_type)
+            else:
+                i = attr.value_type
+            return type_map["table_name"].get(i)
+
+
+CITYPE_RELATION_TYPES = ["connect", "deploy", "install", "contain"]
+CI_RELATION_TYPES = ["connect", "deploy", "install", "contain"]

lib/decorator.py

@@ -0,0 +1,74 @@
+# -*- coding:utf-8 -*- 
+
+
+import time
+from functools import wraps
+
+from flask import request
+from flask import render_template
+from flask import current_app
+
+from lib.exception import InvalidUsageError
+
+
+def templated(template=None):
+    def decorator(f):
+        @wraps(f)
+        def decorated_function(*args, **kwargs):
+            template_name = template
+            if template_name is None:
+                template_name = request.endpoint.replace('.', '/') + '.html'
+            ctx = f(*args, **kwargs)
+            if ctx is None:
+                ctx = {}
+            elif not isinstance(ctx, dict):
+                return ctx
+            return render_template(template_name, **ctx)
+
+        return decorated_function
+
+    return decorator
+
+
+def argument_required1(*args_required):
+    from manage import InvalidUsageError
+
+    def decorator(f):
+        @wraps(f)
+        def decorated_function(*args, **kwargs):
+            for arg in args_required:
+                if request.values.get(arg, None) is None:
+                    raise InvalidUsageError(
+                        "argument {0} is required".format(arg), 400)
+            return f(*args, **kwargs)
+
+        return decorated_function
+
+    return decorator
+
+
+class argument_required(object):
+    def __init__(self, *args):
+        self.args = args
+
+    def __enter__(self):
+        for arg in self.args:
+            if not request.values.get(arg):
+                raise InvalidUsageError(
+                    "argument {0} is required".format(arg), status_code=400)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+
+def url_statistic(f):
+    @wraps(f)
+    def decorated_func(*args, **kwargs):
+        start = time.time()
+        r = f(*args, **kwargs)
+        spend = time.time() - start
+        url = request.path
+        current_app.logger.info(url)
+        current_app.logger.info(spend)
+        return r
+    return decorated_func

lib/exception.py

@@ -0,0 +1,17 @@
+# -*- coding:utf-8 -*- 
+
+
+class InvalidUsageError(Exception):
+    status_code = 400
+
+    def __init__(self, message, status_code=None, payload=None):
+        Exception.__init__(self)
+        self.message = message
+        if status_code is not None:
+            self.status_code = status_code
+        self.payload = payload
+
+    def to_dict(self):
+        rv = dict(self.payload or ())
+        rv['message'] = self.message
+        return rv

lib/history.py

@@ -0,0 +1,75 @@
+# -*- coding:utf-8 -*- 
+
+
+import datetime
+
+from flask import current_app
+from flask import g
+
+from extensions import db
+from models.history import OperationRecord
+from models.history import CIAttributeHistory
+from models.history import CIRelationHistory
+
+
+class CIAttributeHistoryManger(object):
+    def __init__(self):
+        pass
+
+    def add(self, ci_id, history_list):
+        if history_list:
+            record = OperationRecord()
+            record.uid = g.user.uid
+            record.timestamp = datetime.datetime.now()
+            db.session.add(record)
+            db.session.commit()
+            for attr_id, operate_type, old, new in history_list:
+                history = CIAttributeHistory()
+                history.attr_id = attr_id
+                history.operate_type = operate_type
+                history.old = old
+                history.new = new
+                history.ci_id = ci_id
+                history.record_id = record.record_id
+                db.session.add(history)
+
+            try:
+                db.session.commit()
+            except Exception as e:
+                db.session.rollback()
+                db.session.rollback()
+                current_app.logger.error(
+                    "add attribute history error, {0}".format(str(e)))
+                return False, "add attribute history error, {0}".format(str(e))
+        return True, None
+
+
+class CIRelationHistoryManager(object):
+    def __init__(self):
+        pass
+
+    def add(self, relation, first_ci, second_ci,
+            relation_type, operate_type="add"):
+        record = OperationRecord()
+        record.uid = g.user.uid
+        record.timestamp = datetime.datetime.now()
+        db.session.add(record)
+        db.session.flush()
+
+        history = CIRelationHistory()
+        history.relation = relation
+        history.record_id = record.record_id
+        history.operate_type = operate_type
+        history.first_ci_id = first_ci
+        history.second_ci_id = second_ci
+        history.relation_type = relation_type
+        db.session.add(history)
+
+        try:
+            db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error(
+                "add relation history is error, {0}".format(str(e)))
+            return False, "add relation history is error, {0}".format(str(e))
+        return True, None

lib/mail.py

@@ -0,0 +1,86 @@
+# -*- coding:utf-8 -*- 
+
+
+import requests
+
+from flask import current_app
+from flask.ext.mail import Message
+
+from extensions import mail
+from models.account import User
+
+
+def sendmail(users, subject, message, html=False, app=None):
+    if app:
+        mail.app = app
+    else:
+        app = current_app
+    recipients = [x.email for x in users if isinstance(x, User)]
+    recipients.extend(
+        [x for x in users if isinstance(x, basestring) and '@' in x])
+    sender = app.config.get('DEFAULT_MAIL_SENDER')
+    if html:
+        msg = Message(recipients=recipients,
+                      html=message,
+                      subject=subject,
+                      sender=sender)
+    else:
+        msg = Message(recipients=recipients,
+                      body=message,
+                      subject=subject,
+                      sender=sender)
+    mail.send(msg)
+
+
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from email.header import Header
+from email.mime.image import MIMEImage
+import smtplib
+import time
+from email import Utils
+
+
+def send_mail(sender, receiver, subject, content, ctype="html", pics=(),
+              smtpserver='mail.51ping.com',
+              username="networkbench@51ping.com", password="12qwaszx"):
+    """subject and body are unicode objects"""
+    if ctype == "html":
+        msg = MIMEText(content, 'html', 'utf-8')
+    else:
+        msg = MIMEText(content, 'plain', 'utf-8')
+
+    if len(pics) != 0:
+        msgRoot = MIMEMultipart('related')
+        msgText = MIMEText(content, 'html', 'utf-8')
+        msgRoot.attach(msgText)
+        i = 1
+        for pic in pics:
+            fp = open(pic, "rb")
+            image = MIMEImage(fp.read())
+            fp.close()
+            image.add_header('Content-ID', '<img%02d>' % i)
+            msgRoot.attach(image)
+            i += 1
+        msg = msgRoot
+
+    msg['Subject'] = Header(subject, 'utf-8')
+    msg['From'] = sender
+    msg['To'] = ';'.join(receiver)
+    msg['Message-ID'] = Utils.make_msgid()
+    msg['date'] = time.strftime('%a, %d %b %Y %H:%M:%S %z')
+
+    smtp = smtplib.SMTP()
+    smtp.connect(smtpserver, 25)
+    smtp.login(username, password)
+    smtp.sendmail(sender, receiver, msg.as_string())
+    smtp.quit()
+
+
+def send_sms(mobile, content):
+    sms_uri = current_app.config.get("SMS_URI") % (mobile, content)
+    try:
+        current_app.logger.info(sms_uri)
+        requests.get(sms_uri)
+    except Exception as e:
+        current_app.logger.error("send sms error, %s" % str(e))

lib/query_sql.py

@@ -0,0 +1,107 @@
+# -*- coding:utf-8 -*- 
+
+
+QUERY_HOSTS_BY_APP = """
+    SELECT *
+    FROM cis
+    INNER JOIN ci_relations AS cr ON cis.`ci_id`=cr.`second_ci`
+    WHERE cr.`first_ci` = {0:d} LIMIT {1:d}, {2:d};
+"""
+
+QUERY_HOSTS_NUM_BY_PROJECT = """
+    SELECT cr.first_ci_id,
+          count(DISTINCT cr.second_ci_id)
+    FROM ci_relations AS cr
+    WHERE cr.first_ci_id IN {0}
+    GROUP BY cr.first_ci_id
+"""
+
+QUERY_HOSTS_NUM_BY_BU = """
+    SELECT B.first_ci_id,
+           count(DISTINCT cr.second_ci_id)
+    FROM
+      (SELECT A.first_ci_id,
+              cr.second_ci_id
+       FROM
+         (SELECT cr.first_ci_id,
+                 cis.ci_id
+          FROM cis
+          INNER JOIN ci_relations AS cr ON cis.ci_id=cr.second_ci_id
+          WHERE cr.first_ci_id IN {0}) AS A
+       INNER JOIN ci_relations AS cr ON cr.first_ci_id=A.ci_id) AS B
+    INNER JOIN ci_relations AS cr ON B.second_ci_id=cr.first_ci_id
+    GROUP BY B.first_ci_id
+"""
+
+QUERY_HOSTS_NUM_BY_PRODUCT = """
+    SELECT A.first_ci_id,
+           count(DISTINCT cr.second_ci_id)
+    FROM
+      (SELECT cr.first_ci_id,
+              cis.ci_id
+       FROM cis
+       INNER JOIN ci_relations AS cr ON cis.ci_id=cr.second_ci_id
+       WHERE cr.first_ci_id IN {0}) AS A
+    INNER JOIN ci_relations AS cr ON cr.first_ci_id=A.ci_id
+    GROUP BY A.first_ci_id;
+"""
+
+QUERY_CIS_BY_VALUE_TABLE = """
+    SELECT attr.attr_name,
+                  attr.attr_alias,
+                  attr.value_type,
+                  attr.is_multivalue,
+                  cis.type_id,
+                  {0}.ci_id,
+                  {0}.attr_id,
+                  {0}.value
+           FROM {0}
+           INNER JOIN cis ON {0}.ci_id=cis.ci_id
+           AND {0}.`ci_id` IN ({1})
+           INNER JOIN ci_attributes as attr ON attr.attr_id = {0}.attr_id
+"""
+
+QUERY_CIS_BY_IDS = """
+    SELECT A.ci_id,
+           A.type_id,
+           A.attr_id,
+           A.attr_name,
+           A.attr_alias,
+           A.value,
+           A.value_type,
+           A.is_multivalue
+    FROM
+      ({2}) AS A {1}
+       ORDER BY A.ci_id;
+"""
+
+FACET_QUERY1 = """
+    SELECT {0}.value,
+           count({0}.ci_id)
+    FROM {0}
+    INNER JOIN ci_attributes AS attr ON attr.attr_id={0}.attr_id
+    WHERE attr.attr_name="{1}"
+    GROUP BY {0}.ci_id;
+"""
+
+FACET_QUERY = """
+    SELECT {0}.value,
+           count({0}.ci_id)
+    FROM {0}
+    INNER JOIN ({1}) AS B ON B.ci_id={0}.ci_id
+    WHERE {0}.attr_id={2:d}
+    GROUP BY {0}.ci_id
+"""
+
+QUERY_CI_BY_ATTR_NAME = """
+    SELECT {0}.ci_id
+    FROM {0}
+    WHERE {0}.attr_id={1:d}
+      AND {0}.value {2}
+"""
+
+QUERY_CI_BY_TYPE = """
+    SELECT cis.ci_id
+    FROM cis
+    WHERE cis.type_id in ({0})
+"""

lib/search.py

@@ -0,0 +1,348 @@
+# -*- coding:utf-8 -*- 
+
+
+import time
+
+from flask import current_app
+
+from lib.const import TableMap
+from models.attribute import CIAttributeCache
+from models.ci_type import CITypeCache
+from extensions import db
+from models import CI
+from lib.ci import get_cis_by_ids
+from lib.query_sql import FACET_QUERY
+from lib.query_sql import QUERY_CI_BY_TYPE
+from lib.query_sql import QUERY_CI_BY_ATTR_NAME
+
+
+class SearchError(Exception):
+    def __init__(self, v):
+        self.v = v
+
+    def __str__(self):
+        return self.v
+
+
+class Search(object):
+    def __init__(self, query=None, fl=None, facet_field=None,
+                 page=1, ret_key="name", count=1, sort=None):
+        self.orig_query = query
+        self.fl = fl
+        self.facet_field = facet_field
+        self.page = page
+        self.ret_key = ret_key
+        try:
+            self.count = int(count)
+        except ValueError:
+            self.count = current_app.config.get("DEFAULT_PAGE_COUNT")
+        self.sort = sort
+        self.query_sql = ""
+        self.type_id_list = []
+
+    def tor_proc(self, key):
+        tor = list()
+        if key.startswith("+"):
+            tor.append('&')
+            key = key[1:].strip()
+        elif key.startswith("-"):
+            tor.append('|')
+            key = key[1:].strip()
+        elif key.startswith("~"):
+            tor.append('~')
+            key = key[1:].strip()
+        if not tor:
+            tor = ['&', '']
+        if len(tor) < 2:
+            tor.append('')
+        return tor, key
+
+    def attr_name_proc(self, key):
+        tor, key = self.tor_proc(key)
+        if key in ('ci_type', 'type', '_type'):
+            return '_type', 'text', tor, None
+        if key in ('id', 'ci_id', '_id'):
+            return '_id', 'text', tor, None
+        attr = CIAttributeCache.get(key)
+        if attr is not None:
+            # if not attr.is_index:
+            #     raise SearchError("{0} is not indexed".format(attr.attr_name))
+            field_name = attr.attr_name
+            return field_name, attr.value_type, tor, attr
+        else:
+            raise SearchError("{0} is not existed".format(key))
+
+    def type_query_handler(self, v, only_type_query):
+        new_v = [v]
+        if v.startswith("(") and v.endswith(")"):
+            new_v = v[1:-1].split(";")
+        for _v in new_v:
+            ci_type = CITypeCache.get(_v)
+            if ci_type is not None:
+                self.type_id_list.append(str(ci_type.type_id))
+        if self.type_id_list:
+            type_ids = ",".join(self.type_id_list)
+            _query_sql = QUERY_CI_BY_TYPE.format(type_ids)
+            if only_type_query:
+                return _query_sql
+            else:
+                return ""
+        return ""
+
+    def in_query_handler(self, attr, v):
+        new_v = v[1:-1].split(";")
+        table_name = TableMap(attr_name=attr.attr_name).table_name
+        _query_sql = QUERY_CI_BY_ATTR_NAME.format(
+            table_name, attr.attr_id,
+            " OR {0}.value ".format(table_name).join(['LIKE "{0}"'.format(
+                _v.replace("*", "%")) for _v in new_v]))
+        return _query_sql
+
+    def range_query_handler(self, attr, v):
+        start, end = [x.strip() for x in v[1:-1].split("_TO_")]
+        table_name = TableMap(attr_name=attr.attr_name).table_name
+        _query_sql = QUERY_CI_BY_ATTR_NAME.format(
+            table_name, attr.attr_id, "BETWEEN '{0}' AND '{1}'".format(
+                start.replace("*", "%"), end.replace("*", "%")))
+        return _query_sql
+
+    def comparison_query_handler(self, attr, v):
+        table_name = TableMap(attr_name=attr.attr_name).table_name
+        if (v.startswith("<") and not v.startswith("<=")) or \
+                (v.startswith(">") and not v.startswith(">=")):
+            _query_sql = QUERY_CI_BY_ATTR_NAME.format(
+                table_name, attr.attr_id, "{0} '{1}'".format(
+                    v[0], v[1:].replace("*", "%")))
+        elif v.startswith(">=") or v.startswith("<="):
+            _query_sql = QUERY_CI_BY_ATTR_NAME.format(
+                table_name, attr.attr_id, "{0} '{1}'".format(
+                    v[:2], v[2:].replace("*", "%")))
+        return _query_sql
+
+    def sort_query_handler(self, field, query_sql, only_type_query):
+        if field is None:
+            field = ""
+        if field.startswith("+"):
+            field = field[1:]
+            sort_type = "ASC"
+        elif field.startswith("-"):
+            field = field[1:]
+            sort_type = "DESC"
+        else:
+            sort_type = "ASC"
+
+        if field in ("_id", "ci_id") or not field:
+            if only_type_query:
+                return """SELECT SQL_CALC_FOUND_ROWS DISTINCT B.ci_id
+                FROM ({0}) AS B {1}""".format(
+                    query_sql,
+                    "ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format(
+                        (self.page - 1) * self.count, sort_type, self.count))
+            elif self.type_id_list:
+                return """SELECT SQL_CALC_FOUND_ROWS DISTINCT B.ci_id
+                FROM ({0}) AS B {1}""".format(
+                    query_sql,
+                    "INNER JOIN cis on cis.ci_id=B.ci_id "
+                    "WHERE cis.type_id in ({3}) "
+                    "ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format(
+                        (self.page - 1) * self.count, sort_type, self.count,
+                        ",".join(self.type_id_list)))
+            else:
+                return """SELECT SQL_CALC_FOUND_ROWS DISTINCT B.ci_id
+                FROM ({0}) AS B {1}""".format(
+                    query_sql,
+                    "INNER JOIN cis on cis.ci_id=B.ci_id "
+                    "ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format(
+                        (self.page - 1) * self.count, sort_type, self.count))
+        else:
+            attr = CIAttributeCache.get(field)
+            attr_id = attr.attr_id
+
+            table_name = TableMap(attr_name=attr.attr_name).table_name
+            _v_query_sql = """SELECT {0}.ci_id, {1}.value FROM
+                ({2}) AS {0} INNER JOIN {1} ON {1}.ci_id = {0}.ci_id
+                WHERE {1}.attr_id = {3}""".format("ALIAS", table_name,
+                                                  query_sql, attr_id)
+            new_table = _v_query_sql
+            if only_type_query:
+                return "SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id " \
+                       "FROM ({0}) AS C " \
+                       "ORDER BY C.value {2} " \
+                       "LIMIT {1:d}, {3};".format(new_table,
+                                                  (self.page - 1) * self.count,
+                                                  sort_type, self.count)
+            elif self.type_id_list:
+                return """SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id
+                    FROM ({0}) AS C
+                    INNER JOIN cis on cis.ci_id=C.ci_id
+                    WHERE cis.type_id in ({4})
+                    ORDER BY C.value {2}
+                    LIMIT {1:d}, {3};""".format(new_table,
+                                                (self.page - 1) * self.count,
+                                                sort_type, self.count,
+                                                ",".join(self.type_id_list))
+            else:
+                return """SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id
+                    FROM ({0}) AS C
+                    ORDER BY C.value {2}
+                    LIMIT {1:d}, {3};""".format(new_table,
+                                                (self.page - 1) * self.count,
+                                                sort_type, self.count)
+
+    def _wrap_sql(self, tor, alias, _query_sql, query_sql):
+        if tor[0] == "&":
+            query_sql = """SELECT * FROM ({0}) as {1}
+                INNER JOIN ({2}) as {3} USING(ci_id)""".format(
+                query_sql, alias, _query_sql, alias + "A")
+        elif tor[0] == "|":
+            query_sql = "SELECT * FROM ({0}) as {1} UNION ALL ({2})".format(
+                query_sql, alias, _query_sql)
+        elif tor[0] == "~":
+            query_sql = "SELECT * FROM ({0}) as {1} LEFT JOIN ({2}) as {3} " \
+                        "USING(ci_id) WHERE {3}.ci_id is NULL".format(
+                        query_sql, alias, _query_sql, alias + "A")
+        return query_sql
+
+    def _execute_sql(self, query_sql, only_type_query):
+        v_query_sql = self.sort_query_handler(self.sort, query_sql,
+                                              only_type_query)
+        start = time.time()
+        execute = db.session.execute
+        current_app.logger.debug(v_query_sql)
+        res = execute(v_query_sql).fetchall()
+        end_time = time.time()
+        current_app.logger.debug("query ci ids time is: {0}".format(
+            end_time - start))
+        numfound = execute("SELECT FOUND_ROWS();").fetchall()[0][0]
+        current_app.logger.debug("statistics ci ids time is: {0}".format(
+            time.time() - end_time)
+        )
+        return numfound, res
+
+    def query_build_raw(self):
+        query_sql, alias, tor = "", "A", ["&"]
+        is_first = True
+        only_type_query = False
+        queries = self.orig_query.split(",")
+        queries = filter(lambda x: x != "", queries)
+        for q in queries:
+            if q.startswith("_type"):
+                queries.remove(q)
+                queries.insert(0, q)
+                if len(queries) == 1 or queries[1].startswith("-") or \
+                        queries[1].startswith("~"):
+                    only_type_query = True
+                break
+        current_app.logger.debug(queries)
+        special = True
+        for q in queries:
+            _query_sql = ""
+            if ":" in q:
+                k = q.split(":")[0].strip()
+                v = ":".join(q.split(":")[1:]).strip()
+                current_app.logger.info(v)
+                field, field_type, tor, attr = self.attr_name_proc(k)
+                if field == "_type":
+                    _query_sql = self.type_query_handler(v, only_type_query)
+                    current_app.logger.debug(_query_sql)
+                elif field == "_id":  # exclude all others
+                    _ci_ids = [str(v)]
+                    ci = db.session.query(CI.ci_id).filter(
+                        CI.ci_id == int(v)).first()
+                    if ci is not None:
+                        return 1, _ci_ids
+                elif field:
+                    if attr is None:
+                        raise SearchError("{0} is not found".format(field))
+                    # in query
+                    if v.startswith("(") and v.endswith(")"):
+                        _query_sql = self.in_query_handler(attr, v)
+                    # range query
+                    elif v.startswith("[") and v.endswith("]") and "_TO_" in v:
+                        _query_sql = self.range_query_handler(attr, v)
+                    # comparison query
+                    elif v.startswith(">=") or v.startswith("<=") or \
+                            v.startswith(">") or v.startswith("<"):
+                        _query_sql = self.comparison_query_handler(attr, v)
+                    else:
+                        table_name = \
+                            TableMap(attr_name=attr.attr_name).table_name
+                        _query_sql = QUERY_CI_BY_ATTR_NAME.format(
+                            table_name, attr.attr_id,
+                            'LIKE "{0}"'.format(v.replace("*", "%")))
+                else:
+                    return 0, []
+            elif q:
+                return 0, []
+
+            if is_first and _query_sql and not only_type_query:
+                query_sql = "SELECT * FROM ({0}) AS {1}".format(_query_sql,
+                                                                alias)
+                is_first = False
+                alias += "A"
+            elif only_type_query and special:
+                is_first = False
+                special = False
+                query_sql = _query_sql
+            elif _query_sql:
+                query_sql = self._wrap_sql(tor, alias, _query_sql, query_sql)
+                alias += "AA"
+
+        _start = time.time()
+        if query_sql:
+            self.query_sql = query_sql
+            current_app.logger.debug(query_sql)
+            numfound, res = self._execute_sql(query_sql, only_type_query)
+            current_app.logger.info("query ci ids is: {0}".format(
+                time.time() - _start))
+            return numfound, [_res[0] for _res in res]
+        return 0, []
+
+    def facet_build(self):
+        facet = {}
+        for f in self.facet_field:
+            k, field_type, _, attr = self.attr_name_proc(f)
+            if k:
+                table_name = TableMap(attr_name=k).table_name
+                query_sql = FACET_QUERY.format(
+                    table_name, self.query_sql, attr.attr_id)
+                result = db.session.execute(query_sql).fetchall()
+                facet[k] = result
+        facet_result = dict()
+        for k, v in facet.items():
+            if not k.startswith('_'):
+                a = getattr(CIAttributeCache.get(k), "attr_%s" % self.ret_key)
+                facet_result[a] = list()
+                for f in v:
+                    if f[1] != 0:
+                        facet_result[a].append((f[0], f[1], a))
+        return facet_result
+
+    def fl_build(self):
+        _fl = list()
+        for f in self.fl:
+            k, _, _, _ = self.attr_name_proc(f)
+            if k:
+                _fl.append(k)
+        return _fl
+
+    def search(self):
+        numfound, ci_ids = self.query_build_raw()
+        ci_ids = map(str, ci_ids)
+        _fl = self.fl_build()
+
+        if self.facet_field and numfound:
+            facet = self.facet_build()
+        else:
+            facet = dict()
+
+        response, counter = [], {}
+        if ci_ids:
+            response = get_cis_by_ids(ci_ids, ret_key=self.ret_key, fields=_fl)
+        for res in response:
+            ci_type = res.get("ci_type")
+            if ci_type not in counter.keys():
+                counter[ci_type] = 0
+            counter[ci_type] += 1
+        total = len(response)
+        return response, counter, total, self.page, numfound, facet

lib/template/__init__.py

@@ -0,0 +1 @@
+# -*- coding:utf-8 -*-

lib/template/filters.py

@@ -0,0 +1,9 @@
+# -*- coding:utf-8 -*- 
+
+
+def convert_to_list(v):
+    if isinstance(v, list):
+        return v
+    if isinstance(v, tuple):
+        return list(v)
+    return [v, ]

lib/utils.py

@@ -0,0 +1,74 @@
+# -*- coding:utf-8 -*-
+
+
+import redis
+
+from flask import current_app
+import settings
+
+
+class RedisHandler(object):
+    def __init__(self):
+        try:
+            pool = redis.ConnectionPool(
+                max_connections=settings.REDIS_MAX_CONN,
+                host=settings.REDIS_HOST,
+                port=settings.REDIS_PORT,
+                db=settings.REDIS_DB)
+            self.r = redis.Redis(connection_pool=pool)
+        except Exception as e:
+            print e
+            current_app.logger.error("init redis connection failed")
+
+    @classmethod
+    def instance(cls):
+        if not hasattr(cls, "_instance"):
+            cls._instance = cls()
+        return cls._instance
+
+    def get(self, ci_ids, key="CMDB_CI"):
+        try:
+            value = self.r.hmget(key, ci_ids)
+        except Exception as e:
+            current_app.logger.error("get redis error, %s" % str(e))
+            return
+        return value
+
+    def _set(self, ci, key="CMDB_CI"):
+        try:
+            self.r.hmset(key, ci)
+        except Exception as e:
+            current_app.logger.error("set redis error, %s" % str(e))
+
+    def add(self, ci):
+        self._set(ci)
+
+    def delete(self, ci_id, key="CMDB_CI"):
+        try:
+            ret = self.r.hdel(key, ci_id)
+            if not ret:
+                current_app.logger.warn("ci [%d] is not in redis" % ci_id)
+        except Exception as e:
+            current_app.logger.error("delete redis key error, %s" % str(e))
+
+rd = RedisHandler.instance()
+
+
+def get_page(page):
+    try:
+        page = int(page)
+    except ValueError:
+        page = 1
+    if page < 1:
+        page = 1
+    return page
+
+
+def get_per_page(per_page):
+    try:
+        per_page = int(per_page)
+    except:
+        per_page = current_app.config.get("DEFAULT_PAGE_COUNT")
+    if per_page < 1:
+        per_page = current_app.config.get("DEFAULT_PAGE_COUNT")
+    return per_page

lib/value.py

@@ -0,0 +1,170 @@
+# -*- coding:utf-8 -*- 
+
+
+import datetime
+
+from flask import current_app
+
+from extensions import db
+from models.attribute import CIAttributeCache
+from lib.attribute import AttributeManager
+from lib.const import type_map
+from lib.const import TableMap
+
+
+class AttributeValueManager(object):
+    """
+    manage CI attribute values
+    """
+
+    def __init__(self):
+        pass
+
+    def _get_attr(self, key):
+        """key is one of attr_id, attr_name and attr_alias
+        """
+        attr = CIAttributeCache.get(key)
+        return attr
+
+    def _get_attr_values(self, fields, ci_id,
+                         ret_key="name",
+                         uniq_key=None,
+                         use_master=False):
+        res = dict()
+        for field in fields:
+            attr = CIAttributeCache.get(field)
+            if not attr:
+                current_app.logger.warn('attribute %s not found' % field)
+                return res
+            table = TableMap(attr_name=attr.attr_name).table
+            if use_master:
+                rs = db.session().using_bind("master").query(
+                    table.value).filter_by(ci_id=ci_id).filter_by(
+                        attr_id=attr.attr_id)
+            else:
+                rs = db.session.query(table.value).filter_by(
+                    ci_id=ci_id).filter_by(attr_id=attr.attr_id)
+            field_name = getattr(attr, "attr_{0}".format(ret_key))
+            try:
+                if attr.is_multivalue:
+                    if attr.value_type == 'datetime':
+                        res[field_name] = [datetime.datetime.strftime(
+                            x.value, '%Y-%m-%d %H:%M:%S') for x in rs.all()]
+                    else:
+                        res[field_name] = [x.value for x in rs.all()]
+                else:
+                    x = rs.first()
+                    if x:
+                        if attr.value_type == 'datetime':
+                            res[field_name] = datetime.datetime.strftime(
+                                rs.first().value, '%Y-%m-%d %H:%M:%S')
+                        else:
+                            res[field_name] = rs.first().value
+                    else:
+                        res[field_name] = None
+            except AttributeError as e:
+                current_app.logger.warn("get ci by id error, {0}".format(e))
+                if attr.is_multivalue:
+                    res[field_name] = list()
+                else:
+                    res[field_name] = ""
+            if uniq_key is not None and attr.attr_id == uniq_key.attr_id \
+                    and rs.first() is not None:
+                res['unique'] = uniq_key.attr_name
+        return res
+
+    def _validate(self, attr, value, table, ci_id):
+        converter = type_map.get("converter").get(attr.value_type)
+        try:
+            v = converter(value)
+        except ValueError:
+            return False, "attribute value {0} converter fail".format(value)
+        if attr.is_choice:
+            choice_list = AttributeManager()._get_choice_value(
+                attr.attr_id, attr.value_type)
+            if v not in choice_list:
+                return False, "{0} is not existed in choice values".format(
+                    value)
+        elif attr.is_uniq:
+            old_value = db.session.query(table.attr_id).filter(
+                table.attr_id == attr.attr_id).filter(
+                    table.value == v).filter(table.ci_id != ci_id).first()
+            if old_value is not None:
+                return False, "attribute {0} value {1} must be unique".format(
+                    attr.attr_name, value)
+        return True, v
+
+    def add_attr_value(self, key, value, ci_id, ci_type,
+                       _no_attribute_policy="ignore", ci_existed=False):
+        """key is one of attr_id, attr_name and attr_alias
+        """
+        attr = self._get_attr(key)
+        if attr is None:
+            if _no_attribute_policy == 'ignore':
+                return True, None
+            if _no_attribute_policy == 'reject':
+                return False, 'attribute {0} not exist'.format(key)
+        table, old_value, old_value_table = TableMap(
+            attr_name=attr.attr_name).table, None, None
+        if ci_existed:
+            old_value_table = db.session.query(table).filter(
+                table.attr_id == attr.attr_id).filter(
+                    table.ci_id == ci_id).first()
+            if old_value_table is not None:
+                old_value = old_value_table.value
+        if not value and ci_existed:
+            db.session.query(table).filter(
+                table.attr_id == attr.attr_id).filter(
+                    table.ci_id == ci_id).delete()
+            if old_value:
+                return True, (attr.attr_id, "delete", old_value, None)
+            else:
+                return True, None
+        elif not value:
+            return True, None
+        if not attr.is_multivalue:
+            ret, res = self._validate(attr, value, table, ci_id)
+            if not ret:
+                return False, res
+            value_table = table()
+            if ci_existed:  # for history
+                old = db.session.query(table).filter(
+                    table.attr_id == attr.attr_id).filter(
+                        table.value == value).filter(
+                            table.ci_id == ci_id).first()
+                if old is not None:
+                    return True, None
+                elif old_value_table:
+                    value_table = old_value_table
+            value_table.ci_id = ci_id
+            value_table.attr_id = attr.attr_id
+            value_table.value = res
+            db.session.add(value_table)
+        elif attr.is_multivalue:
+            if ci_existed:
+                db.session.query(table).filter(
+                    table.attr_id == attr.attr_id).filter(
+                        table.ci_id == ci_id).delete()
+
+            for v in value.strip().split(","):
+                ret, res = self._validate(attr, v, table, ci_id)
+                if not ret:
+                    return False, res
+                value_table = table()
+                value_table.ci_id = ci_id
+                value_table.attr_id = attr.attr_id
+                value_table.value = res
+                db.session.add(value_table)
+        try:
+            db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error(
+                "add attribute value is error, {0}".format(str(e)))
+            return False, "add attribute value is error, {0}".format(str(e))
+        if ci_existed:
+            if old_value != value:
+                return True, (attr.attr_id, "update", old_value, value)
+            else:
+                return True, None
+        return True, (attr.attr_id, "add", None, value)