Modify code organization

cmdb-api/.env

@@ -0,0 +1,7 @@
+# Environment variable overrides for local development
+FLASK_APP=autoapp.py
+FLASK_DEBUG=1
+FLASK_ENV=development
+GUNICORN_WORKERS=2
+LOG_LEVEL=debug
+SECRET_KEY='<YourSecretKey>'

cmdb-api/Pipfile

@@ -0,0 +1,60 @@
+[[source]]
+url = "https://mirrors.aliyun.com/pypi/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+# Flask
+Flask = "==1.0.3"
+Werkzeug = "==0.15.4"
+click = ">=5.0"
+# Api
+Flask-RESTful = "==0.3.7"
+# Database
+Flask-SQLAlchemy = "==2.4.0"
+SQLAlchemy = "==1.3.5"
+PyMySQL = "==0.9.3"
+redis = "==3.2.1"
+# Migrations
+Flask-Migrate = "==2.5.2"
+# Deployment
+gevent = "==1.4.0"
+gunicorn = "==19.5.0"
+supervisor = "==4.0.3"
+# Auth
+Flask-Login = "==0.4.1"
+Flask-Bcrypt = "==0.7.1"
+Flask-Cors = ">=3.0.8"
+# Caching
+Flask-Caching = ">=1.0.0"
+# Environment variable parsing
+environs = "==4.2.0"
+marshmallow = "==2.20.2"
+# async tasks
+celery = "==4.3.0"
+more-itertools = "==5.0.0"
+kombu = "==4.4.0"
+# other
+six = "==1.12.0"
+bs4 = ">=0.0.1"
+toposort = ">=1.5"
+requests = ">=2.22.0"
+PyJWT = ">=1.7.1"
+elasticsearch = "==7.0.4"
+
+[dev-packages]
+# Testing
+pytest = "==4.6.5"
+WebTest = "==2.0.33"
+factory-boy = "==2.12.*"
+pdbpp = "==0.10.0"
+# Lint and code style
+flake8 = "==3.7.7"
+flake8-blind-except = "==0.1.1"
+flake8-debugger = "==3.1.0"
+flake8-docstrings = "==1.3.0"
+flake8-isort = "==2.7.0"
+isort = "==4.3.21"
+pep8-naming = "==0.8.2"
+pydocstyle = "==3.0.0"
+

cmdb-api/api/__init__.py

@@ -0,0 +1 @@
+# -*- coding:utf-8 -*-

cmdb-api/api/app.py

@@ -0,0 +1,169 @@
+# -*- coding: utf-8 -*-
+"""The app module, containing the app factory function."""
+import logging
+import os
+import sys
+from inspect import getmembers
+from logging.handlers import RotatingFileHandler
+
+from api.flask_cas import CAS
+from flask import Flask
+from flask import make_response, jsonify
+from flask.blueprints import Blueprint
+from flask.cli import click
+
+import api.views
+from api.extensions import (
+    bcrypt,
+    cors,
+    cache,
+    db,
+    login_manager,
+    migrate,
+    celery,
+    rd,
+    es
+)
+from .models.acl import User
+
+HERE = os.path.abspath(os.path.dirname(__file__))
+PROJECT_ROOT = os.path.join(HERE, os.pardir)
+API_PACKAGE = "api"
+
+
+@login_manager.user_loader
+def load_user(user_id):
+    """Load user by ID."""
+    return User.get_by(uid=int(user_id), first=True, to_dict=False)
+
+
+class ReverseProxy(object):
+    """Wrap the application in this middleware and configure the
+    front-end server to add these headers, to let you quietly bind
+    this to a URL other than / and to an HTTP scheme that is
+    different than what is used locally.
+
+    In nginx:
+    location /myprefix {
+        proxy_pass http://192.168.0.1:5001;
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Scheme $scheme;
+        proxy_set_header X-Script-Name /myprefix;
+        }
+
+    :param app: the WSGI application
+    """
+
+    def __init__(self, app):
+        self.app = app
+
+    def __call__(self, environ, start_response):
+        script_name = environ.get('HTTP_X_SCRIPT_NAME', '')
+        if script_name:
+            environ['SCRIPT_NAME'] = script_name
+            path_info = environ['PATH_INFO']
+            if path_info.startswith(script_name):
+                environ['PATH_INFO'] = path_info[len(script_name):]
+
+        scheme = environ.get('HTTP_X_SCHEME', '')
+        if scheme:
+            environ['wsgi.url_scheme'] = scheme
+        return self.app(environ, start_response)
+
+
+def create_app(config_object="settings"):
+    """Create application factory, as explained here: http://flask.pocoo.org/docs/patterns/appfactories/.
+
+    :param config_object: The configuration object to use.
+    """
+    app = Flask(__name__.split(".")[0])
+    app.config.from_object(config_object)
+    register_extensions(app)
+    register_blueprints(app)
+    register_error_handlers(app)
+    register_shell_context(app)
+    register_commands(app)
+    configure_logger(app)
+    CAS(app)
+    app.wsgi_app = ReverseProxy(app.wsgi_app)
+    return app
+
+
+def register_extensions(app):
+    """Register Flask extensions."""
+    bcrypt.init_app(app)
+    cache.init_app(app)
+    db.init_app(app)
+    cors.init_app(app)
+    login_manager.init_app(app)
+    migrate.init_app(app, db)
+    rd.init_app(app)
+    if app.config.get("USE_ES"):
+        es.init_app(app)
+    celery.conf.update(app.config)
+
+
+def register_blueprints(app):
+    for item in getmembers(api.views):
+        if item[0].startswith("blueprint") and isinstance(item[1], Blueprint):
+            app.register_blueprint(item[1])
+
+
+def register_error_handlers(app):
+    """Register error handlers."""
+
+    def render_error(error):
+        """Render error template."""
+        import traceback
+        app.logger.error(traceback.format_exc())
+        error_code = getattr(error, "code", 500)
+        return make_response(jsonify(message=str(error)), error_code)
+
+    for errcode in app.config.get("ERROR_CODES") or [400, 401, 403, 404, 405, 500, 502]:
+        app.errorhandler(errcode)(render_error)
+    app.handle_exception = render_error
+
+
+def register_shell_context(app):
+    """Register shell context objects."""
+
+    def shell_context():
+        """Shell context objects."""
+        return {"db": db, "User": User}
+
+    app.shell_context_processor(shell_context)
+
+
+def register_commands(app):
+    """Register Click commands."""
+    for root, _, files in os.walk(os.path.join(HERE, "commands")):
+        for filename in files:
+            if not filename.startswith("_") and filename.endswith("py"):
+                module_path = os.path.join(API_PACKAGE, root[root.index("commands"):])
+                if module_path not in sys.path:
+                    sys.path.insert(1, module_path)
+                command = __import__(os.path.splitext(filename)[0])
+                func_list = [o[0] for o in getmembers(command) if isinstance(o[1], click.core.Command)]
+                for func_name in func_list:
+                    app.cli.add_command(getattr(command, func_name))
+
+
+def configure_logger(app):
+    """Configure loggers."""
+    handler = logging.StreamHandler(sys.stdout)
+    formatter = logging.Formatter(
+        "%(asctime)s %(levelname)s %(pathname)s %(lineno)d - %(message)s")
+
+    if app.debug:
+        handler.setFormatter(formatter)
+        app.logger.addHandler(handler)
+
+    log_file = app.config['LOG_PATH']
+    file_handler = RotatingFileHandler(log_file,
+                                       maxBytes=2 ** 30,
+                                       backupCount=7)
+    file_handler.setLevel(getattr(logging, app.config['LOG_LEVEL']))
+    file_handler.setFormatter(formatter)
+    app.logger.addHandler(file_handler)
+    app.logger.setLevel(getattr(logging, app.config['LOG_LEVEL']))

cmdb-api/api/commands/__init__.py

@@ -0,0 +1 @@
+# -*- coding:utf-8 -*- 

cmdb-api/api/commands/click_cmdb.py

@@ -0,0 +1,137 @@
+# -*- coding:utf-8 -*-
+
+
+import json
+
+import click
+from flask import current_app
+from flask.cli import with_appcontext
+
+import api.lib.cmdb.ci
+from api.extensions import db
+from api.extensions import rd
+from api.lib.cmdb.const import PermEnum
+from api.lib.cmdb.const import REDIS_PREFIX_CI
+from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
+from api.lib.cmdb.const import ResourceTypeEnum
+from api.lib.cmdb.const import RoleEnum
+from api.lib.cmdb.const import ValueTypeEnum
+from api.lib.exception import AbortException
+from api.lib.perm.acl.acl import ACLManager
+from api.lib.perm.acl.cache import AppCache
+from api.lib.perm.acl.resource import ResourceCRUD
+from api.lib.perm.acl.resource import ResourceTypeCRUD
+from api.lib.perm.acl.role import RoleCRUD
+from api.models.acl import ResourceType
+from api.models.cmdb import CI
+from api.models.cmdb import CIRelation
+from api.models.cmdb import CIType
+from api.models.cmdb import PreferenceRelationView
+
+
+@click.command()
+@with_appcontext
+def init_cache():
+    db.session.remove()
+
+    if current_app.config.get("USE_ES"):
+        from api.extensions import es
+        from api.models.cmdb import Attribute
+        from api.lib.cmdb.utils import ValueTypeMap
+        attributes = Attribute.get_by(to_dict=False)
+        for attr in attributes:
+            other = dict()
+            other['index'] = True if attr.is_index else False
+            if attr.value_type == ValueTypeEnum.TEXT:
+                other['analyzer'] = 'ik_max_word'
+                other['search_analyzer'] = 'ik_smart'
+                if attr.is_index:
+                    other["fields"] = {
+                        "keyword": {
+                            "type": "keyword",
+                            "ignore_above": 256
+                        }
+                    }
+            try:
+                es.update_mapping(attr.name, ValueTypeMap.es_type[attr.value_type], other)
+            except Exception as e:
+                print(e)
+
+    cis = CI.get_by(to_dict=False)
+    for ci in cis:
+        if current_app.config.get("USE_ES"):
+            res = es.get_index_id(ci.id)
+            if res:
+                continue
+        else:
+            res = rd.get([ci.id], REDIS_PREFIX_CI)
+            if res and list(filter(lambda x: x, res)):
+                continue
+
+        m = api.lib.cmdb.ci.CIManager()
+        ci_dict = m.get_ci_by_id_from_db(ci.id, need_children=False, use_master=False)
+
+        if current_app.config.get("USE_ES"):
+            es.create(ci_dict)
+        else:
+            rd.create_or_update({ci.id: json.dumps(ci_dict)}, REDIS_PREFIX_CI)
+
+    ci_relations = CIRelation.get_by(to_dict=False)
+    relations = dict()
+    for cr in ci_relations:
+        relations.setdefault(cr.first_ci_id, {}).update({cr.second_ci_id: cr.second_ci.type_id})
+    for i in relations:
+        relations[i] = json.dumps(relations[i])
+    if relations:
+        rd.create_or_update(relations, REDIS_PREFIX_CI_RELATION)
+
+    db.session.remove()
+
+
+@click.command()
+@with_appcontext
+def init_acl():
+    app_id = AppCache.get('cmdb').id
+    # 1. add resource type
+    for resource_type in ResourceTypeEnum.all():
+        try:
+            ResourceTypeCRUD.add(app_id, resource_type, '', PermEnum.all())
+        except AbortException:
+            pass
+
+    # 2. add role
+    try:
+        RoleCRUD.add_role(RoleEnum.CONFIG, app_id, True)
+    except AbortException:
+        pass
+    try:
+        RoleCRUD.add_role(RoleEnum.CMDB_READ_ALL, app_id, False)
+    except AbortException:
+        pass
+
+    # 3. add resource and grant
+    ci_types = CIType.get_by(to_dict=False)
+    type_id = ResourceType.get_by(name=ResourceTypeEnum.CI, first=True, to_dict=False).id
+    for ci_type in ci_types:
+        try:
+            ResourceCRUD.add(ci_type.name, type_id, app_id)
+        except AbortException:
+            pass
+
+        ACLManager().grant_resource_to_role(ci_type.name,
+                                            RoleEnum.CMDB_READ_ALL,
+                                            ResourceTypeEnum.CI,
+                                            [PermEnum.READ])
+
+    relation_views = PreferenceRelationView.get_by(to_dict=False)
+    type_id = ResourceType.get_by(name=ResourceTypeEnum.RELATION_VIEW, first=True, to_dict=False).id
+    for view in relation_views:
+        try:
+            ResourceCRUD.add(view.name, type_id, app_id)
+        except AbortException:
+            pass
+
+        ACLManager().grant_resource_to_role(view.name,
+                                            RoleEnum.CMDB_READ_ALL,
+                                            ResourceTypeEnum.RELATION_VIEW,
+                                            [PermEnum.READ])

cmdb-api/api/commands/common.py

@@ -0,0 +1,152 @@
+# -*- coding: utf-8 -*-
+"""Click commands."""
+import os
+from glob import glob
+from subprocess import call
+
+import click
+from flask import current_app
+from flask.cli import with_appcontext
+from werkzeug.exceptions import MethodNotAllowed, NotFound
+
+from api.extensions import db
+
+HERE = os.path.abspath(os.path.dirname(__file__))
+PROJECT_ROOT = os.path.join(HERE, os.pardir, os.pardir)
+TEST_PATH = os.path.join(PROJECT_ROOT, "tests")
+
+
+@click.command()
+def test():
+    """Run the tests."""
+    import pytest
+
+    rv = pytest.main([TEST_PATH, "--verbose"])
+    exit(rv)
+
+
+@click.command()
+@click.option(
+    "-f",
+    "--fix-imports",
+    default=True,
+    is_flag=True,
+    help="Fix imports using isort, before linting",
+)
+@click.option(
+    "-c",
+    "--check",
+    default=False,
+    is_flag=True,
+    help="Don't make any changes to files, just confirm they are formatted correctly",
+)
+def lint(fix_imports, check):
+    """Lint and check code style with black, flake8 and isort."""
+    skip = ["node_modules", "requirements", "migrations"]
+    root_files = glob("*.py")
+    root_directories = [
+        name for name in next(os.walk("."))[1] if not name.startswith(".")
+    ]
+    files_and_directories = [
+        arg for arg in root_files + root_directories if arg not in skip
+    ]
+
+    def execute_tool(description, *args):
+        """Execute a checking tool with its arguments."""
+        command_line = list(args) + files_and_directories
+        click.echo("{}: {}".format(description, " ".join(command_line)))
+        rv = call(command_line)
+        if rv != 0:
+            exit(rv)
+
+    isort_args = ["-rc"]
+    black_args = []
+    if check:
+        isort_args.append("-c")
+        black_args.append("--check")
+    if fix_imports:
+        execute_tool("Fixing import order", "isort", *isort_args)
+    execute_tool("Formatting style", "black", *black_args)
+    execute_tool("Checking code style", "flake8")
+
+
+@click.command()
+def clean():
+    """Remove *.pyc and *.pyo files recursively starting at current directory.
+
+    Borrowed from Flask-Script, converted to use Click.
+    """
+    for dirpath, dirnames, filenames in os.walk("."):
+        for filename in filenames:
+            if filename.endswith(".pyc") or filename.endswith(".pyo"):
+                full_pathname = os.path.join(dirpath, filename)
+                click.echo("Removing {}".format(full_pathname))
+                os.remove(full_pathname)
+
+
+@click.command()
+@click.option("--url", default=None, help="Url to test (ex. /static/image.png)")
+@click.option(
+    "--order", default="rule", help="Property on Rule to order by (default: rule)"
+)
+@with_appcontext
+def urls(url, order):
+    """Display all of the url matching routes for the project.
+
+    Borrowed from Flask-Script, converted to use Click.
+    """
+    rows = []
+    column_headers = ("Rule", "Endpoint", "Arguments")
+
+    if url:
+        try:
+            rule, arguments = current_app.url_map.bind("localhost").match(
+                url, return_rule=True
+            )
+            rows.append((rule.rule, rule.endpoint, arguments))
+            column_length = 3
+        except (NotFound, MethodNotAllowed) as e:
+            rows.append(("<{}>".format(e), None, None))
+            column_length = 1
+    else:
+        rules = sorted(
+            current_app.url_map.iter_rules(), key=lambda x: getattr(x, order)
+        )
+        for rule in rules:
+            rows.append((rule.rule, rule.endpoint, None))
+        column_length = 2
+
+    str_template = ""
+    table_width = 0
+
+    if column_length >= 1:
+        max_rule_length = max(len(r[0]) for r in rows)
+        max_rule_length = max_rule_length if max_rule_length > 4 else 4
+        str_template += "{:" + str(max_rule_length) + "}"
+        table_width += max_rule_length
+
+    if column_length >= 2:
+        max_endpoint_length = max(len(str(r[1])) for r in rows)
+        max_endpoint_length = max_endpoint_length if max_endpoint_length > 8 else 8
+        str_template += "  {:" + str(max_endpoint_length) + "}"
+        table_width += 2 + max_endpoint_length
+
+    if column_length >= 3:
+        max_arguments_length = max(len(str(r[2])) for r in rows)
+        max_arguments_length = max_arguments_length if max_arguments_length > 9 else 9
+        str_template += "  {:" + str(max_arguments_length) + "}"
+        table_width += 2 + max_arguments_length
+
+    click.echo(str_template.format(*column_headers[:column_length]))
+    click.echo("-" * table_width)
+
+    for row in rows:
+        click.echo(str_template.format(*row[:column_length]))
+
+
+@click.command()
+@with_appcontext
+def db_setup():
+    """create tables
+    """
+    db.create_all()

cmdb-api/api/extensions.py

@@ -0,0 +1,23 @@
+# -*- coding:utf-8 -*-
+
+
+from celery import Celery
+from flask_bcrypt import Bcrypt
+from flask_caching import Cache
+from flask_cors import CORS
+from flask_login import LoginManager
+from flask_migrate import Migrate
+from flask_sqlalchemy import SQLAlchemy
+
+from api.lib.utils import ESHandler
+from api.lib.utils import RedisHandler
+
+bcrypt = Bcrypt()
+login_manager = LoginManager()
+db = SQLAlchemy()
+migrate = Migrate()
+cache = Cache()
+celery = Celery()
+cors = CORS(supports_credentials=True)
+rd = RedisHandler()
+es = ESHandler()

cmdb-api/api/flask_cas/__init__.py

@@ -0,0 +1,78 @@
+# -*- coding:utf-8 -*-
+
+"""
+flask_cas.__init__
+"""
+
+import flask
+from flask import current_app
+
+# Find the stack on which we want to store the database connection.
+# Starting with Flask 0.9, the _app_ctx_stack is the correct one,
+# before that we need to use the _request_ctx_stack.
+try:
+    from flask import _app_ctx_stack as stack
+except ImportError:
+    from flask import _request_ctx_stack as stack
+
+from api.flask_cas import routing
+
+
+class CAS(object):
+    """
+    Required Configs:
+
+    |Key             |
+    |----------------|
+    |CAS_SERVER      |
+    |CAS_AFTER_LOGIN |
+
+    Optional Configs:
+
+    |Key                      | Default        |
+    |-------------------------|----------------|
+    |CAS_TOKEN_SESSION_KEY    | _CAS_TOKEN     |
+    |CAS_USERNAME_SESSION_KEY | CAS_USERNAME   |
+    |CAS_LOGIN_ROUTE          | '/cas'         |
+    |CAS_LOGOUT_ROUTE         | '/cas/logout'  |
+    |CAS_VALIDATE_ROUTE       | '/cas/validate'|
+    """
+
+    def __init__(self, app=None, url_prefix=None):
+        self._app = app
+        if app is not None:
+            self.init_app(app, url_prefix)
+
+    def init_app(self, app, url_prefix=None):
+        # Configuration defaults
+        app.config.setdefault('CAS_TOKEN_SESSION_KEY', '_CAS_TOKEN')
+        app.config.setdefault('CAS_USERNAME_SESSION_KEY', 'CAS_USERNAME')
+        app.config.setdefault('CAS_LOGIN_ROUTE', '/login')
+        app.config.setdefault('CAS_LOGOUT_ROUTE', '/logout')
+        app.config.setdefault('CAS_VALIDATE_ROUTE', '/serviceValidate')
+        # Register Blueprint
+        app.register_blueprint(routing.blueprint, url_prefix=url_prefix)
+
+        # Use the newstyle teardown_appcontext if it's available,
+        # otherwise fall back to the request context
+        if hasattr(app, 'teardown_appcontext'):
+            app.teardown_appcontext(self.teardown)
+        else:
+            app.teardown_request(self.teardown)
+
+    def teardown(self, exception):
+        ctx = stack.top
+
+    @property
+    def app(self):
+        return self._app or current_app
+
+    @property
+    def username(self):
+        return flask.session.get(
+            self.app.config['CAS_USERNAME_SESSION_KEY'], None)
+
+    @property
+    def token(self):
+        return flask.session.get(
+            self.app.config['CAS_TOKEN_SESSION_KEY'], None)

cmdb-api/api/flask_cas/cas_urls.py

@@ -0,0 +1,122 @@
+# -*- coding:utf-8 -*-
+
+"""
+flask_cas.cas_urls
+
+Functions for creating urls to access CAS.
+"""
+from six.moves.urllib.parse import quote
+from six.moves.urllib.parse import urlencode
+from six.moves.urllib.parse import urljoin
+
+
+def create_url(base, path=None, *query):
+    """ Create a url.
+
+    Creates a url by combining base, path, and the query's list of
+    key/value pairs. Escaping is handled automatically. Any
+    key/value pair with a value that is None is ignored.
+
+    Keyword arguments:
+    base -- The left most part of the url (ex. http://localhost:5000).
+    path -- The path after the base (ex. /foo/bar).
+    query -- A list of key value pairs (ex. [('key', 'value')]).
+
+    Example usage:
+    >>> create_url(
+    ...     'http://localhost:5000',
+    ...     'foo/bar',
+    ...     ('key1', 'value'),
+    ...     ('key2', None),     # Will not include None
+    ...     ('url', 'http://example.com'),
+    ... )
+    'http://localhost:5000/foo/bar?key1=value&url=http%3A%2F%2Fexample.com'
+    """
+    url = base
+    # Add the path to the url if it's not None.
+    if path is not None:
+        url = urljoin(url, quote(path))
+        # Remove key/value pairs with None values.
+    query = filter(lambda pair: pair[1] is not None, query)
+    # Add the query string to the url
+    url = urljoin(url, '?{0}'.format(urlencode(list(query))))
+    return url
+
+
+def create_cas_login_url(cas_url, cas_route, service,
+                         renew=None, gateway=None):
+    """ Create a CAS login URL .
+
+    Keyword arguments:
+    cas_url -- The url to the CAS (ex. http://sso.pdx.edu)
+    cas_route -- The route where the CAS lives on server (ex. /cas)
+    service -- (ex.  http://localhost:5000/login)
+    renew -- "true" or "false"
+    gateway -- "true" or "false"
+
+    Example usage:
+    >>> create_cas_login_url(
+    ...     'http://sso.pdx.edu',
+    ...     '/cas',
+    ...     'http://localhost:5000',
+    ... )
+    'http://sso.pdx.edu/cas?service=http%3A%2F%2Flocalhost%3A5000'
+    """
+    return create_url(
+        cas_url,
+        cas_route,
+        ('service', service),
+        ('renew', renew),
+        ('gateway', gateway),
+    )
+
+
+def create_cas_logout_url(cas_url, cas_route, url=None):
+    """ Create a CAS logout URL.
+
+    Keyword arguments:
+    cas_url -- The url to the CAS (ex. http://sso.pdx.edu)
+    cas_route -- The route where the CAS lives on server (ex. /cas/logout)
+    url -- (ex.  http://localhost:5000/login)
+
+    Example usage:
+    >>> create_cas_logout_url(
+    ...     'http://sso.pdx.edu',
+    ...     '/cas/logout',
+    ...     'http://localhost:5000',
+    ... )
+    'http://sso.pdx.edu/cas/logout?url=http%3A%2F%2Flocalhost%3A5000'
+    """
+    return create_url(
+        cas_url,
+        cas_route,
+        ('service', url),
+    )
+
+
+def create_cas_validate_url(cas_url, cas_route, service, ticket,
+                            renew=None):
+    """ Create a CAS validate URL.
+
+    Keyword arguments:
+    cas_url -- The url to the CAS (ex. http://sso.pdx.edu)
+    cas_route -- The route where the CAS lives on server (ex. /cas/validate)
+    service -- (ex.  http://localhost:5000/login)
+    ticket -- (ex. 'ST-58274-x839euFek492ou832Eena7ee-cas')
+    renew -- "true" or "false"
+
+    Example usage:
+    >>> create_cas_validate_url(
+    ...     'http://sso.pdx.edu',
+    ...     '/cas/validate',
+    ...     'http://localhost:5000/login',
+    ...     'ST-58274-x839euFek492ou832Eena7ee-cas'
+    ... )
+    """
+    return create_url(
+        cas_url,
+        cas_route,
+        ('service', service),
+        ('ticket', ticket),
+        ('renew', renew),
+    )

cmdb-api/api/flask_cas/routing.py

@@ -0,0 +1,162 @@
+# -*- coding:utf-8 -*-
+
+import json
+
+import bs4
+from flask import Blueprint
+from flask import current_app, session, request, url_for, redirect
+from flask_login import login_user, logout_user
+from six.moves.urllib_request import urlopen
+
+from api.lib.perm.acl.cache import UserCache
+from .cas_urls import create_cas_login_url
+from .cas_urls import create_cas_logout_url
+from .cas_urls import create_cas_validate_url
+
+blueprint = Blueprint('cas', __name__)
+
+
+@blueprint.route('/api/sso/login')
+def login():
+    """
+    This route has two purposes. First, it is used by the user
+    to login. Second, it is used by the CAS to respond with the
+    `ticket` after the user logs in successfully.
+
+    When the user accesses this url, they are redirected to the CAS
+    to login. If the login was successful, the CAS will respond to this
+    route with the ticket in the url. The ticket is then validated.
+    If validation was successful the logged in username is saved in
+    the user's session under the key `CAS_USERNAME_SESSION_KEY`.
+    """
+
+    cas_token_session_key = current_app.config['CAS_TOKEN_SESSION_KEY']
+    if request.values.get("next"):
+        session["next"] = request.values.get("next")
+
+    _service = url_for('cas.login', _external=True, next=session["next"]) \
+        if session.get("next") else url_for('cas.login', _external=True)
+    redirect_url = create_cas_login_url(
+        current_app.config['CAS_SERVER'],
+        current_app.config['CAS_LOGIN_ROUTE'],
+        _service)
+
+    if 'ticket' in request.args:
+        session[cas_token_session_key] = request.args.get('ticket')
+
+    if request.args.get('ticket'):
+
+        if validate(request.args['ticket']):
+            redirect_url = session.get("next") or \
+                           current_app.config.get("CAS_AFTER_LOGIN")
+            username = session.get("CAS_USERNAME")
+            user = UserCache.get(username)
+            login_user(user)
+
+            session.permanent = True
+
+        else:
+            del session[cas_token_session_key]
+            redirect_url = create_cas_login_url(
+                current_app.config['CAS_SERVER'],
+                current_app.config['CAS_LOGIN_ROUTE'],
+                url_for('cas.login', _external=True),
+                renew=True)
+    current_app.logger.info("redirect to: {0}".format(redirect_url))
+    return redirect(redirect_url)
+
+
+@blueprint.route('/api/sso/logout')
+def logout():
+    """
+    When the user accesses this route they are logged out.
+    """
+
+    cas_username_session_key = current_app.config['CAS_USERNAME_SESSION_KEY']
+    cas_token_session_key = current_app.config['CAS_TOKEN_SESSION_KEY']
+
+    cas_username_session_key in session and session.pop(cas_username_session_key)
+    "acl" in session and session.pop("acl")
+    "uid" in session and session.pop("uid")
+    cas_token_session_key in session and session.pop(cas_token_session_key)
+    "next" in session and session.pop("next")
+
+    redirect_url = create_cas_logout_url(
+        current_app.config['CAS_SERVER'],
+        current_app.config['CAS_LOGOUT_ROUTE'],
+        url_for('cas.login', _external=True, next=request.referrer))
+
+    logout_user()
+
+    current_app.logger.debug('Redirecting to: {0}'.format(redirect_url))
+
+    return redirect(redirect_url)
+
+
+def validate(ticket):
+    """
+    Will attempt to validate the ticket. If validation fails, then False
+    is returned. If validation is successful, then True is returned
+    and the validated username is saved in the session under the
+    key `CAS_USERNAME_SESSION_KEY`.
+    """
+
+    cas_username_session_key = current_app.config['CAS_USERNAME_SESSION_KEY']
+
+    current_app.logger.debug("validating token {0}".format(ticket))
+
+    cas_validate_url = create_cas_validate_url(
+        current_app.config['CAS_VALIDATE_SERVER'],
+        current_app.config['CAS_VALIDATE_ROUTE'],
+        url_for('cas.login', _external=True),
+        ticket)
+
+    current_app.logger.debug("Making GET request to {0}".format(cas_validate_url))
+
+    try:
+        response = urlopen(cas_validate_url).read()
+        ticketid = _parse_tag(response, "cas:user")
+        strs = [s.strip() for s in ticketid.split('|') if s.strip()]
+        username, is_valid = None, False
+        if len(strs) == 1:
+            username = strs[0]
+            is_valid = True
+        user_info = json.loads(_parse_tag(response, "cas:other"))
+        current_app.logger.info(user_info)
+    except ValueError:
+        current_app.logger.error("CAS returned unexpected result")
+        is_valid = False
+        return is_valid
+
+    if is_valid:
+        current_app.logger.debug("valid")
+        session[cas_username_session_key] = username
+        user = UserCache.get(username)
+        session["acl"] = dict(uid=user_info.get("uuid"),
+                              avatar=user.avatar if user else user_info.get("avatar"),
+                              userId=user_info.get("id"),
+                              userName=user_info.get("name"),
+                              nickName=user_info.get("nickname"),
+                              parentRoles=user_info.get("parents"),
+                              childRoles=user_info.get("children"),
+                              roleName=user_info.get("role"))
+        session["uid"] = user_info.get("uuid")
+        current_app.logger.debug(session)
+        current_app.logger.debug(request.url)
+    else:
+        current_app.logger.debug("invalid")
+
+    return is_valid
+
+
+def _parse_tag(string, tag):
+    """
+    Used for parsing xml.  Search string for the first occurence of
+    <tag>.....</tag> and return text (stripped of leading and tailing
+    whitespace) between tags.  Return "" if tag not found.
+    """
+    soup = bs4.BeautifulSoup(string)
+
+    if soup.find(tag) is None:
+        return ''
+    return soup.find(tag).string.strip()

cmdb-api/api/lib/__init__.py

@@ -0,0 +1 @@
+# -*- coding:utf-8 -*- 

cmdb-api/api/lib/cmdb/__init__.py

@@ -0,0 +1 @@
+# -*- coding:utf-8 -*-

cmdb-api/api/lib/cmdb/attribute.py

@@ -0,0 +1,190 @@
+# -*- coding:utf-8 -*- 
+
+from flask import abort
+from flask import current_app
+
+from api.extensions import db
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.const import ValueTypeEnum
+from api.lib.cmdb.utils import ValueTypeMap
+from api.lib.decorator import kwargs_required
+from api.models.cmdb import Attribute
+from api.models.cmdb import CITypeAttribute
+from api.models.cmdb import PreferenceShowAttributes
+
+
+class AttributeManager(object):
+    """
+    CI attributes manager
+    """
+
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def get_choice_values(attr_id, value_type):
+        choice_table = ValueTypeMap.choice.get(value_type)
+        choice_values = choice_table.get_by(fl=["value"], attr_id=attr_id)
+        return [choice_value["value"] for choice_value in choice_values]
+
+    @staticmethod
+    def _add_choice_values(_id, value_type, choice_values):
+        choice_table = ValueTypeMap.choice.get(value_type)
+
+        db.session.query(choice_table).filter(choice_table.attr_id == _id).delete()
+        db.session.flush()
+        choice_values = choice_values
+        for v in choice_values:
+            table = choice_table(attr_id=_id, value=v)
+            db.session.add(table)
+        db.session.flush()
+
+    @classmethod
+    def search_attributes(cls, name=None, alias=None, page=1, page_size=None):
+        """
+        :param name: 
+        :param alias: 
+        :param page: 
+        :param page_size: 
+        :return: attribute, if name is None, then return all attributes
+        """
+        if name is not None:
+            attrs = Attribute.get_by_like(name=name)
+        elif alias is not None:
+            attrs = Attribute.get_by_like(alias=alias)
+        else:
+            attrs = Attribute.get_by()
+
+        numfound = len(attrs)
+        attrs = attrs[(page - 1) * page_size:][:page_size]
+        res = list()
+        for attr in attrs:
+            attr["is_choice"] and attr.update(dict(choice_value=cls.get_choice_values(attr["id"], attr["value_type"])))
+            res.append(attr)
+
+        return numfound, res
+
+    def get_attribute_by_name(self, name):
+        attr = Attribute.get_by(name=name, first=True)
+        if attr and attr["is_choice"]:
+            attr.update(dict(choice_value=self.get_choice_values(attr["id"], attr["value_type"])))
+        return attr
+
+    def get_attribute_by_alias(self, alias):
+        attr = Attribute.get_by(alias=alias, first=True)
+        if attr and attr["is_choice"]:
+            attr.update(dict(choice_value=self.get_choice_values(attr["id"], attr["value_type"])))
+        return attr
+
+    def get_attribute_by_id(self, _id):
+        attr = Attribute.get_by_id(_id).to_dict()
+        if attr and attr["is_choice"]:
+            attr.update(dict(choice_value=self.get_choice_values(attr["id"], attr["value_type"])))
+        return attr
+
+    def get_attribute(self, key):
+        attr = AttributeCache.get(key).to_dict()
+        if attr and attr["is_choice"]:
+            attr.update(dict(choice_value=self.get_choice_values(attr["id"], attr["value_type"])))
+        return attr
+
+    @classmethod
+    @kwargs_required("name")
+    def add(cls, **kwargs):
+        choice_value = kwargs.pop("choice_value", [])
+        kwargs.pop("is_choice", None)
+        is_choice = True if choice_value else False
+        name = kwargs.pop("name")
+        alias = kwargs.pop("alias", "")
+        alias = name if not alias else alias
+        Attribute.get_by(name=name, first=True) and abort(400, "attribute name <{0}> is duplicated".format(name))
+        Attribute.get_by(alias=alias, first=True) and abort(400, "attribute alias <{0}> is duplicated".format(name))
+
+        attr = Attribute.create(flush=True,
+                                name=name,
+                                alias=alias,
+                                is_choice=is_choice,
+                                **kwargs)
+
+        if choice_value:
+            cls._add_choice_values(attr.id, attr.value_type, choice_value)
+
+        try:
+            db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error("add attribute error, {0}".format(str(e)))
+            return abort(400, "add attribute <{0}> failed".format(name))
+
+        AttributeCache.clean(attr)
+
+        if current_app.config.get("USE_ES"):
+            from api.extensions import es
+            other = dict()
+            other['index'] = True if attr.is_index else False
+            if attr.value_type == ValueTypeEnum.TEXT:
+                other['analyzer'] = 'ik_max_word'
+                other['search_analyzer'] = 'ik_smart'
+                if attr.is_index:
+                    other["fields"] = {
+                        "keyword": {
+                            "type": "keyword",
+                            "ignore_above": 256
+                        }
+                    }
+            es.update_mapping(name, ValueTypeMap.es_type[attr.value_type], other)
+
+        return attr.id
+
+    def update(self, _id, **kwargs):
+        attr = Attribute.get_by_id(_id) or abort(404, "Attribute <{0}> does not exist".format(_id))
+
+        if kwargs.get("name"):
+            other = Attribute.get_by(name=kwargs['name'], first=True, to_dict=False)
+            if other and other.id != attr.id:
+                return abort(400, "Attribute name <{0}> cannot be duplicate!".format(kwargs['name']))
+        if kwargs.get("alias"):
+            other = Attribute.get_by(alias=kwargs['alias'], first=True, to_dict=False)
+            if other and other.id != attr.id:
+                return abort(400, "Attribute alias <{0}> cannot be duplicate!".format(kwargs['alias']))
+
+        choice_value = kwargs.pop("choice_value", False)
+        is_choice = True if choice_value else False
+
+        attr.update(flush=True, **kwargs)
+
+        if is_choice:
+            self._add_choice_values(attr.id, attr.value_type, choice_value)
+
+        try:
+            db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error("update attribute error, {0}".format(str(e)))
+            return abort(400, "update attribute <{0}> failed".format(_id))
+
+        AttributeCache.clean(attr)
+
+        return attr.id
+
+    @staticmethod
+    def delete(_id):
+        attr = Attribute.get_by_id(_id) or abort(404, "Attribute <{0}> does not exist".format(_id))
+        name = attr.name
+
+        if attr.is_choice:
+            choice_table = ValueTypeMap.choice.get(attr.value_type)
+            db.session.query(choice_table).filter(choice_table.attr_id == _id).delete()  # FIXME: session conflict
+            db.session.flush()
+
+        AttributeCache.clean(attr)
+
+        attr.soft_delete()
+
+        for i in CITypeAttribute.get_by(attr_id=_id, to_dict=False):
+            i.soft_delete()
+
+        for i in PreferenceShowAttributes.get_by(attr_id=_id, to_dict=False):
+            i.soft_delete()
+
+        return name

cmdb-api/api/lib/cmdb/cache.py

@@ -0,0 +1,148 @@
+# -*- coding:utf-8 -*-
+
+from __future__ import unicode_literals
+
+from api.extensions import cache
+from api.models.cmdb import Attribute
+from api.models.cmdb import CIType
+from api.models.cmdb import CITypeAttribute
+from api.models.cmdb import RelationType
+
+
+class AttributeCache(object):
+    PREFIX_ID = 'Field::ID::{0}'
+    PREFIX_NAME = 'Field::Name::{0}'
+    PREFIX_ALIAS = 'Field::Alias::{0}'
+
+    @classmethod
+    def get(cls, key):
+        if key is None:
+            return
+        attr = cache.get(cls.PREFIX_NAME.format(key))
+        attr = attr or cache.get(cls.PREFIX_ID.format(key))
+        attr = attr or cache.get(cls.PREFIX_ALIAS.format(key))
+
+        if attr is None:
+            attr = Attribute.get_by(name=key, first=True, to_dict=False)
+            attr = attr or Attribute.get_by_id(key)
+            attr = attr or Attribute.get_by(alias=key, first=True, to_dict=False)
+            if attr is not None:
+                cls.set(attr)
+        return attr
+
+    @classmethod
+    def set(cls, attr):
+        cache.set(cls.PREFIX_ID.format(attr.id), attr)
+        cache.set(cls.PREFIX_NAME.format(attr.name), attr)
+        cache.set(cls.PREFIX_ALIAS.format(attr.alias), attr)
+
+    @classmethod
+    def clean(cls, attr):
+        cache.delete(cls.PREFIX_ID.format(attr.id))
+        cache.delete(cls.PREFIX_NAME.format(attr.name))
+        cache.delete(cls.PREFIX_ALIAS.format(attr.alias))
+
+
+class CITypeCache(object):
+    PREFIX_ID = "CIType::ID::{0}"
+    PREFIX_NAME = "CIType::Name::{0}"
+    PREFIX_ALIAS = "CIType::Alias::{0}"
+
+    @classmethod
+    def get(cls, key):
+        if key is None:
+            return
+        ct = cache.get(cls.PREFIX_NAME.format(key))
+        ct = ct or cache.get(cls.PREFIX_ID.format(key))
+        ct = ct or cache.get(cls.PREFIX_ALIAS.format(key))
+        if ct is None:
+            ct = CIType.get_by(name=key, first=True, to_dict=False)
+            ct = ct or CIType.get_by_id(key)
+            ct = ct or CIType.get_by(alias=key, first=True, to_dict=False)
+            if ct is not None:
+                cls.set(ct)
+        return ct
+
+    @classmethod
+    def set(cls, ct):
+        cache.set(cls.PREFIX_NAME.format(ct.name), ct)
+        cache.set(cls.PREFIX_ID.format(ct.id), ct)
+        cache.set(cls.PREFIX_ALIAS.format(ct.alias), ct)
+
+    @classmethod
+    def clean(cls, key):
+        ct = cls.get(key)
+        if ct is not None:
+            cache.delete(cls.PREFIX_NAME.format(ct.name))
+            cache.delete(cls.PREFIX_ID.format(ct.id))
+            cache.delete(cls.PREFIX_ALIAS.format(ct.alias))
+
+
+class RelationTypeCache(object):
+    PREFIX_ID = "RelationType::ID::{0}"
+    PREFIX_NAME = "RelationType::Name::{0}"
+
+    @classmethod
+    def get(cls, key):
+        if key is None:
+            return
+        ct = cache.get(cls.PREFIX_NAME.format(key))
+        ct = ct or cache.get(cls.PREFIX_ID.format(key))
+        if ct is None:
+            ct = RelationType.get_by(name=key, first=True, to_dict=False) or RelationType.get_by_id(key)
+            if ct is not None:
+                cls.set(ct)
+        return ct
+
+    @classmethod
+    def set(cls, ct):
+        cache.set(cls.PREFIX_NAME.format(ct.name), ct)
+        cache.set(cls.PREFIX_ID.format(ct.id), ct)
+
+    @classmethod
+    def clean(cls, key):
+        ct = cls.get(key)
+        if ct is not None:
+            cache.delete(cls.PREFIX_NAME.format(ct.name))
+            cache.delete(cls.PREFIX_ID.format(ct.id))
+
+
+class CITypeAttributeCache(object):
+    """
+    key is type_id or type_name
+    """
+
+    PREFIX_ID = "CITypeAttribute::ID::{0}"
+    PREFIX_NAME = "CITypeAttribute::Name::{0}"
+
+    @classmethod
+    def get(cls, key):
+        if key is None:
+            return
+
+        attrs = cache.get(cls.PREFIX_NAME.format(key))
+        attrs = attrs or cache.get(cls.PREFIX_ID.format(key))
+        if not attrs:
+            attrs = CITypeAttribute.get_by(type_id=key, to_dict=False)
+            if not attrs:
+                ci_type = CIType.get_by(name=key, first=True, to_dict=False)
+                if ci_type is not None:
+                    attrs = CITypeAttribute.get_by(type_id=ci_type.id, to_dict=False)
+            if attrs is not None:
+                cls.set(key, attrs)
+        return attrs
+
+    @classmethod
+    def set(cls, key, values):
+        ci_type = CITypeCache.get(key)
+        if ci_type is not None:
+            cache.set(cls.PREFIX_ID.format(ci_type.id), values)
+            cache.set(cls.PREFIX_NAME.format(ci_type.name), values)
+
+    @classmethod
+    def clean(cls, key):
+        ci_type = CITypeCache.get(key)
+        attrs = cls.get(key)
+        if attrs is not None and ci_type:
+            cache.delete(cls.PREFIX_ID.format(ci_type.id))
+            cache.delete(cls.PREFIX_NAME.format(ci_type.name))

cmdb-api/api/lib/cmdb/ci.py

@@ -0,0 +1,545 @@
+# -*- coding:utf-8 -*- 
+
+
+import datetime
+import json
+
+from flask import abort
+from flask import current_app
+from werkzeug.exceptions import BadRequest
+
+from api.extensions import db
+from api.extensions import rd
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.cache import CITypeCache
+from api.lib.cmdb.ci_type import CITypeAttributeManager
+from api.lib.cmdb.ci_type import CITypeManager
+from api.lib.cmdb.const import CMDB_QUEUE
+from api.lib.cmdb.const import ExistPolicy
+from api.lib.cmdb.const import OperateType
+from api.lib.cmdb.const import REDIS_PREFIX_CI
+from api.lib.cmdb.const import RetKey
+from api.lib.cmdb.history import AttributeHistoryManger
+from api.lib.cmdb.history import CIRelationHistoryManager
+from api.lib.cmdb.search.ci.db.query_sql import QUERY_CIS_BY_IDS
+from api.lib.cmdb.search.ci.db.query_sql import QUERY_CIS_BY_VALUE_TABLE
+from api.lib.cmdb.utils import TableMap
+from api.lib.cmdb.utils import ValueTypeMap
+from api.lib.cmdb.value import AttributeValueManager
+from api.lib.decorator import kwargs_required
+from api.lib.utils import handle_arg_list
+from api.models.cmdb import CI
+from api.models.cmdb import CIRelation
+from api.models.cmdb import CITypeAttribute
+from api.models.cmdb import CITypeRelation
+from api.tasks.cmdb import ci_cache
+from api.tasks.cmdb import ci_delete
+from api.tasks.cmdb import ci_relation_cache
+from api.tasks.cmdb import ci_relation_delete
+
+
+class CIManager(object):
+    """ manage CI interface
+    """
+
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def get_type_name(ci_id):
+        ci = CI.get_by_id(ci_id) or abort(404, "CI <{0}> is not existed".format(ci_id))
+        return CITypeCache.get(ci.type_id).name
+
+    @staticmethod
+    def confirm_ci_existed(ci_id):
+        return CI.get_by_id(ci_id) or abort(404, "CI <{0}> is not existed".format(ci_id))
+
+    @classmethod
+    def get_ci_by_id(cls, ci_id, ret_key=RetKey.NAME, fields=None, need_children=True):
+        """
+        
+        :param ci_id: 
+        :param ret_key: name, id, or alias
+        :param fields:  attribute list
+        :param need_children: 
+        :return: 
+        """
+
+        ci = CI.get_by_id(ci_id) or abort(404, "CI <{0}> is not existed".format(ci_id))
+
+        res = dict()
+
+        if need_children:
+            children = CIRelationManager.get_children(ci_id, ret_key=ret_key)  # one floor
+            res.update(children)
+
+        ci_type = CITypeCache.get(ci.type_id)
+        res["ci_type"] = ci_type.name
+
+        res.update(cls.get_cis_by_ids([str(ci_id)], fields=fields, ret_key=ret_key))
+
+        res['_type'] = ci_type.id
+        res['_id'] = ci_id
+
+        return res
+
+    @staticmethod
+    def get_ci_by_id_from_db(ci_id, ret_key=RetKey.NAME, fields=None, need_children=True, use_master=False):
+        """
+        
+        :param ci_id: 
+        :param ret_key: name, id or alias
+        :param fields: list
+        :param need_children: 
+        :param use_master: whether to use master db
+        :return: 
+        """
+
+        ci = CI.get_by_id(ci_id) or abort(404, "CI <{0}> is not existed".format(ci_id))
+
+        res = dict()
+
+        if need_children:
+            children = CIRelationManager.get_children(ci_id, ret_key=ret_key)  # one floor
+            res.update(children)
+
+        ci_type = CITypeCache.get(ci.type_id)
+        res["ci_type"] = ci_type.name
+
+        fields = CITypeAttributeManager.get_attr_names_by_type_id(ci.type_id) if not fields else fields
+        unique_key = AttributeCache.get(ci_type.unique_id)
+        _res = AttributeValueManager().get_attr_values(fields,
+                                                       ci_id,
+                                                       ret_key=ret_key,
+                                                       unique_key=unique_key,
+                                                       use_master=use_master)
+        res.update(_res)
+
+        res['type_id'] = ci_type.id
+        res['ci_id'] = ci_id
+
+        return res
+
+    def get_ci_by_ids(self, ci_id_list, ret_key=RetKey.NAME, fields=None):
+        return [self.get_ci_by_id(ci_id, ret_key=ret_key, fields=fields) for ci_id in ci_id_list]
+
+    @classmethod
+    def get_cis_by_type(cls, type_id, ret_key=RetKey.NAME, fields="", page=1, per_page=None):
+        cis = db.session.query(CI.id).filter(CI.type_id == type_id).filter(CI.deleted.is_(False))
+        numfound = cis.count()
+
+        cis = cis.offset((page - 1) * per_page).limit(per_page)
+        ci_ids = [str(ci.id) for ci in cis]
+        res = cls.get_cis_by_ids(ci_ids, ret_key, fields)
+
+        return numfound, page, res
+
+    @staticmethod
+    def ci_is_exist(unique_key, unique_value):
+        """
+        
+        :param unique_key: is a attribute
+        :param unique_value: 
+        :return: 
+        """
+        value_table = TableMap(attr_name=unique_key.name).table
+        unique = value_table.get_by(attr_id=unique_key.id,
+                                    value=unique_value,
+                                    to_dict=False,
+                                    first=True)
+        if unique:
+            return CI.get_by_id(unique.ci_id)
+
+    @staticmethod
+    def _delete_ci_by_id(ci_id):
+        ci = CI.get_by_id(ci_id)
+        ci.delete()  # TODO: soft delete
+
+    @classmethod
+    def add(cls, ci_type_name, exist_policy=ExistPolicy.REPLACE, _no_attribute_policy=ExistPolicy.IGNORE, **ci_dict):
+        """
+        
+        :param ci_type_name: 
+        :param exist_policy: replace or reject or need
+        :param _no_attribute_policy: ignore or reject
+        :param ci_dict: 
+        :return: 
+        """
+
+        ci_type = CITypeManager.check_is_existed(ci_type_name)
+
+        unique_key = AttributeCache.get(ci_type.unique_id) or abort(400, 'illegality unique attribute')
+
+        unique_value = ci_dict.get(unique_key.name)
+        unique_value = unique_value or ci_dict.get(unique_key.alias)
+        unique_value = unique_value or ci_dict.get(unique_key.id)
+        unique_value = unique_value or abort(400, '{0} missing'.format(unique_key.name))
+
+        existed = cls.ci_is_exist(unique_key, unique_value)
+        if existed is not None:
+            if exist_policy == ExistPolicy.REJECT:
+                return abort(400, 'CI is already existed')
+            if existed.type_id != ci_type.id:
+                existed.update(type_id=ci_type.id)
+            ci = existed
+        else:
+            if exist_policy == ExistPolicy.NEED:
+                return abort(404, 'CI <{0}> does not exist'.format(unique_value))
+            ci = CI.create(type_id=ci_type.id)
+
+        value_manager = AttributeValueManager()
+        for p, v in ci_dict.items():
+            try:
+                value_manager.create_or_update_attr_value(p, v, ci.id, _no_attribute_policy)
+            except BadRequest as e:
+                if existed is None:
+                    cls.delete(ci.id)
+                raise e
+
+        ci_cache.apply_async([ci.id], queue=CMDB_QUEUE)
+
+        return ci.id
+
+    def update(self, ci_id, **ci_dict):
+        self.confirm_ci_existed(ci_id)
+        value_manager = AttributeValueManager()
+        for p, v in ci_dict.items():
+            try:
+                value_manager.create_or_update_attr_value(p, v, ci_id)
+            except BadRequest as e:
+                raise e
+
+        ci_cache.apply_async([ci_id], queue=CMDB_QUEUE)
+
+    @staticmethod
+    def update_unique_value(ci_id, unique_name, unique_value):
+        CI.get_by_id(ci_id) or abort(404, "CI <{0}> is not found".format(ci_id))
+
+        AttributeValueManager().create_or_update_attr_value(unique_name, unique_value, ci_id)
+
+        ci_cache.apply_async([ci_id], queue=CMDB_QUEUE)
+
+    @staticmethod
+    def delete(ci_id):
+        ci = CI.get_by_id(ci_id) or abort(404, "CI <{0}> is not found".format(ci_id))
+
+        attrs = CITypeAttribute.get_by(type_id=ci.type_id, to_dict=False)
+        attr_names = set([AttributeCache.get(attr.attr_id).name for attr in attrs])
+        for attr_name in attr_names:
+            value_table = TableMap(attr_name=attr_name).table
+            for item in value_table.get_by(ci_id=ci_id, to_dict=False):
+                item.delete()
+
+        for item in CIRelation.get_by(first_ci_id=ci_id, to_dict=False):
+            ci_relation_delete.apply_async(args=(item.first_ci_id, item.second_ci_id), queue=CMDB_QUEUE)
+            item.delete()
+
+        for item in CIRelation.get_by(second_ci_id=ci_id, to_dict=False):
+            ci_relation_delete.apply_async(args=(item.first_ci_id, item.second_ci_id), queue=CMDB_QUEUE)
+            item.delete()
+
+        ci.delete()  # TODO: soft delete
+
+        AttributeHistoryManger.add(ci_id, [(None, OperateType.DELETE, None, None)])
+
+        ci_delete.apply_async([ci.id], queue=CMDB_QUEUE)
+
+        return ci_id
+
+    @staticmethod
+    def add_heartbeat(ci_type, unique_value):
+        ci_type = CITypeManager().check_is_existed(ci_type)
+
+        unique_key = AttributeCache.get(ci_type.unique_id)
+        value_table = TableMap(attr_name=unique_key.name).table
+
+        v = value_table.get_by(attr_id=unique_key.id,
+                               value=unique_value,
+                               to_dict=False,
+                               first=True) \
+            or abort(404, "not found")
+
+        ci = CI.get_by_id(v.ci_id) or abort(404, "CI <{0}> is not found".format(v.ci_id))
+
+        ci.update(heartbeat=datetime.datetime.now())
+
+    @classmethod
+    @kwargs_required("type_id", "page")
+    def get_heartbeat(cls, **kwargs):
+        query = db.session.query(CI.id, CI.heartbeat).filter(CI.deleted.is_(False))
+
+        expire = datetime.datetime.now() - datetime.timedelta(minutes=72)
+        type_ids = handle_arg_list(kwargs["type_id"])
+
+        query = query.filter(CI.type_id.in_(type_ids))
+
+        page = kwargs.get("page")
+        agent_status = kwargs.get("agent_status")
+        if agent_status == -1:
+            query = query.filter(CI.heartbeat.is_(None))
+        elif agent_status == 0:
+            query = query.filter(CI.heartbeat <= expire)
+        elif agent_status == 1:
+            query = query.filter(CI.heartbeat > expire)
+
+        numfound = query.count()
+        per_page_count = current_app.config.get("DEFAULT_PAGE_COUNT")
+        cis = query.offset((page - 1) * per_page_count).limit(per_page_count).all()
+        ci_ids = [ci.id for ci in cis]
+        heartbeat_dict = {}
+        for ci in cis:
+            if agent_status is not None:
+                heartbeat_dict[ci.id] = agent_status
+            else:
+                if ci.heartbeat is None:
+                    heartbeat_dict[ci.id] = -1
+                elif ci.heartbeat <= expire:
+                    heartbeat_dict[ci.id] = 0
+                else:
+                    heartbeat_dict[ci.id] = 1
+        current_app.logger.debug(heartbeat_dict)
+        ci_ids = list(map(str, ci_ids))
+        res = cls.get_cis_by_ids(ci_ids, fields=["hostname", "private_ip"])
+        result = [(i.get("hostname"), i.get("private_ip")[0], i.get("ci_type"),
+                   heartbeat_dict.get(i.get("_id"))) for i in res
+                  if i.get("private_ip")]
+        return numfound, result
+
+    @staticmethod
+    def _get_cis_from_cache(ci_ids, ret_key=RetKey.NAME, fields=None):
+        res = rd.get(ci_ids, REDIS_PREFIX_CI)
+        if res is not None and None not in res and ret_key == RetKey.NAME:
+            res = list(map(json.loads, res))
+            if not fields:
+                return res
+            else:
+                _res = []
+                for d in res:
+                    _d = dict()
+                    _d["_id"], _d["_type"] = d.get("_id"), d.get("_type")
+                    _d["ci_type"] = d.get("ci_type")
+                    for field in fields:
+                        _d[field] = d.get(field)
+                    _res.append(_d)
+                return _res
+
+    @staticmethod
+    def _get_cis_from_db(ci_ids, ret_key=RetKey.NAME, fields=None, value_tables=None):
+        if not fields:
+            filter_fields_sql = ""
+        else:
+            _fields = list()
+            for field in fields:
+                attr = AttributeCache.get(field)
+                if attr is not None:
+                    _fields.append(str(attr.id))
+            filter_fields_sql = "WHERE A.attr_id in ({0})".format(",".join(_fields))
+
+        ci_ids = ",".join(ci_ids)
+        if value_tables is None:
+            value_tables = ValueTypeMap.table_name.values()
+
+        value_sql = " UNION ".join([QUERY_CIS_BY_VALUE_TABLE.format(value_table, ci_ids)
+                                    for value_table in value_tables])
+        query_sql = QUERY_CIS_BY_IDS.format(filter_fields_sql, value_sql)
+        # current_app.logger.debug(query_sql)
+        cis = db.session.execute(query_sql).fetchall()
+        ci_set = set()
+        res = list()
+        ci_dict = dict()
+        for ci_id, type_id, attr_id, attr_name, attr_alias, value, value_type, is_list in cis:
+            if ci_id not in ci_set:
+                ci_dict = dict()
+                ci_type = CITypeCache.get(type_id)
+                ci_dict["ci_id"] = ci_id
+                ci_dict["ci_type"] = type_id
+                ci_dict["ci_type"] = ci_type.name
+                ci_dict["ci_type_alias"] = ci_type.alias
+                ci_set.add(ci_id)
+                res.append(ci_dict)
+
+            if ret_key == RetKey.NAME:
+                attr_key = attr_name
+            elif ret_key == RetKey.ALIAS:
+                attr_key = attr_alias
+            elif ret_key == RetKey.ID:
+                attr_key = attr_id
+            else:
+                return abort(400, "invalid ret key")
+
+            value = ValueTypeMap.serialize2[value_type](value)
+            if is_list:
+                ci_dict.setdefault(attr_key, []).append(value)
+            else:
+                ci_dict[attr_key] = value
+
+        return res
+
+    @classmethod
+    def get_cis_by_ids(cls, ci_ids, ret_key=RetKey.NAME, fields=None, value_tables=None):
+        """
+
+        :param ci_ids: list of CI instance ID, eg. ['1', '2']
+        :param ret_key: name, id or alias
+        :param fields: 
+        :param value_tables: 
+        :return: 
+        """
+
+        if not ci_ids:
+            return []
+
+        fields = [] if fields is None or not isinstance(fields, list) else fields
+
+        ci_id_tuple = tuple(map(int, ci_ids))
+        res = cls._get_cis_from_cache(ci_id_tuple, ret_key, fields)
+        if res is not None:
+            return res
+
+        current_app.logger.warning("cache not hit...............")
+        return cls._get_cis_from_db(ci_ids, ret_key, fields, value_tables)
+
+
+class CIRelationManager(object):
+    """
+    Manage relation between CIs
+    """
+
+    def __init__(self):
+        pass
+
+    @classmethod
+    def get_children(cls, ci_id, ret_key=RetKey.NAME):
+        second_cis = CIRelation.get_by(first_ci_id=ci_id, to_dict=False)
+        second_ci_ids = (second_ci.second_ci_id for second_ci in second_cis)
+        ci_type2ci_ids = dict()
+        for ci_id in second_ci_ids:
+            type_id = CI.get_by_id(ci_id).type_id
+            ci_type2ci_ids.setdefault(type_id, []).append(ci_id)
+
+        res = {}
+        for type_id in ci_type2ci_ids:
+            ci_type = CITypeCache.get(type_id)
+            children = CIManager.get_cis_by_ids(list(map(str, ci_type2ci_ids[type_id])), ret_key=ret_key)
+            res[ci_type.name] = children
+        return res
+
+    @staticmethod
+    def get_second_cis(first_ci_id, relation_type_id=None, page=1, per_page=None):
+        second_cis = db.session.query(CI.id).filter(CI.deleted.is_(False)).join(
+            CIRelation, CIRelation.second_ci_id == CI.id).filter(
+            CIRelation.first_ci_id == first_ci_id).filter(CIRelation.deleted.is_(False))
+
+        if relation_type_id is not None:
+            second_cis = second_cis.filter(CIRelation.relation_type_id == relation_type_id)
+
+        numfound = second_cis.count()
+        if per_page != "all":
+            second_cis = second_cis.offset((page - 1) * per_page).limit(per_page).all()
+        ci_ids = [str(son.id) for son in second_cis]
+        result = CIManager.get_cis_by_ids(ci_ids)
+
+        return numfound, len(ci_ids), result
+
+    @staticmethod
+    def _sort_handler(sort_by, query_sql):
+
+        if sort_by.startswith("+"):
+            sort_type = "asc"
+            sort_by = sort_by[1:]
+        elif sort_by.startswith("-"):
+            sort_type = "desc"
+            sort_by = sort_by[1:]
+        else:
+            sort_type = "asc"
+        attr = AttributeCache.get(sort_by)
+        if attr is None:
+            return query_sql
+
+        attr_id = attr.id
+        value_table = TableMap(attr_name=sort_by).table
+
+        ci_table = query_sql.subquery()
+        query_sql = db.session.query(ci_table.c.id, value_table.value).join(
+            value_table, value_table.ci_id == ci_table.c.id).filter(
+            value_table.attr_id == attr_id).filter(ci_table.deleted.is_(False)).order_by(
+            getattr(value_table.value, sort_type)())
+
+        return query_sql
+
+    @classmethod
+    def get_first_cis(cls, second_ci, relation_type_id=None, page=1, per_page=None):
+        first_cis = db.session.query(CIRelation.first_ci_id).filter(
+            CIRelation.second_ci_id == second_ci).filter(CIRelation.deleted.is_(False))
+        if relation_type_id is not None:
+            first_cis = first_cis.filter(CIRelation.relation_type_id == relation_type_id)
+
+        numfound = first_cis.count()
+        if per_page != "all":
+            first_cis = first_cis.offset((page - 1) * per_page).limit(per_page).all()
+
+        first_ci_ids = [str(first_ci.first_ci_id) for first_ci in first_cis]
+        result = CIManager.get_cis_by_ids(first_ci_ids)
+
+        return numfound, len(first_ci_ids), result
+
+    @classmethod
+    def add(cls, first_ci_id, second_ci_id, more=None, relation_type_id=None):
+
+        first_ci = CIManager.confirm_ci_existed(first_ci_id)
+        second_ci = CIManager.confirm_ci_existed(second_ci_id)
+
+        existed = CIRelation.get_by(first_ci_id=first_ci_id,
+                                    second_ci_id=second_ci_id,
+                                    to_dict=False,
+                                    first=True)
+        if existed is not None:
+            if existed.relation_type_id != relation_type_id:
+                existed.update(relation_type_id=relation_type_id)
+
+                CIRelationHistoryManager().add(existed, OperateType.UPDATE)
+        else:
+            if relation_type_id is None:
+                type_relation = CITypeRelation.get_by(parent_id=first_ci.type_id,
+                                                      child_id=second_ci.type_id,
+                                                      first=True,
+                                                      to_dict=False)
+                relation_type_id = type_relation and type_relation.relation_type_id
+                relation_type_id or abort(404, "Relation {0} <-> {1} is not found".format(
+                    first_ci.ci_type.name, second_ci.ci_type.name))
+
+            existed = CIRelation.create(first_ci_id=first_ci_id,
+                                        second_ci_id=second_ci_id,
+                                        relation_type_id=relation_type_id)
+
+            CIRelationHistoryManager().add(existed, OperateType.ADD)
+
+            ci_relation_cache.apply_async(args=(first_ci_id, second_ci_id), queue=CMDB_QUEUE)
+
+        if more is not None:
+            existed.upadte(more=more)
+
+        return existed.id
+
+    @staticmethod
+    def delete(cr_id):
+        cr = CIRelation.get_by_id(cr_id) or abort(404, "CIRelation <{0}> is not existed".format(cr_id))
+        cr.delete()
+
+        his_manager = CIRelationHistoryManager()
+        his_manager.add(cr, operate_type=OperateType.DELETE)
+
+        ci_relation_delete.apply_async(args=(cr.first_ci_id, cr.second_ci_id), queue=CMDB_QUEUE)
+
+        return cr_id
+
+    @classmethod
+    def delete_2(cls, first_ci_id, second_ci_id):
+        cr = CIRelation.get_by(first_ci_id=first_ci_id,
+                               second_ci_id=second_ci_id,
+                               to_dict=False,
+                               first=True)
+
+        ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id), queue=CMDB_QUEUE)
+
+        return cls.delete(cr.id)

cmdb-api/api/lib/cmdb/ci_type.py

@@ -0,0 +1,443 @@
+# -*- coding:utf-8 -*- 
+
+
+from flask import abort
+from flask import current_app
+
+from api.extensions import db
+from api.lib.cmdb.attribute import AttributeManager
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.cache import CITypeAttributeCache
+from api.lib.cmdb.cache import CITypeCache
+from api.lib.decorator import kwargs_required
+from api.models.cmdb import CI
+from api.models.cmdb import CIType
+from api.models.cmdb import CITypeAttribute
+from api.models.cmdb import CITypeAttributeGroup
+from api.models.cmdb import CITypeAttributeGroupItem
+from api.models.cmdb import CITypeGroup
+from api.models.cmdb import CITypeGroupItem
+from api.models.cmdb import CITypeRelation
+from api.models.cmdb import PreferenceTreeView
+from api.models.cmdb import PreferenceShowAttributes
+
+
+class CITypeManager(object):
+    """
+    manage CIType
+    """
+
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def get_name_by_id(type_id):
+        return CITypeCache.get(type_id).name
+
+    @staticmethod
+    def check_is_existed(key):
+        return CITypeCache.get(key) or abort(404, "CIType <{0}> is not existed".format(key))
+
+    @staticmethod
+    def get_ci_types(type_name=None):
+        ci_types = CIType.get_by() if type_name is None else CIType.get_by_like(name=type_name)
+        res = list()
+        for type_dict in ci_types:
+            type_dict["unique_key"] = AttributeCache.get(type_dict["unique_id"]).name
+            res.append(type_dict)
+        return res
+
+    @staticmethod
+    def query(_type):
+        ci_type = CITypeCache.get(_type) or abort(404, "CIType <{0}> is not found".format(_type))
+        return ci_type.to_dict()
+
+    @classmethod
+    @kwargs_required("name")
+    def add(cls, **kwargs):
+        unique_key = kwargs.pop("unique_key", None)
+        unique_key = AttributeCache.get(unique_key) or abort(404, "Unique key is not defined")
+
+        CIType.get_by(name=kwargs['name']) and abort(404, "CIType <{0}> is already existed".format(kwargs.get("name")))
+
+        kwargs["alias"] = kwargs["name"] if not kwargs.get("alias") else kwargs["alias"]
+
+        kwargs["unique_id"] = unique_key.id
+        ci_type = CIType.create(**kwargs)
+
+        CITypeAttributeManager.add(ci_type.id, [unique_key.id], is_required=True)
+
+        CITypeCache.clean(ci_type.name)
+
+        if current_app.config.get("USE_ACL"):
+            from api.lib.perm.acl.acl import ACLManager
+            from api.lib.cmdb.const import ResourceTypeEnum, RoleEnum, PermEnum
+            ACLManager().add_resource(ci_type.name, ResourceTypeEnum.CI)
+            ACLManager().grant_resource_to_role(ci_type.name,
+                                                RoleEnum.CMDB_READ_ALL,
+                                                ResourceTypeEnum.CI,
+                                                permissions=[PermEnum.READ])
+
+        return ci_type.id
+
+    @classmethod
+    def update(cls, type_id, **kwargs):
+
+        ci_type = cls.check_is_existed(type_id)
+
+        unique_key = kwargs.pop("unique_key", None)
+        unique_key = AttributeCache.get(unique_key)
+        if unique_key is not None:
+            kwargs["unique_id"] = unique_key.id
+            type_attr = CITypeAttribute.get_by(type_id=type_id,
+                                               attr_id=unique_key.id,
+                                               first=True,
+                                               to_dict=False)
+            if type_attr is None:
+                CITypeAttributeManager.add(type_id, [unique_key.id], is_required=True)
+
+        ci_type.update(**kwargs)
+
+        CITypeCache.clean(type_id)
+
+        return type_id
+
+    @classmethod
+    def set_enabled(cls, type_id, enabled=True):
+        ci_type = cls.check_is_existed(type_id)
+        ci_type.update(enabled=enabled)
+        return type_id
+
+    @classmethod
+    def delete(cls, type_id):
+        ci_type = cls.check_is_existed(type_id)
+
+        if CI.get_by(type_id=type_id, first=True, to_dict=False) is not None:
+            return abort(400, "cannot delete, because CI instance exists")
+
+        for item in CITypeRelation.get_by(parent_id=type_id, to_dict=False):
+            item.soft_delete()
+
+        for item in CITypeRelation.get_by(child_id=type_id, to_dict=False):
+            item.soft_delete()
+
+        for item in PreferenceTreeView.get_by(type_id=type_id, to_dict=False):
+            item.soft_delete()
+
+        for item in PreferenceShowAttributes.get_by(type_id=type_id, to_dict=False):
+            item.soft_delete()
+
+        ci_type.soft_delete()
+
+        CITypeCache.clean(type_id)
+
+        if current_app.config.get("USE_ACL"):
+            from api.lib.perm.acl.acl import ACLManager
+            from api.lib.cmdb.const import ResourceTypeEnum, RoleEnum, PermEnum
+            ACLManager().del_resource(ci_type.name, ResourceTypeEnum.CI)
+
+
+class CITypeGroupManager(object):
+    @staticmethod
+    def get(need_other=None):
+        groups = CITypeGroup.get_by()
+        group_types = set()
+        for group in groups:
+            for t in sorted(CITypeGroupItem.get_by(group_id=group['id']), key=lambda x: x['order']):
+                group.setdefault("ci_types", []).append(CITypeCache.get(t['type_id']).to_dict())
+                group_types.add(t["type_id"])
+
+        if need_other:
+            ci_types = CITypeManager.get_ci_types()
+            other_types = dict(ci_types=[ci_type for ci_type in ci_types if ci_type["id"] not in group_types])
+            groups.append(other_types)
+
+        return groups
+
+    @staticmethod
+    def add(name):
+        CITypeGroup.get_by(name=name, first=True) and abort(400, "Group {0} does exist".format(name))
+        return CITypeGroup.create(name=name)
+
+    @staticmethod
+    def update(gid, name, type_ids):
+        """
+        update all
+        :param gid: 
+        :param name: 
+        :param type_ids: 
+        :return: 
+        """
+        existed = CITypeGroup.get_by_id(gid) or abort(404, "Group <{0}> does not exist".format(gid))
+        if name is not None:
+            existed.update(name=name)
+
+        for idx, type_id in enumerate(type_ids):
+
+            item = CITypeGroupItem.get_by(group_id=gid, type_id=type_id, first=True, to_dict=False)
+            if item is not None:
+                item.update(order=idx)
+            else:
+                CITypeGroupItem.create(group_id=gid, type_id=type_id, order=idx)
+
+    @staticmethod
+    def delete(gid):
+        existed = CITypeGroup.get_by_id(gid) or abort(404, "Group <{0}> does not exist".format(gid))
+
+        items = CITypeGroupItem.get_by(group_id=gid, to_dict=False)
+        for item in items:
+            item.soft_delete()
+
+        existed.soft_delete()
+
+
+class CITypeAttributeManager(object):
+    """
+    manage CIType's attributes, include query, add, update, delete
+    """
+
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def get_attr_names_by_type_id(type_id):
+        return [AttributeCache.get(attr.attr_id).name for attr in CITypeAttributeCache.get(type_id)]
+
+    @staticmethod
+    def get_attributes_by_type_id(type_id):
+        attrs = CITypeAttributeCache.get(type_id)
+        result = list()
+        for attr in sorted(attrs, key=lambda x: (x.order, x.id)):
+            attr_dict = AttributeManager().get_attribute(attr.attr_id)
+            attr_dict["is_required"] = attr.is_required
+            attr_dict["order"] = attr.order
+            attr_dict["default_show"] = attr.default_show
+            result.append(attr_dict)
+        return result
+
+    @staticmethod
+    def _check(type_id, attr_ids):
+        CITypeManager.check_is_existed(type_id)
+
+        if not attr_ids or not isinstance(attr_ids, list):
+            return abort(400, "Attributes are required")
+
+        for attr_id in attr_ids:
+            AttributeCache.get(attr_id) or abort(404, "Attribute <{0}> is not existed".format(attr_id))
+
+    @classmethod
+    def add(cls, type_id, attr_ids=None, **kwargs):
+        """
+        add attributes to CIType
+        :param type_id: 
+        :param attr_ids: list
+        :param kwargs: 
+        :return: 
+        """
+        cls._check(type_id, attr_ids)
+
+        for attr_id in attr_ids:
+            existed = CITypeAttribute.get_by(type_id=type_id,
+                                             attr_id=attr_id,
+                                             first=True,
+                                             to_dict=False)
+            if existed is not None:
+                continue
+
+            current_app.logger.debug(attr_id)
+            CITypeAttribute.create(type_id=type_id, attr_id=attr_id, **kwargs)
+
+        CITypeAttributeCache.clean(type_id)
+
+    @classmethod
+    def update(cls, type_id, attributes):
+        """
+        update attributes to CIType
+        :param type_id: 
+        :param attributes: list
+        :return: 
+        """
+        cls._check(type_id, [i.get('attr_id') for i in attributes])
+
+        for attr in attributes:
+            existed = CITypeAttribute.get_by(type_id=type_id,
+                                             attr_id=attr.get("attr_id"),
+                                             first=True,
+                                             to_dict=False)
+            if existed is None:
+                continue
+
+            existed.update(**attr)
+
+        CITypeAttributeCache.clean(type_id)
+
+    @classmethod
+    def delete(cls, type_id, attr_ids=None):
+        """
+        delete attributes from CIType
+        :param type_id: 
+        :param attr_ids: list
+        :return: 
+        """
+        cls._check(type_id, attr_ids)
+
+        for attr_id in attr_ids:
+            existed = CITypeAttribute.get_by(type_id=type_id,
+                                             attr_id=attr_id,
+                                             first=True,
+                                             to_dict=False)
+            if existed is not None:
+                existed.soft_delete()
+
+        CITypeAttributeCache.clean(type_id)
+
+
+class CITypeRelationManager(object):
+    """
+    manage relation between CITypes
+    """
+
+    @staticmethod
+    def get():
+        res = CITypeRelation.get_by(to_dict=False)
+        for idx, item in enumerate(res):
+            _item = item.to_dict()
+            res[idx] = _item
+            res[idx]['parent'] = item.parent.to_dict()
+            res[idx]['child'] = item.child.to_dict()
+            res[idx]['relation_type'] = item.relation_type.to_dict()
+
+        return res
+
+    @staticmethod
+    def get_child_type_ids(type_id, level):
+        ids = [type_id]
+        query = db.session.query(CITypeRelation).filter(CITypeRelation.deleted.is_(False))
+        for _ in range(0, level):
+            ids = [i.child_id for i in query.filter(CITypeRelation.parent_id.in_(ids))]
+
+        return ids
+
+    @staticmethod
+    def _wrap_relation_type_dict(type_id, relation_inst):
+        ci_type_dict = CITypeCache.get(type_id).to_dict()
+        ci_type_dict["ctr_id"] = relation_inst.id
+        ci_type_dict["attributes"] = CITypeAttributeManager.get_attributes_by_type_id(ci_type_dict["id"])
+        ci_type_dict["relation_type"] = relation_inst.relation_type.name
+        return ci_type_dict
+
+    @classmethod
+    def get_children(cls, parent_id):
+        children = CITypeRelation.get_by(parent_id=parent_id, to_dict=False)
+
+        return [cls._wrap_relation_type_dict(child.child_id, child) for child in children]
+
+    @classmethod
+    def get_parents(cls, child_id):
+        parents = CITypeRelation.get_by(child_id=child_id, to_dict=False)
+
+        return [cls._wrap_relation_type_dict(parent.parent_id, parent) for parent in parents]
+
+    @staticmethod
+    def _get(parent_id, child_id):
+        return CITypeRelation.get_by(parent_id=parent_id,
+                                     child_id=child_id,
+                                     to_dict=False,
+                                     first=True)
+
+    @classmethod
+    def add(cls, parent, child, relation_type_id):
+        p = CITypeManager.check_is_existed(parent)
+        c = CITypeManager.check_is_existed(child)
+
+        existed = cls._get(p.id, c.id)
+        if existed is not None:
+            existed.update(relation_type_id=relation_type_id)
+        else:
+            existed = CITypeRelation.create(parent_id=p.id,
+                                            child_id=c.id,
+                                            relation_type_id=relation_type_id)
+        return existed.id
+
+    @staticmethod
+    def delete(_id):
+        ctr = CITypeRelation.get_by_id(_id) or abort(404, "Type relation <{0}> is not found".format(_id))
+        ctr.soft_delete()
+
+    @classmethod
+    def delete_2(cls, parent, child):
+        ctr = cls._get(parent, child)
+        return cls.delete(ctr.id)
+
+
+class CITypeAttributeGroupManager(object):
+    @staticmethod
+    def get_by_type_id(type_id, need_other=None):
+        groups = CITypeAttributeGroup.get_by(type_id=type_id)
+        groups = sorted(groups, key=lambda x: x["order"])
+        grouped = list()
+        for group in groups:
+            items = CITypeAttributeGroupItem.get_by(group_id=group["id"], to_dict=False)
+            items = sorted(items, key=lambda x: x.order)
+            group["attributes"] = [AttributeCache.get(i.attr_id).to_dict() for i in items]
+            grouped.extend([i.attr_id for i in items])
+
+        if need_other is not None:
+            grouped = set(grouped)
+            attributes = CITypeAttributeManager.get_attributes_by_type_id(type_id)
+            other_attributes = [attr for attr in attributes if attr["id"] not in grouped]
+            groups.append(dict(attributes=other_attributes))
+
+        return groups
+
+    @staticmethod
+    def create_or_update(type_id, name, attr_order, group_order=0):
+        """
+        create or update
+        :param type_id:
+        :param name:
+        :param group_order: group order
+        :param attr_order:
+        :return:
+        """
+        existed = CITypeAttributeGroup.get_by(type_id=type_id, name=name, first=True, to_dict=False)
+        existed = existed or CITypeAttributeGroup.create(type_id=type_id, name=name, order=group_order)
+        existed.update(order=group_order)
+        attr_order = dict(attr_order)
+        current_app.logger.info(attr_order)
+        existed_items = CITypeAttributeGroupItem.get_by(group_id=existed.id, to_dict=False)
+        for item in existed_items:
+            if item.attr_id not in attr_order:
+                item.soft_delete()
+            else:
+                item.update(order=attr_order[item.attr_id])
+
+        existed_items = {item.attr_id: 1 for item in existed_items}
+        for attr_id, order in attr_order.items():
+            if attr_id not in existed_items:
+                CITypeAttributeGroupItem.create(group_id=existed.id, attr_id=attr_id, order=order)
+
+        return existed
+
+    @classmethod
+    def update(cls, group_id, name, attr_order, group_order=0):
+        group = CITypeAttributeGroup.get_by_id(group_id) or abort(404, "Group <{0}> does not exist".format(group_id))
+        other = CITypeAttributeGroup.get_by(type_id=group.type_id, name=name, first=True, to_dict=False)
+        if other is not None and other.id != group.id:
+            return abort(400, "Group <{0}> duplicate".format(name))
+        if name is not None:
+            group.update(name=name)
+
+        cls.create_or_update(group.type_id, name, attr_order, group_order)
+
+    @staticmethod
+    def delete(group_id):
+        group = CITypeAttributeGroup.get_by_id(group_id) \
+                or abort(404, "AttributeGroup <{0}> does not exist".format(group_id))
+        group.soft_delete()
+
+        items = CITypeAttributeGroupItem.get_by(group_id=group_id, to_dict=False)
+        for item in items:
+            item.soft_delete()
+
+        return group_id

cmdb-api/api/lib/cmdb/const.py

@@ -0,0 +1,59 @@
+# -*- coding:utf-8 -*- 
+
+
+from api.lib.utils import BaseEnum
+
+
+class ValueTypeEnum(BaseEnum):
+    INT = "0"
+    FLOAT = "1"
+    TEXT = "2"
+    DATETIME = "3"
+    DATE = "4"
+    TIME = "5"
+
+
+class CIStatusEnum(BaseEnum):
+    REVIEW = "0"
+    VALIDATE = "1"
+
+
+class ExistPolicy(BaseEnum):
+    REJECT = "reject"
+    NEED = "need"
+    IGNORE = "ignore"
+    REPLACE = "replace"
+
+
+class OperateType(BaseEnum):
+    ADD = "0"
+    DELETE = "1"
+    UPDATE = "2"
+
+
+class RetKey(BaseEnum):
+    ID = "id"
+    NAME = "name"
+    ALIAS = "alias"
+
+
+class ResourceTypeEnum(BaseEnum):
+    CI = "CIType"
+    RELATION_VIEW = "RelationView"
+
+
+class PermEnum(BaseEnum):
+    ADD = "add"
+    UPDATE = "update"
+    DELETE = "delete"
+    READ = "read"
+
+
+class RoleEnum(BaseEnum):
+    CONFIG = "admin"
+    CMDB_READ_ALL = "CMDB_READ_ALL"
+
+
+CMDB_QUEUE = "cmdb_async"
+REDIS_PREFIX_CI = "CMDB_CI"
+REDIS_PREFIX_CI_RELATION = "CMDB_CI_RELATION"

cmdb-api/api/lib/cmdb/history.py

@@ -0,0 +1,124 @@
+# -*- coding:utf-8 -*- 
+
+
+from flask import abort
+from flask import g
+
+from api.extensions import db
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.cache import RelationTypeCache
+from api.lib.cmdb.const import OperateType
+from api.lib.perm.acl.cache import UserCache
+from api.models.cmdb import Attribute
+from api.models.cmdb import AttributeHistory
+from api.models.cmdb import CIRelationHistory
+from api.models.cmdb import OperationRecord
+
+
+class AttributeHistoryManger(object):
+    @staticmethod
+    def get_records(start, end, username, page, page_size):
+        records = db.session.query(OperationRecord).filter(OperationRecord.deleted.is_(False))
+        numfound = db.session.query(db.func.count(OperationRecord.id)).filter(OperationRecord.deleted.is_(False))
+        if start:
+            records = records.filter(OperationRecord.created_at >= start)
+            numfound = numfound.filter(OperationRecord.created_at >= start)
+        if end:
+            records = records.filter(OperationRecord.created_at <= end)
+            numfound = records.filter(OperationRecord.created_at <= end)
+        if username:
+            user = UserCache.get(username)
+            if user:
+                records = records.filter(OperationRecord.uid == user.uid)
+            else:
+                return abort(404, "User <{0}> is not found".format(username))
+
+        records = records.order_by(-OperationRecord.id).offset(page_size * (page - 1)).limit(page_size).all()
+        total = len(records)
+        numfound = numfound.first()[0]
+        res = []
+        for record in records:
+            _res = record.to_dict()
+            _res["user"] = UserCache.get(_res.get("uid")).nickname or UserCache.get(_res.get("uid")).username
+
+            attr_history = AttributeHistory.get_by(record_id=_res.get("id"), to_dict=False)
+            _res["attr_history"] = [AttributeCache.get(h.attr_id).attr_alias for h in attr_history]
+
+            rel_history = CIRelationHistory.get_by(record_id=_res.get("id"), to_dict=False)
+            rel_statis = {}
+            for rel in rel_history:
+                if rel.operate_type not in rel_statis:
+                    rel_statis[rel.operate_type] = 1
+                else:
+                    rel_statis[rel.operate_type] += 1
+            _res["rel_history"] = rel_statis
+            res.append(_res)
+
+        return numfound, total, res
+
+    @staticmethod
+    def get_by_ci_id(ci_id):
+        res = db.session.query(AttributeHistory, Attribute, OperationRecord).join(
+            Attribute, Attribute.id == AttributeHistory.attr_id).join(
+            OperationRecord, OperationRecord.id == AttributeHistory.record_id).filter(
+            AttributeHistory.ci_id == ci_id).order_by(OperationRecord.id.desc())
+        return [dict(attr_name=i.Attribute.name,
+                     attr_alias=i.Attribute.alias,
+                     operate_type=i.AttributeHistory.operate_type,
+                     username=UserCache.get(i.OperationRecord.uid).nickname,
+                     old=i.AttributeHistory.old,
+                     new=i.AttributeHistory.new,
+                     created_at=i.OperationRecord.created_at.strftime('%Y-%m-%d %H:%M:%S'),
+                     record_id=i.OperationRecord.id,
+                     hid=i.AttributeHistory.id
+                     ) for i in res]
+
+    @staticmethod
+    def get_record_detail(record_id):
+        from api.lib.cmdb.ci import CIManager
+
+        record = OperationRecord.get_by_id(record_id) or abort(404, "Record <{0}> is not found".format(record_id))
+
+        username = UserCache.get(record.uid).nickname or UserCache.get(record.uid).username
+        timestamp = record.created_at.strftime("%Y-%m-%d %H:%M:%S")
+        attr_history = AttributeHistory.get_By(record_id=record_id, to_dict=False)
+        rel_history = CIRelationHistory.get_by(record_id=record_id, to_dict=False)
+
+        attr_dict, rel_dict = dict(), {"add": [], "delete": []}
+        for attr_h in attr_history:
+            attr_dict[AttributeCache.get(attr_h.attr_id).alias] = dict(
+                old=attr_h.old,
+                new=attr_h.new,
+                operate_type=attr_h.operate_type)
+
+        for rel_h in rel_history:
+            first = CIManager.get_ci_by_id(rel_h.first_ci_id)
+            second = CIManager.get_ci_by_id(rel_h.second_ci_id)
+            rel_dict[rel_h.operate_type].append((first, RelationTypeCache.get(rel_h.relation_type_id).name, second))
+
+        return username, timestamp, attr_dict, rel_dict
+
+    @staticmethod
+    def add(ci_id, history_list):
+        record = OperationRecord.create(uid=g.user.uid)
+
+        for attr_id, operate_type, old, new in history_list or []:
+            AttributeHistory.create(attr_id=attr_id,
+                                    operate_type=operate_type,
+                                    old=old,
+                                    new=new,
+                                    ci_id=ci_id,
+                                    record_id=record.id)
+
+
+class CIRelationHistoryManager(object):
+    @staticmethod
+    def add(rel_obj, operate_type=OperateType.ADD):
+        record = OperationRecord.create(uid=g.user.uid)
+
+        CIRelationHistory.create(relation_id=rel_obj.id,
+                                 record_id=record.id,
+                                 operate_type=operate_type,
+                                 first_ci_id=rel_obj.first_ci_id,
+                                 second_ci_id=rel_obj.second_ci_id,
+                                 relation_type_id=rel_obj.relation_type_id)

cmdb-api/api/lib/cmdb/preference.py

@@ -0,0 +1,208 @@
+# -*- coding:utf-8 -*-
+
+
+import copy
+import json
+
+import six
+import toposort
+from flask import abort
+from flask import current_app
+from flask import g
+
+from api.extensions import db
+from api.lib.cmdb.attribute import AttributeManager
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.cache import CITypeAttributeCache
+from api.lib.cmdb.cache import CITypeCache
+from api.lib.cmdb.const import ResourceTypeEnum, RoleEnum, PermEnum
+from api.lib.exception import AbortException
+from api.lib.perm.acl.acl import ACLManager
+from api.models.cmdb import CITypeAttribute
+from api.models.cmdb import CITypeRelation
+from api.models.cmdb import PreferenceRelationView
+from api.models.cmdb import PreferenceShowAttributes
+from api.models.cmdb import PreferenceTreeView
+
+
+class PreferenceManager(object):
+    @staticmethod
+    def get_types(instance=False, tree=False):
+        types = db.session.query(PreferenceShowAttributes.type_id).filter(
+            PreferenceShowAttributes.uid == g.user.uid).filter(
+            PreferenceShowAttributes.deleted.is_(False)).group_by(PreferenceShowAttributes.type_id).all() \
+            if instance else []
+        tree_types = PreferenceTreeView.get_by(uid=g.user.uid, to_dict=False) if tree else []
+        type_ids = list(set([i.type_id for i in types + tree_types]))
+        return [CITypeCache.get(type_id).to_dict() for type_id in type_ids]
+
+    @staticmethod
+    def get_show_attributes(type_id):
+        if not isinstance(type_id, six.integer_types):
+            type_id = CITypeCache.get(type_id).id
+
+        attrs = db.session.query(PreferenceShowAttributes, CITypeAttribute.order).join(
+            CITypeAttribute, CITypeAttribute.attr_id == PreferenceShowAttributes.attr_id).filter(
+            PreferenceShowAttributes.uid == g.user.uid).filter(
+            PreferenceShowAttributes.type_id == type_id).filter(
+            PreferenceShowAttributes.deleted.is_(False)).filter(CITypeAttribute.deleted.is_(False)).filter(
+            CITypeAttribute.type_id == type_id).order_by(
+            CITypeAttribute.order).all()
+        result = [i.PreferenceShowAttributes.attr.to_dict() for i in attrs]
+        is_subscribed = True
+        if not attrs:
+            attrs = db.session.query(CITypeAttribute).filter(
+                CITypeAttribute.type_id == type_id).filter(
+                CITypeAttribute.deleted.is_(False)).filter(
+                CITypeAttribute.default_show.is_(True)).order_by(CITypeAttribute.order)
+            result = [i.attr.to_dict() for i in attrs]
+            is_subscribed = False
+
+        for i in result:
+            if i["is_choice"]:
+                i.update(dict(choice_value=AttributeManager.get_choice_values(i["id"], i["value_type"])))
+
+        return is_subscribed, result
+
+    @classmethod
+    def create_or_update_show_attributes(cls, type_id, attr_order):
+        existed_all = PreferenceShowAttributes.get_by(type_id=type_id, uid=g.user.uid, to_dict=False)
+        for _attr, order in attr_order:
+            attr = AttributeCache.get(_attr) or abort(404, "Attribute <{0}> does not exist".format(_attr))
+            existed = PreferenceShowAttributes.get_by(type_id=type_id,
+                                                      uid=g.user.uid,
+                                                      attr_id=attr.id,
+                                                      first=True,
+                                                      to_dict=False)
+            if existed is None:
+                PreferenceShowAttributes.create(type_id=type_id,
+                                                uid=g.user.uid,
+                                                attr_id=attr.id,
+                                                order=order)
+            else:
+                existed.update(order=order)
+
+        attr_dict = {int(i): j for i, j in attr_order}
+        for i in existed_all:
+            if i.attr_id not in attr_dict:
+                i.soft_delete()
+
+    @staticmethod
+    def get_tree_view():
+        res = PreferenceTreeView.get_by(uid=g.user.uid, to_dict=True)
+        for item in res:
+            if item["levels"]:
+                item.update(CITypeCache.get(item['type_id']).to_dict())
+                item.update(dict(levels=[AttributeCache.get(l).to_dict()
+                                         for l in item["levels"].split(",") if AttributeCache.get(l)]))
+
+        return res
+
+    @staticmethod
+    def create_or_update_tree_view(type_id, levels):
+        attrs = CITypeAttributeCache.get(type_id)
+        for idx, i in enumerate(levels):
+            for attr in attrs:
+                attr = AttributeCache.get(attr.attr_id)
+                if i == attr.id or i == attr.name or i == attr.alias:
+                    levels[idx] = str(attr.id)
+        levels = ",".join(levels)
+
+        existed = PreferenceTreeView.get_by(uid=g.user.uid, type_id=type_id, to_dict=False, first=True)
+        if existed is not None:
+            if not levels:
+                existed.soft_delete()
+                return existed
+            return existed.update(levels=levels)
+        elif levels:
+            return PreferenceTreeView.create(levels=levels, type_id=type_id, uid=g.user.uid)
+
+    @staticmethod
+    def get_relation_view():
+        _views = PreferenceRelationView.get_by(to_dict=True)
+        views = []
+        if current_app.config.get("USE_ACL"):
+            for i in _views:
+                try:
+                    if ACLManager().has_permission(i.get('name'),
+                                                   ResourceTypeEnum.RELATION_VIEW,
+                                                   PermEnum.READ):
+                        views.append(i)
+                except AbortException:
+                    pass
+        else:
+            views = _views
+
+        view2cr_ids = dict()
+        result = dict()
+        name2id = list()
+        for view in views:
+            view2cr_ids.setdefault(view['name'], []).extend(json.loads(view['cr_ids']))
+            name2id.append([view['name'], view['id']])
+
+        id2type = dict()
+        for view_name in view2cr_ids:
+            for i in view2cr_ids[view_name]:
+                id2type[i['parent_id']] = None
+                id2type[i['child_id']] = None
+            topo = {i['child_id']: {i['parent_id']} for i in view2cr_ids[view_name]}
+            leaf = list(set(toposort.toposort_flatten(topo)) - set([j for i in topo.values() for j in i]))
+
+            leaf2show_types = {i: [t['child_id'] for t in CITypeRelation.get_by(parent_id=i)] for i in leaf}
+            node2show_types = copy.deepcopy(leaf2show_types)
+
+            def _find_parent(_node_id):
+                parents = topo.get(_node_id, {})
+                for parent in parents:
+                    node2show_types.setdefault(parent, []).extend(node2show_types.get(_node_id, []))
+                    _find_parent(parent)
+                if not parents:
+                    return
+
+            for l in leaf:
+                _find_parent(l)
+
+            for node_id in node2show_types:
+                node2show_types[node_id] = [CITypeCache.get(i).to_dict() for i in set(node2show_types[node_id])]
+
+            result[view_name] = dict(topo=list(map(list, toposort.toposort(topo))),
+                                     topo_flatten=list(toposort.toposort_flatten(topo)),
+                                     leaf=leaf,
+                                     leaf2show_types=leaf2show_types,
+                                     node2show_types=node2show_types,
+                                     show_types=[CITypeCache.get(j).to_dict()
+                                                 for i in leaf2show_types.values() for j in i])
+
+        for type_id in id2type:
+            id2type[type_id] = CITypeCache.get(type_id).to_dict()
+
+        return result, id2type, sorted(name2id, key=lambda x: x[1])
+
+    @classmethod
+    def create_or_update_relation_view(cls, name, cr_ids):
+        if not cr_ids:
+            return abort(400, "Node must be selected")
+
+        existed = PreferenceRelationView.get_by(name=name, to_dict=False, first=True)
+        current_app.logger.debug(existed)
+        if existed is None:
+            PreferenceRelationView.create(name=name, cr_ids=json.dumps(cr_ids))
+
+            if current_app.config.get("USE_ACL"):
+                ACLManager().add_resource(name, ResourceTypeEnum.RELATION_VIEW)
+                ACLManager().grant_resource_to_role(name,
+                                                    RoleEnum.CMDB_READ_ALL,
+                                                    ResourceTypeEnum.RELATION_VIEW,
+                                                    permissions=[PermEnum.READ])
+
+        return cls.get_relation_view()
+
+    @staticmethod
+    def delete_relation_view(name):
+        for existed in PreferenceRelationView.get_by(name=name, to_dict=False):
+            existed.soft_delete()
+
+        if current_app.config.get("USE_ACL"):
+            ACLManager().del_resource(name, ResourceTypeEnum.RELATION_VIEW)
+
+        return name

cmdb-api/api/lib/cmdb/relation_type.py

@@ -0,0 +1,37 @@
+# -*- coding:utf-8 -*-
+
+
+from flask import abort
+
+from api.models.cmdb import RelationType
+
+
+class RelationTypeManager(object):
+    @staticmethod
+    def get_all():
+        return RelationType.get_by(to_dict=False)
+
+    @classmethod
+    def get_names(cls):
+        return [i.name for i in cls.get_all()]
+
+    @classmethod
+    def get_pairs(cls):
+        return [(i.id, i.name) for i in cls.get_all()]
+
+    @staticmethod
+    def add(name):
+        RelationType.get_by(name=name, first=True, to_dict=False) and abort(400, "It's already existed")
+        return RelationType.create(name=name)
+
+    @staticmethod
+    def update(rel_id, name):
+        existed = RelationType.get_by_id(rel_id) or abort(404, "RelationType <{0}> does not exist".format(rel_id))
+
+        return existed.update(name=name)
+
+    @staticmethod
+    def delete(rel_id):
+        existed = RelationType.get_by_id(rel_id) or abort(404, "RelationType <{0}> does not exist".format(rel_id))
+
+        existed.soft_delete()

cmdb-api/api/lib/cmdb/search/__init__.py

@@ -0,0 +1,11 @@
+# -*- coding:utf-8 -*-
+
+__all__ = ['ci', 'ci_relation', 'SearchError']
+
+
+class SearchError(Exception):
+    def __init__(self, v):
+        self.v = v
+
+    def __str__(self):
+        return self.v

cmdb-api/api/lib/cmdb/search/ci/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding:utf-8 -*-
+
+__all__ = ['db', 'es']

cmdb-api/api/lib/cmdb/search/ci/db/__init__.py

@@ -0,0 +1 @@
+# -*- coding:utf-8 -*-

cmdb-api/api/lib/cmdb/search/ci/db/query_sql.py

@@ -0,0 +1,66 @@
+# -*- coding:utf-8 -*- 
+
+from __future__ import unicode_literals
+
+
+QUERY_CIS_BY_VALUE_TABLE = """
+    SELECT attr.name AS attr_name,
+          attr.alias AS attr_alias,
+          attr.value_type,
+          attr.is_list,
+          c_cis.type_id,
+          {0}.ci_id,
+          {0}.attr_id,
+          {0}.value
+   FROM {0}
+   INNER JOIN c_cis ON {0}.ci_id=c_cis.id
+   AND {0}.`ci_id` IN ({1})
+   INNER JOIN c_attributes as attr ON attr.id = {0}.attr_id
+"""
+
+#  {2}: value_table
+QUERY_CIS_BY_IDS = """
+    SELECT A.ci_id,
+           A.type_id,
+           A.attr_id,
+           A.attr_name,
+           A.attr_alias,
+           A.value,
+           A.value_type,
+           A.is_list
+    FROM
+      ({1}) AS A {0}
+       ORDER BY A.ci_id;
+"""
+
+FACET_QUERY1 = """
+    SELECT {0}.value,
+           count({0}.ci_id)
+    FROM {0}
+    INNER JOIN c_attributes AS attr ON attr.id={0}.attr_id
+    WHERE attr.name="{1}"
+    GROUP BY {0}.ci_id;
+"""
+
+FACET_QUERY = """
+    SELECT {0}.value,
+           count({0}.ci_id)
+    FROM {0}
+    INNER JOIN ({1}) AS F ON F.ci_id={0}.ci_id
+    WHERE {0}.attr_id={2:d}
+    GROUP BY {0}.value
+"""
+
+QUERY_CI_BY_ATTR_NAME = """
+    SELECT {0}.ci_id
+    FROM {0}
+    WHERE {0}.attr_id={1:d}
+      AND {0}.value {2}
+"""
+
+QUERY_CI_BY_TYPE = """
+    SELECT c_cis.id AS ci_id
+    FROM c_cis
+    WHERE c_cis.type_id in ({0})
+"""
+

cmdb-api/api/lib/cmdb/search/ci/db/search.py

@@ -0,0 +1,367 @@
+# -*- coding:utf-8 -*- 
+
+
+from __future__ import unicode_literals
+
+import time
+
+from flask import current_app
+
+from api.extensions import db
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.cache import CITypeCache
+from api.lib.cmdb.ci import CIManager
+from api.lib.cmdb.const import RetKey
+from api.lib.cmdb.const import ValueTypeEnum
+from api.lib.cmdb.search import SearchError
+from api.lib.cmdb.search.ci.db.query_sql import FACET_QUERY
+from api.lib.cmdb.search.ci.db.query_sql import QUERY_CI_BY_ATTR_NAME
+from api.lib.cmdb.search.ci.db.query_sql import QUERY_CI_BY_TYPE
+from api.lib.cmdb.utils import TableMap
+from api.lib.utils import handle_arg_list
+from api.models.cmdb import CI
+
+
+class Search(object):
+    def __init__(self, query=None,
+                 fl=None,
+                 facet_field=None,
+                 page=1,
+                 ret_key=RetKey.NAME,
+                 count=1,
+                 sort=None,
+                 ci_ids=None):
+        self.orig_query = query
+        self.fl = fl
+        self.facet_field = facet_field
+        self.page = page
+        self.ret_key = ret_key
+        self.count = count
+        self.sort = sort
+        self.ci_ids = ci_ids or []
+        self.query_sql = ""
+        self.type_id_list = []
+        self.only_type_query = False
+
+    @staticmethod
+    def _operator_proc(key):
+        operator = "&"
+        if key.startswith("+"):
+            key = key[1:].strip()
+        elif key.startswith("-"):
+            operator = "|"
+            key = key[1:].strip()
+        elif key.startswith("~"):
+            operator = "~"
+            key = key[1:].strip()
+
+        return operator, key
+
+    def _attr_name_proc(self, key):
+        operator, key = self._operator_proc(key)
+
+        if key in ('ci_type', 'type', '_type'):
+            return '_type', ValueTypeEnum.TEXT, operator, None
+
+        if key in ('id', 'ci_id', '_id'):
+            return '_id', ValueTypeEnum.TEXT, operator, None
+
+        attr = AttributeCache.get(key)
+        if attr:
+            return attr.name, attr.value_type, operator, attr
+        else:
+            raise SearchError("{0} is not existed".format(key))
+
+    def _type_query_handler(self, v):
+        new_v = v[1:-1].split(";") if v.startswith("(") and v.endswith(")") else [v]
+        for _v in new_v:
+            ci_type = CITypeCache.get(_v)
+            if ci_type is not None:
+                self.type_id_list.append(str(ci_type.id))
+
+        if self.type_id_list:
+            type_ids = ",".join(self.type_id_list)
+            _query_sql = QUERY_CI_BY_TYPE.format(type_ids)
+            if self.only_type_query:
+                return _query_sql
+            else:
+                return ""
+        return ""
+
+    @staticmethod
+    def _in_query_handler(attr, v):
+        new_v = v[1:-1].split(";")
+        table_name = TableMap(attr_name=attr.name).table_name
+        in_query = " OR {0}.value ".format(table_name).join(['LIKE "{0}"'.format(_v.replace("*", "%")) for _v in new_v])
+        _query_sql = QUERY_CI_BY_ATTR_NAME.format(table_name, attr.id, in_query)
+        return _query_sql
+
+    @staticmethod
+    def _range_query_handler(attr, v):
+        start, end = [x.strip() for x in v[1:-1].split("_TO_")]
+        table_name = TableMap(attr_name=attr.name).table_name
+        range_query = "BETWEEN '{0}' AND '{1}'".format(start.replace("*", "%"), end.replace("*", "%"))
+        _query_sql = QUERY_CI_BY_ATTR_NAME.format(table_name, attr.id, range_query)
+        return _query_sql
+
+    @staticmethod
+    def _comparison_query_handler(attr, v):
+        table_name = TableMap(attr_name=attr.name).table_name
+        if v.startswith(">=") or v.startswith("<="):
+            comparison_query = "{0} '{1}'".format(v[:2], v[2:].replace("*", "%"))
+        else:
+            comparison_query = "{0} '{1}'".format(v[0], v[1:].replace("*", "%"))
+        _query_sql = QUERY_CI_BY_ATTR_NAME.format(table_name, attr.id, comparison_query)
+        return _query_sql
+
+    @staticmethod
+    def __sort_by(field):
+        field = field or ""
+        sort_type = "ASC"
+        if field.startswith("+"):
+            field = field[1:]
+        elif field.startswith("-"):
+            field = field[1:]
+            sort_type = "DESC"
+        return field, sort_type
+
+    def __sort_by_id(self, sort_type, query_sql):
+        ret_sql = "SELECT SQL_CALC_FOUND_ROWS DISTINCT B.ci_id FROM ({0}) AS B {1}"
+
+        if self.only_type_query:
+            return ret_sql.format(query_sql, "ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format(
+                (self.page - 1) * self.count, sort_type, self.count))
+
+        elif self.type_id_list:
+            self.query_sql = "SELECT B.ci_id FROM ({0}) AS B {1}".format(
+                query_sql,
+                "INNER JOIN c_cis on c_cis.id=B.ci_id WHERE c_cis.type_id IN ({0}) ".format(
+                    ",".join(self.type_id_list)))
+
+            return ret_sql.format(
+                query_sql,
+                "INNER JOIN c_cis on c_cis.id=B.ci_id WHERE c_cis.type_id IN ({3}) "
+                "ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format(
+                    (self.page - 1) * self.count, sort_type, self.count, ",".join(self.type_id_list)))
+
+        else:
+            self.query_sql = "SELECT B.ci_id FROM ({0}) AS B {1}".format(
+                query_sql,
+                "INNER JOIN c_cis on c_cis.id=B.ci_id ")
+
+            return ret_sql.format(
+                query_sql,
+                "INNER JOIN c_cis on c_cis.id=B.ci_id "
+                "ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format((self.page - 1) * self.count, sort_type, self.count))
+
+    def __sort_by_field(self, field, sort_type, query_sql):
+        attr = AttributeCache.get(field)
+        attr_id = attr.id
+
+        table_name = TableMap(attr_name=attr.name).table_name
+        _v_query_sql = """SELECT {0}.ci_id, {1}.value 
+                          FROM ({2}) AS {0} INNER JOIN {1} ON {1}.ci_id = {0}.ci_id
+                          WHERE {1}.attr_id = {3}""".format("ALIAS", table_name, query_sql, attr_id)
+        new_table = _v_query_sql
+
+        if self.only_type_query or not self.type_id_list:
+            return "SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id, C.value " \
+                   "FROM ({0}) AS C " \
+                   "ORDER BY C.value {2} " \
+                   "LIMIT {1:d}, {3};".format(new_table, (self.page - 1) * self.count, sort_type, self.count)
+
+        elif self.type_id_list:
+            self.query_sql = """SELECT C.ci_id
+                                FROM ({0}) AS C
+                                INNER JOIN c_cis on c_cis.id=C.ci_id
+                                WHERE c_cis.type_id IN ({1})""".format(new_table, ",".join(self.type_id_list))
+
+            return """SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id, C.value
+                      FROM ({0}) AS C
+                      INNER JOIN c_cis on c_cis.id=C.ci_id
+                      WHERE c_cis.type_id IN ({4})
+                      ORDER BY C.value {2}
+                      LIMIT {1:d}, {3};""".format(new_table,
+                                                  (self.page - 1) * self.count,
+                                                  sort_type, self.count,
+                                                  ",".join(self.type_id_list))
+
+    def _sort_query_handler(self, field, query_sql):
+
+        field, sort_type = self.__sort_by(field)
+
+        if field in ("_id", "ci_id") or not field:
+            return self.__sort_by_id(sort_type, query_sql)
+        else:
+            return self.__sort_by_field(field, sort_type, query_sql)
+
+    @staticmethod
+    def _wrap_sql(operator, alias, _query_sql, query_sql):
+        if operator == "&":
+            query_sql = """SELECT * FROM ({0}) as {1}
+                           INNER JOIN ({2}) as {3} USING(ci_id)""".format(query_sql, alias, _query_sql, alias + "A")
+
+        elif operator == "|":
+            query_sql = "SELECT * FROM ({0}) as {1} UNION ALL ({2})".format(query_sql, alias, _query_sql)
+
+        elif operator == "~":
+            query_sql = """SELECT * FROM ({0}) as {1} LEFT JOIN ({2}) as {3} USING(ci_id) 
+                           WHERE {3}.ci_id is NULL""".format(query_sql, alias, _query_sql, alias + "A")
+
+        return query_sql
+
+    def _execute_sql(self, query_sql):
+        v_query_sql = self._sort_query_handler(self.sort, query_sql)
+
+        start = time.time()
+        execute = db.session.execute
+        current_app.logger.debug(v_query_sql)
+        res = execute(v_query_sql).fetchall()
+        end_time = time.time()
+        current_app.logger.debug("query ci ids time is: {0}".format(end_time - start))
+
+        numfound = execute("SELECT FOUND_ROWS();").fetchall()[0][0]
+        current_app.logger.debug("statistics ci ids time is: {0}".format(time.time() - end_time))
+
+        return numfound, res
+
+    def __confirm_type_first(self, queries):
+        for q in queries:
+            if q.startswith("_type"):
+                queries.remove(q)
+                queries.insert(0, q)
+                if len(queries) == 1 or queries[1].startswith("-") or queries[1].startswith("~"):
+                    self.only_type_query = True
+        return queries
+
+    def __query_build_by_field(self, queries):
+        query_sql, alias, operator = "", "A", "&"
+        is_first, only_type_query_special = True, True
+
+        for q in queries:
+            _query_sql = ""
+            if ":" in q:
+                k = q.split(":")[0].strip()
+                v = ":".join(q.split(":")[1:]).strip()
+                current_app.logger.debug(v)
+                field, field_type, operator, attr = self._attr_name_proc(k)
+                if field == "_type":
+                    _query_sql = self._type_query_handler(v)
+                    current_app.logger.debug(_query_sql)
+                elif field == "_id":  # exclude all others
+                    ci = CI.get_by_id(v)
+                    if ci is not None:
+                        return 1, [str(v)]
+                elif field:
+                    if attr is None:
+                        raise SearchError("{0} is not found".format(field))
+
+                    # in query
+                    if v.startswith("(") and v.endswith(")"):
+                        _query_sql = self._in_query_handler(attr, v)
+                    # range query
+                    elif v.startswith("[") and v.endswith("]") and "_TO_" in v:
+                        _query_sql = self._range_query_handler(attr, v)
+                    # comparison query
+                    elif v.startswith(">=") or v.startswith("<=") or v.startswith(">") or v.startswith("<"):
+                        _query_sql = self._comparison_query_handler(attr, v)
+                    else:
+                        table_name = TableMap(attr_name=attr.name).table_name
+                        _query_sql = QUERY_CI_BY_ATTR_NAME.format(
+                            table_name, attr.id, 'LIKE "{0}"'.format(v.replace("*", "%")))
+                else:
+                    raise SearchError("argument q format invalid: {0}".format(q))
+            elif q:
+                raise SearchError("argument q format invalid: {0}".format(q))
+
+            if is_first and _query_sql and not self.only_type_query:
+                query_sql = "SELECT * FROM ({0}) AS {1}".format(_query_sql, alias)
+                is_first = False
+                alias += "A"
+            elif self.only_type_query and only_type_query_special:
+                is_first = False
+                only_type_query_special = False
+                query_sql = _query_sql
+            elif _query_sql:
+                query_sql = self._wrap_sql(operator, alias, _query_sql, query_sql)
+                alias += "AA"
+        return None, query_sql
+
+    def _filter_ids(self, query_sql):
+        if self.ci_ids:
+            return "SELECT * FROM ({0}) AS IN_QUERY WHERE IN_QUERY.ci_id IN ({1})".format(
+                query_sql, ",".join(list(map(str, self.ci_ids))))
+
+        return query_sql
+
+    def _query_build_raw(self):
+
+        queries = handle_arg_list(self.orig_query)
+        queries = self.__confirm_type_first(queries)
+        current_app.logger.debug(queries)
+
+        ret, query_sql = self.__query_build_by_field(queries)
+        if ret is not None:
+            return ret, query_sql
+
+        s = time.time()
+        if query_sql:
+            query_sql = self._filter_ids(query_sql)
+            self.query_sql = query_sql
+            current_app.logger.debug(query_sql)
+            numfound, res = self._execute_sql(query_sql)
+            current_app.logger.debug("query ci ids is: {0}".format(time.time() - s))
+            return numfound, [_res[0] for _res in res]
+
+        return 0, []
+
+    def _facet_build(self):
+        facet = {}
+        for f in self.facet_field:
+            k, field_type, _, attr = self._attr_name_proc(f)
+            if k:
+                table_name = TableMap(attr_name=k).table_name
+                query_sql = FACET_QUERY.format(table_name, self.query_sql, attr.id)
+                current_app.logger.debug(query_sql)
+                result = db.session.execute(query_sql).fetchall()
+                facet[k] = result
+
+        facet_result = dict()
+        for k, v in facet.items():
+            if not k.startswith('_'):
+                a = getattr(AttributeCache.get(k), self.ret_key)
+                facet_result[a] = [(f[0], f[1], a) for f in v]
+
+        return facet_result
+
+    def _fl_build(self):
+        _fl = list()
+        for f in self.fl:
+            k, _, _, _ = self._attr_name_proc(f)
+            if k:
+                _fl.append(k)
+
+        return _fl
+
+    def search(self):
+        numfound, ci_ids = self._query_build_raw()
+        ci_ids = list(map(str, ci_ids))
+
+        _fl = self._fl_build()
+
+        if self.facet_field and numfound:
+            facet = self._facet_build()
+        else:
+            facet = dict()
+
+        response, counter = [], {}
+        if ci_ids:
+            response = CIManager.get_cis_by_ids(ci_ids, ret_key=self.ret_key, fields=_fl)
+        for res in response:
+            ci_type = res.get("ci_type")
+            if ci_type not in counter.keys():
+                counter[ci_type] = 0
+            counter[ci_type] += 1
+        total = len(response)
+
+        return response, counter, total, self.page, numfound, facet

cmdb-api/api/lib/cmdb/search/ci/es/__init__.py

@@ -0,0 +1 @@
+# -*- coding:utf-8 -*-

cmdb-api/api/lib/cmdb/search/ci/es/search.py

@@ -0,0 +1,259 @@
+# -*- coding:utf-8 -*-
+
+
+from __future__ import unicode_literals
+
+from flask import current_app
+
+from api.extensions import es
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.const import RetKey
+from api.lib.cmdb.const import ValueTypeEnum
+from api.lib.cmdb.search import SearchError
+from api.lib.utils import handle_arg_list
+
+
+class Search(object):
+    def __init__(self, query=None,
+                 fl=None,
+                 facet_field=None,
+                 page=1,
+                 ret_key=RetKey.NAME,
+                 count=1,
+                 sort=None,
+                 ci_ids=None):
+        self.orig_query = query
+        self.fl = fl
+        self.facet_field = facet_field
+        self.page = page
+        self.ret_key = ret_key
+        self.count = count or current_app.config.get("DEFAULT_PAGE_COUNT")
+        self.sort = sort or "ci_id"
+        self.ci_ids = ci_ids or []
+
+        self.query = dict(query=dict(bool=dict(should=[], must=[], must_not=[])))
+
+    @staticmethod
+    def _operator_proc(key):
+        operator = "&"
+        if key.startswith("+"):
+            key = key[1:].strip()
+        elif key.startswith("-"):
+            operator = "|"
+            key = key[1:].strip()
+        elif key.startswith("~"):
+            operator = "~"
+            key = key[1:].strip()
+
+        return operator, key
+
+    def _operator2query(self, operator):
+        if operator == "&":
+            return self.query['query']['bool']['must']
+        elif operator == "|":
+            return self.query['query']['bool']['should']
+        else:
+            return self.query['query']['bool']['must_not']
+
+    def _attr_name_proc(self, key):
+        operator, key = self._operator_proc(key)
+
+        if key in ('ci_type', 'type', '_type'):
+            return 'ci_type', ValueTypeEnum.TEXT, operator
+
+        if key in ('id', 'ci_id', '_id'):
+            return 'ci_id', ValueTypeEnum.TEXT, operator
+
+        attr = AttributeCache.get(key)
+        if attr:
+            return attr.name, attr.value_type, operator
+        else:
+            raise SearchError("{0} is not existed".format(key))
+
+    def _in_query_handle(self, attr, v):
+        terms = v[1:-1].split(";")
+        operator = "|"
+        if attr in ('_type', 'ci_type', 'type_id') and terms and terms[0].isdigit():
+            attr = "type_id"
+            terms = map(int, terms)
+        current_app.logger.warning(terms)
+        for term in terms:
+            self._operator2query(operator).append({
+                "term": {
+                    attr: term
+                }
+            })
+
+    def _filter_ids(self):
+        if self.ci_ids:
+            self.query['query']['bool'].update(dict(filter=dict(terms=dict(ci_id=self.ci_ids))))
+
+    @staticmethod
+    def _digit(s):
+        if s.isdigit():
+            return int(float(s))
+        return s
+
+    def _range_query_handle(self, attr, v, operator):
+        left, right = v.split("_TO_")
+        left, right = left.strip()[1:], right.strip()[:-1]
+        self._operator2query(operator).append({
+            "range": {
+                attr: {
+                    "lte": self._digit(right),
+                    "gte": self._digit(left),
+                    "boost": 2.0
+                }
+            }
+        })
+
+    def _comparison_query_handle(self, attr, v, operator):
+        if v.startswith(">="):
+            _query = dict(gte=self._digit(v[2:]), boost=2.0)
+        elif v.startswith("<="):
+            _query = dict(lte=self._digit(v[2:]), boost=2.0)
+        elif v.startswith(">"):
+            _query = dict(gt=self._digit(v[1:]), boost=2.0)
+        elif v.startswith("<"):
+            _query = dict(lt=self._digit(v[1:]), boost=2.0)
+        else:
+            return
+
+        self._operator2query(operator).append({
+            "range": {
+                attr: _query
+            }
+        })
+
+    def _match_query_handle(self, attr, v, operator):
+        if "*" in v:
+            self._operator2query(operator).append({
+                "wildcard": {
+                    attr: v
+                }
+            })
+        else:
+            if attr == "ci_type" and v.isdigit():
+                attr = "type_id"
+            self._operator2query(operator).append({
+                "term": {
+                    attr: v
+                }
+            })
+
+    def __query_build_by_field(self, queries):
+
+        for q in queries:
+            if ":" in q:
+                k = q.split(":")[0].strip()
+                v = ":".join(q.split(":")[1:]).strip()
+                field_name, field_type, operator = self._attr_name_proc(k)
+                if field_name:
+                    # in query
+                    if v.startswith("(") and v.endswith(")"):
+                        self._in_query_handle(field_name, v)
+                    # range query
+                    elif v.startswith("[") and v.endswith("]") and "_TO_" in v:
+                        self._range_query_handle(field_name, v, operator)
+                    # comparison query
+                    elif v.startswith(">=") or v.startswith("<=") or v.startswith(">") or v.startswith("<"):
+                        self._comparison_query_handle(field_name, v, operator)
+                    else:
+                        self._match_query_handle(field_name, v, operator)
+                else:
+                    raise SearchError("argument q format invalid: {0}".format(q))
+            elif q:
+                raise SearchError("argument q format invalid: {0}".format(q))
+
+    def _query_build_raw(self):
+
+        queries = handle_arg_list(self.orig_query)
+
+        current_app.logger.debug(queries)
+
+        self.__query_build_by_field(queries)
+
+        self._paginate_build()
+
+        filter_path = self._fl_build()
+
+        self._sort_build()
+
+        self._facet_build()
+
+        self._filter_ids()
+
+        return es.read(self.query, filter_path=filter_path)
+
+    def _facet_build(self):
+        aggregations = dict(aggs={})
+        for field in self.facet_field:
+            attr = AttributeCache.get(field)
+            if not attr:
+                raise SearchError("Facet by <{0}> does not exist".format(field))
+            aggregations['aggs'].update({
+                field: {
+                    "terms": {
+                        "field": "{0}.keyword".format(field)
+                        if attr.value_type not in (ValueTypeEnum.INT, ValueTypeEnum.FLOAT) else field
+                    }
+                }
+            })
+
+        if aggregations['aggs']:
+            self.query.update(aggregations)
+
+    def _sort_build(self):
+        fields = list(filter(lambda x: x != "", (self.sort or "").split(",")))
+        sorts = []
+        for field in fields:
+            sort_type = "asc"
+            if field.startswith("+"):
+                field = field[1:]
+            elif field.startswith("-"):
+                field = field[1:]
+                sort_type = "desc"
+            else:
+                field = field
+            if field == "ci_id":
+                sorts.append({field: {"order": sort_type}})
+                continue
+
+            attr = AttributeCache.get(field)
+            if not attr:
+                raise SearchError("Sort by <{0}> does not exist".format(field))
+
+            sort_by = "{0}.keyword".format(field) \
+                if attr.value_type not in (ValueTypeEnum.INT, ValueTypeEnum.FLOAT) else field
+            sorts.append({sort_by: {"order": sort_type}})
+
+        self.query.update(dict(sort=sorts))
+
+    def _paginate_build(self):
+        self.query.update({"from": (self.page - 1) * self.count,
+                           "size": self.count})
+
+    def _fl_build(self):
+        return ['hits.hits._source.{0}'.format(i) for i in self.fl]
+
+    def search(self):
+        try:
+            numfound, cis, facet = self._query_build_raw()
+        except Exception as e:
+            current_app.logger.error(str(e))
+            raise SearchError("unknown search error")
+
+        total = len(cis)
+
+        counter = dict()
+        for ci in cis:
+            ci_type = ci.get("ci_type")
+            if ci_type not in counter.keys():
+                counter[ci_type] = 0
+            counter[ci_type] += 1
+
+        facet_ = dict()
+        for k in facet:
+            facet_[k] = [[i['key'], i['doc_count'], k] for i in facet[k]["buckets"]]
+
+        return cis, counter, total, self.page, numfound, facet_

cmdb-api/api/lib/cmdb/search/ci_relation/__init__.py

@@ -0,0 +1 @@
+# -*- coding:utf-8 -*-

cmdb-api/api/lib/cmdb/search/ci_relation/search.py

@@ -0,0 +1,108 @@
+# -*- coding:utf-8 -*-
+
+
+import json
+
+from flask import abort
+from flask import current_app
+
+from api.extensions import rd
+from api.lib.cmdb.ci_type import CITypeRelationManager
+from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
+from api.lib.cmdb.search.ci.db.search import Search as SearchFromDB
+from api.lib.cmdb.search.ci.es.search import Search as SearchFromES
+from api.models.cmdb import CI
+
+
+class Search(object):
+    def __init__(self, root_id,
+                 level=1,
+                 query=None,
+                 fl=None,
+                 facet_field=None,
+                 page=1,
+                 count=None,
+                 sort=None):
+        self.orig_query = query
+        self.fl = fl
+        self.facet_field = facet_field
+        self.page = page
+        self.count = count or current_app.config.get("DEFAULT_PAGE_COUNT")
+        self.sort = sort or ("ci_id" if current_app.config.get("USE_ES") else None)
+
+        self.root_id = root_id
+        self.level = level
+
+    def search(self):
+        ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
+        cis = [CI.get_by_id(_id) or abort(404, "CI <{0}> does not exist".format(_id)) for _id in ids]
+
+        merge_ids = []
+        for level in self.level:
+            ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
+            for _ in range(0, level):
+                _tmp = list(map(lambda x: list(json.loads(x).keys()),
+                                filter(lambda x: x is not None, rd.get(ids, REDIS_PREFIX_CI_RELATION) or [])))
+                ids = [j for i in _tmp for j in i]
+
+            merge_ids.extend(ids)
+
+        if not self.orig_query or ("_type:" not in self.orig_query
+                                   and "type_id:" not in self.orig_query
+                                   and "ci_type:" not in self.orig_query):
+            type_ids = []
+            for level in self.level:
+                for ci in cis:
+                    type_ids.extend(CITypeRelationManager.get_child_type_ids(ci.type_id, level))
+            type_ids = list(set(type_ids))
+            if self.orig_query:
+                self.orig_query = "_type:({0}),{1}".format(";".join(list(map(str, type_ids))), self.orig_query)
+            else:
+                self.orig_query = "_type:({0})".format(";".join(list(map(str, type_ids))))
+
+        if not merge_ids:
+            # cis, counter, total, self.page, numfound, facet_
+            return [], {}, 0, self.page, 0, {}
+
+        if current_app.config.get("USE_ES"):
+            return SearchFromES(self.orig_query,
+                                fl=self.fl,
+                                facet_field=self.facet_field,
+                                page=self.page,
+                                count=self.count,
+                                sort=self.sort,
+                                ci_ids=merge_ids).search()
+        else:
+            return SearchFromDB(self.orig_query,
+                                fl=self.fl,
+                                facet_field=self.facet_field,
+                                page=self.page,
+                                count=self.count,
+                                sort=self.sort,
+                                ci_ids=merge_ids).search()
+
+    def statistics(self, type_ids):
+        ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
+        for l in range(0, int(self.level)):
+            if l == 0:
+                _tmp = list(map(lambda x: list(json.loads(x).keys()),
+                                [i or '{}' for i in rd.get(ids, REDIS_PREFIX_CI_RELATION) or []]))
+            else:
+                for idx, i in enumerate(_tmp):
+                    if i:
+                        if type_ids and l == self.level - 1:
+                            __tmp = list(
+                                map(lambda x: list({_id: 1 for _id, type_id in json.loads(x).items()
+                                                    if type_id in type_ids}.keys()),
+                                    filter(lambda x: x is not None,
+                                           rd.get(i, REDIS_PREFIX_CI_RELATION) or [])))
+                        else:
+
+                            __tmp = list(map(lambda x: list(json.loads(x).keys()),
+                                             filter(lambda x: x is not None,
+                                                    rd.get(i, REDIS_PREFIX_CI_RELATION) or [])))
+                        _tmp[idx] = [j for i in __tmp for j in i]
+                    else:
+                        _tmp[idx] = []
+
+        return {_id: len(_tmp[idx]) for idx, _id in enumerate(ids)}

cmdb-api/api/lib/cmdb/utils.py

@@ -0,0 +1,116 @@
+# -*- coding:utf-8 -*-
+
+from __future__ import unicode_literals
+
+import datetime
+
+import six
+from markupsafe import escape
+
+import api.models.cmdb as model
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.const import ValueTypeEnum
+
+
+def string2int(x):
+    return int(float(x))
+
+
+def str2datetime(x):
+    try:
+        return datetime.datetime.strptime(x, "%Y-%m-%d")
+    except ValueError:
+        pass
+
+    return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")
+
+
+class ValueTypeMap(object):
+    deserialize = {
+        ValueTypeEnum.INT: string2int,
+        ValueTypeEnum.FLOAT: float,
+        ValueTypeEnum.TEXT: lambda x: escape(x).encode('utf-8').decode('utf-8'),
+        ValueTypeEnum.TIME: lambda x: escape(x).encode('utf-8').decode('utf-8'),
+        ValueTypeEnum.DATETIME: str2datetime,
+        ValueTypeEnum.DATE: str2datetime,
+    }
+
+    serialize = {
+        ValueTypeEnum.INT: int,
+        ValueTypeEnum.FLOAT: float,
+        ValueTypeEnum.TEXT: lambda x: x if isinstance(x, six.text_type) else str(x),
+        ValueTypeEnum.TIME: lambda x: x if isinstance(x, six.text_type) else str(x),
+        ValueTypeEnum.DATE: lambda x: x.strftime("%Y-%m-%d"),
+        ValueTypeEnum.DATETIME: lambda x: x.strftime("%Y-%m-%d %H:%M:%S"),
+    }
+
+    serialize2 = {
+        ValueTypeEnum.INT: int,
+        ValueTypeEnum.FLOAT: float,
+        ValueTypeEnum.TEXT: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
+        ValueTypeEnum.TIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
+        ValueTypeEnum.DATE: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
+        ValueTypeEnum.DATETIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
+    }
+
+    choice = {
+        ValueTypeEnum.INT: model.IntegerChoice,
+        ValueTypeEnum.FLOAT: model.FloatChoice,
+        ValueTypeEnum.TEXT: model.TextChoice,
+    }
+
+    table = {
+        ValueTypeEnum.INT: model.CIValueInteger,
+        ValueTypeEnum.TEXT: model.CIValueText,
+        ValueTypeEnum.DATETIME: model.CIValueDateTime,
+        ValueTypeEnum.DATE: model.CIValueDateTime,
+        ValueTypeEnum.TIME: model.CIValueText,
+        ValueTypeEnum.FLOAT: model.CIValueFloat,
+        'index_{0}'.format(ValueTypeEnum.INT): model.CIIndexValueInteger,
+        'index_{0}'.format(ValueTypeEnum.TEXT): model.CIIndexValueText,
+        'index_{0}'.format(ValueTypeEnum.DATETIME): model.CIIndexValueDateTime,
+        'index_{0}'.format(ValueTypeEnum.DATE): model.CIIndexValueDateTime,
+        'index_{0}'.format(ValueTypeEnum.TIME): model.CIIndexValueText,
+        'index_{0}'.format(ValueTypeEnum.FLOAT): model.CIIndexValueFloat,
+    }
+
+    table_name = {
+        ValueTypeEnum.INT: 'c_value_integers',
+        ValueTypeEnum.TEXT: 'c_value_texts',
+        ValueTypeEnum.DATETIME: 'c_value_datetime',
+        ValueTypeEnum.DATE: 'c_value_datetime',
+        ValueTypeEnum.TIME: 'c_value_texts',
+        ValueTypeEnum.FLOAT: 'c_value_floats',
+        'index_{0}'.format(ValueTypeEnum.INT): 'c_value_index_integers',
+        'index_{0}'.format(ValueTypeEnum.TEXT): 'c_value_index_texts',
+        'index_{0}'.format(ValueTypeEnum.DATETIME): 'c_value_index_datetime',
+        'index_{0}'.format(ValueTypeEnum.DATE): 'c_value_index_datetime',
+        'index_{0}'.format(ValueTypeEnum.TIME): 'c_value_index_texts',
+        'index_{0}'.format(ValueTypeEnum.FLOAT): 'c_value_index_floats',
+    }
+
+    es_type = {
+        ValueTypeEnum.INT: 'long',
+        ValueTypeEnum.TEXT: 'text',
+        ValueTypeEnum.DATETIME: 'text',
+        ValueTypeEnum.DATE: 'text',
+        ValueTypeEnum.TIME: 'text',
+        ValueTypeEnum.FLOAT: 'float'
+    }
+
+
+class TableMap(object):
+    def __init__(self, attr_name=None):
+        self.attr_name = attr_name
+
+    @property
+    def table(self):
+        attr = AttributeCache.get(self.attr_name)
+        i = "index_{0}".format(attr.value_type) if attr.is_index else attr.value_type
+        return ValueTypeMap.table.get(i)
+
+    @property
+    def table_name(self):
+        attr = AttributeCache.get(self.attr_name)
+        i = "index_{0}".format(attr.value_type) if attr.is_index else attr.value_type
+        return ValueTypeMap.table_name.get(i)

cmdb-api/api/lib/cmdb/value.py

@@ -0,0 +1,150 @@
+# -*- coding:utf-8 -*- 
+
+
+from __future__ import unicode_literals
+
+from flask import abort
+
+from api.extensions import db
+from api.lib.cmdb.attribute import AttributeManager
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.const import ExistPolicy
+from api.lib.cmdb.const import OperateType
+from api.lib.cmdb.history import AttributeHistoryManger
+from api.lib.cmdb.utils import TableMap
+from api.lib.cmdb.utils import ValueTypeMap
+from api.lib.utils import handle_arg_list
+
+
+class AttributeValueManager(object):
+    """
+    manage CI attribute values
+    """
+
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def _get_attr(key):
+        """
+        :param key: id, name or alias
+        :return: attribute instance
+        """
+        return AttributeCache.get(key)
+
+    def get_attr_values(self, fields, ci_id, ret_key="name", unique_key=None, use_master=False):
+        """
+
+        :param fields:
+        :param ci_id:
+        :param ret_key: It can be name or alias
+        :param unique_key: primary attribute
+        :param use_master: Only for master-slave read-write separation
+        :return:
+        """
+        res = dict()
+        for field in fields:
+            attr = self._get_attr(field)
+            if not attr:
+                continue
+
+            value_table = TableMap(attr_name=attr.name).table
+            rs = value_table.get_by(ci_id=ci_id,
+                                    attr_id=attr.id,
+                                    use_master=use_master,
+                                    to_dict=False)
+            field_name = getattr(attr, ret_key)
+
+            if attr.is_list:
+                res[field_name] = [ValueTypeMap.serialize[attr.value_type](i.value) for i in rs]
+            else:
+                res[field_name] = ValueTypeMap.serialize[attr.value_type](rs[0].value) if rs else None
+
+            if unique_key is not None and attr.id == unique_key.id and rs:
+                res['unique'] = unique_key.name
+
+        return res
+
+    @staticmethod
+    def __deserialize_value(value_type, value):
+        if not value:
+            return value
+        deserialize = ValueTypeMap.deserialize[value_type]
+        try:
+            v = deserialize(value)
+            return v
+        except ValueError:
+            return abort(400, "attribute value <{0}> is invalid".format(value))
+
+    @staticmethod
+    def __check_is_choice(attr_id, value_type, value):
+        choice_values = AttributeManager.get_choice_values(attr_id, value_type)
+        if value not in choice_values:
+            return abort(400, "{0} does not existed in choice values".format(value))
+
+    @staticmethod
+    def __check_is_unique(value_table, attr_id, ci_id, value):
+        existed = db.session.query(value_table.attr_id).filter(
+            value_table.attr_id == attr_id).filter(value_table.deleted.is_(False)).filter(
+            value_table.value == value).filter(value_table.ci_id != ci_id).first()
+        existed and abort(400, "attribute <{0}> value {1} must be unique".format(attr_id, value))
+
+    def _validate(self, attr, value, value_table, ci_id):
+        v = self.__deserialize_value(attr.value_type, value)
+
+        attr.is_choice and value and self.__check_is_choice(attr.id, attr.value_type, v)
+        attr.is_unique and self.__check_is_unique(value_table, attr.id, ci_id, v)
+
+        return v
+
+    @staticmethod
+    def _write_change(ci_id, attr_id, operate_type, old, new):
+        AttributeHistoryManger.add(ci_id, [(attr_id, operate_type, old, new)])
+
+    def create_or_update_attr_value(self, key, value, ci_id, _no_attribute_policy=ExistPolicy.IGNORE):
+        """
+        add or update attribute value, then write history
+        :param key: id, name or alias
+        :param value:
+        :param ci_id:
+        :param _no_attribute_policy: ignore or reject
+        :return:
+        """
+        attr = self._get_attr(key)
+        if attr is None:
+            if _no_attribute_policy == ExistPolicy.IGNORE:
+                return
+            if _no_attribute_policy == ExistPolicy.REJECT:
+                return abort(400, 'attribute {0} does not exist'.format(key))
+
+        value_table = TableMap(attr_name=attr.name).table
+
+        if attr.is_list:
+            value_list = [self._validate(attr, i, value_table, ci_id) for i in handle_arg_list(value)]
+            existed_attrs = value_table.get_by(attr_id=attr.id,
+                                               ci_id=ci_id,
+                                               to_dict=False)
+            existed_values = [i.value for i in existed_attrs]
+            added = set(value_list) - set(existed_values)
+            deleted = set(existed_values) - set(value_list)
+            for v in added:
+                value_table.create(ci_id=ci_id, attr_id=attr.id, value=v)
+                self._write_change(ci_id, attr.id, OperateType.ADD, None, v)
+
+            for v in deleted:
+                existed_attr = existed_attrs[existed_values.index(v)]
+                existed_attr.delete()
+                self._write_change(ci_id, attr.id, OperateType.DELETE, v, None)
+        else:
+            value = self._validate(attr, value, value_table, ci_id)
+            existed_attr = value_table.get_by(attr_id=attr.id,
+                                              ci_id=ci_id,
+                                              first=True,
+                                              to_dict=False)
+            existed_value = existed_attr and existed_attr.value
+            if existed_value is None:
+                value_table.create(ci_id=ci_id, attr_id=attr.id, value=value)
+                self._write_change(ci_id, attr.id, OperateType.ADD, None, value)
+            else:
+                existed_attr.update(value=value)
+                self._write_change(ci_id, attr.id, OperateType.UPDATE, existed_value, value)

cmdb-api/api/lib/database.py

@@ -0,0 +1,127 @@
+# -*- coding:utf-8 -*-
+
+import datetime
+
+import six
+
+from api.extensions import db
+from api.lib.exception import CommitException
+
+
+class FormatMixin(object):
+    def to_dict(self):
+        res = dict()
+        for k in getattr(self, "__table__").columns:
+            if not isinstance(getattr(self, k.name), datetime.datetime):
+                res[k.name] = getattr(self, k.name)
+            else:
+                res[k.name] = getattr(self, k.name).strftime('%Y-%m-%d %H:%M:%S')
+
+        return res
+
+    @classmethod
+    def get_columns(cls):
+        return {k.name: 1 for k in getattr(cls, "__mapper__").c.values()}
+
+
+class CRUDMixin(FormatMixin):
+    @classmethod
+    def create(cls, flush=False, **kwargs):
+        return cls(**kwargs).save(flush=flush)
+
+    def update(self, flush=False, **kwargs):
+        kwargs.pop("id", None)
+        for attr, value in six.iteritems(kwargs):
+            if value is not None:
+                setattr(self, attr, value)
+        if flush:
+            return self.save(flush=flush)
+        return self.save()
+
+    def save(self, commit=True, flush=False):
+        db.session.add(self)
+        try:
+            if flush:
+                db.session.flush()
+            elif commit:
+                db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            raise CommitException(str(e))
+
+        return self
+
+    def delete(self, flush=False):
+        db.session.delete(self)
+        try:
+            if flush:
+                return db.session.flush()
+            return db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            raise CommitException(str(e))
+
+    def soft_delete(self, flush=False):
+        setattr(self, "deleted", True)
+        setattr(self, "deleted_at", datetime.datetime.now())
+        self.save(flush=flush)
+
+    @classmethod
+    def get_by_id(cls, _id):
+        if any((isinstance(_id, six.string_types) and _id.isdigit(),
+                isinstance(_id, (six.integer_types, float))), ):
+            return getattr(cls, "query").get(int(_id)) or None
+
+    @classmethod
+    def get_by(cls, first=False, to_dict=True, fl=None, exclude=None, deleted=False, use_master=False, **kwargs):
+        db_session = db.session if not use_master else db.session().using_bind("master")
+        fl = fl.strip().split(",") if fl and isinstance(fl, six.string_types) else (fl or [])
+        exclude = exclude.strip().split(",") if exclude and isinstance(exclude, six.string_types) else (exclude or [])
+
+        keys = cls.get_columns()
+        fl = [k for k in fl if k in keys]
+        fl = [k for k in keys if k not in exclude and not k.isupper()] if exclude else fl
+        fl = list(filter(lambda x: "." not in x, fl))
+
+        if hasattr(cls, "deleted") and deleted is not None:
+            kwargs["deleted"] = deleted
+
+        if fl:
+            query = db_session.query(*[getattr(cls, k) for k in fl])
+            query = query.filter_by(**kwargs)
+            result = [{k: getattr(i, k) for k in fl} for i in query]
+        else:
+            result = [i.to_dict() if to_dict else i for i in getattr(cls, 'query').filter_by(**kwargs)]
+
+        return result[0] if first and result else (None if first else result)
+
+    @classmethod
+    def get_by_like(cls, to_dict=True, **kwargs):
+        query = db.session.query(cls)
+        for k, v in kwargs.items():
+            query = query.filter(getattr(cls, k).ilike('%{0}%'.format(v)))
+        return [i.to_dict() if to_dict else i for i in query]
+
+
+class SoftDeleteMixin(object):
+    deleted_at = db.Column(db.DateTime)
+    deleted = db.Column(db.Boolean, index=True, default=False)
+
+
+class TimestampMixin(object):
+    created_at = db.Column(db.DateTime, default=lambda: datetime.datetime.now())
+    updated_at = db.Column(db.DateTime, onupdate=lambda: datetime.datetime.now())
+
+
+class SurrogatePK(object):
+    __table_args__ = {"extend_existing": True}
+
+    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
+
+
+class Model(SoftDeleteMixin, TimestampMixin, CRUDMixin, db.Model, SurrogatePK):
+    __abstract__ = True
+
+
+class CRUDModel(db.Model, CRUDMixin):
+    __abstract__ = True

cmdb-api/api/lib/decorator.py

@@ -0,0 +1,35 @@
+# -*- coding:utf-8 -*-
+
+
+from functools import wraps
+
+from flask import abort
+from flask import request
+
+
+def kwargs_required(*required_args):
+    def decorate(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            for arg in required_args:
+                if arg not in kwargs:
+                    return abort(400, "Argument <{0}> is required".format(arg))
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorate
+
+
+def args_required(*required_args):
+    def decorate(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            for arg in required_args:
+                if arg not in request.values:
+                    return abort(400, "Argument <{0}> is required".format(arg))
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorate

cmdb-api/api/lib/exception.py

@@ -0,0 +1,11 @@
+# -*- coding:utf-8 -*-
+
+
+from werkzeug.exceptions import NotFound, Forbidden, BadRequest
+
+
+class CommitException(Exception):
+    pass
+
+
+AbortException = (NotFound, Forbidden, BadRequest)

cmdb-api/api/lib/http_cli.py

@@ -0,0 +1,49 @@
+# -*- coding:utf-8 -*-
+
+from __future__ import unicode_literals
+
+import hashlib
+
+import requests
+from flask import abort
+from flask import current_app
+from flask import g
+from future.moves.urllib.parse import urlparse
+
+
+def build_api_key(path, params):
+    g.user is not None or abort(403, "您得登陆才能进行该操作")
+    key = g.user.key
+    secret = g.user.secret
+    values = "".join([str(params[k]) for k in sorted(params.keys())
+                      if params[k] is not None]) if params.keys() else ""
+    _secret = "".join([path, secret, values]).encode("utf-8")
+    params["_secret"] = hashlib.sha1(_secret).hexdigest()
+    params["_key"] = key
+    return params
+
+
+def api_request(url, method="get", params=None, ret_key=None):
+    params = params or {}
+    resp = None
+    try:
+        method = method.lower()
+        params = build_api_key(urlparse(url).path, params)
+        if method == "get":
+            resp = getattr(requests, method)(url, params=params)
+        else:
+            resp = getattr(requests, method)(url, data=params)
+        if resp.status_code != 200:
+            return abort(resp.status_code, resp.json().get("message"))
+        resp = resp.json()
+        if ret_key is not None:
+            return resp.get(ret_key)
+        return resp
+    except Exception as e:
+        code = e.code if hasattr(e, "code") else None
+        if isinstance(code, int) and resp is not None:
+            return abort(code, resp.json().get("message"))
+        current_app.logger.warning(url)
+        current_app.logger.warning(params)
+        current_app.logger.error(str(e))
+        return abort(500, "server unknown error")

cmdb-api/api/lib/mail.py

@@ -0,0 +1,51 @@
+# -*- coding:utf-8 -*- 
+
+
+import smtplib
+import time
+from email import Utils
+from email.header import Header
+from email.mime.image import MIMEImage
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+
+from flask import current_app
+
+
+def send_mail(sender, receiver, subject, content, ctype="html", pics=()):
+    """subject and body are unicode objects"""
+    if not sender:
+        sender = current_app.config.get("DEFAULT_MAIL_SENDER")
+    smtp_server = current_app.config.get("MAIL_SERVER")
+    if ctype == "html":
+        msg = MIMEText(content, 'html', 'utf-8')
+    else:
+        msg = MIMEText(content, 'plain', 'utf-8')
+
+    if len(pics) != 0:
+        msg_root = MIMEMultipart('related')
+        msg_text = MIMEText(content, 'html', 'utf-8')
+        msg_root.attach(msg_text)
+        i = 1
+        for pic in pics:
+            fp = open(pic, "rb")
+            image = MIMEImage(fp.read())
+            fp.close()
+            image.add_header('Content-ID', '<img%02d>' % i)
+            msg_root.attach(image)
+            i += 1
+        msg = msg_root
+
+    msg['Subject'] = Header(subject, 'utf-8')
+    msg['From'] = sender
+    msg['To'] = ';'.join(receiver)
+    msg['Message-ID'] = Utils.make_msgid()
+    msg['date'] = time.strftime('%a, %d %b %Y %H:%M:%S %z')
+
+    smtp = smtplib.SMTP()
+    smtp.connect(smtp_server, 25)
+    username, password = current_app.config.get("MAIL_USERNAME"), current_app.config.get("MAIL_PASSWORD")
+    if username and password:
+        smtp.login(username, password)
+    smtp.sendmail(sender, receiver, msg.as_string())
+    smtp.quit()

cmdb-api/api/lib/perm/__init__.py

@@ -0,0 +1 @@
+# -*- coding:utf-8 -*-

cmdb-api/api/lib/perm/acl/__init__.py

@@ -0,0 +1,23 @@
+# -*- coding:utf-8 -*-
+
+
+from functools import wraps
+
+from flask import request
+from flask import abort
+
+from api.lib.perm.acl.cache import AppCache
+
+
+def validate_app(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        app_id = request.values.get('app_id')
+        app = AppCache.get(app_id)
+        if app is None:
+            return abort(400, "App <{0}> does not exist".format(app_id))
+        request.values['app_id'] = app.id
+
+        return func(*args, **kwargs)
+
+    return wrapper

cmdb-api/api/lib/perm/acl/acl.py

@@ -0,0 +1,174 @@
+# -*- coding:utf-8 -*-
+
+import functools
+
+import six
+from flask import current_app, g, request
+from flask import session, abort
+
+from api.lib.cmdb.const import ResourceTypeEnum as CmdbResourceType
+from api.lib.cmdb.const import RoleEnum
+from api.lib.perm.acl.cache import AppCache
+from api.lib.perm.acl.cache import RoleCache
+from api.lib.perm.acl.cache import UserCache
+from api.lib.perm.acl.permission import PermissionCRUD
+from api.lib.perm.acl.resource import ResourceCRUD
+from api.lib.perm.acl.role import RoleCRUD
+from api.models.acl import Resource
+from api.models.acl import ResourceGroup
+from api.models.acl import ResourceType
+from api.models.acl import Role
+
+CMDB_RESOURCE_TYPES = CmdbResourceType.all()
+
+
+class ACLManager(object):
+    def __init__(self):
+        self.app_id = AppCache.get('cmdb')
+        if not self.app_id:
+            raise Exception("cmdb not in acl apps")
+        self.app_id = self.app_id.id
+
+    def _get_resource(self, name, resource_type_name):
+        resource_type = ResourceType.get_by(name=resource_type_name, first=True, to_dict=False)
+        resource_type or abort(404, "ResourceType <{0}> cannot be found".format(resource_type_name))
+
+        return Resource.get_by(resource_type_id=resource_type.id,
+                               app_id=self.app_id,
+                               name=name,
+                               first=True,
+                               to_dict=False)
+
+    def _get_resource_group(self, name):
+        return ResourceGroup.get_by(
+            app_id=self.app_id,
+            name=name,
+            first=True,
+            to_dict=False
+        )
+
+    def _get_role(self, name):
+        user = UserCache.get(name)
+        if user:
+            return Role.get_by(name=name, uid=user.uid, first=True, to_dict=False)
+
+        return Role.get_by(name=name, app_id=self.app_id, first=True, to_dict=False)
+
+    def add_resource(self, name, resource_type_name=None):
+        resource_type = ResourceType.get_by(name=resource_type_name, first=True, to_dict=False)
+        resource_type or abort(404, "ResourceType <{0}> cannot be found".format(resource_type_name))
+
+        ResourceCRUD.add(name, resource_type.id, self.app_id)
+
+    def grant_resource_to_role(self, name, role, resource_type_name=None, permissions=None):
+        resource = self._get_resource(name, resource_type_name)
+
+        role = self._get_role(role)
+
+        if resource:
+            PermissionCRUD.grant(role.id, permissions, resource_id=resource.id)
+        else:
+            group = self._get_resource_group(name)
+            if group:
+                PermissionCRUD.grant(role.id, permissions, group_id=group.id)
+
+    def del_resource(self, name, resource_type_name=None):
+        resource = self._get_resource(name, resource_type_name)
+        if resource:
+            ResourceCRUD.delete(resource.id)
+
+    def has_permission(self, resource_name, resource_type, perm):
+
+        role = self._get_role(g.user.username)
+
+        role or abort(404, "Role <{0}> is not found".format(g.user.username))
+
+        return RoleCRUD.has_permission(role.id, resource_name, resource_type, self.app_id, perm)
+
+
+def validate_permission(resources, resource_type, perm):
+    if not resources:
+        return
+
+    if current_app.config.get("USE_ACL"):
+        if g.user.username == "worker":
+            return
+
+        resources = [resources] if isinstance(resources, six.string_types) else resources
+        for resource in resources:
+            if not ACLManager().has_permission(resource, resource_type, perm):
+                return abort(403, "has no permission")
+
+
+def has_perm(resources, resource_type, perm):
+    def decorator_has_perm(func):
+        @functools.wraps(func)
+        def wrapper_has_perm(*args, **kwargs):
+            if not resources:
+                return
+
+            if current_app.config.get("USE_ACL"):
+                if is_app_admin():
+                    return func(*args, **kwargs)
+
+                validate_permission(resources, resource_type, perm)
+
+            return func(*args, **kwargs)
+
+        return wrapper_has_perm
+
+    return decorator_has_perm
+
+
+def is_app_admin(app=None):
+    if RoleEnum.CONFIG in session.get("acl", {}).get("parentRoles", []):
+        return True
+
+    app = app or 'cmdb'
+    app_id = AppCache.get(app).id
+
+    for role in session.get("acl", {}).get("parentRoles", []):
+        if RoleCache.get_by_name(app_id, role).is_app_admin:
+            return True
+
+    return False
+
+
+def has_perm_from_args(arg_name, resource_type, perm, callback=None):
+    def decorator_has_perm(func):
+        @functools.wraps(func)
+        def wrapper_has_perm(*args, **kwargs):
+            if not arg_name:
+                return
+            resource = request.view_args.get(arg_name) or request.values.get(arg_name)
+            if callback is not None and resource:
+                resource = callback(resource)
+
+            if current_app.config.get("USE_ACL") and resource:
+                if is_app_admin():
+                    return func(*args, **kwargs)
+
+                validate_permission(resource, resource_type, perm)
+
+            return func(*args, **kwargs)
+
+        return wrapper_has_perm
+
+    return decorator_has_perm
+
+
+def role_required(role_name):
+    def decorator_role_required(func):
+        @functools.wraps(func)
+        def wrapper_role_required(*args, **kwargs):
+            if not role_name:
+                return
+
+            if current_app.config.get("USE_ACL"):
+                if role_name not in session.get("acl", {}).get("parentRoles", []) and not is_app_admin():
+                    return abort(403, "Role {0} is required".format(role_name))
+            return func(*args, **kwargs)
+
+        return wrapper_role_required
+
+    return decorator_role_required

cmdb-api/api/lib/perm/acl/cache.py

@@ -0,0 +1,174 @@
+# -*- coding:utf-8 -*-
+
+from api.extensions import cache
+from api.extensions import db
+from api.models.acl import App
+from api.models.acl import Permission
+from api.models.acl import Role
+from api.models.acl import User
+
+
+class AppCache(object):
+    PREFIX_ID = "App::id::{0}"
+    PREFIX_NAME = "App::name::{0}"
+
+    @classmethod
+    def get(cls, key):
+        app = cache.get(cls.PREFIX_ID.format(key)) or cache.get(cls.PREFIX_NAME.format(key))
+        if app is None:
+            app = App.get_by_id(key) or App.get_by(name=key, to_dict=False, first=True)
+        if app is not None:
+            cls.set(app)
+
+        return app
+
+    @classmethod
+    def set(cls, app):
+        cache.set(cls.PREFIX_ID.format(app.id), app)
+        cache.set(cls.PREFIX_NAME.format(app.name), app)
+
+    @classmethod
+    def clean(cls, app):
+        cache.delete(cls.PREFIX_ID.format(app.id))
+        cache.delete(cls.PREFIX_NAME.format(app.name))
+
+
+class UserCache(object):
+    PREFIX_ID = "User::uid::{0}"
+    PREFIX_NAME = "User::username::{0}"
+    PREFIX_NICK = "User::nickname::{0}"
+
+    @classmethod
+    def get(cls, key):
+        user = cache.get(cls.PREFIX_ID.format(key)) or \
+               cache.get(cls.PREFIX_NAME.format(key)) or \
+               cache.get(cls.PREFIX_NICK.format(key))
+        if not user:
+            user = User.query.get(key) or \
+                   User.query.get_by_username(key) or \
+                   User.query.get_by_nickname(key)
+        if user:
+            cls.set(user)
+
+        return user
+
+    @classmethod
+    def set(cls, user):
+        cache.set(cls.PREFIX_ID.format(user.uid), user)
+        cache.set(cls.PREFIX_NAME.format(user.username), user)
+        cache.set(cls.PREFIX_NICK.format(user.nickname), user)
+
+    @classmethod
+    def clean(cls, user):
+        cache.delete(cls.PREFIX_ID.format(user.uid))
+        cache.delete(cls.PREFIX_NAME.format(user.username))
+        cache.delete(cls.PREFIX_NICK.format(user.nickname))
+
+
+class RoleCache(object):
+    PREFIX_ID = "Role::id::{0}"
+    PREFIX_NAME = "Role::app_id::{0}::name::{1}"
+
+    @classmethod
+    def get_by_name(cls, app_id, name):
+        role = cache.get(cls.PREFIX_NAME.format(app_id, name))
+        if role is None:
+            role = Role.get_by(app_id=app_id, name=name, first=True, to_dict=False)
+            if role is not None:
+                cache.set(cls.PREFIX_NAME.format(app_id, name), role)
+
+        return role
+
+    @classmethod
+    def get(cls, rid):
+        role = cache.get(cls.PREFIX_ID.format(rid))
+        if role is None:
+            role = Role.get_by_id(rid)
+            if role is not None:
+                cache.set(cls.PREFIX_ID.format(rid), role)
+
+        return role
+
+    @classmethod
+    def clean(cls, rid):
+        cache.delete(cls.PREFIX_ID.format(rid))
+
+    @classmethod
+    def clean_by_name(cls, app_id, name):
+        cache.delete(cls.PREFIX_NAME.format(app_id, name))
+
+
+class RoleRelationCache(object):
+    PREFIX_PARENT = "RoleRelationParent::id::{0}"
+    PREFIX_CHILDREN = "RoleRelationChildren::id::{0}"
+    PREFIX_RESOURCES = "RoleRelationResources::id::{0}"
+
+    @classmethod
+    def get_parent_ids(cls, rid):
+        parent_ids = cache.get(cls.PREFIX_PARENT.format(rid))
+        if not parent_ids:
+            from api.lib.perm.acl.role import RoleRelationCRUD
+            parent_ids = RoleRelationCRUD.get_parent_ids(rid)
+            cache.set(cls.PREFIX_PARENT.format(rid), parent_ids, timeout=0)
+
+        return parent_ids
+
+    @classmethod
+    def get_child_ids(cls, rid):
+        child_ids = cache.get(cls.PREFIX_CHILDREN.format(rid))
+        if not child_ids:
+            from api.lib.perm.acl.role import RoleRelationCRUD
+            child_ids = RoleRelationCRUD.get_child_ids(rid)
+            cache.set(cls.PREFIX_CHILDREN.format(rid), child_ids, timeout=0)
+
+        return child_ids
+
+    @classmethod
+    def get_resources(cls, rid):
+        """
+        :param rid: 
+        :return: {id2perms: {resource_id: [perm,]}, group2perms: {group_id: [perm, ]}}
+        """
+        resources = cache.get(cls.PREFIX_RESOURCES.format(rid))
+        if not resources:
+            from api.lib.perm.acl.role import RoleCRUD
+            resources = RoleCRUD.get_resources(rid)
+            cache.set(cls.PREFIX_RESOURCES.format(rid), resources, timeout=0)
+
+        return resources or {}
+
+    @classmethod
+    def rebuild(cls, rid):
+        cls.clean(rid)
+        db.session.close()
+        cls.get_parent_ids(rid)
+        cls.get_child_ids(rid)
+        cls.get_resources(rid)
+
+    @classmethod
+    def clean(cls, rid):
+        cache.delete(cls.PREFIX_PARENT.format(rid))
+        cache.delete(cls.PREFIX_CHILDREN.format(rid))
+        cache.delete(cls.PREFIX_RESOURCES.format(rid))
+
+
+class PermissionCache(object):
+    PREFIX_ID = "Permission::id::{0}"
+    PREFIX_NAME = "Permission::name::{0}"
+
+    @classmethod
+    def get(cls, key):
+        perm = cache.get(cls.PREFIX_ID.format(key))
+        perm = perm or cache.get(cls.PREFIX_NAME.format(key))
+        if perm is None:
+            perm = Permission.get_by_id(key)
+            perm = perm or Permission.get_by(name=key, first=True, to_dict=False)
+            if perm is not None:
+                cache.set(cls.PREFIX_ID.format(key), perm)
+
+        return perm
+
+    @classmethod
+    def clean(cls, key):
+        cache.delete(cls.PREFIX_ID.format(key))
+        cache.delete(cls.PREFIX_NAME.format(key))

cmdb-api/api/lib/perm/acl/const.py

@@ -0,0 +1,5 @@
+# -*- coding:utf-8 -*-
+
+from api.lib.cmdb.const import CMDB_QUEUE
+
+ACL_QUEUE = CMDB_QUEUE

cmdb-api/api/lib/perm/acl/permission.py

@@ -0,0 +1,48 @@
+# -*- coding:utf-8 -*-
+
+
+from api.lib.perm.acl.cache import PermissionCache
+from api.lib.perm.acl.cache import RoleCache
+from api.lib.perm.acl.const import ACL_QUEUE
+from api.models.acl import RolePermission
+from api.tasks.acl import role_rebuild
+
+
+class PermissionCRUD(object):
+    @staticmethod
+    def get_all(resource_id=None, group_id=None):
+        result = dict()
+        if resource_id is not None:
+            perms = RolePermission.get_by(resource_id=resource_id, to_dict=False)
+        else:
+            perms = RolePermission.get_by(group_id=group_id, to_dict=False)
+
+        for perm in perms:
+            perm_dict = PermissionCache.get(perm.perm_id).to_dict()
+            perm_dict.update(dict(rid=perm.rid))
+            result.setdefault(RoleCache.get(perm.rid).name, []).append(perm_dict)
+
+        return result
+
+    @staticmethod
+    def grant(rid, perms, resource_id=None, group_id=None):
+        for perm in perms:
+            perm = PermissionCache.get(perm)
+            existed = RolePermission.get_by(rid=rid, perm_id=perm.id, group_id=group_id, resource_id=resource_id)
+            existed or RolePermission.create(rid=rid, perm_id=perm.id, group_id=group_id, resource_id=resource_id)
+
+        role_rebuild.apply_async(args=(rid,), queue=ACL_QUEUE)
+
+    @staticmethod
+    def revoke(rid, perms, resource_id=None, group_id=None):
+        for perm in perms:
+            perm = PermissionCache.get(perm)
+            existed = RolePermission.get_by(rid=rid,
+                                            perm_id=perm.id,
+                                            group_id=group_id,
+                                            resource_id=resource_id,
+                                            first=True,
+                                            to_dict=False)
+            existed and existed.soft_delete()
+
+        role_rebuild.apply_async(args=(rid,), queue=ACL_QUEUE)

cmdb-api/api/lib/perm/acl/resource.py

@@ -0,0 +1,186 @@
+# -*- coding:utf-8 -*-
+
+
+from flask import abort
+
+from api.extensions import db
+from api.lib.perm.acl.const import ACL_QUEUE
+from api.models.acl import Permission
+from api.models.acl import Resource
+from api.models.acl import ResourceGroup
+from api.models.acl import ResourceGroupItems
+from api.models.acl import ResourceType
+from api.models.acl import RolePermission
+from api.tasks.acl import role_rebuild
+
+
+class ResourceTypeCRUD(object):
+    @staticmethod
+    def search(q, app_id, page=1, page_size=None):
+        query = db.session.query(ResourceType).filter(
+            ResourceType.deleted.is_(False)).filter(ResourceType.app_id == app_id)
+        if q:
+            query = query.filter(ResourceType.name.ilike('%{0}%'.format(q)))
+
+        numfound = query.count()
+        res = query.offset((page - 1) * page_size).limit(page_size)
+        rt_ids = [i.id for i in res]
+        perms = db.session.query(Permission).filter(Permission.deleted.is_(False)).filter(
+            Permission.resource_type_id.in_(rt_ids))
+        id2perms = dict()
+        for perm in perms:
+            id2perms.setdefault(perm.resource_type_id, []).append(perm.to_dict())
+
+        return numfound, res, id2perms
+
+    @staticmethod
+    def get_perms(rt_id):
+        perms = Permission.get_by(resource_type_id=rt_id, to_dict=False)
+        return [i.to_dict() for i in perms]
+
+    @classmethod
+    def add(cls, app_id, name, description, perms):
+        ResourceType.get_by(name=name, app_id=app_id) and abort(
+            400, "ResourceType <{0}> is already existed".format(name))
+
+        rt = ResourceType.create(name=name, description=description, app_id=app_id)
+
+        cls.update_perms(rt.id, perms, app_id)
+
+        return rt
+
+    @classmethod
+    def update(cls, rt_id, **kwargs):
+        kwargs.pop('app_id', None)
+
+        rt = ResourceType.get_by_id(rt_id) or abort(404, "ResourceType <{0}> is not found".format(rt_id))
+        if 'name' in kwargs:
+            other = ResourceType.get_by(name=kwargs['name'], app_id=rt.app_id, to_dict=False, first=True)
+            if other and other.id != rt_id:
+                return abort(400, "ResourceType <{0}> is duplicated".format(kwargs['name']))
+
+        if 'perms' in kwargs:
+            cls.update_perms(rt_id, kwargs.pop('perms'), rt.app_id)
+
+        return rt.update(**kwargs)
+
+    @classmethod
+    def delete(cls, rt_id):
+        rt = ResourceType.get_by_id(rt_id) or abort(404, "ResourceType <{0}> is not found".format(rt_id))
+
+        cls.update_perms(rt_id, [], rt.app_id)
+
+        rt.soft_delete()
+
+    @classmethod
+    def update_perms(cls, rt_id, perms, app_id):
+        existed = Permission.get_by(resource_type_id=rt_id, to_dict=False)
+        existed_names = [i.name for i in existed]
+
+        for i in existed:
+            if i.name not in perms:
+                i.soft_delete()
+
+        for i in perms:
+            if i not in existed_names:
+                Permission.create(resource_type_id=rt_id,
+                                  name=i,
+                                  app_id=app_id)
+
+
+class ResourceGroupCRUD(object):
+    @staticmethod
+    def search(q, app_id, page=1, page_size=None):
+        query = db.session.query(ResourceGroup).filter(
+            ResourceGroup.deleted.is_(False)).filter(ResourceGroup.app_id == app_id)
+
+        if q:
+            query = query.filter(ResourceGroup.name.ilike("%{0}%".format(q)))
+
+        numfound = query.count()
+
+        return numfound, query.offset((page - 1) * page_size).limit(page_size)
+
+    @staticmethod
+    def get_items(rg_id):
+        items = ResourceGroupItems.get_by(group_id=rg_id, to_dict=False)
+
+        return [i.resource.to_dict() for i in items]
+
+    @staticmethod
+    def add(name, type_id, app_id):
+        ResourceGroup.get_by(name=name, resource_type_id=type_id, app_id=app_id) and abort(
+            400, "ResourceGroup <{0}> is already existed".format(name))
+
+        return ResourceGroup.create(name=name, resource_type_id=type_id, app_id=app_id)
+
+    @staticmethod
+    def update(rg_id, items):
+        existed = ResourceGroupItems.get_by(group_id=rg_id, to_dict=False)
+        existed_ids = [i.resource_id for i in existed]
+
+        for i in existed:
+            if i.resource_id not in items:
+                i.soft_delete()
+
+        for _id in items:
+            if _id not in existed_ids:
+                ResourceGroupItems.create(group_id=rg_id, resource_id=_id)
+
+    @staticmethod
+    def delete(rg_id):
+        rg = ResourceGroup.get_by_id(rg_id) or abort(404, "ResourceGroup <{0}> is not found".format(rg_id))
+
+        rg.soft_delete()
+
+        items = ResourceGroupItems.get_by(group_id=rg_id, to_dict=False)
+        for item in items:
+            item.soft_delete()
+
+        for i in RolePermission.get_by(group_id=rg_id, to_dict=False):
+            i.soft_delete()
+            role_rebuild.apply_async(args=(i.rid,), queue=ACL_QUEUE)
+
+
+class ResourceCRUD(object):
+    @staticmethod
+    def search(q, app_id, resource_type_id=None, page=1, page_size=None):
+        query = db.session.query(Resource).filter(
+            Resource.deleted.is_(False)).filter(Resource.app_id == app_id)
+
+        if q:
+            query = query.filter(Resource.name.ilike("%{0}%".format(q)))
+
+        if resource_type_id:
+            query = query.filter(Resource.resource_type_id == resource_type_id)
+
+        numfound = query.count()
+
+        return numfound, query.offset((page - 1) * page_size).limit(page_size)
+
+    @staticmethod
+    def add(name, type_id, app_id):
+        Resource.get_by(name=name, resource_type_id=type_id, app_id=app_id) and abort(
+            400, "Resource <{0}> is already existed".format(name))
+
+        return Resource.create(name=name, resource_type_id=type_id, app_id=app_id)
+
+    @staticmethod
+    def update(_id, name):
+        resource = Resource.get_by_id(_id) or abort(404, "Resource <{0}> is not found".format(_id))
+
+        other = Resource.get_by(name=name, resource_type_id=resource.resource_type_id, to_dict=False, first=True)
+        if other and other.id != _id:
+            return abort(400, "Resource <{0}> is duplicated".format(name))
+
+        return resource.update(name=name)
+
+    @staticmethod
+    def delete(_id):
+        resource = Resource.get_by_id(_id) or abort(404, "Resource <{0}> is not found".format(_id))
+
+        resource.soft_delete()
+
+        for i in RolePermission.get_by(resource_id=_id, to_dict=False):
+            i.soft_delete()
+            role_rebuild.apply_async(args=(i.rid,), queue=ACL_QUEUE)

cmdb-api/api/lib/perm/acl/role.py

@@ -0,0 +1,228 @@
+# -*- coding:utf-8 -*-
+
+
+import six
+from flask import abort
+
+from api.extensions import db
+from api.lib.perm.acl.cache import RoleCache
+from api.lib.perm.acl.cache import RoleRelationCache
+from api.lib.perm.acl.const import ACL_QUEUE
+from api.models.acl import Resource
+from api.models.acl import ResourceGroupItems
+from api.models.acl import ResourceType
+from api.models.acl import Role
+from api.models.acl import RolePermission
+from api.models.acl import RoleRelation
+from api.tasks.acl import role_rebuild
+
+
+class RoleRelationCRUD(object):
+    @staticmethod
+    def get_parents(rids=None, uids=None):
+        rid2uid = dict()
+        if uids is not None:
+            uids = [uids] if isinstance(uids, six.integer_types) else uids
+            rids = db.session.query(Role).filter(Role.deleted.is_(False)).filter(Role.uid.in_(uids))
+            rid2uid = {i.id: i.uid for i in rids}
+            rids = [i.id for i in rids]
+        else:
+            rids = [rids] if isinstance(rids, six.integer_types) else rids
+
+        res = db.session.query(RoleRelation).filter(
+            RoleRelation.child_id.in_(rids)).filter(RoleRelation.deleted.is_(False))
+        id2parents = {}
+        for i in res:
+            id2parents.setdefault(rid2uid.get(i.child_id, i.child_id), []).append(RoleCache.get(i.parent_id).to_dict())
+
+        return id2parents
+
+    @staticmethod
+    def get_parent_ids(rid):
+        res = RoleRelation.get_by(child_id=rid, to_dict=False)
+
+        return [i.parent_id for i in res]
+
+    @staticmethod
+    def get_child_ids(rid):
+        res = RoleRelation.get_by(parent_id=rid, to_dict=False)
+
+        return [i.parent_id for i in res]
+
+    @classmethod
+    def recursive_parent_ids(cls, rid):
+        all_parent_ids = set()
+
+        def _get_parent(_id):
+            all_parent_ids.add(_id)
+            parent_ids = RoleRelationCache.get_parent_ids(_id)
+            for parent_id in parent_ids:
+                _get_parent(parent_id)
+
+        _get_parent(rid)
+
+        return all_parent_ids
+
+    @classmethod
+    def recursive_child_ids(cls, rid):
+        all_child_ids = set()
+
+        def _get_children(_id):
+            all_child_ids.add(_id)
+            child_ids = RoleRelationCache.get_child_ids(_id)
+            for child_id in child_ids:
+                _get_children(child_id)
+
+        _get_children(rid)
+
+        return all_child_ids
+
+    @staticmethod
+    def add(parent_id, child_id):
+        RoleRelation.get_by(parent_id=parent_id, child_id=child_id) and abort(400, "It's already existed")
+
+        RoleRelationCache.clean(parent_id)
+        RoleRelationCache.clean(child_id)
+
+        return RoleRelation.create(parent_id=parent_id, child_id=child_id)
+
+    @classmethod
+    def delete(cls, _id):
+        existed = RoleRelation.get_by_id(_id) or abort(400, "RoleRelation <{0}> does not exist".format(_id))
+
+        child_ids = cls.recursive_child_ids(existed.child_id)
+        for child_id in child_ids:
+            role_rebuild.apply_async(args=(child_id,), queue=ACL_QUEUE)
+
+        RoleRelationCache.clean(existed.parent_id)
+        RoleRelationCache.clean(existed.child_id)
+
+        existed.soft_delete()
+
+    @classmethod
+    def delete2(cls, parent_id, child_id):
+        existed = RoleRelation.get_by(parent_id=parent_id, child_id=child_id, first=True, to_dict=False)
+        existed or abort(400, "RoleRelation < {0} -> {1} > does not exist".format(parent_id, child_id))
+
+        child_ids = cls.recursive_child_ids(existed.child_id)
+        for child_id in child_ids:
+            role_rebuild.apply_async(args=(child_id,), queue=ACL_QUEUE)
+
+        RoleRelationCache.clean(existed.parent_id)
+        RoleRelationCache.clean(existed.child_id)
+
+        existed.soft_delete()
+
+
+class RoleCRUD(object):
+    @staticmethod
+    def search(q, app_id, page=1, page_size=None, user_role=True):
+        query = db.session.query(Role).filter(Role.deleted.is_(False))
+        query = query.filter(Role.app_id == app_id).filter(Role.uid.is_(None))
+
+        if user_role:
+            query1 = db.session.query(Role).filter(Role.deleted.is_(False)).filter(Role.uid.isnot(None))
+            query = query.union(query1)
+
+        if q:
+            query = query.filter(Role.name.ilike('%{0}%'.format(q)))
+
+        numfound = query.count()
+
+        return numfound, query.offset((page - 1) * page_size).limit(page_size)
+
+    @staticmethod
+    def add_role(name, app_id=None, is_app_admin=False, uid=None):
+        Role.get_by(name=name, app_id=app_id) and abort(400, "Role <{0}> is already existed".format(name))
+
+        return Role.create(name=name,
+                           app_id=app_id,
+                           is_app_admin=is_app_admin,
+                           uid=uid)
+
+    @staticmethod
+    def update_role(rid, **kwargs):
+        kwargs.pop('app_id', None)
+
+        role = Role.get_by_id(rid) or abort(404, "Role <{0}> does not exist".format(rid))
+
+        RoleCache.clean(rid)
+
+        return role.update(**kwargs)
+
+    @classmethod
+    def delete_role(cls, rid):
+        role = Role.get_by_id(rid) or abort(404, "Role <{0}> does not exist".format(rid))
+
+        for i in RoleRelation.get_by(parent_id=rid, to_dict=False):
+            i.soft_delete()
+        for i in RoleRelation.get_by(child_id=rid, to_dict=False):
+            i.soft_delete()
+
+        for i in RolePermission.get_by(rid=rid, to_dict=False):
+            i.soft_delete()
+
+        role_rebuild.apply_async(args=(list(RoleRelationCRUD.recursive_child_ids(rid)), ), queue=ACL_QUEUE)
+
+        RoleCache.clean(rid)
+        RoleRelationCache.clean(rid)
+
+        role.soft_delete()
+
+    @staticmethod
+    def get_resources(rid):
+        res = RolePermission.get_by(rid=rid, to_dict=False)
+        id2perms = dict(id2perms={}, group2perms={})
+        for i in res:
+            if i.resource_id:
+                id2perms['id2perms'].setdefault(i.resource_id, []).append(i.perm.name)
+            elif i.group_id:
+                id2perms['group2perms'].setdefault(i.group_id, []).append(i.perm.name)
+
+        return id2perms
+
+    @staticmethod
+    def get_group_ids(resource_id):
+        return [i.group_id for i in ResourceGroupItems.get_by(resource_id=resource_id, to_dict=False)]
+
+    @classmethod
+    def has_permission(cls, rid, resource_name, resource_type, app_id, perm):
+        resource_type = ResourceType.get_by(app_id=app_id, name=resource_type, first=True, to_dict=False)
+        resource_type or abort(404, "ResourceType <{0}> is not found".format(resource_type))
+        type_id = resource_type.id
+        resource = Resource.get_by(name=resource_name, resource_type_id=type_id, first=True, to_dict=False)
+        resource = resource or abort(403, "Resource <{0}> is not in ACL".format(resource_name))
+
+        parent_ids = RoleRelationCRUD.recursive_parent_ids(rid)
+
+        group_ids = cls.get_group_ids(resource.id)
+        for parent_id in parent_ids:
+            id2perms = RoleRelationCache.get_resources(parent_id)
+            perms = id2perms['id2perms'].get(resource.id, [])
+            if perms and {perm}.issubset(set(perms)):
+                return True
+
+            for group_id in group_ids:
+                perms = id2perms['group2perms'].get(group_id, [])
+                if perms and {perm}.issubset(set(perms)):
+                    return True
+
+        return False
+
+    @classmethod
+    def get_permissions(cls, rid, resource_name):
+        resource = Resource.get_by(name=resource_name, first=True, to_dict=False)
+        resource = resource or abort(403, "Resource <{0}> is not in ACL".format(resource_name))
+
+        parent_ids = RoleRelationCRUD.recursive_parent_ids(rid)
+        group_ids = cls.get_group_ids(resource.id)
+
+        perms = []
+        for parent_id in parent_ids:
+            id2perms = RoleRelationCache.get_resources(parent_id)
+            perms += id2perms['id2perms'].get(parent_id, [])
+
+            for group_id in group_ids:
+                perms += id2perms['group2perms'].get(group_id, [])
+
+        return set(perms)

cmdb-api/api/lib/perm/acl/user.py

@@ -0,0 +1,82 @@
+# -*- coding:utf-8 -*-
+
+
+import random
+import string
+import uuid
+
+from flask import abort
+from flask import g
+
+from api.extensions import db
+from api.lib.perm.acl.cache import UserCache
+from api.lib.perm.acl.role import RoleCRUD
+from api.models.acl import Role
+from api.models.acl import User
+
+
+class UserCRUD(object):
+    @staticmethod
+    def search(q, page=1, page_size=None):
+        query = db.session.query(User).filter(User.deleted.is_(False))
+        if q:
+            query = query.filter(User.username.ilike('%{0}%'.format(q)))
+
+        numfound = query.count()
+
+        return numfound, query.offset((page - 1) * page_size).limit(page_size)
+
+    @staticmethod
+    def _gen_key_secret():
+        key = uuid.uuid4().hex
+        secret = ''.join(random.sample(string.ascii_letters + string.digits + '~!@#$%^&*?', 32))
+
+        return key, secret
+
+    @classmethod
+    def add(cls, **kwargs):
+        existed = User.get_by(username=kwargs['username'], email=kwargs['email'])
+        existed and abort(400, "User <{0}> is already existed".format(kwargs['username']))
+
+        kwargs['nickname'] = kwargs.get('nickname') or kwargs['username']
+        kwargs['block'] = 0
+        kwargs['key'], kwargs['secret'] = cls._gen_key_secret()
+
+        user = User.create(**kwargs)
+
+        RoleCRUD.add_role(user.username, uid=user.uid)
+
+        return user
+
+    @staticmethod
+    def update(uid, **kwargs):
+        user = User.get_by(uid=uid, to_dict=False, first=True) or abort(404, "User <{0}> does not exist".format(uid))
+
+        if kwargs.get("username"):
+            other = User.get_by(username=kwargs['username'], first=True, to_dict=False)
+            if other is not None and other.uid != user.uid:
+                return abort(400, "User <{0}> cannot be duplicated".format(kwargs['username']))
+
+        UserCache.clean(user)
+
+        if kwargs.get("username") and kwargs['username'] != user.username:
+            role = Role.get_by(name=user.username, first=True, to_dict=False)
+            if role is not None:
+                RoleCRUD.update_role(role.id, **dict(name=kwargs['name']))
+
+        return user.update(**kwargs)
+
+    @classmethod
+    def reset_key_secret(cls):
+        key, secret = cls._gen_key_secret()
+        g.user.update(key=key, secret=secret)
+
+        return key, secret
+
+    @classmethod
+    def delete(cls, uid):
+        user = User.get_by(uid=uid, to_dict=False, first=True) or abort(404, "User <{0}> does not exist".format(uid))
+
+        UserCache.clean(user)
+
+        user.soft_delete()

cmdb-api/api/lib/perm/auth.py

@@ -0,0 +1,104 @@
+# -*- coding:utf-8 -*- 
+
+
+from __future__ import unicode_literals
+
+from functools import wraps
+
+import jwt
+from flask import abort
+from flask import current_app
+from flask import g
+from flask import request
+from flask import session
+from flask_login import login_user
+
+from api.models.acl import User
+from api.lib.perm.acl.cache import UserCache
+
+
+def _auth_with_key():
+    key = request.values.get('_key')
+    secret = request.values.get('_secret')
+    path = request.path
+    keys = sorted(request.values.keys())
+    req_args = [request.values[k] for k in keys if k not in ("_key", "_secret")]
+    user, authenticated = User.query.authenticate_with_key(key, secret, req_args, path)
+    if user and authenticated:
+        login_user(user)
+        return True
+    return False
+
+
+def _auth_with_session():
+    if isinstance(getattr(g, 'user', None), User):
+        login_user(g.user)
+        return True
+    if "acl" in session and "userName" in (session["acl"] or {}):
+        login_user(UserCache.get(session["acl"]["userName"]))
+        return True
+    return False
+
+
+def _auth_with_token():
+    auth_headers = request.headers.get('Access-Token', '').strip()
+    if not auth_headers:
+        return False
+
+    try:
+        token = auth_headers
+        data = jwt.decode(token, current_app.config['SECRET_KEY'])
+        user = User.query.filter_by(email=data['sub']).first()
+        if not user:
+            return False
+
+        login_user(user)
+        return True
+    except jwt.ExpiredSignatureError:
+        return False
+    except (jwt.InvalidTokenError, Exception):
+        return False
+
+
+def _auth_with_ip_white_list():
+    ip = request.remote_addr
+    key = request.values.get('_key')
+    secret = request.values.get('_secret')
+
+    if not key and not secret and ip.strip() in current_app.config.get("WHITE_LIST", []):  # TODO
+        user = UserCache.get("worker")
+        login_user(user)
+        return True
+    return False
+
+
+def auth_required(func):
+    if request.json is not None:
+        setattr(request, 'values', request.json)
+    else:
+        setattr(request, 'values', request.values.to_dict())
+
+    current_app.logger.debug(request.values)
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+
+        if not getattr(func, 'authenticated', True):
+            return func(*args, **kwargs)
+
+        if _auth_with_session() or _auth_with_key() or _auth_with_token() or _auth_with_ip_white_list():
+            return func(*args, **kwargs)
+
+        abort(401)
+
+    return wrapper
+
+
+def auth_abandoned(func):
+    setattr(func, "authenticated", False)
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        return func(*args, **kwargs)
+
+    return wrapper

cmdb-api/api/lib/utils.py

@@ -0,0 +1,165 @@
+# -*- coding:utf-8 -*- 
+
+import redis
+import six
+from elasticsearch import Elasticsearch
+from flask import current_app
+
+
+def get_page(page):
+    try:
+        page = int(page)
+    except ValueError:
+        page = 1
+    return page if page >= 1 else 1
+
+
+def get_page_size(page_size):
+    if page_size == "all":
+        return page_size
+
+    try:
+        page_size = int(page_size)
+    except (ValueError, TypeError):
+        page_size = current_app.config.get("DEFAULT_PAGE_COUNT")
+    return page_size if page_size >= 1 else current_app.config.get("DEFAULT_PAGE_COUNT")
+
+
+def handle_arg_list(arg):
+    return list(filter(lambda x: x != "", arg.strip().split(","))) if isinstance(arg, six.string_types) else arg
+
+
+class BaseEnum(object):
+    _ALL_ = set()  # type: Set[str]
+
+    @classmethod
+    def is_valid(cls, item):
+        return item in cls.all()
+
+    @classmethod
+    def all(cls):
+        if not cls._ALL_:
+            cls._ALL_ = {
+                getattr(cls, attr)
+                for attr in dir(cls)
+                if not attr.startswith("_") and not callable(getattr(cls, attr))
+            }
+        return cls._ALL_
+
+
+class RedisHandler(object):
+    def __init__(self, flask_app=None):
+        self.flask_app = flask_app
+        self.r = None
+
+    def init_app(self, app):
+        self.flask_app = app
+        config = self.flask_app.config
+        try:
+            pool = redis.ConnectionPool(
+                max_connections=config.get("REDIS_MAX_CONN"),
+                host=config.get("CACHE_REDIS_HOST"),
+                port=config.get("CACHE_REDIS_PORT"),
+                db=config.get("REDIS_DB"))
+            self.r = redis.Redis(connection_pool=pool)
+        except Exception as e:
+            current_app.logger.warning(str(e))
+            current_app.logger.error("init redis connection failed")
+
+    def get(self, key_ids, prefix):
+        try:
+            value = self.r.hmget(prefix, key_ids)
+        except Exception as e:
+            current_app.logger.error("get redis error, {0}".format(str(e)))
+            return
+        return value
+
+    def _set(self, obj, prefix):
+        try:
+            self.r.hmset(prefix, obj)
+        except Exception as e:
+            current_app.logger.error("set redis error, {0}".format(str(e)))
+
+    def create_or_update(self, obj, prefix):
+        self._set(obj, prefix)
+
+    def delete(self, key_id, prefix):
+        try:
+            ret = self.r.hdel(prefix, key_id)
+            if not ret:
+                current_app.logger.warn("[{0}] is not in redis".format(key_id))
+        except Exception as e:
+            current_app.logger.error("delete redis key error, {0}".format(str(e)))
+
+
+class ESHandler(object):
+    def __init__(self, flask_app=None):
+        self.flask_app = flask_app
+        self.es = None
+        self.index = "cmdb"
+
+    def init_app(self, app):
+        self.flask_app = app
+        config = self.flask_app.config
+        self.es = Elasticsearch(config.get("ES_HOST"))
+        if not self.es.indices.exists(index=self.index):
+            self.es.indices.create(index=self.index)
+
+    def update_mapping(self, field, value_type, other):
+        body = {
+            "properties": {
+                field: {"type": value_type},
+            }}
+        body['properties'][field].update(other)
+
+        self.es.indices.put_mapping(
+            index=self.index,
+            body=body
+        )
+
+    def get_index_id(self, ci_id):
+        query = {
+            'query': {
+                'match': {'ci_id': ci_id}
+            },
+        }
+        res = self.es.search(index=self.index, body=query)
+        if res['hits']['hits']:
+            return res['hits']['hits'][-1].get('_id')
+
+    def create(self, body):
+        return self.es.index(index=self.index, body=body).get("_id")
+
+    def update(self, ci_id, body):
+        _id = self.get_index_id(ci_id)
+
+        if _id:
+            return self.es.index(index=self.index, id=_id, body=body).get("_id")
+
+    def create_or_update(self, ci_id, body):
+        try:
+            self.update(ci_id, body) or self.create(body)
+        except KeyError:
+            self.create(body)
+
+    def delete(self, ci_id):
+        try:
+            _id = self.get_index_id(ci_id)
+        except KeyError:
+            return
+
+        if _id:
+            self.es.delete(index=self.index, id=_id)
+
+    def read(self, query, filter_path=None):
+        filter_path = filter_path or []
+        if filter_path:
+            filter_path.append('hits.total')
+
+        res = self.es.search(index=self.index, body=query, filter_path=filter_path)
+        if res['hits'].get('hits'):
+            return res['hits']['total']['value'], \
+                   [i['_source'] for i in res['hits']['hits']], \
+                   res.get("aggregations", {})
+        else:
+            return 0, [], {}

cmdb-api/api/models/__init__.py

@@ -0,0 +1,5 @@
+# -*- coding:utf-8 -*- 
+
+
+from .cmdb import *
+from .acl import *

cmdb-api/api/models/acl.py

@@ -0,0 +1,190 @@
+# -*- coding:utf-8 -*-
+
+
+import copy
+import hashlib
+from datetime import datetime
+
+from flask import current_app
+from flask_sqlalchemy import BaseQuery
+
+from api.extensions import db
+from api.lib.database import CRUDModel
+from api.lib.database import Model
+from api.lib.database import SoftDeleteMixin
+
+
+class App(Model):
+    __tablename__ = "acl_apps"
+
+    name = db.Column(db.String(64), index=True)
+    description = db.Column(db.Text)
+    app_id = db.Column(db.Text)
+    secret_key = db.Column(db.Text)
+
+
+class UserQuery(BaseQuery):
+    def _join(self, *args, **kwargs):
+        super(UserQuery, self)._join(*args, **kwargs)
+
+    def authenticate(self, login, password):
+        user = self.filter(db.or_(User.username == login,
+                                  User.email == login)).filter(User.deleted.is_(False)).first()
+        if user:
+            current_app.logger.info(user)
+            authenticated = user.check_password(password)
+        else:
+            authenticated = False
+
+        return user, authenticated
+
+    def authenticate_with_key(self, key, secret, args, path):
+        user = self.filter(User.key == key).filter(User.deleted.is_(False)).filter(User.block == 0).first()
+        if not user:
+            return None, False
+        if user and hashlib.sha1('{0}{1}{2}'.format(
+                path, user.secret, "".join(args)).encode("utf-8")).hexdigest() == secret:
+            authenticated = True
+        else:
+            authenticated = False
+
+        return user, authenticated
+
+    def search(self, key):
+        query = self.filter(db.or_(User.email == key,
+                                   User.nickname.ilike('%' + key + '%'),
+                                   User.username.ilike('%' + key + '%'))).filter(User.deleted.is_(False))
+        return query
+
+    def get_by_username(self, username):
+        user = self.filter(User.username == username).filter(User.deleted.is_(False)).first()
+
+        return user
+
+    def get_by_nickname(self, nickname):
+        user = self.filter(User.nickname == nickname).filter(User.deleted.is_(False)).first()
+
+        return user
+
+    def get(self, uid):
+        user = self.filter(User.uid == uid).filter(User.deleted.is_(False)).first()
+
+        return copy.deepcopy(user)
+
+
+class User(CRUDModel, SoftDeleteMixin):
+    __tablename__ = 'users'
+    # __bind_key__ = "user"
+    query_class = UserQuery
+
+    uid = db.Column(db.Integer, primary_key=True, autoincrement=True)
+    username = db.Column(db.String(32), unique=True)
+    nickname = db.Column(db.String(20), nullable=True)
+    department = db.Column(db.String(20))
+    catalog = db.Column(db.String(64))
+    email = db.Column(db.String(100), unique=True, nullable=False)
+    mobile = db.Column(db.String(14), unique=True)
+    _password = db.Column("password", db.String(80))
+    key = db.Column(db.String(32), nullable=False)
+    secret = db.Column(db.String(32), nullable=False)
+    date_joined = db.Column(db.DateTime, default=datetime.utcnow)
+    last_login = db.Column(db.DateTime, default=datetime.utcnow)
+    block = db.Column(db.Boolean, default=False)
+    has_logined = db.Column(db.Boolean, default=False)
+    wx_id = db.Column(db.String(32))
+    avatar = db.Column(db.String(128))
+
+    def __str__(self):
+        return self.username
+
+    def is_active(self):
+        return not self.block
+
+    def get_id(self):
+        return self.uid
+
+    @staticmethod
+    def is_authenticated():
+        return True
+
+    def _get_password(self):
+        return self._password
+
+    def _set_password(self, password):
+        self._password = hashlib.md5(password.encode('utf-8')).hexdigest()
+
+    password = db.synonym("_password", descriptor=property(_get_password, _set_password))
+
+    def check_password(self, password):
+        if self.password is None:
+            return False
+        return self.password == password
+
+
+class Role(Model):
+    __tablename__ = "acl_roles"
+
+    name = db.Column(db.Text, nullable=False)
+    is_app_admin = db.Column(db.Boolean, default=False)
+    app_id = db.Column(db.Integer, db.ForeignKey("acl_apps.id"))
+    uid = db.Column(db.Integer, db.ForeignKey("users.uid"))
+
+
+class RoleRelation(Model):
+    __tablename__ = "acl_role_relations"
+
+    parent_id = db.Column(db.Integer, db.ForeignKey('acl_roles.id'))
+    child_id = db.Column(db.Integer, db.ForeignKey('acl_roles.id'))
+
+
+class ResourceType(Model):
+    __tablename__ = "acl_resource_types"
+
+    name = db.Column(db.String(64), index=True)
+    description = db.Column(db.Text)
+    app_id = db.Column(db.Integer, db.ForeignKey('acl_apps.id'))
+
+
+class ResourceGroup(Model):
+    __tablename__ = "acl_resource_groups"
+
+    name = db.Column(db.String(64), index=True, nullable=False)
+    resource_type_id = db.Column(db.Integer, db.ForeignKey("acl_resource_types.id"))
+
+    app_id = db.Column(db.Integer, db.ForeignKey('acl_apps.id'))
+
+
+class Resource(Model):
+    __tablename__ = "acl_resources"
+
+    name = db.Column(db.String(128), nullable=False)
+    resource_type_id = db.Column(db.Integer, db.ForeignKey("acl_resource_types.id"))
+
+    app_id = db.Column(db.Integer, db.ForeignKey("acl_apps.id"))
+
+
+class ResourceGroupItems(Model):
+    __tablename__ = "acl_resource_group_items"
+
+    group_id = db.Column(db.Integer, db.ForeignKey('acl_resource_groups.id'), nullable=False)
+    resource_id = db.Column(db.Integer, db.ForeignKey('acl_resources.id'), nullable=False)
+
+
+class Permission(Model):
+    __tablename__ = "acl_permissions"
+
+    name = db.Column(db.String(64), nullable=False)
+    resource_type_id = db.Column(db.Integer, db.ForeignKey("acl_resource_types.id"))
+
+    app_id = db.Column(db.Integer, db.ForeignKey("acl_apps.id"))
+
+
+class RolePermission(Model):
+    __tablename__ = "acl_role_permissions"
+
+    rid = db.Column(db.Integer, db.ForeignKey('acl_roles.id'))
+    resource_id = db.Column(db.Integer, db.ForeignKey('acl_resources.id'))
+    group_id = db.Column(db.Integer, db.ForeignKey('acl_resource_groups.id'))
+    perm_id = db.Column(db.Integer, db.ForeignKey('acl_permissions.id'))
+
+    perm = db.relationship("Permission", backref='acl_role_permissions.perm_id')

cmdb-api/api/models/cmdb.py

@@ -0,0 +1,310 @@
+# -*- coding:utf-8 -*-
+
+
+import datetime
+
+from api.extensions import db
+from api.lib.cmdb.const import CIStatusEnum
+from api.lib.cmdb.const import OperateType
+from api.lib.cmdb.const import ValueTypeEnum
+from api.lib.database import Model
+
+
+# template
+
+class RelationType(Model):
+    __tablename__ = "c_relation_types"
+
+    name = db.Column(db.String(16), index=True)
+
+
+class CITypeGroup(Model):
+    __tablename__ = "c_ci_type_groups"
+
+    name = db.Column(db.String(32))
+
+
+class CITypeGroupItem(Model):
+    __tablename__ = "c_ci_type_group_items"
+
+    group_id = db.Column(db.Integer, db.ForeignKey("c_ci_type_groups.id"), nullable=False)
+    type_id = db.Column(db.Integer, db.ForeignKey("c_ci_types.id"), nullable=False)
+    order = db.Column(db.SmallInteger, default=0)
+
+
+class CIType(Model):
+    __tablename__ = "c_ci_types"
+
+    name = db.Column(db.String(32))
+    alias = db.Column(db.String(32))
+    unique_id = db.Column(db.Integer, db.ForeignKey("c_attributes.id"), nullable=False)
+    enabled = db.Column(db.Boolean, default=True, nullable=False)
+    is_attached = db.Column(db.Boolean, default=False, nullable=False)
+    icon_url = db.Column(db.String(256))
+    order = db.Column(db.SmallInteger, default=0, nullable=False)
+
+    unique_key = db.relationship("Attribute", backref="c_ci_types.unique_id")
+
+
+class CITypeRelation(Model):
+    __tablename__ = "c_ci_type_relations"
+
+    parent_id = db.Column(db.Integer, db.ForeignKey("c_ci_types.id"), nullable=False)
+    child_id = db.Column(db.Integer, db.ForeignKey("c_ci_types.id"), nullable=False)
+    relation_type_id = db.Column(db.Integer, db.ForeignKey("c_relation_types.id"), nullable=False)
+
+    parent = db.relationship("CIType", primaryjoin="CIType.id==CITypeRelation.parent_id")
+    child = db.relationship("CIType", primaryjoin="CIType.id==CITypeRelation.child_id")
+    relation_type = db.relationship("RelationType", backref="c_ci_type_relations.relation_type_id")
+
+
+class Attribute(Model):
+    __tablename__ = "c_attributes"
+
+    name = db.Column(db.String(32), nullable=False)
+    alias = db.Column(db.String(32), nullable=False)
+    value_type = db.Column(db.Enum(*ValueTypeEnum.all()), default=ValueTypeEnum.TEXT, nullable=False)
+
+    is_choice = db.Column(db.Boolean, default=False)
+    is_list = db.Column(db.Boolean, default=False)
+    is_unique = db.Column(db.Boolean, default=False)
+    is_index = db.Column(db.Boolean, default=False)
+    is_link = db.Column(db.Boolean, default=False)
+    is_password = db.Column(db.Boolean, default=False)
+    is_sortable = db.Column(db.Boolean, default=False)
+
+
+class CITypeAttribute(Model):
+    __tablename__ = "c_ci_type_attributes"
+
+    type_id = db.Column(db.Integer, db.ForeignKey("c_ci_types.id"), nullable=False)
+    attr_id = db.Column(db.Integer, db.ForeignKey("c_attributes.id"), nullable=False)
+    order = db.Column(db.Integer, default=0)
+    is_required = db.Column(db.Boolean, default=False)
+    default_show = db.Column(db.Boolean, default=True)
+
+    attr = db.relationship("Attribute", backref="c_ci_type_attributes.attr_id")
+
+
+class CITypeAttributeGroup(Model):
+    __tablename__ = "c_ci_type_attribute_groups"
+
+    name = db.Column(db.String(64))
+    type_id = db.Column(db.Integer, db.ForeignKey("c_ci_types.id"), nullable=False)
+    order = db.Column(db.SmallInteger, default=0)
+
+
+class CITypeAttributeGroupItem(Model):
+    __tablename__ = "c_ci_type_attribute_group_items"
+
+    group_id = db.Column(db.Integer, db.ForeignKey("c_ci_type_attribute_groups.id"), nullable=False)
+    attr_id = db.Column(db.Integer, db.ForeignKey("c_attributes.id"), nullable=False)
+    order = db.Column(db.SmallInteger, default=0)
+
+
+# instance
+
+class CI(Model):
+    __tablename__ = "c_cis"
+
+    type_id = db.Column(db.Integer, db.ForeignKey("c_ci_types.id"), nullable=False)
+    status = db.Column(db.Enum(*CIStatusEnum.all(), name="status"))
+    heartbeat = db.Column(db.DateTime, default=lambda: datetime.datetime.now())
+
+    ci_type = db.relationship("CIType", backref="c_cis.type_id")
+
+
+class CIRelation(Model):
+    __tablename__ = "c_ci_relations"
+
+    first_ci_id = db.Column(db.Integer, db.ForeignKey("c_cis.id"), nullable=False)
+    second_ci_id = db.Column(db.Integer, db.ForeignKey("c_cis.id"), nullable=False)
+    relation_type_id = db.Column(db.Integer, db.ForeignKey("c_relation_types.id"), nullable=False)
+    more = db.Column(db.Integer, db.ForeignKey("c_cis.id"))
+
+    first_ci = db.relationship("CI", primaryjoin="CI.id==CIRelation.first_ci_id")
+    second_ci = db.relationship("CI", primaryjoin="CI.id==CIRelation.second_ci_id")
+    relation_type = db.relationship("RelationType", backref="c_ci_relations.relation_type_id")
+
+
+class IntegerChoice(Model):
+    __tablename__ = 'c_choice_integers'
+
+    attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False)
+    value = db.Column(db.Integer, nullable=False)
+
+    attr = db.relationship("Attribute", backref="c_choice_integers.attr_id")
+
+
+class FloatChoice(Model):
+    __tablename__ = 'c_choice_floats'
+
+    attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False)
+    value = db.Column(db.Float, nullable=False)
+
+    attr = db.relationship("Attribute", backref="c_choice_floats.attr_id")
+
+
+class TextChoice(Model):
+    __tablename__ = 'c_choice_texts'
+
+    attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False)
+    value = db.Column(db.Text, nullable=False)
+
+    attr = db.relationship("Attribute", backref="c_choice_texts.attr_id")
+
+
+class CIIndexValueInteger(Model):
+    __tablename__ = "c_value_index_integers"
+
+    ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False)
+    attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False)
+    value = db.Column(db.Integer, nullable=False)
+
+    ci = db.relationship("CI", backref="c_value_index_integers.ci_id")
+    attr = db.relationship("Attribute", backref="c_value_index_integers.attr_id")
+
+    __table_args__ = (db.Index("integer_attr_value_index", "attr_id", "value"),)
+
+
+class CIIndexValueFloat(Model):
+    __tablename__ = "c_value_index_floats"
+
+    ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False)
+    attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False)
+    value = db.Column(db.Float, nullable=False)
+
+    ci = db.relationship("CI", backref="c_value_index_floats.ci_id")
+    attr = db.relationship("Attribute", backref="c_value_index_floats.attr_id")
+
+    __table_args__ = (db.Index("float_attr_value_index", "attr_id", "value"),)
+
+
+class CIIndexValueText(Model):
+    __tablename__ = "c_value_index_texts"
+
+    ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False)
+    attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False)
+    value = db.Column(db.String(128), nullable=False)
+
+    ci = db.relationship("CI", backref="c_value_index_texts.ci_id")
+    attr = db.relationship("Attribute", backref="c_value_index_texts.attr_id")
+
+    __table_args__ = (db.Index("text_attr_value_index", "attr_id", "value"),)
+
+
+class CIIndexValueDateTime(Model):
+    __tablename__ = "c_value_index_datetime"
+
+    ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False)
+    attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False)
+    value = db.Column(db.DateTime, nullable=False)
+
+    ci = db.relationship("CI", backref="c_value_index_datetime.ci_id")
+    attr = db.relationship("Attribute", backref="c_value_index_datetime.attr_id")
+
+    __table_args__ = (db.Index("datetime_attr_value_index", "attr_id", "value"),)
+
+
+class CIValueInteger(Model):
+    __tablename__ = "c_value_integers"
+
+    ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False)
+    attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False)
+    value = db.Column(db.Integer, nullable=False)
+
+    ci = db.relationship("CI", backref="c_value_integers.ci_id")
+    attr = db.relationship("Attribute", backref="c_value_integers.attr_id")
+
+
+class CIValueFloat(Model):
+    __tablename__ = "c_value_floats"
+
+    ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False)
+    attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False)
+    value = db.Column(db.Float, nullable=False)
+
+    ci = db.relationship("CI", backref="c_value_floats.ci_id")
+    attr = db.relationship("Attribute", backref="c_value_floats.attr_id")
+
+
+class CIValueText(Model):
+    __tablename__ = "c_value_texts"
+
+    ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False)
+    attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False)
+    value = db.Column(db.Text, nullable=False)
+
+    ci = db.relationship("CI", backref="c_value_texts.ci_id")
+    attr = db.relationship("Attribute", backref="c_value_texts.attr_id")
+
+
+class CIValueDateTime(Model):
+    __tablename__ = "c_value_datetime"
+
+    ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False)
+    attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False)
+    value = db.Column(db.DateTime, nullable=False)
+
+    ci = db.relationship("CI", backref="c_value_datetime.ci_id")
+    attr = db.relationship("Attribute", backref="c_value_datetime.attr_id")
+
+
+# history
+class OperationRecord(Model):
+    __tablename__ = "c_records"
+
+    uid = db.Column(db.Integer, index=True, nullable=False)
+    origin = db.Column(db.String(32))
+    ticket_id = db.Column(db.String(32))
+    reason = db.Column(db.Text)
+
+
+class AttributeHistory(Model):
+    __tablename__ = "c_attribute_histories"
+
+    operate_type = db.Column(db.Enum(*OperateType.all(), name="operate_type"))
+    record_id = db.Column(db.Integer, db.ForeignKey("c_records.id"), nullable=False)
+    ci_id = db.Column(db.Integer, index=True, nullable=False)
+    attr_id = db.Column(db.Integer, index=True)
+    old = db.Column(db.Text)
+    new = db.Column(db.Text)
+
+
+class CIRelationHistory(Model):
+    __tablename__ = "c_relation_histories"
+
+    operate_type = db.Column(db.Enum(OperateType.ADD, OperateType.DELETE, name="operate_type"))
+    record_id = db.Column(db.Integer, db.ForeignKey("c_records.id"), nullable=False)
+    first_ci_id = db.Column(db.Integer)
+    second_ci_id = db.Column(db.Integer)
+    relation_type_id = db.Column(db.Integer, db.ForeignKey("c_relation_types.id"))
+    relation_id = db.Column(db.Integer, nullable=False)
+
+
+# preference
+class PreferenceShowAttributes(Model):
+    __tablename__ = "c_preference_show_attributes"
+
+    uid = db.Column(db.Integer, index=True, nullable=False)
+    type_id = db.Column(db.Integer, db.ForeignKey("c_ci_types.id"), nullable=False)
+    attr_id = db.Column(db.Integer, db.ForeignKey("c_attributes.id"))
+    order = db.Column(db.SmallInteger, default=0)
+
+    ci_type = db.relationship("CIType", backref="c_preference_show_attributes.type_id")
+    attr = db.relationship("Attribute", backref="c_preference_show_attributes.attr_id")
+
+
+class PreferenceTreeView(Model):
+    __tablename__ = "c_preference_tree_views"
+
+    uid = db.Column(db.Integer, index=True, nullable=False)
+    type_id = db.Column(db.Integer, db.ForeignKey("c_ci_types.id"), nullable=False)
+    levels = db.Column(db.Text)  # TODO: JSON
+
+
+class PreferenceRelationView(Model):
+    __tablename__ = "c_preference_relation_views"
+
+    name = db.Column(db.String(8), index=True, nullable=False)
+    cr_ids = db.Column(db.TEXT)  # [{parent_id: x, child_id: y}] TODO: JSON

cmdb-api/api/resource.py

@@ -0,0 +1,45 @@
+# -*- coding:utf-8 -*-
+
+import os
+import sys
+from inspect import getmembers, isclass
+
+import six
+from flask import jsonify
+from flask_restful import Resource
+
+from api.lib.perm.auth import auth_required
+
+
+class APIView(Resource):
+    method_decorators = [auth_required]
+
+    def __init__(self):
+        super(APIView, self).__init__()
+
+    @staticmethod
+    def jsonify(*args, **kwargs):
+        return jsonify(*args, **kwargs)
+
+
+API_PACKAGE = "api"
+
+
+def register_resources(resource_path, rest_api):
+    for root, _, files in os.walk(os.path.join(resource_path)):
+        for filename in files:
+            if not filename.startswith("_") and filename.endswith("py"):
+                module_path = os.path.join(API_PACKAGE, root[root.index("views"):])
+                if module_path not in sys.path:
+                    sys.path.insert(1, module_path)
+                view = __import__(os.path.splitext(filename)[0])
+                resource_list = [o[0] for o in getmembers(view) if isclass(o[1]) and issubclass(o[1], Resource)]
+                resource_list = [i for i in resource_list if i != "APIView"]
+                for resource_cls_name in resource_list:
+                    resource_cls = getattr(view, resource_cls_name)
+                    if not hasattr(resource_cls, "url_prefix"):
+                        resource_cls.url_prefix = ("",)
+                    if isinstance(resource_cls.url_prefix, six.string_types):
+                        resource_cls.url_prefix = (resource_cls.url_prefix,)
+
+                    rest_api.add_resource(resource_cls, *resource_cls.url_prefix)

cmdb-api/api/tasks/__init__.py

@@ -0,0 +1 @@
+# -*- coding:utf-8 -*- 

cmdb-api/api/tasks/acl.py

@@ -0,0 +1,16 @@
+# -*- coding:utf-8 -*-
+
+from flask import current_app
+
+from api.extensions import celery
+from api.lib.perm.acl.cache import RoleRelationCache
+from api.lib.perm.acl.const import ACL_QUEUE
+
+
+@celery.task(name="acl.role_rebuild", queue=ACL_QUEUE)
+def role_rebuild(rids):
+    rids = rids if isinstance(rids, list) else [rids]
+    for rid in rids:
+        RoleRelationCache.rebuild(rid)
+
+    current_app.logger.info("Role {0} rebuild..........".format(rids))

cmdb-api/api/tasks/cmdb.py

@@ -0,0 +1,73 @@
+# -*- coding:utf-8 -*- 
+
+
+import json
+import time
+
+from flask import current_app
+
+import api.lib.cmdb.ci
+from api.extensions import celery
+from api.extensions import db
+from api.extensions import es
+from api.extensions import rd
+from api.lib.cmdb.const import CMDB_QUEUE
+from api.lib.cmdb.const import REDIS_PREFIX_CI
+from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
+from api.models.cmdb import CIRelation
+
+
+@celery.task(name="cmdb.ci_cache", queue=CMDB_QUEUE)
+def ci_cache(ci_id):
+    time.sleep(0.01)
+    db.session.close()
+
+    m = api.lib.cmdb.ci.CIManager()
+    ci = m.get_ci_by_id_from_db(ci_id, need_children=False, use_master=False)
+    if current_app.config.get("USE_ES"):
+        es.create_or_update(ci_id, ci)
+    else:
+        rd.create_or_update({ci_id: json.dumps(ci)}, REDIS_PREFIX_CI)
+
+    current_app.logger.info("{0} flush..........".format(ci_id))
+
+
+@celery.task(name="cmdb.ci_delete", queue=CMDB_QUEUE)
+def ci_delete(ci_id):
+    current_app.logger.info(ci_id)
+
+    if current_app.config.get("USE_ES"):
+        es.delete(ci_id)
+    else:
+        rd.delete(ci_id, REDIS_PREFIX_CI)
+
+    current_app.logger.info("{0} delete..........".format(ci_id))
+
+
+@celery.task(name="cmdb.ci_relation_cache", queue=CMDB_QUEUE)
+def ci_relation_cache(parent_id, child_id):
+    db.session.close()
+
+    children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0]
+    children = json.loads(children) if children is not None else {}
+
+    cr = CIRelation.get_by(first_ci_id=parent_id, second_ci_id=child_id, first=True, to_dict=False)
+    if str(child_id) not in children:
+        children[str(child_id)] = cr.second_ci.type_id
+
+    rd.create_or_update({parent_id: json.dumps(children)}, REDIS_PREFIX_CI_RELATION)
+
+    current_app.logger.info("ADD ci relation cache: {0} -> {1}".format(parent_id, child_id))
+
+
+@celery.task(name="cmdb.ci_relation_delete", queue=CMDB_QUEUE)
+def ci_relation_delete(parent_id, child_id):
+    children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0]
+    children = json.loads(children) if children is not None else {}
+
+    if str(child_id) in children:
+        children.pop(str(child_id))
+
+    rd.create_or_update({parent_id: json.dumps(children)}, REDIS_PREFIX_CI_RELATION)
+
+    current_app.logger.info("DELETE ci relation cache: {0} -> {1}".format(parent_id, child_id))

cmdb-api/api/tasks/test.py

@@ -0,0 +1,10 @@
+# -*- coding:utf-8 -*-
+
+from flask import current_app
+
+from api.extensions import celery
+
+
+@celery.task(queue="ticket_web")
+def test_task():
+    current_app.logger.info("test task.............................")

cmdb-api/api/views/__init__.py

@@ -0,0 +1,29 @@
+# -*- coding:utf-8 -*-
+
+import os
+
+from flask import Blueprint
+from flask_restful import Api
+
+from api.resource import register_resources
+from api.views.account import LoginView, LogoutView
+
+HERE = os.path.abspath(os.path.dirname(__file__))
+
+# account
+blueprint_account = Blueprint('account_api', __name__, url_prefix='/api')
+account_rest = Api(blueprint_account)
+account_rest.add_resource(LoginView, LoginView.url_prefix)
+account_rest.add_resource(LogoutView, LogoutView.url_prefix)
+
+
+# cmdb
+blueprint_cmdb_v01 = Blueprint('cmdb_api_v01', __name__, url_prefix='/api/v0.1')
+rest = Api(blueprint_cmdb_v01)
+register_resources(os.path.join(HERE, "cmdb"), rest)
+
+
+# acl
+blueprint_acl_v1 = Blueprint('acl_api_v1', __name__, url_prefix='/api/v1/acl')
+rest = Api(blueprint_acl_v1)
+register_resources(os.path.join(HERE, "acl"), rest)

cmdb-api/api/views/account.py

@@ -0,0 +1,64 @@
+# -*- coding:utf-8 -*-
+
+import datetime
+
+import jwt
+from flask import abort
+from flask import current_app
+from flask import request
+from flask import session
+from flask_login import login_user, logout_user
+
+from api.lib.decorator import args_required
+from api.lib.perm.auth import auth_abandoned
+from api.models.acl import User, Role
+from api.resource import APIView
+from api.lib.perm.acl.role import RoleRelationCRUD
+from api.lib.perm.acl.cache import RoleCache
+
+
+class LoginView(APIView):
+    url_prefix = "/login"
+
+    @args_required("username")
+    @args_required("password")
+    @auth_abandoned
+    def post(self):
+        username = request.values.get("username") or request.values.get("email")
+        password = request.values.get("password")
+        user, authenticated = User.query.authenticate(username, password)
+        if not user:
+            return abort(403, "User <{0}> does not exist".format(username))
+        if not authenticated:
+            return abort(403, "invalid username or password")
+
+        login_user(user)
+
+        token = jwt.encode({
+            'sub': user.email,
+            'iat': datetime.datetime.now(),
+            'exp': datetime.datetime.now() + datetime.timedelta(minutes=24 * 60 * 7)},
+            current_app.config['SECRET_KEY'])
+
+        role = Role.get_by(uid=user.uid, first=True, to_dict=False)
+        if role:
+            parent_ids = RoleRelationCRUD.recursive_parent_ids(role.id)
+            parent_roles = [RoleCache.get(i).name for i in parent_ids]
+        else:
+            parent_roles = []
+        session["acl"] = dict(uid=user.uid,
+                              avatar=user.avatar,
+                              userName=user.username,
+                              nickName=user.nickname,
+                              parentRoles=parent_roles)
+
+        return self.jsonify(token=token.decode())
+
+
+class LogoutView(APIView):
+    url_prefix = "/logout"
+
+    @auth_abandoned
+    def post(self):
+        logout_user()
+        self.jsonify(code=200)

cmdb-api/api/views/acl/__init__.py

@@ -0,0 +1 @@
+# -*- coding:utf-8 -*-

cmdb-api/api/views/acl/permission.py

@@ -0,0 +1,40 @@
+# -*- coding:utf-8 -*-
+
+
+from flask import request
+
+from api.lib.decorator import args_required
+from api.lib.perm.acl.permission import PermissionCRUD
+from api.lib.utils import handle_arg_list
+from api.resource import APIView
+
+
+class ResourcePermissionView(APIView):
+    url_prefix = ("/resources/<int:resource_id>/permissions", "/resource_groups/<int:group_id>/permissions")
+
+    def get(self, resource_id=None, group_id=None):
+        return self.jsonify(PermissionCRUD.get_all(resource_id, group_id))
+
+
+class RolePermissionGrantView(APIView):
+    url_prefix = ('/roles/<int:rid>/resources/<int:resource_id>/grant',
+                  '/roles/<int:rid>/resource_groups/<int:group_id>/grant')
+
+    @args_required('perms')
+    def post(self, rid, resource_id=None, group_id=None):
+        perms = handle_arg_list(request.values.get("perms"))
+        PermissionCRUD.grant(rid, perms, resource_id=resource_id, group_id=group_id)
+
+        return self.jsonify(rid=rid, resource_id=resource_id, group_id=group_id, perms=perms)
+
+
+class RolePermissionRevokeView(APIView):
+    url_prefix = ('/roles/<int:rid>/resources/<int:resource_id>/revoke',
+                  '/roles/<int:rid>/resource_groups/<int:group_id>/revoke')
+
+    @args_required('perms')
+    def post(self, rid, resource_id=None, group_id=None):
+        perms = handle_arg_list(request.values.get("perms"))
+        PermissionCRUD.revoke(rid, perms, resource_id=resource_id, group_id=group_id)
+
+        return self.jsonify(rid=rid, resource_id=resource_id, group_id=group_id, perms=perms)

cmdb-api/api/views/acl/resources.py

@@ -0,0 +1,166 @@
+# -*- coding:utf-8 -*-
+
+from flask import request
+
+from api.lib.decorator import args_required
+from api.lib.perm.acl import validate_app
+from api.lib.perm.acl.resource import ResourceCRUD
+from api.lib.perm.acl.resource import ResourceGroupCRUD
+from api.lib.perm.acl.resource import ResourceTypeCRUD
+from api.lib.utils import get_page
+from api.lib.utils import get_page_size
+from api.lib.utils import handle_arg_list
+from api.resource import APIView
+
+
+class ResourceTypeView(APIView):
+    url_prefix = ("/resource_types", "/resource_types/<int:type_id>")
+
+    @args_required('app_id')
+    @validate_app
+    def get(self):
+        page = get_page(request.values.get("page", 1))
+        page_size = get_page_size(request.values.get("page_size"))
+        q = request.values.get('q')
+        app_id = request.values.get('app_id')
+
+        numfound, res, id2perms = ResourceTypeCRUD.search(q, app_id, page, page_size)
+
+        return self.jsonify(numfound=numfound,
+                            page=page,
+                            page_size=page_size,
+                            groups=[i.to_dict() for i in res],
+                            id2perms=id2perms)
+
+    @args_required('name')
+    @args_required('app_id')
+    @args_required('perms')
+    @validate_app
+    def post(self):
+        name = request.values.get('name')
+        app_id = request.values.get('app_id')
+        description = request.values.get('description', '')
+        perms = request.values.get('perms')
+
+        rt = ResourceTypeCRUD.add(app_id, name, description, perms)
+
+        return self.jsonify(rt.to_dict())
+
+    def put(self, type_id):
+        rt = ResourceTypeCRUD.update(type_id, **request.values)
+
+        return self.jsonify(rt.to_dict())
+
+    def delete(self, type_id):
+        ResourceTypeCRUD.delete(type_id)
+
+        return self.jsonify(type_id=type_id)
+
+
+class ResourceTypePermsView(APIView):
+    url_prefix = "/resource_types/<int:type_id>/perms"
+
+    def get(self, type_id):
+        return self.jsonify(ResourceTypeCRUD.get_perms(type_id))
+
+
+class ResourceView(APIView):
+    url_prefix = ("/resources", "/resources/<int:resource_id>")
+
+    @args_required('app_id')
+    @validate_app
+    def get(self):
+        page = get_page(request.values.get("page", 1))
+        page_size = get_page_size(request.values.get("page_size"))
+        q = request.values.get('q')
+        resource_type_id = request.values.get('resource_type_id')
+        app_id = request.values.get('app_id')
+
+        numfound, res = ResourceCRUD.search(q, app_id, resource_type_id, page, page_size)
+
+        return self.jsonify(numfound=numfound,
+                            page=page,
+                            page_size=page_size,
+                            resources=[i.to_dict() for i in res])
+
+    @args_required('name')
+    @args_required('type_id')
+    @args_required('app_id')
+    @validate_app
+    def post(self):
+        name = request.values.get('name')
+        type_id = request.values.get('type_id')
+        app_id = request.values.get('app_id')
+
+        resource = ResourceCRUD.add(name, type_id, app_id)
+
+        return self.jsonify(resource.to_dict())
+
+    @args_required('name')
+    def put(self, resource_id):
+        name = request.values.get('name')
+
+        resource = ResourceCRUD.update(resource_id, name)
+
+        return self.jsonify(resource.to_dict())
+
+    def delete(self, resource_id):
+        ResourceCRUD.delete(resource_id)
+
+        return self.jsonify(resource_id=resource_id)
+
+
+class ResourceGroupView(APIView):
+    url_prefix = ("/resource_groups", "/resource_groups/<int:group_id>")
+
+    @args_required('app_id')
+    @validate_app
+    def get(self):
+        page = get_page(request.values.get("page", 1))
+        page_size = get_page_size(request.values.get("page_size"))
+        q = request.values.get('q')
+        app_id = request.values.get('app_id')
+
+        numfound, res = ResourceGroupCRUD.search(q, app_id, page, page_size)
+
+        return self.jsonify(numfound=numfound,
+                            page=page,
+                            page_size=page_size,
+                            groups=[i.to_dict() for i in res])
+
+    @args_required('name')
+    @args_required('type_id')
+    @args_required('app_id')
+    @validate_app
+    def post(self):
+        name = request.values.get('name')
+        type_id = request.values.get('type_id')
+        app_id = request.values.get('app_id')
+
+        group = ResourceGroupCRUD.add(name, type_id, app_id)
+
+        return self.jsonify(group.to_dict())
+
+    @args_required('items')
+    def put(self, group_id):
+        items = handle_arg_list(request.values.get("items"))
+
+        ResourceGroupCRUD.update(group_id, items)
+
+        items = ResourceGroupCRUD.get_items(group_id)
+
+        return self.jsonify(items)
+
+    def delete(self, group_id):
+        ResourceGroupCRUD.delete(group_id)
+
+        return self.jsonify(group_id=group_id)
+
+
+class ResourceGroupItemsView(APIView):
+    url_prefix = "/resource_groups/<int:group_id>/items"
+
+    def get(self, group_id):
+        items = ResourceGroupCRUD.get_items(group_id)
+
+        return self.jsonify(items)

cmdb-api/api/views/acl/role.py

@@ -0,0 +1,77 @@
+# -*- coding:utf-8 -*-
+
+from flask import current_app
+from flask import request
+
+from api.lib.decorator import args_required
+from api.lib.perm.acl import validate_app
+from api.lib.perm.acl.role import RoleCRUD
+from api.lib.perm.acl.role import RoleRelationCRUD
+from api.lib.utils import get_page
+from api.lib.utils import get_page_size
+from api.resource import APIView
+
+
+class RoleView(APIView):
+    url_prefix = ("/roles", "/roles/<int:rid>")
+
+    @args_required('app_id')
+    @validate_app
+    def get(self):
+        page = get_page(request.values.get("page", 1))
+        page_size = get_page_size(request.values.get("page_size"))
+        q = request.values.get('q')
+        app_id = request.values.get('app_id')
+        user_role = request.values.get('user_role', True)
+        user_role = True if user_role in current_app.config.get("BOOL_TRUE") else False
+
+        numfound, roles = RoleCRUD.search(q, app_id, page, page_size, user_role)
+
+        id2parents = RoleRelationCRUD.get_parents([i.id for i in roles])
+
+        return self.jsonify(numfound=numfound,
+                            page=page,
+                            page_size=page_size,
+                            id2parents=id2parents,
+                            roles=[i.to_dict() for i in roles])
+
+    @args_required('name')
+    @args_required('app_id')
+    @validate_app
+    def post(self):
+        name = request.values.get('name')
+        app_id = request.values.get('app_id')
+        is_app_admin = request.values.get('is_app_admin', False)
+
+        role = RoleCRUD.add_role(name, app_id, is_app_admin=is_app_admin)
+
+        return self.jsonify(role.to_dict())
+
+    def put(self, rid):
+        role = RoleCRUD.update_role(rid, **request.values)
+
+        return self.jsonify(role.to_dict())
+
+    def delete(self, rid):
+        RoleCRUD.delete_role(rid)
+
+        return self.jsonify(rid=rid)
+
+
+class RoleRelationView(APIView):
+    url_prefix = "/roles/<int:child_id>/parents"
+
+    @args_required('parent_id')
+    def post(self, child_id):
+        parent_id = request.values.get('parent_id')
+        res = RoleRelationCRUD.add(parent_id, child_id)
+
+        return self.jsonify(res.to_dict())
+
+    @args_required('parent_id')
+    def delete(self, child_id):
+        parent_id = request.values.get('parent_id')
+
+        RoleRelationCRUD.delete2(parent_id, child_id)
+
+        return self.jsonify(parent_id=parent_id, child_id=child_id)

cmdb-api/api/views/acl/user.py

@@ -0,0 +1,78 @@
+# -*- coding:utf-8 -*-
+
+
+from flask import request
+from flask import session
+from flask_login import current_user
+
+from api.lib.decorator import args_required
+from api.lib.perm.acl.role import RoleRelationCRUD
+from api.lib.perm.acl.user import UserCRUD
+from api.lib.utils import get_page
+from api.lib.utils import get_page_size
+from api.resource import APIView
+
+
+class GetUserInfoView(APIView):
+    url_prefix = "/users/info"
+
+    def get(self):
+        name = session.get("CAS_USERNAME") or current_user.nickname
+        role = dict(permissions=session.get("acl", {}).get("parentRoles", []))
+        avatar = current_user.avatar
+
+        return self.jsonify(result=dict(name=name,
+                                        role=role,
+                                        avatar=avatar))
+
+
+class UserView(APIView):
+    url_prefix = ("/users", "/users/<int:uid>")
+
+    def get(self):
+        page = get_page(request.values.get('page', 1))
+        page_size = get_page_size(request.values.get('page_size'))
+        q = request.values.get("q")
+        numfound, users = UserCRUD.search(q, page, page_size)
+        id2parents = RoleRelationCRUD.get_parents(uids=[i.uid for i in users])
+
+        users = [i.to_dict() for i in users]
+        for u in users:
+            u.pop('password', None)
+            u.pop('key', None)
+            u.pop('secret', None)
+
+        return self.jsonify(numfound=numfound,
+                            page=page,
+                            page_size=page_size,
+                            id2parents=id2parents,
+                            users=users)
+
+    @args_required('username')
+    @args_required('email')
+    def post(self):
+        user = UserCRUD.add(**request.values)
+
+        return self.jsonify(user.to_dict())
+
+    def put(self, uid):
+        user = UserCRUD.update(uid, **request.values)
+
+        return self.jsonify(user.to_dict())
+
+    def delete(self, uid):
+        UserCRUD.delete(uid)
+
+        return self.jsonify(uid=uid)
+
+
+class UserResetKeySecretView(APIView):
+    url_prefix = "/users/reset_key_secret"
+
+    def post(self):
+        key, secret = UserCRUD.reset_key_secret()
+
+        return self.jsonify(key=key, secret=secret)
+
+    def put(self):
+        return self.post()

cmdb-api/api/views/cmdb/__init__.py

@@ -0,0 +1 @@
+# -*- coding:utf-8 -*- 

cmdb-api/api/views/cmdb/attribute.py

@@ -0,0 +1,74 @@
+# -*- coding:utf-8 -*- 
+
+
+from flask import abort
+from flask import current_app
+from flask import request
+
+from api.lib.cmdb.attribute import AttributeManager
+from api.lib.cmdb.const import RoleEnum
+from api.lib.decorator import args_required
+from api.lib.perm.acl.acl import role_required
+from api.lib.utils import get_page
+from api.lib.utils import get_page_size
+from api.lib.utils import handle_arg_list
+from api.resource import APIView
+
+
+class AttributeSearchView(APIView):
+    url_prefix = ("/attributes/s", "/attributes/search")
+
+    def get(self):
+        name = request.values.get("name")
+        alias = request.values.get("alias")
+        page = get_page(request.values.get("page", 1))
+        page_size = get_page_size(request.values.get("page_size"))
+        numfound, res = AttributeManager.search_attributes(name=name, alias=alias, page=page, page_size=page_size)
+
+        return self.jsonify(page=page,
+                            page_size=page_size,
+                            numfound=numfound,
+                            total=len(res),
+                            attributes=res)
+
+
+class AttributeView(APIView):
+    url_prefix = ("/attributes", "/attributes/<string:attr_name>", "/attributes/<int:attr_id>")
+
+    def get(self, attr_name=None, attr_id=None):
+        attr_manager = AttributeManager()
+        attr_dict = None
+        if attr_name is not None:
+            attr_dict = attr_manager.get_attribute_by_name(attr_name)
+            if attr_dict is None:
+                attr_dict = attr_manager.get_attribute_by_alias(attr_name)
+        elif attr_id is not None:
+            attr_dict = attr_manager.get_attribute_by_id(attr_id)
+        if attr_dict is not None:
+            return self.jsonify(attribute=attr_dict)
+        abort(404, "Attribute is not found")
+
+    @role_required(RoleEnum.CONFIG)
+    @args_required("name")
+    def post(self):
+        choice_value = handle_arg_list(request.values.get("choice_value"))
+        params = request.values
+        params["choice_value"] = choice_value
+        current_app.logger.debug(params)
+
+        attr_id = AttributeManager.add(**params)
+        return self.jsonify(attr_id=attr_id)
+
+    @role_required(RoleEnum.CONFIG)
+    def put(self, attr_id):
+        choice_value = handle_arg_list(request.values.get("choice_value"))
+        params = request.values
+        params["choice_value"] = choice_value
+        current_app.logger.debug(params)
+        AttributeManager().update(attr_id, **params)
+        return self.jsonify(attr_id=attr_id)
+
+    @role_required(RoleEnum.CONFIG)
+    def delete(self, attr_id):
+        attr_name = AttributeManager.delete(attr_id)
+        return self.jsonify(message="attribute {0} deleted".format(attr_name))

cmdb-api/api/views/cmdb/ci.py

@@ -0,0 +1,217 @@
+# -*- coding:utf-8 -*- 
+
+import time
+
+import six
+from flask import abort
+from flask import current_app
+from flask import request
+
+from api.lib.cmdb.cache import CITypeCache
+from api.lib.cmdb.ci import CIManager
+from api.lib.cmdb.const import ExistPolicy
+from api.lib.cmdb.const import ResourceTypeEnum, PermEnum
+from api.lib.cmdb.const import RetKey
+from api.lib.cmdb.search import SearchError
+from api.lib.cmdb.search.ci.db.search import Search as SearchFromDB
+from api.lib.cmdb.search.ci.es.search import Search as SearchFromES
+from api.lib.perm.acl.acl import has_perm_from_args
+from api.lib.perm.auth import auth_abandoned
+from api.lib.utils import get_page
+from api.lib.utils import get_page_size
+from api.lib.utils import handle_arg_list
+from api.models.cmdb import CI
+from api.resource import APIView
+
+
+class CIsByTypeView(APIView):
+    url_prefix = "/ci/type/<int:type_id>"
+
+    def get(self, type_id):
+        fields = handle_arg_list(request.values.get("fields", ""))
+
+        ret_key = request.values.get("ret_key", RetKey.NAME)
+        if ret_key not in (RetKey.NAME, RetKey.ALIAS, RetKey.ID):
+            ret_key = RetKey.NAME
+
+        page = get_page(request.values.get("page", 1))
+        count = get_page_size(request.values.get("count"))
+
+        manager = CIManager()
+        res = manager.get_cis_by_type(type_id,
+                                      ret_key=ret_key,
+                                      fields=fields,
+                                      page=page,
+                                      per_page=count)
+
+        return self.jsonify(type_id=type_id,
+                            numfound=res[0],
+                            total=len(res[2]),
+                            page=res[1],
+                            cis=res[2])
+
+
+class CIView(APIView):
+    url_prefix = ("/ci/<int:ci_id>", "/ci")
+
+    def get(self, ci_id):
+        fields = handle_arg_list(request.values.get("fields", ""))
+
+        ret_key = request.values.get("ret_key", RetKey.NAME)
+        if ret_key not in (RetKey.NAME, RetKey.ALIAS, RetKey.ID):
+            ret_key = RetKey.NAME
+
+        manager = CIManager()
+        ci = manager.get_ci_by_id_from_db(ci_id, ret_key=ret_key, fields=fields)
+        return self.jsonify(ci_id=ci_id, ci=ci)
+
+    @staticmethod
+    def _wrap_ci_dict():
+        ci_dict = dict()
+        for k, v in request.values.items():
+            if k != "ci_type" and not k.startswith("_"):
+                ci_dict[k] = v.strip() if isinstance(v, six.string_types) else v
+        return ci_dict
+
+    @has_perm_from_args("ci_type", ResourceTypeEnum.CI, PermEnum.ADD)
+    def post(self):
+        ci_type = request.values.get("ci_type")
+        _no_attribute_policy = request.values.get("_no_attribute_policy", ExistPolicy.IGNORE)
+
+        ci_dict = self._wrap_ci_dict()
+
+        manager = CIManager()
+        current_app.logger.debug(ci_dict)
+        ci_id = manager.add(ci_type,
+                            exist_policy=ExistPolicy.REJECT,
+                            _no_attribute_policy=_no_attribute_policy, **ci_dict)
+        return self.jsonify(ci_id=ci_id)
+
+    @has_perm_from_args("ci_id", ResourceTypeEnum.CI, PermEnum.UPDATE, CIManager.get_type_name)
+    def put(self, ci_id=None):
+        args = request.values
+        ci_type = args.get("ci_type")
+        _no_attribute_policy = args.get("_no_attribute_policy", ExistPolicy.IGNORE)
+
+        ci_dict = self._wrap_ci_dict()
+        manager = CIManager()
+        if ci_id is not None:
+            manager.update(ci_id, **ci_dict)
+        else:
+            ci_id = manager.add(ci_type,
+                                exist_policy=ExistPolicy.REPLACE,
+                                _no_attribute_policy=_no_attribute_policy,
+                                **ci_dict)
+        return self.jsonify(ci_id=ci_id)
+
+    @has_perm_from_args("ci_id", ResourceTypeEnum.CI, PermEnum.DELETE, CIManager.get_type_name)
+    def delete(self, ci_id):
+        manager = CIManager()
+        manager.delete(ci_id)
+        return self.jsonify(message="ok")
+
+
+class CIDetailView(APIView):
+    url_prefix = "/ci/<int:ci_id>/detail"
+
+    def get(self, ci_id):
+        _ci = CI.get_by_id(ci_id).to_dict()
+        return self.jsonify(**_ci)
+
+
+class CISearchView(APIView):
+    url_prefix = ("/ci/s", "/ci/search")
+
+    @auth_abandoned
+    def get(self):
+        """@params: q: query statement
+                    fl: filter by column
+                    count/page_size: the number of ci
+                    ret_key: id, name, alias
+                    facet: statistic
+        """
+
+        page = get_page(request.values.get("page", 1))
+        count = get_page_size(request.values.get("count") or request.values.get("page_size"))
+
+        query = request.values.get('q', "")
+        fl = handle_arg_list(request.values.get('fl', ""))
+        ret_key = request.values.get('ret_key', RetKey.NAME)
+        if ret_key not in (RetKey.NAME, RetKey.ALIAS, RetKey.ID):
+            ret_key = RetKey.NAME
+        facet = handle_arg_list(request.values.get("facet", ""))
+        sort = request.values.get("sort")
+
+        start = time.time()
+        if current_app.config.get("USE_ES"):
+            s = SearchFromES(query, fl, facet, page, ret_key, count, sort)
+        else:
+            s = SearchFromDB(query, fl, facet, page, ret_key, count, sort)
+        try:
+            response, counter, total, page, numfound, facet = s.search()
+        except SearchError as e:
+            return abort(400, str(e))
+        current_app.logger.debug("search time is :{0}".format(time.time() - start))
+        return self.jsonify(numfound=numfound,
+                            total=total,
+                            page=page,
+                            facet=facet,
+                            counter=counter,
+                            result=response)
+
+
+class CIUnique(APIView):
+    url_prefix = "/ci/<int:ci_id>/unique"
+
+    @has_perm_from_args("ci_id", ResourceTypeEnum.CI, PermEnum.UPDATE, CIManager.get_type_name)
+    def put(self, ci_id):
+        params = request.values
+        unique_name = params.keys()[0]
+        unique_value = params.values()[0]
+
+        CIManager.update_unique_value(ci_id, unique_name, unique_value)
+
+        return self.jsonify(ci_id=ci_id)
+
+
+class CIHeartbeatView(APIView):
+    url_prefix = ("/ci/heartbeat", "/ci/heartbeat/<string:ci_type>/<string:unique>")
+
+    def get(self):
+        page = get_page(request.values.get("page", 1))
+        ci_type = request.values.get("ci_type", "").strip()
+        try:
+            type_id = CITypeCache.get(ci_type).type_id
+        except AttributeError:
+            return self.jsonify(numfound=0, result=[])
+        agent_status = request.values.get("agent_status")
+        if agent_status:
+            agent_status = int(agent_status)
+
+        numfound, result = CIManager.get_heartbeat(page, type_id, agent_status=agent_status)
+
+        return self.jsonify(numfound=numfound, result=result)
+
+    def post(self, ci_type, unique):
+        if not unique or not ci_type:
+            return self.jsonify(message="error")
+
+        msg, cmd = CIManager().add_heartbeat(ci_type, unique)
+
+        return self.jsonify(message=msg, cmd=cmd)
+
+
+class CIFlushView(APIView):
+    url_prefix = ("/ci/flush", "/ci/<int:ci_id>/flush")
+
+    @auth_abandoned
+    def get(self, ci_id=None):
+        from api.tasks.cmdb import ci_cache
+        from api.lib.cmdb.const import CMDB_QUEUE
+        if ci_id is not None:
+            ci_cache.apply_async([ci_id], queue=CMDB_QUEUE)
+        else:
+            cis = CI.get_by(to_dict=False)
+            for ci in cis:
+                ci_cache.apply_async([ci.id], queue=CMDB_QUEUE)
+        return self.jsonify(code=200)

cmdb-api/api/views/cmdb/ci_relation.py

@@ -0,0 +1,139 @@
+# -*- coding:utf-8 -*- 
+
+
+import time
+
+from flask import abort
+from flask import current_app
+from flask import request
+
+from api.lib.cmdb.cache import RelationTypeCache
+from api.lib.cmdb.ci import CIRelationManager
+from api.lib.cmdb.search import SearchError
+from api.lib.cmdb.search.ci_relation.search import Search
+from api.lib.perm.auth import auth_abandoned
+from api.lib.utils import get_page
+from api.lib.utils import get_page_size
+from api.lib.utils import handle_arg_list
+from api.resource import APIView
+
+
+class CIRelationSearchView(APIView):
+    url_prefix = ("/ci_relations/s", "/ci_relations/search")
+
+    @auth_abandoned
+    def get(self):
+        """@params: q: query statement
+                    fl: filter by column
+                    count: the number of ci
+                    root_id: ci id
+                    level: default is 1
+                    facet: statistic
+        """
+
+        page = get_page(request.values.get("page", 1))
+        count = get_page_size(request.values.get("count") or request.values.get("page_size"))
+
+        root_id = request.values.get('root_id')
+        level = list(map(int, handle_arg_list(request.values.get('level', '1'))))
+
+        query = request.values.get('q', "")
+        fl = handle_arg_list(request.values.get('fl', ""))
+        facet = handle_arg_list(request.values.get("facet", ""))
+        sort = request.values.get("sort")
+
+        start = time.time()
+        s = Search(root_id, level, query, fl, facet, page, count, sort)
+        try:
+            response, counter, total, page, numfound, facet = s.search()
+        except SearchError as e:
+            return abort(400, str(e))
+        current_app.logger.debug("search time is :{0}".format(time.time() - start))
+
+        return self.jsonify(numfound=numfound,
+                            total=total,
+                            page=page,
+                            facet=facet,
+                            counter=counter,
+                            result=response)
+
+
+class CIRelationStatisticsView(APIView):
+    url_prefix = "/ci_relations/statistics"
+
+    @auth_abandoned
+    def get(self):
+        root_ids = list(map(int, handle_arg_list(request.values.get('root_ids'))))
+        level = request.values.get('level', 1)
+        type_ids = set(map(int, handle_arg_list(request.values.get('type_ids', []))))
+
+        start = time.time()
+        s = Search(root_ids, level)
+        try:
+            result = s.statistics(type_ids)
+        except SearchError as e:
+            return abort(400, str(e))
+        current_app.logger.debug("search time is :{0}".format(time.time() - start))
+
+        return self.jsonify(result)
+
+
+class GetSecondCIsView(APIView):
+    url_prefix = "/ci_relations/<int:first_ci_id>/second_cis"
+
+    def get(self, first_ci_id):
+        page = get_page(request.values.get("page", 1))
+        count = get_page_size(request.values.get("count"))
+        relation_type = request.values.get("relation_type")
+        try:
+            relation_type_id = RelationTypeCache.get(relation_type).id if relation_type else None
+        except AttributeError:
+            return abort(400, "invalid relation type <{0}>".format(relation_type))
+
+        manager = CIRelationManager()
+        numfound, total, second_cis = manager.get_second_cis(
+            first_ci_id, page=page, per_page=count, relation_type_id=relation_type_id)
+
+        return self.jsonify(numfound=numfound,
+                            total=total,
+                            page=page,
+                            second_cis=second_cis)
+
+
+class GetFirstCIsView(APIView):
+    url_prefix = "/ci_relations/<int:second_ci_id>/first_cis"
+
+    def get(self, second_ci_id):
+        page = get_page(request.values.get("page", 1))
+        count = get_page_size(request.values.get("count"))
+
+        manager = CIRelationManager()
+        numfound, total, first_cis = manager.get_first_cis(second_ci_id, per_page=count, page=page)
+
+        return self.jsonify(numfound=numfound,
+                            total=total,
+                            page=page,
+                            first_cis=first_cis)
+
+
+class CIRelationView(APIView):
+    url_prefix = "/ci_relations/<int:first_ci_id>/<int:second_ci_id>"
+
+    def post(self, first_ci_id, second_ci_id):
+        manager = CIRelationManager()
+        res = manager.add(first_ci_id, second_ci_id)
+        return self.jsonify(cr_id=res)
+
+    def delete(self, first_ci_id, second_ci_id):
+        manager = CIRelationManager()
+        manager.delete_2(first_ci_id, second_ci_id)
+        return self.jsonify(message="CIType Relation is deleted")
+
+
+class DeleteCIRelationView(APIView):
+    url_prefix = "/ci_relations/<int:cr_id>"
+
+    def delete(self, cr_id):
+        manager = CIRelationManager()
+        manager.delete(cr_id)
+        return self.jsonify(message="CIType Relation is deleted")

cmdb-api/api/views/cmdb/ci_type.py

@@ -0,0 +1,202 @@
+# -*- coding:utf-8 -*- 
+
+
+from flask import abort
+from flask import current_app
+from flask import request
+
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.cache import CITypeCache
+from api.lib.cmdb.ci_type import CITypeAttributeGroupManager
+from api.lib.cmdb.ci_type import CITypeAttributeManager
+from api.lib.cmdb.ci_type import CITypeGroupManager
+from api.lib.cmdb.ci_type import CITypeManager
+from api.lib.cmdb.const import RoleEnum
+from api.lib.decorator import args_required
+from api.lib.perm.acl.acl import role_required
+from api.lib.utils import handle_arg_list
+from api.resource import APIView
+
+
+class CITypeView(APIView):
+    url_prefix = ("/ci_types", "/ci_types/<int:type_id>", "/ci_types/<string:type_name>")
+
+    def get(self, type_id=None, type_name=None):
+        q = request.args.get("type_name")
+
+        if type_id is not None:
+            ci_types = [CITypeCache.get(type_id).to_dict()]
+        elif type_name is not None:
+            ci_types = [CITypeCache.get(type_name).to_dict()]
+        else:
+            ci_types = CITypeManager().get_ci_types(q)
+        count = len(ci_types)
+
+        return self.jsonify(numfound=count, ci_types=ci_types)
+
+    @role_required(RoleEnum.CONFIG)
+    @args_required("name")
+    def post(self):
+        params = request.values
+
+        type_name = params.get("name")
+        type_alias = params.get("alias")
+        type_alias = type_name if not type_alias else type_alias
+        params['alias'] = type_alias
+
+        manager = CITypeManager()
+        type_id = manager.add(**params)
+
+        return self.jsonify(type_id=type_id)
+
+    @role_required(RoleEnum.CONFIG)
+    def put(self, type_id):
+        params = request.values
+
+        manager = CITypeManager()
+        manager.update(type_id, **params)
+        return self.jsonify(type_id=type_id)
+
+    @role_required(RoleEnum.CONFIG)
+    def delete(self, type_id):
+        CITypeManager.delete(type_id)
+        return self.jsonify(type_id=type_id)
+
+
+class CITypeGroupView(APIView):
+    url_prefix = ("/ci_types/groups", "/ci_types/groups/<int:gid>")
+
+    def get(self):
+        need_other = request.values.get("need_other")
+        return self.jsonify(CITypeGroupManager.get(need_other))
+
+    @role_required(RoleEnum.CONFIG)
+    @args_required("name")
+    def post(self):
+        name = request.values.get("name")
+        group = CITypeGroupManager.add(name)
+        return self.jsonify(group.to_dict())
+
+    @role_required(RoleEnum.CONFIG)
+    def put(self, gid):
+        name = request.values.get('name')
+        type_ids = request.values.get('type_ids')
+        CITypeGroupManager.update(gid, name, type_ids)
+        return self.jsonify(gid=gid)
+
+    @role_required(RoleEnum.CONFIG)
+    def delete(self, gid):
+        CITypeGroupManager.delete(gid)
+        return self.jsonify(gid=gid)
+
+
+class CITypeQueryView(APIView):
+    url_prefix = "/ci_types/query"
+
+    @args_required("q")
+    def get(self):
+        q = request.args.get("q")
+        res = CITypeManager.query(q)
+        return self.jsonify(ci_type=res)
+
+
+class EnableCITypeView(APIView):
+    url_prefix = "/ci_types/<int:type_id>/enable"
+
+    @role_required(RoleEnum.CONFIG)
+    def post(self, type_id):
+        enable = request.values.get("enable", True)
+        CITypeManager.set_enabled(type_id, enabled=enable)
+        return self.jsonify(type_id=type_id, enable=enable)
+
+
+class CITypeAttributeView(APIView):
+    url_prefix = ("/ci_types/<int:type_id>/attributes", "/ci_types/<string:type_name>/attributes")
+
+    def get(self, type_id=None, type_name=None):
+        t = CITypeCache.get(type_id) or CITypeCache.get(type_name) or abort(404, "CIType does not exist")
+        type_id = t.id
+        unique_id = t.unique_id
+        unique = AttributeCache.get(unique_id).name
+        return self.jsonify(attributes=CITypeAttributeManager.get_attributes_by_type_id(type_id),
+                            type_id=type_id,
+                            unique_id=unique_id,
+                            unique=unique)
+
+    @role_required(RoleEnum.CONFIG)
+    @args_required("attr_id")
+    def post(self, type_id=None):
+        attr_id_list = handle_arg_list(request.values.get("attr_id"))
+        params = request.values
+        params.pop("attr_id", "")
+
+        CITypeAttributeManager.add(type_id, attr_id_list, **params)
+        return self.jsonify(attributes=attr_id_list)
+
+    @role_required(RoleEnum.CONFIG)
+    @args_required("attributes")
+    def put(self, type_id=None):
+        """
+        attributes is list, only support raw data request
+        :param type_id: 
+        :return: 
+        """
+        attributes = request.values.get("attributes")
+        current_app.logger.debug(attributes)
+        if not isinstance(attributes, list):
+            return abort(400, "attributes must be list")
+        CITypeAttributeManager.update(type_id, attributes)
+        return self.jsonify(attributes=attributes)
+
+    @role_required(RoleEnum.CONFIG)
+    @args_required("attr_id")
+    def delete(self, type_id=None):
+        """
+        Form request: attr_id is a string, separated by commas
+        Raw data request: attr_id is a list
+        :param type_id: 
+        :return: 
+        """
+        attr_id_list = handle_arg_list(request.values.get("attr_id", ""))
+
+        CITypeAttributeManager.delete(type_id, attr_id_list)
+
+        return self.jsonify(attributes=attr_id_list)
+
+
+class CITypeAttributeGroupView(APIView):
+    url_prefix = ("/ci_types/<int:type_id>/attribute_groups",
+                  "/ci_types/attribute_groups/<int:group_id>")
+
+    def get(self, type_id):
+        need_other = request.values.get("need_other")
+        return self.jsonify(CITypeAttributeGroupManager.get_by_type_id(type_id, need_other))
+
+    @role_required(RoleEnum.CONFIG)
+    @args_required("name")
+    def post(self, type_id):
+        name = request.values.get("name").strip()
+        order = request.values.get("order") or 0
+        attrs = handle_arg_list(request.values.get("attributes", ""))
+        orders = list(range(len(attrs)))
+
+        attr_order = list(zip(attrs, orders))
+        group = CITypeAttributeGroupManager.create_or_update(type_id, name, attr_order, order)
+        current_app.logger.warning(group.id)
+        return self.jsonify(group_id=group.id)
+
+    @role_required(RoleEnum.CONFIG)
+    def put(self, group_id):
+        name = request.values.get("name")
+        order = request.values.get("order") or 0
+        attrs = handle_arg_list(request.values.get("attributes", ""))
+        orders = list(range(len(attrs)))
+
+        attr_order = list(zip(attrs, orders))
+        CITypeAttributeGroupManager.update(group_id, name, attr_order, order)
+        return self.jsonify(group_id=group_id)
+
+    @role_required(RoleEnum.CONFIG)
+    def delete(self, group_id):
+        CITypeAttributeGroupManager.delete(group_id)
+        return self.jsonify(group_id=group_id)

cmdb-api/api/views/cmdb/ci_type_relation.py

@@ -0,0 +1,58 @@
+# -*- coding:utf-8 -*- 
+
+
+from flask import request
+
+from api.lib.cmdb.ci_type import CITypeRelationManager
+from api.lib.cmdb.const import RoleEnum
+from api.lib.decorator import args_required
+from api.lib.perm.acl.acl import role_required
+from api.resource import APIView
+
+
+class GetChildrenView(APIView):
+    url_prefix = "/ci_type_relations/<int:parent_id>/children"
+
+    def get(self, parent_id):
+        return self.jsonify(children=CITypeRelationManager.get_children(parent_id))
+
+
+class GetParentsView(APIView):
+    url_prefix = "/ci_type_relations/<int:child_id>/parents"
+
+    def get(self, child_id):
+        return self.jsonify(parents=CITypeRelationManager.get_parents(child_id))
+
+
+class CITypeRelationView(APIView):
+    url_prefix = ("/ci_type_relations", "/ci_type_relations/<int:parent_id>/<int:child_id>")
+
+    @role_required(RoleEnum.CONFIG)
+    def get(self):
+        res = CITypeRelationManager.get()
+
+        return self.jsonify(res)
+
+    @role_required(RoleEnum.CONFIG)
+    @args_required("relation_type_id")
+    def post(self, parent_id, child_id):
+        relation_type_id = request.values.get("relation_type_id")
+        ctr_id = CITypeRelationManager.add(parent_id, child_id, relation_type_id)
+
+        return self.jsonify(ctr_id=ctr_id)
+
+    @role_required(RoleEnum.CONFIG)
+    def delete(self, parent_id, child_id):
+        CITypeRelationManager.delete_2(parent_id, child_id)
+
+        return self.jsonify(code=200, parent_id=parent_id, child_id=child_id)
+
+
+class CITypeRelationDelete2View(APIView):
+    url_prefix = "/ci_type_relations/<int:ctr_id>"
+
+    @role_required(RoleEnum.CONFIG)
+    def delete(self, ctr_id):
+        CITypeRelationManager.delete(ctr_id)
+
+        return self.jsonify(code=200, ctr_id=ctr_id)

cmdb-api/api/views/cmdb/history.py

@@ -0,0 +1,63 @@
+# -*- coding:utf-8 -*- 
+
+
+import datetime
+
+from flask import abort
+from flask import request
+
+from api.lib.cmdb.history import AttributeHistoryManger
+from api.lib.utils import get_page
+from api.lib.utils import get_page_size
+from api.resource import APIView
+
+
+class RecordView(APIView):
+    url_prefix = "/history/records"
+
+    def get(self):
+        page = get_page(request.values.get("page", 1))
+        page_size = get_page_size(request.values.get("page_size"))
+        _start = request.values.get("start")
+        _end = request.values.get("end")
+        username = request.values.get("username", "")
+        start, end = None, None
+        if _start:
+            try:
+                start = datetime.datetime.strptime(_start, '%Y-%m-%d %H:%M:%S')
+            except ValueError:
+                abort(400, 'incorrect start date time')
+        if _end:
+            try:
+                end = datetime.datetime.strptime(_end, '%Y-%m-%d %H:%M:%S')
+            except ValueError:
+                abort(400, 'incorrect end date time')
+
+        numfound, total, res = AttributeHistoryManger.get_records(start, end, username, page, page_size)
+
+        return self.jsonify(numfound=numfound,
+                            records=res,
+                            page=page,
+                            total=total,
+                            start=_start,
+                            end=_end,
+                            username=username)
+
+
+class CIHistoryView(APIView):
+    url_prefix = "/history/ci/<int:ci_id>"
+
+    def get(self, ci_id):
+        result = AttributeHistoryManger.get_by_ci_id(ci_id)
+        return self.jsonify(result)
+
+
+class RecordDetailView(APIView):
+    url_prefix = "/history/records/<int:record_id>"
+
+    def get(self, record_id):
+        username, timestamp, attr_dict, rel_dict = AttributeHistoryManger.get_record_detail(record_id)
+        return self.jsonify(username=username,
+                            timestamp=timestamp,
+                            attr_history=attr_dict,
+                            rel_history=rel_dict)

cmdb-api/api/views/cmdb/preference.py

@@ -0,0 +1,98 @@
+# -*- coding:utf-8 -*-
+
+
+from flask import request
+
+from api.lib.cmdb.ci_type import CITypeManager
+from api.lib.cmdb.const import ResourceTypeEnum, PermEnum, RoleEnum
+from api.lib.cmdb.preference import PreferenceManager
+from api.lib.decorator import args_required
+from api.lib.perm.acl.acl import has_perm_from_args
+from api.lib.perm.acl.acl import role_required
+from api.lib.utils import handle_arg_list
+from api.resource import APIView
+
+
+class PreferenceShowCITypesView(APIView):
+    url_prefix = "/preference/ci_types"
+
+    def get(self):
+        instance = request.values.get("instance")
+        tree = request.values.get("tree")
+
+        return self.jsonify(PreferenceManager.get_types(instance, tree))
+
+
+class PreferenceShowAttributesView(APIView):
+    url_prefix = "/preference/ci_types/<id_or_name>/attributes"
+
+    def get(self, id_or_name):
+        is_subscribed, attributes = PreferenceManager.get_show_attributes(id_or_name)
+
+        return self.jsonify(attributes=attributes, is_subscribed=is_subscribed)
+
+    @has_perm_from_args("id_or_name", ResourceTypeEnum.CI, PermEnum.READ, CITypeManager.get_name_by_id)
+    @args_required("attr")
+    def post(self, id_or_name):
+        id_or_name = int(id_or_name)
+        attr_list = handle_arg_list(request.values.get("attr", ""))
+        orders = list(range(len(attr_list)))
+        PreferenceManager.create_or_update_show_attributes(id_or_name, list(zip(attr_list, orders)))
+
+        return self.jsonify(type_id=id_or_name,
+                            attr_order=list(zip(attr_list, orders)))
+
+    @has_perm_from_args("id_or_name", ResourceTypeEnum.CI, PermEnum.READ, CITypeManager.get_name_by_id)
+    def put(self, id_or_name):
+        return self.post(id_or_name)
+
+
+class PreferenceTreeApiView(APIView):
+    url_prefix = "/preference/tree/view"
+
+    def get(self):
+        return self.jsonify(PreferenceManager.get_tree_view())
+
+    @has_perm_from_args("type_id", ResourceTypeEnum.CI, PermEnum.READ, CITypeManager.get_name_by_id)
+    @args_required("type_id")
+    @args_required("levels")
+    def post(self):
+        type_id = request.values.get("type_id")
+        levels = handle_arg_list(request.values.get("levels"))
+        res = PreferenceManager.create_or_update_tree_view(type_id, levels)
+
+        return self.jsonify(res and res.to_dict() or {})
+
+    def put(self):
+        return self.post()
+
+
+class PreferenceRelationApiView(APIView):
+    url_prefix = "/preference/relation/view"
+
+    def get(self):
+        views, id2type, name2id = PreferenceManager.get_relation_view()
+
+        return self.jsonify(views=views, id2type=id2type, name2id=name2id)
+
+    @role_required(RoleEnum.CONFIG)
+    @args_required("name")
+    @args_required("cr_ids")
+    def post(self):
+        name = request.values.get("name")
+        cr_ids = request.values.get("cr_ids")
+        views, id2type, name2id = PreferenceManager.create_or_update_relation_view(name, cr_ids)
+
+        return self.jsonify(views=views, id2type=id2type, name2id=name2id)
+
+    @role_required(RoleEnum.CONFIG)
+    def put(self):
+        return self.post()
+
+    @role_required(RoleEnum.CONFIG)
+    @args_required("name")
+    def delete(self):
+        name = request.values.get("name")
+        PreferenceManager.delete_relation_view(name)
+
+        return self.jsonify(name=name)

cmdb-api/api/views/cmdb/relation_type.py

@@ -0,0 +1,37 @@
+# -*- coding:utf-8 -*-
+
+
+from flask import abort
+from flask import request
+
+from api.lib.cmdb.const import RoleEnum
+from api.lib.cmdb.relation_type import RelationTypeManager
+from api.lib.decorator import args_required
+from api.lib.perm.acl.acl import role_required
+from api.resource import APIView
+
+
+class RelationTypeView(APIView):
+    url_prefix = ("/relation_types", "/relation_types/<int:rel_id>")
+
+    def get(self):
+        return self.jsonify([i.to_dict() for i in RelationTypeManager.get_all()])
+
+    @role_required(RoleEnum.CONFIG)
+    @args_required("name")
+    def post(self):
+        name = request.values.get("name") or abort(400, "Name cannot be empty")
+        rel = RelationTypeManager.add(name)
+        return self.jsonify(rel.to_dict())
+
+    @role_required(RoleEnum.CONFIG)
+    @args_required("name")
+    def put(self, rel_id):
+        name = request.values.get("name") or abort(400, "Name cannot be empty")
+        rel = RelationTypeManager.update(rel_id, name)
+        return self.jsonify(rel.to_dict())
+
+    @role_required(RoleEnum.CONFIG)
+    def delete(self, rel_id):
+        RelationTypeManager.delete(rel_id)
+        return self.jsonify(rel_id=rel_id)

cmdb-api/autoapp.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+
+
+from flask import g
+from flask_login import current_user
+
+from api.app import create_app
+
+app = create_app()
+
+
+@app.before_request
+def before_request():
+    g.user = current_user

cmdb-api/celery_worker.py

@@ -0,0 +1,10 @@
+# -*- coding:utf-8 -*-
+
+from api.app import create_app
+from api.extensions import celery
+
+# celery worker -A celery_worker.celery -l DEBUG -E -Q <queue_name> --concurrency=1
+print(celery)
+
+app = create_app()
+app.app_context().push()

cmdb-api/requirements.txt

@@ -0,0 +1,38 @@
+-i https://mirrors.aliyun.com/pypi/simple
+# Flask
+Flask==1.0.3
+Werkzeug==0.15.4
+click>=5.0
+# Api
+Flask-RESTful ==0.3.7
+# Database
+Flask-SQLAlchemy ==2.4.0
+SQLAlchemy ==1.3.5
+PyMySQL ==0.9.3
+redis ==3.2.1
+# Migrations
+Flask-Migrate == 2.5.2
+# Deployment
+gevent ==1.4.0
+gunicorn == 19.5.0
+supervisor ==4.0.3
+# Auth
+Flask-Login ==0.4.1
+Flask-Bcrypt ==0.7.1
+Flask-Cors>=3.0.8
+# Caching
+Flask-Caching>=1.0.0
+# Environment variable parsing
+environs ==4.2.0
+marshmallow ==2.20.2
+# async tasks
+celery ==4.3.0
+more-itertools ==5.0.0
+kombu ==4.4.0
+# other
+six ==1.12.0
+bs4>=0.0.1
+toposort>=1.5
+requests>=2.22.0
+PyJWT>=1.7.1
+elasticsearch ==7.0.4

cmdb-api/settings.py.example

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+"""Application configuration.
+
+Most configuration is set via environment variables.
+
+For local development, use a .env file to set
+environment variables.
+"""
+from environs import Env
+
+env = Env()
+env.read_env()
+
+ENV = env.str("FLASK_ENV", default="production")
+DEBUG = ENV == "development"
+SECRET_KEY = env.str("SECRET_KEY")
+BCRYPT_LOG_ROUNDS = env.int("BCRYPT_LOG_ROUNDS", default=13)
+DEBUG_TB_ENABLED = DEBUG
+DEBUG_TB_INTERCEPT_REDIRECTS = False
+
+
+ERROR_CODES = [400, 401, 403, 404, 405, 500, 502]
+
+# # database
+SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://{user}:{password}@127.0.0.1:3306/{db}?charset=utf8'
+SQLALCHEMY_BINDS = {
+    "user": 'mysql+pymysql://{user}:{password}@127.0.0.1:3306/{db}?charset=utf8'
+}
+SQLALCHEMY_ECHO = False
+SQLALCHEMY_TRACK_MODIFICATIONS = False
+SQLALCHEMY_ENGINE_OPTIONS = {
+    'pool_recycle': 300,
+}
+
+# # cache
+CACHE_TYPE = "redis"
+CACHE_REDIS_HOST = "127.0.0.1"
+CACHE_REDIS_PORT = 6379
+CACHE_KEY_PREFIX = "CMDB::"
+CACHE_DEFAULT_TIMEOUT = 3000
+
+# # log
+LOG_PATH = './logs/app.log'
+LOG_LEVEL = 'DEBUG'
+
+
+# # mail
+MAIL_SERVER = ''
+MAIL_PORT = 25
+MAIL_USE_TLS = False
+MAIL_USE_SSL = False
+MAIL_DEBUG = True
+MAIL_USERNAME = ''
+MAIL_PASSWORD = ''
+DEFAULT_MAIL_SENDER = ''
+
+# # queue
+CELERY_RESULT_BACKEND = "redis://127.0.0.1:6379/2"
+BROKER_URL = 'redis://127.0.0.1:6379/2'
+BROKER_VHOST = '/'
+
+
+# # SSO
+CAS_SERVER = "http://sso.xxx.com"
+CAS_VALIDATE_SERVER = "http://sso.xxx.com"
+CAS_LOGIN_ROUTE = "/cas/login"
+CAS_LOGOUT_ROUTE = "/cas/logout"
+CAS_VALIDATE_ROUTE = "/cas/serviceValidate"
+CAS_AFTER_LOGIN = "/"
+DEFAULT_SERVICE = "http://127.0.0.1:8000"
+
+# # pagination
+DEFAULT_PAGE_COUNT = 50
+
+# # permission
+WHITE_LIST = ["127.0.0.1"]
+USE_ACL = False
+
+# # elastic search
+ES_HOST = '127.0.0.1'
+USE_ES = False
+
+
+BOOL_TRUE = ['true', 'TRUE', 'True', True, '1', 1, "Yes", "YES", "yes", 'Y', 'y']

cmdb-api/setup.cfg

@@ -0,0 +1,12 @@
+[flake8]
+ignore = D401,D202,E226,E302,E41
+max-line-length = 120
+exclude = migrations/*
+max-complexity = 10
+
+
+[isort]
+line_length = 88
+multi_line_output = 3
+skip = migrations/*
+include_trailing_comma = true

cmdb-api/tests/__init__.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

cmdb-api/tests/conftest.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+"""Defines fixtures available to all tests."""
+
+import pytest
+from webtest import TestApp
+
+from api.app import create_app
+
+
+@pytest.fixture
+def app():
+    """Create application for the tests."""
+    _app = create_app("tests.settings")
+    ctx = _app.test_request_context()
+    ctx.push()
+    yield _app
+
+    ctx.pop()
+
+
+@pytest.fixture
+def testapp(app):
+    """Create Webtest app."""
+    return TestApp(app)

cmdb-api/tests/test_cmdb_attribute.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

cmdb-api/tests/test_cmdb_ci.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+
+
+class TestCI:
+
+    def test_ci_search_only_type_query(self, app):
+        with app.test_client() as c:
+            rv = c.get('/api/v0.1/ci/s?q=_type:server', json={})
+            json_data = rv.get_json()
+            assert type(json_data.get("result")) is list

cmdb-api/tests/test_cmdb_ci_realtion.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

cmdb-api/tests/test_cmdb_ci_type.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

cmdb-api/tests/test_cmdb_ci_type_relation.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

cmdb-api/tests/test_cmdb_history.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

cmdb-api/tests/test_cmdb_preference.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

cmdb-api/tests/test_cmdb_relation_type.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-