From a91f409525baa275d215b971e0f236d7a5368b01 Mon Sep 17 00:00:00 2001 From: pycook Date: Tue, 29 Dec 2015 10:35:10 +0800 Subject: [PATCH] first commit to github --- .gitignore | 14 + README.md | 4 + cmdb-api/__init__.py | 126 ++++++ cmdb-api/cmdb_api.md | 637 ++++++++++++++++++++++++++ cmdb-api/cmdb_query_api.md | 99 ++++ cmdb-api/command/__init__.py | 1 + cmdb-api/config-sample.cfg | 69 +++ cmdb-api/core/__init__.py | 11 + cmdb-api/core/account.py | 98 ++++ cmdb-api/core/attribute.py | 145 ++++++ cmdb-api/core/ci.py | 189 ++++++++ cmdb-api/core/ci_relation.py | 70 +++ cmdb-api/core/ci_type.py | 89 ++++ cmdb-api/core/ci_type_relation.py | 55 +++ cmdb-api/core/history.py | 116 +++++ cmdb-api/core/special.py | 16 + cmdb-api/core/statis.py | 12 + cmdb-api/extensions.py | 16 + cmdb-api/gunicornserver.py | 72 +++ cmdb-api/lib/__init__.py | 4 + cmdb-api/lib/account.py | 145 ++++++ cmdb-api/lib/attribute.py | 167 +++++++ cmdb-api/lib/auth.py | 68 +++ cmdb-api/lib/ci.py | 677 ++++++++++++++++++++++++++++ cmdb-api/lib/ci_type.py | 315 +++++++++++++ cmdb-api/lib/const.py | 99 ++++ cmdb-api/lib/decorator.py | 74 +++ cmdb-api/lib/exception.py | 17 + cmdb-api/lib/history.py | 75 +++ cmdb-api/lib/mail.py | 86 ++++ cmdb-api/lib/query_sql.py | 107 +++++ cmdb-api/lib/search.py | 348 ++++++++++++++ cmdb-api/lib/template/__init__.py | 1 + cmdb-api/lib/template/filters.py | 9 + cmdb-api/lib/utils.py | 74 +++ cmdb-api/lib/value.py | 170 +++++++ cmdb-api/manage.py | 77 ++++ cmdb-api/models/__init__.py | 24 + cmdb-api/models/account.py | 230 ++++++++++ cmdb-api/models/attribute.py | 87 ++++ cmdb-api/models/ci.py | 20 + cmdb-api/models/ci_relation.py | 26 ++ cmdb-api/models/ci_type.py | 128 ++++++ cmdb-api/models/ci_type_relation.py | 27 ++ cmdb-api/models/ci_value.py | 117 +++++ cmdb-api/models/history.py | 51 +++ cmdb-api/models/statis.py | 20 + cmdb-api/permissions.py | 9 + cmdb-api/requirements/default.txt | 14 + cmdb-api/settings.py | 7 + cmdb-api/tasks/__init__.py | 1 + cmdb-api/tasks/cmdb.py | 30 ++ cmdb-api/tasks/statis.py | 21 + cmdb-api/templates/search.xml | 27 ++ cmdb-api/templates/search_tidy.xml | 19 + 55 files changed, 5210 insertions(+) create mode 100755 .gitignore create mode 100644 README.md create mode 100644 cmdb-api/__init__.py create mode 100644 cmdb-api/cmdb_api.md create mode 100644 cmdb-api/cmdb_query_api.md create mode 100644 cmdb-api/command/__init__.py create mode 100644 cmdb-api/config-sample.cfg create mode 100644 cmdb-api/core/__init__.py create mode 100644 cmdb-api/core/account.py create mode 100644 cmdb-api/core/attribute.py create mode 100644 cmdb-api/core/ci.py create mode 100644 cmdb-api/core/ci_relation.py create mode 100644 cmdb-api/core/ci_type.py create mode 100644 cmdb-api/core/ci_type_relation.py create mode 100644 cmdb-api/core/history.py create mode 100644 cmdb-api/core/special.py create mode 100644 cmdb-api/core/statis.py create mode 100644 cmdb-api/extensions.py create mode 100644 cmdb-api/gunicornserver.py create mode 100644 cmdb-api/lib/__init__.py create mode 100644 cmdb-api/lib/account.py create mode 100644 cmdb-api/lib/attribute.py create mode 100644 cmdb-api/lib/auth.py create mode 100644 cmdb-api/lib/ci.py create mode 100644 cmdb-api/lib/ci_type.py create mode 100644 cmdb-api/lib/const.py create mode 100644 cmdb-api/lib/decorator.py create mode 100644 cmdb-api/lib/exception.py create mode 100644 cmdb-api/lib/history.py create mode 100644 cmdb-api/lib/mail.py create mode 100644 cmdb-api/lib/query_sql.py create mode 100644 cmdb-api/lib/search.py create mode 100644 cmdb-api/lib/template/__init__.py create mode 100644 cmdb-api/lib/template/filters.py create mode 100644 cmdb-api/lib/utils.py create mode 100644 cmdb-api/lib/value.py create mode 100644 cmdb-api/manage.py create mode 100644 cmdb-api/models/__init__.py create mode 100644 cmdb-api/models/account.py create mode 100644 cmdb-api/models/attribute.py create mode 100644 cmdb-api/models/ci.py create mode 100644 cmdb-api/models/ci_relation.py create mode 100644 cmdb-api/models/ci_type.py create mode 100644 cmdb-api/models/ci_type_relation.py create mode 100644 cmdb-api/models/ci_value.py create mode 100644 cmdb-api/models/history.py create mode 100644 cmdb-api/models/statis.py create mode 100644 cmdb-api/permissions.py create mode 100644 cmdb-api/requirements/default.txt create mode 100644 cmdb-api/settings.py create mode 100644 cmdb-api/tasks/__init__.py create mode 100644 cmdb-api/tasks/cmdb.py create mode 100644 cmdb-api/tasks/statis.py create mode 100644 cmdb-api/templates/search.xml create mode 100644 cmdb-api/templates/search_tidy.xml diff --git a/.gitignore b/.gitignore new file mode 100755 index 0000000..30b7fe8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +*~ +*.pyc +.idea +data +logs +*/logs/* +*.sql +test/* +tools/* +*.log +*.orig +*.zip +*.swp +*.tar.gz diff --git a/README.md b/README.md new file mode 100644 index 0000000..8fd6703 --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +## cmdb + +### cmdb即配置管理数据库 +### 目前只发布api部分，安装脚本和portal计划很快push \ No newline at end of file diff --git a/cmdb-api/__init__.py b/cmdb-api/__init__.py new file mode 100644 index 0000000..a4d8219 --- /dev/null +++ b/cmdb-api/__init__.py @@ -0,0 +1,126 @@ +# encoding=utf-8 + +import os +import logging +from logging.handlers import SMTPHandler +from logging.handlers import TimedRotatingFileHandler + +from flask import Flask +from flask import request +from flask import g +from flask.ext.babel import Babel +from flask.ext.principal import identity_loaded + +from extensions import db +from extensions import mail +from extensions import cache +from extensions import celery +from core import attribute +from core import citype +from core import cityperelation +from core import cirelation +from core import ci +from core import history +from core import account +from core import special +from models.account import User +from lib.template import filters + + +APP_NAME = "CMDB-API" + +MODULES = ( + (attribute, "/api/v0.1/attributes"), + (citype, "/api/v0.1/citypes"), + (cityperelation, "/api/v0.1/cityperelations"), + (cirelation, "/api/v0.1/cirelations"), + (ci, "/api/v0.1/ci"), + (history, "/api/v0.1/history"), + (account, "/api/v0.1/accounts"), + (special, ""), +) + + +def make_app(config=None, modules=None): + modules = modules + if not modules: + modules = MODULES + app = Flask(APP_NAME) + app.config.from_pyfile(config) + configure_extensions(app) + configure_i18n(app) + configure_identity(app) + configure_blueprints(app, modules) + configure_logging(app) + configure_template_filters(app) + return app + + +def configure_extensions(app): + db.app = app + celery.init_app(app) + db.init_app(app) + mail.init_app(app) + cache.init_app(app) + celery.init_app(app) + + +def configure_i18n(app): + babel = Babel(app) + + @babel.localeselector + def get_locale(): + accept_languages = app.config.get('ACCEPT_LANGUAGES', ['en', 'zh']) + return request.accept_languages.best_match(accept_languages) + + +def configure_modules(app, modules): + for module, url_prefix in modules: + app.register_module(module, url_prefix=url_prefix) + + +def configure_blueprints(app, modules): + for module, url_prefix in modules: + app.register_blueprint(module, url_prefix=url_prefix) + + +def configure_identity(app): + + @identity_loaded.connect_via(app) + def on_identity_loaded(sender, identity): + g.user = User.query.from_identity(identity) + + +def configure_logging(app): + hostname = os.uname()[1] + mail_handler = SMTPHandler( + app.config['MAIL_SERVER'], + app.config['DEFAULT_MAIL_SENDER'], + app.config['ADMINS'], + '[%s] CMDB error' % hostname, + ( + app.config['MAIL_USERNAME'], + app.config['MAIL_PASSWORD'], + ) + ) + mail_formater = logging.Formatter( + "%(asctime)s %(levelname)s %(pathname)s %(lineno)d\n%(message)s") + mail_handler.setFormatter(mail_formater) + mail_handler.setLevel(logging.ERROR) + if not app.debug: + app.logger.addHandler(mail_handler) + formatter = logging.Formatter( + "%(asctime)s %(levelname)s %(pathname)s %(lineno)d - %(message)s") + log_file = app.config['LOG_PATH'] + file_handler = TimedRotatingFileHandler( + log_file, when='d', interval=1, backupCount=7) + file_handler.setLevel(getattr(logging, app.config['LOG_LEVEL'])) + file_handler.setFormatter(formatter) + app.logger.addHandler(file_handler) + app.logger.setLevel(getattr(logging, app.config['LOG_LEVEL'])) + + +def configure_template_filters(app): + for name in dir(filters): + if callable(getattr(filters, name)): + app.add_template_filter(getattr(filters, name)) diff --git a/cmdb-api/cmdb_api.md b/cmdb-api/cmdb_api.md new file mode 100644 index 0000000..c4e237f --- /dev/null +++ b/cmdb-api/cmdb_api.md @@ -0,0 +1,637 @@ +# CMDB API文档 + +## 状态返回码的定义 +* 200: 成功 +* 400：失败 +* 401：未授权 +* 404：url not found +* 408：超时 +* 410：资源删除 +* 500: 服务器错误 + + +## 用户接口 + +### CI搜索接口 + +* GET `/api/v0.1/ci/s` +* 参数 + * `string:_type` 搜索的ci_type，多个用分号隔开, 例如: _type:(server;vservser) + * `string:q` 搜索表达式, 例如`q=hostname:cmdb*` + * `string:fl` 返回字段(id, attr_name, attr_alias均可)，英文半角逗号分隔 + * `string:ret_key` 返回字段类型 `Enum("id", "name", "alias")` 默认 `name` + * `count` 指定一次返回CI数 + * `facet` 属性字段，逗号分隔，返回属性字段对应的所有值 + +* 搜索表达式： + * 简单的字符串 + * `attribute:value` 指定属性搜索， `attribute`可以是`id`,`attr_name`和`attr_alias` + * 以上的组合，逗号分隔 + +* 组合查询支持 + * `AND`关系-`默认关系` + * `OR`关系 - eg.`-hostname:cmdb*`、 + * `NOT`关系-属性字段前加`~`eg. `~hostname:cmdb*` + * `IN`查询. eg. `hostname:(cmdb*;cmdb-web*)` 小括号, 分号分隔 + * `RANGE`查询. eg. `hostname:[cmdb* _TO_ cmdb-web*]` `_TO_`分隔 + * `COMPARISON`查询. eg. `cpu_core_num:>5` 支持`>, >=, <, <=` + +## api key 认证 + +每个用户会自动生成一个 `api key` 和 一个`secret`, 通过API接口使用的时候，需要提供一个参数 `_key`值为您的`api key`， 以及参数`_secret`值为除`_key`以外的参数，按照**参数名的字典序**排列，并连接到`url path` + `secret`之后的`sha1`**十六进制**值。 + + +## 管理接口 + +### Attribute管理接口 +* GET `/api/v0.1/attributes` 列出所有属性 + * param + * `string:q` 属性名称或者别名，允许为空 + * return + + ``` + { + "numfound": 1, + "attributes": [ + { + "attr_name": "idc", + "is_choice": true, + "choice_value": ["南汇", "欧阳路"], + "attr_id": 1, + "is_multivalue": false, + "attr_alias": "IDC", + "value_type": "text", + "is_uniq": false + } + } + ``` + + * error 无 + + +* GET `/api/v0.1/attributes/`、 `/api/v0.1/attributes/` 根据属性名称、别名或ID获取属性 + * param + * `string:attr_name` 属性名称或别名 + * `int:attr_id` 属性ID + * `attr_id`和`attr_name`选其一 + * return + + ``` + { + "attribute": { + "attr_name": "idc", + "is_choice": true, + "choice_value": ["南汇", "欧阳路"], + "attr_id": 1, + "is_multivalue": false, + "attr_alias": "IDC", + "value_type": "text", + "is_uniq": false + }, + } + ``` + + * error + * `404` 找不到属性 + +* POST `/api/v0.1/attributes` 增加新的属性 + * param + * `string:attr_name` 属性名称 + * `string:attr_alias` 属性别名，可为空，为空时等于`attr_name` + * `boolean:choice_value` 若属性有预定义值, 则不能为空 + * `boolean:is_multivalue` 属性是否允许多值，默认`False` + * `boolean:is_uniq` 属性是否唯一，默认`False` + * `string:value_type` 属性值类型， `Enum("text", "int", "float", "date")`, 默认`text` + + * return + + ``` + { + "attr_id":1 + } + ``` + + * error + * `500` 属性已存在 + * `500` 属性增加失败 + + * PUT `/api/v0.1/attributes/` 修改属性 + * param + * `string:attr_name` 属性名称 + * `string:attr_alias` 属性别名，可为空，为空时等于`attr_name` + * `boolean:choice_value` 若属性有预定义值, 则不能为空 + * `boolean:is_multivalue` 属性是否允许多值，值为0或者1，默认`False` + * `boolean:is_uniq` 属性是否唯一，值为0或者1，默认`False` + * `string:value_type` 属性值类型， `Enum("text", "int", "float", "date")`, 默认`text` + + * return + + ``` + { + "attr_id":1 + } + ``` + + * error + * `500` 属性已存在 + * `500` 属性增加失败 + + * DELETE `/api/v0.1/attributes/` 根据ID删除属性 + * param + * `int:attr_id` 属性ID + * return + + ``` + { + "message":"attribute %s deleted" % attr_name + } + ``` + + * error + * `404` 属性不存在 + * `500` 删除属性失败 + +#### CIType属性管理 + + * GET `/api/v0.1/attributes/citype/` 根据type_id查询固有属性列表 + * return + + ``` + { + "attributes": [ + { + "attr_name": "idc", + "is_choice": true, + "choice_value": ["南汇", "欧阳路"], + "attr_id": 1, + "is_multivalue": false, + "attr_alias": "IDC", + "value_type": "text", + "is_uniq": false + }, + ], + "type_id": 1, + } + ``` + +* POST `/api/v0.1/attributes/citype/` 根据`attr_id`增加CIType的属性 + * param + * `string:attr_id` `,`分隔的`attr_id` + * `int:is_required` 0或者1 + + * return + + ``` + { + "attributes":[1, 2, 3] + } + ``` + + * error + * `404` CIType不存在 + * `404` 属性不存在 + * `500` 增加失败 + +* DELETE `/api/v0.1/attributes/citype/` 删除CIType的属性 + * param + * `string:attr_id` `,`分隔的`attr_id` + + * return + + ``` + { + "attributes":[1, 2, 3] + } + ``` + + * error + * `404` CIType不存在 + * `404` 属性不存在 + * `500` 增加失败 + + +### CIType管理接口 + +* `/api/v0.1/citypes` 列出所有CI类型 + * param `string:type_name` 类型名称，允许为空 + * return + + ``` + { + "numfound": 2, + "citypes": [ + { + "uniq_key": "sn", + "type_name": "物理机", + "type_id": 1, + "enabled": True, + "icon_url": "" + }, + { + "uniq_key": "uuid", + "type_name": "KVM", + "type_id": 2, + "enabled": True, + "icon_url": "" + } + ], + } + ``` + * error 无 + +* GET `/api/v0.1/citypes/query` 查询CI类型 + * param `string:type` 可以是type_id, type_name, type_alias + * return + + ``` + { + "citype": { + "type_name": "software", + "type_id": 4, + "icon_url": "", + "type_alias": "\u8f6f\u4ef6", + "enabled": true, + "uniq_key": 21 + } + } + ``` + * error + * `400` message=输入参数缺失 + * `404` message='citype is not found' + +* POST `/api/v0.1/citypes` 增加新CIType + * param (下列参数任意一个或多个) + * `string:type_name` CIType名称 + * `string:type_alias` 类型别名，可为空 + * `int:_id` 唯一属性ID + * `string:unique` 唯一属性名称 + * `_id`和`unique`只能二选一 + * `icon_url` + * `enabled` 0/1 + * return + + ``` + { + "type_id": 2 + } + ``` + + * error + * `400` message=输入参数缺失 + * `500` message=CIType已存在 + * `500` message=唯一属性不存在 + * `500` message=唯一属性不是唯一的 + +* PUT `/api/v0.1/citypes/` 修改CIType + * param (下列参数任意一个或多个) + * `string:type_name` CIType名称 + * `string:type_alias` 类型别名，可为空 + * `int:_id` 唯一属性ID + * `string:unique` 唯一属性名称 + * `_id`和`unique`只能二选一 + * `icon_url` + * `enabled` 0/1 + * return + + ``` + { + "type_id": 2 + } + ``` + + * error + * `400` message=输入参数缺失 + * `500` message=CIType已存在 + * `500` message=唯一属性不存在 + * `500` message=唯一属性不是唯一的 + +* GET/POST `/api/v0.1/citypes/enable/` 修改CIType + * param + * `enabled` 0 or 1 + * return + + ``` + { + "type_id": 2 + } + ``` + + * error + * `500` 设置失败 + * `404` CIType不存在 + +* DELETE `/api/v0.1/citypes/` 根据ID删除CIType + * return + + ``` + { + "message":"ci type %s deleted" % type_name + } + ``` + * error + * `500` 删除失败 + * `404` CIType不存在 + +### CITypeRelation管理接口 + +* GET `/api/v0.1/cityperelations/types` 列出所有CIType关系类型名 + * return + + ``` + { + "relation_types": ["连接", "位置", "附属", "部署"], + } + ``` + * error 无 + +* GET `/api/v0.1/cityperelations//children` 返回所有child id + * return + + ``` + { + "children": [ + { + "ctr_id": 1, + "type_name": "project", + "type_id": 2, + "icon_url": "", + "type_alias": "应用", + "enabled": true, + "uniq_key": 3 + } + ] + } + ``` + * error 无 + +* GET `/api/v0.1/cityperelations//parents` 返回parent id + * return + + ``` + { + "parents": [{'parent':1, 'relaltion_type': 'containes', "ctr_id":1}], + } + ``` + * error 无 + + +* POST `/api/v0.1/cityperelations//` 增加CIType关系 + * param + * `string:relation_type` 类型名称 + * return + + ``` + { + "ctr_id":1 + } + ``` + * error + * `500` 增加失败 + * `404` CIType不存在 + +* DELETE `/api/v0.1/cityperelations/` 根据`ctr_id`删除CIType关系 + * return + + ``` + { + "message":"CIType relation %s deleted" % type + } + ``` + * error + * `500` 删除失败 + * `404` 关系不存在 + + + +### CI管理接口 + +* GET `/api/v0.1/ci/type/` 查询CIType的所有CI,一次返回25条记录 + * param + * `string:fields` 返回属性名、id，逗号隔开 + * `string:ret_key` 返回属性key,默认'name',还可是'id', 'alias' + * `int:page` 页码 + + * return + + ``` + { + "numfound": 1, + "type_id":1, + "page": 1, + "cis": [ + { + "ci_type": "KVM", + "_type": 1, + "nic": [ + 2 + ], + "hostname": "xxxxxx", + "_unique": "xxxxxx", + "_id": 1 + } + ] + } + ``` + * erorr + * `404` CIType不存在 + +* GET `/api/v0.1/ci/` 查询CI + + * return + + ``` + { + "ci": { + "ci_type": "KVM", + "_type": 1, + "nic": [2], + "hostname": "xxxxx", + "_unique": "xxxxx", + "_id": 1 + }, + "ci_id": 1 + } + ``` + * erorr 无 + + + +* POST `/api/v0.1/ci` 增加CI + * param + * `string:ci_type` CIType name 或者id + * `string:_no_attribute_policy` 当添加不存在的attribute时的策略， 默认`ignore` + * 其他url参数`k=v`： `k` 为属性名(id或别名亦可)， `v`为对应的值 + * 此CIType的`unique`字段必须包含在url参数中 + * return + + ``` + { + "ci_id":1, + } + ``` + * erorr + * `500` 添加失败 + +* PUT `/api/v0.1/ci` 修改CI + * param + * `string:ci_type` CIType name 或者id + * `string:_no_attribute_policy` 当添加不存在的attribute时的策略， 默认`ignore` + * 其他url参数`k=v`： `k` 为属性名(id或别名亦可)， `v`为对应的值 + * 此CIType的`unique`字段必须包含在url参数中 + * return + + ``` + { + "ci_id":1, + } + ``` + * erorr + * `500` 添加失败 + +* DELETE `/api/v0.1/ci/` 删除ci + * return + + ``` + { + "message":"ok", + } + ``` + * erorr + * `500` 删除失败 + + +## CIRelaiton管理接口 + +* GET `/api/v0.1/cirelations/types` 列出所有CI关系类型名 + * return + + ``` + { + "relation_types": ["connect", "install", "deploy", "contain"], + } + ``` + * error 无 + +* GET `/api/v0.1/cirelations//second_cis` 返回所有second id + * return + + ``` + { + "numfound": 1, + "second_cis": [ + { + "ci_type": "project", + "ci_type_alias": "应用", + "_type": 2, + "_id": 18, + "project_name": "cmdb-api" + } + ] + } + ``` + * error 无 + +* GET `/api/v0.1/cirelations//first_cis` 返回first ci id + * return + + ``` + { + "first_cis": [ + { + "ci_type": "project", + "ci_type_alias": "应用", + "_type": 2, + "_id": 18, + "project_name": "cmdb-api" + } + ], + "numfound": 1 + } + ``` + * error 无 + + +* POST `/api/v0.1/cirelations//` 增加CI关系 + * param + * `int: more` more实例 + * `string:relation_type` 类型名称 + * return + + ``` + { + "cr_id":1 + } + ``` + * error + * `500` 增加失败 + * `404` CI不存在 + +* DELETE `/api/v0.1/cirelations/delete/` 根据`cr_id`删除CI关系 + * return + + ``` + { + "message":"CIType relation %s deleted" % type + } + ``` + * error + * `500` 删除失败 + * `404` 关系不存在 + + + +## 历史记录管理接口 +* GET `/api/v0.1/history/record` 查询历史记录 + * param + * `int: page` + * `string: username` 变更用户 + * `string: start` 变更开始时间 + * `string: end` 变更结束时间 + * return + + ``` + { + "username": "", + "start": "2014-12-31 14:57:43", + "end": "2015-01-07 14:57:43", + "records": [ + { + "origin": null, + "attr_history": [], + "timestamp": "2015-01-01 22:12:39", + "reason": null, + "rel_history": { + "add": 1 + }, + "user": 1, + "record_id": 1234, + "ticket_id": null + } + ] + } + ``` + * error 无 + + * GET `/api/v0.1/history/` 历史记录详情 + * return + + ``` + { + "username": "pycook", + "timestamp": "2015-01-02 20:21:16", + "rel_history": { + "add": [ + [ + 123, + "deploy", + 234 + ] + ], + "delete": [] + }, + "attr_history": {} + } + ``` + + * error + * `404` 该记录不存在 diff --git a/cmdb-api/cmdb_query_api.md b/cmdb-api/cmdb_query_api.md new file mode 100644 index 0000000..ed51597 --- /dev/null +++ b/cmdb-api/cmdb_query_api.md @@ -0,0 +1,99 @@ +# CMDB查询 API文档 + + + +## 用户接口 + +### CI通用搜索接口 + +* GET `/api/v0.1/ci/s` +* 参数 + * `string:_type` 搜索的ci_type，多个用分号隔开, 例如: _type:(docker;kvm) + * `string:q` 搜索表达式, 例如`q=hostname:cmdb*` + * `string:fl` 返回字段(id, attr_name, attr_alias均可)，英文半角逗号分隔 + * `string:ret_key` 返回字段类型 `Enum("id", "name", "alias")` 默认 `name` + * `count` 指定一次返回CI数 + * `facet` 属性字段，逗号分隔，返回属性字段对应的所有值 + * `wt` 返回的数据格式，默认为`json`, 可选参数为`xml` + +* 搜索表达式： + * 简单的字符串 + * `attribute:value` 指定属性搜索， `attribute`可以是`id`,`attr_name`和`attr_alias` + * 以上的组合，逗号分隔 + +* 组合查询支持 + * `AND`关系-`默认关系` + * `OR`关系 - eg.`-hostname:cmdb*`、 + * `NOT`关系-属性字段前加`~`eg. `~hostname:cmdb*` + * `IN`查询. eg. `hostname:(cmdb*;cmdb-web*)` 小括号, 分号分隔 + * `RANGE`查询. eg. `hostname:[cmdb* _TO_ cmdb-web*]` `_TO_`分隔 + * `COMPARISON`查询. eg. `cpu_count:>5` 支持`>, >=, <, <=` + +* 返回结果 + * 搜索表达式 `/api/v0.1/ci/s?q=_type:kvm,status:在线,idc:南汇,private_ip:10.1.1.1*&page=1&fl=hostname,private_ip&facet=private_ip&count=1` + * 返回数据（默认json） + + ``` + { + facet: { + private_ip: [ + [ + "10.1.1.11", + 1, + "private_ip" + ], + [ + "10.1.1.12", + 1, + "private_ip" + ], + [ + "10.1.1.13", + 1, + "private_ip" + ] + ] + }, + total: 1, + numfound: 3, + result: [ + { + ci_type: "kvm", + _type: 8, + _id: 3213, + hostname: "xxx11", + private_ip: [ + "10.1.1.11" + ] + }, + { + ci_type: "kvm", + _type: 8, + _id: 123232, + hostname: "xxx12", + private_ip: [ + "10.1.1.12" + ] + }, + { + ci_type: "kvm", + _type: 8, + _id: 123513, + hostname: "xxx13", + private_ip: [ + "10.1.1.13" + ] + } + ], + counter: { + kvm: 3 + }, + page: 1 + } +``` + + +### CI专用搜索接口 +##### 根据需求实现 + + \ No newline at end of file diff --git a/cmdb-api/command/__init__.py b/cmdb-api/command/__init__.py new file mode 100644 index 0000000..44d37d3 --- /dev/null +++ b/cmdb-api/command/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- \ No newline at end of file diff --git a/cmdb-api/config-sample.cfg b/cmdb-api/config-sample.cfg new file mode 100644 index 0000000..fb1eba7 --- /dev/null +++ b/cmdb-api/config-sample.cfg @@ -0,0 +1,69 @@ +# coding: utf-8 +# common + +DEBUG = True +SECRET_KEY = 'dsfdjsf@3213!@JKJWL' +HOST = 'http://127.0.0.1:5000' + +# # database +SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://mysqluser:password@127.0.0.1:3306/cmdb?charset=utf8' + +SQLALCHEMY_ECHO = False +SQLALCHEMY_POOL_SIZE = 10 +SQLALCHEMY_POOL_RECYCLE = 300 + +# # upload +UPLOADS_DEFAULT_DEST = './static/' +UPLOADS_DEFAULT_URL = '/static' + +# # cache +CACHE_TYPE = "redis" +CACHE_REDIS_HOST = "127.0.0.1" +CACHE_REDIS_PORT = 6379 +CACHE_KEY_PREFIX = "CMDB-API" +CACHE_DEFAULT_TIMEOUT = 3000 + +# # i18n +ACCEPT_LANGUAGES = ['en', 'zh'] +BABEL_DEFAULT_LOCALE = 'zh' +BABEL_DEFAULT_TIMEZONE = 'Asia/Shanghai' + +# # log +LOG_PATH = './logs/app.log' + +LOG_LEVEL = 'DEBUG' +ADMINS = ('@') + +# # mail +MAIL_SERVER = '' +MAIL_PORT = 25 +MAIL_USE_TLS = False +MAIL_USE_SSL = False +MAIL_DEBUG = True +MAIL_USERNAME = '' +MAIL_PASSWORD = '' +DEFAULT_MAIL_SENDER = '' + +# # LDAP +LDAP_SERVER = 'ldap://' +LDAP_DOMAIN = '' + +# # queue +CELERY_RESULT_BACKEND = "redis://127.0.0.1//" +BROKER_URL = 'redis://127.0.0.1//' +BROKER_VHOST = '/' + +# # zookeeper +ZK_SERVERS = "127.0.0.1:2181" + +# # assets +ASSETS_DEBUG = False + +# application + +# # pagination +PER_PAGE_COUNT_RANGE = (10, 25, 50, 100) +DEFAULT_PAGE_COUNT = 25 + + +WHITE_LIST = ["127.0.0.1"] diff --git a/cmdb-api/core/__init__.py b/cmdb-api/core/__init__.py new file mode 100644 index 0000000..99d8104 --- /dev/null +++ b/cmdb-api/core/__init__.py @@ -0,0 +1,11 @@ +# -*- coding:utf-8 -*- + + +from attribute import attribute +from ci_type import citype +from ci_type_relation import cityperelation +from ci_relation import cirelation +from ci import ci +from history import history +from account import account +from special import special \ No newline at end of file diff --git a/cmdb-api/core/account.py b/cmdb-api/core/account.py new file mode 100644 index 0000000..1a1ae82 --- /dev/null +++ b/cmdb-api/core/account.py @@ -0,0 +1,98 @@ +# -*- coding:utf-8 -*- + + +from flask import Blueprint +from flask import request + +from flask import g +from flask import abort +from flask import jsonify + +from models import row2dict +from lib.account import AccountManager +from lib.auth import auth_with_key + + +account = Blueprint('account', __name__) + + +@account.route("/", methods=["GET"]) +@auth_with_key +def get_user(uid=None): + manager = AccountManager() + user = manager.get_user_by_uid(uid) + if user: + return jsonify(rolenames=user.rolenames, user=row2dict(user)) + else: + return jsonify(user=None) + + +@account.route("", methods=["POST"]) +@auth_with_key +def create_user(): + manager = AccountManager() + params = {} + for k, v in request.values.iteritems(): + params[k] = v + user = manager.create_user(**params) + return jsonify(user=row2dict(user)) + + +@account.route("/", methods=["PUT"]) +@auth_with_key +def update_user(uid=None): + manager = AccountManager() + params = {} + for k, v in request.values.iteritems(): + params[k] = v + ret, res = manager.update_user(uid, **params) + if not ret: + abort(res[0], res[1]) + return jsonify(user=row2dict(res), rolenames=res.rolenames) + + +@account.route("/", methods=["DELETE"]) +@auth_with_key +def delete_user(uid=None): + manager = AccountManager() + ret, res = manager.delete_user(uid) + if not ret: + abort(res[0], res[1]) + return jsonify(uid=uid) + + +@account.route("/validate", methods=["POST"]) +@auth_with_key +def validate(): + username = request.values.get("username") + password = request.values.get("password") + manager = AccountManager() + user, authenticated = manager.validate(username, password) + if user and not authenticated: + return jsonify(code=401, user=row2dict(user), rolenames=user.rolenames) + elif not user: + return jsonify(code=404, message="user is not existed") + return jsonify(code=200, user=row2dict(user), rolenames=user.rolenames) + + +@account.route("/key", methods=["PUT"]) +@auth_with_key +def update_key(): + manager = AccountManager() + ret, res = manager.reset_key(g.user.uid) + if not ret: + abort(res[0], res[1]) + return jsonify(user=row2dict(res), rolenames=res.rolenames) + + +@account.route("/password", methods=["PUT"]) +@auth_with_key +def update_password(): + manager = AccountManager() + old = request.values.get("password") + new = request.values.get("new_password") + confirm = request.values.get("confirm") + ret, res = manager.update_password(g.user.uid, old, new, confirm) + if not ret: + abort(res[0], res[1]) + return jsonify(user=row2dict(res), rolenames=res.rolenames) diff --git a/cmdb-api/core/attribute.py b/cmdb-api/core/attribute.py new file mode 100644 index 0000000..edff09e --- /dev/null +++ b/cmdb-api/core/attribute.py @@ -0,0 +1,145 @@ +# -*- coding:utf-8 -*- + + +from flask import jsonify +from flask import request +from flask import Blueprint +from flask import abort +from flask import current_app + +from lib.attribute import AttributeManager +from lib.ci_type import CITypeAttributeManager +from lib.decorator import argument_required +from lib.exception import InvalidUsageError +from lib.auth import auth_with_key + +attribute = Blueprint("attribute", __name__) + + +@attribute.route("", methods=["GET"]) +def get_attributes(): + q = request.values.get("q") + attrs = AttributeManager().get_attributes(name=q) + count = len(attrs) + return jsonify(numfound=count, attributes=attrs) + + +@attribute.route("/", methods=["GET"]) +@attribute.route("/", methods=["GET"]) +def get_attribute(attr_name=None, attr_id=None): + attr_manager = AttributeManager() + attr_dict = None + if attr_name is not None: + attr_dict = attr_manager.get_attribute_by_name(attr_name) + if attr_dict is None: + attr_dict = attr_manager.get_attribute_by_alias(attr_name) + elif attr_id is not None: + attr_dict = attr_manager.get_attribute_by_id(attr_id) + if attr_dict is not None: + return jsonify(attribute=attr_dict) + abort(404, "attribute not found") + + +@attribute.route("", methods=["POST"]) +@auth_with_key +def create_attribute(): + with argument_required("attr_name"): + attr_name = request.values.get("attr_name") + current_app.logger.info(attr_name) + attr_alias = request.values.get("attr_alias", attr_name) + choice_value = request.values.get("choice_value") + is_multivalue = request.values.get("is_multivalue", False) + is_uniq = request.values.get("is_uniq", False) + is_index = request.values.get("is_index", False) + value_type = request.values.get("value_type", "text") + try: + is_multivalue = int(is_multivalue) + is_uniq = int(is_uniq) + is_index = int(is_index) + except ValueError: + raise InvalidUsageError("argument format is error") + attr_manager = AttributeManager() + kwargs = {"choice_value": choice_value, "is_multivalue": is_multivalue, + "is_uniq": is_uniq, "value_type": value_type, + "is_index": is_index} + ret, res = attr_manager.add(attr_name, attr_alias, **kwargs) + if not ret: + return abort(500, res) + return jsonify(attr_id=res) + + +@attribute.route("/", methods=["PUT"]) +@auth_with_key +def update_attribute(attr_id=None): + with argument_required("attr_name"): + attr_name = request.values.get("attr_name") + attr_alias = request.values.get("attr_alias", attr_name) + choice_value = request.values.get("choice_value") + is_multivalue = request.values.get("is_multivalue", False) + is_uniq = request.values.get("is_uniq", False) + value_type = request.values.get("value_type", "text") + try: + is_multivalue = int(is_multivalue) + is_uniq = int(is_uniq) + except ValueError: + raise InvalidUsageError("argument format is error") + attr_manager = AttributeManager() + kwargs = {"choice_value": choice_value, "is_multivalue": is_multivalue, + "is_uniq": is_uniq, "value_type": value_type} + ret, res = attr_manager.update(attr_id, attr_name, + attr_alias, **kwargs) + if not ret: + return abort(500, res) + return jsonify(attr_id=res) + + +@attribute.route("/", methods=["DELETE"]) +@auth_with_key +def delete_attribute(attr_id=None): + attr_manager = AttributeManager() + res = attr_manager.delete(attr_id) + return jsonify(message="attribute {0} deleted".format(res)) + + +@attribute.route("/citype/", methods=["GET"]) +def get_attributes_by_type(type_id=None): + manager = CITypeAttributeManager() + from models.cmdb import CITypeCache, CIAttributeCache + + t = CITypeCache.get(type_id) + if not t: + return abort(400, "CIType {0} is not existed".format(type_id)) + uniq_id = t.uniq_id + unique = CIAttributeCache.get(uniq_id).attr_name + return jsonify(attributes=manager.get_attributes_by_type_id(type_id), + type_id=type_id, uniq_id=uniq_id, unique=unique) + + +@attribute.route("/citype/", methods=["POST"]) +@auth_with_key +def create_attributes_to_citype(type_id=None): + with argument_required("attr_id"): + attr_ids = request.values.get("attr_id", "") + is_required = request.values.get("is_required", False) + attr_id_list = attr_ids.strip().split(",") + if "" in attr_id_list: + attr_id_list.remove("") + attr_id_list = map(int, attr_id_list) + try: + is_required = int(is_required) + except ValueError: + abort(500, "argument format is error") + manager = CITypeAttributeManager() + manager.add(type_id, attr_id_list, is_required=is_required) + return jsonify(attributes=attr_id_list) + + +@attribute.route("/citype/", methods=["DELETE"]) +@auth_with_key +def delete_attribute_in_type(type_id=None): + with argument_required("attr_id"): + attr_ids = request.values.get("attr_id", "") + attr_id_list = attr_ids.strip().split(",") + manager = CITypeAttributeManager() + manager.delete(type_id, attr_id_list) + return jsonify(attributes=attr_id_list) \ No newline at end of file diff --git a/cmdb-api/core/ci.py b/cmdb-api/core/ci.py new file mode 100644 index 0000000..151e0aa --- /dev/null +++ b/cmdb-api/core/ci.py @@ -0,0 +1,189 @@ +# -*- coding:utf-8 -*- + +import sys +reload(sys) +sys.setdefaultencoding("utf-8") +import time +import urllib + +from flask import Blueprint +from flask import request +from flask import jsonify +from flask import current_app +from flask import make_response +from flask import render_template +from flask import abort + +from lib.auth import auth_with_key +from lib.ci import CIManager +from lib.search import Search +from lib.search import SearchError +from lib.utils import get_page +from lib.utils import get_per_page +from models.ci_type import CITypeCache + +ci = Blueprint("ci", __name__) + + +@ci.route("/type/", methods=["GET"]) +def get_cis_by_type(type_id=None): + fields = request.args.get("fields", "").strip().split(",") + fields = filter(lambda x: x != "", fields) + + ret_key = request.args.get("ret_key", "name") + if ret_key not in ('name', 'alias', 'id'): + ret_key = 'name' + + page = get_page(request.values.get("page", 1)) + count = get_per_page(request.values.get("count")) + manager = CIManager() + res = manager.get_cis_by_type(type_id, ret_key=ret_key, + fields=fields, page=page, per_page=count) + return jsonify(type_id=type_id, numfound=res[0], + total=len(res[2]), page=res[1], cis=res[2]) + + +@ci.route("/", methods=['GET']) +def get_ci(ci_id=None): + fields = request.args.get("fields", "").strip().split(",") + fields = filter(lambda x: x != "", fields) + + ret_key = request.args.get("ret_key", "name") + if ret_key not in ('name', 'alias', 'id'): + ret_key = 'name' + + manager = CIManager() + ci = manager.get_ci_by_id(ci_id, ret_key=ret_key, fields=fields) + return jsonify(ci_id=ci_id, ci=ci) + + +@ci.route("/s", methods=["GET"]) +@ci.route("/search", methods=["GET"]) +def search(): + """@params: q: query statement + fl: filter by column + count: the number of ci + ret_key: id, name, alias + facet: statistic + wt: result format + """ + page = get_page(request.values.get("page", 1)) + count = get_per_page(request.values.get("count")) + + query = request.values.get('q', "") + fl = request.values.get('fl', "").split(",") + ret_key = request.values.get('ret_key', "name") + if ret_key not in ('name', 'alias', 'id'): + ret_key = 'name' + facet = request.values.get("facet", "").split(",") + wt = request.values.get('wt', 'json') + fl = filter(lambda x: x != "", fl) + facet = filter(lambda x: x != "", facet) + sort = request.values.get("sort") + + start = time.time() + s = Search(query, fl, facet, page, ret_key, count, sort) + try: + response, counter, total, page, numfound, facet = s.search() + except SearchError, e: + return abort(400, str(e)) + except Exception, e: + current_app.logger.error(str(e)) + return abort(500, "search unknown error") + + if wt == 'xml': + res = make_response( + render_template("search.xml", + counter=counter, + total=total, + result=response, + page=page, + numfound=numfound, + facet=facet)) + res.headers['Content-type'] = 'text/xml' + return res + current_app.logger.debug("search time is :{0}".format( + time.time() - start)) + return jsonify(numfound=numfound, + total=total, + page=page, + facet=facet, + counter=counter, + result=response) + + +@ci.route("", methods=["POST"]) +@auth_with_key +def create_ci(): + ci_type = request.values.get("ci_type") + _no_attribute_policy = request.values.get("_no_attribute_policy", "ignore") + + ci_dict = dict() + for k, v in request.values.iteritems(): + if k != "ci_type" and not k.startswith("_"): + ci_dict[k] = v.strip() + + manager = CIManager() + current_app.logger.debug(ci_dict) + ci_id = manager.add(ci_type, exist_policy="reject", + _no_attribute_policy=_no_attribute_policy, **ci_dict) + return jsonify(ci_id=ci_id) + + +@ci.route("", methods=["PUT"]) +@auth_with_key +def update_ci(): + if request.data: + args = dict() + _args = request.data.split("&") + for arg in _args: + if arg: + args[arg.split("=")[0]] = \ + urllib.unquote(urllib.unquote(arg.split("=")[1])) + else: + args = request.values + + ci_type = args.get("ci_type") + _no_attribute_policy = args.get("_no_attribute_policy", "ignore") + ci_dict = dict() + for k, v in args.items(): + if k != "ci_type" and not k.startswith("_"): + ci_dict[k] = v.strip() + + manager = CIManager() + ci_id = manager.add(ci_type, exist_policy="replace", + _no_attribute_policy=_no_attribute_policy, **ci_dict) + return jsonify(ci_id=ci_id) + + +@ci.route("/", methods=["DELETE"]) +@auth_with_key +def delete_ci(ci_id=None): + manager = CIManager() + manager.delete(ci_id) + return jsonify(message="ok") + + +@ci.route("/heartbeat//", methods=["POST"]) +def add_heartbeat(ci_type, unique): + if not unique or not ci_type: + return jsonify(message="error") + # return jsonify(message="ok") + return jsonify(message=CIManager().add_heartbeat(ci_type, unique)) + + +@ci.route("/heartbeat", methods=["GET"]) +def get_heartbeat(): + page = get_page(request.values.get("page", 1)) + ci_type = request.values.get("ci_type", "").strip() + try: + ci_type = CITypeCache.get(ci_type).type_id + except: + return jsonify(numfound=0, result=[]) + agent_status = request.values.get("agent_status", None) + if agent_status: + agent_status = int(agent_status) + numfound, result = CIManager().get_heartbeat(page, + ci_type, + agent_status=agent_status) + return jsonify(numfound=numfound, result=result) \ No newline at end of file diff --git a/cmdb-api/core/ci_relation.py b/cmdb-api/core/ci_relation.py new file mode 100644 index 0000000..63233df --- /dev/null +++ b/cmdb-api/core/ci_relation.py @@ -0,0 +1,70 @@ +# -*- coding:utf-8 -*- + + +from flask import Blueprint +from flask import jsonify +from flask import request + +from lib.ci import CIRelationManager +from lib.utils import get_page +from lib.utils import get_per_page +from lib.auth import auth_with_key + + +cirelation = Blueprint("cirelation", __name__) + + +@cirelation.route("/types", methods=["GET"]) +def get_types(): + manager = CIRelationManager() + return jsonify(relation_types=manager.relation_types) + + +@cirelation.route("//second_cis", methods=["GET"]) +def get_second_cis_by_first_ci(first_ci=None): + page = get_page(request.values.get("page", 1)) + count = get_per_page(request.values.get("count")) + relation_type = request.values.get("relation_type", "contain") + manager = CIRelationManager() + numfound, total, second_cis = manager.get_second_cis( + first_ci, page=page, per_page=count, relation_type=relation_type) + return jsonify(numfound=numfound, total=total, + page=page, second_cis=second_cis) + + +@cirelation.route("//first_cis", methods=["GET"]) +def get_first_cis_by_second_ci(second_ci=None): + page = get_page(request.values.get("page", 1)) + count = get_per_page(request.values.get("count")) + relation_type = request.values.get("relation_type", "contain") + + manager = CIRelationManager() + numfound, total, first_cis = manager.get_first_cis( + second_ci, per_page=count, page=page, relation_type=relation_type) + return jsonify(numfound=numfound, total=total, + page=page, first_cis=first_cis) + + +@cirelation.route("//", methods=["POST"]) +@auth_with_key +def create_ci_relation(first_ci=None, second_ci=None): + relation_type = request.values.get("relation_type", "contain") + manager = CIRelationManager() + res = manager.add(first_ci, second_ci, relation_type=relation_type) + return jsonify(cr_id=res) + + +@cirelation.route("/", methods=["DELETE"]) +@auth_with_key +def delete_ci_relation(cr_id=None): + manager = CIRelationManager() + manager.delete(cr_id) + return jsonify(message="CIType Relation is deleted") + + +@cirelation.route("//", methods=["DELETE"]) +@auth_with_key +def delete_ci_relation_2(first_ci, second_ci): + manager = CIRelationManager() + manager.delete_2(first_ci, second_ci) + return jsonify(message="CIType Relation is deleted") \ No newline at end of file diff --git a/cmdb-api/core/ci_type.py b/cmdb-api/core/ci_type.py new file mode 100644 index 0000000..85efa12 --- /dev/null +++ b/cmdb-api/core/ci_type.py @@ -0,0 +1,89 @@ +# -*- coding:utf-8 -*- + + +from flask import Blueprint +from flask import jsonify +from flask import request +from flask import abort + +from lib.ci_type import CITypeManager +from lib.decorator import argument_required +from lib.auth import auth_with_key + + +citype = Blueprint("citype", __name__) + + +@citype.route("", methods=["GET"]) +def get_citypes(): + type_name = request.args.get("type_name") + manager = CITypeManager() + citypes = manager.get_citypes(type_name) + count = len(citypes) + return jsonify(numfound=count, citypes=citypes) + + +@citype.route("/query", methods=["GET"]) +def query(): + with argument_required("type"): + _type = request.args.get("type") + manager = CITypeManager() + res = manager.query(_type) + return jsonify(citype=res) + + +@citype.route("", methods=["POST"]) +@auth_with_key +def create_citype(): + with argument_required("type_name"): + type_name = request.values.get("type_name") + type_alias = request.values.get("type_alias") + if type_alias is None: + type_alias = type_name + _id = request.values.get("_id") + unique = request.values.get("unique") + enabled = request.values.get("enabled", True) + icon_url = request.values.get("icon_url", "") + manager = CITypeManager() + ret, res = manager.add(type_name, type_alias, _id=_id, + unique=unique, enabled=enabled, + icon_url=icon_url) + if ret: + return jsonify(type_id=res) + abort(500, res) + + +@citype.route("/", methods=["PUT"]) +@auth_with_key +def update_citype(type_id=None): + type_name = request.values.get("type_name") + type_alias = request.values.get("type_alias") + _id = request.values.get("_id") + unique = request.values.get("unique") + icon_url = request.values.get("icon_url") + enabled = request.values.get("enabled") + enabled = False if enabled in (0, "0") else True \ + if enabled is not None else None + manager = CITypeManager() + ret, res = manager.update(type_id, type_name, type_alias, _id=_id, + unique=unique, icon_url=icon_url, + enabled=enabled) + if ret: + return jsonify(type_id=type_id) + abort(500, res) + + +@citype.route("/", methods=["DELETE"]) +@auth_with_key +def delete_citype(type_id=None): + manager = CITypeManager() + res = manager.delete(type_id) + return jsonify(message=res) + + +@citype.route("/enable/", methods=["GET", "POST"]) +def enable(type_id=None): + enable = request.values.get("enable", True) + manager = CITypeManager() + manager.set_enabled(type_id, enabled=enable) + return jsonify(type_id=type_id) \ No newline at end of file diff --git a/cmdb-api/core/ci_type_relation.py b/cmdb-api/core/ci_type_relation.py new file mode 100644 index 0000000..36d72ca --- /dev/null +++ b/cmdb-api/core/ci_type_relation.py @@ -0,0 +1,55 @@ +# -*- coding:utf-8 -*- + + +from flask import Blueprint +from flask import jsonify +from flask import request + +from lib.ci_type import CITypeRelationManager +from lib.auth import auth_with_key + + +cityperelation = Blueprint("cityperelation", __name__) + + +@cityperelation.route("/types", methods=["GET"]) +def get_types(): + manager = CITypeRelationManager() + return jsonify(relation_types=manager.relation_types) + + +@cityperelation.route("//children", methods=["GET"]) +def get_children_by_parent(parent=None): + manager = CITypeRelationManager() + return jsonify(children=manager.get_children(parent)) + + +@cityperelation.route("//parents", methods=["GET"]) +def get_parents_by_child(child=None): + manager = CITypeRelationManager() + return jsonify(parents=manager.get_parents(child)) + + +@cityperelation.route("//", methods=["POST"]) +@auth_with_key +def create_citype_realtions(parent=None, child=None): + relation_type = request.values.get("relation_type", "contain") + manager = CITypeRelationManager() + res = manager.add(parent, child, relation_type=relation_type) + return jsonify(ctr_id=res) + + +@cityperelation.route("/", methods=["DELETE"]) +@auth_with_key +def delete_citype_relation(ctr_id=None): + manager = CITypeRelationManager() + manager.delete(ctr_id) + return jsonify(message="CIType Relation is deleted") + + +@cityperelation.route("//", methods=["DELETE"]) +@auth_with_key +def delete_citype_relation_2(parent=None, child=None): + manager = CITypeRelationManager() + manager.delete_2(parent, child) + return jsonify(message="CIType Relation is deleted") diff --git a/cmdb-api/core/history.py b/cmdb-api/core/history.py new file mode 100644 index 0000000..5eee828 --- /dev/null +++ b/cmdb-api/core/history.py @@ -0,0 +1,116 @@ +# -*- coding:utf-8 -*- + + +import datetime + +from flask import jsonify +from flask import current_app +from flask import Blueprint +from flask import request +from flask import abort + +from models.history import OperationRecord +from models.history import CIRelationHistory +from models.history import CIAttributeHistory +from models.attribute import CIAttributeCache +from extensions import db +from models import row2dict +from models.account import UserCache +from lib.ci import CIManager +from lib.utils import get_page + +history = Blueprint("history", __name__) + + +@history.route("/record", methods=["GET"]) +def get_record(): + page = get_page(request.values.get("page", 1)) + _start = request.values.get("start") + _end = request.values.get("end") + username = request.values.get("username", "") + per_page_cnt = current_app.config.get("DEFAULT_PAGE_COUNT") + start, end = None, None + if _start: + try: + start = datetime.datetime.strptime(_start, '%Y-%m-%d %H:%M:%S') + except ValueError: + abort(400, 'incorrect start date time') + if _end: + try: + end = datetime.datetime.strptime(_end, '%Y-%m-%d %H:%M:%S') + except ValueError: + abort(400, 'incorrect end date time') + records = db.session.query(OperationRecord) + numfound = db.session.query(db.func.count(OperationRecord.record_id)) + if start: + records = records.filter(OperationRecord.timestamp >= start) + numfound = numfound.filter(OperationRecord.timestamp >= start) + if end: + records = records.filter(OperationRecord.timestamp <= end) + numfound = records.filter(OperationRecord.timestamp <= end) + if username: + user = UserCache.get(username) + if user: + records = records.filter(OperationRecord.uid == user.uid) + else: + return jsonify(numfound=0, records=[], + page=1, total=0, start=_start, + end=_end, username=username) + records = records.order_by(-OperationRecord.record_id).offset( + per_page_cnt * (page - 1)).limit(per_page_cnt).all() + total = len(records) + numfound = numfound.first()[0] + res = [] + for record in records: + _res = row2dict(record) + _res["user"] = UserCache.get(_res.get("uid")).nickname \ + if UserCache.get(_res.get("uid")).nickname \ + else UserCache.get(_res.get("uid")).username + attr_history = db.session.query(CIAttributeHistory.attr_id).filter( + CIAttributeHistory.record_id == _res.get("record_id")).all() + _res["attr_history"] = [CIAttributeCache.get(h.attr_id).attr_alias + for h in attr_history] + rel_history = db.session.query(CIRelationHistory.operate_type).filter( + CIRelationHistory.record_id == _res.get("record_id")).all() + rel_statis = {} + for rel in rel_history: + if rel.operate_type not in rel_statis: + rel_statis[rel.operate_type] = 1 + else: + rel_statis[rel.res.operate_type] += 1 + _res["rel_history"] = rel_statis + res.append(_res) + + return jsonify(numfound=numfound, records=res, page=page, total=total, + start=_start, end=_end, username=username) + + +@history.route("/", methods=["GET"]) +def get_detail_by_record(record_id=None): + record = db.session.query(OperationRecord).filter( + OperationRecord.record_id == record_id).first() + if record is None: + abort(404, "record is not found") + username = UserCache.get(record.uid).nickname \ + if UserCache.get(record.uid).nickname \ + else UserCache.get(record.uid).username + timestamp = record.timestamp.strftime("%Y-%m-%d %H:%M:%S") + attr_history = db.session.query(CIAttributeHistory).filter( + CIAttributeHistory.record_id == record_id).all() + rel_history = db.session.query(CIRelationHistory).filter( + CIRelationHistory.record_id == record_id).all() + attr_dict, rel_dict = dict(), {"add": [], "delete": []} + for attr_h in attr_history: + attr_dict[CIAttributeCache.get(attr_h.attr_id).attr_alias] = { + "old": attr_h.old, "new": attr_h.new, + "operate_type": attr_h.operate_type} + manager = CIManager() + for rel_h in rel_history: + _, first = manager.get_ci_by_id(rel_h.first_ci_id) + _, second = manager.get_ci_by_id(rel_h.second_ci_id) + rel_dict[rel_h.operate_type].append( + (first, rel_h.relation_type, second)) + + return jsonify(username=username, timestamp=timestamp, + attr_history=attr_dict, + rel_history=rel_dict) diff --git a/cmdb-api/core/special.py b/cmdb-api/core/special.py new file mode 100644 index 0000000..019e5f5 --- /dev/null +++ b/cmdb-api/core/special.py @@ -0,0 +1,16 @@ +# -*- coding:utf-8 -*- + + +from flask import Blueprint +from flask import jsonify + + +special = Blueprint(__name__, "special") + + +@special.route("/api/v0.1/special", methods=["GET"]) +def index(): + """ + 定义专用接口 + """ + return jsonify(code=200) \ No newline at end of file diff --git a/cmdb-api/core/statis.py b/cmdb-api/core/statis.py new file mode 100644 index 0000000..20b5061 --- /dev/null +++ b/cmdb-api/core/statis.py @@ -0,0 +1,12 @@ +# -*- coding:utf-8 -*- + + +from flask import Blueprint + + +statis = Blueprint("statis", __name__) + + +@statis.route("") +def statis(): + pass \ No newline at end of file diff --git a/cmdb-api/extensions.py b/cmdb-api/extensions.py new file mode 100644 index 0000000..5dfb606 --- /dev/null +++ b/cmdb-api/extensions.py @@ -0,0 +1,16 @@ +# encoding=utf-8 + + +from flask.ext.mail import Mail +from flask.ext.sqlalchemy import SQLAlchemy +from flask.ext.cache import Cache +from flask.ext.celery import Celery + + +__all__ = ['mail', 'db', 'cache', 'celery'] + + +mail = Mail() +db = SQLAlchemy() +cache = Cache() +celery = Celery() \ No newline at end of file diff --git a/cmdb-api/gunicornserver.py b/cmdb-api/gunicornserver.py new file mode 100644 index 0000000..6c5feab --- /dev/null +++ b/cmdb-api/gunicornserver.py @@ -0,0 +1,72 @@ +# encoding=utf-8 + +from flask_script import Command, Option + + +class GunicornServer(Command): + description = 'Run the app within Gunicorn' + + def __init__(self, host='127.0.0.1', port=5000, workers=8, + worker_class="gevent", daemon=False): + self.port = port + self.host = host + self.workers = workers + self.worker_class = worker_class + self.daemon = daemon + + def get_options(self): + return ( + Option('-H', '--host', + dest='host', + default=self.host), + + Option('-p', '--port', + dest='port', + type=int, + default=self.port), + + Option('-w', '--workers', + dest='workers', + type=int, + default=self.workers), + + Option("-c", "--worker_class", + dest='worker_class', + type=str, + default=self.worker_class), + + Option("-d", "--daemon", + dest="daemon", + type=bool, + default=self.daemon) + ) + + def handle(self, app, host, port, workers, worker_class, daemon): + + from gunicorn import version_info + + if version_info < (0, 9, 0): + from gunicorn.arbiter import Arbiter + from gunicorn.config import Config + + arbiter = Arbiter(Config({'bind': "%s:%d" % (host, int(port)), + 'workers': workers, + 'worker_class': worker_class, + 'daemon': daemon}), app) + arbiter.run() + else: + from gunicorn.app.base import Application + + class FlaskApplication(Application): + def init(self, parser, opts, args): + return { + 'bind': '{0}:{1}'.format(host, port), + 'workers': workers, + 'worker_class': worker_class, + 'daemon': daemon + } + + def load(self): + return app + + FlaskApplication().run() diff --git a/cmdb-api/lib/__init__.py b/cmdb-api/lib/__init__.py new file mode 100644 index 0000000..ef612ed --- /dev/null +++ b/cmdb-api/lib/__init__.py @@ -0,0 +1,4 @@ +# -*- coding:utf-8 -*- + + +__all__ = [] \ No newline at end of file diff --git a/cmdb-api/lib/account.py b/cmdb-api/lib/account.py new file mode 100644 index 0000000..31bb2b1 --- /dev/null +++ b/cmdb-api/lib/account.py @@ -0,0 +1,145 @@ +# -*- coding:utf-8 -*- + + +import uuid +import random +import string +import datetime + +from flask import current_app +from flask import abort + +from extensions import db +from models.account import UserCache +from models.account import User +from models.account import UserRole + + +class AccountManager(object): + def __init__(self): + pass + + def get_user_by_uid(self, uid): + user = UserCache.get(uid) + return user + + def _generate_key(self): + key = uuid.uuid4().hex + secret = ''.join(random.sample(string.ascii_letters + + string.digits + '~!@#$%^&*?', 32)) + return key, secret + + def validate(self, username, password): + user, authenticated = User.query.authenticate(username, password) + return user, authenticated + + def create_user(self, **kwargs): + username = kwargs.get("username") + if username: + user = UserCache.get(username) + if user is not None: + user, authenticated = self.validate( + username, kwargs.get("password")) + if authenticated: + return user + else: + return abort(401, "authenticate validate failed") + else: + return abort(400, "argument username is required") + user = User() + email = kwargs.get("email", "") + if not email: + return abort(400, "argument email is required") + user.email = email + user.password = kwargs.get("password") + user.username = kwargs.get("username", "") + user.nickname = kwargs.get("nickname") if kwargs.get("nickname") \ + else kwargs.get("username", "") + key, secret = self._generate_key() + user.key = key + user.secret = secret + user.date_joined = datetime.datetime.now() + user.block = 0 + + db.session.add(user) + try: + db.session.commit() + except Exception as e: + db.session.rollback() + current_app.logger.error("create user is error {0}".format(str(e))) + return abort(500, "create user is error, {0}".format(str(e))) + return user + + def update_user(self, uid, **kwargs): + user = UserCache.get(uid) + if user is None: + return abort(400, "the user[{0}] is not existed".format(uid)) + user.username = kwargs.get("username", "") \ + if kwargs.get("username") else user.username + user.nickname = kwargs.get("nickname") \ + if kwargs.get("nickname") else user.nickname + user.department = kwargs.get("department") \ + if kwargs.get("department") else user.department + user.catalog = kwargs.get("catalog") \ + if kwargs.get("catalog") else user.catalog + user.email = kwargs.get("email") \ + if kwargs.get("email") else user.email + user.mobile = kwargs.get("mobile") \ + if kwargs.get("mobile") else user.mobile + db.session.add(user) + try: + db.session.commit() + except Exception as e: + db.session.rollback() + current_app.logger.error("create user is error {0}".format(str(e))) + return abort(500, "create user is error, {0}".format(str(e))) + return True, user + + def delete_user(self, uid): + user = UserCache.get(uid) + if user is None: + return abort(400, "the user[{0}] is not existed".format(uid)) + db.session.query(UserRole).filter(UserRole.uid == uid).delete() + db.session.delete(user) + try: + db.session.commit() + except Exception as e: + db.session.rollback() + current_app.logger.error("delete user error, {0}".format(str(e))) + return abort(500, "delete user error, {0}".format(str(e))) + return True, uid + + def update_password(self, uid, old, new, confirm): + user = User.query.get(uid) + if not user: + return abort(400, "user is not existed") + if not user.check_password(old): + return abort(400, "invalidate old password") + if not (new and confirm and new == confirm): + return abort(400, """Password cannot be empty, + two inputs must be the same""") + user.password = new + db.session.add(user) + try: + db.session.commit() + except Exception as e: + db.session.rollback() + current_app.logger.error("set password error, %s" % str(e)) + return abort(500, "set password errors, {0:s}".format(str(e))) + return True, user + + def reset_key(self, uid): + user = UserCache.get(uid) + if user is None: + return abort(400, "the user[{0}] is not existed".format(uid)) + key, secret = self._generate_key() + user.key = key + user.secret = secret + db.session.add(user) + try: + db.session.commit() + except Exception as e: + db.session.rollback() + current_app.logger.error("reset key is error, {0}".format(str(e))) + return abort(500, "reset key is error, {0}".format(str(e))) + return True, user \ No newline at end of file diff --git a/cmdb-api/lib/attribute.py b/cmdb-api/lib/attribute.py new file mode 100644 index 0000000..19e889d --- /dev/null +++ b/cmdb-api/lib/attribute.py @@ -0,0 +1,167 @@ +# -*- coding:utf-8 -*- + +from flask import current_app +from flask import abort + +from extensions import db +from models.attribute import CIAttribute +from models.attribute import CIAttributeCache +from models import row2dict +from lib.const import type_map + + +class AttributeManager(object): + """ + CI attributes manager + """ + + def __init__(self): + pass + + def _get_choice_value(self, attr_id, value_type): + _table = type_map.get("choice").get(value_type) + choice_values = db.session.query(_table.value).filter( + _table.attr_id == attr_id).all() + return [choice_value.value for choice_value in choice_values] + + def _add_choice_value(self, choice_value, attr_id, value_type): + _table = type_map.get("choice").get(value_type) + db.session.query(_table).filter(_table.attr_id == attr_id).delete() + db.session.flush() + for v in choice_value.strip().split(","): + table = _table() + table.attr_id = attr_id + table.value = v + db.session.add(table) + db.session.flush() + + def get_attributes(self, name=None): + """ + return attribute by name, + if name is None, then return all attributes + """ + attrs = db.session.query(CIAttribute).filter( + CIAttribute.attr_name.ilike("%{0}%".format(name))).all() \ + if name is not None else db.session.query(CIAttribute).all() + res = list() + for attr in attrs: + attr_dict = row2dict(attr) + if attr.is_choice: + attr_dict["choice_value"] = self._get_choice_value( + attr.attr_id, attr.value_type) + res.append(attr_dict) + return res + + def get_attribute_by_name(self, attr_name): + attr = db.session.query(CIAttribute).filter( + CIAttribute.attr_name == attr_name).first() + if attr: + attr_dict = row2dict(attr) + if attr.is_choice: + attr_dict["choice_value"] = self._get_choice_value( + attr.attr_id, attr.value_type) + return attr_dict + + def get_attribute_by_alias(self, attr_alias): + attr = db.session.query(CIAttribute).filter( + CIAttribute.attr_alias == attr_alias).first() + if attr: + attr_dict = row2dict(attr) + if attr.is_choice: + attr_dict["choice_value"] = self._get_choice_value( + attr.attr_id, attr.value_type) + return attr_dict + + def get_attribute_by_id(self, attr_id): + attr = db.session.query(CIAttribute).filter( + CIAttribute.attr_id == attr_id).first() + if attr: + attr_dict = row2dict(attr) + if attr.is_choice: + attr_dict["choice_value"] = self._get_choice_value( + attr.attr_id, attr.value_type) + return attr_dict + + def add(self, attr_name, attr_alias, **kwargs): + choice_value = kwargs.get("choice_value", False) + attr = CIAttributeCache.get(attr_name) + if attr is not None: + return False, "attribute {0} is already existed".format(attr_name) + is_choice = False + if choice_value: + is_choice = True + if not attr_alias: + attr_alias = attr_name + attr = CIAttribute() + attr.attr_name = attr_name + attr.attr_alias = attr_alias + attr.is_choice = is_choice + attr.is_multivalue = kwargs.get("is_multivalue", False) + attr.is_uniq = kwargs.get("is_uniq", False) + attr.is_index = kwargs.get("is_index", False) + attr.value_type = kwargs.get("value_type", "text") + db.session.add(attr) + db.session.flush() + + if choice_value: + self._add_choice_value(choice_value, attr.attr_id, attr.value_type) + try: + db.session.commit() + except Exception as e: + db.session.rollback() + current_app.logger.error("add attribute error, {0}".format(str(e))) + return False, str(e) + CIAttributeCache.clean(attr) + return True, attr.attr_id + + def update(self, attr_id, *args, **kwargs): + attr = db.session.query(CIAttribute).filter_by(attr_id=attr_id).first() + if not attr: + return False, "CI attribute you want to update is not existed" + choice_value = kwargs.get("choice_value", False) + is_choice = False + if choice_value: + is_choice = True + attr.attr_name = args[0] + attr.attr_alias = args[1] + if not args[1]: + attr.attr_alias = args[0] + attr.is_choice = is_choice + attr.is_multivalue = kwargs.get("is_multivalue", False) + attr.is_uniq = kwargs.get("is_uniq", False) + attr.value_type = kwargs.get("value_type", "text") + db.session.add(attr) + db.session.flush() + if is_choice: + self._add_choice_value(choice_value, attr.attr_id, attr.value_type) + try: + db.session.commit() + except Exception as e: + db.session.rollback() + current_app.logger.error("update attribute error, {0}".format( + str(e))) + return False, str(e) + CIAttributeCache.clean(attr) + return True, attr.attr_id + + def delete(self, attr_id): + attr, name = db.session.query(CIAttribute).filter_by( + attr_id=attr_id).first(), None + if attr: + if attr.is_choice: + choice_table = type_map["choice"].get(attr.value_type) + db.session.query(choice_table).filter( + choice_table.attr_id == attr_id).delete() + name = attr.attr_name + CIAttributeCache.clean(attr) + db.session.delete(attr) + try: + db.session.commit() + except Exception as e: + db.session.rollback() + current_app.logger.error("delete attribute error, {0}".format( + str(e))) + return abort(500, str(e)) + else: + return abort(404, "attribute you want to delete is not existed") + return name \ No newline at end of file diff --git a/cmdb-api/lib/auth.py b/cmdb-api/lib/auth.py new file mode 100644 index 0000000..ce43d43 --- /dev/null +++ b/cmdb-api/lib/auth.py @@ -0,0 +1,68 @@ +# -*- coding:utf-8 -*- + +import urllib +from functools import wraps + +from flask import current_app +from flask import g +from flask import request +from flask import abort +from flask.ext.principal import identity_changed +from flask.ext.principal import Identity +from flask.ext.principal import AnonymousIdentity + +from models.account import User +from models.account import UserCache + + +def auth_with_key(func): + @wraps(func) + def wrapper(*args, **kwargs): + if isinstance(getattr(g, 'user', None), User): + identity_changed.send(current_app._get_current_object(), + identity=Identity(g.user.uid)) + return func(*args, **kwargs) + ip = request.remote_addr + if request.data: + request_args = dict() + _args = request.data.split("&") + for arg in _args: + if arg: + request_args[arg.split("=")[0]] = \ + urllib.unquote(arg.split("=")[1]) + else: + request_args = request.values + + key = request_args.get('_key') + secret = request_args.get('_secret') + if not key and not secret and \ + ip.strip() in current_app.config.get("WHITE_LIST"): + ip = ip.strip() + user = UserCache.get(ip) + if user: + identity_changed.send(current_app._get_current_object(), + identity=Identity(user.uid)) + return func(*args, **kwargs) + else: + identity_changed.send(current_app._get_current_object(), + identity=AnonymousIdentity()) + return abort(400, "invalid _key and _secret") + + path = request.path + + keys = sorted(request_args.keys()) + req_args = [request_args[k] for k in keys + if str(k) not in ("_key", "_secret")] + current_app.logger.debug('args is %s' % req_args) + user, authenticated = User.query.authenticate_with_key( + key, secret, req_args, path) + if user and authenticated: + identity_changed.send(current_app._get_current_object(), + identity=Identity(user.get("uid"))) + return func(*args, **kwargs) + else: + identity_changed.send(current_app._get_current_object(), + identity=AnonymousIdentity()) + return abort(400, "invalid _key and _secret") + + return wrapper diff --git a/cmdb-api/lib/ci.py b/cmdb-api/lib/ci.py new file mode 100644 index 0000000..a6282b0 --- /dev/null +++ b/cmdb-api/lib/ci.py @@ -0,0 +1,677 @@ +# -*- coding:utf-8 -*- + + +import uuid +import time +import datetime +import json + +from flask import current_app +from flask import abort +from sqlalchemy import or_ + +from extensions import db +from models.ci import CI +from models.ci_relation import CIRelation +from models.ci_type import CITypeAttribute +from models.ci_type import CITypeCache +from models.ci_type import CITypeSpecCache +from models.history import CIAttributeHistory +from models.attribute import CIAttributeCache +from lib.const import TableMap +from lib.const import type_map +from lib.value import AttributeValueManager +from lib.history import CIAttributeHistoryManger +from lib.history import CIRelationHistoryManager +from lib.query_sql import QUERY_HOSTS_NUM_BY_PRODUCT +from lib.query_sql import QUERY_HOSTS_NUM_BY_BU +from lib.query_sql import QUERY_HOSTS_NUM_BY_PROJECT +from lib.query_sql import QUERY_CIS_BY_IDS +from lib.query_sql import QUERY_CIS_BY_VALUE_TABLE +from lib.utils import rd +from tasks.cmdb import ci_cache +from tasks.cmdb import ci_delete + + +class CIManager(object): + """ manage CI interface + """ + + def __init__(self): + pass + + def get_ci_by_id(self, ci_id, ret_key="name", + fields=None, need_children=True, use_master=False): + """@params: `ret_key` is one of 'name', 'id', 'alias' + `fields` is list of attribute name/alias/id + """ + ci = CI.query.get(ci_id) or \ + abort(404, "CI {0} is not existed".format(ci_id)) + + res = dict() + + if need_children: + children = self.get_children(ci_id, ret_key=ret_key) # one floor + res.update(children) + ci_type = CITypeCache.get(ci.type_id) + res["ci_type"] = ci_type.type_name + uniq_key = CIAttributeCache.get(ci_type.uniq_id) + if not fields: # fields are all attributes + attr_ids = db.session.query(CITypeAttribute.attr_id).filter_by( + type_id=ci.type_id) + fields = [CIAttributeCache.get(_.attr_id).attr_name + for _ in attr_ids] + + if uniq_key.attr_name not in fields: + fields.append(uniq_key.attr_name) + if fields: + value_manager = AttributeValueManager() + _res = value_manager._get_attr_values( + fields, ci_id, + ret_key=ret_key, uniq_key=uniq_key, use_master=use_master) + res.update(_res) + res['_type'] = ci_type.type_id + res['_id'] = ci_id + return res + + def get_ci_by_ids(self, ci_id_list, ret_key="name", fields=None): + result = list() + for ci_id in ci_id_list: + res = self.get_ci_by_id(ci_id, ret_key=ret_key, fields=fields) + result.append(res) + return result + + def get_children(self, ci_id, ret_key='name', relation_type="contain"): + second_cis = db.session.query(CIRelation.second_ci_id).filter( + CIRelation.first_ci_id == ci_id).filter(or_( + CIRelation.relation_type == relation_type, + CIRelation.relation_type == "deploy")) + second_ci_ids = (second_ci.second_ci_id for second_ci in second_cis) + ci_types = {} + for ci_id in second_ci_ids: + type_id = db.session.query(CI.type_id).filter( + CI.ci_id == ci_id).first().type_id + if type_id not in ci_types: + ci_types[type_id] = [ci_id] + else: + ci_types[type_id].append(ci_id) + res = {} + for type_id in ci_types: + ci_type = CITypeCache.get(type_id) + children = get_cis_by_ids(map(str, ci_types.get(type_id)), + ret_key=ret_key) + res[ci_type.type_name] = children + return res + + def get_cis_by_type(self, type_id, ret_key="name", fields="", + page=1, per_page=None): + if per_page is None: + per_page = current_app.config.get("DEFAULT_PAGE_COUNT") + cis = db.session.query(CI.ci_id).filter(CI.type_id == type_id) + numfound = cis.count() + cis = cis.offset((page - 1) * per_page).limit(per_page) + res = list() + ci_ids = [str(ci.ci_id) for ci in cis] + if ci_ids: + res = get_cis_by_ids(ci_ids, ret_key, fields) + return numfound, page, res + + def ci_is_exist(self, ci_type, unique_key, unique): + table = TableMap(attr_name=unique_key.attr_name).table + unique = db.session.query(table).filter( + table.attr_id == unique_key.attr_id).filter( + table.value == unique).first() + if unique: + return db.session.query(CI).filter( + CI.ci_id == unique.ci_id).first() + + def _delete_ci_by_id(self, ci_id): + db.session.query(CI.ci_id).filter(CI.ci_id == ci_id).delete() + try: + db.session.commit() + except Exception as e: + db.session.rollback() + current_app.logger.error("delete ci is error, {0}".format(str(e))) + + def add(self, ci_type_name, exist_policy="replace", + _no_attribute_policy="ignore", **ci_dict): + ci_existed = False + ci_type = CITypeCache.get(ci_type_name) or \ + abort(404, "CIType {0} is not existed".format(ci_type_name)) + + unique_key = CIAttributeCache.get(ci_type.uniq_id) \ + or abort(500, 'illegality unique attribute') + + unique = ci_dict.get(unique_key.attr_name) \ + or abort(500, '{0} missing'.format(unique_key.attr_name)) + + old_ci = self.ci_is_exist(ci_type, unique_key, unique) + if old_ci is not None: + ci_existed = True + if exist_policy == 'reject': + return abort(500, 'CI is existed') + if old_ci.type_id != ci_type.type_id: # update ci_type + old_ci.type_id = ci_type.type_id + db.session.add(old_ci) + db.session.flush() + ci = old_ci + else: + if exist_policy == 'need': + return abort(404, 'CI {0} not exist'.format(unique)) + ci = CI() + ci.type_id = ci_type.type_id + _uuid = uuid.uuid4().hex + ci.uuid = _uuid + ci.created_time = datetime.datetime.now() + db.session.add(ci) + try: + db.session.commit() + except Exception as e: + db.session.rollback() + current_app.logger.error('add CI error: {0}'.format(str(e))) + return abort(500, 'add CI error') + value_manager = AttributeValueManager() + histories = list() + for p, v in ci_dict.items(): + ret, res = value_manager.add_attr_value( + p, v, ci.ci_id, ci_type, + _no_attribute_policy=_no_attribute_policy, + ci_existed=ci_existed) + if not ret: + db.session.rollback() + if not ci_existed: + self.delete(ci.ci_id) + current_app.logger.info(res) + return abort(500, res) + if res is not None: + histories.append(res) + try: + db.session.commit() + except Exception as e: + current_app.logger.error(str(e)) + db.session.rollback() + if not ci_existed: # only add + self.delete(ci.ci_id) + return abort(500, "add CI error") + his_manager = CIAttributeHistoryManger() + his_manager.add(ci.ci_id, histories) + ci_cache.apply_async([ci.ci_id], queue="cmdb_async") + return ci.ci_id + + def delete(self, ci_id): + ci = db.session.query(CI).filter(CI.ci_id == ci_id).first() + if ci is not None: + attrs = db.session.query(CITypeAttribute.attr_id).filter( + CITypeAttribute.type_id == ci.type_id).all() + attr_names = [] + for attr in attrs: + attr_names.append(CIAttributeCache.get(attr.attr_id).attr_name) + attr_names = set(attr_names) + for attr_name in attr_names: + Table = TableMap(attr_name=attr_name).table + db.session.query(Table).filter(Table.ci_id == ci_id).delete() + db.session.query(CIRelation).filter( + CIRelation.first_ci_id == ci_id).delete() + db.session.query(CIRelation).filter( + CIRelation.second_ci_id == ci_id).delete() + db.session.query(CIAttributeHistory).filter( + CIAttributeHistory.ci_id == ci_id).delete() + + db.session.flush() + db.session.delete(ci) + try: + db.session.commit() + except Exception as e: + db.session.rollback() + current_app.logger.error("delete CI error, {0}".format(str(e))) + return abort(500, "delete CI error, {0}".format(str(e))) + # TODO: write history + ci_delete.apply_async([ci.ci_id], queue="cmdb_async") + return ci_id + return abort(404, "CI {0} not found".format(ci_id)) + + def add_heartbeat(self, ci_type, unique): + ci_type = CITypeCache.get(ci_type) + if not ci_type: + return 'error' + uniq_key = CIAttributeCache.get(ci_type.uniq_id) + Table = TableMap(attr_name=uniq_key.attr_name).table + ci_id = db.session.query(Table.ci_id).filter( + Table.attr_id == uniq_key.attr_id).filter( + Table.value == unique).first() + if ci_id is None: + return 'error' + ci = db.session.query(CI).filter(CI.ci_id == ci_id.ci_id).first() + if ci is None: + return 'error' + + ci.heartbeat = datetime.datetime.now() + + db.session.add(ci) + db.session.commit() + return "ok" + + def get_heartbeat(self, page, type_id, agent_status=None): + query = db.session.query(CI.ci_id, CI.heartbeat) + expire = datetime.datetime.now() - datetime.timedelta(minutes=72) + if type_id: + query = query.filter(CI.type_id == type_id) + else: + query = query.filter(db.or_(CI.type_id == 7, CI.type_id == 8)) + if agent_status == -1: + query = query.filter(CI.heartbeat == None) + elif agent_status == 0: + query = query.filter(CI.heartbeat <= expire) + elif agent_status == 1: + query = query.filter(CI.heartbeat > expire) + numfound = query.count() + per_page_count = current_app.config.get("DEFAULT_PAGE_COUNT") + cis = query.offset((page - 1) * per_page_count).limit( + per_page_count).all() + ci_ids = [ci.ci_id for ci in cis] + heartbeat_dict = {} + for ci in cis: + if agent_status is not None: + heartbeat_dict[ci.ci_id] = agent_status + else: + if ci.heartbeat is None: + heartbeat_dict[ci.ci_id] = -1 + elif ci.heartbeat <= expire: + heartbeat_dict[ci.ci_id] = 0 + else: + heartbeat_dict[ci.ci_id] = 1 + current_app.logger.debug(heartbeat_dict) + ci_ids = map(str, ci_ids) + res = get_cis_by_ids(ci_ids, fields=["hostname", "private_ip"]) + result = [(i.get("hostname"), i.get("private_ip")[0], i.get("ci_type"), + heartbeat_dict.get(i.get("_id"))) for i in res + if i.get("private_ip")] + return numfound, result + + +class CIRelationManager(object): + """ + manage relation between CIs + """ + + def __init__(self): + pass + + @property + def relation_types(self): + """ all CIType relation types + """ + from lib.const import CI_RELATION_TYPES + + return CI_RELATION_TYPES + + def get_second_cis(self, first_ci, relation_type="contain", + page=1, per_page=None, **kwargs): + if per_page is None: + per_page = current_app.config.get("DEFAULT_PAGE_COUNT") + second_cis = db.session.query( + CI.ci_id).join( + CIRelation, CIRelation.second_ci_id == CI.ci_id).filter( + CIRelation.first_ci_id == first_ci).filter( + CIRelation.relation_type == relation_type) + if kwargs: # special for devices + second_cis = self._query_wrap_for_device(second_cis, **kwargs) + numfound = second_cis.count() + second_cis = second_cis.offset( + (page - 1) * per_page).limit(per_page).all() + ci_ids = [str(son.ci_id) for son in second_cis] + total = len(ci_ids) + result = get_cis_by_ids(ci_ids) + return numfound, total, result + + def get_grandsons(self, ci_id, page=1, per_page=None, **kwargs): + if per_page is None: + per_page = current_app.config.get("DEFAULT_PAGE_COUNT") + children = db.session.query(CIRelation.second_ci_id).filter( + CIRelation.first_ci_id == ci_id).subquery() + grandsons = db.session.query(CIRelation.second_ci_id).join( + children, + children.c.second_ci_id == CIRelation.first_ci_id).subquery() + grandsons = db.session.query(CI.ci_id).join( + grandsons, grandsons.c.second_ci_id == CI.ci_id) + if kwargs: + grandsons = self._query_wrap_for_device(grandsons, **kwargs) + + numfound = grandsons.count() + grandsons = grandsons.offset( + (page - 1) * per_page).limit(per_page).all() + if not grandsons: + return 0, 0, [] + ci_ids = [str(son.ci_id) for son in grandsons] + total = len(ci_ids) + result = get_cis_by_ids(ci_ids) + + return numfound, total, result + + def _sort_handler(self, sort_by, query_sql): + + if sort_by.startswith("+"): + sort_type = "asc" + sort_by = sort_by[1:] + elif sort_by.startswith("-"): + sort_type = "desc" + sort_by = sort_by[1:] + else: + sort_type = "asc" + attr = CIAttributeCache.get(sort_by) + if attr is None: + return query_sql + + attr_id = attr.attr_id + Table = TableMap(attr_name=sort_by).table + + CI_table = query_sql.subquery() + query_sql = db.session.query(CI_table.c.ci_id, Table.value).join( + Table, Table.ci_id == CI_table.c.ci_id).filter( + Table.attr_id == attr_id).order_by( + getattr(Table.value, sort_type)()) + + return query_sql + + def _query_wrap_for_device(self, query_sql, **kwargs): + _type = kwargs.pop("_type", False) or kwargs.pop("type", False) \ + or kwargs.pop("ci_type", False) + if _type: + ci_type = CITypeCache.get(_type) + if ci_type is None: + return + query_sql = query_sql.filter(CI.type_id == ci_type.type_id) + + for k, v in kwargs.iteritems(): + attr = CIAttributeCache.get(k) + if attr is None: + continue + Table = TableMap(attr_name=k).table + CI_table = query_sql.subquery() + query_sql = db.session.query(CI_table.c.ci_id).join( + Table, Table.ci_id == CI_table.c.ci_id).filter( + Table.attr_id == attr.attr_id).filter( + Table.value.ilike(v.replace("*", "%"))) + + current_app.logger.debug(query_sql) + sort_by = kwargs.pop("sort", False) + if sort_by: + query_sql = self._sort_handler(sort_by, query_sql) + return query_sql + + def get_great_grandsons(self, ci_id, page=1, per_page=None, **kwargs): + if per_page is None: + per_page = current_app.config.get("DEFAULT_PAGE_COUNT") + + children = db.session.query(CIRelation.second_ci_id).filter( + CIRelation.first_ci_id == ci_id).subquery() + grandsons = db.session.query(CIRelation.second_ci_id).join( + children, + children.c.second_ci_id == CIRelation.first_ci_id).subquery() + great_grandsons = db.session.query(CIRelation.second_ci_id).join( + grandsons, + grandsons.c.second_ci_id == CIRelation.first_ci_id).subquery() + great_grandsons = db.session.query(CI.ci_id).join( + great_grandsons, great_grandsons.c.second_ci_id == CI.ci_id) + if kwargs: + great_grandsons = self._query_wrap_for_device( + great_grandsons, **kwargs) + if great_grandsons is None: + return 0, 0, [] + numfound = great_grandsons.count() + great_grandsons = great_grandsons.offset( + (page - 1) * per_page).limit(per_page).all() + ci_ids = [str(son.ci_id) for son in great_grandsons] + total = len(ci_ids) + result = get_cis_by_ids(ci_ids) + + return numfound, total, result + + def get_first_cis(self, second_ci, relation_type="contain", + page=1, per_page=None): + """only for CI Type + """ + if per_page is None: + per_page = current_app.config.get("DEFAULT_PAGE_COUNT") + first_cis = db.session.query(CIRelation.first_ci_id).filter( + CIRelation.second_ci_id == second_ci).filter( + CIRelation.relation_type == relation_type) + numfound = first_cis.count() + first_cis = first_cis.offset( + (page - 1) * per_page).limit(per_page).all() + result = [] + first_ci_ids = [str(first_ci.first_ci_id) for first_ci in first_cis] + total = len(first_ci_ids) + if first_ci_ids: + result = get_cis_by_ids(first_ci_ids) + return numfound, total, result + + def get_grandfather(self, ci_id, relation_type="contain"): + """only for CI Type + """ + grandfather = db.session.query(CIRelation.first_ci_id).filter( + CIRelation.second_ci_id.in_(db.session.query( + CIRelation.first_ci_id).filter( + CIRelation.second_ci_id == ci_id).filter( + CIRelation.relation_type == relation_type))).filter( + CIRelation.relation_type == relation_type).first() + if grandfather: + return CIManager().get_ci_by_id(grandfather.first_ci_id, + need_children=False) + + def add(self, first_ci, second_ci, more=None, relation_type="contain"): + ci = db.session.query(CI.ci_id).filter(CI.ci_id == first_ci).first() + if ci is None: + return abort(404, "first_ci {0} is not existed".format(first_ci)) + c = db.session.query(CI.ci_id).filter(CI.ci_id == second_ci).first() + if c is None: + return abort(404, "second_ci {0} is not existed".format( + second_ci)) + existed = db.session.query(CIRelation.cr_id).filter( + CIRelation.first_ci_id == first_ci).filter( + CIRelation.second_ci_id == second_ci).first() + if existed is not None: + return existed.cr_id + cr = CIRelation() + cr.first_ci_id = first_ci + cr.second_ci_id = second_ci + if more is not None: + cr.more = more + cr.relation_type = relation_type + db.session.add(cr) + try: + db.session.commit() + except Exception as e: + db.session.rollback() + current_app.logger.error("add CIRelation is error, {0}".format( + str(e))) + return abort(500, "add CIRelation is error, {0}".format(str(e))) + # write history + his_manager = CIRelationHistoryManager() + his_manager.add(cr.cr_id, cr.first_ci_id, cr.second_ci_id, + relation_type, operate_type="add") + return cr.cr_id + + def delete(self, cr_id): + cr = db.session.query(CIRelation).filter( + CIRelation.cr_id == cr_id).first() + cr_id = cr.cr_id + first_ci = cr.first_ci_id + second_ci = cr.second_ci_id + if cr is not None: + db.session.delete(cr) + try: + db.session.commit() + except Exception as e: + db.session.rollback() + current_app.logger.error( + "delete CIRelation is error, {0}".format(str(e))) + return abort( + 500, "delete CIRelation is error, {0}".format(str(e))) + his_manager = CIRelationHistoryManager() + his_manager.add(cr_id, first_ci, second_ci, + cr.relation_type, operate_type="delete") + return True + return abort(404, "CI relation is not existed") + + def delete_2(self, first_ci, second_ci): + cr = db.session.query(CIRelation).filter( + CIRelation.first_ci_id == first_ci).filter( + CIRelation.second_ci_id == second_ci).first() + return self.delete(cr.cr_id) + + +class HostNumStatis(object): + def __init__(self): + pass + + def get_hosts_by_project(self, project_id_list=None): + res = {} + if not project_id_list: + project = CITypeCache.get("project") + projects = db.session.query(CI.ci_id).filter( + CI.type_id == project.type_id).all() + project_id_list = (project.ci_id for project in projects) + project_id_list = map(str, project_id_list) + project_ids = ",".join(project_id_list) + nums = db.session.execute(QUERY_HOSTS_NUM_BY_PROJECT.format( + "".join(["(", project_ids, ")"]))).fetchall() + if nums: + for ci_id in project_id_list: + res[int(ci_id)] = 0 + for ci_id, num in nums: + res[ci_id] = num + return res + + def get_hosts_by_product(self, product_id_list=None): + res = {} + if not product_id_list: + product = CITypeCache.get("product") + products = db.session.query(CI.ci_id).filter( + CI.type_id == product.type_id).all() + product_id_list = (product.ci_id for product in products) + product_id_list = map(str, product_id_list) + product_ids = ",".join(product_id_list) + nums = db.session.execute(QUERY_HOSTS_NUM_BY_PRODUCT.format( + "".join(["(", product_ids, ")"]))).fetchall() + if nums: + for ci_id in product_id_list: + res[int(ci_id)] = 0 + for ci_id, num in nums: + res[ci_id] = num + return res + + def get_hosts_by_bu(self, bu_id_list=None): + res = {} + if not bu_id_list: + bu = CITypeCache.get("bu") + bus = db.session.query(CI.ci_id).filter( + CI.type_id == bu.type_id).all() + bu_id_list = (bu.ci_id for bu in bus) + bu_id_list = map(str, bu_id_list) + bu_ids = ",".join(bu_id_list) + current_app.logger.debug(QUERY_HOSTS_NUM_BY_BU.format( + "".join(["(", bu_ids, ")"]))) + if not bu_ids: + return res + nums = db.session.execute( + QUERY_HOSTS_NUM_BY_BU.format( + "".join(["(", bu_ids, ")"]))).fetchall() + if nums: + for ci_id in bu_id_list: + res[int(ci_id)] = 0 + for ci_id, num in nums: + res[ci_id] = num + return res + + +def get_cis_by_ids(ci_ids, ret_key="name", fields="", value_tables=None): + """ argument ci_ids are string list of CI instance ID, eg. ['1', '2'] + """ + if not ci_ids: + return [] + start = time.time() + ci_id_tuple = tuple(map(int, ci_ids)) + res = rd.get(ci_id_tuple) + if res is not None and None not in res and ret_key == "name": + res = map(json.loads, res) + if not fields: + return res + else: + _res = [] + for d in res: + _d = dict() + _d["_id"], _d["_type"] = d.get("_id"), d.get("_type") + _d["ci_type"] = d.get("ci_type") + for field in fields: + _d[field] = d.get(field) + _res.append(_d) + current_app.logger.debug("filter time: %s" % (time.time() - start)) + return _res + current_app.logger.warning("cache not hit...............") + if not fields: + _fields = "" + else: + _fields = list() + for field in fields: + attr = CIAttributeCache.get(field) + if attr is not None: + _fields.append(str(attr.attr_id)) + _fields = "WHERE A.attr_id in ({0})".format(",".join(_fields)) + ci_ids = ",".join(ci_ids) + if value_tables is None: + value_tables = type_map["table_name"].values() + current_app.logger.debug(value_tables) + value_sql = " UNION ".join([QUERY_CIS_BY_VALUE_TABLE.format(value_table, + ci_ids) + for value_table in value_tables]) + query_sql = QUERY_CIS_BY_IDS.format(ci_ids, _fields, value_sql) + current_app.logger.debug(query_sql) + start = time.time() + hosts = db.session.execute(query_sql).fetchall() + current_app.logger.info("get cis time is: {0}".format( + time.time() - start)) + + ci_list = set() + res = list() + ci_dict = dict() + start = time.time() + for ci_id, type_id, attr_id, attr_name, \ + attr_alias, value, value_type, is_multivalue in hosts: + if ci_id not in ci_list: + ci_dict = dict() + ci_type = CITypeSpecCache.get(type_id) + ci_dict["_id"] = ci_id + ci_dict["_type"] = type_id + ci_dict["ci_type"] = ci_type.type_name + ci_dict["ci_type_alias"] = ci_type.type_alias + ci_list.add(ci_id) + res.append(ci_dict) + if ret_key == "name": + if is_multivalue: + if isinstance(ci_dict.get(attr_name), list): + ci_dict[attr_name].append(value) + else: + ci_dict[attr_name] = [value] + else: + ci_dict[attr_name] = value + elif ret_key == "alias": + if is_multivalue: + if isinstance(ci_dict.get(attr_alias), list): + ci_dict[attr_alias].append(value) + else: + ci_dict[attr_alias] = [value] + else: + ci_dict[attr_alias] = value + elif ret_key == "id": + if is_multivalue: + if isinstance(ci_dict.get(attr_id), list): + ci_dict[attr_id].append(value) + else: + ci_dict[attr_id] = [value] + else: + ci_dict[attr_id] = value + + current_app.logger.debug("result parser time is: {0}".format( + time.time() - start)) + return res \ No newline at end of file diff --git a/cmdb-api/lib/ci_type.py b/cmdb-api/lib/ci_type.py new file mode 100644 index 0000000..728beed --- /dev/null +++ b/cmdb-api/lib/ci_type.py @@ -0,0 +1,315 @@ +# -*- coding:utf-8 -*- + + +from flask import current_app +from flask import abort + +from extensions import db +from models import row2dict +from models.ci_type import CITypeAttribute +from models.ci_type import CIType +from models.ci_type import CITypeAttributeCache +from models.ci_type import CITypeCache +from models.ci_type_relation import CITypeRelation +from models.attribute import CIAttributeCache +from lib.attribute import AttributeManager + + +class CITypeAttributeManager(object): + """ + manage CIType's attributes, include query, add, update, delete + """ + + def __init__(self): + pass + + def get_attributes_by_type_id(self, type_id): + attrs = CITypeAttributeCache.get(type_id) + attr_manager = AttributeManager() + result = list() + for attr in attrs: + attr_dict = attr_manager.get_attribute_by_id(attr.attr_id) + attr_dict["is_required"] = attr.is_required + result.append(attr_dict) + return result + + def add(self, type_id, attr_ids=None, is_required=False): + """ + add attributes to CIType, attr_ids are list + """ + if not attr_ids or not isinstance(attr_ids, list): + return abort(500, "attr_ids must be required") + ci_type = CITypeCache.get(type_id) + if ci_type is None: + return abort(404, "CIType ID({0}) is not existed".format(type_id)) + for attr_id in attr_ids: + attr = CIAttributeCache.get(attr_id) + if attr is None: + return abort(404, + "attribute id {0} is not existed".format(attr_id)) + existed = db.session.query(CITypeAttribute.attr_id).filter_by( + type_id=type_id).filter_by(attr_id=attr_id).first() + if existed is not None: + continue + current_app.logger.debug(attr_id) + db.session.add(CITypeAttribute( + type_id=type_id, attr_id=attr_id, is_required=is_required)) + try: + db.session.commit() + except Exception as e: + db.session.rollback() + current_app.logger.error( + "add attribute to CIType is error, {0}".format(str(e))) + return abort( + 500, "add attribute to CIType is error, maybe duplicate entry") + + CITypeAttributeCache.clean(type_id) + return True + + def delete(self, type_id, attr_ids=None): + """ + delete attributes at CIType, attr_ids are list + """ + if not attr_ids or not isinstance(attr_ids, list): + return abort( + 500, "delete attribute of CIType, attr_ids must be required") + ci_type = CITypeCache.get(type_id) + if ci_type is None: + return abort( + 404, "CIType ID({0}) is not existed".format(type_id)) + for attr_id in attr_ids: + attr = CIAttributeCache.get(attr_id) + if attr is None: + return abort( + 404, "attribute id {0} is not existed".format(attr_id)) + db.session.query(CITypeAttribute).filter_by( + type_id=type_id).filter_by(attr_id=attr_id).delete() + try: + db.session.commit() + except Exception as e: + db.session.rollback() + current_app.logger.error( + "delete attributes of CIType is error, {0}".format(str(e))) + return abort(500, "delete attributes of CIType is error") + CITypeAttributeCache.clean(type_id) + return True + + +class CITypeManager(object): + """ + manage CIType + """ + + def __init__(self): + pass + + def get_citypes(self, type_name=None): + ci_types = db.session.query(CIType).all() if type_name is None else \ + db.session.query(CIType).filter( + CIType.type_name.ilike("%{0}%".format(type_name))).all() + res = list() + for ci_type in ci_types: + type_dict = row2dict(ci_type) + type_dict["uniq_key"] = CIAttributeCache.get( + type_dict["uniq_id"]).attr_name + res.append(type_dict) + return res + + def query(self, _type): + citype = CITypeCache.get(_type) + if citype: + return row2dict(citype) + return abort(404, "citype is not found") + + def add(self, type_name, type_alias, _id=None, unique=None, + icon_url="", enabled=True): + uniq_key = CIAttributeCache.get(_id) or CIAttributeCache.get(unique) + if uniq_key is None: + return False, "uniq_key is not existed" + citype = CITypeCache.get(type_name) + if citype: + return False, "this CIType {0} is existed".format(type_name) + _citype = CIType() + _citype.type_name = type_name + _citype.type_alias = type_alias + _citype.uniq_id = uniq_key.attr_id + _citype.enabled = enabled + _citype.icon_url = icon_url + db.session.add(_citype) + db.session.flush() + _citype_attr = CITypeAttribute() + _citype_attr.attr_id = uniq_key.attr_id + _citype_attr.type_id = _citype.type_id + _citype_attr.is_required = True + db.session.add(_citype_attr) + try: + db.session.commit() + except Exception as e: + db.session.rollback() + current_app.logger.error("add CIType is error, {0}".format(str(e))) + return False, str(e) + CITypeCache.clean(type_name) + return True, _citype.type_id + + def update(self, type_id, type_name, type_alias, _id=None, unique=None, + icon_url="", enabled=None): + citype = CITypeCache.get(type_id) + if citype is None: + return False, "CIType {0} is not existed".format(type_name) + uniq_key = CIAttributeCache.get(_id) or CIAttributeCache.get(unique) + if uniq_key is not None: + citype.uniq_id = uniq_key.attr_id + citype_attr = db.session.query(CITypeAttribute).filter( + CITypeAttribute.type_id == type_id).filter( + CITypeAttribute.attr_id == uniq_key.attr_id).first() + if citype_attr is None: + citype_attr = CITypeAttribute() + citype_attr.attr_id = uniq_key.attr_id + citype_attr.type_id = type_id + citype_attr.is_required = True + db.session.add(citype_attr) + if type_name: + citype.type_name = type_name + if type_alias: + citype.type_alias = type_alias + if icon_url: + citype.icon_url = icon_url + if enabled is not None: + citype.enabled = enabled + db.session.add(citype) + try: + db.session.commit() + except Exception as e: + db.session.rollback() + current_app.logger.error("add CIType is error, {0}".format(str(e))) + return False, str(e) + CITypeCache.clean(type_id) + return True, type_id + + def set_enabled(self, type_id, enabled=True): + citype = CITypeCache.get(type_id) + if citype is None: + return abort(404, "CIType[{0}] is not existed".format(type_id)) + citype.enabled = enabled + db.session.add(citype) + try: + db.session.commit() + except Exception as e: + db.session.rollback() + current_app.logger.error( + "set CIType enabled is error, {0}".format(str(e))) + return abort(500, str(e)) + return type_id + + def delete(self, type_id): + citype = db.session.query(CIType).filter_by(type_id=type_id).first() + type_name = citype.type_name + if citype: + db.session.delete(citype) + try: + db.session.commit() + except Exception as e: + db.session.rollback() + current_app.logger.error( + "delete CIType is error, {0}".format(str(e))) + return abort(500, str(e)) + CITypeCache.clean(type_id) + return "CIType {0} deleted".format(type_name) + return abort(404, "CIType is not existed") + + +class CITypeRelationManager(object): + """ + manage relation between CITypes + """ + + def __init__(self): + pass + + @property + def relation_types(self): + """ all CIType relation types + """ + from lib.const import CITYPE_RELATION_TYPES + + return CITYPE_RELATION_TYPES + + def get_children(self, parent_id): + children = db.session.query(CITypeRelation).filter( + CITypeRelation.parent_id == parent_id).all() + result = [] + for child in children: + ctr_id = child.ctr_id + citype = CITypeCache.get(child.child_id) + citype_dict = row2dict(citype) + citype_dict["ctr_id"] = ctr_id + manager = CITypeAttributeManager() + citype_dict["attributes"] = manager.get_attributes_by_type_id( + citype.type_id) + citype_dict["relation_type"] = child.relation_type + result.append(citype_dict) + return result + + def get_parents(self, child_id): + parents = db.session.query(CITypeRelation).filter( + CITypeRelation.child_id == child_id).all() + result = [] + for parent in parents: + ctr_id = parent.ctr_id + citype = CITypeCache.get(parent.parent_id) + citype_dict = row2dict(citype) + citype_dict["ctr_id"] = ctr_id + manager = CITypeAttributeManager() + citype_dict["attributes"] = manager.get_attributes_by_type_id( + citype.type_id) + citype_dict["relation_type"] = parent.relation_type + result.append(citype_dict) + return result + + def add(self, parent, child, relation_type="contain"): + p = CITypeCache.get(parent) + if p is None: + return abort(404, "parent {0} is not existed".format(parent)) + c = CITypeCache.get(child) + if c is None: + return abort(404, "child {0} is not existed".format(child)) + existed = db.session.query(CITypeRelation.ctr_id).filter_by( + parent_id=parent).filter_by(child_id=child).first() + if existed is not None: + return True, existed.ctr_id + ctr = CITypeRelation() + ctr.parent_id = parent + ctr.child_id = child + ctr.relation_type = relation_type + db.session.add(ctr) + try: + db.session.commit() + except Exception as e: + db.session.rollback() + current_app.logger.error( + "add CITypeRelation is error, {0}".format(str(e))) + return abort( + 500, "add CITypeRelation is error, {0}".format(str(e))) + return ctr.ctr_id + + def delete(self, ctr_id): + ctr = db.session.query(CITypeRelation).filter( + CITypeRelation.ctr_id == ctr_id).first() + if ctr: + db.session.delete(ctr) + try: + db.session.commit() + except Exception as e: + db.session.rollback() + current_app.logger.error( + "delete CITypeRelation is error, {0}".format(str(e))) + return abort( + 500, "delete CITypeRelation is error, {0}".format(str(e))) + return True + return abort(404, "CIType relation is not existed") + + def delete_2(self, parent, child): + ctr = db.session.query(CITypeRelation).filter( + CITypeRelation.parent_id == parent).filter( + CITypeRelation.child_id == child).first() + return self.delete(ctr.ctr_id) \ No newline at end of file diff --git a/cmdb-api/lib/const.py b/cmdb-api/lib/const.py new file mode 100644 index 0000000..51409a4 --- /dev/null +++ b/cmdb-api/lib/const.py @@ -0,0 +1,99 @@ +# -*- coding:utf-8 -*- + + +import datetime + +from models.attribute import TextChoice +from models.attribute import FloatChoice +from models.attribute import IntegerChoice +from models.attribute import CIAttributeCache +from models.ci_value import CIValueText +from models.ci_value import CIValueInteger +from models.ci_value import CIValueFloat +from models.ci_value import CIValueDateTime +from models.ci_value import CIIndexValueDateTime +from models.ci_value import CIIndexValueFloat +from models.ci_value import CIIndexValueInteger +from models.ci_value import CIIndexValueText + + +def string2int(x): + return int(float(x)) + + +def str2datetime(x): + try: + v = datetime.datetime.strptime(x, "%Y-%m-%d") + return v + except ValueError: + pass + try: + v = datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S") + return v + except ValueError: + pass + raise ValueError + + +type_map = { + 'converter': { + 'int': string2int, + 'float': float, + 'text': unicode, + 'datetime': str2datetime, + }, + 'choice': { + 'int': IntegerChoice, + 'float': FloatChoice, + 'text': TextChoice, + }, + 'table': { + 'int': CIValueInteger, + 'text': CIValueText, + 'datetime': CIValueDateTime, + 'float': CIValueFloat, + 'index_int': CIIndexValueInteger, + 'index_text': CIIndexValueText, + 'index_datetime': CIIndexValueDateTime, + 'index_float': CIIndexValueFloat, + }, + 'table_name': { + 'int': 'integers', + 'text': 'texts', + 'datetime': 'datetime', + 'float': 'floats', + 'index_int': 'index_integers', + 'index_text': 'index_texts', + 'index_datetime': 'index_datetime', + 'index_float': 'index_floats', + } +} + + +class TableMap(): + def __init__(self, attr_name=None): + self.attr_name = attr_name + + @property + def table(self): + if self.attr_name is not None: + attr = CIAttributeCache.get(self.attr_name) + if attr.is_index: + i = "index_{0}".format(attr.value_type) + else: + i = attr.value_type + return type_map["table"].get(i) + + @property + def table_name(self): + if self.attr_name is not None: + attr = CIAttributeCache.get(self.attr_name) + if attr.is_index: + i = "index_{0}".format(attr.value_type) + else: + i = attr.value_type + return type_map["table_name"].get(i) + + +CITYPE_RELATION_TYPES = ["connect", "deploy", "install", "contain"] +CI_RELATION_TYPES = ["connect", "deploy", "install", "contain"] \ No newline at end of file diff --git a/cmdb-api/lib/decorator.py b/cmdb-api/lib/decorator.py new file mode 100644 index 0000000..8cafba7 --- /dev/null +++ b/cmdb-api/lib/decorator.py @@ -0,0 +1,74 @@ +# -*- coding:utf-8 -*- + + +import time +from functools import wraps + +from flask import request +from flask import render_template +from flask import current_app + +from lib.exception import InvalidUsageError + + +def templated(template=None): + def decorator(f): + @wraps(f) + def decorated_function(*args, **kwargs): + template_name = template + if template_name is None: + template_name = request.endpoint.replace('.', '/') + '.html' + ctx = f(*args, **kwargs) + if ctx is None: + ctx = {} + elif not isinstance(ctx, dict): + return ctx + return render_template(template_name, **ctx) + + return decorated_function + + return decorator + + +def argument_required1(*args_required): + from manage import InvalidUsageError + + def decorator(f): + @wraps(f) + def decorated_function(*args, **kwargs): + for arg in args_required: + if request.values.get(arg, None) is None: + raise InvalidUsageError( + "argument {0} is required".format(arg), 400) + return f(*args, **kwargs) + + return decorated_function + + return decorator + + +class argument_required(object): + def __init__(self, *args): + self.args = args + + def __enter__(self): + for arg in self.args: + if not request.values.get(arg): + raise InvalidUsageError( + "argument {0} is required".format(arg), status_code=400) + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + + +def url_statistic(f): + @wraps(f) + def decorated_func(*args, **kwargs): + start = time.time() + r = f(*args, **kwargs) + spend = time.time() - start + url = request.path + current_app.logger.info(url) + current_app.logger.info(spend) + return r + return decorated_func \ No newline at end of file diff --git a/cmdb-api/lib/exception.py b/cmdb-api/lib/exception.py new file mode 100644 index 0000000..a11af85 --- /dev/null +++ b/cmdb-api/lib/exception.py @@ -0,0 +1,17 @@ +# -*- coding:utf-8 -*- + + +class InvalidUsageError(Exception): + status_code = 400 + + def __init__(self, message, status_code=None, payload=None): + Exception.__init__(self) + self.message = message + if status_code is not None: + self.status_code = status_code + self.payload = payload + + def to_dict(self): + rv = dict(self.payload or ()) + rv['message'] = self.message + return rv \ No newline at end of file diff --git a/cmdb-api/lib/history.py b/cmdb-api/lib/history.py new file mode 100644 index 0000000..ebcef62 --- /dev/null +++ b/cmdb-api/lib/history.py @@ -0,0 +1,75 @@ +# -*- coding:utf-8 -*- + + +import datetime + +from flask import current_app +from flask import g + +from extensions import db +from models.history import OperationRecord +from models.history import CIAttributeHistory +from models.history import CIRelationHistory + + +class CIAttributeHistoryManger(object): + def __init__(self): + pass + + def add(self, ci_id, history_list): + if history_list: + record = OperationRecord() + record.uid = g.user.uid + record.timestamp = datetime.datetime.now() + db.session.add(record) + db.session.commit() + for attr_id, operate_type, old, new in history_list: + history = CIAttributeHistory() + history.attr_id = attr_id + history.operate_type = operate_type + history.old = old + history.new = new + history.ci_id = ci_id + history.record_id = record.record_id + db.session.add(history) + + try: + db.session.commit() + except Exception as e: + db.session.rollback() + db.session.rollback() + current_app.logger.error( + "add attribute history error, {0}".format(str(e))) + return False, "add attribute history error, {0}".format(str(e)) + return True, None + + +class CIRelationHistoryManager(object): + def __init__(self): + pass + + def add(self, relation, first_ci, second_ci, + relation_type, operate_type="add"): + record = OperationRecord() + record.uid = g.user.uid + record.timestamp = datetime.datetime.now() + db.session.add(record) + db.session.flush() + + history = CIRelationHistory() + history.relation = relation + history.record_id = record.record_id + history.operate_type = operate_type + history.first_ci_id = first_ci + history.second_ci_id = second_ci + history.relation_type = relation_type + db.session.add(history) + + try: + db.session.commit() + except Exception as e: + db.session.rollback() + current_app.logger.error( + "add relation history is error, {0}".format(str(e))) + return False, "add relation history is error, {0}".format(str(e)) + return True, None diff --git a/cmdb-api/lib/mail.py b/cmdb-api/lib/mail.py new file mode 100644 index 0000000..57d8e03 --- /dev/null +++ b/cmdb-api/lib/mail.py @@ -0,0 +1,86 @@ +# -*- coding:utf-8 -*- + + +import requests + +from flask import current_app +from flask.ext.mail import Message + +from extensions import mail +from models.account import User + + +def sendmail(users, subject, message, html=False, app=None): + if app: + mail.app = app + else: + app = current_app + recipients = [x.email for x in users if isinstance(x, User)] + recipients.extend( + [x for x in users if isinstance(x, basestring) and '@' in x]) + sender = app.config.get('DEFAULT_MAIL_SENDER') + if html: + msg = Message(recipients=recipients, + html=message, + subject=subject, + sender=sender) + else: + msg = Message(recipients=recipients, + body=message, + subject=subject, + sender=sender) + mail.send(msg) + + +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.header import Header +from email.mime.image import MIMEImage +import smtplib +import time +from email import Utils + + +def send_mail(sender, receiver, subject, content, ctype="html", pics=(), + smtpserver='mail.51ping.com', + username="networkbench@51ping.com", password="12qwaszx"): + """subject and body are unicode objects""" + if ctype == "html": + msg = MIMEText(content, 'html', 'utf-8') + else: + msg = MIMEText(content, 'plain', 'utf-8') + + if len(pics) != 0: + msgRoot = MIMEMultipart('related') + msgText = MIMEText(content, 'html', 'utf-8') + msgRoot.attach(msgText) + i = 1 + for pic in pics: + fp = open(pic, "rb") + image = MIMEImage(fp.read()) + fp.close() + image.add_header('Content-ID', '' % i) + msgRoot.attach(image) + i += 1 + msg = msgRoot + + msg['Subject'] = Header(subject, 'utf-8') + msg['From'] = sender + msg['To'] = ';'.join(receiver) + msg['Message-ID'] = Utils.make_msgid() + msg['date'] = time.strftime('%a, %d %b %Y %H:%M:%S %z') + + smtp = smtplib.SMTP() + smtp.connect(smtpserver, 25) + smtp.login(username, password) + smtp.sendmail(sender, receiver, msg.as_string()) + smtp.quit() + + +def send_sms(mobile, content): + sms_uri = current_app.config.get("SMS_URI") % (mobile, content) + try: + current_app.logger.info(sms_uri) + requests.get(sms_uri) + except Exception as e: + current_app.logger.error("send sms error, %s" % str(e)) \ No newline at end of file diff --git a/cmdb-api/lib/query_sql.py b/cmdb-api/lib/query_sql.py new file mode 100644 index 0000000..960f02b --- /dev/null +++ b/cmdb-api/lib/query_sql.py @@ -0,0 +1,107 @@ +# -*- coding:utf-8 -*- + + +QUERY_HOSTS_BY_APP = """ + SELECT * + FROM cis + INNER JOIN ci_relations AS cr ON cis.`ci_id`=cr.`second_ci` + WHERE cr.`first_ci` = {0:d} LIMIT {1:d}, {2:d}; +""" + +QUERY_HOSTS_NUM_BY_PROJECT = """ + SELECT cr.first_ci_id, + count(DISTINCT cr.second_ci_id) + FROM ci_relations AS cr + WHERE cr.first_ci_id IN {0} + GROUP BY cr.first_ci_id +""" + +QUERY_HOSTS_NUM_BY_BU = """ + SELECT B.first_ci_id, + count(DISTINCT cr.second_ci_id) + FROM + (SELECT A.first_ci_id, + cr.second_ci_id + FROM + (SELECT cr.first_ci_id, + cis.ci_id + FROM cis + INNER JOIN ci_relations AS cr ON cis.ci_id=cr.second_ci_id + WHERE cr.first_ci_id IN {0}) AS A + INNER JOIN ci_relations AS cr ON cr.first_ci_id=A.ci_id) AS B + INNER JOIN ci_relations AS cr ON B.second_ci_id=cr.first_ci_id + GROUP BY B.first_ci_id +""" + +QUERY_HOSTS_NUM_BY_PRODUCT = """ + SELECT A.first_ci_id, + count(DISTINCT cr.second_ci_id) + FROM + (SELECT cr.first_ci_id, + cis.ci_id + FROM cis + INNER JOIN ci_relations AS cr ON cis.ci_id=cr.second_ci_id + WHERE cr.first_ci_id IN {0}) AS A + INNER JOIN ci_relations AS cr ON cr.first_ci_id=A.ci_id + GROUP BY A.first_ci_id; +""" + +QUERY_CIS_BY_VALUE_TABLE = """ + SELECT attr.attr_name, + attr.attr_alias, + attr.value_type, + attr.is_multivalue, + cis.type_id, + {0}.ci_id, + {0}.attr_id, + {0}.value + FROM {0} + INNER JOIN cis ON {0}.ci_id=cis.ci_id + AND {0}.`ci_id` IN ({1}) + INNER JOIN ci_attributes as attr ON attr.attr_id = {0}.attr_id +""" + +QUERY_CIS_BY_IDS = """ + SELECT A.ci_id, + A.type_id, + A.attr_id, + A.attr_name, + A.attr_alias, + A.value, + A.value_type, + A.is_multivalue + FROM + ({2}) AS A {1} + ORDER BY A.ci_id; +""" + +FACET_QUERY1 = """ + SELECT {0}.value, + count({0}.ci_id) + FROM {0} + INNER JOIN ci_attributes AS attr ON attr.attr_id={0}.attr_id + WHERE attr.attr_name="{1}" + GROUP BY {0}.ci_id; +""" + +FACET_QUERY = """ + SELECT {0}.value, + count({0}.ci_id) + FROM {0} + INNER JOIN ({1}) AS B ON B.ci_id={0}.ci_id + WHERE {0}.attr_id={2:d} + GROUP BY {0}.ci_id +""" + +QUERY_CI_BY_ATTR_NAME = """ + SELECT {0}.ci_id + FROM {0} + WHERE {0}.attr_id={1:d} + AND {0}.value {2} +""" + +QUERY_CI_BY_TYPE = """ + SELECT cis.ci_id + FROM cis + WHERE cis.type_id in ({0}) +""" \ No newline at end of file diff --git a/cmdb-api/lib/search.py b/cmdb-api/lib/search.py new file mode 100644 index 0000000..341950d --- /dev/null +++ b/cmdb-api/lib/search.py @@ -0,0 +1,348 @@ +# -*- coding:utf-8 -*- + + +import time + +from flask import current_app + +from lib.const import TableMap +from models.attribute import CIAttributeCache +from models.ci_type import CITypeCache +from extensions import db +from models import CI +from lib.ci import get_cis_by_ids +from lib.query_sql import FACET_QUERY +from lib.query_sql import QUERY_CI_BY_TYPE +from lib.query_sql import QUERY_CI_BY_ATTR_NAME + + +class SearchError(Exception): + def __init__(self, v): + self.v = v + + def __str__(self): + return self.v + + +class Search(object): + def __init__(self, query=None, fl=None, facet_field=None, + page=1, ret_key="name", count=1, sort=None): + self.orig_query = query + self.fl = fl + self.facet_field = facet_field + self.page = page + self.ret_key = ret_key + try: + self.count = int(count) + except ValueError: + self.count = current_app.config.get("DEFAULT_PAGE_COUNT") + self.sort = sort + self.query_sql = "" + self.type_id_list = [] + + def tor_proc(self, key): + tor = list() + if key.startswith("+"): + tor.append('&') + key = key[1:].strip() + elif key.startswith("-"): + tor.append('|') + key = key[1:].strip() + elif key.startswith("~"): + tor.append('~') + key = key[1:].strip() + if not tor: + tor = ['&', ''] + if len(tor) < 2: + tor.append('') + return tor, key + + def attr_name_proc(self, key): + tor, key = self.tor_proc(key) + if key in ('ci_type', 'type', '_type'): + return '_type', 'text', tor, None + if key in ('id', 'ci_id', '_id'): + return '_id', 'text', tor, None + attr = CIAttributeCache.get(key) + if attr is not None: + # if not attr.is_index: + # raise SearchError("{0} is not indexed".format(attr.attr_name)) + field_name = attr.attr_name + return field_name, attr.value_type, tor, attr + else: + raise SearchError("{0} is not existed".format(key)) + + def type_query_handler(self, v, only_type_query): + new_v = [v] + if v.startswith("(") and v.endswith(")"): + new_v = v[1:-1].split(";") + for _v in new_v: + ci_type = CITypeCache.get(_v) + if ci_type is not None: + self.type_id_list.append(str(ci_type.type_id)) + if self.type_id_list: + type_ids = ",".join(self.type_id_list) + _query_sql = QUERY_CI_BY_TYPE.format(type_ids) + if only_type_query: + return _query_sql + else: + return "" + return "" + + def in_query_handler(self, attr, v): + new_v = v[1:-1].split(";") + table_name = TableMap(attr_name=attr.attr_name).table_name + _query_sql = QUERY_CI_BY_ATTR_NAME.format( + table_name, attr.attr_id, + " OR {0}.value ".format(table_name).join(['LIKE "{0}"'.format( + _v.replace("*", "%")) for _v in new_v])) + return _query_sql + + def range_query_handler(self, attr, v): + start, end = [x.strip() for x in v[1:-1].split("_TO_")] + table_name = TableMap(attr_name=attr.attr_name).table_name + _query_sql = QUERY_CI_BY_ATTR_NAME.format( + table_name, attr.attr_id, "BETWEEN '{0}' AND '{1}'".format( + start.replace("*", "%"), end.replace("*", "%"))) + return _query_sql + + def comparison_query_handler(self, attr, v): + table_name = TableMap(attr_name=attr.attr_name).table_name + if (v.startswith("<") and not v.startswith("<=")) or \ + (v.startswith(">") and not v.startswith(">=")): + _query_sql = QUERY_CI_BY_ATTR_NAME.format( + table_name, attr.attr_id, "{0} '{1}'".format( + v[0], v[1:].replace("*", "%"))) + elif v.startswith(">=") or v.startswith("<="): + _query_sql = QUERY_CI_BY_ATTR_NAME.format( + table_name, attr.attr_id, "{0} '{1}'".format( + v[:2], v[2:].replace("*", "%"))) + return _query_sql + + def sort_query_handler(self, field, query_sql, only_type_query): + if field is None: + field = "" + if field.startswith("+"): + field = field[1:] + sort_type = "ASC" + elif field.startswith("-"): + field = field[1:] + sort_type = "DESC" + else: + sort_type = "ASC" + + if field in ("_id", "ci_id") or not field: + if only_type_query: + return """SELECT SQL_CALC_FOUND_ROWS DISTINCT B.ci_id + FROM ({0}) AS B {1}""".format( + query_sql, + "ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format( + (self.page - 1) * self.count, sort_type, self.count)) + elif self.type_id_list: + return """SELECT SQL_CALC_FOUND_ROWS DISTINCT B.ci_id + FROM ({0}) AS B {1}""".format( + query_sql, + "INNER JOIN cis on cis.ci_id=B.ci_id " + "WHERE cis.type_id in ({3}) " + "ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format( + (self.page - 1) * self.count, sort_type, self.count, + ",".join(self.type_id_list))) + else: + return """SELECT SQL_CALC_FOUND_ROWS DISTINCT B.ci_id + FROM ({0}) AS B {1}""".format( + query_sql, + "INNER JOIN cis on cis.ci_id=B.ci_id " + "ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format( + (self.page - 1) * self.count, sort_type, self.count)) + else: + attr = CIAttributeCache.get(field) + attr_id = attr.attr_id + + table_name = TableMap(attr_name=attr.attr_name).table_name + _v_query_sql = """SELECT {0}.ci_id, {1}.value FROM + ({2}) AS {0} INNER JOIN {1} ON {1}.ci_id = {0}.ci_id + WHERE {1}.attr_id = {3}""".format("ALIAS", table_name, + query_sql, attr_id) + new_table = _v_query_sql + if only_type_query: + return "SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id " \ + "FROM ({0}) AS C " \ + "ORDER BY C.value {2} " \ + "LIMIT {1:d}, {3};".format(new_table, + (self.page - 1) * self.count, + sort_type, self.count) + elif self.type_id_list: + return """SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id + FROM ({0}) AS C + INNER JOIN cis on cis.ci_id=C.ci_id + WHERE cis.type_id in ({4}) + ORDER BY C.value {2} + LIMIT {1:d}, {3};""".format(new_table, + (self.page - 1) * self.count, + sort_type, self.count, + ",".join(self.type_id_list)) + else: + return """SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id + FROM ({0}) AS C + ORDER BY C.value {2} + LIMIT {1:d}, {3};""".format(new_table, + (self.page - 1) * self.count, + sort_type, self.count) + + def _wrap_sql(self, tor, alias, _query_sql, query_sql): + if tor[0] == "&": + query_sql = """SELECT * FROM ({0}) as {1} + INNER JOIN ({2}) as {3} USING(ci_id)""".format( + query_sql, alias, _query_sql, alias + "A") + elif tor[0] == "|": + query_sql = "SELECT * FROM ({0}) as {1} UNION ALL ({2})".format( + query_sql, alias, _query_sql) + elif tor[0] == "~": + query_sql = "SELECT * FROM ({0}) as {1} LEFT JOIN ({2}) as {3} " \ + "USING(ci_id) WHERE {3}.ci_id is NULL".format( + query_sql, alias, _query_sql, alias + "A") + return query_sql + + def _execute_sql(self, query_sql, only_type_query): + v_query_sql = self.sort_query_handler(self.sort, query_sql, + only_type_query) + start = time.time() + execute = db.session.execute + current_app.logger.debug(v_query_sql) + res = execute(v_query_sql).fetchall() + end_time = time.time() + current_app.logger.debug("query ci ids time is: {0}".format( + end_time - start)) + numfound = execute("SELECT FOUND_ROWS();").fetchall()[0][0] + current_app.logger.debug("statistics ci ids time is: {0}".format( + time.time() - end_time) + ) + return numfound, res + + def query_build_raw(self): + query_sql, alias, tor = "", "A", ["&"] + is_first = True + only_type_query = False + queries = self.orig_query.split(",") + queries = filter(lambda x: x != "", queries) + for q in queries: + if q.startswith("_type"): + queries.remove(q) + queries.insert(0, q) + if len(queries) == 1 or queries[1].startswith("-") or \ + queries[1].startswith("~"): + only_type_query = True + break + current_app.logger.debug(queries) + special = True + for q in queries: + _query_sql = "" + if ":" in q: + k = q.split(":")[0].strip() + v = ":".join(q.split(":")[1:]).strip() + current_app.logger.info(v) + field, field_type, tor, attr = self.attr_name_proc(k) + if field == "_type": + _query_sql = self.type_query_handler(v, only_type_query) + current_app.logger.debug(_query_sql) + elif field == "_id": # exclude all others + _ci_ids = [str(v)] + ci = db.session.query(CI.ci_id).filter( + CI.ci_id == int(v)).first() + if ci is not None: + return 1, _ci_ids + elif field: + if attr is None: + raise SearchError("{0} is not found".format(field)) + # in query + if v.startswith("(") and v.endswith(")"): + _query_sql = self.in_query_handler(attr, v) + # range query + elif v.startswith("[") and v.endswith("]") and "_TO_" in v: + _query_sql = self.range_query_handler(attr, v) + # comparison query + elif v.startswith(">=") or v.startswith("<=") or \ + v.startswith(">") or v.startswith("<"): + _query_sql = self.comparison_query_handler(attr, v) + else: + table_name = \ + TableMap(attr_name=attr.attr_name).table_name + _query_sql = QUERY_CI_BY_ATTR_NAME.format( + table_name, attr.attr_id, + 'LIKE "{0}"'.format(v.replace("*", "%"))) + else: + return 0, [] + elif q: + return 0, [] + + if is_first and _query_sql and not only_type_query: + query_sql = "SELECT * FROM ({0}) AS {1}".format(_query_sql, + alias) + is_first = False + alias += "A" + elif only_type_query and special: + is_first = False + special = False + query_sql = _query_sql + elif _query_sql: + query_sql = self._wrap_sql(tor, alias, _query_sql, query_sql) + alias += "AA" + + _start = time.time() + if query_sql: + self.query_sql = query_sql + current_app.logger.debug(query_sql) + numfound, res = self._execute_sql(query_sql, only_type_query) + current_app.logger.info("query ci ids is: {0}".format( + time.time() - _start)) + return numfound, [_res[0] for _res in res] + return 0, [] + + def facet_build(self): + facet = {} + for f in self.facet_field: + k, field_type, _, attr = self.attr_name_proc(f) + if k: + table_name = TableMap(attr_name=k).table_name + query_sql = FACET_QUERY.format( + table_name, self.query_sql, attr.attr_id) + result = db.session.execute(query_sql).fetchall() + facet[k] = result + facet_result = dict() + for k, v in facet.items(): + if not k.startswith('_'): + a = getattr(CIAttributeCache.get(k), "attr_%s" % self.ret_key) + facet_result[a] = list() + for f in v: + if f[1] != 0: + facet_result[a].append((f[0], f[1], a)) + return facet_result + + def fl_build(self): + _fl = list() + for f in self.fl: + k, _, _, _ = self.attr_name_proc(f) + if k: + _fl.append(k) + return _fl + + def search(self): + numfound, ci_ids = self.query_build_raw() + ci_ids = map(str, ci_ids) + _fl = self.fl_build() + + if self.facet_field and numfound: + facet = self.facet_build() + else: + facet = dict() + + response, counter = [], {} + if ci_ids: + response = get_cis_by_ids(ci_ids, ret_key=self.ret_key, fields=_fl) + for res in response: + ci_type = res.get("ci_type") + if ci_type not in counter.keys(): + counter[ci_type] = 0 + counter[ci_type] += 1 + total = len(response) + return response, counter, total, self.page, numfound, facet \ No newline at end of file diff --git a/cmdb-api/lib/template/__init__.py b/cmdb-api/lib/template/__init__.py new file mode 100644 index 0000000..44d37d3 --- /dev/null +++ b/cmdb-api/lib/template/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- \ No newline at end of file diff --git a/cmdb-api/lib/template/filters.py b/cmdb-api/lib/template/filters.py new file mode 100644 index 0000000..149d4f7 --- /dev/null +++ b/cmdb-api/lib/template/filters.py @@ -0,0 +1,9 @@ +# -*- coding:utf-8 -*- + + +def convert_to_list(v): + if isinstance(v, list): + return v + if isinstance(v, tuple): + return list(v) + return [v, ] diff --git a/cmdb-api/lib/utils.py b/cmdb-api/lib/utils.py new file mode 100644 index 0000000..2a31655 --- /dev/null +++ b/cmdb-api/lib/utils.py @@ -0,0 +1,74 @@ +# -*- coding:utf-8 -*- + + +import redis + +from flask import current_app +import settings + + +class RedisHandler(object): + def __init__(self): + try: + pool = redis.ConnectionPool( + max_connections=settings.REDIS_MAX_CONN, + host=settings.REDIS_HOST, + port=settings.REDIS_PORT, + db=settings.REDIS_DB) + self.r = redis.Redis(connection_pool=pool) + except Exception as e: + print e + current_app.logger.error("init redis connection failed") + + @classmethod + def instance(cls): + if not hasattr(cls, "_instance"): + cls._instance = cls() + return cls._instance + + def get(self, ci_ids, key="CMDB_CI"): + try: + value = self.r.hmget(key, ci_ids) + except Exception as e: + current_app.logger.error("get redis error, %s" % str(e)) + return + return value + + def _set(self, ci, key="CMDB_CI"): + try: + self.r.hmset(key, ci) + except Exception as e: + current_app.logger.error("set redis error, %s" % str(e)) + + def add(self, ci): + self._set(ci) + + def delete(self, ci_id, key="CMDB_CI"): + try: + ret = self.r.hdel(key, ci_id) + if not ret: + current_app.logger.warn("ci [%d] is not in redis" % ci_id) + except Exception as e: + current_app.logger.error("delete redis key error, %s" % str(e)) + +rd = RedisHandler.instance() + + +def get_page(page): + try: + page = int(page) + except ValueError: + page = 1 + if page < 1: + page = 1 + return page + + +def get_per_page(per_page): + try: + per_page = int(per_page) + except: + per_page = current_app.config.get("DEFAULT_PAGE_COUNT") + if per_page < 1: + per_page = current_app.config.get("DEFAULT_PAGE_COUNT") + return per_page \ No newline at end of file diff --git a/cmdb-api/lib/value.py b/cmdb-api/lib/value.py new file mode 100644 index 0000000..f9916f7 --- /dev/null +++ b/cmdb-api/lib/value.py @@ -0,0 +1,170 @@ +# -*- coding:utf-8 -*- + + +import datetime + +from flask import current_app + +from extensions import db +from models.attribute import CIAttributeCache +from lib.attribute import AttributeManager +from lib.const import type_map +from lib.const import TableMap + + +class AttributeValueManager(object): + """ + manage CI attribute values + """ + + def __init__(self): + pass + + def _get_attr(self, key): + """key is one of attr_id, attr_name and attr_alias + """ + attr = CIAttributeCache.get(key) + return attr + + def _get_attr_values(self, fields, ci_id, + ret_key="name", + uniq_key=None, + use_master=False): + res = dict() + for field in fields: + attr = CIAttributeCache.get(field) + if not attr: + current_app.logger.warn('attribute %s not found' % field) + return res + table = TableMap(attr_name=attr.attr_name).table + if use_master: + rs = db.session().using_bind("master").query( + table.value).filter_by(ci_id=ci_id).filter_by( + attr_id=attr.attr_id) + else: + rs = db.session.query(table.value).filter_by( + ci_id=ci_id).filter_by(attr_id=attr.attr_id) + field_name = getattr(attr, "attr_{0}".format(ret_key)) + try: + if attr.is_multivalue: + if attr.value_type == 'datetime': + res[field_name] = [datetime.datetime.strftime( + x.value, '%Y-%m-%d %H:%M:%S') for x in rs.all()] + else: + res[field_name] = [x.value for x in rs.all()] + else: + x = rs.first() + if x: + if attr.value_type == 'datetime': + res[field_name] = datetime.datetime.strftime( + rs.first().value, '%Y-%m-%d %H:%M:%S') + else: + res[field_name] = rs.first().value + else: + res[field_name] = None + except AttributeError as e: + current_app.logger.warn("get ci by id error, {0}".format(e)) + if attr.is_multivalue: + res[field_name] = list() + else: + res[field_name] = "" + if uniq_key is not None and attr.attr_id == uniq_key.attr_id \ + and rs.first() is not None: + res['unique'] = uniq_key.attr_name + return res + + def _validate(self, attr, value, table, ci_id): + converter = type_map.get("converter").get(attr.value_type) + try: + v = converter(value) + except ValueError: + return False, "attribute value {0} converter fail".format(value) + if attr.is_choice: + choice_list = AttributeManager()._get_choice_value( + attr.attr_id, attr.value_type) + if v not in choice_list: + return False, "{0} is not existed in choice values".format( + value) + elif attr.is_uniq: + old_value = db.session.query(table.attr_id).filter( + table.attr_id == attr.attr_id).filter( + table.value == v).filter(table.ci_id != ci_id).first() + if old_value is not None: + return False, "attribute {0} value {1} must be unique".format( + attr.attr_name, value) + return True, v + + def add_attr_value(self, key, value, ci_id, ci_type, + _no_attribute_policy="ignore", ci_existed=False): + """key is one of attr_id, attr_name and attr_alias + """ + attr = self._get_attr(key) + if attr is None: + if _no_attribute_policy == 'ignore': + return True, None + if _no_attribute_policy == 'reject': + return False, 'attribute {0} not exist'.format(key) + table, old_value, old_value_table = TableMap( + attr_name=attr.attr_name).table, None, None + if ci_existed: + old_value_table = db.session.query(table).filter( + table.attr_id == attr.attr_id).filter( + table.ci_id == ci_id).first() + if old_value_table is not None: + old_value = old_value_table.value + if not value and ci_existed: + db.session.query(table).filter( + table.attr_id == attr.attr_id).filter( + table.ci_id == ci_id).delete() + if old_value: + return True, (attr.attr_id, "delete", old_value, None) + else: + return True, None + elif not value: + return True, None + if not attr.is_multivalue: + ret, res = self._validate(attr, value, table, ci_id) + if not ret: + return False, res + value_table = table() + if ci_existed: # for history + old = db.session.query(table).filter( + table.attr_id == attr.attr_id).filter( + table.value == value).filter( + table.ci_id == ci_id).first() + if old is not None: + return True, None + elif old_value_table: + value_table = old_value_table + value_table.ci_id = ci_id + value_table.attr_id = attr.attr_id + value_table.value = res + db.session.add(value_table) + elif attr.is_multivalue: + if ci_existed: + db.session.query(table).filter( + table.attr_id == attr.attr_id).filter( + table.ci_id == ci_id).delete() + + for v in value.strip().split(","): + ret, res = self._validate(attr, v, table, ci_id) + if not ret: + return False, res + value_table = table() + value_table.ci_id = ci_id + value_table.attr_id = attr.attr_id + value_table.value = res + db.session.add(value_table) + try: + db.session.commit() + except Exception as e: + db.session.rollback() + current_app.logger.error( + "add attribute value is error, {0}".format(str(e))) + return False, "add attribute value is error, {0}".format(str(e)) + if ci_existed: + if old_value != value: + return True, (attr.attr_id, "update", old_value, value) + else: + return True, None + return True, (attr.attr_id, "add", None, value) \ No newline at end of file diff --git a/cmdb-api/manage.py b/cmdb-api/manage.py new file mode 100644 index 0000000..3488926 --- /dev/null +++ b/cmdb-api/manage.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- + + +from flask import jsonify +from flask import make_response +from flask.ext.script import Manager +from flask.ext.script import prompt_bool +from flask.ext.celery import install_commands as install_celery_command + +from __init__ import make_app +from extensions import db +from gunicornserver import GunicornServer +from lib.exception import InvalidUsageError + + +app = make_app('config.cfg') + + +@app.errorhandler(InvalidUsageError) +def handle_invalid_usage(error): + response = jsonify(error.to_dict()) + response.status_code = error.status_code + return response + + +@app.errorhandler(404) +def not_found(error): + return make_response(jsonify({'message': error.description}), 404) + + +@app.errorhandler(400) +def bad_request(error): + return make_response(jsonify({'message': error.description}), 400) + + +@app.errorhandler(401) +def auth_lack(error): + return make_response(jsonify({'message': error.description}), 401) + + +@app.errorhandler(403) +def exception_403(error): + return make_response(jsonify({'message': error.description}), 403) + + +@app.errorhandler(405) +def exception_405(error): + return make_response(jsonify({'message': error.description}), 405) + + +@app.errorhandler(500) +def server_error(error): + return make_response(jsonify({"message": error.description}), 500) + + +manager = Manager(app) + +install_celery_command(manager) + + +@manager.command +def db_setup(): + "create all database tables" + db.create_all() + + +@manager.command +def db_dropall(): + "drop all databse tables" + if prompt_bool("Are you sure ? You will lose all your data !"): + db.drop_all() + + +manager.add_command("run", GunicornServer()) + +if __name__ == '__main__': + manager.run(default_command="runserver") diff --git a/cmdb-api/models/__init__.py b/cmdb-api/models/__init__.py new file mode 100644 index 0000000..b5f28a6 --- /dev/null +++ b/cmdb-api/models/__init__.py @@ -0,0 +1,24 @@ +# -*- coding:utf-8 -*- + + +def row2dict(row): + d = dict() + for c in row.__table__.columns: + if not isinstance(getattr(row, c.name), + (basestring, long, int, float, list, tuple, dict)) \ + and getattr(row, c.name): + d[c.name] = getattr(row, c.name).strftime("%Y-%m-%d %H:%M:%S") + else: + d[c.name] = getattr(row, c.name) + return d + + +from account import * +from attribute import * +from ci import * +from ci_relation import * +from ci_type import * +from ci_type_relation import * +from ci_value import * +from history import * +from statis import * diff --git a/cmdb-api/models/account.py b/cmdb-api/models/account.py new file mode 100644 index 0000000..0a3b322 --- /dev/null +++ b/cmdb-api/models/account.py @@ -0,0 +1,230 @@ +# -*- coding:utf-8 -*- + + +import hashlib +import copy +from datetime import datetime + +from werkzeug.utils import cached_property +from flask.ext.sqlalchemy import BaseQuery +from flask.ext.principal import RoleNeed +from flask.ext.principal import UserNeed +from flask.ext.principal import Permission + +from extensions import db +from extensions import cache +from permissions import admin +from models import row2dict + + +class UserQuery(BaseQuery): + def from_identity(self, identity): + """ + Loads user from flask.ext.principal.Identity instance and + assigns permissions from user. + + A "user" instance is monkey patched to the identity instance. + + If no user found then None is returned. + """ + + try: + _id = identity.id + if _id: + _id = int(_id) + user = self.get(_id) + except ValueError: + user = None + except Exception: + user = None + if user: + identity.provides.update(user.provides) + identity.user = user + return user + + def authenticate(self, login, password): + user = self.filter(db.or_(User.username == login, + User.email == login)).first() + if user: + authenticated = user.check_password(password) + else: + authenticated = False + return user, authenticated + + def authenticate_with_key(self, key, secret, args, path): + user = self.filter(User.key == key).filter(User.block == 0).first() + if not user: + return None, False + if user and hashlib.sha1('%s%s%s' % ( + path, user.secret, "".join(args))).hexdigest() == secret: + authenticated = True + else: + authenticated = False + return row2dict(user), authenticated + + def search(self, key): + query = self.filter(db.or_(User.email == key, + User.nickname.ilike('%' + key + '%'), + User.username.ilike('%' + key + '%'))) + return query + + def get_by_username(self, username): + user = self.filter(User.username == username).first() + return user + + def get_by_nickname(self, nickname): + user = self.filter(User.nickname == nickname).first() + return user + + def get(self, uid): + user = self.filter(User.uid == uid).first() + return copy.deepcopy(user) + + def is_exits(self, username): + user = self.filter(User.username == username).first() + return user is not None + + +class User(db.Model): + __tablename__ = 'users' + query_class = UserQuery + + ADMIN = 1 + + uid = db.Column(db.Integer, primary_key=True, autoincrement=True) + username = db.Column(db.String(32), unique=True) + nickname = db.Column(db.String(20), nullable=True) + department = db.Column(db.String(20)) + catalog = db.Column(db.String(64)) + email = db.Column(db.String(100), unique=True, nullable=False) + mobile = db.Column(db.String(14), unique=True) + _password = db.Column("password", db.String(80), nullable=False) + key = db.Column(db.String(32), nullable=False) + secret = db.Column(db.String(32), nullable=False) + date_joined = db.Column(db.DateTime, default=datetime.utcnow) + last_login = db.Column(db.DateTime, default=datetime.utcnow) + block = db.Column(db.Boolean, default=False) + has_logined = db.Column(db.Boolean, default=False) + + class Permissions(object): + def __init__(self, obj): + self.obj = obj + + @cached_property + def is_admin(self): + return Permission(UserNeed(self.obj.id)) & admin + + def __init__(self, *args, **kwargs): + super(User, self).__init__(*args, **kwargs) + + def __str__(self): + return self.username + + @cached_property + def permissions(self): + return self.Permissions(self) + + def _get_password(self): + return self._password + + def _set_password(self, password): + self._password = password + + password = db.synonym("_password", descriptor=property( + _get_password, _set_password)) + + def check_password(self, password): + return self.password == password + + @cached_property + def provides(self): + needs = [RoleNeed('authenticated'), UserNeed(self.uid)] + for r in self.rolenames: + needs.append(RoleNeed(r)) + if self.is_admin: + needs.append(RoleNeed('admin')) + return needs + + @property + def roles(self): + urs = db.session.query(UserRole.rid).filter( + UserRole.uid == self.uid).all() + return [x.rid for x in urs] + + @property + def rolenames(self): + return [db.session.query(Role.role_name).filter( + Role.rid == rid).first().role_name for rid in self.roles] + + @property + def is_admin(self): + return self.ADMIN in self.roles + + +class Role(db.Model): + __tablename__ = 'roles' + + rid = db.Column(db.Integer, primary_key=True, autoincrement=True) + role_name = db.Column(db.String(64), nullable=False, unique=True) + + +class UserRole(db.Model): + __tablename__ = 'users_roles' + + uid = db.Column(db.Integer, db.ForeignKey('users.uid'), primary_key=True) + rid = db.Column(db.Integer, db.ForeignKey('roles.rid'), primary_key=True) + + +class UserCache(object): + @classmethod + def get(cls, key): + user = cache.get("User::uid::%s" % key) or \ + cache.get("User::username::%s" % key) or \ + cache.get("User::nickname::%s" % key) + if not user: + user = User.query.get(key) or \ + User.query.get_by_username(key) or \ + User.query.get_by_nickname(key) + if user: + cls.set(user) + return user + + @classmethod + def set(cls, user): + cache.set("User::uid::%s" % user.uid, user) + cache.set("User::username::%s" % user.username, user) + cache.set("User::nickname::%s" % user.nickname, user) + + @classmethod + def clean(cls, user): + cache.delete("User::uid::%s" % user.uid) + cache.delete("User::username::%s" % user.username) + cache.delete("User::nickname::%s" % user.nickname) + + +class RoleCache(object): + @classmethod + def get(cls, rid): + role = None + if isinstance(rid, (int, long)): + role = cache.get("Role::rid::%s" % rid) + if not role: + role = db.session.query(Role).filter(Role.rid == rid).first() + cls.set(role) + elif isinstance(rid, basestring): + role = cache.get("Role::role_name::%s" % rid) + if not role: + role = db.session.query(Role).filter( + Role.role_name == rid).first() + cls.set(role) + return role + + @classmethod + def set(cls, role): + cache.set("Role::rid::%s" % role.rid, role) + cache.set("Role::role_name::%s" % role.role_name, role) + + @classmethod + def clean(cls, role): + cache.delete("Role::rid::%s" % role.rid, role) + cache.delete("Role::role_name::%s" % role.role_name, role) \ No newline at end of file diff --git a/cmdb-api/models/attribute.py b/cmdb-api/models/attribute.py new file mode 100644 index 0000000..fbc5c0b --- /dev/null +++ b/cmdb-api/models/attribute.py @@ -0,0 +1,87 @@ +# -*- coding:utf-8 -*- + +from extensions import db, cache + + +class CIAttribute(db.Model): + __tablename__ = "ci_attributes" + + attr_id = db.Column(db.Integer, primary_key=True, autoincrement=True) + attr_name = db.Column(db.String(32), nullable=False, unique=True) + attr_alias = db.Column(db.String(32), nullable=False, unique=True) + value_type = db.Column( + db.String(8), + db.Enum("int", "float", "text", "datetime", name='value_type'), + default="text", + nullable=False) + is_choice = db.Column(db.Boolean, default=False) + is_multivalue = db.Column(db.Boolean, default=False) + is_uniq = db.Column(db.Boolean, default=False) + is_index = db.Column(db.Boolean, default=False) + + +class IntegerChoice(db.Model): + __tablename__ = 'choice_integers' + + choice_id = db.Column(db.Integer, primary_key=True, autoincrement=True) + attr_id = db.Column(db.Integer, + db.ForeignKey('ci_attributes.attr_id'), + nullable=False) + attr = db.relationship("CIAttribute", backref="choice_integers") + value = db.Column(db.Integer, nullable=False) + + +class FloatChoice(db.Model): + __tablename__ = 'choice_floats' + + choice_id = db.Column(db.Integer, primary_key=True, autoincrement=True) + attr_id = db.Column(db.Integer, + db.ForeignKey('ci_attributes.attr_id'), + nullable=False) + attr = db.relationship("CIAttribute", backref="choice_floats") + value = db.Column(db.Float, nullable=False) + + +class TextChoice(db.Model): + __tablename__ = 'choice_texts' + + choice_id = db.Column(db.Integer, primary_key=True, autoincrement=True) + attr_id = db.Column(db.Integer, + db.ForeignKey('ci_attributes.attr_id'), + nullable=False) + attr = db.relationship("CIAttribute", backref="choice_texts") + value = db.Column(db.Text, nullable=False) + + +class CIAttributeCache(object): + @classmethod + def get(cls, key): + if key is None: + return + attr = cache.get('Field::Name::%s' % key) or \ + cache.get('Field::ID::%s' % key) or \ + cache.get('Field::Alias::%s' % key) + if attr is None: + attr = db.session.query(CIAttribute).filter_by( + attr_name=key).first() or \ + db.session.query(CIAttribute).filter( + CIAttribute.attr_id == key).first() or \ + db.session.query(CIAttribute).filter( + CIAttribute.attr_alias == key).first() + db.session.close() + if attr is not None: + CIAttributeCache.set(attr) + return attr + + @classmethod + def set(cls, attr): + cache.set('Field::ID::%s' % attr.attr_id, attr) + cache.set('Field::Name::%s' % attr.attr_name, attr) + cache.set('Field::Alias::%s' % attr.attr_alias, attr) + + @classmethod + def clean(cls, attr): + if cache.get('Field::ID::%s' % attr.attr_id): + cache.delete('Field::ID::%s' % attr.attr_id) + cache.delete('Field::Name::%s' % attr.attr_name) + cache.delete('Field::Alias::%s' % attr.attr_alias) \ No newline at end of file diff --git a/cmdb-api/models/ci.py b/cmdb-api/models/ci.py new file mode 100644 index 0000000..b4ee4cf --- /dev/null +++ b/cmdb-api/models/ci.py @@ -0,0 +1,20 @@ +# -*- coding:utf-8 -*- + +import datetime + +from extensions import db + + +class CI(db.Model): + __tablename__ = "cis" + + ci_id = db.Column(db.Integer, primary_key=True, autoincrement=True) + uuid = db.Column(db.String(32), nullable=False) + type_id = db.Column(db.Integer, + db.ForeignKey("ci_types.type_id"), + nullable=False) + ci_type = db.relationship("CIType", backref="cis") + status = db.Column(db.String(8), + db.Enum("review", "validate", name="stauts")) + created_time = db.Column(db.DateTime, default=datetime.datetime.now()) + heartbeat = db.Column(db.DateTime, default=datetime.datetime.now()) \ No newline at end of file diff --git a/cmdb-api/models/ci_relation.py b/cmdb-api/models/ci_relation.py new file mode 100644 index 0000000..6a347ea --- /dev/null +++ b/cmdb-api/models/ci_relation.py @@ -0,0 +1,26 @@ +# -*- coding:utf-8 -*- + + +from extensions import db + + +class CIRelation(db.Model): + __tablename__ = "ci_relations" + cr_id = db.Column(db.Integer, primary_key=True, autoincrement=True) + first_ci_id = db.Column(db.Integer, + db.ForeignKey("cis.ci_id"), + primary_key=True) + second_ci_id = db.Column(db.Integer, + db.ForeignKey("cis.ci_id"), + primary_key=True) + first_ci = db.relationship("CI", + primaryjoin="CI.ci_id==CIRelation.first_ci_id") + second_ci = db.relationship( + "CI", primaryjoin="CI.ci_id==CIRelation.second_ci_id") + relation_type = db.Column( + db.String(8), db.Enum("connect", "deploy", "install", "contain", + name="relation_type"), nullable=False) + more = db.Column(db.Integer, db.ForeignKey("cis.ci_id")) + + __table_args__ = (db.UniqueConstraint("first_ci_id", "second_ci_id", + name="first_second_uniq"), ) diff --git a/cmdb-api/models/ci_type.py b/cmdb-api/models/ci_type.py new file mode 100644 index 0000000..a995c82 --- /dev/null +++ b/cmdb-api/models/ci_type.py @@ -0,0 +1,128 @@ +# -*- coding:utf-8 -*- + +from extensions import db +from extensions import cache + + +class CIType(db.Model): + __tablename__ = "ci_types" + + type_id = db.Column(db.Integer, primary_key=True, autoincrement=True) + type_name = db.Column(db.String(32)) + type_alias = db.Column(db.String(32)) + uniq_id = db.Column(db.Integer, + db.ForeignKey("ci_attributes.attr_id"), + nullable=False) + uniq_key = db.relationship("CIAttribute", backref="ci_types") + enabled = db.Column(db.Boolean, default=True, nullable=False) + is_attached = db.Column(db.Boolean, default=False, nullable=False) + icon_url = db.Column(db.String(256)) + order = db.Column(db.SmallInteger, default=0, nullable=False) + + +class CITypeAttribute(db.Model): + __tablename__ = "type_attributes" + + type_id = db.Column(db.Integer, + db.ForeignKey("ci_types.type_id"), + primary_key=True) + attr_id = db.Column(db.Integer, + db.ForeignKey("ci_attributes.attr_id"), + primary_key=True) + is_required = db.Column(db.Boolean, default=False) + + __table_args__ = (db.UniqueConstraint("type_id", "attr_id", + name="type_attr_uniq"), ) + + +class CITypeCache(object): + @classmethod + def get(cls, key): + if key is None: + return + ct = cache.get("CIType::ID::%s" % key) or \ + cache.get("CIType::Name::%s" % key) + if ct is None: + ct = db.session.query(CIType).filter( + CIType.type_name == key).first() or \ + db.session.query(CIType).filter(CIType.type_id == key).first() + if ct is not None: + CITypeCache.set(ct) + return ct + + @classmethod + def set(cls, ct): + cache.set("CIType::Name::%s" % ct.type_name, ct) + cache.set("CIType::ID::%d" % ct.type_id, ct) + + @classmethod + def clean(cls, key): + ct = CITypeCache.get(key) + if ct is not None: + cache.delete("CIType::Name::%s" % ct.type_name) + cache.delete("CIType::ID::%s" % ct.type_id) + + +class CITypeSpecCache(object): + @classmethod + def get(cls, key): + if key is None: + return + ct = cache.get("CITypeSPEC::ID::%d" % key) + if ct is None: + ct = db.session.query(CIType).filter(CIType.type_id == key).first() + if ct is not None: + CITypeSpecCache.set(ct) + return ct + + @classmethod + def set(cls, ct): + cache.set("CITypeSPEC::ID::%d" % ct.type_id, ct) + + @classmethod + def clean(cls, key): + ct = CITypeCache.get(key) + if ct is not None: + cache.delete("CITypeSPEC::ID::%d" % ct.type_id) + + +class CITypeAttributeCache(object): + """ + key is type_id or type_name + """ + + @classmethod + def get(cls, key): + if key is None: + return + if isinstance(key, basestring) and isinstance(key, unicode): + key = unicode(key, 'utf8') + citypes = cache.get("CITypeAttribute::Name::%s" % key) or \ + cache.get("CITypeAttribute::ID::%s" % key) + if not citypes: + citypes = db.session.query(CITypeAttribute).filter( + CITypeAttribute.type_id == key).all() + if citypes is None: + ci_type = db.session.query(CIType).filter( + CIType.type_name == key).first() + if ci_type is not None: + citypes = db.session.query(CITypeAttribute).filter_by( + type_id=ci_type.type_id).all() + if citypes is not None: + CITypeAttributeCache.set(key, citypes) + return citypes + + @classmethod + def set(cls, key, values): + citype = CITypeCache.get(key) + if citype is not None: + cache.set("CITypeAttribute::ID::%s" % citype.type_id, values) + cache.set("CITypeAttribute::Name::%s" % citype.type_name, values) + + @classmethod + def clean(cls, key): + citype = CITypeCache.get(key) + attrs = CITypeAttributeCache.get(key) + if attrs is not None and citype: + cache.delete("CITypeAttribute::ID::%s" % citype.type_id) + cache.delete("CITypeAttribute::Name::%s" % citype.type_name) \ No newline at end of file diff --git a/cmdb-api/models/ci_type_relation.py b/cmdb-api/models/ci_type_relation.py new file mode 100644 index 0000000..cde0486 --- /dev/null +++ b/cmdb-api/models/ci_type_relation.py @@ -0,0 +1,27 @@ +# -*- coding:utf-8 -*- + +from extensions import db + + +class CITypeRelation(db.Model): + __tablename__ = "ci_type_relations" + + ctr_id = db.Column(db.Integer, primary_key=True, autoincrement=True) + parent_id = db.Column(db.Integer, + db.ForeignKey("ci_types.type_id"), + primary_key=True) + parent = db.relationship( + "CIType", primaryjoin="CIType.type_id==CITypeRelation.parent_id") + child_id = db.Column(db.Integer, + db.ForeignKey("ci_types.type_id"), + primary_key=True) + child = db.relationship( + "CIType", primaryjoin="CIType.type_id==CITypeRelation.child_id") + relation_type = db.Column( + db.String(7), + db.Enum("contain", "connect", "deploy", "install", + name="relation_type"), + default="contain") + + __table_args__ = (db.UniqueConstraint("parent_id", "child_id", + name="parent_child_uniq"), ) \ No newline at end of file diff --git a/cmdb-api/models/ci_value.py b/cmdb-api/models/ci_value.py new file mode 100644 index 0000000..f99068c --- /dev/null +++ b/cmdb-api/models/ci_value.py @@ -0,0 +1,117 @@ +# -*- coding:utf-8 -*- + + +from extensions import db +from sqlalchemy import Index + + +class CIIndexValueInteger(db.Model): + __tablename__ = "index_integers" + + value_id = db.Column(db.Integer, primary_key=True, autoincrement=True) + ci_id = db.Column(db.Integer, db.ForeignKey('cis.ci_id'), nullable=False) + attr_id = db.Column(db.Integer, + db.ForeignKey('ci_attributes.attr_id'), + nullable=False) + ci = db.relationship("CI", backref="index_integers") + attr = db.relationship("CIAttribute", backref="index_integers") + value = db.Column(db.Integer, nullable=False) + + __table_args__ = (Index("attr_value_index", "attr_id", "value"), ) + + +class CIIndexValueFloat(db.Model): + __tablename__ = "index_floats" + + value_id = db.Column(db.Integer, primary_key=True, autoincrement=True) + ci_id = db.Column(db.Integer, db.ForeignKey('cis.ci_id'), nullable=False) + attr_id = db.Column(db.Integer, + db.ForeignKey('ci_attributes.attr_id'), + nullable=False) + ci = db.relationship("CI", backref="index_floats") + attr = db.relationship("CIAttribute", backref="index_floats") + value = db.Column(db.Float, nullable=False) + + __table_args__ = (Index("attr_value_index", "attr_id", "value"), ) + + +class CIIndexValueText(db.Model): + __tablename__ = "index_texts" + + value_id = db.Column(db.Integer, primary_key=True, autoincrement=True) + ci_id = db.Column(db.Integer, db.ForeignKey('cis.ci_id'), nullable=False) + attr_id = db.Column(db.Integer, + db.ForeignKey('ci_attributes.attr_id'), + nullable=False) + ci = db.relationship("CI", backref="index_texts") + attr = db.relationship("CIAttribute", backref="index_texts") + value = db.Column(db.String(128), nullable=False) + + __table_args__ = (Index("attr_value_index", "attr_id", "value"), ) + + +class CIIndexValueDateTime(db.Model): + __tablename__ = "index_datetime" + + value_id = db.Column(db.Integer, primary_key=True, autoincrement=True) + ci_id = db.Column(db.Integer, db.ForeignKey('cis.ci_id'), nullable=False) + attr_id = db.Column(db.Integer, + db.ForeignKey('ci_attributes.attr_id'), + nullable=False) + ci = db.relationship("CI", backref="index_datetime") + attr = db.relationship("CIAttribute", backref="index_datetime") + value = db.Column(db.DateTime, nullable=False) + + __table_args__ = (Index("attr_value_index", "attr_id", "value"), ) + + +class CIValueInteger(db.Model): + __tablename__ = "integers" + + value_id = db.Column(db.Integer, primary_key=True, autoincrement=True) + ci_id = db.Column(db.Integer, db.ForeignKey('cis.ci_id'), nullable=False) + attr_id = db.Column(db.Integer, + db.ForeignKey('ci_attributes.attr_id'), + nullable=False) + ci = db.relationship("CI", backref="integers") + attr = db.relationship("CIAttribute", backref="integers") + value = db.Column(db.Integer, nullable=False) + + +class CIValueFloat(db.Model): + __tablename__ = "floats" + + value_id = db.Column(db.Integer, primary_key=True, autoincrement=True) + ci_id = db.Column(db.Integer, db.ForeignKey('cis.ci_id'), nullable=False) + attr_id = db.Column(db.Integer, + db.ForeignKey('ci_attributes.attr_id'), + nullable=False) + ci = db.relationship("CI", backref="floats") + attr = db.relationship("CIAttribute", backref="floats") + value = db.Column(db.Float, nullable=False) + + +class CIValueText(db.Model): + __tablename__ = "texts" + + value_id = db.Column(db.Integer, primary_key=True, autoincrement=True) + ci_id = db.Column(db.Integer, db.ForeignKey('cis.ci_id'), nullable=False) + attr_id = db.Column(db.Integer, + db.ForeignKey('ci_attributes.attr_id'), + nullable=False) + ci = db.relationship("CI", backref="texts") + attr = db.relationship("CIAttribute", backref="texts") + value = db.Column(db.Text, nullable=False) + + +class CIValueDateTime(db.Model): + __tablename__ = "datetime" + + value_id = db.Column(db.Integer, primary_key=True, autoincrement=True) + ci_id = db.Column(db.Integer, db.ForeignKey('cis.ci_id'), nullable=False) + attr_id = db.Column(db.Integer, + db.ForeignKey('ci_attributes.attr_id'), + nullable=False) + ci = db.relationship("CI", backref="datetime") + attr = db.relationship("CIAttribute", backref="datetime") + value = db.Column(db.DateTime, nullable=False) diff --git a/cmdb-api/models/history.py b/cmdb-api/models/history.py new file mode 100644 index 0000000..8fb9e7a --- /dev/null +++ b/cmdb-api/models/history.py @@ -0,0 +1,51 @@ +# -*- coding:utf-8 -*- + + +import datetime + +from extensions import db + + +class OperationRecord(db.Model): + __tablename__ = "records" + + record_id = db.Column(db.Integer, primary_key=True, autoincrement=True) + uid = db.Column(db.Integer, db.ForeignKey('users.uid'), nullable=False) + timestamp = db.Column(db.DateTime, + nullable=False, + default=datetime.datetime.now()) + origin = db.Column(db.String(32)) + ticket_id = db.Column(db.String(32)) + reason = db.Column(db.Text) + + +class CIAttributeHistory(db.Model): + __tablename__ = "histories" + + h_id = db.Column(db.Integer, primary_key=True, autoincrement=True) + operate_type = db.Column(db.String(6), db.Enum("add", "delete", "update", + name="operate_type")) + record_id = db.Column(db.Integer, + db.ForeignKey("records.record_id"), + nullable=False) + ci_id = db.Column(db.Integer, nullable=False) + attr_id = db.Column(db.Integer, nullable=False) + old = db.Column(db.Text) + new = db.Column(db.Text) + + +class CIRelationHistory(db.Model): + __tablename__ = "relation_histories" + + rh_id = db.Column(db.Integer, primary_key=True, autoincrement=True) + operate_type = db.Column(db.String(6), + db.Enum("add", "delete", name="operate_type")) + record_id = db.Column(db.Integer, + db.ForeignKey("records.record_id"), + nullable=False) + first_ci_id = db.Column(db.Integer) + second_ci_id = db.Column(db.Integer) + relation_type = db.Column( + db.String(8), db.Enum("connect", "deploy", "install", "contain", + name="relation_type")) + relation = db.Column(db.Integer, nullable=False) diff --git a/cmdb-api/models/statis.py b/cmdb-api/models/statis.py new file mode 100644 index 0000000..64e03a4 --- /dev/null +++ b/cmdb-api/models/statis.py @@ -0,0 +1,20 @@ +# -*- coding:utf-8 -*- + + +import datetime + +from extensions import db + + +class UrlRecord(db.Model): + + url_id = db.Column(db.Integer, primary_key=True, autoincrement=True) + + url = db.Column(db.String(64), nullable=False) + response_time = db.Column(db.Float, nullable=False) + is_ok = db.Column(db.Boolean, default=True) + source = db.Column(db.String(32)) + remote_addr = db.Column(db.String(20)) + hits = db.Column(db.Integer) + method = db.Column(db.String(5), default="GET") + created_at = db.Column(db.DateTime, default=datetime.datetime.now()) \ No newline at end of file diff --git a/cmdb-api/permissions.py b/cmdb-api/permissions.py new file mode 100644 index 0000000..891b952 --- /dev/null +++ b/cmdb-api/permissions.py @@ -0,0 +1,9 @@ +# -*- coding:utf-8 -*- + + +from flask.ext.principal import RoleNeed, Permission + + +admin = Permission(RoleNeed('admin')) +auth = Permission(RoleNeed('authenticated')) +null = Permission(RoleNeed('null')) \ No newline at end of file diff --git a/cmdb-api/requirements/default.txt b/cmdb-api/requirements/default.txt new file mode 100644 index 0000000..6ee0a59 --- /dev/null +++ b/cmdb-api/requirements/default.txt @@ -0,0 +1,14 @@ +Flask==0.9 +Flask-Script==0.5.2 +Flask-Babel==0.8 +Flask-principal==0.3.5 +Flask-mail==0.7.4 +pymysql==0.5 +sqlalchemy==0.8.2 +Flask-sqlalchemy==0.16 +Flask-cache==0.9.2 +redis==2.7.2 +gunicorn==0.17.4 +celery==3.0.18 +flask-celery=2.4.3 +Jinja2==2.7.1 \ No newline at end of file diff --git a/cmdb-api/settings.py b/cmdb-api/settings.py new file mode 100644 index 0000000..9819a83 --- /dev/null +++ b/cmdb-api/settings.py @@ -0,0 +1,7 @@ +# -*- coding:utf-8 -*- + +## CI cache +REDIS_HOST = "127.0.0.1" +REDIS_PORT = 6379 +REDIS_DB = 0 +REDIS_MAX_CONN = 30 diff --git a/cmdb-api/tasks/__init__.py b/cmdb-api/tasks/__init__.py new file mode 100644 index 0000000..44d37d3 --- /dev/null +++ b/cmdb-api/tasks/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- \ No newline at end of file diff --git a/cmdb-api/tasks/cmdb.py b/cmdb-api/tasks/cmdb.py new file mode 100644 index 0000000..11f5cbe --- /dev/null +++ b/cmdb-api/tasks/cmdb.py @@ -0,0 +1,30 @@ +# -*- coding:utf-8 -*- + + +import json +import time + +from flask import current_app + +from extensions import celery +from extensions import db +from lib.utils import rd +import lib.ci + + +@celery.task(name="cmdb.ci_cache", queue="cmdb_async") +def ci_cache(ci_id): + time.sleep(0.1) + db.session.close() + m = lib.ci.CIManager() + ci = m.get_ci_by_id(ci_id, need_children=False, use_master=True) + rd.delete(ci_id) + rd.add({ci_id: json.dumps(ci)}) + current_app.logger.info("%d caching.........." % ci_id) + + +@celery.task(name="cmdb.ci_delete", queue="cmdb_async") +def ci_delete(ci_id): + current_app.logger.info(ci_id) + rd.delete(ci_id) + current_app.logger.info("%d delete.........." % ci_id) diff --git a/cmdb-api/tasks/statis.py b/cmdb-api/tasks/statis.py new file mode 100644 index 0000000..0971117 --- /dev/null +++ b/cmdb-api/tasks/statis.py @@ -0,0 +1,21 @@ +# -*- coding:utf-8 -*- + + +import datetime + +from flask import current_app + +from extensions import celery +from extensions import db +from models.statis import UrlRecord + + +@celery.task(name="statis.url_record", queue="statis_async") +def url_record(url, method, remote_addr, response_time, status_code, source): + current_app.logger.info("%s add 1" % url) + now = datetime.datetime.now() + u = UrlRecord(url=url, response_time=response_time, is_ok=1, + source="default", hits=1, method=method, created_at=now, + remote_addr=remote_addr) + db.session.add(u) + db.session.commit() \ No newline at end of file diff --git a/cmdb-api/templates/search.xml b/cmdb-api/templates/search.xml new file mode 100644 index 0000000..e30f791 --- /dev/null +++ b/cmdb-api/templates/search.xml @@ -0,0 +1,27 @@ + + + {{ numfound }} + {{ page }} + + {% for ci in result %} + + {% for k, v in ci.items() %} + {% if not k.startswith('_') %} + {% for item in v | convert_to_list %} + {{ item }} + {% endfor %} + {% endif %} + {% endfor %} + + {% endfor %} + + + {% for k,v in facet.items() %} + + {% for item in v %} + {{ item[1] }} + {% endfor %} + + {% endfor %} + + \ No newline at end of file diff --git a/cmdb-api/templates/search_tidy.xml b/cmdb-api/templates/search_tidy.xml new file mode 100644 index 0000000..52832a0 --- /dev/null +++ b/cmdb-api/templates/search_tidy.xml @@ -0,0 +1,19 @@ + + + {{ code }} + + {% for k, v in ret.items() %} + + {% for ci in v %} + + + {% for item in ci|convert_to_list %} + {{ item }} + {% endfor %} + + + {% endfor %} + + {% endfor %} + + \ No newline at end of file