feat(api): Replace imp with importlib

feat(ui): update relative views menu display

.env

@@ -0,0 +1,6 @@
+ MYSQL_ROOT_PASSWORD='123456'
+ MYSQL_HOST='mysql'
+ MYSQL_PORT=3306
+ MYSQL_USER='cmdb'
+ MYSQL_DATABASE='cmdb'
+ MYSQL_PASSWORD='123456'

.gitattributes

@@ -0,0 +1 @@
+*.vue linguist-language=python

.github/ISSUE_TEMPLATE/1bug.yaml

@@ -0,0 +1,60 @@
+name: Bug Report
+description: File a bug report
+title: "[Bug]: "
+labels: ["☢️ bug"]
+assignees:
+  - Selina316
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report!
+  - type: input
+    id: contact
+    attributes:
+      label: Contact Details
+      description: How can we get in touch with you if we need more info?
+      placeholder: ex. email@example.com
+    validations:
+      required: false
+  - type: dropdown
+    id: aspects
+    attributes:
+      label: This bug is related to UI or API?
+      multiple: true
+      options:
+        - UI
+        - API
+  - type: textarea
+    id: happened
+    attributes:
+      label: What happened?
+      description: Also tell us, what did you expect to happen?
+      placeholder: Tell us what you see!
+      value: "A bug happened!"
+    validations:
+      required: true
+  - type: input
+    id: version
+    attributes:
+      label: Version
+      description: What version of our software are you running?
+      value: "newest"
+    validations:
+      required: true
+  - type: dropdown
+    id: browsers
+    attributes:
+      label: What browsers are you seeing the problem on?
+      multiple: true
+      options:
+        - Firefox
+        - Chrome
+        - Safari
+        - Microsoft Edge
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output
+      description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
+      render: shell

.github/ISSUE_TEMPLATE/2feature.yaml

@@ -0,0 +1,44 @@
+name: Feature wanted
+description: A new feature would be good
+title: "[Feature]: "
+labels: ["✏️ feature"]
+assignees:
+  - pycook
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thank you for your feature suggestion; we will evaluate it carefully!
+  - type: input
+    id: contact
+    attributes:
+      label: Contact Details
+      description: How can we get in touch with you if we need more info?
+      placeholder: ex. email@example.com
+    validations:
+      required: false
+  - type: dropdown
+    id: aspects
+    attributes:
+      label: feature is related to UI or API aspects?
+      multiple: true
+      options:
+        - UI
+        - API
+  - type: textarea
+    id: feature
+    attributes:
+      label: What is your advice?
+      description: Also tell us, what did you expect to happen?
+      placeholder: Tell us what you want!
+      value: "everyone wants this feature!"
+    validations:
+      required: true
+  - type: input
+    id: version
+    attributes:
+      label: Version
+      description: What version of our software are you running?
+      value: "newest"
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/3consultation.yaml

@@ -0,0 +1,36 @@
+name: Help wanted
+description: I have a question
+title: "[help wanted]: "
+labels: ["help wanted"]
+assignees:
+  - ivonGwy
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Please tell us what's you need!
+  - type: input
+    id: contact
+    attributes:
+      label: Contact Details
+      description: How can we get in touch with you if we need more info?
+      placeholder: ex. email@example.com
+    validations:
+      required: false
+  - type: textarea
+    id: question
+    attributes:
+      label: What is your question?
+      description: Also tell us, how can we help?
+      placeholder: Tell us what you need!
+      value: "i have a question!"
+    validations:
+      required: true
+  - type: input
+    id: version
+    attributes:
+      label: Version
+      description: What version of our software are you running?
+      value: "newest"
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/bug.yaml

@@ -0,0 +1,60 @@
+name: Bug Report
+description: File a bug report
+title: "[Bug]: "
+labels: ["bug"]
+assignees:
+  - pycook
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report!
+  - type: input
+    id: contact
+    attributes:
+      label: Contact Details
+      description: How can we get in touch with you if we need more info?
+      placeholder: ex. email@example.com
+    validations:
+      required: false
+  - type: dropdown
+      id: type
+      attributes:
+        label: bug is related to UI or API aspects?
+        multiple: true
+        options:
+          - UI
+          - API
+    - type: textarea
+      id: what-happened
+      attributes:
+        label: What happened?
+        description: Also tell us, what did you expect to happen?
+        placeholder: Tell us what you see!
+        value: "A bug happened!"
+      validations:
+        required: true
+    - type: textarea
+      id: version
+      attributes:
+        label: Version
+        description: What version of our software are you running?
+        default: 2.3.5
+      validations:
+        required: true
+    - type: dropdown
+      id: browsers
+      attributes:
+        label: What browsers are you seeing the problem on?
+        multiple: true
+        options:
+          - Firefox
+          - Chrome
+          - Safari
+          - Microsoft Edge
+    - type: textarea
+      id: logs
+      attributes:
+        label: Relevant log output
+        description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
+        render: shell

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,6 @@
+blank_issues_enabled: false
+contact_links:
+  - name: veops official website
+    url: https://veops.cn/#hero
+    about: you can contact us here.
+

.github/ISSUE_TEMPLATE/feature.yaml

@@ -0,0 +1,44 @@
+name: Feature wanted
+description: A new feature would be good
+title: "[Feature]: "
+labels: ["feature"]
+assignees:
+  - pycook
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thank you for your feature suggestion; we will evaluate it carefully!
+  - type: input
+    id: contact
+    attributes:
+      label: Contact Details
+      description: How can we get in touch with you if we need more info?
+      placeholder: ex. email@example.com
+    validations:
+      required: false
+  - type: dropdown
+      id: type
+      attributes:
+        label: feature is related to UI or API aspects?
+        multiple: true
+        options:
+          - UI
+          - API
+    - type: textarea
+      id: describe the feature
+      attributes:
+        label: What is your advice?
+        description: Also tell us, what did you expect to happen?
+        placeholder: Tell us what you want!
+        value: "everyone wants this feature!"
+      validations:
+        required: true
+    - type: textarea
+      id: version
+      attributes:
+        label: Version
+        description: What version of our software are you running?
+        default: 2.3.5
+      validations:
+        required: true

.github/workflows/docker-build-and-release.yaml

@@ -0,0 +1,79 @@
+name: docker-images-build-and-release
+
+on:
+  push:
+    branches:
+      - master
+    tags: ["v*"]
+#  pull_request:
+#    branches:
+#      - master
+
+env:
+  # Use docker.io for Docker Hub if empty
+  REGISTRY_SERVER_ADDRESS: ghcr.io/veops
+  TAG: ${{ github.sha }}
+
+jobs:
+  setup-environment:
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+  release-api-images:
+    runs-on: ubuntu-latest
+    needs: [setup-environment]
+    permissions:
+      contents: read
+      packages: write
+    timeout-minutes: 90
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+      - name: Login to GitHub Package Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push CMDB-API Docker image
+        uses: docker/build-push-action@v6
+        with:
+          file: docker/Dockerfile-API
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ env.REGISTRY_SERVER_ADDRESS }}/cmdb-api:${{ env.TAG }}
+#  release-ui-images:
+#    runs-on: ubuntu-latest
+#    needs: [setup-environment]
+#    permissions:
+#      contents: read
+#      packages: write
+#    timeout-minutes: 90
+#    steps:
+#      - name: Checkout Repo
+#        uses: actions/checkout@v4
+#      - name: Login to GitHub Package Registry
+#        uses: docker/login-action@v2
+#        with:
+#          registry: ghcr.io
+#          username: ${{ github.repository_owner }}
+#          password: ${{ secrets.GITHUB_TOKEN }}
+#      - name: Set up QEMU
+#        uses: docker/setup-qemu-action@v3
+#      - name: Set up Docker Buildx
+#        uses: docker/setup-buildx-action@v3
+#      - name: Build and push CMDB-UI Docker image
+#        uses: docker/build-push-action@v6
+#        with:
+#          file: docker/Dockerfile-UI
+#          context: .
+#          platforms: linux/amd64,linux/arm64
+#          push: true
+#          tags: ${{ env.REGISTRY_SERVER_ADDRESS }}/cmdb-ui:${{ env.TAG }}

.gitignore

@@ -39,9 +39,12 @@ pip-log.txt
 nosetests.xml
 .pytest_cache
 cmdb-api/test-output
+cmdb-api/api/uploaded_files
+cmdb-api/migrations/versions
 
 # Translations
-*.mo
+#*.mo
+messages.pot
 
 # Mr Developer
 .mr.developer.cfg
@@ -68,6 +71,7 @@ settings.py
 # UI
 cmdb-ui/node_modules
 cmdb-ui/dist
+cmdb-ui/yarn.lock
 
 # Log files
 cmdb-ui/npm-debug.log*

CODE_OF_CONDUCT.md

@@ -0,0 +1,13 @@
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free experience for everyone, regardless of the level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, age, or religion.
+
+Examples of unacceptable behavior by participants include the use of sexual language or imagery, derogatory comments or personal attacks, trolling, public or private harassment, insults, or other unprofessional conduct.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. Project maintainers who do not follow the Code of Conduct may be removed from the project team.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers.
+
+This Code of Conduct is adapted from the [Contributor Covenant](https://contributor-covenant.org), version 1.0.0, available at [http://contributor-covenant.org/version/1/0/0/](http://contributor-covenant.org/version/1/0/0/)

Dockerfile

@@ -1,48 +0,0 @@
-# ================================= UI ================================
-FROM node:16.0.0-alpine AS builder
-
-LABEL description="cmdb-ui"
-
-COPY cmdb-ui /data/apps/cmdb-ui
-
-WORKDIR /data/apps/cmdb-ui
-
-RUN sed -i "s#http://127.0.0.1:5000##g" .env && yarn install  && yarn build
-
-
-FROM nginx:alpine AS cmdb-ui
-
-RUN mkdir /etc/nginx/html && rm -f /etc/nginx/conf.d/default.conf
-
-COPY --from=builder /data/apps/cmdb-ui/dist /etc/nginx/html/
-
-
-# ================================= API ================================
-FROM python:3.8-alpine AS cmdb-api
-
-LABEL description="Python3.8,cmdb"
-
-COPY cmdb-api /data/apps/cmdb
-
-WORKDIR /data/apps/cmdb
-
-RUN apk add --no-cache tzdata gcc musl-dev libffi-dev openldap-dev python3-dev jpeg-dev zlib-dev build-base
-
-ENV TZ=Asia/Shanghai
-
-RUN pip install  --no-cache-dir -r requirements.txt \
-    && cp ./settings.example.py settings.py \
-    && sed -i "s#{user}:{password}@127.0.0.1:3306/{db}#cmdb:123456@mysql:3306/cmdb#g" settings.py \
-    && sed -i "s#redis://127.0.0.1#redis://redis#g" settings.py \
-    && sed -i 's#CACHE_REDIS_HOST = "127.0.0.1"#CACHE_REDIS_HOST = "redis"#g' settings.py
-
-ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.7.3/wait /wait
-RUN chmod +x /wait
-
-CMD ["bash", "-c", "flask run"]
-
-
-# ================================= Search ================================
-FROM docker.elastic.co/elasticsearch/elasticsearch:7.4.2 AS cmdb-search
-
-RUN yes | ./bin/elasticsearch-plugin install https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v7.4.2/elasticsearch-analysis-ik-7.4.2.zip

Makefile

@@ -1,37 +1,72 @@
-.PHONY: env clean api ui worker
+include ./Makefile.variable
 
-help:
+default: help
-	@echo "  env         create a development environment using pipenv"
+help:  ## display this help
-	@echo "  deps        install dependencies using pip"
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
-	@echo "  clean       remove unwanted files like .pyc's"
+.PHONY: help
-	@echo "  lint        check style with flake8"
-	@echo "  api         start api server"
-	@echo "  ui          start ui  server"
-	@echo "  worker      start async tasks worker"
 
-env:
+env: ## create a development environment using pipenv
 	sudo easy_install pip && \
-	pip install pipenv -i https://pypi.douban.com/simple && \
+	pip install pipenv -i https://repo.huaweicloud.com/repository/pypi/simple && \
 	npm install yarn && \
 	make deps
+.PHONY: env
 
-deps:
+docker-mysql: ## deploy MySQL use docker
+	@docker run --name mysql -p ${MYSQL_PORT}:3306 -e MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} -d mysql:latest
+.PHONY: docker-mysql
+
+docker-redis: ## deploy Redis use docker
+	@docker run --name redis -p ${REDIS_PORT}:6379 -d redis:latest
+.PHONY: docker-redis
+
+deps: ## install dependencies using pip
+	cd cmdb-api && \
 	pipenv install --dev && \
 	pipenv run flask db-setup && \
 	pipenv run flask cmdb-init-cache && \
+	cd .. && \
     cd cmdb-ui && yarn install && cd ..
+.PHONY: deps
 
-api:
+api: ## start api server
 	cd cmdb-api && pipenv run flask run -h 0.0.0.0
+.PHONY: api
 
-worker:
+worker: ## start async tasks worker
-	cd cmdb-api && pipenv run celery worker -A celery_worker.celery -E -Q one_cmdb_async --concurrency=1 -D && pipenv run celery worker -A celery_worker.celery -E -Q acl_async --concurrency=1 -D
+	cd cmdb-api && pipenv run celery -A celery_worker.celery worker -E -Q one_cmdb_async --autoscale=5,2 --logfile=one_cmdb_async.log -D && pipenv run celery -A celery_worker.celery worker -E -Q acl_async --autoscale=2,1 --logfile=one_acl_async.log -D
+.PHONY: worker
 
-ui:
+ui: ## start ui server
 	cd cmdb-ui && yarn run serve
+.PHONY: ui
 
-clean:
+clean: ## remove unwanted files like .pyc's
 	pipenv run flask clean
+.PHONY: clean
 
-lint:
+lint: ## check style with flake8
 	flake8 --exclude=env .
+.PHONY: lint
+
+api-docker-build:
+	export DOCKER_CLI_EXPERIMENTAL=enabled ;\
+	! ( docker buildx ls | grep multi-platform-builder ) && docker buildx create --use --platform=$(BUILD_ARCH) --name multi-platform-builder ;\
+	docker buildx build \
+    			--builder multi-platform-builder \
+    			--platform=$(BUILD_ARCH) \
+    			--tag $(REGISTRY)/cmdb-api:$(CMDB_DOCKER_VERSION)  \
+    			--tag $(REGISTRY)/cmdb-api:latest  \
+    			-f docker/Dockerfile-API \
+    			.
+
+ui-docker-build:
+	export DOCKER_CLI_EXPERIMENTAL=enabled ;\
+	! ( docker buildx ls | grep multi-platform-builder ) && docker buildx create --use --platform=$(BUILD_ARCH) --name multi-platform-builder ;\
+	docker buildx build \
+    			--builder multi-platform-builder \
+    			--platform=$(BUILD_ARCH) \
+    			--tag $(REGISTRY)/cmdb-ui:$(CMDB_DOCKER_VERSION)  \
+    			--tag $(REGISTRY)/cmdb-ui:latest  \
+    			-f docker/Dockerfile-UI \
+    			.

Makefile.variable

@@ -0,0 +1,21 @@
+SHELL := /bin/bash -o pipefail
+
+MYSQL_ROOT_PASSWORD ?= root
+MYSQL_PORT ?= 3306
+REDIS_PORT ?= 6379
+
+LATEST_TAG_DIFF:=$(shell git describe --tags --abbrev=8)
+LATEST_COMMIT:=$(VERSION)-dev-$(shell git rev-parse --short=8 HEAD)
+BUILD_ARCH ?= linux/amd64,linux/arm64
+
+# Set your version by env or using latest tags from git
+CMDB_VERSION?=$(LATEST_TAG_DIFF)
+ifeq ($(CMDB_VERSION),)
+ 	#fall back to last commit
+	CMDB_VERSION=$(LATEST_COMMIT)
+endif
+COMMIT_VERSION:=$(LATEST_COMMIT)
+CMDB_DOCKER_VERSION:=${CMDB_VERSION}
+CMDB_CHART_VERSION:=$(shell echo ${CMDB_VERSION} | sed  's/^v//g' )
+
+REGISTRY ?= local

README.md

@@ -1,80 +1,138 @@
-![基础资源视图](docs/logo.png)
 
-[![License](https://img.shields.io/badge/License-AGPLv3-brightgreen)](https://github.com/veops/cmdb/blob/master/LICENSE)
+<p align="center">
-[![UI](https://img.shields.io/badge/UI-Ant%20Design%20Pro%20Vue-brightgreen)](https://github.com/sendya/ant-design-pro-vue)
+  <a href="https://veops.cn">
-[![API](https://img.shields.io/badge/API-Flask-brightgreen)](https://github.com/pallets/flask)
+    <img src="https://github.com/user-attachments/assets/c5cfb272-899b-418d-9e69-8e1dd07db0f6" alt="维易CMDB"/>
+  </a>
+</p>
 
-[English](README_en.md) / [中文](README.md)
+<h4 align="center">简单、轻量、通用的运维配置管理数据库</h4>
 
-- 在线体验: <a href="https://cmdb.veops.cn" target="_blank">CMDB</a>
+<p align="center">
-  - username: demo
+  <a href="https://github.com/veops/cmdb/blob/master/LICENSE"><img src="https://img.shields.io/badge/License-AGPLv3-brightgreen" alt="License: GPLv3"></a>
-  - password: 123456
+  <a href="https://github.com/veops/cmdb/releases"><img alt="the latest release version" src="https://img.shields.io/github/v/release/veops/cmdb?color=75C1C4&include_prereleases&label=Release&logo=github&logoColor=white"></a>
-
+  <a href="https:https://github.com/sendya/ant-design-pro-vue"><img src="https://img.shields.io/badge/UI-Ant%20Design%20Pro%20Vue-green" alt="UI"></a>
-> **重要提示**: `master` 分支在开发过程中可能处于 _不稳定的状态_ 。
+  <a href="https://github.com/pallets/flask"><img src="https://img.shields.io/badge/API-Flask-bright" alt="API"></a>
-> 请通过[releases](https://github.com/veops/cmdb/releases)获取
+  <a href="https://github.com/veops/cmdb/stargazers"><img src="https://img.shields.io/github/stars/veops/cmdb" alt="Stars Badge"/></a>
+  <a href="https://github.com/veops/cmdb"><img src="https://img.shields.io/github/forks/veops/cmdb" alt="Forks Badge"/></a>
+</p>
+<p align="center">
+  中文(简体) · <a href="docs/README_en.md">English</a>
+</p>
 
 ## 系统介绍
 
-### 整体架构
+维易CMDB是一个简洁、轻量且高度可定制的运维配置管理数据库（CMDB）。它支持灵活的模型配置和资源自动发现，旨在为企业提供便捷的资产管理解决方案，帮助运维团队高效地管理 IT 基础设施和服务。
 
-<img src=docs/view.jpg />
+- 产品文档：[https://veops.cn/docs/](https://veops.cn/docs/)
-
+- 在线体验：[https://cmdb.veops.cn](https://cmdb.veops.cn)
-### 相关文档
+  - 用户名：demo 或者 admin
-
+  - 密码：123456
-- <a href="https://zhuanlan.zhihu.com/p/98453732" target="_blank">设计文档</a>
+- **重要提示**：`master` 分支在开发过程中可能处于**不稳定的状态**。请通过 [releases](https://github.com/veops/cmdb/releases) 获取最新稳定版本。
-- <a href="https://github.com/veops/cmdb/tree/master/docs/cmdb_api.md" target="_blank">API 文档</a>
-- <a href="https://mp.weixin.qq.com/s/EflmmJ-qdUkddTx2hRt3pA" target="_blank">树形视图实践</a>
-
-### 特点
-
-- 灵活性
-  1.  规范并统一纳管复杂数据资产
-  2.  自动发现、入库 IT 资产
-- 安全性
-  1. 细粒度访问控制
-  2. 完备操作日志
-- 多应用
-  1. 丰富视图展示维度
-  2. 提供 Restful API
-  3. 自定义字段触发器
 
 ### 主要功能
 
-- 模型属性支持索引、多值、默认排序、字体颜色，支持计算属性
+- **自定义模型和模型关系**：支持模型属性的自定义，包括下拉列表、字体颜色、计算属性等高级功能，满足不同业务需求。
-- 支持自动发现、定时巡检、文件导入
+- **自动发现资源**：支持计算机、网络设备、存储设备、数据库、中间件、公有云资源等自动发现。
-- 支持资源、树形、关系视图展示
+- **多维度视图展示**：包括资源视图、层级视图、关系视图等，帮助运维人员全面管理资源。
-- 支持模型间关系配置和展示
+- **细粒度权限控制**：通过精确的访问控制和完备的操作日志保障系统的安全性。
-- 细粒度访问控制，完备的操作日志
+- **全面的资源搜索功能**：支持灵活的资源和关系搜索，快速定位和操作资源。
-- 支持跨模型搜索
+- **集成 IP 地址管理（IPAM）和数据中心基础设施管理（DCIM）**：简化网络资源和数据中心设备的管理。
+
+更多详细功能，请移步 [维易科技官网](https://veops.cn) 进行了解。
+
+### 系统优势
+
+- 灵活性
+  + 无需指定固定运维场景，支持自由配置并内置多种模板
+  + 支持自动发现和入库 IT 资产，快速搭建资产管理系统
+- 安全性
+  + 细粒度的权限控制机制，确保资源管理的安全性
+  + 完整的操作日志记录，便于审计和问题追踪
+- 多应用
+  + 提供多种视图展示方式，满足不同场景的需求
+  + 强大的 API 接口，支持深度集成
+  + 支持定义属性触发器和计算属性，增强数据处理能力
+
+### 技术栈
+
++ 后端：Python [3.8-3.11]
++ 数据存储：MySQL、Redis
++ 前端：Vue.js
++ UI组件库：Ant Design Vue
 
 ### 系统概览
 
-- 服务树
+<table style="border-collapse: collapse;">
+  <tr>
+    <td style="padding: 5px;background-color:#fff;">
+      <img width="400" src="https://github.com/user-attachments/assets/6d2df835-ae93-4d91-9bd9-213c270eca7a"/>
+    </td>
+    <td style="padding: 5px;background-color:#fff;">
+      <img width="400" src="https://github.com/user-attachments/assets/cb8b598a-a1f9-4c74-adf1-6e59aea2c9b3"/>
+    </td>
+  </tr>
 
-![1](docs/0.png "首页展示")
+  <tr>
+    <td style="padding: 5px;background-color:#fff;">
+      <img width="400" src="https://github.com/user-attachments/assets/b440224f-53c3-4b7f-a9be-285d7a4b848f"/>
+    </td>
+    <td style="padding: 5px;background-color:#fff;">
+      <img width="400" src="https://github.com/user-attachments/assets/f457d5a0-b60b-4949-b94e-020f4c61444b"/>
+    </td>
+  </tr>
+</table>
 
-[查看更多展示](docs/screenshot.md)
+## 关注我们
 
+欢迎 Star 加关注，第一时间获取更新动态！
 
+![star us](https://github.com/user-attachments/assets/f9056d5a-171c-4f53-9fec-d40c9e5ff94d)
 
-### 更多功能
+## 快速开始
 
-> 也欢迎移步[维易科技官网](https://veops.cn)，发现更多免费运维系统。
+### 1. 搭建
+
++ 方案一：Docker 一键快速构建
+
+  - 第1步: 安装 Docker 环境和 Docker Compose（v2）
+  - 第2步: 拷贝项目代码, `git clone https://github.com/veops/cmdb.git`
+  - 第3步：进入主目录并启动, `docker compose up -d`
+
++ 方案二：[本地开发环境搭建](docs/local.md)
++ 方案三：[Makefile 安装](docs/makefile.md)
+
+### 2. 访问
+- 打开浏览器并访问: [http://127.0.0.1:8000](http://127.0.0.1:8000)
+- 用户名: demo 或者 admin
+- 密码: 123456
 
 ## 接入公司
 
-> 欢迎使用CMDB的公司，在 [#112](https://github.com/veops/cmdb/issues/112) 登记
++ 欢迎使用开源CMDB的公司和团队，在 [#112](https://github.com/veops/cmdb/issues/112) 登记
 
-## 安装
+## 代码贡献
+我们欢迎所有开发者贡献代码，改善和扩展这个项目。请先阅读我们的[贡献指南](docs/CONTRIBUTING.md)。此外，您还可以通过社交媒体、活动和分享来支持 Veops 的开源。
 
-### [Docker 一键快速构建](docs/docker.md)
+<a href="https://github.com/veops/cmdb/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=veops/cmdb" />
+</a>
 
-### [本地搭建](docs/local.md)
+## 更多开源
 
-### [Makefile 安装](docs/makefile.md)
+- [OneTerm](https://github.com/veops/oneterm): 一款简单、轻量、灵活的堡垒机服务。
+- [messenger](https://github.com/veops/messenger): 一个简单轻量的消息发送服务。
+- [ACL](https://github.com/veops/acl): 一个简单通用的权限管理系统设计与实践。
 
----
+## 相关文章
 
-_**欢迎关注我们的公众号，点击联系我们，加入微信、qq运维群，获得更多产品、行业相关资讯**_
+- <a href="https://mp.weixin.qq.com/s/v3eANth64UBW5xdyOkK3tg" target="_blank">尽可能通用的运维CMDB的设计与实践(Ⅰ) - 概览</a>
+- <a href="https://mp.weixin.qq.com/s/rQaf4AES7YJsyNQG_MKOLg" target="_blank">尽可能通用的运维CMDB的设计与实践(ⅠⅠ) - 自动发现</a>
+- <a href="https://github.com/veops/cmdb/tree/master/docs/cmdb_api.md" target="_blank">CMDB接口文档</a>
 
-![公众号](docs/qrcode_for_gzh.jpg)
+更多文章可以在公众号 **维易科技OneOps** 里查看
+
+## 与我联系
+
++ 邮箱: <a href="mailto:bd@veops.cn">bd@veops.cn</a>
++ 公众号：**维易科技OneOps**。关注后可以加入微信群，参与产品和技术交流  
+  <img src="docs/images/wechat.png" alt="公众号: 维易科技OneOps" />

README_en.md

@@ -1,82 +0,0 @@
-![基础资源视图](docs/logo.png)
-
-[![License](https://img.shields.io/badge/License-AGPLv3-brightgreen)](https://github.com/veops/cmdb/blob/master/LICENSE)
-[![UI](https://img.shields.io/badge/UI-Ant%20Design%20Pro%20Vue-brightgreen)](https://github.com/sendya/ant-design-pro-vue)
-[![API](https://img.shields.io/badge/API-Flask-brightgreen)](https://github.com/pallets/flask)
-
-[English](README_en.md) / [中文](README.md)
-
-## DEMO ONLINE
-
-- Preview online: <a href="https://cmdb.veops.cn" target="_blank">CMDB</a>
-  - username: demo
-  - password: 123456
-
-> **ATTENTION**: branch `master` may be unstable as the result of continued development, please pull code from [releases](https://github.com/veops/cmdb/releases)
-
-## Overview
-
-### Technical Architecture
-
-<img src=docs/view.png />
-
-### Document
-
-- <a href="https://zhuanlan.zhihu.com/p/98453732" target="_blank">Design Document</a>
-- <a href="https://github.com/veops/cmdb/tree/master/docs/cmdb_api.md" target="_blank">API Documentation</a>
-- <a href="https://mp.weixin.qq.com/s/EflmmJ-qdUkddTx2hRt3pA" target="_blank">Practice of Tree View</a>
-
-### Features
-
-- Flexibility
-  1. Standardize and manage complex data assets
-  2. Automatically discover and inventory IT assets
-- Security
-  1. Fine-grained access control
-  2. Comprehensive operation logs
-- Multi-application
-  1. Rich view display dimensions
-  2. Provide Restful API
-  3. Custom field triggers
-
-### Main Features
-
-- Model attributes support indexing, multiple values, default sorting, font color, and computed properties.
-- Support automatic discovery, scheduled inspections, and file import.
-- Support resource, tree view, and relationship view display.
-- Support configuration and display of relationships between models.
-- Fine-grained access control and comprehensive operation logs.
-- Support cross-model search.
-
-### System Overview
-
-- Service Tree
-  ![1](docs/0.png "首页展示")
-
-[View more screenshots](docs/screenshot.md)
-
-### More Features
-
-> Welcome to visit VeOps official website to discover more free operations and maintenance systems.
-
-## Installation
-
-### [One-Click Docker Quick Build](docs/docker_en.md)
-
-### [Local Setup](docs/local_en.md)
-
-### [Installation with Makefile](docs/makefile_en.md)
-
-## Contributing
-
-1. Fork it
-1. Create your feature branch (`git checkout -b my-feature`)
-1. Commit your changes (`git commit -am 'Add some feature'`)
-1. Push to the branch (`git push origin my-feature`)
-1. Create new Pull Request
-
----
-
-_**Welcome to join us through QQ group（336164978）**_
-
-![QQgroup](docs/qr_code.jpg)

cmdb-api/.ruff.toml

@@ -0,0 +1,79 @@
+line-length = 120
+cache-dir = ".ruff_cache"
+target-version = "py310"
+unsafe-fixes = true
+show-fixes = true
+
+[lint]
+select = [
+    "E",
+    "F",
+    "I",
+    "TCH",
+    # W
+    "W505",
+    # PT
+    "PT018",
+    # SIM
+    "SIM101",
+    "SIM114",
+    # PGH
+    "PGH004",
+    # PL
+    "PLE1142",
+    # RUF
+    "RUF100",
+    # UP
+    "UP007"
+]
+preview = true
+ignore = ["FURB101"]
+
+[lint.flake8-pytest-style]
+mark-parentheses = false
+parametrize-names-type = "list"
+parametrize-values-row-type = "list"
+parametrize-values-type = "tuple"
+
+[lint.flake8-unused-arguments]
+ignore-variadic-names = true
+
+[lint.isort]
+lines-between-types = 1
+order-by-type = true
+
+[lint.per-file-ignores]
+"**/api/v1/*.py" = ["TCH"]
+"**/model/*.py" = ["TCH003"]
+"**/models/__init__.py" = ["F401", "F403"]
+"**/tests/*.py" = ["E402"]
+"celery_worker.py" = ["F401"]
+"api/views/entry.py" = ["I001"]
+"migrations/*.py" = ["I001", "E402"]
+"*.py" = ["I001"]
+"api/views/common_setting/department.py" = ["F841"]
+"api/lib/common_setting/upload_file.py" = ["F841"]
+"api/lib/common_setting/acl.py" = ["F841"]
+"**/__init__.py" = ["F822"]
+"api/tasks/*.py" = ["E722"]
+"api/views/cmdb/*.py" = ["E722"]
+"api/views/acl/*.py" = ["E722"]
+"api/lib/secrets/*.py" = ["E722", "F841"]
+"api/lib/utils.py" = ["E722", "E731"]
+"api/lib/perm/authentication/cas/*" = ["E113", "F841"]
+"api/lib/perm/acl/*" = ["E722"]
+"api/lib/*" = ["E721", "F722"]
+"api/lib/cmdb/*" = ["F722", "E722"]
+"api/lib/cmdb/search/ci/es/search.py" = ["F841", "SIM114"]
+"api/lib/cmdb/search/ci/db/search.py" = ["F841"]
+"api/lib/cmdb/value.py" = ["F841"]
+"api/lib/cmdb/history.py" = ["E501"]
+"api/commands/common.py" = ["E722"]
+"api/commands/click_cmdb.py" = ["E722"]
+"api/lib/perm/auth.py" = ["SIM114"]
+
+[format]
+preview = true
+quote-style = "single"
+docstring-code-format = true
+skip-magic-trailing-comma = false

cmdb-api/Makefile

@@ -1,16 +0,0 @@
-default: help
-
-test: ## test in local environment
-	pytest -s  --html=test-output/test/index.html --cov-report html:test-output/coverage --cov=api tests
-
-clean_test: ## clean test output
-	rm -f .coverage
-	rm -rf .pytest_cache
-	rm -rf test-output
-
-
-docker_test: ## test all case in docker container
-	@echo "TODO"
-
-help:
-	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' ./Makefile | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'

cmdb-api/Pipfile

@@ -5,60 +5,71 @@ name = "pypi"
 
 [packages]
 # Flask
-Flask = "==1.0.3"
+Flask = "==2.2.5"
-Werkzeug = "==0.15.5"
+Werkzeug = "==2.2.3"
 click = ">=5.0"
 # Api
-Flask-RESTful = "==0.3.7"
+Flask-RESTful = "==0.3.10"
 # Database
-Flask-SQLAlchemy = "==2.4.0"
+Flask-SQLAlchemy = "==3.0.5"
-SQLAlchemy = "==1.3.5"
+SQLAlchemy = "==1.4.49"
-PyMySQL = "==0.9.3"
+PyMySQL = "==1.1.0"
-redis = "==3.2.1"
+redis = "==4.6.0"
+python-redis-lock = "==4.0.0"
 # Migrations
 Flask-Migrate = "==2.5.2"
 # Deployment
-gunicorn = "==19.5.0"
+gunicorn = "==21.0.1"
 supervisor = "==4.0.3"
 # Auth
-Flask-Login = "==0.4.1"
+Flask-Login = ">=0.6.2"
-Flask-Bcrypt = "==0.7.1"
+Flask-Bcrypt = "==1.0.1"
 Flask-Cors = ">=3.0.8"
-python-ldap = "==3.2.0"
+ldap3 = "==2.9.1"
 pycryptodome = "==3.12.0"
+cryptography = ">=41.0.2"
+# i18n
+flask-babel = "==4.0.0"
 # Caching
 Flask-Caching = ">=1.0.0"
 # Environment variable parsing
 environs = "==4.2.0"
 marshmallow = "==2.20.2"
 # async tasks
-celery = "==4.3.0"
+celery = "==5.3.1"
 celery_once = "==3.0.1"
 more-itertools = "==5.0.0"
-kombu = "==4.4.0"
+kombu = ">=5.3.1"
 # common setting
-Flask-APScheduler = "==1.12.4"
 timeout-decorator = "==0.5.0"
-numpy = "==1.18.5"
-pandas = "==1.3.2"
 WTForms = "==3.0.0"
 email-validator = "==1.3.1"
 treelib = "==1.6.1"
 flasgger = "==0.9.5"
-Pillow = "==8.3.2"
+Pillow = ">=10.0.1"
 # other
-six = "==1.12.0"
+six = "==1.16.0"
 bs4 = ">=0.0.1"
 toposort = ">=1.5"
 requests = ">=2.22.0"
+requests_oauthlib = "==1.3.1"
+markdownify = "==0.11.6"
 PyJWT = "==2.4.0"
 elasticsearch = "==7.17.9"
-future = "==0.18.2"
+future = "==0.18.3"
-itsdangerous = "==2.0.1"
+itsdangerous = "==2.1.2"
-Jinja2 = "==3.0.1"
+Jinja2 = "==3.1.2"
 jinja2schema = "==0.1.4"
 msgpack-python = "==0.5.6"
 alembic = "==1.7.7"
+hvac = "==2.0.0"
+colorama = ">=0.4.6"
+pycryptodomex = ">=3.19.0"
+lz4 = ">=4.3.2"
+python-magic = "==0.4.27"
+jsonpath = "==0.82.2"
+networkx = ">=3.1"
+ipaddress = ">=1.0.23"
 
 [dev-packages]
 # Testing
@@ -75,4 +86,3 @@ flake8-isort = "==2.7.0"
 isort = "==4.3.21"
 pep8-naming = "==0.8.2"
 pydocstyle = "==3.0.0"
-

cmdb-api/api/app.py

@@ -7,31 +7,28 @@ import os
 import sys
 from inspect import getmembers
 from logging.handlers import RotatingFileHandler
+from pathlib import Path
 
 from flask import Flask
-from flask import make_response, jsonify
+from flask import jsonify
+from flask import make_response
+from flask import request
 from flask.blueprints import Blueprint
 from flask.cli import click
-from flask.json import JSONEncoder
+from flask.json.provider import DefaultJSONProvider
+from flask_babel.speaklater import LazyString
 
 import api.views.entry
-from api.extensions import (
+from api.extensions import (bcrypt, babel, cache, celery, cors, db, es, login_manager, migrate, rd)
-    bcrypt,
+from api.extensions import inner_secrets
-    cors,
+from api.lib.perm.authentication.cas import CAS
-    cache,
+from api.lib.perm.authentication.oauth2 import OAuth2
-    db,
+from api.lib.secrets.secrets import InnerKVManger
-    login_manager,
-    migrate,
-    celery,
-    rd,
-    es,
-)
-from api.flask_cas import CAS
 from api.models.acl import User
 
 HERE = os.path.abspath(os.path.dirname(__file__))
 PROJECT_ROOT = os.path.join(HERE, os.pardir)
-API_PACKAGE = "api"
+BASE_DIR = Path(__file__).resolve().parent.parent
 
 
 @login_manager.user_loader
@@ -75,9 +72,9 @@ class ReverseProxy(object):
         return self.app(environ, start_response)
 
 
-class MyJSONEncoder(JSONEncoder):
+class MyJSONEncoder(DefaultJSONProvider):
     def default(self, o):
-        if isinstance(o, (decimal.Decimal, datetime.date, datetime.time)):
+        if isinstance(o, (decimal.Decimal, datetime.date, datetime.time, LazyString)):
             return str(o)
 
         if isinstance(o, datetime.datetime):
@@ -86,15 +83,6 @@ class MyJSONEncoder(JSONEncoder):
         return o
 
 
-def create_acl_app(config_object="settings"):
-    app = Flask(__name__.split(".")[0])
-    app.config.from_object(config_object)
-
-    register_extensions(app)
-
-    return app
-
-
 def create_app(config_object="settings"):
     """Create application factory, as explained here: http://flask.pocoo.org/docs/patterns/appfactories/.
 
@@ -103,7 +91,7 @@ def create_app(config_object="settings"):
     app = Flask(__name__.split(".")[0])
 
     app.config.from_object(config_object)
-    app.json_encoder = MyJSONEncoder
+    app.json = MyJSONEncoder(app)
     configure_logger(app)
     register_extensions(app)
     register_blueprints(app)
@@ -111,6 +99,7 @@ def create_app(config_object="settings"):
     register_shell_context(app)
     register_commands(app)
     CAS(app)
+    OAuth2(app)
     app.wsgi_app = ReverseProxy(app.wsgi_app)
     configure_upload_dir(app)
 
@@ -130,17 +119,29 @@ def configure_upload_dir(app):
 
 def register_extensions(app):
     """Register Flask extensions."""
+
+    def get_locale():
+        accept_languages = app.config.get('ACCEPT_LANGUAGES', ['en', 'zh'])
+        return request.accept_languages.best_match(accept_languages)
+
     bcrypt.init_app(app)
+    babel.init_app(app, locale_selector=get_locale)
     cache.init_app(app)
     db.init_app(app)
     cors.init_app(app)
     login_manager.init_app(app)
-    migrate.init_app(app, db)
+    migrate.init_app(app, db, directory=f"{BASE_DIR}/migrations")
     rd.init_app(app)
     if app.config.get('USE_ES'):
         es.init_app(app)
+
+    app.config.update(app.config.get("CELERY"))
     celery.conf.update(app.config)
 
+    if app.config.get('SECRETS_ENGINE') == 'inner':
+        with app.app_context():
+            inner_secrets.init_app(app, InnerKVManger())
+
 
 def register_blueprints(app):
     for item in getmembers(api.views.entry):
@@ -158,10 +159,8 @@ def register_error_handlers(app):
         error_code = getattr(error, "code", 500)
         if not str(error_code).isdigit():
             error_code = 400
-        if error_code != 500:
+
-            return make_response(jsonify(message=str(error)), error_code)
+        return make_response(jsonify(message=str(error)), error_code)
-        else:
-            return make_response(jsonify(message=traceback.format_exc(-1)), error_code)
 
     for errcode in app.config.get("ERROR_CODES") or [400, 401, 403, 404, 405, 500, 502]:
         app.errorhandler(errcode)(render_error)
@@ -184,9 +183,8 @@ def register_commands(app):
     for root, _, files in os.walk(os.path.join(HERE, "commands")):
         for filename in files:
             if not filename.startswith("_") and filename.endswith("py"):
-                module_path = os.path.join(API_PACKAGE, root[root.index("commands"):])
+                if root not in sys.path:
-                if module_path not in sys.path:
+                    sys.path.insert(1, root)
-                    sys.path.insert(1, module_path)
                 command = __import__(os.path.splitext(filename)[0])
                 func_list = [o[0] for o in getmembers(command) if isinstance(o[1], click.core.Command)]
                 for func_name in func_list:
@@ -204,10 +202,11 @@ def configure_logger(app):
         app.logger.addHandler(handler)
 
     log_file = app.config['LOG_PATH']
-    file_handler = RotatingFileHandler(log_file,
+    if log_file and log_file != "/dev/stdout":
-                                       maxBytes=2 ** 30,
+        file_handler = RotatingFileHandler(log_file,
-                                       backupCount=7)
+                                           maxBytes=2 ** 30,
-    file_handler.setLevel(getattr(logging, app.config['LOG_LEVEL']))
+                                           backupCount=7)
-    file_handler.setFormatter(formatter)
+        file_handler.setLevel(getattr(logging, app.config['LOG_LEVEL']))
-    app.logger.addHandler(file_handler)
+        file_handler.setFormatter(formatter)
+        app.logger.addHandler(file_handler)
     app.logger.setLevel(getattr(logging, app.config['LOG_LEVEL']))

cmdb-api/api/commands/click_acl.py

@@ -1,10 +1,15 @@
 import click
 from flask.cli import with_appcontext
 
+from api.lib.perm.acl.user import UserCRUD
+
 
 @click.command()
 @with_appcontext
 def init_acl():
+    """
+    acl init
+    """
     from api.models.acl import Role
     from api.models.acl import App
     from api.tasks.acl import role_rebuild
@@ -20,50 +25,32 @@ def init_acl():
                 role_rebuild.apply_async(args=(role.id, app.id), queue=ACL_QUEUE)
 
 
-# @click.command()
+@click.command()
-# @with_appcontext
+@with_appcontext
-# def acl_clean():
+def add_user():
-#     from api.models.acl import Resource
+    """
-#     from api.models.acl import Permission
+    create a user
-#     from api.models.acl import RolePermission
+
-#
+    is_admin: default is False
-#     perms = RolePermission.get_by(to_dict=False)
+
-#
+    """
-#     for r in perms:
+
-#         perm = Permission.get_by_id(r.perm_id)
+    from api.models.acl import App
-#         if perm and perm.app_id != r.app_id:
+    from api.lib.perm.acl.cache import AppCache
-#             resource_id = r.resource_id
+    from api.lib.perm.acl.cache import RoleCache
-#             resource = Resource.get_by_id(resource_id)
+    from api.lib.perm.acl.role import RoleCRUD
-#             perm_name = perm.name
+    from api.lib.perm.acl.role import RoleRelationCRUD
-#             existed = Permission.get_by(resource_type_id=resource.resource_type_id, name=perm_name, first=True,
+
-#                                         to_dict=False)
+    username = click.prompt('Enter username', confirmation_prompt=False)
-#             if existed is not None:
+    password = click.prompt('Enter password', hide_input=True, confirmation_prompt=True)
-#                 other = RolePermission.get_by(rid=r.rid, perm_id=existed.id, resource_id=resource_id)
+    email = click.prompt('Enter email   ', confirmation_prompt=False)
-#                 if not other:
+    is_admin = click.prompt('Admin (Y/N)   ', confirmation_prompt=False, type=bool, default=False)
-#                     r.update(perm_id=existed.id)
+
-#                 else:
+    UserCRUD.add(username=username, password=password, email=email)
-#                     r.soft_delete()
+
-#             else:
+    if is_admin:
-#                 r.soft_delete()
+        app = AppCache.get('acl') or App.create(name='acl')
-#
+        acl_admin = RoleCache.get_by_name(app.id, 'acl_admin') or RoleCRUD.add_role('acl_admin', app.id, True)
-#
+        rid = RoleCache.get_by_name(None, username).id
-# @click.command()
+
-# @with_appcontext
+        RoleRelationCRUD.add(acl_admin, acl_admin.id, [rid], app.id)
-# def acl_has_resource_role():
-#     from api.models.acl import Role
-#     from api.models.acl import App
-#     from api.lib.perm.acl.cache import HasResourceRoleCache
-#     from api.lib.perm.acl.role import RoleCRUD
-#
-#     roles = Role.get_by(to_dict=False)
-#     apps = App.get_by(to_dict=False)
-#     for role in roles:
-#         if role.app_id:
-#             res = RoleCRUD.recursive_resources(role.id, role.app_id)
-#             if res.get('resources') or res.get('groups'):
-#                 HasResourceRoleCache.add(role.id, role.app_id)
-#         else:
-#             for app in apps:
-#                 res = RoleCRUD.recursive_resources(role.id, app.id)
-#                 if res.get('resources') or res.get('groups'):
-#                     HasResourceRoleCache.add(role.id, app.id)

cmdb-api/api/commands/click_cmdb.py

@@ -1,39 +1,50 @@
 # -*- coding:utf-8 -*-
 
 
+import click
 import copy
 import datetime
 import json
+import requests
 import time
-
+import uuid
-import click
 from flask import current_app
 from flask.cli import with_appcontext
+from flask_login import login_user
 
 import api.lib.cmdb.ci
 from api.extensions import db
 from api.extensions import rd
-from api.lib.cmdb.ci_type import CITypeTriggerManager
+from api.lib.cmdb.cache import AttributeCache
 from api.lib.cmdb.const import PermEnum
 from api.lib.cmdb.const import REDIS_PREFIX_CI
 from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
+from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION2
 from api.lib.cmdb.const import ResourceTypeEnum
 from api.lib.cmdb.const import RoleEnum
 from api.lib.cmdb.const import ValueTypeEnum
+from api.lib.cmdb.dcim.rack import RackManager
 from api.lib.exception import AbortException
 from api.lib.perm.acl.acl import ACLManager
+from api.lib.perm.acl.acl import UserCache
 from api.lib.perm.acl.cache import AppCache
 from api.lib.perm.acl.resource import ResourceCRUD
 from api.lib.perm.acl.resource import ResourceTypeCRUD
 from api.lib.perm.acl.role import RoleCRUD
-from api.lib.perm.acl.user import UserCRUD
+from api.lib.secrets.inner import KeyManage
+from api.lib.secrets.inner import global_key_threshold
+from api.lib.secrets.secrets import InnerKVManger
 from api.models.acl import App
 from api.models.acl import ResourceType
+from api.models.cmdb import Attribute
+from api.models.cmdb import AttributeHistory
 from api.models.cmdb import CI
 from api.models.cmdb import CIRelation
 from api.models.cmdb import CIType
 from api.models.cmdb import CITypeTrigger
+from api.models.cmdb import OperationRecord
 from api.models.cmdb import PreferenceRelationView
+from api.tasks.cmdb import batch_ci_cache
 
 
 @click.command()
@@ -43,13 +54,22 @@ def cmdb_init_cache():
 
     ci_relations = CIRelation.get_by(to_dict=False)
     relations = dict()
+    relations2 = dict()
     for cr in ci_relations:
         relations.setdefault(cr.first_ci_id, {}).update({cr.second_ci_id: cr.second_ci.type_id})
+        if cr.ancestor_ids:
+            relations2.setdefault('{},{}'.format(cr.ancestor_ids, cr.first_ci_id), {}).update(
+                {cr.second_ci_id: cr.second_ci.type_id})
     for i in relations:
         relations[i] = json.dumps(relations[i])
+    for i in relations2:
+        relations2[i] = json.dumps(relations2[i])
     if relations:
         rd.create_or_update(relations, REDIS_PREFIX_CI_RELATION)
+    if relations2:
+        rd.create_or_update(relations2, REDIS_PREFIX_CI_RELATION2)
 
+    es = None
     if current_app.config.get("USE_ES"):
         from api.extensions import es
         from api.models.cmdb import Attribute
@@ -101,10 +121,20 @@ def cmdb_init_acl():
     _app = AppCache.get('cmdb') or App.create(name='cmdb')
     app_id = _app.id
 
+    current_app.test_request_context().push()
+
     # 1. add resource type
     for resource_type in ResourceTypeEnum.all():
         try:
-            ResourceTypeCRUD.add(app_id, resource_type, '', PermEnum.all())
+            perms = PermEnum.all()
+            if resource_type in (ResourceTypeEnum.CI_FILTER, ResourceTypeEnum.PAGE):
+                perms = [PermEnum.READ]
+            elif resource_type == ResourceTypeEnum.CI_TYPE_RELATION:
+                perms = [PermEnum.ADD, PermEnum.DELETE, PermEnum.GRANT]
+            elif resource_type in (ResourceTypeEnum.RELATION_VIEW, ResourceTypeEnum.TOPOLOGY_VIEW):
+                perms = [PermEnum.READ, PermEnum.UPDATE, PermEnum.DELETE, PermEnum.GRANT]
+
+            ResourceTypeCRUD.add(app_id, resource_type, '', perms)
         except AbortException:
             pass
 
@@ -120,10 +150,10 @@ def cmdb_init_acl():
 
     # 3. add resource and grant
     ci_types = CIType.get_by(to_dict=False)
-    type_id = ResourceType.get_by(name=ResourceTypeEnum.CI, first=True, to_dict=False).id
+    resource_type_id = ResourceType.get_by(name=ResourceTypeEnum.CI, first=True, to_dict=False).id
     for ci_type in ci_types:
         try:
-            ResourceCRUD.add(ci_type.name, type_id, app_id)
+            ResourceCRUD.add(ci_type.name, resource_type_id, app_id)
         except AbortException:
             pass
 
@@ -133,10 +163,10 @@ def cmdb_init_acl():
                                             [PermEnum.READ])
 
     relation_views = PreferenceRelationView.get_by(to_dict=False)
-    type_id = ResourceType.get_by(name=ResourceTypeEnum.RELATION_VIEW, first=True, to_dict=False).id
+    resource_type_id = ResourceType.get_by(name=ResourceTypeEnum.RELATION_VIEW, first=True, to_dict=False).id
     for view in relation_views:
         try:
-            ResourceCRUD.add(view.name, type_id, app_id)
+            ResourceCRUD.add(view.name, resource_type_id, app_id)
         except AbortException:
             pass
 
@@ -146,67 +176,43 @@ def cmdb_init_acl():
                                             [PermEnum.READ])
 
 
-@click.command()
-@click.option(
-    '-u',
-    '--user',
-    help='username'
-)
-@click.option(
-    '-p',
-    '--password',
-    help='password'
-)
-@click.option(
-    '-m',
-    '--mail',
-    help='mail'
-)
-@with_appcontext
-def add_user(user, password, mail):
-    """
-    create a user
-
-    is_admin: default is False
-
-    Example:  flask add-user -u <username> -p <password> -m <mail>
-    """
-    assert user is not None
-    assert password is not None
-    assert mail is not None
-    UserCRUD.add(username=user, password=password, email=mail)
-
-
-@click.command()
-@click.option(
-    '-u',
-    '--user',
-    help='username'
-)
-@with_appcontext
-def del_user(user):
-    """
-    delete a user
-
-    Example:  flask del-user -u <username>
-    """
-    assert user is not None
-    from api.models.acl import User
-
-    u = User.get_by(username=user, first=True, to_dict=False)
-    u and UserCRUD.delete(u.uid)
-
-
 @click.command()
 @with_appcontext
 def cmdb_counter():
+    """
+    Dashboard calculations
+    """
     from api.lib.cmdb.cache import CMDBCounterCache
 
+    current_app.test_request_context().push()
+    if not UserCache.get('worker'):
+        from api.lib.perm.acl.user import UserCRUD
+
+        UserCRUD.add(username='worker', password=uuid.uuid4().hex, email='worker@xxx.com')
+
+    login_user(UserCache.get('worker'))
+
+    i = 0
+    today = datetime.date.today()
     while True:
         try:
-            db.session.remove()
+            db.session.commit()
 
             CMDBCounterCache.reset()
+
+            if i % 5 == 0:
+                CMDBCounterCache.flush_adc_counter()
+                i = 0
+
+            if datetime.date.today() != today:
+                CMDBCounterCache.clear_ad_exec_history()
+                today = datetime.date.today()
+
+            CMDBCounterCache.flush_sub_counter()
+
+            RackManager().check_u_slot()
+
+            i += 1
         except:
             import traceback
             print(traceback.format_exc())
@@ -217,45 +223,365 @@ def cmdb_counter():
 @click.command()
 @with_appcontext
 def cmdb_trigger():
+    """
+    Trigger execution for date attribute
+    """
+    from api.lib.cmdb.ci import CITriggerManager
+
+    current_app.test_request_context().push()
+    if not UserCache.get('worker'):
+        from api.lib.perm.acl.user import UserCRUD
+        UserCRUD.add(username='worker', password=uuid.uuid4().hex, email='worker@xxx.com')
+    login_user(UserCache.get('worker'))
+
     current_day = datetime.datetime.today().strftime("%Y-%m-%d")
     trigger2cis = dict()
     trigger2completed = dict()
 
     i = 0
     while True:
-        db.session.remove()
+        try:
-        if datetime.datetime.today().strftime("%Y-%m-%d") != current_day:
+            db.session.remove()
-            trigger2cis = dict()
-            trigger2completed = dict()
-            current_day = datetime.datetime.today().strftime("%Y-%m-%d")
 
-        if i == 360 or i == 0:
+            if datetime.datetime.today().strftime("%Y-%m-%d") != current_day:
-            i = 0
+                trigger2cis = dict()
-            try:
+                trigger2completed = dict()
-                triggers = CITypeTrigger.get_by(to_dict=False)
+                current_day = datetime.datetime.today().strftime("%Y-%m-%d")
 
+            if i == 3 or i == 0:
+                i = 0
+                triggers = CITypeTrigger.get_by(to_dict=False, __func_isnot__key_attr_id=None)
                 for trigger in triggers:
-                    ready_cis = CITypeTriggerManager.waiting_cis(trigger)
+                    try:
+                        ready_cis = CITriggerManager.waiting_cis(trigger)
+                    except Exception as e:
+                        print(e)
+                        continue
+
                     if trigger.id not in trigger2cis:
                         trigger2cis[trigger.id] = (trigger, ready_cis)
                     else:
                         cur = trigger2cis[trigger.id]
-                        cur_ci_ids = {i.ci_id for i in cur[1]}
+                        cur_ci_ids = {_ci.ci_id for _ci in cur[1]}
-                        trigger2cis[trigger.id] = (trigger, cur[1] + [i for i in ready_cis if i.ci_id not in cur_ci_ids
+                        trigger2cis[trigger.id] = (
-                                                                      and i.ci_id not in trigger2completed[trigger.id]])
+                            trigger, cur[1] + [_ci for _ci in ready_cis if _ci.ci_id not in cur_ci_ids
+                                               and _ci.ci_id not in trigger2completed.get(trigger.id, {})])
 
-            except Exception as e:
+            for tid in trigger2cis:
-                print(e)
+                trigger, cis = trigger2cis[tid]
+                for ci in copy.deepcopy(cis):
+                    if CITriggerManager.trigger_notify(trigger, ci):
+                        trigger2completed.setdefault(trigger.id, set()).add(ci.ci_id)
 
-        for tid in trigger2cis:
+                        for _ci in cis:
-            trigger, cis = trigger2cis[tid]
+                            if _ci.ci_id == ci.ci_id:
-            for ci in copy.deepcopy(cis):
+                                cis.remove(_ci)
-                if CITypeTriggerManager.trigger_notify(trigger, ci):
-                    trigger2completed.setdefault(trigger.id, set()).add(ci.ci_id)
 
-                    for _ci in cis:
+            i += 1
-                        if _ci.ci_id == ci.ci_id:
+            time.sleep(10)
-                            cis.remove(_ci)
+        except Exception as e:
+            import traceback
+            print(traceback.format_exc())
+            current_app.logger.error("cmdb trigger exception: {}".format(e))
+            time.sleep(60)
 
-        i += 1
+
-        time.sleep(10)
+@click.command()
+@with_appcontext
+def cmdb_index_table_upgrade():
+    """
+    Migrate data from tables c_value_integers, c_value_floats, and c_value_datetime
+    """
+    for attr in Attribute.get_by(to_dict=False):
+        if attr.value_type not in {ValueTypeEnum.TEXT, ValueTypeEnum.JSON} and not attr.is_index:
+            attr.update(is_index=True)
+            AttributeCache.clean(attr)
+
+    from api.models.cmdb import CIValueInteger, CIIndexValueInteger
+    from api.models.cmdb import CIValueFloat, CIIndexValueFloat
+    from api.models.cmdb import CIValueDateTime, CIIndexValueDateTime
+
+    for i in CIValueInteger.get_by(to_dict=False):
+        CIIndexValueInteger.create(ci_id=i.ci_id, attr_id=i.attr_id, value=i.value, commit=False)
+        i.delete(commit=False)
+    db.session.commit()
+
+    for i in CIValueFloat.get_by(to_dict=False):
+        CIIndexValueFloat.create(ci_id=i.ci_id, attr_id=i.attr_id, value=i.value, commit=False)
+        i.delete(commit=False)
+    db.session.commit()
+
+    for i in CIValueDateTime.get_by(to_dict=False):
+        CIIndexValueDateTime.create(ci_id=i.ci_id, attr_id=i.attr_id, value=i.value, commit=False)
+        i.delete(commit=False)
+    db.session.commit()
+
+
+def valid_address(address):
+    if not address:
+        return False
+
+    if not address.startswith(("http://127.0.0.1", "https://127.0.0.1")):
+        response = {
+            "message": "Address should start with http://127.0.0.1 or https://127.0.0.1",
+            "status": "failed"
+        }
+        KeyManage.print_response(response)
+        return False
+    return True
+
+
+@click.command()
+@click.option(
+    '-a',
+    '--address',
+    help='inner cmdb api, http://127.0.0.1:8000',
+)
+@with_appcontext
+def cmdb_inner_secrets_init(address):
+    """
+    init inner secrets for password feature
+    """
+    res, ok = KeyManage(backend=InnerKVManger()).init()
+    if not ok:
+        if res.get("status") == "failed":
+            KeyManage.print_response(res)
+            return
+
+    token = res.get("details", {}).get("root_token", "")
+    if valid_address(address):
+        token = current_app.config.get("INNER_TRIGGER_TOKEN", "") if not token else token
+        if not token:
+            token = click.prompt('Enter root token', hide_input=True, confirmation_prompt=False)
+        assert token is not None
+        resp = requests.post("{}/api/v0.1/secrets/auto_seal".format(address.strip("/")),
+                             headers={"Inner-Token": token})
+        if resp.status_code == 200:
+            KeyManage.print_response(resp.json())
+        else:
+            KeyManage.print_response({"message": resp.text or resp.status_code, "status": "failed"})
+    else:
+        KeyManage.print_response(res)
+
+
+@click.command()
+@click.option(
+    '-a',
+    '--address',
+    help='inner cmdb api, http://127.0.0.1:8000',
+    required=True,
+)
+@with_appcontext
+def cmdb_inner_secrets_unseal(address):
+    """
+    unseal the secrets feature
+    """
+    # if not valid_address(address):
+    #     return
+    address = "{}/api/v0.1/secrets/unseal".format(address.strip("/"))
+    for i in range(global_key_threshold):
+        token = click.prompt(f'Enter unseal token {i + 1}', hide_input=True, confirmation_prompt=False)
+        assert token is not None
+        resp = requests.post(address, headers={"Unseal-Token": token}, timeout=5)
+        if resp.status_code == 200:
+            KeyManage.print_response(resp.json())
+            if resp.json().get("status") in ["success", "skip"]:
+                return
+        else:
+            KeyManage.print_response({"message": resp.status_code, "status": "failed"})
+            return
+
+
+@click.command()
+@click.option(
+    '-a',
+    '--address',
+    help='inner cmdb api, http://127.0.0.1:8000',
+    required=True,
+)
+@click.option(
+    '-k',
+    '--token',
+    help='root token',
+    prompt=True,
+    hide_input=True,
+)
+@with_appcontext
+def cmdb_inner_secrets_seal(address, token):
+    """
+    seal the secrets feature
+    """
+    assert address is not None
+    assert token is not None
+    if not valid_address(address):
+        return
+    address = "{}/api/v0.1/secrets/seal".format(address.strip("/"))
+    resp = requests.post(address, headers={
+        "Inner-Token": token,
+    })
+    if resp.status_code == 200:
+        KeyManage.print_response(resp.json())
+    else:
+        KeyManage.print_response({"message": resp.status_code, "status": "failed"})
+
+
+@click.command()
+@with_appcontext
+def cmdb_password_data_migrate():
+    """
+    Migrate CI password data, version >= v2.3.6
+    """
+    from api.models.cmdb import CIIndexValueText
+    from api.models.cmdb import CIValueText
+    from api.lib.secrets.inner import InnerCrypt
+    from api.lib.secrets.vault import VaultClient
+
+    attrs = Attribute.get_by(to_dict=False)
+    for attr in attrs:
+        if attr.is_password:
+
+            value_table = CIIndexValueText if attr.is_index else CIValueText
+
+            failed = False
+            for i in value_table.get_by(attr_id=attr.id, to_dict=False):
+                if current_app.config.get("SECRETS_ENGINE", 'inner') == 'inner':
+                    _, status = InnerCrypt().decrypt(i.value)
+                    if status:
+                        continue
+
+                    encrypt_value, status = InnerCrypt().encrypt(i.value)
+                    if status:
+                        CIValueText.create(ci_id=i.ci_id, attr_id=attr.id, value=encrypt_value)
+                    else:
+                        failed = True
+                        continue
+                elif current_app.config.get("SECRETS_ENGINE") == 'vault':
+                    if i.value == '******':
+                        continue
+
+                    vault = VaultClient(current_app.config.get('VAULT_URL'), current_app.config.get('VAULT_TOKEN'))
+                    try:
+                        vault.update("/{}/{}".format(i.ci_id, i.attr_id), dict(v=i.value))
+                    except Exception as e:
+                        print('save password to vault failed: {}'.format(e))
+                        failed = True
+                        continue
+                else:
+                    continue
+
+                i.delete()
+
+                if not failed and attr.is_index:
+                    attr.update(is_index=False)
+
+
+@click.command()
+@with_appcontext
+def cmdb_agent_init():
+    """
+    Initialize the agent's permissions and obtain the key and secret
+    """
+
+    from api.models.acl import User
+
+    user = User.get_by(username="cmdb_agent", first=True, to_dict=False)
+    if user is None:
+        click.echo(
+            click.style('user cmdb_agent does not exist, please use flask add-user to create it first', fg='red'))
+        return
+
+    # grant
+    _app = AppCache.get('cmdb') or App.create(name='cmdb')
+    app_id = _app.id
+
+    ci_types = CIType.get_by(to_dict=False)
+    resource_type_id = ResourceType.get_by(name=ResourceTypeEnum.CI, first=True, to_dict=False).id
+    for ci_type in ci_types:
+        try:
+            ResourceCRUD.add(ci_type.name, resource_type_id, app_id)
+        except AbortException:
+            pass
+
+        ACLManager().grant_resource_to_role(ci_type.name,
+                                            "cmdb_agent",
+                                            ResourceTypeEnum.CI,
+                                            [PermEnum.READ, PermEnum.UPDATE, PermEnum.ADD, PermEnum.DELETE])
+
+    click.echo("Key   : {}".format(click.style(user.key, bg='red')))
+    click.echo("Secret: {}".format(click.style(user.secret, bg='red')))
+
+
+@click.command()
+@click.option(
+    '-v',
+    '--version',
+    help='input cmdb version, e.g. 2.4.6',
+    required=True,
+)
+@with_appcontext
+def cmdb_patch(version):
+    """
+    CMDB upgrade patch
+    """
+
+    version = version[1:] if version.lower().startswith("v") else version
+
+    try:
+        if version >= '2.4.6':
+
+            from api.models.cmdb import CITypeRelation
+            for cr in CITypeRelation.get_by(to_dict=False):
+                if hasattr(cr, 'parent_attr_id') and cr.parent_attr_id and not cr.parent_attr_ids:
+                    parent_attr_ids, child_attr_ids = [cr.parent_attr_id], [cr.child_attr_id]
+                    cr.update(parent_attr_ids=parent_attr_ids, child_attr_ids=child_attr_ids, commit=False)
+            db.session.commit()
+
+            from api.models.cmdb import AutoDiscoveryCIType, AutoDiscoveryCITypeRelation
+            from api.lib.cmdb.cache import CITypeCache, AttributeCache
+            for adt in AutoDiscoveryCIType.get_by(to_dict=False):
+                if adt.relation:
+                    if not AutoDiscoveryCITypeRelation.get_by(ad_type_id=adt.type_id):
+                        peer_type = CITypeCache.get(list(adt.relation.values())[0]['type_name'])
+                        peer_type_id = peer_type and peer_type.id
+                        peer_attr = AttributeCache.get(list(adt.relation.values())[0]['attr_name'])
+                        peer_attr_id = peer_attr and peer_attr.id
+                        if peer_type_id and peer_attr_id:
+                            AutoDiscoveryCITypeRelation.create(ad_type_id=adt.type_id,
+                                                               ad_key=list(adt.relation.keys())[0],
+                                                               peer_type_id=peer_type_id,
+                                                               peer_attr_id=peer_attr_id,
+                                                               commit=False)
+                if hasattr(adt, 'interval') and adt.interval and not adt.cron:
+                    adt.cron = "*/{} * * * *".format(adt.interval // 60 or 1)
+
+            db.session.commit()
+
+        if version >= "2.4.7":
+            from api.lib.cmdb.auto_discovery.const import DEFAULT_INNER
+            from api.models.cmdb import AutoDiscoveryRule
+            for i in DEFAULT_INNER:
+                existed = AutoDiscoveryRule.get_by(name=i['name'], first=True, to_dict=False)
+                if existed is not None:
+                    if "en" in i['option'] and 'en' not in (existed.option or {}):
+                        option = copy.deepcopy(existed.option)
+                        option['en'] = i['option']['en']
+                        existed.update(option=option, commit=False)
+
+            db.session.commit()
+
+        if version >= "2.4.14":  # update ci columns: updated_at and updated_by
+            ci_ids = []
+            for i in CI.get_by(only_query=True).filter(CI.updated_at.is_(None)):
+                hist = AttributeHistory.get_by(ci_id=i.id, only_query=True).order_by(AttributeHistory.id.desc()).first()
+                if hist is not None:
+                    record = OperationRecord.get_by_id(hist.record_id)
+                    if record is not None:
+                        u = UserCache.get(record.uid)
+                        i.update(updated_at=record.created_at, updated_by=u and u.nickname, flush=True)
+                        ci_ids.append(i.id)
+
+            db.session.commit()
+
+            batch_ci_cache.apply_async(args=(ci_ids,))
+    except Exception as e:
+        print("cmdb patch failed: {}".format(e))

cmdb-api/api/commands/click_common_setting.py

@@ -4,31 +4,40 @@ from flask.cli import with_appcontext
 from werkzeug.datastructures import MultiDict
 
 from api.lib.common_setting.acl import ACLManager
-from api.lib.common_setting.employee import EmployeeAddForm
+from api.lib.common_setting.employee import EmployeeAddForm, GrantEmployeeACLPerm
 from api.lib.common_setting.resp_format import ErrFormat
+from api.lib.common_setting.utils import CheckNewColumn
 from api.models.common_setting import Employee, Department
 
 
 class InitEmployee(object):
-    """
-    初始化员工
-    """
 
     def __init__(self):
         self.log = current_app.logger
 
     def import_user_from_acl(self):
         """
-        从ACL导入用户
+        Import users from ACL
         """
 
+        InitDepartment().init()
         acl = ACLManager('acl')
         user_list = acl.get_all_users()
 
         username_list = [e['username'] for e in Employee.get_by()]
 
         for user in user_list:
+            acl_uid = user['uid']
+            block = 1 if user['block'] else 0
+            acl_rid = self.get_rid_by_uid(acl_uid)
             if user['username'] in username_list:
+                existed = Employee.get_by(first=True, username=user['username'], to_dict=False)
+                if existed:
+                    existed.update(
+                        acl_uid=acl_uid,
+                        acl_rid=acl_rid,
+                        block=block,
+                    )
                 continue
             try:
                 form = EmployeeAddForm(MultiDict(user))
@@ -36,8 +45,9 @@ class InitEmployee(object):
                     raise Exception(
                         ','.join(['{}: {}'.format(filed, ','.join(msg)) for filed, msg in form.errors.items()]))
                 data = form.data
-                data['acl_uid'] = user['uid']
+                data['acl_uid'] = acl_uid
-                data['block'] = 1 if user['block'] else 0
+                data['acl_rid'] = acl_rid
+                data['block'] = block
                 data.pop('password')
                 Employee.create(
                     **data
@@ -46,6 +56,12 @@ class InitEmployee(object):
                 self.log.error(ErrFormat.acl_import_user_failed.format(user['username'], str(e)))
                 self.log.error(e)
 
+    @staticmethod
+    def get_rid_by_uid(uid):
+        from api.models.acl import Role
+        role = Role.get_by(first=True, uid=uid)
+        return role['id'] if role is not None else 0
+
 
 class InitDepartment(object):
     def __init__(self):
@@ -54,7 +70,8 @@ class InitDepartment(object):
     def init(self):
         self.init_wide_company()
 
-    def hard_delete(self, department_id, department_name):
+    @staticmethod
+    def hard_delete(department_id, department_name):
         existed_deleted_list = Department.query.filter(
             Department.department_name == department_name,
             Department.department_id == department_id,
@@ -63,11 +80,12 @@ class InitDepartment(object):
         for existed in existed_deleted_list:
             existed.delete()
 
-    def get_department(self, department_name):
+    @staticmethod
+    def get_department(department_name):
         return Department.query.filter(
             Department.department_name == department_name,
             Department.deleted == 0,
-        ).order_by(Department.created_at.asc()).first()
+        ).first()
 
     def run(self, department_id, department_name, department_parent_id):
         self.hard_delete(department_id, department_name)
@@ -77,7 +95,7 @@ class InitDepartment(object):
             if res.department_id == department_id:
                 return
             else:
-                new_d = res.update(
+                res.update(
                     department_id=department_id,
                     department_parent_id=department_parent_id,
                 )
@@ -91,11 +109,11 @@ class InitDepartment(object):
         new_d = self.get_department(department_name)
 
         if new_d.department_id != department_id:
-            new_d = new_d.update(
+            new_d.update(
                 department_id=department_id,
                 department_parent_id=department_parent_id,
             )
-        self.log.info(f"初始化 {department_name} 部门成功.")
+        self.log.info(f"init {department_name} success.")
 
     def run_common(self, department_id, department_name, department_parent_id):
         try:
@@ -106,19 +124,14 @@ class InitDepartment(object):
             raise Exception(e)
 
     def init_wide_company(self):
-        """
-        创建 id 0, name 全公司 的部门
-        """
         department_id = 0
         department_name = '全公司'
         department_parent_id = -1
 
         self.run_common(department_id, department_name, department_parent_id)
 
-    def create_acl_role_with_department(self):
+    @staticmethod
-        """
+    def create_acl_role_with_department():
-        当前所有部门，在ACL创建 role
-        """
         acl = ACLManager('acl')
         role_name_map = {role['name']: role for role in acl.get_all_roles()}
 
@@ -129,7 +142,7 @@ class InitDepartment(object):
                 continue
 
             role = role_name_map.get(department.department_name)
-            if role is None:
+            if not role:
                 payload = {
                     'app_id': 'acl',
                     'name': department.department_name,
@@ -144,12 +157,37 @@ class InitDepartment(object):
             info = f"update department acl_rid: {acl_rid}"
             current_app.logger.info(info)
 
+    def init_backend_resource(self):
+        acl = self.check_app('backend')
+        acl_rid = self.get_admin_user_rid()
+
+        if acl_rid == 0:
+            return
+        GrantEmployeeACLPerm(acl).grant_by_rid(acl_rid, True)
+
+    @staticmethod
+    def check_app(app_name):
+        acl = ACLManager(app_name)
+        payload = dict(
+            name=app_name,
+            description=app_name
+        )
+        app = acl.validate_app()
+        if not app:
+            acl.create_app(payload)
+        return acl
+
+    @staticmethod
+    def get_admin_user_rid():
+        admin = Employee.get_by(first=True, username='admin', to_dict=False)
+        return admin.acl_rid if admin else 0
+
 
 @click.command()
 @with_appcontext
 def init_import_user_from_acl():
     """
-    从ACL导入用户
+    Import users from ACL
     """
     InitEmployee().import_user_from_acl()
 
@@ -158,7 +196,35 @@ def init_import_user_from_acl():
 @with_appcontext
 def init_department():
     """
-    初始化 部门
+    Department initialization
     """
-    InitDepartment().init()
+    cli = InitDepartment()
-    InitDepartment().create_acl_role_with_department()
+    cli.init_wide_company()
+    cli.create_acl_role_with_department()
+    cli.init_backend_resource()
+
+
+@click.command()
+@with_appcontext
+def common_check_new_columns():
+    """
+    add new columns to tables
+    """
+    CheckNewColumn().run()
+
+
+@click.command()
+@with_appcontext
+def common_sync_file_to_db():
+    from api.lib.common_setting.upload_file import CommonFileCRUD
+    CommonFileCRUD.sync_file_to_db()
+
+
+@click.command()
+@with_appcontext
+@click.option('--value', type=click.INT, default=-1)
+def set_auth_auto_redirect_enable(value):
+    if value < 0:
+        return
+    from api.lib.common_setting.common_data import CommonDataCRUD
+    CommonDataCRUD.set_auth_auto_redirect_enable(value)

cmdb-api/api/commands/common.py

@@ -5,9 +5,7 @@ from glob import glob
 from subprocess import call
 
 import click
-from flask import current_app
 from flask.cli import with_appcontext
-from werkzeug.exceptions import MethodNotAllowed, NotFound
 
 from api.extensions import db
 
@@ -84,69 +82,59 @@ def clean():
                 os.remove(full_pathname)
 
 
-@click.command()
-@click.option("--url", default=None, help="Url to test (ex. /static/image.png)")
-@click.option(
-    "--order", default="rule", help="Property on Rule to order by (default: rule)"
-)
-@with_appcontext
-def urls(url, order):
-    """Display all of the url matching routes for the project.
-
-    Borrowed from Flask-Script, converted to use Click.
-    """
-    rows = []
-    column_headers = ("Rule", "Endpoint", "Arguments")
-
-    if url:
-        try:
-            rule, arguments = current_app.url_map.bind("localhost").match(
-                url, return_rule=True
-            )
-            rows.append((rule.rule, rule.endpoint, arguments))
-            column_length = 3
-        except (NotFound, MethodNotAllowed) as e:
-            rows.append(("<{}>".format(e), None, None))
-            column_length = 1
-    else:
-        rules = sorted(
-            current_app.url_map.iter_rules(), key=lambda rule: getattr(rule, order)
-        )
-        for rule in rules:
-            rows.append((rule.rule, rule.endpoint, None))
-        column_length = 2
-
-    str_template = ""
-    table_width = 0
-
-    if column_length >= 1:
-        max_rule_length = max(len(r[0]) for r in rows)
-        max_rule_length = max_rule_length if max_rule_length > 4 else 4
-        str_template += "{:" + str(max_rule_length) + "}"
-        table_width += max_rule_length
-
-    if column_length >= 2:
-        max_endpoint_length = max(len(str(r[1])) for r in rows)
-        max_endpoint_length = max_endpoint_length if max_endpoint_length > 8 else 8
-        str_template += "  {:" + str(max_endpoint_length) + "}"
-        table_width += 2 + max_endpoint_length
-
-    if column_length >= 3:
-        max_arguments_length = max(len(str(r[2])) for r in rows)
-        max_arguments_length = max_arguments_length if max_arguments_length > 9 else 9
-        str_template += "  {:" + str(max_arguments_length) + "}"
-        table_width += 2 + max_arguments_length
-
-    click.echo(str_template.format(*column_headers[:column_length]))
-    click.echo("-" * table_width)
-
-    for row in rows:
-        click.echo(str_template.format(*row[:column_length]))
-
-
 @click.command()
 @with_appcontext
 def db_setup():
     """create tables
     """
     db.create_all()
+
+    try:
+        db.session.execute("set global sql_mode='STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,"
+                           "ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'")
+        db.session.commit()
+    except:
+        pass
+
+    try:
+        db.session.execute("set global tidb_enable_noop_functions='ON'")
+        db.session.commit()
+    except:
+        pass
+
+
+@click.group()
+def translate():
+    """Translation and localization commands."""
+
+
+@translate.command()
+@click.argument('lang')
+def init(lang):
+    """Initialize a new language."""
+
+    if os.system('pybabel extract -F babel.cfg -k _l -o messages.pot .'):
+        raise RuntimeError('extract command failed')
+    if os.system(
+            'pybabel init -i messages.pot -d api/translations -l ' + lang):
+        raise RuntimeError('init command failed')
+    os.remove('messages.pot')
+
+
+@translate.command()
+def update():
+    """Update all languages."""
+
+    if os.system('pybabel extract -F babel.cfg -k _l -o messages.pot .'):
+        raise RuntimeError('extract command failed')
+    if os.system('pybabel update -i messages.pot -d api/translations'):
+        raise RuntimeError('update command failed')
+    os.remove('messages.pot')
+
+
+@translate.command()
+def compile():
+    """Compile all languages."""
+
+    if os.system('pybabel compile -d api/translations'):
+        raise RuntimeError('compile command failed')

cmdb-api/api/extensions.py

@@ -2,6 +2,7 @@
 
 
 from celery import Celery
+from flask_babel import Babel
 from flask_bcrypt import Bcrypt
 from flask_caching import Cache
 from flask_cors import CORS
@@ -9,10 +10,12 @@ from flask_login import LoginManager
 from flask_migrate import Migrate
 from flask_sqlalchemy import SQLAlchemy
 
+from api.lib.secrets.inner import KeyManage
 from api.lib.utils import ESHandler
 from api.lib.utils import RedisHandler
 
 bcrypt = Bcrypt()
+babel = Babel()
 login_manager = LoginManager()
 db = SQLAlchemy(session_options={"autoflush": False})
 migrate = Migrate()
@@ -21,3 +24,4 @@ celery = Celery()
 cors = CORS(supports_credentials=True)
 rd = RedisHandler()
 es = ESHandler()
+inner_secrets = KeyManage()

cmdb-api/api/lib/cmdb/attribute.py

@@ -1,15 +1,20 @@
-# -*- coding:utf-8 -*- 
+# -*- coding:utf-8 -*-
 
-import requests
 from flask import abort
 from flask import current_app
-from flask import g
 from flask import session
+from flask_login import current_user
 
 from api.extensions import db
 from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.cache import CITypeAttributesCache
+from api.lib.cmdb.cache import CITypeCache
+from api.lib.cmdb.const import BUILTIN_KEYWORDS
 from api.lib.cmdb.const import CITypeOperateType
-from api.lib.cmdb.const import ResourceTypeEnum, RoleEnum, PermEnum
+from api.lib.cmdb.const import CMDB_QUEUE
+from api.lib.cmdb.const import PermEnum
+from api.lib.cmdb.const import ResourceTypeEnum
+from api.lib.cmdb.const import RoleEnum
 from api.lib.cmdb.const import ValueTypeEnum
 from api.lib.cmdb.history import CITypeHistoryManager
 from api.lib.cmdb.resp_format import ErrFormat
@@ -17,7 +22,9 @@ from api.lib.cmdb.utils import ValueTypeMap
 from api.lib.decorator import kwargs_required
 from api.lib.perm.acl.acl import is_app_admin
 from api.lib.perm.acl.acl import validate_permission
+from api.lib.webhook import webhook_request
 from api.models.cmdb import Attribute
+from api.models.cmdb import CIType
 from api.models.cmdb import CITypeAttribute
 from api.models.cmdb import CITypeAttributeGroupItem
 from api.models.cmdb import PreferenceShowAttributes
@@ -33,15 +40,11 @@ class AttributeManager(object):
         pass
 
     @staticmethod
-    def _get_choice_values_from_web_hook(choice_web_hook):
+    def _get_choice_values_from_webhook(choice_webhook, payload=None):
-        url = choice_web_hook.get('url')
+        ret_key = choice_webhook.get('ret_key')
-        ret_key = choice_web_hook.get('ret_key')
-        headers = choice_web_hook.get('headers') or {}
-        payload = choice_web_hook.get('payload') or {}
-        method = choice_web_hook.get('method', 'GET').lower()
 
         try:
-            res = getattr(requests, method)(url, headers=headers, data=payload).json()
+            res = webhook_request(choice_webhook, payload or {}).json()
             if ret_key:
                 ret_key_list = ret_key.strip().split("##")
                 for key in ret_key_list[:-1]:
@@ -53,52 +56,102 @@ class AttributeManager(object):
                 return [[i, {}] for i in (res.get(ret_key_list[-1]) or [])]
 
         except Exception as e:
-            current_app.logger.error(str(e))
+            current_app.logger.error("get choice values failed: {}".format(e))
             return []
 
+    @staticmethod
+    def _get_choice_values_from_other(choice_other):
+        from api.lib.cmdb.search import SearchError
+        from api.lib.cmdb.search.ci import search
+
+        if choice_other.get('type_ids'):
+            type_ids = choice_other.get('type_ids')
+            attr_id = choice_other.get('attr_id')
+            other_filter = choice_other.get('filter') or ''
+
+            query = "_type:({}),{}".format(";".join(map(str, type_ids)), other_filter)
+            s = search(query, fl=[str(attr_id)], facet=[str(attr_id)], count=1)
+            try:
+                _, _, _, _, _, facet = s.search()
+                return [[i[0], {}] for i in (list(facet.values()) or [[]])[0]]
+            except SearchError as e:
+                current_app.logger.error("get choice values from other ci failed: {}".format(e))
+                return []
+
+        elif choice_other.get('script'):
+            try:
+                x = compile(choice_other['script'], '', "exec")
+                local_ns = {}
+                exec(x, {}, local_ns)
+                res = local_ns['ChoiceValue']().values() or []
+                return [[i, {}] for i in res]
+            except Exception as e:
+                current_app.logger.error("get choice values from script: {}".format(e))
+                return []
+
     @classmethod
-    def get_choice_values(cls, attr_id, value_type, choice_web_hook, choice_web_hook_parse=True):
+    def get_choice_values(cls, attr_id, value_type, choice_web_hook, choice_other,
-        if choice_web_hook and isinstance(choice_web_hook, dict) and choice_web_hook_parse:
+                          choice_web_hook_parse=True, choice_other_parse=True):
-            return cls._get_choice_values_from_web_hook(choice_web_hook)
+        if choice_web_hook:
-        elif choice_web_hook and not choice_web_hook_parse:
+            if choice_web_hook_parse and isinstance(choice_web_hook, dict):
-            return []
+                return cls._get_choice_values_from_webhook(choice_web_hook)
+            else:
+                return []
+        elif choice_other:
+            if choice_other_parse and isinstance(choice_other, dict):
+                return cls._get_choice_values_from_other(choice_other)
+            else:
+                return []
 
         choice_table = ValueTypeMap.choice.get(value_type)
+        if not choice_table:
+            return []
         choice_values = choice_table.get_by(fl=["value", "option"], attr_id=attr_id)
 
-        return [[choice_value['value'], choice_value['option']] for choice_value in choice_values]
+        return [[ValueTypeMap.serialize[value_type](choice_value['value']), choice_value['option'] or
+                 {"label": ValueTypeMap.serialize[value_type](choice_value['value'])}]
+                for choice_value in choice_values]
 
     @staticmethod
     def add_choice_values(_id, value_type, choice_values):
         choice_table = ValueTypeMap.choice.get(value_type)
+        if choice_table is None:
+            return
+
+        choice_table.get_by(attr_id=_id, only_query=True).delete()
 
-        db.session.query(choice_table).filter(choice_table.attr_id == _id).delete()
-        db.session.flush()
-        choice_values = choice_values
         for v, option in choice_values:
-            table = choice_table(attr_id=_id, value=v, option=option)
+            choice_table.create(attr_id=_id, value=v, option=option, commit=False)
-
-            db.session.add(table)
 
         try:
             db.session.flush()
-        except:
+        except Exception as e:
+            current_app.logger.warning("add choice values failed: {}".format(e))
             return abort(400, ErrFormat.invalid_choice_values)
 
     @staticmethod
     def _del_choice_values(_id, value_type):
         choice_table = ValueTypeMap.choice.get(value_type)
 
-        db.session.query(choice_table).filter(choice_table.attr_id == _id).delete()
+        choice_table and choice_table.get_by(attr_id=_id, only_query=True).delete()
         db.session.flush()
 
+    @classmethod
+    def get_enum_map(cls, _attr_id, _attr=None):
+        attr = AttributeCache.get(_attr_id) if _attr_id else _attr
+        if attr and attr.is_choice:
+            choice_values = cls.get_choice_values(attr.id, attr.value_type, None, None)
+            return {i[0]: i[1]['label'] for i in choice_values if i[1] and i[1].get('label')}
+
+        return {}
+
     @classmethod
     def search_attributes(cls, name=None, alias=None, page=1, page_size=None):
         """
-        :param name: 
+        :param name:
-        :param alias: 
+        :param alias:
-        :param page: 
+        :param page:
-        :param page_size: 
+        :param page_size:
         :return: attribute, if name is None, then return all attributes
         """
         if name is not None:
@@ -112,8 +165,9 @@ class AttributeManager(object):
         attrs = attrs[(page - 1) * page_size:][:page_size]
         res = list()
         for attr in attrs:
-            attr["is_choice"] and attr.update(dict(choice_value=cls.get_choice_values(
+            attr["is_choice"] and attr.update(
-                attr["id"], attr["value_type"], attr["choice_web_hook"])))
+                dict(choice_value=cls.get_choice_values(attr["id"], attr["value_type"],
+                                                        attr["choice_web_hook"], attr.get("choice_other"))))
             attr['is_choice'] and attr.pop('choice_web_hook', None)
 
             res.append(attr)
@@ -122,30 +176,47 @@ class AttributeManager(object):
 
     def get_attribute_by_name(self, name):
         attr = Attribute.get_by(name=name, first=True)
-        if attr and attr["is_choice"]:
+        if attr.get("is_choice"):
-            attr.update(dict(choice_value=self.get_choice_values(
+            attr["choice_value"] = self.get_choice_values(attr["id"],
-                attr["id"], attr["value_type"], attr["choice_web_hook"])))
+                                                          attr["value_type"],
+                                                          attr["choice_web_hook"],
+                                                          attr.get("choice_other"))
+
         return attr
 
     def get_attribute_by_alias(self, alias):
         attr = Attribute.get_by(alias=alias, first=True)
-        if attr and attr["is_choice"]:
+        if attr.get("is_choice"):
-            attr.update(dict(choice_value=self.get_choice_values(
+            attr["choice_value"] = self.get_choice_values(attr["id"],
-                attr["id"], attr["value_type"], attr["choice_web_hook"])))
+                                                          attr["value_type"],
+                                                          attr["choice_web_hook"],
+                                                          attr.get("choice_other"))
+
         return attr
 
     def get_attribute_by_id(self, _id):
         attr = Attribute.get_by_id(_id).to_dict()
-        if attr and attr["is_choice"]:
+        if attr.get("is_choice"):
-            attr.update(dict(choice_value=self.get_choice_values(
+            attr["choice_value"] = self.get_choice_values(attr["id"],
-                attr["id"], attr["value_type"], attr["choice_web_hook"])))
+                                                          attr["value_type"],
+                                                          attr["choice_web_hook"],
+                                                          attr.get("choice_other"))
+
         return attr
 
-    def get_attribute(self, key, choice_web_hook_parse=True):
+    def get_attribute(self, key, choice_web_hook_parse=True, choice_other_parse=True):
-        attr = AttributeCache.get(key).to_dict()
+        attr = AttributeCache.get(key) or dict()
-        if attr and attr["is_choice"]:
+        attr = attr and attr.to_dict()
-            attr.update(dict(choice_value=self.get_choice_values(
+        if attr.get("is_choice"):
-                attr["id"], attr["value_type"], attr["choice_web_hook"])), choice_web_hook_parse=choice_web_hook_parse)
+            attr["choice_value"] = self.get_choice_values(
+                attr["id"],
+                attr["value_type"],
+                attr["choice_web_hook"],
+                attr.get("choice_other"),
+                choice_web_hook_parse=choice_web_hook_parse,
+                choice_other_parse=choice_other_parse,
+            )
+
         return attr
 
     @staticmethod
@@ -153,16 +224,40 @@ class AttributeManager(object):
         if RoleEnum.CONFIG not in session.get("acl", {}).get("parentRoles", []) and not is_app_admin('cmdb'):
             return abort(403, ErrFormat.role_required.format(RoleEnum.CONFIG))
 
+    @classmethod
+    def calc_computed_attribute(cls, attr_id):
+        """
+        calculate computed attribute for all ci
+        :param attr_id:
+        :return:
+        """
+        cls.can_create_computed_attribute()
+
+        from api.tasks.cmdb import calc_computed_attribute
+
+        calc_computed_attribute.apply_async(args=(attr_id, current_user.uid), queue=CMDB_QUEUE)
+
     @classmethod
     @kwargs_required("name")
     def add(cls, **kwargs):
         choice_value = kwargs.pop("choice_value", [])
         kwargs.pop("is_choice", None)
-        is_choice = True if choice_value or kwargs.get('choice_web_hook') else False
+        is_choice = True if choice_value or kwargs.get('choice_web_hook') or kwargs.get('choice_other') else False
 
         name = kwargs.pop("name")
-        if name in {'id', '_id', 'ci_id', 'type', '_type', 'ci_type'}:
+        if name in BUILTIN_KEYWORDS or kwargs.get('alias') in BUILTIN_KEYWORDS:
             return abort(400, ErrFormat.attribute_name_cannot_be_builtin)
+
+        while kwargs.get('choice_other'):
+            if isinstance(kwargs['choice_other'], dict):
+                if kwargs['choice_other'].get('script'):
+                    break
+
+                if kwargs['choice_other'].get('type_ids') and kwargs['choice_other'].get('attr_id'):
+                    break
+
+            return abort(400, ErrFormat.attribute_choice_other_invalid)
+
         alias = kwargs.pop("alias", "")
         alias = name if not alias else alias
         Attribute.get_by(name=name, first=True) and abort(400, ErrFormat.attribute_name_duplicate.format(name))
@@ -172,11 +267,13 @@ class AttributeManager(object):
 
         kwargs.get('is_computed') and cls.can_create_computed_attribute()
 
+        kwargs.get('choice_other') and kwargs['choice_other'].get('script') and cls.can_create_computed_attribute()
+
         attr = Attribute.create(flush=True,
                                 name=name,
                                 alias=alias,
                                 is_choice=is_choice,
-                                uid=g.user.uid,
+                                uid=current_user.uid,
                                 **kwargs)
 
         if choice_value:
@@ -210,6 +307,11 @@ class AttributeManager(object):
 
         return attr.id
 
+    @staticmethod
+    def _clean_ci_type_attributes_cache(attr_id):
+        for i in CITypeAttribute.get_by(attr_id=attr_id, to_dict=False):
+            CITypeAttributesCache.clean(i.type_id)
+
     @staticmethod
     def _change_index(attr, old, new):
         from api.lib.cmdb.utils import TableMap
@@ -220,11 +322,11 @@ class AttributeManager(object):
         new_table = TableMap(attr=attr, is_index=new).table
 
         ci_ids = []
-        for i in db.session.query(old_table).filter(getattr(old_table, 'attr_id') == attr.id):
+        for i in old_table.get_by(attr_id=attr.id, to_dict=False):
             new_table.create(ci_id=i.ci_id, attr_id=attr.id, value=i.value, flush=True)
             ci_ids.append(i.ci_id)
 
-        db.session.query(old_table).filter(getattr(old_table, 'attr_id') == attr.id).delete()
+        old_table.get_by(attr_id=attr.id, only_query=True).delete()
 
         try:
             db.session.commit()
@@ -239,7 +341,7 @@ class AttributeManager(object):
     def _can_edit_attribute(attr):
         from api.lib.cmdb.ci_type import CITypeManager
 
-        if attr.uid == g.user.uid:
+        if attr.uid == current_user.uid:
             return True
 
         for i in CITypeAttribute.get_by(attr_id=attr.id, to_dict=False):
@@ -252,9 +354,6 @@ class AttributeManager(object):
     def update(self, _id, **kwargs):
         attr = Attribute.get_by_id(_id) or abort(404, ErrFormat.attribute_not_found.format("id={}".format(_id)))
 
-        if not self._can_edit_attribute(attr):
-            return abort(403, ErrFormat.cannot_edit_attribute)
-
         if kwargs.get("name"):
             other = Attribute.get_by(name=kwargs['name'], first=True, to_dict=False)
             if other and other.id != attr.id:
@@ -272,12 +371,22 @@ class AttributeManager(object):
 
             self._change_index(attr, attr.is_index, kwargs['is_index'])
 
+        while kwargs.get('choice_other'):
+            if isinstance(kwargs['choice_other'], dict):
+                if kwargs['choice_other'].get('script'):
+                    break
+
+                if kwargs['choice_other'].get('type_ids') and kwargs['choice_other'].get('attr_id'):
+                    break
+
+            return abort(400, ErrFormat.attribute_choice_other_invalid)
+
         existed2 = attr.to_dict()
-        if not existed2['choice_web_hook'] and existed2['is_choice']:
+        if not existed2['choice_web_hook'] and not existed2.get('choice_other') and existed2['is_choice']:
-            existed2['choice_value'] = self.get_choice_values(attr.id, attr.value_type, attr.choice_web_hook)
+            existed2['choice_value'] = self.get_choice_values(attr.id, attr.value_type, None, None)
 
         choice_value = kwargs.pop("choice_value", False)
-        is_choice = True if choice_value or kwargs.get('choice_web_hook') else False
+        is_choice = True if choice_value or kwargs.get('choice_web_hook') or kwargs.get('choice_other') else False
         kwargs['is_choice'] = is_choice
 
         if kwargs.get('default') and not (isinstance(kwargs['default'], dict) and 'default' in kwargs['default']):
@@ -285,11 +394,19 @@ class AttributeManager(object):
 
         kwargs.get('is_computed') and self.can_create_computed_attribute()
 
+        is_changed = False
+        for k in kwargs:
+            if kwargs[k] != getattr(attr, k, None):
+                is_changed = True
+
+        if is_changed and not self._can_edit_attribute(attr):
+            return abort(403, ErrFormat.cannot_edit_attribute)
+
         attr.update(flush=True, filter_none=False, **kwargs)
 
         if is_choice and choice_value:
             self.add_choice_values(attr.id, attr.value_type, choice_value)
-        elif is_choice:
+        elif existed2['is_choice']:
             self._del_choice_values(attr.id, attr.value_type)
 
         try:
@@ -298,7 +415,7 @@ class AttributeManager(object):
             db.session.rollback()
             current_app.logger.error("update attribute error, {0}".format(str(e)))
 
-            return abort(400, ErrFormat.update_attribute_failed.format(("id=".format(_id))))
+            return abort(400, ErrFormat.update_attribute_failed.format(("id={}".format(_id))))
 
         new = attr.to_dict()
         if not new['choice_web_hook'] and new['is_choice']:
@@ -308,6 +425,8 @@ class AttributeManager(object):
 
         AttributeCache.clean(attr)
 
+        self._clean_ci_type_attributes_cache(_id)
+
         return attr.id
 
     @staticmethod
@@ -315,25 +434,31 @@ class AttributeManager(object):
         attr = Attribute.get_by_id(_id) or abort(404, ErrFormat.attribute_not_found.format("id={}".format(_id)))
         name = attr.name
 
-        if attr.uid and attr.uid != g.user.uid:
+        if CIType.get_by(unique_id=attr.id, first=True, to_dict=False) is not None:
+            return abort(400, ErrFormat.attribute_is_unique_id)
+
+        ref = CITypeAttribute.get_by(attr_id=_id, to_dict=False, first=True)
+        if ref is not None:
+            ci_type = CITypeCache.get(ref.type_id)
+            return abort(400, ErrFormat.attribute_is_ref_by_type.format(ci_type and ci_type.alias or ref.type_id))
+
+        if attr.uid != current_user.uid and not is_app_admin('cmdb'):
             return abort(403, ErrFormat.cannot_delete_attribute)
 
         if attr.is_choice:
             choice_table = ValueTypeMap.choice.get(attr.value_type)
-            db.session.query(choice_table).filter(choice_table.attr_id == _id).delete()  # FIXME: session conflict
+            choice_table.get_by(attr_id=_id, only_query=True).delete()
-            db.session.flush()
-
-        AttributeCache.clean(attr)
 
         attr.soft_delete()
 
-        for i in CITypeAttribute.get_by(attr_id=_id, to_dict=False):
+        AttributeCache.clean(attr)
-            i.soft_delete()
 
         for i in PreferenceShowAttributes.get_by(attr_id=_id, to_dict=False):
-            i.soft_delete()
+            i.soft_delete(commit=False)
 
         for i in CITypeAttributeGroupItem.get_by(attr_id=_id, to_dict=False):
-            i.soft_delete()
+            i.soft_delete(commit=False)
+
+        db.session.commit()
 
         return name

cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py

@@ -1,44 +1,61 @@
 # -*- coding:utf-8 -*-
+import copy
 import datetime
 import json
+import jsonpath
 import os
-
 from flask import abort
 from flask import current_app
-from flask import g
+from flask_login import current_user
 from sqlalchemy import func
 
 from api.extensions import db
-from api.lib.cmdb.auto_discovery.const import ClOUD_MAP
+from api.lib.cmdb.auto_discovery.const import CLOUD_MAP
+from api.lib.cmdb.auto_discovery.const import DEFAULT_INNER
+from api.lib.cmdb.auto_discovery.const import PRIVILEGED_USERS
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.cache import AutoDiscoveryMappingCache
 from api.lib.cmdb.cache import CITypeAttributeCache
 from api.lib.cmdb.cache import CITypeCache
 from api.lib.cmdb.ci import CIManager
-from api.lib.cmdb.ci import CIRelationManager
 from api.lib.cmdb.ci_type import CITypeGroupManager
 from api.lib.cmdb.const import AutoDiscoveryType
+from api.lib.cmdb.const import CMDB_QUEUE
 from api.lib.cmdb.const import PermEnum
 from api.lib.cmdb.const import ResourceTypeEnum
+from api.lib.cmdb.custom_dashboard import SystemConfigManager
 from api.lib.cmdb.resp_format import ErrFormat
 from api.lib.cmdb.search import SearchError
-from api.lib.cmdb.search.ci import search
+from api.lib.cmdb.search.ci import search as ci_search
+from api.lib.common_setting.role_perm_base import CMDBApp
 from api.lib.mixin import DBMixin
+from api.lib.perm.acl.acl import ACLManager
 from api.lib.perm.acl.acl import is_app_admin
 from api.lib.perm.acl.acl import validate_permission
 from api.lib.utils import AESCrypto
+from api.models.cmdb import AutoDiscoveryAccount
 from api.models.cmdb import AutoDiscoveryCI
 from api.models.cmdb import AutoDiscoveryCIType
+from api.models.cmdb import AutoDiscoveryCITypeRelation
+from api.models.cmdb import AutoDiscoveryCounter
+from api.models.cmdb import AutoDiscoveryExecHistory
 from api.models.cmdb import AutoDiscoveryRule
+from api.models.cmdb import AutoDiscoveryRuleSyncHistory
+from api.tasks.cmdb import build_relations_for_ad_accept
+from api.tasks.cmdb import write_ad_rule_sync_history
 
 PWD = os.path.abspath(os.path.dirname(__file__))
+app_cli = CMDBApp()
 
 
 def parse_plugin_script(script):
     attributes = []
     try:
         x = compile(script, '', "exec")
-        exec(x)
+        local_ns = {}
-        unique_key = locals()['AutoDiscovery']().unique_key
+        exec(x, {}, local_ns)
-        attrs = locals()['AutoDiscovery']().attributes() or []
+        unique_key = local_ns['AutoDiscovery']().unique_key
+        attrs = local_ns['AutoDiscovery']().attributes() or []
     except Exception as e:
         return abort(400, str(e))
 
@@ -96,14 +113,30 @@ class AutoDiscoveryRuleCRUD(DBMixin):
             else:
                 self.cls.create(**rule)
 
-    def _can_add(self, **kwargs):
+    def _can_add(self, valid=True, **kwargs):
         self.cls.get_by(name=kwargs['name']) and abort(400, ErrFormat.adr_duplicate.format(kwargs['name']))
-        if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
+        if kwargs.get('is_plugin') and kwargs.get('plugin_script') and valid:
             kwargs = check_plugin_script(**kwargs)
+            acl = ACLManager(app_cli.app_name)
+            has_perm = True
+            try:
+                if not acl.has_permission(app_cli.op.Auto_Discovery,
+                                          app_cli.resource_type_name,
+                                          app_cli.op.create_plugin) and not is_app_admin(app_cli.app_name):
+                    has_perm = False
+            except Exception:
+                if not is_app_admin(app_cli.app_name):
+                    return abort(403, ErrFormat.role_required.format(app_cli.admin_name))
+
+            if not has_perm:
+                return abort(403, ErrFormat.no_permission.format(
+                    app_cli.op.Auto_Discovery, app_cli.op.create_plugin))
+
+        kwargs['owner'] = current_user.uid
 
         return kwargs
 
-    def _can_update(self, **kwargs):
+    def _can_update(self, valid=True, **kwargs):
         existed = self.cls.get_by_id(kwargs['_id']) or abort(
             404, ErrFormat.adr_not_found.format("id={}".format(kwargs['_id'])))
 
@@ -115,6 +148,22 @@ class AutoDiscoveryRuleCRUD(DBMixin):
             if other and other.id != existed.id:
                 return abort(400, ErrFormat.adr_duplicate.format(kwargs['name']))
 
+        if existed.is_plugin and valid:
+            acl = ACLManager(app_cli.app_name)
+            has_perm = True
+            try:
+                if not acl.has_permission(app_cli.op.Auto_Discovery,
+                                          app_cli.resource_type_name,
+                                          app_cli.op.update_plugin) and not is_app_admin(app_cli.app_name):
+                    has_perm = False
+            except Exception:
+                if not is_app_admin(app_cli.app_name):
+                    return abort(403, ErrFormat.role_required.format(app_cli.admin_name))
+
+            if not has_perm:
+                return abort(403, ErrFormat.no_permission.format(
+                    app_cli.op.Auto_Discovery, app_cli.op.update_plugin))
+
         return existed
 
     def update(self, _id, **kwargs):
@@ -122,21 +171,44 @@ class AutoDiscoveryRuleCRUD(DBMixin):
         if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
             kwargs = check_plugin_script(**kwargs)
 
+            for item in AutoDiscoveryCIType.get_by(adr_id=_id, to_dict=False):
+                item.update(updated_at=datetime.datetime.now())
+
         return super(AutoDiscoveryRuleCRUD, self).update(_id, filter_none=False, **kwargs)
 
     def _can_delete(self, **kwargs):
         if AutoDiscoveryCIType.get_by(adr_id=kwargs['_id'], first=True):
             return abort(400, ErrFormat.adr_referenced)
 
-        return self._can_update(**kwargs)
+        existed = self.cls.get_by_id(kwargs['_id']) or abort(
+            404, ErrFormat.adr_not_found.format("id={}".format(kwargs['_id'])))
+
+        if existed.is_plugin:
+            acl = ACLManager(app_cli.app_name)
+            has_perm = True
+            try:
+                if not acl.has_permission(app_cli.op.Auto_Discovery,
+                                          app_cli.resource_type_name,
+                                          app_cli.op.delete_plugin) and not is_app_admin(app_cli.app_name):
+                    has_perm = False
+            except Exception:
+                if not is_app_admin(app_cli.app_name):
+                    return abort(403, ErrFormat.role_required.format(app_cli.admin_name))
+
+            if not has_perm:
+                return abort(403, ErrFormat.no_permission.format(
+                    app_cli.op.Auto_Discovery, app_cli.op.delete_plugin))
+
+        return existed
 
 
 class AutoDiscoveryCITypeCRUD(DBMixin):
     cls = AutoDiscoveryCIType
 
     @classmethod
-    def get_all(cls):
+    def get_all(cls, type_ids=None):
-        return cls.cls.get_by(to_dict=False)
+        res = cls.cls.get_by(to_dict=False)
+        return [i for i in res if type_ids is None or i.type_id in type_ids]
 
     @classmethod
     def get_by_id(cls, _id):
@@ -147,25 +219,59 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
         return cls.cls.get_by(type_id=type_id, to_dict=False)
 
     @classmethod
-    def get(cls, ci_id, oneagent_id, last_update_at=None):
+    def get_ad_attributes(cls, type_id):
+        result = []
+        adts = cls.get_by_type_id(type_id)
+        for adt in adts:
+            adr = AutoDiscoveryRuleCRUD.get_by_id(adt.adr_id)
+            if not adr:
+                continue
+            if adr.type == AutoDiscoveryType.HTTP:
+                for i in DEFAULT_INNER:
+                    if adr.name == i['name']:
+                        attrs = AutoDiscoveryHTTPManager.get_attributes(
+                            i['en'], (adt.extra_option or {}).get('category')) or []
+                        result.extend([i.get('name') for i in attrs])
+                        break
+            elif adr.type == AutoDiscoveryType.SNMP:
+                attributes = AutoDiscoverySNMPManager.get_attributes()
+                result.extend([i.get('name') for i in (attributes or [])])
+            else:
+                result.extend([i.get('name') for i in (adr.attributes or [])])
+
+        return sorted(list(set(result)))
+
+    @classmethod
+    def get(cls, ci_id, oneagent_id, oneagent_name, last_update_at=None):
+        """
+        OneAgent sync rules
+        :param ci_id:
+        :param oneagent_id:
+        :param oneagent_name:
+        :param last_update_at:
+        :return:
+        """
         result = []
         rules = cls.cls.get_by(to_dict=True)
 
         for rule in rules:
-            if rule.get('relation'):
+            if not rule['enabled']:
                 continue
 
-            if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('secret'):
+            if isinstance(rule.get("extra_option"), dict):
-                if not (g.user.username == "cmdb_agent" or g.user.uid == rule['uid']):
+                decrypt_account(rule['extra_option'], rule['uid'])
+
+                if rule['extra_option'].get('_reference'):
+                    rule['extra_option'].pop('password', None)
                     rule['extra_option'].pop('secret', None)
-                else:
+                    rule['extra_option'].update(
-                    rule['extra_option']['secret'] = AESCrypto.decrypt(rule['extra_option']['secret'])
+                        AutoDiscoveryAccountCRUD().get_config_by_id(rule['extra_option']['_reference']))
 
             if oneagent_id and rule['agent_id'] == oneagent_id:
                 result.append(rule)
             elif rule['query_expr']:
                 query = rule['query_expr'].lstrip('q').lstrip('=')
-                s = search(query, fl=['_id'], count=1000000)
+                s = ci_search(query, fl=['_id'], count=1000000)
                 try:
                     response, _, _, _, _, _ = s.search()
                 except SearchError as e:
@@ -176,25 +282,32 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
                         result.append(rule)
                         break
             elif not rule['agent_id'] and not rule['query_expr'] and rule['adr_id']:
+                try:
+                    if not int(oneagent_id, 16):  # excludes master
+                        continue
+                except Exception:
+                    pass
+
                 adr = AutoDiscoveryRuleCRUD.get_by_id(rule['adr_id'])
                 if not adr:
                     continue
                 if adr.type in (AutoDiscoveryType.SNMP, AutoDiscoveryType.HTTP):
                     continue
 
-                if not rule['updated_at']:
-                    continue
-
                 result.append(rule)
 
+        ad_rules_updated_at = (SystemConfigManager.get('ad_rules_updated_at') or {}).get('option', {}).get('v') or ""
         new_last_update_at = ""
         for i in result:
             i['adr'] = AutoDiscoveryRule.get_by_id(i['adr_id']).to_dict()
+            i['adr'].pop("attributes", None)
             __last_update_at = max([i['updated_at'] or "", i['created_at'] or "",
-                                    i['adr']['created_at'] or "", i['adr']['updated_at'] or ""])
+                                    i['adr']['created_at'] or "", i['adr']['updated_at'] or "", ad_rules_updated_at])
             if new_last_update_at < __last_update_at:
                 new_last_update_at = __last_update_at
 
+        write_ad_rule_sync_history.apply_async(args=(result, oneagent_id, oneagent_name, datetime.datetime.now()),
+                                               queue=CMDB_QUEUE)
         if not last_update_at or new_last_update_at > last_update_at:
             return result, new_last_update_at
         else:
@@ -213,7 +326,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
             agent_id = agent_id.strip()
             q = "op_duty:{0},-rd_duty:{0},oneagent_id:{1}"
 
-            s = search(q.format(g.user.username, agent_id.strip()))
+            s = ci_search(q.format(current_user.username, agent_id.strip()))
             try:
                 response, _, _, _, _, _ = s.search()
                 if response:
@@ -222,7 +335,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
                 current_app.logger.warning(e)
                 return abort(400, str(e))
 
-            s = search(q.format(g.user.nickname, agent_id.strip()))
+            s = ci_search(q.format(current_user.nickname, agent_id.strip()))
             try:
                 response, _, _, _, _, _ = s.search()
                 if response:
@@ -236,41 +349,59 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
             if query_expr.startswith('q='):
                 query_expr = query_expr[2:]
 
-            s = search(query_expr, count=1000000)
+            s = ci_search(query_expr, count=1000000)
             try:
                 response, _, _, _, _, _ = s.search()
                 for i in response:
-                    if g.user.username not in (i.get('rd_duty') or []) and g.user.username not in \
+                    if (current_user.username not in (i.get('rd_duty') or []) and
-                            (i.get('op_duty') or []) and g.user.nickname not in (i.get('rd_duty') or []) and \
+                            current_user.username not in (i.get('op_duty') or []) and
-                            g.user.nickname not in (i.get('op_duty') or []):
+                            current_user.nickname not in (i.get('rd_duty') or []) and
+                            current_user.nickname not in (i.get('op_duty') or [])):
                         return abort(403, ErrFormat.adt_target_expr_no_permission.format(
                             i.get("{}_name".format(i.get('ci_type')))))
             except SearchError as e:
                 current_app.logger.warning(e)
                 return abort(400, str(e))
 
-    def _can_add(self, **kwargs):
+    @staticmethod
-        self.cls.get_by(type_id=kwargs['type_id'], adr_id=kwargs.get('adr_id') or None) and abort(
+    def _can_add(**kwargs):
-            400, ErrFormat.ad_duplicate)
-
-        # self.__valid_exec_target(kwargs.get('agent_id'), kwargs.get('query_expr'))
 
         if kwargs.get('adr_id'):
             adr = AutoDiscoveryRule.get_by_id(kwargs['adr_id']) or abort(
                 404, ErrFormat.adr_not_found.format("id={}".format(kwargs['adr_id'])))
-            if not adr.is_plugin:
+            if adr.type == AutoDiscoveryType.HTTP:
-                other = self.cls.get_by(adr_id=adr.id, first=True, to_dict=False)
+                kwargs.setdefault('extra_option', dict())
-                if other:
+                en_name = None
-                    ci_type = CITypeCache.get(other.type_id)
+                for i in DEFAULT_INNER:
-                    return abort(400, ErrFormat.adr_default_ref_once.format(ci_type.alias))
+                    if i['name'] == adr.name:
+                        en_name = i['en']
+                        break
+                if en_name and kwargs['extra_option'].get('category'):
+                    for item in CLOUD_MAP[en_name]:
+                        if item["collect_key_map"].get(kwargs['extra_option']['category']):
+                            kwargs["extra_option"]["collect_key"] = item["collect_key_map"][
+                                kwargs['extra_option']['category']]
+                            kwargs["extra_option"]["provider"] = en_name
+                            break
+
+            if adr.type == AutoDiscoveryType.COMPONENTS and kwargs.get('extra_option'):
+                for i in DEFAULT_INNER:
+                    if i['name'] == adr.name:
+                        kwargs['extra_option']['collect_key'] = i['option'].get('collect_key')
+                        break
 
         if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
             kwargs = check_plugin_script(**kwargs)
 
-        if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'):
+        encrypt_account(kwargs.get('extra_option'))
-            kwargs['extra_option']['secret'] = AESCrypto.encrypt(kwargs['extra_option']['secret'])
 
-        kwargs['uid'] = g.user.uid
+        ci_type = CITypeCache.get(kwargs['type_id'])
+        unique = AttributeCache.get(ci_type.unique_id)
+        if unique and unique.name not in (kwargs.get('attributes') or {}).values():
+            current_app.logger.warning((unique.name, kwargs.get('attributes'), ci_type.alias))
+            return abort(400, ErrFormat.ad_not_unique_key.format(unique.name))
+
+        kwargs['uid'] = current_user.uid
 
         return kwargs
 
@@ -278,10 +409,43 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
         existed = self.cls.get_by_id(kwargs['_id']) or abort(
             404, ErrFormat.ad_not_found.format("id={}".format(kwargs['_id'])))
 
-        self.__valid_exec_target(kwargs.get('agent_id'), kwargs.get('query_expr'))
+        adr = AutoDiscoveryRule.get_by_id(existed.adr_id) or abort(
+            404, ErrFormat.adr_not_found.format("id={}".format(existed.adr_id)))
+        if adr.type == AutoDiscoveryType.HTTP:
+            kwargs.setdefault('extra_option', dict())
+            en_name = None
+            for i in DEFAULT_INNER:
+                if i['name'] == adr.name:
+                    en_name = i['en']
+                    break
+            if en_name and kwargs['extra_option'].get('category'):
+                for item in CLOUD_MAP[en_name]:
+                    if item["collect_key_map"].get(kwargs['extra_option']['category']):
+                        kwargs["extra_option"]["collect_key"] = item["collect_key_map"][
+                            kwargs['extra_option']['category']]
+                        kwargs["extra_option"]["provider"] = en_name
+                        break
+
+        if adr.type == AutoDiscoveryType.COMPONENTS and kwargs.get('extra_option'):
+            for i in DEFAULT_INNER:
+                if i['name'] == adr.name:
+                    kwargs['extra_option']['collect_key'] = i['option'].get('collect_key')
+                    break
+
+        if 'attributes' in kwargs:
+            self.__valid_exec_target(kwargs.get('agent_id'), kwargs.get('query_expr'))
+
+            ci_type = CITypeCache.get(existed.type_id)
+            unique = AttributeCache.get(ci_type.unique_id)
+            if unique and unique.name not in (kwargs.get('attributes') or {}).values():
+                current_app.logger.warning((unique.name, kwargs.get('attributes'), ci_type.alias))
+                return abort(400, ErrFormat.ad_not_unique_key.format(unique.name))
 
         if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'):
-            if g.user.uid != existed.uid:
+            if current_user.uid != existed.uid:
+                return abort(403, ErrFormat.adt_secret_no_permission)
+        if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('password'):
+            if current_user.uid != existed.uid:
                 return abort(403, ErrFormat.adt_secret_no_permission)
 
         return existed
@@ -291,10 +455,22 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
         if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
             kwargs = check_plugin_script(**kwargs)
 
-        if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'):
+        encrypt_account(kwargs.get('extra_option'))
-            kwargs['extra_option']['secret'] = AESCrypto.encrypt(kwargs['extra_option']['secret'])
 
-        return super(AutoDiscoveryCITypeCRUD, self).update(_id, filter_none=False, **kwargs)
+        inst = self._can_update(_id=_id, **kwargs)
+        if len(kwargs) == 1 and 'enabled' in kwargs:  # enable or disable
+            pass
+        elif inst.agent_id != kwargs.get('agent_id') or inst.query_expr != kwargs.get('query_expr'):
+            for item in AutoDiscoveryRuleSyncHistory.get_by(adt_id=inst.id, to_dict=False):
+                item.delete(commit=False)
+            db.session.commit()
+
+            SystemConfigManager.create_or_update("ad_rules_updated_at",
+                                                 dict(v=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
+
+        obj = inst.update(_id=_id, filter_none=False, **kwargs)
+
+        return obj
 
     def _can_delete(self, **kwargs):
         if AutoDiscoveryCICRUD.get_by_adt_id(kwargs['_id']):
@@ -305,6 +481,61 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
 
         return existed
 
+    def delete(self, _id):
+        inst = self._can_delete(_id=_id)
+
+        inst.soft_delete()
+
+        for item in AutoDiscoveryRuleSyncHistory.get_by(adt_id=inst.id, to_dict=False):
+            item.delete(commit=False)
+        db.session.commit()
+
+        attributes = self.get_ad_attributes(inst.type_id)
+        for item in AutoDiscoveryCITypeRelationCRUD.get_by_type_id(inst.type_id):
+            if item.ad_key not in attributes:
+                item.soft_delete()
+
+        return inst
+
+
+class AutoDiscoveryCITypeRelationCRUD(DBMixin):
+    cls = AutoDiscoveryCITypeRelation
+
+    @classmethod
+    def get_all(cls, type_ids=None):
+        res = cls.cls.get_by(to_dict=False)
+        return [i for i in res if type_ids is None or i.ad_type_id in type_ids]
+
+    @classmethod
+    def get_by_type_id(cls, type_id, to_dict=False):
+        return cls.cls.get_by(ad_type_id=type_id, to_dict=to_dict)
+
+    def upsert(self, ad_type_id, relations):
+        existed = self.cls.get_by(ad_type_id=ad_type_id, to_dict=False)
+        existed = {(i.ad_key, i.peer_type_id, i.peer_attr_id): i for i in existed}
+
+        new = []
+        for r in relations:
+            k = (r.get('ad_key'), r.get('peer_type_id'), r.get('peer_attr_id'))
+            if len(list(filter(lambda x: x, k))) == 3 and k not in existed:
+                self.cls.create(ad_type_id=ad_type_id, **r)
+
+            new.append(k)
+
+        for deleted in set(existed.keys()) - set(new):
+            existed[deleted].soft_delete()
+
+        return self.get_by_type_id(ad_type_id, to_dict=True)
+
+    def _can_add(self, **kwargs):
+        pass
+
+    def _can_update(self, **kwargs):
+        pass
+
+    def _can_delete(self, **kwargs):
+        pass
+
 
 class AutoDiscoveryCICRUD(DBMixin):
     cls = AutoDiscoveryCI
@@ -332,15 +563,14 @@ class AutoDiscoveryCICRUD(DBMixin):
 
     @staticmethod
     def get_attributes_by_type_id(type_id):
-        from api.lib.cmdb.cache import CITypeAttributesCache
+        from api.lib.cmdb.ci_type import CITypeAttributeManager
-        attributes = [i[1] for i in CITypeAttributesCache.get2(type_id) or []]
+        attributes = [i for i in CITypeAttributeManager.get_attributes_by_type_id(type_id) or []]
 
         attr_names = set()
         adts = AutoDiscoveryCITypeCRUD.get_by_type_id(type_id)
         for adt in adts:
             attr_names |= set((adt.attributes or {}).values())
-
+        return [attr for attr in attributes if attr['name'] in attr_names]
-        return [attr.to_dict() for attr in attributes if attr.name in attr_names]
 
     @classmethod
     def search(cls, page, page_size, fl=None, **kwargs):
@@ -393,16 +623,24 @@ class AutoDiscoveryCICRUD(DBMixin):
         changed = False
         if existed is not None:
             if existed.instance != kwargs['instance']:
+                instance = copy.deepcopy(existed.instance) or {}
+                instance.update(kwargs['instance'])
+                kwargs['instance'] = instance
                 existed.update(filter_none=False, **kwargs)
+                AutoDiscoveryExecHistoryCRUD().add(type_id=adt.type_id,
+                                                   stdout="update resource: {}".format(kwargs.get('unique_value')))
                 changed = True
         else:
             existed = self.cls.create(**kwargs)
+            AutoDiscoveryExecHistoryCRUD().add(type_id=adt.type_id,
+                                               stdout="add resource: {}".format(kwargs.get('unique_value')))
             changed = True
 
         if adt.auto_accept and changed:
             try:
                 self.accept(existed)
             except Exception as e:
+                current_app.logger.error(e)
                 return abort(400, str(e))
         elif changed:
             existed.update(is_accept=False, accept_time=None, accept_by=None, filter_none=False)
@@ -422,6 +660,13 @@ class AutoDiscoveryCICRUD(DBMixin):
 
         inst.delete()
 
+        adt = AutoDiscoveryCIType.get_by_id(inst.adt_id)
+        if adt:
+            adt.update(updated_at=datetime.datetime.now())
+
+        AutoDiscoveryExecHistoryCRUD().add(type_id=inst.type_id,
+                                           stdout="delete resource: {}".format(inst.unique_value))
+
         self._after_delete(inst)
 
         return inst
@@ -437,6 +682,13 @@ class AutoDiscoveryCICRUD(DBMixin):
             not is_app_admin("cmdb") and validate_permission(ci_type.name, ResourceTypeEnum.CI, PermEnum.DELETE, "cmdb")
 
         existed.delete()
+
+        adt = AutoDiscoveryCIType.get_by_id(existed.adt_id)
+        if adt:
+            adt.update(updated_at=datetime.datetime.now())
+
+        AutoDiscoveryExecHistoryCRUD().add(type_id=type_id,
+                                           stdout="delete resource: {}".format(unique_value))
         # TODO: delete ci
 
     @classmethod
@@ -447,52 +699,103 @@ class AutoDiscoveryCICRUD(DBMixin):
         adt = AutoDiscoveryCITypeCRUD.get_by_id(adc.adt_id) or abort(404, ErrFormat.adt_not_found)
 
         ci_id = None
-        if adt.attributes:
-            ci_dict = {adt.attributes[k]: v for k, v in adc.instance.items() if k in adt.attributes}
-            ci_id = CIManager.add(adc.type_id, is_auto_discovery=True, **ci_dict)
 
-        relation_adts = AutoDiscoveryCIType.get_by(type_id=adt.type_id, adr_id=None, to_dict=False)
+        ad_key2attr = adt.attributes or {}
-        for r_adt in relation_adts:
+        if ad_key2attr:
-            if r_adt.relation and ci_id is not None:
+            ci_dict = {ad_key2attr[k]: None if not v and isinstance(v, (list, dict)) else v
-                ad_key, cmdb_key = None, {}
+                       for k, v in adc.instance.items() if k in ad_key2attr}
-                for ad_key in r_adt.relation:
+            extra_option = adt.extra_option or {}
-                    cmdb_key = r_adt.relation[ad_key]
+            mapping, path_mapping = AutoDiscoveryHTTPManager.get_predefined_value_mapping(
-                query = "_type:{},{}:{}".format(cmdb_key.get('type_name'), cmdb_key.get('attr_name'),
+                extra_option.get('provider'), extra_option.get('category'))
-                                                adc.instance.get(ad_key))
+            if mapping:
-                s = search(query)
+                ci_dict = {k: (mapping.get(k) or {}).get(str(v), v) for k, v in ci_dict.items()}
-                try:
+            if path_mapping:
-                    response, _, _, _, _, _ = s.search()
+                ci_dict = {k: jsonpath.jsonpath(v, path_mapping[k]) if k in path_mapping else v
-                except SearchError as e:
+                           for k, v in ci_dict.items()}
-                    current_app.logger.warning(e)
+            ci_id = CIManager.add(adc.type_id, is_auto_discovery=True, _is_admin=True, **ci_dict)
-                    return abort(400, str(e))
+            AutoDiscoveryExecHistoryCRUD().add(type_id=adt.type_id,
+                                               stdout="accept resource: {}".format(adc.unique_value))
 
-                relation_ci_id = response and response[0]['_id']
+        build_relations_for_ad_accept.apply_async(args=(adc.to_dict(), ci_id, ad_key2attr), queue=CMDB_QUEUE)
-                if relation_ci_id:
-                    try:
-                        CIRelationManager.add(ci_id, relation_ci_id)
-                    except:
-                        try:
-                            CIRelationManager.add(relation_ci_id, ci_id)
-                        except:
-                            pass
 
-        adc.update(is_accept=True, accept_by=nickname or g.user.nickname, accept_time=datetime.datetime.now())
+        adc.update(is_accept=True,
+                   accept_by=nickname or current_user.nickname,
+                   accept_time=datetime.datetime.now(),
+                   ci_id=ci_id)
 
 
 class AutoDiscoveryHTTPManager(object):
     @staticmethod
     def get_categories(name):
-        return (ClOUD_MAP.get(name) or {}).get('categories') or []
+        categories = (CLOUD_MAP.get(name) or {}) or []
+        for item in copy.deepcopy(categories):
+            item.pop('map', None)
+            item.pop('collect_key_map', None)
 
-    @staticmethod
+        return categories
-    def get_attributes(name, category):
+
-        tpt = ((ClOUD_MAP.get(name) or {}).get('map') or {}).get(category)
+    def get_resources(self, name):
-        if tpt and os.path.exists(os.path.join(PWD, tpt)):
+        en_name = None
-            with open(os.path.join(PWD, tpt)) as f:
+        for i in DEFAULT_INNER:
-                return json.loads(f.read())
+            if i['name'] == name:
+                en_name = i['en']
+                break
+
+        if en_name:
+            categories = self.get_categories(en_name)
+
+            return [j for i in categories for j in i['items']]
 
         return []
 
+    @staticmethod
+    def get_attributes(provider, resource):
+        for item in (CLOUD_MAP.get(provider) or {}):
+            for _resource in (item.get('map') or {}):
+                if _resource == resource:
+                    tpt = item['map'][_resource]
+                    if isinstance(tpt, dict):
+                        tpt = tpt.get('template')
+                    if tpt and os.path.exists(os.path.join(PWD, tpt)):
+                        with open(os.path.join(PWD, tpt)) as f:
+                            return json.loads(f.read())
+
+        return []
+
+    @staticmethod
+    def get_mapping(provider, resource):
+        for item in (CLOUD_MAP.get(provider) or {}):
+            for _resource in (item.get('map') or {}):
+                if _resource == resource:
+                    mapping = item['map'][_resource]
+                    if not isinstance(mapping, dict):
+                        return {}
+                    name = mapping.get('mapping')
+                    mapping = AutoDiscoveryMappingCache.get(name)
+                    if isinstance(mapping, dict):
+                        return {mapping[key][provider]['key'].split('.')[0]: key for key in mapping if
+                                (mapping[key].get(provider) or {}).get('key')}
+
+        return {}
+
+    @staticmethod
+    def get_predefined_value_mapping(provider, resource):
+        for item in (CLOUD_MAP.get(provider) or {}):
+            for _resource in (item.get('map') or {}):
+                if _resource == resource:
+                    mapping = item['map'][_resource]
+                    if not isinstance(mapping, dict):
+                        return {}, {}
+                    name = mapping.get('mapping')
+                    mapping = AutoDiscoveryMappingCache.get(name)
+                    if isinstance(mapping, dict):
+                        return ({key: mapping[key][provider].get('map') for key in mapping if
+                                 mapping[key].get(provider, {}).get('map')},
+                                {key: mapping[key][provider]['key'].split('.', 1)[1] for key in mapping if
+                                 ((mapping[key].get(provider) or {}).get('key') or '').split('.')[1:]})
+
+        return {}, {}
+
 
 class AutoDiscoverySNMPManager(object):
 
@@ -503,3 +806,191 @@ class AutoDiscoverySNMPManager(object):
                 return json.loads(f.read())
 
         return []
+
+
+class AutoDiscoveryComponentsManager(object):
+
+    @staticmethod
+    def get_attributes(name):
+        if os.path.exists(os.path.join(PWD, "templates/{}.json".format(name))):
+            with open(os.path.join(PWD, "templates/{}.json".format(name))) as f:
+                return json.loads(f.read())
+
+        return []
+
+
+class AutoDiscoveryRuleSyncHistoryCRUD(DBMixin):
+    cls = AutoDiscoveryRuleSyncHistory
+
+    def _can_add(self, **kwargs):
+        pass
+
+    def _can_update(self, **kwargs):
+        pass
+
+    def _can_delete(self, **kwargs):
+        pass
+
+    def upsert(self, **kwargs):
+        existed = self.cls.get_by(adt_id=kwargs.get('adt_id'),
+                                  oneagent_id=kwargs.get('oneagent_id'),
+                                  oneagent_name=kwargs.get('oneagent_name'),
+                                  first=True,
+                                  to_dict=False)
+
+        if existed is not None:
+            existed.update(**kwargs)
+        else:
+            self.cls.create(**kwargs)
+
+
+class AutoDiscoveryExecHistoryCRUD(DBMixin):
+    cls = AutoDiscoveryExecHistory
+
+    def _can_add(self, **kwargs):
+        pass
+
+    def _can_update(self, **kwargs):
+        pass
+
+    def _can_delete(self, **kwargs):
+        pass
+
+
+class AutoDiscoveryCounterCRUD(DBMixin):
+    cls = AutoDiscoveryCounter
+
+    def get(self, type_id):
+        res = self.cls.get_by(type_id=type_id, first=True, to_dict=True)
+        if res is None:
+            return dict(rule_count=0, exec_target_count=0, instance_count=0, accept_count=0,
+                        this_month_count=0, this_week_count=0, last_month_count=0, last_week_count=0)
+
+        return res
+
+    def _can_add(self, **kwargs):
+        pass
+
+    def _can_update(self, **kwargs):
+        pass
+
+    def _can_delete(self, **kwargs):
+        pass
+
+
+def encrypt_account(config):
+    if isinstance(config, dict):
+        if config.get('secret'):
+            config['secret'] = AESCrypto.encrypt(config['secret'])
+        if config.get('password'):
+            config['password'] = AESCrypto.encrypt(config['password'])
+
+
+def decrypt_account(config, uid):
+    if isinstance(config, dict):
+        if config.get('password'):
+            if not (current_user.username in PRIVILEGED_USERS or current_user.uid == uid):
+                config.pop('password', None)
+            else:
+                try:
+                    config['password'] = AESCrypto.decrypt(config['password'])
+                except Exception as e:
+                    current_app.logger.error('decrypt account failed: {}'.format(e))
+
+        if config.get('secret'):
+            if not (current_user.username in PRIVILEGED_USERS or current_user.uid == uid):
+                config.pop('secret', None)
+            else:
+                try:
+                    config['secret'] = AESCrypto.decrypt(config['secret'])
+                except Exception as e:
+                    current_app.logger.error('decrypt account failed: {}'.format(e))
+
+
+class AutoDiscoveryAccountCRUD(DBMixin):
+    cls = AutoDiscoveryAccount
+
+    def get(self, adr_id):
+        res = self.cls.get_by(adr_id=adr_id, to_dict=True)
+
+        for i in res:
+            decrypt_account(i.get('config'), i['uid'])
+
+        return res
+
+    def get_config_by_id(self, _id):
+        res = self.cls.get_by_id(_id)
+        if not res:
+            return {}
+
+        config = res.to_dict().get('config') or {}
+
+        decrypt_account(config, res.uid)
+
+        return config
+
+    def _can_add(self, **kwargs):
+        encrypt_account(kwargs.get('config'))
+
+        kwargs['uid'] = current_user.uid
+
+        return kwargs
+
+    def upsert(self, adr_id, accounts):
+        existed_all = self.cls.get_by(adr_id=adr_id, to_dict=False)
+        account_names = {i['name'] for i in accounts}
+
+        name_changed = dict()
+        for account in accounts:
+            existed = None
+            if account.get('id'):
+                existed = self.cls.get_by_id(account.get('id'))
+                if existed is None:
+                    continue
+
+                account.pop('id')
+                name_changed[existed.name] = account.get('name')
+            else:
+                account = self._can_add(**account)
+
+            if existed is not None:
+                if current_user.uid == existed.uid:
+                    config = copy.deepcopy(existed.config) or {}
+                    config.update(account.get('config') or {})
+                    account['config'] = config
+                    existed.update(**account)
+            else:
+                self.cls.create(adr_id=adr_id, **account)
+
+        for item in existed_all:
+            if name_changed.get(item.name, item.name) not in account_names:
+                if current_user.uid == item.uid:
+                    item.soft_delete()
+
+    def _can_update(self, **kwargs):
+        existed = self.cls.get_by_id(kwargs['_id']) or abort(404, ErrFormat.not_found)
+
+        if isinstance(kwargs.get('config'), dict) and kwargs['config'].get('secret'):
+            if current_user.uid != existed.uid:
+                return abort(403, ErrFormat.adt_secret_no_permission)
+        if isinstance(kwargs.get('config'), dict) and kwargs['config'].get('password'):
+            if current_user.uid != existed.uid:
+                return abort(403, ErrFormat.adt_secret_no_permission)
+
+        return existed
+
+    def update(self, _id, **kwargs):
+
+        if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
+            kwargs = check_plugin_script(**kwargs)
+
+        encrypt_account(kwargs.get('config'))
+
+        inst = self._can_update(_id=_id, **kwargs)
+
+        obj = inst.update(_id=_id, filter_none=False, **kwargs)
+
+        return obj
+
+    def _can_delete(self, **kwargs):
+        pass

cmdb-api/api/lib/cmdb/auto_discovery/const.py

@@ -2,15 +2,38 @@
 
 from api.lib.cmdb.const import AutoDiscoveryType
 
-DEFAULT_HTTP = [
+PRIVILEGED_USERS = ("cmdb_agent", "worker", "admin")
-    dict(name="阿里云", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
+
-         option={'icon': {'name': 'caise-aliyun'}}),
+DEFAULT_INNER = [
-    dict(name="腾讯云", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
+    dict(name="阿里云", en="aliyun", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
-         option={'icon': {'name': 'caise-tengxunyun'}}),
+         option={'icon': {'name': 'caise-aliyun'}, "en": "aliyun"}),
-    dict(name="华为云", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
+    dict(name="腾讯云", en="tencentcloud", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
-         option={'icon': {'name': 'caise-huaweiyun'}}),
+         option={'icon': {'name': 'caise-tengxunyun'}, "en": "tencentcloud"}),
-    dict(name="AWS", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
+    dict(name="华为云", en="huaweicloud", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
-         option={'icon': {'name': 'caise-aws'}}),
+         option={'icon': {'name': 'caise-huaweiyun'}, "en": "huaweicloud"}),
+    dict(name="AWS", en="aws", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-aws'}, "en": "aws"}),
+
+    dict(name="VCenter", en="vcenter", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'cmdb-vcenter'}, "category": "private_cloud", "en": "vcenter"}),
+    dict(name="KVM", en="kvm", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'ops-KVM'}, "category": "private_cloud", "en": "kvm"}),
+
+
+    dict(name="Nginx", en="nginx", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-nginx'}, "en": "nginx", "collect_key": "nginx"}),
+    dict(name="Apache", en="apache", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-apache'}, "en": "apache", "collect_key": "apache"}),
+    dict(name="Tomcat", en="tomcat", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-tomcat'}, "en": "tomcat", "collect_key": "tomcat"}),
+    dict(name="MySQL", en="mysql", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-mySQL'}, "en": "mysql", "collect_key": "mysql"}),
+    dict(name="MSSQL", en="mssql", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+             option={'icon': {'name': 'caise-SQLServer'}, "en": "mssql", "collect_key": "sqlserver"}),
+    dict(name="Oracle", en="oracle", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-oracle'}, "en": "oracle", "collect_key": "oracle"}),
+    dict(name="Redis", en="redis", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-redis'}, "en": "redis", "collect_key": "redis"}),
 
     dict(name="交换机", type=AutoDiscoveryType.SNMP, is_inner=True, is_plugin=False,
          option={'icon': {'name': 'caise-jiaohuanji'}}),
@@ -22,32 +45,307 @@ DEFAULT_HTTP = [
          option={'icon': {'name': 'caise-dayinji'}}),
 ]
 
-ClOUD_MAP = {
+CLOUD_MAP = {
-    "aliyun": {
+    "aliyun": [
-        "categories": ["云服务器 ECS"],
+        {
-        "map": {
+            "category": "计算",
-            "云服务器 ECS": "templates/aliyun_ecs.json",
+            "items": ["云服务器 ECS", "云服务器 Disk"],
-        }
+            "map": {
-    },
+                "云服务器 ECS": {"template": "templates/aliyun_ecs.json", "mapping": "ecs"},
-
+                "云服务器 Disk": {"template": "templates/aliyun_ecs_disk.json", "mapping": "evs"},
-    "tencentcloud": {
+            },
-        "categories": ["云服务器 CVM"],
+            "collect_key_map": {
-        "map": {
+                "云服务器 ECS": "ali.ecs",
-            "云服务器 CVM": "templates/tencent_cvm.json",
+                "云服务器 Disk": "ali.ecs_disk",
-        }
+            },
-    },
+        },
-
+        {
-    "huaweicloud": {
+            "category": "网络与CDN",
-        "categories": ["云服务器 ECS"],
+            "items": [
-        "map": {
+                "内容分发CDN",
-            "云服务器 ECS": "templates/huaweicloud_ecs.json",
+                "负载均衡SLB",
-        }
+                "专有网络VPC",
-    },
+                "交换机Switch",
-
+            ],
-    "aws": {
+            "map": {
-        "categories": ["云服务器 EC2"],
+                "内容分发CDN": {"template": "templates/aliyun_cdn.json", "mapping": "CDN"},
-        "map": {
+                "负载均衡SLB": {"template": "templates/aliyun_slb.json", "mapping": "loadbalancer"},
-            "云服务器 EC2": "templates/aws_ec2.json",
+                "专有网络VPC": {"template": "templates/aliyun_vpc.json", "mapping": "vpc"},
-        }
+                "交换机Switch": {"template": "templates/aliyun_switch.json", "mapping": "vswitch"},
-    },
+            },
+            "collect_key_map": {
+                "内容分发CDN": "ali.cdn",
+                "负载均衡SLB": "ali.slb",
+                "专有网络VPC": "ali.vpc",
+                "交换机Switch": "ali.switch",
+            },
+        },
+        {
+            "category": "存储",
+            "items": ["块存储EBS", "对象存储OSS"],
+            "map": {
+                "块存储EBS": {"template": "templates/aliyun_ebs.json", "mapping": "evs"},
+                "对象存储OSS": {"template": "templates/aliyun_oss.json", "mapping": "objectStorage"},
+            },
+            "collect_key_map": {
+                "块存储EBS": "ali.ebs",
+                "对象存储OSS": "ali.oss",
+            },
+        },
+        {
+            "category": "数据库",
+            "items": ["云数据库RDS MySQL", "云数据库RDS PostgreSQL", "云数据库 Redis"],
+            "map": {
+                "云数据库RDS MySQL": {"template": "templates/aliyun_rds_mysql.json", "mapping": "mysql"},
+                "云数据库RDS PostgreSQL": {"template": "templates/aliyun_rds_postgre.json", "mapping": "postgresql"},
+                "云数据库 Redis": {"template": "templates/aliyun_redis.json", "mapping": "redis"},
+            },
+            "collect_key_map": {
+                "云数据库RDS MySQL": "ali.rds_mysql",
+                "云数据库RDS PostgreSQL": "ali.rds_postgre",
+                "云数据库 Redis": "ali.redis",
+            },
+        },
+    ],
+    "tencentcloud": [
+        {
+            "category": "计算",
+            "items": ["云服务器 CVM"],
+            "map": {
+                "云服务器 CVM": {"template": "templates/tencent_cvm.json", "mapping": "ecs"},
+            },
+            "collect_key_map": {
+                "云服务器 CVM": "tencent.cvm",
+            },
+        },
+        {
+            "category": "CDN与边缘",
+            "items": ["内容分发CDN"],
+            "map": {
+                "内容分发CDN": {"template": "templates/tencent_cdn.json", "mapping": "CDN"},
+            },
+            "collect_key_map": {
+                "内容分发CDN": "tencent.cdn",
+            },
+        },
+        {
+            "category": "网络",
+            "items": ["负载均衡CLB", "私有网络VPC", "子网"],
+            "map": {
+                "负载均衡CLB": {"template": "templates/tencent_clb.json", "mapping": "loadbalancer"},
+                "私有网络VPC": {"template": "templates/tencent_vpc.json", "mapping": "vpc"},
+                "子网": {"template": "templates/tencent_subnet.json", "mapping": "vswitch"},
+            },
+            "collect_key_map": {
+                "负载均衡CLB": "tencent.clb",
+                "私有网络VPC": "tencent.vpc",
+                "子网": "tencent.subnet",
+            },
+        },
+        {
+            "category": "存储",
+            "items": ["云硬盘CBS", "对象存储COS"],
+            "map": {
+                "云硬盘CBS": {"template": "templates/tencent_cbs.json", "mapping": "evs"},
+                "对象存储COS": {"template": "templates/tencent_cos.json", "mapping": "objectStorage"},
+            },
+            "collect_key_map": {
+                "云硬盘CBS": "tencent.cbs",
+                "对象存储COS": "tencent.cos",
+            },
+        },
+        {
+            "category": "数据库",
+            "items": ["云数据库 MySQL", "云数据库 PostgreSQL", "云数据库 Redis"],
+            "map": {
+                "云数据库 MySQL": {"template": "templates/tencent_rdb.json", "mapping": "mysql"},
+                "云数据库 PostgreSQL": {"template": "templates/tencent_postgres.json", "mapping": "postgresql"},
+                "云数据库 Redis": {"template": "templates/tencent_redis.json", "mapping": "redis"},
+            },
+            "collect_key_map": {
+                "云数据库 MySQL": "tencent.rdb",
+                "云数据库 PostgreSQL": "tencent.rds_postgres",
+                "云数据库 Redis": "tencent.redis",
+            },
+        },
+    ],
+    "huaweicloud": [
+        {
+            "category": "计算",
+            "items": ["云服务器 ECS"],
+            "map": {
+                "云服务器 ECS": {"template": "templates/huaweicloud_ecs.json", "mapping": "ecs"},
+            },
+            "collect_key_map": {
+                "云服务器 ECS": "huawei.ecs",
+            },
+        },
+        {
+            "category": "CDN与智能边缘",
+            "items": ["内容分发网络CDN"],
+            "map": {
+                "内容分发网络CDN": {"template": "templates/huawei_cdn.json", "mapping": "CDN"},
+            },
+            "collect_key_map": {
+                "内容分发网络CDN": "huawei.cdn",
+            },
+        },
+        {
+            "category": "网络",
+            "items": ["弹性负载均衡ELB", "虚拟私有云VPC", "子网"],
+            "map": {
+                "弹性负载均衡ELB": {"template": "templates/huawei_elb.json", "mapping": "loadbalancer"},
+                "虚拟私有云VPC": {"template": "templates/huawei_vpc.json", "mapping": "vpc"},
+                "子网": {"template": "templates/huawei_subnet.json", "mapping": "vswitch"},
+            },
+            "collect_key_map": {
+                "弹性负载均衡ELB": "huawei.elb",
+                "虚拟私有云VPC": "huawei.vpc",
+                "子网": "huawei.subnet",
+            },
+        },
+        {
+            "category": "存储",
+            "items": ["云硬盘EVS", "对象存储OBS"],
+            "map": {
+                "云硬盘EVS": {"template": "templates/huawei_evs.json", "mapping": "evs"},
+                "对象存储OBS": {"template": "templates/huawei_obs.json", "mapping": "objectStorage"},
+            },
+            "collect_key_map": {
+                "云硬盘EVS": "huawei.evs",
+                "对象存储OBS": "huawei.obs",
+            },
+        },
+        {
+            "category": "数据库",
+            "items": ["云数据库RDS MySQL", "云数据库RDS PostgreSQL"],
+            "map": {
+                "云数据库RDS MySQL": {"template": "templates/huawei_rds_mysql.json", "mapping": "mysql"},
+                "云数据库RDS PostgreSQL": {"template": "templates/huawei_rds_postgre.json", "mapping": "postgresql"},
+            },
+            "collect_key_map": {
+                "云数据库RDS MySQL": "huawei.rds_mysql",
+                "云数据库RDS PostgreSQL": "huawei.rds_postgre",
+            },
+        },
+        {
+            "category": "应用中间件",
+            "items": ["分布式缓存Redis"],
+            "map": {
+                "分布式缓存Redis": {"template": "templates/huawei_dcs.json", "mapping": "redis"},
+            },
+            "collect_key_map": {
+                "分布式缓存Redis": "huawei.dcs",
+            },
+        },
+    ],
+    "aws": [
+        {
+            "category": "计算",
+            "items": ["云服务器 EC2"],
+            "map": {
+                "云服务器 EC2": {"template": "templates/aws_ec2.json", "mapping": "ecs"},
+            },
+            "collect_key_map": {
+                "云服务器 EC2": "aws.ec2",
+            },
+        },
+        {"category": "网络与CDN", "items": [], "map": {}, "collect_key_map": {}},
+    ],
+    "vcenter": [
+        {
+            "category": "计算",
+            "items": [
+                "主机",
+                "虚拟机",
+                "主机集群"
+            ],
+            "map": {
+                "主机": "templates/vsphere_host.json",
+                "虚拟机": "templates/vsphere_vm.json",
+                "主机集群": "templates/vsphere_cluster.json",
+            },
+            "collect_key_map": {
+                "主机": "vsphere.host",
+                "虚拟机": "vsphere.vm",
+                "主机集群": "vsphere.cluster",
+            },
+        },
+        {
+            "category": "网络",
+            "items": [
+                "网络",
+                "标准交换机",
+                "分布式交换机",
+            ],
+            "map": {
+                "网络": "templates/vsphere_network.json",
+                "标准交换机": "templates/vsphere_standard_switch.json",
+                "分布式交换机": "templates/vsphere_distributed_switch.json",
+            },
+            "collect_key_map": {
+                "网络": "vsphere.network",
+                "标准交换机": "vsphere.standard_switch",
+                "分布式交换机": "vsphere.distributed_switch",
+            },
+        },
+        {
+            "category": "存储",
+            "items": ["数据存储", "数据存储集群"],
+            "map": {
+                "数据存储": "templates/vsphere_datastore.json",
+                "数据存储集群": "templates/vsphere_storage_pod.json",
+            },
+            "collect_key_map": {
+                "数据存储": "vsphere.datastore",
+                "数据存储集群": "vsphere.storage_pod",
+            },
+        },
+        {
+            "category": "其他",
+            "items": ["资源池", "数据中心", "文件夹"],
+            "map": {
+                "资源池": "templates/vsphere_pool.json",
+                "数据中心": "templates/vsphere_datacenter.json",
+                "文件夹": "templates/vsphere_folder.json",
+            },
+            "collect_key_map": {
+                "资源池": "vsphere.pool",
+                "数据中心": "vsphere.datacenter",
+                "文件夹": "vsphere.folder",
+            },
+        },
+    ],
+    "kvm": [
+        {
+            "category": "计算",
+            "items": ["虚拟机"],
+            "map": {
+                "虚拟机": "templates/kvm_vm.json",
+            },
+            "collect_key_map": {
+                "虚拟机": "kvm.vm",
+            },
+        },
+        {
+            "category": "存储",
+            "items": ["存储"],
+            "map": {
+                "存储": "templates/kvm_storage.json",
+            },
+            "collect_key_map": {
+                "存储": "kvm.storage",
+            },
+        },
+        {
+            "category": "network",
+            "items": ["网络"],
+            "map": {
+                "网络": "templates/kvm_network.json",
+            },
+            "collect_key_map": {
+                "网络": "kvm.network",
+            },
+        },
+    ],
 }

cmdb-api/api/lib/cmdb/auto_discovery/templates/aws_ec2.json

@@ -1,427 +1,344 @@
-[
+[
-    {
+    {
-        "name": "amiLaunchIndex",
+        "name": "amiLaunchIndex",
-        "type": "整数",
+        "type": "Integer",
-        "desc": "The AMI launch index, which can be used to find this instance in the launch group.",
+        "desc": "The AMI launch index, which can be used to find this instance in the launch group.",
-        "example": "0"
+        "example": ""
-    },
+    },
-    {
+    {
-        "name": "architecture",
+        "name": "architecture",
-        "type": "文本",
+        "type": "String",
-        "desc": "The architecture of the image.",
+        "desc": "The architecture of the image.",
-        "example": "x86_64"
+        "example": "i386"
-    },
+    },
-    {
+    {
-        "name": "blockDeviceMapping",
+        "name": "blockDeviceMapping",
-        "type": "json",
+        "type": "Array of InstanceBlockDeviceMapping objects",
-        "desc": "Any block device mapping entries for the instance.",
+        "desc": "Any block device mapping entries for the instance.",
-        "example": {
+        "example": ""
-            "item": {
+    },
-                "deviceName": "/dev/xvda",
+    {
-                "ebs": {
+        "name": "bootMode",
-                    "volumeId": "vol-1234567890abcdef0",
+        "type": "String",
-                    "status": "attached",
+        "desc": "The boot mode that was specified by the AMI. If the value is uefi-preferred, the AMI supports both UEFI and Legacy BIOS. The currentInstanceBootMode parameter is the boot mode that is used to boot the instance at launch or start. For more information, see Boot modes in the Amazon EC2 User Guide.",
-                    "attachTime": "2015-12-22T10:44:09.000Z",
+        "example": "legacy-bios"
-                    "deleteOnTermination": "true"
+    },
-                }
+    {
-            }
+        "name": "capacityReservationId",
-        }
+        "type": "String",
-    },
+        "desc": "The ID of the Capacity Reservation.",
-    {
+        "example": ""
-        "name": "bootMode",
+    },
-        "type": "文本",
+    {
-        "desc": "The boot mode that was specified by the AMI. If the value is uefi-preferred, the AMI supports both UEFI and Legacy BIOS. The currentInstanceBootMode parameter is the boot mode that is used to boot the instance at launch or start.",
+        "name": "capacityReservationSpecification",
-        "example": null
+        "type": "CapacityReservationSpecificationResponse object",
-    },
+        "desc": "Information about the Capacity Reservation targeting option.",
-    {
+        "example": ""
-        "name": "capacityReservationId",
+    },
-        "type": "文本",
+    {
-        "desc": "The ID of the Capacity Reservation.",
+        "name": "clientToken",
-        "example": null
+        "type": "String",
-    },
+        "desc": "The idempotency token you provided when you launched the instance, if applicable.",
-    {
+        "example": ""
-        "name": "capacityReservationSpecification",
+    },
-        "type": "json",
+    {
-        "desc": "Information about the Capacity Reservation targeting option.",
+        "name": "cpuOptions",
-        "example": null
+        "type": "CpuOptions object",
-    },
+        "desc": "The CPU options for the instance.",
-    {
+        "example": ""
-        "name": "clientToken",
+    },
-        "type": "文本",
+    {
-        "desc": "The idempotency token you provided when you launched the instance, if applicable.",
+        "name": "currentInstanceBootMode",
-        "example": "xMcwG14507example"
+        "type": "String",
-    },
+        "desc": "The boot mode that is used to boot the instance at launch or start. For more information, see Boot modes in the Amazon EC2 User Guide.",
-    {
+        "example": "legacy-bios"
-        "name": "cpuOptions",
+    },
-        "type": "json",
+    {
-        "desc": "The CPU options for the instance.",
+        "name": "dnsName",
-        "example": {
+        "type": "String",
-            "coreCount": "1",
+        "desc": "[IPv4 only] The public DNS name assigned to the instance. This name is not available until the instance enters the running state. This name is only available if you've enabled DNS hostnames for your VPC.",
-            "threadsPerCore": "1"
+        "example": ""
-        }
+    },
-    },
+    {
-    {
+        "name": "ebsOptimized",
-        "name": "currentInstanceBootMode",
+        "type": "Boolean",
-        "type": "文本",
+        "desc": "Indicates whether the instance is optimized for Amazon EBS I/O. This optimization provides dedicated throughput to Amazon EBS and an optimized configuration stack to provide optimal I/O performance. This optimization isn't available with all instance types. Additional usage charges apply when using an EBS Optimized instance.",
-        "desc": "The boot mode that is used to boot the instance at launch or start. For more information, see Boot modes in the Amazon EC2 User Guide.",
+        "example": ""
-        "example": null
+    },
-    },
+    {
-    {
+        "name": "elasticGpuAssociationSet",
-        "name": "dnsName",
+        "type": "Array of ElasticGpuAssociation objects",
-        "type": "文本",
+        "desc": "The Elastic GPU associated with the instance.",
-        "desc": "[IPv4 only] The public DNS name assigned to the instance. This name is not available until the instance enters the running state. This name is only available if you've enabled DNS hostnames for your VPC.",
+        "example": ""
-        "example": "ec2-54-194-252-215.eu-west-1.compute.amazonaws.com"
+    },
-    },
+    {
-    {
+        "name": "elasticInferenceAcceleratorAssociationSet",
-        "name": "ebsOptimized",
+        "type": "Array of ElasticInferenceAcceleratorAssociation objects",
-        "type": "Boolean",
+        "desc": "The elastic inference accelerator associated with the instance.",
-        "desc": "Indicates whether the instance is optimized for Amazon EBS I/O. This optimization provides dedicated throughput to Amazon EBS and an optimized configuration stack to provide optimal I/O performance. This optimization isn't available with all instance types. Additional usage charges apply when using an EBS Optimized instance.",
+        "example": ""
-        "example": "false"
+    },
-    },
+    {
-    {
+        "name": "enaSupport",
-        "name": "elasticGpuAssociationSet",
+        "type": "Boolean",
-        "type": "json",
+        "desc": "Specifies whether enhanced networking with ENA is enabled.",
-        "desc": "The Elastic GPU associated with the instance.",
+        "example": ""
-        "example": null
+    },
-    },
+    {
-    {
+        "name": "enclaveOptions",
-        "name": "elasticInferenceAcceleratorAssociationSet",
+        "type": "EnclaveOptions object",
-        "type": "json",
+        "desc": "Indicates whether the instance is enabled for AWS Nitro Enclaves.",
-        "desc": "The elastic inference accelerator associated with the instance.",
+        "example": ""
-        "example": null
+    },
-    },
+    {
-    {
+        "name": "groupSet",
-        "name": "enaSupport",
+        "type": "Array of GroupIdentifier objects",
-        "type": "Boolean",
+        "desc": "The security groups for the instance.",
-        "desc": "Specifies whether enhanced networking with ENA is enabled.",
+        "example": ""
-        "example": null
+    },
-    },
+    {
-    {
+        "name": "hibernationOptions",
-        "name": "enclaveOptions",
+        "type": "HibernationOptions object",
-        "type": "json",
+        "desc": "Indicates whether the instance is enabled for hibernation.",
-        "desc": "Indicates whether the instance is enabled for AWS Nitro Enclaves.",
+        "example": ""
-        "example": null
+    },
-    },
+    {
-    {
+        "name": "hypervisor",
-        "name": "groupSet",
+        "type": "String",
-        "type": "json",
+        "desc": "The hypervisor type of the instance. The value xen is used for both Xen and Nitro hypervisors.",
-        "desc": "The security groups for the instance.",
+        "example": "ovm"
-        "example": {
+    },
-            "item": {
+    {
-                "groupId": "sg-e4076980",
+        "name": "iamInstanceProfile",
-                "groupName": "SecurityGroup1"
+        "type": "IamInstanceProfile object",
-            }
+        "desc": "The IAM instance profile associated with the instance, if applicable.",
-        }
+        "example": ""
-    },
+    },
-    {
+    {
-        "name": "hibernationOptions",
+        "name": "imageId",
-        "type": "json",
+        "type": "String",
-        "desc": "Indicates whether the instance is enabled for hibernation.",
+        "desc": "The ID of the AMI used to launch the instance.",
-        "example": null
+        "example": ""
-    },
+    },
-    {
+    {
-        "name": "hypervisor",
+        "name": "instanceId",
-        "type": "文本",
+        "type": "String",
-        "desc": "The hypervisor type of the instance. The value xen is used for both Xen and Nitro hypervisors.",
+        "desc": "The ID of the instance.",
-        "example": "xen"
+        "example": ""
-    },
+    },
-    {
+    {
-        "name": "iamInstanceProfile",
+        "name": "instanceLifecycle",
-        "type": "json",
+        "type": "String",
-        "desc": "The IAM instance profile associated with the instance, if applicable.",
+        "desc": "Indicates whether this is a Spot Instance or a Scheduled Instance.",
-        "example": {
+        "example": "spot"
-            "arn": "arn:aws:iam::123456789012:instance-profile/AdminRole",
+    },
-            "id": "ABCAJEDNCAA64SSD123AB"
+    {
-        }
+        "name": "instanceState",
-    },
+        "type": "InstanceState object",
-    {
+        "desc": "The current state of the instance.",
-        "name": "imageId",
+        "example": ""
-        "type": "文本",
+    },
-        "desc": "The ID of the AMI used to launch the instance.",
+    {
-        "example": "ami-bff32ccc"
+        "name": "instanceType",
-    },
+        "type": "String",
-    {
+        "desc": "The instance type.",
-        "name": "instanceId",
+        "example": "a1.medium"
-        "type": "文本",
+    },
-        "desc": "The ID of the instance.",
+    {
-        "example": "i-1234567890abcdef0"
+        "name": "ipAddress",
-    },
+        "type": "String",
-    {
+        "desc": "The public IPv4 address, or the Carrier IP address assigned to the instance, if applicable. A Carrier IP address only applies to an instance launched in a subnet associated with a Wavelength Zone.",
-        "name": "instanceLifecycle",
+        "example": "Required: No"
-        "type": "文本",
+    },
-        "desc": "Indicates whether this is a Spot Instance or a Scheduled Instance.",
+    {
-        "example": null
+        "name": "ipv6Address",
-    },
+        "type": "String",
-    {
+        "desc": "The IPv6 address assigned to the instance.",
-        "name": "instanceState",
+        "example": ""
-        "type": "json",
+    },
-        "desc": "The current state of the instance.",
+    {
-        "example": {
+        "name": "kernelId",
-            "code": "16",
+        "type": "String",
-            "name": "running"
+        "desc": "The kernel associated with this instance, if applicable.",
-        }
+        "example": ""
-    },
+    },
-    {
+    {
-        "name": "instanceType",
+        "name": "keyName",
-        "type": "文本",
+        "type": "String",
-        "desc": "The instance type.",
+        "desc": "The name of the key pair, if this instance was launched with an associated key pair.",
-        "example": "t2.micro"
+        "example": ""
-    },
+    },
-    {
+    {
-        "name": "ipAddress",
+        "name": "launchTime",
-        "type": "文本",
+        "type": "Timestamp",
-        "desc": "The public IPv4 address, or the Carrier IP address assigned to the instance, if applicable.",
+        "desc": "The time the instance was launched.",
-        "example": "54.194.252.215"
+        "example": ""
-    },
+    },
-    {
+    {
-        "name": "ipv6Address",
+        "name": "licenseSet",
-        "type": "文本",
+        "type": "Array of LicenseConfiguration objects",
-        "desc": "The IPv6 address assigned to the instance.",
+        "desc": "The license configurations for the instance.",
-        "example": null
+        "example": ""
-    },
+    },
-    {
+    {
-        "name": "kernelId",
+        "name": "maintenanceOptions",
-        "type": "文本",
+        "type": "InstanceMaintenanceOptions object",
-        "desc": "The kernel associated with this instance, if applicable.",
+        "desc": "Provides information on the recovery and maintenance options of your instance.",
-        "example": null
+        "example": ""
-    },
+    },
-    {
+    {
-        "name": "keyName",
+        "name": "metadataOptions",
-        "type": "文本",
+        "type": "InstanceMetadataOptionsResponse object",
-        "desc": "The name of the key pair, if this instance was launched with an associated key pair.",
+        "desc": "The metadata options for the instance.",
-        "example": "my_keypair"
+        "example": ""
-    },
+    },
-    {
+    {
-        "name": "launchTime",
+        "name": "monitoring",
-        "type": "Time",
+        "type": "Monitoring object",
-        "desc": "The time the instance was launched.",
+        "desc": "The monitoring for the instance.",
-        "example": "2018-05-08T16:46:19.000Z"
+        "example": ""
-    },
+    },
-    {
+    {
-        "name": "licenseSet",
+        "name": "networkInterfaceSet",
-        "type": "json",
+        "type": "Array of InstanceNetworkInterface objects",
-        "desc": "The license configurations for the instance.",
+        "desc": "The network interfaces for the instance.",
-        "example": null
+        "example": ""
-    },
+    },
-    {
+    {
-        "name": "maintenanceOptions",
+        "name": "outpostArn",
-        "type": "json",
+        "type": "String",
-        "desc": "Provides information on the recovery and maintenance options of your instance.",
+        "desc": "The Amazon Resource Name (ARN) of the Outpost.",
-        "example": null
+        "example": ""
-    },
+    },
-    {
+    {
-        "name": "metadataOptions",
+        "name": "placement",
-        "type": "json",
+        "type": "Placement object",
-        "desc": "The metadata options for the instance.",
+        "desc": "The location where the instance launched, if applicable.",
-        "example": null
+        "example": ""
-    },
+    },
-    {
+    {
-        "name": "monitoring",
+        "name": "platform",
-        "type": "json",
+        "type": "String",
-        "desc": "The monitoring for the instance.",
+        "desc": "The platform. This value is windows for Windows instances; otherwise, it is empty.",
-        "example": {
+        "example": "windows"
-            "state": "disabled"
+    },
-        }
+    {
-    },
+        "name": "platformDetails",
-    {
+        "type": "String",
-        "name": "networkInterfaceSet",
+        "desc": "The platform details value for the instance. For more information, see AMI billing information fields in the Amazon EC2 User Guide.",
-        "type": "json",
+        "example": ""
-        "desc": "The network interfaces for the instance.",
+    },
-        "example": {
+    {
-            "item": {
+        "name": "privateDnsName",
-                "networkInterfaceId": "eni-551ba033",
+        "type": "String",
-                "subnetId": "subnet-56f5f633",
+        "desc": "[IPv4 only] The private DNS hostname name assigned to the instance. This DNS hostname can only be used inside the Amazon EC2 network. This name is not available until the instance enters the running state. The Amazon-provided DNS server resolves Amazon-provided private DNS hostnames if you've enabled DNS resolution and DNS hostnames in your VPC. If you are not using the Amazon-provided DNS server in your VPC, your custom domain name servers must resolve the hostname as appropriate.",
-                "vpcId": "vpc-11112222",
+        "example": "Required: No"
-                "description": "Primary network interface",
+    },
-                "ownerId": "123456789012",
+    {
-                "status": "in-use",
+        "name": "privateDnsNameOptions",
-                "macAddress": "02:dd:2c:5e:01:69",
+        "type": "PrivateDnsNameOptionsResponse object",
-                "privateIpAddress": "192.168.1.88",
+        "desc": "The options for the instance hostname.",
-                "privateDnsName": "ip-192-168-1-88.eu-west-1.compute.internal",
+        "example": ""
-                "sourceDestCheck": "true",
+    },
-                "groupSet": {
+    {
-                    "item": {
+        "name": "privateIpAddress",
-                        "groupId": "sg-e4076980",
+        "type": "String",
-                        "groupName": "SecurityGroup1"
+        "desc": "The private IPv4 address assigned to the instance.",
-                    }
+        "example": ""
-                },
+    },
-                "attachment": {
+    {
-                    "attachmentId": "eni-attach-39697adc",
+        "name": "productCodes",
-                    "deviceIndex": "0",
+        "type": "Array of ProductCode objects",
-                    "status": "attached",
+        "desc": "The product codes attached to this instance, if applicable.",
-                    "attachTime": "2018-05-08T16:46:19.000Z",
+        "example": ""
-                    "deleteOnTermination": "true"
+    },
-                },
+    {
-                "association": {
+        "name": "ramdiskId",
-                    "publicIp": "54.194.252.215",
+        "type": "String",
-                    "publicDnsName": "ec2-54-194-252-215.eu-west-1.compute.amazonaws.com",
+        "desc": "The RAM disk associated with this instance, if applicable.",
-                    "ipOwnerId": "amazon"
+        "example": ""
-                },
+    },
-                "privateIpAddressesSet": {
+    {
-                    "item": {
+        "name": "reason",
-                        "privateIpAddress": "192.168.1.88",
+        "type": "String",
-                        "privateDnsName": "ip-192-168-1-88.eu-west-1.compute.internal",
+        "desc": "The reason for the most recent state transition. This might be an empty string.",
-                        "primary": "true",
+        "example": ""
-                        "association": {
+    },
-                            "publicIp": "54.194.252.215",
+    {
-                            "publicDnsName": "ec2-54-194-252-215.eu-west-1.compute.amazonaws.com",
+        "name": "rootDeviceName",
-                            "ipOwnerId": "amazon"
+        "type": "String",
-                        }
+        "desc": "The device name of the root device volume (for example, /dev/sda1).",
-                    }
+        "example": ""
-                },
+    },
-                "ipv6AddressesSet": {
+    {
-                    "item": {
+        "name": "rootDeviceType",
-                        "ipv6Address": "2001:db8:1234:1a2b::123"
+        "type": "String",
-                    }
+        "desc": "The root device type used by the AMI. The AMI can use an EBS volume or an instance store volume.",
-                }
+        "example": "ebs"
-            }
+    },
-        }
+    {
-    },
+        "name": "sourceDestCheck",
-    {
+        "type": "Boolean",
-        "name": "outpostArn",
+        "desc": "Indicates whether source/destination checking is enabled.",
-        "type": "文本",
+        "example": ""
-        "desc": "The Amazon Resource Name (ARN) of the Outpost.",
+    },
-        "example": null
+    {
-    },
+        "name": "spotInstanceRequestId",
-    {
+        "type": "String",
-        "name": "placement",
+        "desc": "If the request is a Spot Instance request, the ID of the request.",
-        "type": "json",
+        "example": ""
-        "desc": "The location where the instance launched, if applicable.",
+    },
-        "example": {
+    {
-            "availabilityZone": "eu-west-1c",
+        "name": "sriovNetSupport",
-            "groupName": null,
+        "type": "String",
-            "tenancy": "default"
+        "desc": "Specifies whether enhanced networking with the Intel 82599 Virtual Function interface is enabled.",
-        }
+        "example": ""
-    },
+    },
-    {
+    {
-        "name": "platform",
+        "name": "stateReason",
-        "type": "文本",
+        "type": "StateReason object",
-        "desc": "The value is Windows for Windows instances; otherwise blank.",
+        "desc": "The reason for the most recent state transition.",
-        "example": null
+        "example": ""
-    },
+    },
-    {
+    {
-        "name": "platformDetails",
+        "name": "subnetId",
-        "type": "文本",
+        "type": "String",
-        "desc": "The platform details value for the instance. For more information, see AMI billing information fields in the Amazon EC2 User Guide.",
+        "desc": "The ID of the subnet in which the instance is running.",
-        "example": null
+        "example": ""
-    },
+    },
-    {
+    {
-        "name": "privateDnsName",
+        "name": "tagSet",
-        "type": "文本",
+        "type": "Array of Tag objects",
-        "desc": "[IPv4 only] The private DNS hostname name assigned to the instance. This DNS hostname can only be used inside the Amazon EC2 network. This name is not available until the instance enters the running state.",
+        "desc": "Any tags assigned to the instance.",
-        "example": "ip-192-168-1-88.eu-west-1.compute.internal"
+        "example": ""
-    },
+    },
-    {
+    {
-        "name": "privateDnsNameOptions",
+        "name": "tpmSupport",
-        "type": "json",
+        "type": "String",
-        "desc": "The options for the instance hostname.",
+        "desc": "If the instance is configured for NitroTPM support, the value is v2.0. For more information, see NitroTPM in the Amazon EC2 User Guide.",
-        "example": null
+        "example": ""
-    },
+    },
-    {
+    {
-        "name": "privateIpAddress",
+        "name": "usageOperation",
-        "type": "文本",
+        "type": "String",
-        "desc": "The private IPv4 address assigned to the instance.",
+        "desc": "The usage operation value for the instance. For more information, see AMI billing information fields in the Amazon EC2 User Guide.",
-        "example": "192.168.1.88"
+        "example": ""
-    },
+    },
-    {
+    {
-        "name": "productCodes",
+        "name": "usageOperationUpdateTime",
-        "type": "json",
+        "type": "Timestamp",
-        "desc": "The product codes attached to this instance, if applicable.",
+        "desc": "The time that the usage operation was last updated.",
-        "example": null
+        "example": ""
-    },
+    },
-    {
+    {
-        "name": "ramdiskId",
+        "name": "virtualizationType",
-        "type": "文本",
+        "type": "String",
-        "desc": "The RAM disk associated with this instance, if applicable.",
+        "desc": "The virtualization type of the instance.",
-        "example": null
+        "example": "hvm"
-    },
+    },
-    {
+    {
-        "name": "reason",
+        "name": "vpcId",
-        "type": "文本",
+        "type": "String",
-        "desc": "The reason for the most recent state transition. This might be an empty string.",
+        "desc": "The ID of the VPC in which the instance is running.",
-        "example": null
+        "example": ""
-    },
+    }
-    {
-        "name": "rootDeviceName",
-        "type": "文本",
-        "desc": "The device name of the root device volume (for example, /dev/sda1).",
-        "example": "/dev/xvda"
-    },
-    {
-        "name": "rootDeviceType",
-        "type": "文本",
-        "desc": "The root device type used by the AMI. The AMI can use an EBS volume or an instance store volume.",
-        "example": "ebs"
-    },
-    {
-        "name": "sourceDestCheck",
-        "type": "Boolean",
-        "desc": "Indicates whether source/destination checking is enabled.",
-        "example": "true"
-    },
-    {
-        "name": "spotInstanceRequestId",
-        "type": "文本",
-        "desc": "If the request is a Spot Instance request, the ID of the request.",
-        "example": null
-    },
-    {
-        "name": "sriovNetSupport",
-        "type": "文本",
-        "desc": "Specifies whether enhanced networking with the Intel 82599 Virtual Function interface is enabled.",
-        "example": null
-    },
-    {
-        "name": "stateReason",
-        "type": "json",
-        "desc": "The reason for the most recent state transition.",
-        "example": null
-    },
-    {
-        "name": "subnetId",
-        "type": "文本",
-        "desc": "The ID of the subnet in which the instance is running.",
-        "example": "subnet-56f5f633"
-    },
-    {
-        "name": "tagSet",
-        "type": "json",
-        "desc": "Any tags assigned to the instance.",
-        "example": {
-            "item": {
-                "key": "Name",
-                "value": "Server_1"
-            }
-        }
-    },
-    {
-        "name": "tpmSupport",
-        "type": "文本",
-        "desc": "If the instance is configured for NitroTPM support, the value is v2.0. For more information, see NitroTPM in the Amazon EC2 User Guide.",
-        "example": null
-    },
-    {
-        "name": "usageOperation",
-        "type": "文本",
-        "desc": "The usage operation value for the instance. For more information, see AMI billing information fields in the Amazon EC2 User Guide.",
-        "example": null
-    },
-    {
-        "name": "usageOperationUpdateTime",
-        "type": "Time",
-        "desc": "The time that the usage operation was last updated.",
-        "example": null
-    },
-    {
-        "name": "virtualizationType",
-        "type": "文本",
-        "desc": "The virtualization type of the instance.",
-        "example": "hvm"
-    },
-    {
-        "name": "vpcId",
-        "type": "文本",
-        "desc": "The ID of the VPC in which the instance is running.",
-        "example": "vpc-11112222"
-    }
 ]

cmdb-api/api/lib/cmdb/auto_discovery/templates/huaweicloud_ecs.json

@@ -1,292 +1,284 @@
-[
+[
-  {
+    {
-    "name": "status",
+        "name": "status",
-    "type": "文本",
+        "type": "string",
-    "example": "ACTIVE",
+        "desc": "弹性云服务器状态。\n\n取值范围:\n\nACTIVE、BUILD、DELETED、ERROR、HARD_REBOOT、MIGRATING、PAUSED、REBOOT、REBUILD、RESIZE、REVERT_RESIZE、SHUTOFF、SHELVED、SHELVED_OFFLOADED、SOFT_DELETED、SUSPENDED、VERIFY_RESIZE\n\n弹性云服务器状态说明请参考[云服务器状态](https://support.huaweicloud.com/api-ecs/ecs_08_0002.html)",
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u72b6\u6001\u3002\n\n\u53d6\u503c\u8303\u56f4:\n\nACTIVE\u3001BUILD\u3001DELETED\u3001ERROR\u3001HARD_REBOOT\u3001MIGRATING\u3001PAUSED\u3001REBOOT\u3001REBUILD\u3001RESIZE\u3001REVERT_RESIZE\u3001SHUTOFF\u3001SHELVED\u3001SHELVED_OFFLOADED\u3001SOFT_DELETED\u3001SUSPENDED\u3001VERIFY_RESIZE\n\n\u5f39\u6027\u4e91\u670d\u52a1\u5668\u72b6\u6001\u8bf4\u660e\u8bf7\u53c2\u8003[\u4e91\u670d\u52a1\u5668\u72b6\u6001](https://support.huaweicloud.com/api-ecs/ecs_08_0002.html)"
+        "example": "ACTIVE"
-  },
+    },
-  {
+    {
-    "name": "updated",
+        "name": "updated",
-    "type": "文本",
+        "type": "string",
-    "example": "2019-05-22T03:30:52Z",
+        "desc": "弹性云服务器更新时间。\n\n时间格式例如:2019-05-22T03:30:52Z",
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u66f4\u65b0\u65f6\u95f4\u3002\n\n\u65f6\u95f4\u683c\u5f0f\u4f8b\u5982:2019-05-22T03:30:52Z"
+        "example": "2019-05-22T03:30:52Z"
-  },
+    },
-  {
+    {
-    "name": "auto_terminate_time",
+        "name": "auto_terminate_time",
-    "type": "文本",
+        "type": "string",
-    "example": "2020-01-19T03:30:52Z",
+        "desc": "弹性云服务器定时删除时间。\n\n时间格式例如:2020-01-19T03:30:52Z",
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u81ea\u52a8\u91ca\u653e\u65f6\u95f4\u3002\n\n\u65f6\u95f4\u683c\u5f0f\u4f8b\u5982:2020-01-19T03:30:52Z"
+        "example": "2020-01-19T03:30:52Z"
-  },
+    },
-  {
+    {
-    "name": "hostId",
+        "name": "hostId",
-    "type": "文本",
+        "type": "string",
-    "example": "c7145889b2e3202cd295ceddb1742ff8941b827b586861fd0acedf64",
+        "desc": "弹性云服务器所在主机的主机ID。",
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u6240\u5728\u4e3b\u673a\u7684\u4e3b\u673aID\u3002"
+        "example": "c7145889b2e3202cd295ceddb1742ff8941b827b586861fd0acedf64"
-  },
+    },
-  {
+    {
-    "name": "OS-EXT-SRV-ATTR:host",
+        "name": "OS-EXT-SRV-ATTR:host",
-    "type": "文本",
+        "type": "string",
-    "example": "pod01.cn-north-1c",
+        "desc": "弹性云服务器所在主机的主机名称。",
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u6240\u5728\u4e3b\u673a\u7684\u4e3b\u673a\u540d\u79f0\u3002"
+        "example": "pod01.cn-north-1c"
-  },
+    },
-  {
+    {
-    "name": "addresses",
+        "name": "addresses",
-    "type": "json",
+        "type": "object",
-    "example": null,
+        "desc": "弹性云服务器的网络属性。",
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u7684\u7f51\u7edc\u5c5e\u6027\u3002"
+        "example": ""
-  },
+    },
-  {
+    {
-    "name": "key_name",
+        "name": "key_name",
-    "type": "文本",
+        "type": "string",
-    "example": "KeyPair-test",
+        "desc": "弹性云服务器使用的密钥对名称。",
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u4f7f\u7528\u7684\u5bc6\u94a5\u5bf9\u540d\u79f0\u3002"
+        "example": "KeyPair-test"
-  },
+    },
-  {
+    {
-    "name": "image",
+        "name": "image",
-    "type": "json",
+        "type": "",
-    "example": null,
+        "desc": "弹性云服务器镜像信息。",
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u955c\u50cf\u4fe1\u606f\u3002"
+        "example": ""
-  },
+    },
-  {
+    {
-    "name": "OS-EXT-STS:task_state",
+        "name": "OS-EXT-STS:task_state",
-    "type": "文本",
+        "type": "string",
-    "example": "rebooting",
+        "desc": "扩展属性,弹性云服务器当前任务的状态。\n\n取值范围请参考[云服务器状态](https://support.huaweicloud.com/api-ecs/ecs_08_0002.html)表3。",
-    "desc": "\u6269\u5c55\u5c5e\u6027,\u5f39\u6027\u4e91\u670d\u52a1\u5668\u5f53\u524d\u4efb\u52a1\u7684\u72b6\u6001\u3002\n\n\u53d6\u503c\u8303\u56f4\u8bf7\u53c2\u8003[\u4e91\u670d\u52a1\u5668\u72b6\u6001](https://support.huaweicloud.com/api-ecs/ecs_08_0002.html)\u88683\u3002"
+        "example": "rebooting"
-  },
+    },
-  {
+    {
-    "name": "OS-EXT-STS:vm_state",
+        "name": "OS-EXT-STS:vm_state",
-    "type": "文本",
+        "type": "string",
-    "example": "active",
+        "desc": "扩展属性,弹性云服务器当前状态。\n\n云服务器状态说明请参考[云服务器状态](https://support.huaweicloud.com/api-ecs/ecs_08_0002.html)。",
-    "desc": "\u6269\u5c55\u5c5e\u6027,\u5f39\u6027\u4e91\u670d\u52a1\u5668\u5f53\u524d\u72b6\u6001\u3002\n\n\u4e91\u670d\u52a1\u5668\u72b6\u6001\u8bf4\u660e\u8bf7\u53c2\u8003[\u4e91\u670d\u52a1\u5668\u72b6\u6001](https://support.huaweicloud.com/api-ecs/ecs_08_0002.html)\u3002"
+        "example": "active"
-  },
+    },
-  {
+    {
-    "name": "OS-EXT-SRV-ATTR:instance_name",
+        "name": "OS-EXT-SRV-ATTR:instance_name",
-    "type": "文本",
+        "type": "string",
-    "example": "instance-0048a91b",
+        "desc": "扩展属性,弹性云服务器别名。",
-    "desc": "\u6269\u5c55\u5c5e\u6027,\u5f39\u6027\u4e91\u670d\u52a1\u5668\u522b\u540d\u3002"
+        "example": "instance-0048a91b"
-  },
+    },
-  {
+    {
-    "name": "OS-EXT-SRV-ATTR:hypervisor_hostname",
+        "name": "OS-EXT-SRV-ATTR:hypervisor_hostname",
-    "type": "文本",
+        "type": "string",
-    "example": "nova022@36",
+        "desc": "扩展属性,弹性云服务器所在虚拟化主机名。",
-    "desc": "\u6269\u5c55\u5c5e\u6027,\u5f39\u6027\u4e91\u670d\u52a1\u5668\u6240\u5728\u865a\u62df\u5316\u4e3b\u673a\u540d\u3002"
+        "example": "nova022@36"
-  },
+    },
-  {
+    {
-    "name": "flavor",
+        "name": "flavor",
-    "type": "json",
+        "type": "",
-    "example": null,
+        "desc": "弹性云服务器规格信息。",
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u89c4\u683c\u4fe1\u606f\u3002"
+        "example": ""
-  },
+    },
-  {
+    {
-    "name": "id",
+        "name": "id",
-    "type": "文本",
+        "type": "string",
-    "example": "4f4b3dfa-eb70-47cf-a60a-998a53bd6666",
+        "desc": "弹性云服务器ID,格式为UUID。",
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668ID,\u683c\u5f0f\u4e3aUUID\u3002"
+        "example": "4f4b3dfa-eb70-47cf-a60a-998a53bd6666"
-  },
+    },
-  {
+    {
-    "name": "security_groups",
+        "name": "security_groups",
-    "type": "json",
+        "type": "array",
-    "example": {
+        "desc": "弹性云服务器所属安全组列表。",
-      "$ref": "#/definitions/ServerSecurityGroup"
+        "example": ""
-    },
+    },
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u6240\u5c5e\u5b89\u5168\u7ec4\u5217\u8868\u3002"
+    {
-  },
+        "name": "OS-EXT-AZ:availability_zone",
-  {
+        "type": "string",
-    "name": "OS-EXT-AZ:availability_zone",
+        "desc": "扩展属性,弹性云服务器所在可用区名称。",
-    "type": "文本",
+        "example": "cn-north-1c"
-    "example": "cn-north-1c",
+    },
-    "desc": "\u6269\u5c55\u5c5e\u6027,\u5f39\u6027\u4e91\u670d\u52a1\u5668\u6240\u5728\u53ef\u7528\u533a\u540d\u79f0\u3002"
+    {
-  },
+        "name": "user_id",
-  {
+        "type": "string",
-    "name": "user_id",
+        "desc": "创建弹性云服务器的用户ID,格式为UUID。",
-    "type": "文本",
+        "example": "05498fe56b8010d41f7fc01e280b6666"
-    "example": "05498fe56b8010d41f7fc01e280b6666",
+    },
-    "desc": "\u521b\u5efa\u5f39\u6027\u4e91\u670d\u52a1\u5668\u7684\u7528\u6237ID,\u683c\u5f0f\u4e3aUUID\u3002"
+    {
-  },
+        "name": "name",
-  {
+        "type": "string",
-    "name": "name",
+        "desc": "弹性云服务器名称。",
-    "type": "文本",
+        "example": "ecs-test-server"
-    "example": "ecs-test-server",
+    },
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u540d\u79f0\u3002"
+    {
-  },
+        "name": "created",
-  {
+        "type": "string",
-    "name": "created",
+        "desc": "弹性云服务器创建时间。\n\n时间格式例如:2019-05-22T03:19:19Z",
-    "type": "文本",
+        "example": "2017-07-15T11:30:52Z"
-    "example": "2017-07-15T11:30:52Z",
+    },
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u521b\u5efa\u65f6\u95f4\u3002\n\n\u65f6\u95f4\u683c\u5f0f\u4f8b\u5982:2019-05-22T03:19:19Z"
+    {
-  },
+        "name": "tenant_id",
-  {
+        "type": "string",
-    "name": "tenant_id",
+        "desc": "弹性云服务器所属租户ID,即项目id,和project_id表示相同的概念,格式为UUID。",
-    "type": "文本",
+        "example": "743b4c0428d94531b9f2add666646666"
-    "example": "743b4c0428d94531b9f2add666646666",
+    },
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u6240\u5c5e\u79df\u6237ID,\u5373\u9879\u76eeid,\u548cproject_id\u8868\u793a\u76f8\u540c\u7684\u6982\u5ff5,\u683c\u5f0f\u4e3aUUID\u3002"
+    {
-  },
+        "name": "OS-DCF:diskConfig",
-  {
+        "type": "string",
-    "name": "OS-DCF:diskConfig",
+        "desc": "扩展属性, diskConfig的类型。\n\n- MANUAL,镜像空间不会扩展。\n- AUTO,系统盘镜像空间会自动扩展为与flavor大小一致。",
-    "type": "文本",
+        "example": "AUTO"
-    "example": "AUTO",
+    },
-    "desc": "\u6269\u5c55\u5c5e\u6027, diskConfig\u7684\u7c7b\u578b\u3002\n\n- MANUAL,\u955c\u50cf\u7a7a\u95f4\u4e0d\u4f1a\u6269\u5c55\u3002\n- AUTO,\u7cfb\u7edf\u76d8\u955c\u50cf\u7a7a\u95f4\u4f1a\u81ea\u52a8\u6269\u5c55\u4e3a\u4e0eflavor\u5927\u5c0f\u4e00\u81f4\u3002"
+    {
-  },
+        "name": "accessIPv4",
-  {
+        "type": "string",
-    "name": "accessIPv4",
+        "desc": "预留属性。",
-    "type": "文本",
+        "example": ""
-    "example": null,
+    },
-    "desc": "\u9884\u7559\u5c5e\u6027\u3002"
+    {
-  },
+        "name": "accessIPv6",
-  {
+        "type": "string",
-    "name": "accessIPv6",
+        "desc": "预留属性。",
-    "type": "文本",
+        "example": ""
-    "example": null,
+    },
-    "desc": "\u9884\u7559\u5c5e\u6027\u3002"
+    {
-  },
+        "name": "fault",
-  {
+        "type": "",
-    "name": "fault",
+        "desc": "弹性云服务器故障信息。\n\n可选参数,在弹性云服务器状态为ERROR且存在异常的情况下返回。",
-    "type": "文本",
+        "example": ""
-    "example": null,
+    },
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u6545\u969c\u4fe1\u606f\u3002\n\n\u53ef\u9009\u53c2\u6570,\u5728\u5f39\u6027\u4e91\u670d\u52a1\u5668\u72b6\u6001\u4e3aERROR\u4e14\u5b58\u5728\u5f02\u5e38\u7684\u60c5\u51b5\u4e0b\u8fd4\u56de\u3002"
+    {
-  },
+        "name": "progress",
-  {
+        "type": "integer",
-    "name": "progress",
+        "desc": "弹性云服务器进度。",
-    "type": "整数",
+        "example": 0
-    "example": null,
+    },
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u8fdb\u5ea6\u3002"
+    {
-  },
+        "name": "OS-EXT-STS:power_state",
-  {
+        "type": "integer",
-    "name": "OS-EXT-STS:power_state",
+        "desc": "扩展属性,弹性云服务器电源状态。",
-    "type": "整数",
+        "example": 4
-    "example": 4,
+    },
-    "desc": "\u6269\u5c55\u5c5e\u6027,\u5f39\u6027\u4e91\u670d\u52a1\u5668\u7535\u6e90\u72b6\u6001\u3002"
+    {
-  },
+        "name": "config_drive",
-  {
+        "type": "string",
-    "name": "config_drive",
+        "desc": "config drive信息。",
-    "type": "文本",
+        "example": ""
-    "example": null,
+    },
-    "desc": "config drive\u4fe1\u606f\u3002"
+    {
-  },
+        "name": "metadata",
-  {
+        "type": "object",
-    "name": "metadata",
+        "desc": "弹性云服务器元数据。\n\n> 说明:\n> \n> 元数据包含系统默认添加字段和用户设置的字段。\n\n系统默认添加字段\n\n1. charging_mode\n云服务器的计费类型。\n\n- “0”:按需计费(即postPaid-后付费方式)。\n- “1”:按包年包月计费(即prePaid-预付费方式)。\"2\":竞价实例计费\n\n2. metering.order_id\n按“包年/包月”计费的云服务器对应的订单ID。\n\n3. metering.product_id\n按“包年/包月”计费的云服务器对应的产品ID。\n\n4. vpc_id\n云服务器所属的虚拟私有云ID。\n\n5. EcmResStatus\n云服务器的冻结状态。\n\n- normal:云服务器正常状态(未被冻结)。\n- freeze:云服务器被冻结。\n\n> 当云服务器被冻结或者解冻后,系统默认添加该字段,且该字段必选。\n\n6. metering.image_id\n云服务器操作系统对应的镜像ID\n\n7.  metering.imagetype\n镜像类型,目前支持:\n\n- 公共镜像(gold)\n- 私有镜像(private)\n- 共享镜像(shared)\n\n8. metering.resourcespeccode\n云服务器对应的资源规格。\n\n9. image_name\n云服务器操作系统对应的镜像名称。\n\n10. os_bit\n操作系统位数,一般取值为“32”或者“64”。\n\n11. lockCheckEndpoint\n回调URL,用于检查弹性云服务器的加锁是否有效。\n\n- 如果有效,则云服务器保持锁定状态。\n- 如果无效,解除锁定状态,删除失效的锁。\n\n12. lockSource\n弹性云服务器来自哪个服务。订单加锁(ORDER)\n\n13. lockSourceId\n弹性云服务器的加锁来自哪个ID。lockSource为“ORDER”时,lockSourceId为订单ID。\n\n14. lockScene\n弹性云服务器的加锁类型。\n\n- 按需转包周期(TO_PERIOD_LOCK)\n\n15. virtual_env_type\n\n- IOS镜像创建虚拟机,\"virtual_env_type\": \"IsoImage\" 属性;\n- 非IOS镜像创建虚拟机,在19.5.0版本以后创建的虚拟机将不会添加virtual_env_type 属性,而在此之前的版本创建的虚拟机可能会返回\"virtual_env_type\": \"FusionCompute\"属性 。\n\n> virtual_env_type属性不允许用户增加、删除和修改。\n\n16. metering.resourcetype\n云服务器对应的资源类型。\n\n17. os_type\n操作系统类型,取值为:Linux、Windows。\n\n18. cascaded.instance_extrainfo\n系统内部虚拟机扩展信息。\n\n19. __support_agent_list\n云服务器是否支持企业主机安全、主机监控。\n\n- “hss”:企业主机安全\n-  “ces”:主机监控\n\n20. agency_name\n委托的名称。\n\n委托是由租户管理员在统一身份认证服务(Identity and Access Management,IAM)上创建的,可以为弹性云服务器提供访问云服务的临时凭证。",
-    "type": "json",
+        "example": ""
-    "example": null,
+    },
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u5143\u6570\u636e\u3002\n\n> \u8bf4\u660e:\n> \n> \u5143\u6570\u636e\u5305\u542b\u7cfb\u7edf\u9ed8\u8ba4\u6dfb\u52a0\u5b57\u6bb5\u548c\u7528\u6237\u8bbe\u7f6e\u7684\u5b57\u6bb5\u3002\n\n\u7cfb\u7edf\u9ed8\u8ba4\u6dfb\u52a0\u5b57\u6bb5\n\n1. charging_mode\n\u4e91\u670d\u52a1\u5668\u7684\u8ba1\u8d39\u7c7b\u578b\u3002\n\n- \u201c0\u201d:\u6309\u9700\u8ba1\u8d39(\u5373postPaid-\u540e\u4ed8\u8d39\u65b9\u5f0f)\u3002\n- \u201c1\u201d:\u6309\u5305\u5e74\u5305\u6708\u8ba1\u8d39(\u5373prePaid-\u9884\u4ed8\u8d39\u65b9\u5f0f)\u3002\"2\":\u7ade\u4ef7\u5b9e\u4f8b\u8ba1\u8d39\n\n2. metering.order_id\n\u6309\u201c\u5305\u5e74/\u5305\u6708\u201d\u8ba1\u8d39\u7684\u4e91\u670d\u52a1\u5668\u5bf9\u5e94\u7684\u8ba2\u5355ID\u3002\n\n3. metering.product_id\n\u6309\u201c\u5305\u5e74/\u5305\u6708\u201d\u8ba1\u8d39\u7684\u4e91\u670d\u52a1\u5668\u5bf9\u5e94\u7684\u4ea7\u54c1ID\u3002\n\n4. vpc_id\n\u4e91\u670d\u52a1\u5668\u6240\u5c5e\u7684\u865a\u62df\u79c1\u6709\u4e91ID\u3002\n\n5. EcmResStatus\n\u4e91\u670d\u52a1\u5668\u7684\u51bb\u7ed3\u72b6\u6001\u3002\n\n- normal:\u4e91\u670d\u52a1\u5668\u6b63\u5e38\u72b6\u6001(\u672a\u88ab\u51bb\u7ed3)\u3002\n- freeze:\u4e91\u670d\u52a1\u5668\u88ab\u51bb\u7ed3\u3002\n\n> \u5f53\u4e91\u670d\u52a1\u5668\u88ab\u51bb\u7ed3\u6216\u8005\u89e3\u51bb\u540e,\u7cfb\u7edf\u9ed8\u8ba4\u6dfb\u52a0\u8be5\u5b57\u6bb5,\u4e14\u8be5\u5b57\u6bb5\u5fc5\u9009\u3002\n\n6. metering.image_id\n\u4e91\u670d\u52a1\u5668\u64cd\u4f5c\u7cfb\u7edf\u5bf9\u5e94\u7684\u955c\u50cfID\n\n7.  metering.imagetype\n\u955c\u50cf\u7c7b\u578b,\u76ee\u524d\u652f\u6301:\n\n- \u516c\u5171\u955c\u50cf(gold)\n- \u79c1\u6709\u955c\u50cf(private)\n- \u5171\u4eab\u955c\u50cf(shared)\n\n8. metering.resourcespeccode\n\u4e91\u670d\u52a1\u5668\u5bf9\u5e94\u7684\u8d44\u6e90\u89c4\u683c\u3002\n\n9. image_name\n\u4e91\u670d\u52a1\u5668\u64cd\u4f5c\u7cfb\u7edf\u5bf9\u5e94\u7684\u955c\u50cf\u540d\u79f0\u3002\n\n10. os_bit\n\u64cd\u4f5c\u7cfb\u7edf\u4f4d\u6570,\u4e00\u822c\u53d6\u503c\u4e3a\u201c32\u201d\u6216\u8005\u201c64\u201d\u3002\n\n11. lockCheckEndpoint\n\u56de\u8c03URL,\u7528\u4e8e\u68c0\u67e5\u5f39\u6027\u4e91\u670d\u52a1\u5668\u7684\u52a0\u9501\u662f\u5426\u6709\u6548\u3002\n\n- \u5982\u679c\u6709\u6548,\u5219\u4e91\u670d\u52a1\u5668\u4fdd\u6301\u9501\u5b9a\u72b6\u6001\u3002\n- \u5982\u679c\u65e0\u6548,\u89e3\u9664\u9501\u5b9a\u72b6\u6001,\u5220\u9664\u5931\u6548\u7684\u9501\u3002\n\n12. lockSource\n\u5f39\u6027\u4e91\u670d\u52a1\u5668\u6765\u81ea\u54ea\u4e2a\u670d\u52a1\u3002\u8ba2\u5355\u52a0\u9501(ORDER)\n\n13. lockSourceId\n\u5f39\u6027\u4e91\u670d\u52a1\u5668\u7684\u52a0\u9501\u6765\u81ea\u54ea\u4e2aID\u3002lockSource\u4e3a\u201cORDER\u201d\u65f6,lockSourceId\u4e3a\u8ba2\u5355ID\u3002\n\n14. lockScene\n\u5f39\u6027\u4e91\u670d\u52a1\u5668\u7684\u52a0\u9501\u7c7b\u578b\u3002\n\n- \u6309\u9700\u8f6c\u5305\u5468\u671f(TO_PERIOD_LOCK)\n\n15. virtual_env_type\n\n- IOS\u955c\u50cf\u521b\u5efa\u865a\u62df\u673a,\"virtual_env_type\": \"IsoImage\" \u5c5e\u6027;\n- \u975eIOS\u955c\u50cf\u521b\u5efa\u865a\u62df\u673a,\u572819.5.0\u7248\u672c\u4ee5\u540e\u521b\u5efa\u7684\u865a\u62df\u673a\u5c06\u4e0d\u4f1a\u6dfb\u52a0virtual_env_type \u5c5e\u6027,\u800c\u5728\u6b64\u4e4b\u524d\u7684\u7248\u672c\u521b\u5efa\u7684\u865a\u62df\u673a\u53ef\u80fd\u4f1a\u8fd4\u56de\"virtual_env_type\": \"FusionCompute\"\u5c5e\u6027 \u3002\n\n> virtual_env_type\u5c5e\u6027\u4e0d\u5141\u8bb8\u7528\u6237\u589e\u52a0\u3001\u5220\u9664\u548c\u4fee\u6539\u3002\n\n16. metering.resourcetype\n\u4e91\u670d\u52a1\u5668\u5bf9\u5e94\u7684\u8d44\u6e90\u7c7b\u578b\u3002\n\n17. os_type\n\u64cd\u4f5c\u7cfb\u7edf\u7c7b\u578b,\u53d6\u503c\u4e3a:Linux\u3001Windows\u3002\n\n18. cascaded.instance_extrainfo\n\u7cfb\u7edf\u5185\u90e8\u865a\u62df\u673a\u6269\u5c55\u4fe1\u606f\u3002\n\n19. __support_agent_list\n\u4e91\u670d\u52a1\u5668\u662f\u5426\u652f\u6301\u4f01\u4e1a\u4e3b\u673a\u5b89\u5168\u3001\u4e3b\u673a\u76d1\u63a7\u3002\n\n- \u201chss\u201d:\u4f01\u4e1a\u4e3b\u673a\u5b89\u5168\n-  \u201cces\u201d:\u4e3b\u673a\u76d1\u63a7\n\n20. agency_name\n\u59d4\u6258\u7684\u540d\u79f0\u3002\n\n\u59d4\u6258\u662f\u7531\u79df\u6237\u7ba1\u7406\u5458\u5728\u7edf\u4e00\u8eab\u4efd\u8ba4\u8bc1\u670d\u52a1(Identity and Access Management,IAM)\u4e0a\u521b\u5efa\u7684,\u53ef\u4ee5\u4e3a\u5f39\u6027\u4e91\u670d\u52a1\u5668\u63d0\u4f9b\u8bbf\u95ee\u4e91\u670d\u52a1\u7684\u4e34\u65f6\u51ed\u8bc1\u3002"
+    {
-  },
+        "name": "OS-SRV-USG:launched_at",
-  {
+        "type": "string",
-    "name": "OS-SRV-USG:launched_at",
+        "desc": "弹性云服务器启动时间。时间格式例如:2019-05-22T03:23:59.000000",
-    "type": "文本",
+        "example": "2018-08-15T14:21:22.000000"
-    "example": "2018-08-15T14:21:22.000000",
+    },
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u542f\u52a8\u65f6\u95f4\u3002\u65f6\u95f4\u683c\u5f0f\u4f8b\u5982:2019-05-22T03:23:59.000000"
+    {
-  },
+        "name": "OS-SRV-USG:terminated_at",
-  {
+        "type": "string",
-    "name": "OS-SRV-USG:terminated_at",
+        "desc": "弹性云服务器删除时间。\n\n时间格式例如:2019-05-22T03:23:59.000000",
-    "type": "文本",
+        "example": "2019-05-22T03:23:59.000000"
-    "example": "2019-05-22T03:23:59.000000",
+    },
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u5220\u9664\u65f6\u95f4\u3002\n\n\u65f6\u95f4\u683c\u5f0f\u4f8b\u5982:2019-05-22T03:23:59.000000"
+    {
-  },
+        "name": "os-extended-volumes:volumes_attached",
-  {
+        "type": "array",
-    "name": "os-extended-volumes:volumes_attached",
+        "desc": "挂载到弹性云服务器上的磁盘。",
-    "type": "json",
+        "example": ""
-    "example": {
+    },
-      "$ref": "#/definitions/ServerExtendVolumeAttachment"
+    {
-    },
+        "name": "description",
-    "desc": "\u6302\u8f7d\u5230\u5f39\u6027\u4e91\u670d\u52a1\u5668\u4e0a\u7684\u78c1\u76d8\u3002"
+        "type": "string",
-  },
+        "desc": "弹性云服务器的描述信息。",
-  {
+        "example": "ecs description"
-    "name": "description",
+    },
-    "type": "文本",
+    {
-    "example": "ecs description",
+        "name": "host_status",
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u7684\u63cf\u8ff0\u4fe1\u606f\u3002"
+        "type": "string",
-  },
+        "desc": "nova-compute状态。\n\n- UP:服务正常\n- UNKNOWN:状态未知\n- DOWN:服务异常\n- MAINTENANCE:维护状态\n- 空字符串:弹性云服务器无主机信息",
-  {
+        "example": "UP"
-    "name": "host_status",
+    },
-    "type": "文本",
+    {
-    "example": "UP",
+        "name": "OS-EXT-SRV-ATTR:hostname",
-    "desc": "nova-compute\u72b6\u6001\u3002\n\n- UP:\u670d\u52a1\u6b63\u5e38\n- UNKNOWN:\u72b6\u6001\u672a\u77e5\n- DOWN:\u670d\u52a1\u5f02\u5e38\n- MAINTENANCE:\u7ef4\u62a4\u72b6\u6001\n- \u7a7a\u5b57\u7b26\u4e32:\u5f39\u6027\u4e91\u670d\u52a1\u5668\u65e0\u4e3b\u673a\u4fe1\u606f"
+        "type": "string",
-  },
+        "desc": "弹性云服务器的主机名。",
-  {
+        "example": ""
-    "name": "OS-EXT-SRV-ATTR:hostname",
+    },
-    "type": "文本",
+    {
-    "example": null,
+        "name": "OS-EXT-SRV-ATTR:reservation_id",
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u7684\u4e3b\u673a\u540d\u3002"
+        "type": "string",
-  },
+        "desc": "批量创建场景,弹性云服务器的预留ID。",
-  {
+        "example": "r-f06p3js8"
-    "name": "OS-EXT-SRV-ATTR:reservation_id",
+    },
-    "type": "文本",
+    {
-    "example": "r-f06p3js8",
+        "name": "OS-EXT-SRV-ATTR:launch_index",
-    "desc": "\u6279\u91cf\u521b\u5efa\u573a\u666f,\u5f39\u6027\u4e91\u670d\u52a1\u5668\u7684\u9884\u7559ID\u3002"
+        "type": "integer",
-  },
+        "desc": "批量创建场景,弹性云服务器的启动顺序。",
-  {
+        "example": 0
-    "name": "OS-EXT-SRV-ATTR:launch_index",
+    },
-    "type": "整数",
+    {
-    "example": null,
+        "name": "OS-EXT-SRV-ATTR:kernel_id",
-    "desc": "\u6279\u91cf\u521b\u5efa\u573a\u666f,\u5f39\u6027\u4e91\u670d\u52a1\u5668\u7684\u542f\u52a8\u987a\u5e8f\u3002"
+        "type": "string",
-  },
+        "desc": "若使用AMI格式的镜像,则表示kernel image的UUID;否则,留空。",
-  {
+        "example": ""
-    "name": "OS-EXT-SRV-ATTR:kernel_id",
+    },
-    "type": "文本",
+    {
-    "example": null,
+        "name": "OS-EXT-SRV-ATTR:ramdisk_id",
-    "desc": "\u82e5\u4f7f\u7528AMI\u683c\u5f0f\u7684\u955c\u50cf,\u5219\u8868\u793akernel image\u7684UUID;\u5426\u5219,\u7559\u7a7a\u3002"
+        "type": "string",
-  },
+        "desc": "若使用AMI格式镜像,则表示ramdisk image的UUID;否则,留空。",
-  {
+        "example": ""
-    "name": "OS-EXT-SRV-ATTR:ramdisk_id",
+    },
-    "type": "文本",
+    {
-    "example": null,
+        "name": "OS-EXT-SRV-ATTR:root_device_name",
-    "desc": "\u82e5\u4f7f\u7528AMI\u683c\u5f0f\u955c\u50cf,\u5219\u8868\u793aramdisk image\u7684UUID;\u5426\u5219,\u7559\u7a7a\u3002"
+        "type": "string",
-  },
+        "desc": "弹性云服务器系统盘的设备名称。",
-  {
+        "example": "/dev/vda"
-    "name": "OS-EXT-SRV-ATTR:root_device_name",
+    },
-    "type": "文本",
+    {
-    "example": "/dev/vda",
+        "name": "OS-EXT-SRV-ATTR:user_data",
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u7cfb\u7edf\u76d8\u7684\u8bbe\u5907\u540d\u79f0\u3002"
+        "type": "string",
-  },
+        "desc": "创建弹性云服务器时指定的user_data。",
-  {
+        "example": "IyEvYmluL2Jhc2gKZWNobyAncm9vdDokNiRjcGRkSjckWm5WZHNiR253Z0l0SGlxUjZxbWtLTlJaeU9lZUtKd3dPbG9XSFdUeGFzWjA1STYwdnJYRTdTUTZGbEpFbWlXZ21WNGNmZ1pac1laN1BkMTBLRndyeC8nIHwgY2hwYXNzd2Q6666"
-    "name": "OS-EXT-SRV-ATTR:user_data",
+    },
-    "type": "文本",
+    {
-    "example": "IyEvYmluL2Jhc2gKZWNobyAncm9vdDokNiRjcGRkSjckWm5WZHNiR253Z0l0SGlxUjZxbWtLTlJaeU9lZUtKd3dPbG9XSFdUeGFzWjA1STYwdnJYRTdTUTZGbEpFbWlXZ21WNGNmZ1pac1laN1BkMTBLRndyeC8nIHwgY2hwYXNzd2Q6666",
+        "name": "locked",
-    "desc": "\u521b\u5efa\u5f39\u6027\u4e91\u670d\u52a1\u5668\u65f6\u6307\u5b9a\u7684user_data\u3002"
+        "type": "boolean",
-  },
+        "desc": "弹性云服务器是否为锁定状态。\n\n- true:锁定\n- false:未锁定",
-  {
+        "example": false
-    "name": "locked",
+    },
-    "type": "boolean",
+    {
-    "example": null,
+        "name": "tags",
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u662f\u5426\u4e3a\u9501\u5b9a\u72b6\u6001\u3002\n\n- true:\u9501\u5b9a\n- false:\u672a\u9501\u5b9a"
+        "type": "array",
-  },
+        "desc": "弹性云服务器标签。",
-  {
+        "example": ""
-    "name": "tags",
+    },
-    "type": "文本、多值",
+    {
-    "example": {
+        "name": "os:scheduler_hints",
-      "type": "文本"
+        "type": "",
-    },
+        "desc": "弹性云服务器调度信息",
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u6807\u7b7e\u3002"
+        "example": ""
-  },
+    },
-  {
+    {
-    "name": "os:scheduler_hints",
+        "name": "enterprise_project_id",
-    "type": "json",
+        "type": "string",
-    "example": null,
+        "desc": "弹性云服务器所属的企业项目ID。",
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u8c03\u5ea6\u4fe1\u606f"
+        "example": "0"
-  },
+    },
-  {
+    {
-    "name": "enterprise_project_id",
+        "name": "sys_tags",
-    "type": "文本",
+        "type": "array",
-    "example": "0",
+        "desc": "弹性云服务器系统标签。",
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u6240\u5c5e\u7684\u4f01\u4e1a\u9879\u76eeID\u3002"
+        "example": ""
-  },
+    },
-  {
+    {
-    "name": "sys_tags",
+        "name": "cpu_options",
-    "type": "文本、多值",
+        "type": "",
-    "example": {
+        "desc": "自定义CPU选项。",
-      "$ref": "#/definitions/ServerSystemTag"
+        "example": ""
-    },
+    },
-    "desc": "\u5f39\u6027\u4e91\u670d\u52a1\u5668\u7cfb\u7edf\u6807\u7b7e\u3002"
+    {
-  },
+        "name": "hypervisor",
-  {
+        "type": "",
-    "name": "cpu_options",
+        "desc": "hypervisor信息。",
-    "type": "json",
+        "example": ""
-    "example": null,
+    }
-    "desc": "\u81ea\u5b9a\u4e49CPU\u9009\u9879\u3002"
-  },
-  {
-    "name": "hypervisor",
-    "type": "文本",
-    "example": null,
-    "desc": "hypervisor\u4fe1\u606f\u3002"
-  }
 ]

cmdb-api/api/lib/cmdb/auto_discovery/templates/tencent_cvm.json

@@ -1,297 +1,248 @@
-[
+[
-    {
+    {
-        "name": "Placement",
+        "name": "Placement",
-        "type": "json",
+        "type": "Placement",
-        "desc": "实例所在的位置。",
+        "desc": "实例所在的位置。",
-        "example": {
+        "example": ""
-            "HostId": "host-h3m57oik",
+    },
-            "ProjectId": 1174660,
+    {
-            "HostIds": [],
+        "name": "InstanceId",
-            "Zone": "ap-guangzhou-1",
+        "type": "String",
-            "HostIps": []
+        "desc": "实例ID。",
-        }
+        "example": "ins-9bxebleo"
-    },
+    },
-    {
+    {
-        "name": "InstanceId",
+        "name": "InstanceType",
-        "type": "文本",
+        "type": "String",
-        "desc": "实例ID。",
+        "desc": "实例机型。",
-        "example": "ins-xlsyru2j"
+        "example": "S1.SMALL1"
-    },
+    },
-    {
+    {
-        "name": "InstanceType",
+        "name": "CPU",
-        "type": "文本",
+        "type": "Integer",
-        "desc": "实例机型。",
+        "desc": "实例的CPU核数，单位：核。",
-        "example": "S2.SMALL2"
+        "example": "1"
-    },
+    },
-    {
+    {
-        "name": "CPU",
+        "name": "Memory",
-        "type": "整数",
+        "type": "Integer",
-        "desc": "实例的CPU核数，单位：核。",
+        "desc": "实例内存容量，单位：GB。",
-        "example": 1
+        "example": "1"
-    },
+    },
-    {
+    {
-        "name": "Memory",
+        "name": "RestrictState",
-        "type": "整数",
+        "type": "String",
-        "desc": "实例内存容量，单位：GB。",
+        "desc": "NORMAL：表示正常状态的实例\nEXPIRED：表示过期的实例\nPROTECTIVELY_ISOLATED：表示被安全隔离的实例。",
-        "example": 1
+        "example": "NORMAL"
-    },
+    },
-    {
+    {
-        "name": "RestrictState",
+        "name": "InstanceName",
-        "type": "文本",
+        "type": "String",
-        "desc": "实例业务状态。取值范围： NORMAL：表示正常状态的实例 EXPIRED：表示过期的实例 PROTECTIVELY_ISOLATED：表示被安全隔离的实例。",
+        "desc": "实例名称。",
-        "example": "PROTECTIVELY_ISOLATED"
+        "example": "测试实例"
-    },
+    },
-    {
+    {
-        "name": "InstanceName",
+        "name": "InstanceChargeType",
-        "type": "文本",
+        "type": "String",
-        "desc": "实例名称。",
+        "desc": "PREPAID：表示预付费，即包年包月\nPOSTPAID_BY_HOUR：表示后付费，即按量计费\nCDHPAID：专用宿主机付费，即只对专用宿主机计费，不对专用宿主机上的实例计费。\nSPOTPAID：表示竞价实例付费。",
-        "example": "test"
+        "example": "PREPAID"
-    },
+    },
-    {
+    {
-        "name": "InstanceChargeType",
+        "name": "SystemDisk",
-        "type": "文本",
+        "type": "SystemDisk",
-        "desc": "实例计费模式。取值范围： PREPAID：表示预付费，即包年包月 POSTPAID_BY_HOUR：表示后付费，即按量计费 CDHPAID：专用宿主机付费，即只对专用宿主机计费，不对专用宿主机上的实例计费。 SPOTPAID：表示竞价实例付费。",
+        "desc": "实例系统盘信息。",
-        "example": "POSTPAID_BY_HOUR"
+        "example": ""
-    },
+    },
-    {
+    {
-        "name": "SystemDisk",
+        "name": "DataDisks",
-        "type": "json",
+        "type": "Array of DataDisk",
-        "desc": "实例系统盘信息。",
+        "desc": "实例数据盘信息。",
-        "example": {
+        "example": ""
-            "DiskSize": 50,
+    },
-            "CdcId": null,
+    {
-            "DiskId": "disk-czsodtl1",
+        "name": "PrivateIpAddresses",
-            "DiskType": "CLOUD_SSD"
+        "type": "Array of String",
-        }
+        "desc": "实例主网卡的内网IP列表。",
-    },
+        "example": "[\"172.16.32.78\"]"
-    {
+    },
-        "name": "DataDisks",
+    {
-        "type": "json",
+        "name": "PublicIpAddresses",
-        "desc": "实例数据盘信息。",
+        "type": "Array of String",
-        "example": [
+        "desc": "实例主网卡的公网IP列表。注意：此字段可能返回 null，表示取不到有效值。",
-            {
+        "example": "[\"123.207.11.190\"]"
-                "DeleteWithInstance": true,
+    },
-                "Encrypt": true,
+    {
-                "CdcId": null,
+        "name": "InternetAccessible",
-                "DiskType": "CLOUD_SSD",
+        "type": "InternetAccessible",
-                "ThroughputPerformance": 0,
+        "desc": "实例带宽信息。",
-                "KmsKeyId": null,
+        "example": ""
-                "DiskSize": 50,
+    },
-                "SnapshotId": null,
+    {
-                "DiskId": "disk-bzsodtn1"
+        "name": "VirtualPrivateCloud",
-            }
+        "type": "VirtualPrivateCloud",
-        ]
+        "desc": "实例所属虚拟私有网络信息。",
-    },
+        "example": ""
-    {
+    },
-        "name": "PrivateIpAddresses",
+    {
-        "type": "文本、多值",
+        "name": "ImageId",
-        "desc": "实例主网卡的内网IP列表。",
+        "type": "String",
-        "example": [
+        "desc": "生产实例所使用的镜像ID。",
-            "172.16.32.78"
+        "example": "img-9qabwvbn"
-        ]
+    },
-    },
+    {
-    {
+        "name": "RenewFlag",
-        "name": "PublicIpAddresses",
+        "type": "String",
-        "type": "文本、多值",
+        "desc": "NOTIFY_AND_MANUAL_RENEW：表示通知即将过期，但不自动续费\nNOTIFY_AND_AUTO_RENEW：表示通知即将过期，而且自动续费\nDISABLE_NOTIFY_AND_MANUAL_RENEW：表示不通知即将过期，也不自动续费。\n注意：后付费模式本项为null",
-        "desc": "实例主网卡的公网IP列表。 注意：此字段可能返回 null，表示取不到有效值。",
+        "example": "NOTIFY_AND_MANUAL_RENEW"
-        "example": [
+    },
-            "123.207.11.190"
+    {
-        ]
+        "name": "CreatedTime",
-    },
+        "type": "Timestamp ISO8601",
-    {
+        "desc": "创建时间。按照ISO8601标准表示，并且使用UTC时间。格式为：YYYY-MM-DDThh:mm:ssZ。",
-        "name": "InternetAccessible",
+        "example": "2020-03-10T02:43:51Z"
-        "type": "json",
+    },
-        "desc": "实例带宽信息。",
+    {
-        "example": {
+        "name": "ExpiredTime",
-            "PublicIpAssigned": true,
+        "type": "Timestamp ISO8601",
-            "InternetChargeType": "TRAFFIC_POSTPAID_BY_HOUR",
+        "desc": "到期时间。按照ISO8601标准表示，并且使用UTC时间。格式为：YYYY-MM-DDThh:mm:ssZ。注意：后付费模式本项为null",
-            "BandwidthPackageId": null,
+        "example": "2020-04-10T02:47:36Z"
-            "InternetMaxBandwidthOut": 1
+    },
-        }
+    {
-    },
+        "name": "OsName",
-    {
+        "type": "String",
-        "name": "VirtualPrivateCloud",
+        "desc": "操作系统名称。",
-        "type": "json",
+        "example": "CentOS 7.6 64bit"
-        "desc": "实例所属虚拟私有网络信息。",
+    },
-        "example": {
+    {
-            "SubnetId": "subnet-mv4sn55k",
+        "name": "SecurityGroupIds",
-            "AsVpcGateway": false,
+        "type": "Array of String",
-            "Ipv6AddressCount": 1,
+        "desc": "实例所属安全组。该参数可以通过调用 DescribeSecurityGroups 的返回值中的sgId字段来获取。",
-            "VpcId": "vpc-m0cnatxj",
+        "example": "[\"sg-p1ezv4wz\"]"
-            "PrivateIpAddresses": [
+    },
-                "172.16.3.59"
+    {
-            ]
+        "name": "LoginSettings",
-        }
+        "type": "LoginSettings",
-    },
+        "desc": "实例登录设置。目前只返回实例所关联的密钥。",
-    {
+        "example": ""
-        "name": "ImageId",
+    },
-        "type": "文本",
+    {
-        "desc": "生产实例所使用的镜像ID。",
+        "name": "InstanceState",
-        "example": "img-8toqc6s3"
+        "type": "String",
-    },
+        "desc": "PENDING：表示创建中\nLAUNCH_FAILED：表示创建失败\nRUNNING：表示运行中\nSTOPPED：表示关机\nSTARTING：表示开机中\nSTOPPING：表示关机中\nREBOOTING：表示重启中\nSHUTDOWN：表示停止待销毁\nTERMINATING：表示销毁中。",
-    {
+        "example": ""
-        "name": "RenewFlag",
+    },
-        "type": "文本",
+    {
-        "desc": "自动续费标识。取值范围： NOTIFY_AND_MANUAL_RENEW：表示通知即将过期，但不自动续费 NOTIFY_AND_AUTO_RENEW：表示通知即将过期，而且自动续费 DISABLE_NOTIFY_AND_MANUAL_RENEW：表示不通知即将过期，也不自动续费。 注意：后付费模式本项为null",
+        "name": "Tags",
-        "example": "NOTIFY_AND_MANUAL_RENEW"
+        "type": "Array of Tag",
-    },
+        "desc": "实例关联的标签列表。",
-    {
+        "example": ""
-        "name": "CreatedTime",
+    },
-        "type": "json",
+    {
-        "desc": "创建时间。按照ISO8601标准表示，并且使用UTC时间。格式为：YYYY-MM-DDThh:mm:ssZ。",
+        "name": "StopChargingMode",
-        "example": "2020-09-22T00:00:00+00:00"
+        "type": "String",
-    },
+        "desc": "KEEP_CHARGING：关机继续收费\nSTOP_CHARGING：关机停止收费\nNOT_APPLICABLE：实例处于非关机状态或者不适用关机停止计费的条件",
-    {
+        "example": "NOT_APPLICABLE"
-        "name": "ExpiredTime",
+    },
-        "type": "json",
+    {
-        "desc": "到期时间。按照ISO8601标准表示，并且使用UTC时间。格式为：YYYY-MM-DDThh:mm:ssZ。注意：后付费模式本项为null",
+        "name": "Uuid",
-        "example": "2020-09-22T00:00:00+00:00"
+        "type": "String",
-    },
+        "desc": "实例全局唯一ID",
-    {
+        "example": "68b510db-b4c1-4630-a62b-73d0c7c970f9"
-        "name": "OsName",
+    },
-        "type": "文本",
+    {
-        "desc": "操作系统名称。",
+        "name": "LatestOperation",
-        "example": "CentOS 7.4 64bit"
+        "type": "String",
-    },
+        "desc": "实例的最新操作。例：StopInstances、ResetInstance。注意：此字段可能返回 null，表示取不到有效值。",
-    {
+        "example": "RenewInstances"
-        "name": "SecurityGroupIds",
+    },
-        "type": "文本、多值",
+    {
-        "desc": "实例所属安全组。该参数可以通过调用 DescribeSecurityGroups 的返回值中的sgId字段来获取。",
+        "name": "LatestOperationState",
-        "example": [
+        "type": "String",
-            "sg-p1ezv4wz"
+        "desc": "SUCCESS：表示操作成功\nOPERATING：表示操作执行中\nFAILED：表示操作失败注意：此字段可能返回 null，表示取不到有效值。",
-        ]
+        "example": "SUCCESS"
-    },
+    },
-    {
+    {
-        "name": "LoginSettings",
+        "name": "LatestOperationRequestId",
-        "type": "json",
+        "type": "String",
-        "desc": "实例登录设置。目前只返回实例所关联的密钥。",
+        "desc": "实例最新操作的唯一请求 ID。注意：此字段可能返回 null，表示取不到有效值。",
-        "example": {
+        "example": "3554eb5b-1cfa-471a-ae76-dc436c9d43e8"
-            "Password": "123qwe!@#QWE",
+    },
-            "KeepImageLogin": "False",
+    {
-            "KeyIds": [
+        "name": "DisasterRecoverGroupId",
-                "skey-b4vakk62"
+        "type": "String",
-            ]
+        "desc": "分散置放群组ID。注意：此字段可能返回 null，表示取不到有效值。",
-        }
+        "example": "null"
-    },
+    },
-    {
+    {
-        "name": "InstanceState",
+        "name": "IPv6Addresses",
-        "type": "文本",
+        "type": "Array of String",
-        "desc": "实例状态。取值范围： PENDING：表示创建中 LAUNCH_FAILED：表示创建失败 RUNNING：表示运行中 STOPPED：表示关机 STARTING：表示开机中 STOPPING：表示关机中 REBOOTING：表示重启中 SHUTDOWN：表示停止待销毁 TERMINATING：表示销毁中。",
+        "desc": "实例的IPv6地址。注意：此字段可能返回 null，表示取不到有效值。",
-        "example": "RUNNING"
+        "example": "null"
-    },
+    },
-    {
+    {
-        "name": "Tags",
+        "name": "CamRoleName",
-        "type": "json",
+        "type": "String",
-        "desc": "实例关联的标签列表。",
+        "desc": "CAM角色名。注意：此字段可能返回 null，表示取不到有效值。",
-        "example": [
+        "example": "null"
-            {
+    },
-                "Value": "test",
+    {
-                "Key": "test"
+        "name": "HpcClusterId",
-            }
+        "type": "String",
-        ]
+        "desc": "高性能计算集群ID。注意：此字段可能返回 null，表示取不到有效值。",
-    },
+        "example": "null"
-    {
+    },
-        "name": "StopChargingMode",
+    {
-        "type": "文本",
+        "name": "RdmaIpAddresses",
-        "desc": "实例的关机计费模式。 取值范围： KEEP_CHARGING：关机继续收费 STOP_CHARGING：关机停止收费NOT_APPLICABLE：实例处于非关机状态或者不适用关机停止计费的条件",
+        "type": "Array of String",
-        "example": "NOT_APPLICABLE"
+        "desc": "高性能计算集群IP列表。注意：此字段可能返回 null，表示取不到有效值。",
-    },
+        "example": "null"
-    {
+    },
-        "name": "Uuid",
+    {
-        "type": "文本",
+        "name": "DedicatedClusterId",
-        "desc": "实例全局唯一ID",
+        "type": "String",
-        "example": "e85f1388-0422-410d-8e50-bef540e78c18"
+        "desc": "实例所在的专用集群ID。注意：此字段可能返回 null，表示取不到有效值。",
-    },
+        "example": "cluster-du3jken"
-    {
+    },
-        "name": "LatestOperation",
+    {
-        "type": "文本",
+        "name": "IsolatedSource",
-        "desc": "实例的最新操作。例：StopInstances、ResetInstance。 注意：此字段可能返回 null，表示取不到有效值。",
+        "type": "String",
-        "example": "ResetInstancesType"
+        "desc": "ARREAR：表示欠费隔离\nEXPIRE：表示到期隔离\nMANMADE：表示主动退还隔离\nNOTISOLATED：表示未隔离",
-    },
+        "example": ""
-    {
+    },
-        "name": "LatestOperationState",
+    {
-        "type": "文本",
+        "name": "GPUInfo",
-        "desc": "实例的最新操作状态。取值范围： SUCCESS：表示操作成功 OPERATING：表示操作执行中 FAILED：表示操作失败 注意：此字段可能返回 null，表示取不到有效值。",
+        "type": "GPUInfo",
-        "example": "SUCCESS"
+        "desc": "GPU信息。如果是gpu类型子机，该值会返回GPU信息，如果是其他类型子机则不返回。注意：此字段可能返回 null，表示取不到有效值。",
-    },
+        "example": ""
-    {
+    },
-        "name": "LatestOperationRequestId",
+    {
-        "type": "文本",
+        "name": "LicenseType",
-        "desc": "实例最新操作的唯一请求 ID。 注意：此字段可能返回 null，表示取不到有效值。",
+        "type": "String",
-        "example": "c7de1287-061d-4ace-8caf-6ad8e5a2f29a"
+        "desc": "实例的操作系统许可类型，默认为TencentCloud",
-    },
+        "example": "TencentCloud"
-    {
+    },
-        "name": "DisasterRecoverGroupId",
+    {
-        "type": "文本",
+        "name": "DisableApiTermination",
-        "desc": "分散置放群组ID。 注意：此字段可能返回 null，表示取不到有效值。",
+        "type": "Boolean",
-        "example": ""
+        "desc": "TRUE：表示开启实例保护，不允许通过api接口删除实例\nFALSE：表示关闭实例保护，允许通过api接口删除实例默认取值：FALSE。",
-    },
+        "example": "false"
-    {
+    },
-        "name": "IPv6Addresses",
+    {
-        "type": "文本、多值",
+        "name": "DefaultLoginUser",
-        "desc": "实例的IPv6地址。 注意：此字段可能返回 null，表示取不到有效值。",
+        "type": "String",
-        "example": [
+        "desc": "默认登录用户。",
-            "2001:0db8:86a3:08d3:1319:8a2e:0370:7344"
+        "example": "root"
-        ]
+    },
-    },
+    {
-    {
+        "name": "DefaultLoginPort",
-        "name": "CamRoleName",
+        "type": "Integer",
-        "type": "文本",
+        "desc": "默认登录端口。",
-        "desc": "CAM角色名。 注意：此字段可能返回 null，表示取不到有效值。",
+        "example": "22"
-        "example": ""
+    },
-    },
+    {
-    {
+        "name": "LatestOperationErrorMsg",
-        "name": "HpcClusterId",
+        "type": "String",
-        "type": "文本",
+        "desc": "实例的最新操作错误信息。注意：此字段可能返回 null，表示取不到有效值。",
-        "desc": "高性能计算集群ID。 注意：此字段可能返回 null，表示取不到有效值。",
+        "example": "None"
-        "example": ""
+    }
-    },
-    {
-        "name": "RdmaIpAddresses",
-        "type": "文本、多值",
-        "desc": "高性能计算集群IP列表。 注意：此字段可能返回 null，表示取不到有效值。",
-        "example": []
-    },
-    {
-        "name": "IsolatedSource",
-        "type": "文本",
-        "desc": "实例隔离类型。取值范围： ARREAR：表示欠费隔离 EXPIRE：表示到期隔离 MANMADE：表示主动退还隔离 NOTISOLATED：表示未隔离 注意：此字段可能返回 null，表示取不到有效值。",
-        "example": "NOTISOLATED"
-    },
-    {
-        "name": "GPUInfo",
-        "type": "json",
-        "desc": "GPU信息。如果是gpu类型子机，该值会返回GPU信息，如果是其他类型子机则不返回。 注意：此字段可能返回 null，表示取不到有效值。",
-        "example": null
-    },
-    {
-        "name": "LicenseType",
-        "type": "文本",
-        "desc": "实例的操作系统许可类型，默认为TencentCloud",
-        "example": null
-    },
-    {
-        "name": "DisableApiTermination",
-        "type": "Boolean",
-        "desc": "实例销毁保护标志，表示是否允许通过api接口删除实例。取值范围： TRUE：表示开启实例保护，不允许通过api接口删除实例 FALSE：表示关闭实例保护，允许通过api接口删除实例 默认取值：FALSE。",
-        "example": null
-    },
-    {
-        "name": "DefaultLoginUser",
-        "type": "文本",
-        "desc": "默认登录用户。",
-        "example": null
-    },
-    {
-        "name": "DefaultLoginPort",
-        "type": "整数",
-        "desc": "默认登录端口。",
-        "example": null
-    },
-    {
-        "name": "LatestOperationErrorMsg",
-        "type": "文本",
-        "desc": "实例的最新操作错误信息。 注意：此字段可能返回 null，表示取不到有效值。",
-        "example": null
-    }
 ]

cmdb-api/api/lib/cmdb/cache.py

@@ -2,16 +2,27 @@
 
 from __future__ import unicode_literals
 
-import requests
+from collections import defaultdict
-from flask import current_app
 
+import datetime
+import os
+import yaml
+from flask import current_app
+import json
 from api.extensions import cache
 from api.extensions import db
 from api.lib.cmdb.custom_dashboard import CustomDashboardManager
-from api.models.cmdb import Attribute
+from api.models.cmdb import Attribute, AutoDiscoveryExecHistory
+from api.models.cmdb import AutoDiscoveryCI
+from api.models.cmdb import AutoDiscoveryCIType
+from api.models.cmdb import AutoDiscoveryCITypeRelation
+from api.models.cmdb import AutoDiscoveryCounter
+from api.models.cmdb import AutoDiscoveryRuleSyncHistory
 from api.models.cmdb import CI
 from api.models.cmdb import CIType
 from api.models.cmdb import CITypeAttribute
+from api.models.cmdb import PreferenceShowAttributes
+from api.models.cmdb import PreferenceTreeView
 from api.models.cmdb import RelationType
 
 
@@ -34,6 +45,7 @@ class AttributeCache(object):
             attr = attr or Attribute.get_by(alias=key, first=True, to_dict=False)
             if attr is not None:
                 cls.set(attr)
+
         return attr
 
     @classmethod
@@ -67,6 +79,7 @@ class CITypeCache(object):
             ct = ct or CIType.get_by(alias=key, first=True, to_dict=False)
             if ct is not None:
                 cls.set(ct)
+
         return ct
 
     @classmethod
@@ -98,6 +111,7 @@ class RelationTypeCache(object):
             ct = RelationType.get_by(name=key, first=True, to_dict=False) or RelationType.get_by_id(key)
             if ct is not None:
                 cls.set(ct)
+
         return ct
 
     @classmethod
@@ -133,12 +147,15 @@ class CITypeAttributesCache(object):
         attrs = attrs or cache.get(cls.PREFIX_ID.format(key))
         if not attrs:
             attrs = CITypeAttribute.get_by(type_id=key, to_dict=False)
+
             if not attrs:
                 ci_type = CIType.get_by(name=key, first=True, to_dict=False)
                 if ci_type is not None:
                     attrs = CITypeAttribute.get_by(type_id=ci_type.id, to_dict=False)
+
             if attrs is not None:
                 cls.set(key, attrs)
+
         return attrs
 
     @classmethod
@@ -155,13 +172,16 @@ class CITypeAttributesCache(object):
         attrs = attrs or cache.get(cls.PREFIX_ID2.format(key))
         if not attrs:
             attrs = CITypeAttribute.get_by(type_id=key, to_dict=False)
+
             if not attrs:
                 ci_type = CIType.get_by(name=key, first=True, to_dict=False)
                 if ci_type is not None:
                     attrs = CITypeAttribute.get_by(type_id=ci_type.id, to_dict=False)
+
             if attrs is not None:
                 attrs = [(i, AttributeCache.get(i.attr_id)) for i in attrs]
                 cls.set2(key, attrs)
+
         return attrs
 
     @classmethod
@@ -201,13 +221,13 @@ class CITypeAttributeCache(object):
 
     @classmethod
     def get(cls, type_id, attr_id):
-
         attr = cache.get(cls.PREFIX_ID.format(type_id, attr_id))
         attr = attr or cache.get(cls.PREFIX_ID.format(type_id, attr_id))
-        if not attr:
+        attr = attr or CITypeAttribute.get_by(type_id=type_id, attr_id=attr_id, first=True, to_dict=False)
-            attr = CITypeAttribute.get_by(type_id=type_id, attr_id=attr_id, first=True, to_dict=False)
+
-            if attr is not None:
+        if attr is not None:
-                cls.set(type_id, attr_id, attr)
+            cls.set(type_id, attr_id, attr)
+
         return attr
 
     @classmethod
@@ -220,7 +240,9 @@ class CITypeAttributeCache(object):
 
 
 class CMDBCounterCache(object):
-    KEY = 'CMDB::Counter'
+    KEY = 'CMDB::Counter::dashboard'
+    KEY2 = 'CMDB::Counter::adc'
+    KEY3 = 'CMDB::Counter::sub'
 
     @classmethod
     def get(cls):
@@ -233,7 +255,7 @@ class CMDBCounterCache(object):
 
     @classmethod
     def set(cls, result):
-        cache.set(cls.KEY, result, timeout=0)
+        cache.set(cls.KEY, json.loads(json.dumps(result)), timeout=0)
 
     @classmethod
     def reset(cls):
@@ -241,53 +263,83 @@ class CMDBCounterCache(object):
         result = {}
         for custom in customs:
             if custom['category'] == 0:
-                result[custom['id']] = cls.summary_counter(custom['type_id'])
+                res = cls.sum_counter(custom)
             elif custom['category'] == 1:
-                result[custom['id']] = cls.attribute_counter(custom['type_id'], custom['attr_id'])
+                res = cls.attribute_counter(custom)
-            elif custom['category'] == 2:
+            else:
-                result[custom['id']] = cls.relation_counter(custom['type_id'], custom['level'])
+                res = cls.relation_counter(custom.get('type_id'),
+                                           custom.get('level'),
+                                           custom.get('options', {}).get('filter', ''),
+                                           custom.get('options', {}).get('type_ids', ''))
+
+            if res:
+                result[custom['id']] = res
 
         cls.set(result)
 
-        return result
+        return json.loads(json.dumps(result))
 
     @classmethod
-    def update(cls, custom):
+    def update(cls, custom, flush=True):
         result = cache.get(cls.KEY) or {}
         if not result:
             result = cls.reset()
 
         if custom['category'] == 0:
-            result[custom['id']] = cls.summary_counter(custom['type_id'])
+            res = cls.sum_counter(custom)
         elif custom['category'] == 1:
-            result[custom['id']] = cls.attribute_counter(custom['type_id'], custom['attr_id'])
+            res = cls.attribute_counter(custom)
-        elif custom['category'] == 2:
+        else:
-            result[custom['id']] = cls.relation_counter(custom['type_id'], custom['level'])
+            res = cls.relation_counter(custom.get('type_id'),
+                                       custom.get('level'),
+                                       custom.get('options', {}).get('filter', ''),
+                                       custom.get('options', {}).get('type_ids', ''))
 
-        cls.set(result)
+        if res and flush:
+            result[custom['id']] = res
+            cls.set(result)
 
-    @staticmethod
+        return json.loads(json.dumps(res))
-    def summary_counter(type_id):
-        return db.session.query(CI.id).filter(CI.deleted.is_(False)).filter(CI.type_id == type_id).count()
 
-    @staticmethod
+    @classmethod
-    def relation_counter(type_id, level):
+    def relation_counter(cls, type_id, level, other_filer, type_ids):
+        from api.lib.cmdb.search.ci_relation.search import Search as RelSearch
+        from api.lib.cmdb.search import SearchError
+        from api.lib.cmdb.search.ci import search
+        from api.lib.cmdb.attribute import AttributeManager
 
-        uri = current_app.config.get('CMDB_API')
+        query = "_type:{}".format(type_id)
+        if other_filer:
+            query = "{},{}".format(query, other_filer)
+        s = search(query, count=1000000)
+        try:
+            type_names, _, _, _, _, _ = s.search()
+        except SearchError as e:
+            current_app.logger.error(e)
+            return
+        root_type = CITypeCache.get(type_id)
+        show_attr_id = root_type and root_type.show_id
+        show_attr = AttributeCache.get(show_attr_id)
 
-        type_names = requests.get("{}/ci/s?q=_type:{}&count=10000".format(uri, type_id)).json().get('result')
+        type_id_names = []
-        type_id_names = [(str(i.get('_id')), i.get(i.get('unique'))) for i in type_names]
+        for i in type_names:
+            attr_value = i.get(show_attr and show_attr.name) or i.get(i.get('unique'))
+            enum_map = AttributeManager.get_enum_map(show_attr_id or i.get('unique'))
 
-        url = "{}/ci_relations/statistics?root_ids={}&level={}".format(
+            type_id_names.append((str(i.get('_id')), enum_map.get(attr_value, attr_value)))
-            uri, ','.join([i[0] for i in type_id_names]), level)
+
-        stats = requests.get(url).json()
+        s = RelSearch([i[0] for i in type_id_names], level)
+        try:
+            stats = s.statistics(type_ids, need_filter=False)
+        except SearchError as e:
+            current_app.logger.error(e)
+            return
 
         id2name = dict(type_id_names)
         type_ids = set()
         for i in (stats.get('detail') or []):
             for j in stats['detail'][i]:
                 type_ids.add(j)
-
         for type_id in type_ids:
             _type = CITypeCache.get(type_id)
             id2name[type_id] = _type and _type.alias
@@ -306,10 +358,241 @@ class CMDBCounterCache(object):
 
         return result
 
+    @classmethod
+    def attribute_counter(cls, custom):
+        from api.lib.cmdb.search import SearchError
+        from api.lib.cmdb.search.ci import search
+        from api.lib.cmdb.utils import ValueTypeMap
+        from api.lib.cmdb.attribute import AttributeManager
+
+        custom.setdefault('options', {})
+        type_id = custom.get('type_id')
+        attr_id = custom.get('attr_id')
+        type_ids = custom['options'].get('type_ids') or (type_id and [type_id])
+        attr_ids = list(map(str, custom['options'].get('attr_ids') or (attr_id and [attr_id])))
+        try:
+            attr2value_type = [AttributeCache.get(i).value_type for i in attr_ids]
+        except AttributeError:
+            return
+
+        other_filter = custom['options'].get('filter')
+        other_filter = "{}".format(other_filter) if other_filter else ''
+
+        if custom['options'].get('ret') == 'cis':
+            enum_map = {}
+            for _attr_id in attr_ids:
+                _attr = AttributeCache.get(_attr_id)
+                if _attr:
+                    enum_map[_attr.alias] = AttributeManager.get_enum_map(_attr_id)
+
+            query = "_type:({}),{}".format(";".join(map(str, type_ids)), other_filter)
+            s = search(query, fl=attr_ids, ret_key='alias', count=100)
+            try:
+                cis, _, _, _, _, _ = s.search()
+                cis = [{k: (enum_map.get(k) or {}).get(v, v) for k, v in ci.items()} for ci in cis]
+            except SearchError as e:
+                current_app.logger.error(e)
+                return
+
+            return cis
+
+        origin_result = dict()
+        result = dict()
+        # level = 1
+        query = "_type:({}),{}".format(";".join(map(str, type_ids)), other_filter)
+        s = search(query, fl=attr_ids, facet=[attr_ids[0]], count=1)
+        try:
+            _, _, _, _, _, facet = s.search()
+        except SearchError as e:
+            current_app.logger.error(e)
+            return
+
+        enum_map1 = AttributeManager.get_enum_map(attr_ids[0])
+        for i in (list(facet.values()) or [[]])[0]:
+            k = ValueTypeMap.serialize2[attr2value_type[0]](str(i[0]))
+            result[enum_map1.get(k, k)] = i[1]
+            origin_result[k] = i[1]
+        if len(attr_ids) == 1:
+            return result
+
+        # level = 2
+        enum_map2 = AttributeManager.get_enum_map(attr_ids[1])
+        for v in origin_result:
+            query = "_type:({}),{},{}:{}".format(";".join(map(str, type_ids)), other_filter, attr_ids[0], v)
+            s = search(query, fl=attr_ids, facet=[attr_ids[1]], count=1)
+            try:
+                _, _, _, _, _, facet = s.search()
+            except SearchError as e:
+                current_app.logger.error(e)
+                return
+            result[enum_map1.get(v, v)] = dict()
+            origin_result[v] = dict()
+            for i in (list(facet.values()) or [[]])[0]:
+                k = ValueTypeMap.serialize2[attr2value_type[1]](str(i[0]))
+                result[enum_map1.get(v, v)][enum_map2.get(k, k)] = i[1]
+                origin_result[v][k] = i[1]
+
+        if len(attr_ids) == 2:
+            return result
+
+        # level = 3
+        enum_map3 = AttributeManager.get_enum_map(attr_ids[2])
+        for v1 in origin_result:
+            if not isinstance(result[enum_map1.get(v1, v1)], dict):
+                continue
+            for v2 in origin_result[v1]:
+                query = "_type:({}),{},{}:{},{}:{}".format(";".join(map(str, type_ids)), other_filter,
+                                                           attr_ids[0], v1, attr_ids[1], v2)
+                s = search(query, fl=attr_ids, facet=[attr_ids[2]], count=1)
+                try:
+                    _, _, _, _, _, facet = s.search()
+                except SearchError as e:
+                    current_app.logger.error(e)
+                    return
+                result[enum_map1.get(v1, v1)][enum_map2.get(v2, v2)] = dict()
+                for i in (list(facet.values()) or [[]])[0]:
+                    k = ValueTypeMap.serialize2[attr2value_type[2]](str(i[0]))
+                    result[enum_map1.get(v1, v1)][enum_map2.get(v2, v2)][enum_map3.get(k, k)] = i[1]
+
+        return result
+
     @staticmethod
-    def attribute_counter(type_id, attr_id):
+    def sum_counter(custom):
-        uri = current_app.config.get('CMDB_API')
+        from api.lib.cmdb.search import SearchError
-        url = "{}/ci/s?q=_type:{}&fl={}&facet={}".format(uri, type_id, attr_id, attr_id)
+        from api.lib.cmdb.search.ci import search
-        res = requests.get(url).json()
+
-        if res.get('facet'):
+        custom.setdefault('options', {})
-            return dict([i[:2] for i in list(res.get('facet').values())[0]])
+        type_id = custom.get('type_id')
+        type_ids = custom['options'].get('type_ids') or (type_id and [type_id])
+        other_filter = custom['options'].get('filter') or ''
+
+        query = "_type:({}),{}".format(";".join(map(str, type_ids)), other_filter)
+        s = search(query, count=1)
+        try:
+            _, _, _, _, numfound, _ = s.search()
+        except SearchError as e:
+            current_app.logger.error(e)
+            return
+
+        return numfound
+
+    @classmethod
+    def flush_adc_counter(cls):
+        res = db.session.query(CI.type_id, CI.is_auto_discovery)
+        result = dict()
+        for i in res:
+            result.setdefault(i.type_id, dict(total=0, auto_discovery=0))
+            result[i.type_id]['total'] += 1
+            if i.is_auto_discovery:
+                result[i.type_id]['auto_discovery'] += 1
+
+        cache.set(cls.KEY2, result, timeout=0)
+
+        res = db.session.query(AutoDiscoveryCI.created_at,
+                               AutoDiscoveryCI.updated_at,
+                               AutoDiscoveryCI.adt_id,
+                               AutoDiscoveryCI.type_id,
+                               AutoDiscoveryCI.is_accept).filter(AutoDiscoveryCI.deleted.is_(False))
+
+        today = datetime.datetime.today()
+        this_month = datetime.datetime(today.year, today.month, 1)
+        last_month = this_month - datetime.timedelta(days=1)
+        last_month = datetime.datetime(last_month.year, last_month.month, 1)
+        this_week = today - datetime.timedelta(days=datetime.date.weekday(today))
+        this_week = datetime.datetime(this_week.year, this_week.month, this_week.day)
+        last_week = this_week - datetime.timedelta(days=7)
+        last_week = datetime.datetime(last_week.year, last_week.month, last_week.day)
+        result = dict()
+        for i in res:
+            if i.type_id not in result:
+                result[i.type_id] = dict(instance_count=0, accept_count=0,
+                                         this_month_count=0, this_week_count=0, last_month_count=0, last_week_count=0)
+
+                adts = AutoDiscoveryCIType.get_by(type_id=i.type_id, to_dict=False)
+                result[i.type_id]['rule_count'] = len(adts) + AutoDiscoveryCITypeRelation.get_by(
+                    ad_type_id=i.type_id, only_query=True).count()
+                result[i.type_id]['exec_target_count'] = len(
+                    set([j.oneagent_id for adt in adts for j in db.session.query(
+                        AutoDiscoveryRuleSyncHistory.oneagent_id).filter(
+                        AutoDiscoveryRuleSyncHistory.adt_id == adt.id)]))
+
+            result[i.type_id]['instance_count'] += 1
+            if i.is_accept:
+                result[i.type_id]['accept_count'] += 1
+
+            if last_month <= i.created_at < this_month:
+                result[i.type_id]['last_month_count'] += 1
+            elif i.created_at >= this_month:
+                result[i.type_id]['this_month_count'] += 1
+
+            if last_week <= i.created_at < this_week:
+                result[i.type_id]['last_week_count'] += 1
+            elif i.created_at >= this_week:
+                result[i.type_id]['this_week_count'] += 1
+
+        for type_id in result:
+            existed = AutoDiscoveryCounter.get_by(type_id=type_id, first=True, to_dict=False)
+            if existed is None:
+                AutoDiscoveryCounter.create(type_id=type_id, **result[type_id])
+            else:
+                existed.update(**result[type_id])
+
+        for i in AutoDiscoveryCounter.get_by(to_dict=False):
+            if i.type_id not in result:
+                i.delete()
+
+    @classmethod
+    def clear_ad_exec_history(cls):
+        ci_types = CIType.get_by(to_dict=False)
+        for ci_type in ci_types:
+            for i in AutoDiscoveryExecHistory.get_by(type_id=ci_type.id, only_query=True).order_by(
+                    AutoDiscoveryExecHistory.id.desc()).offset(50000):
+                i.delete(commit=False)
+            db.session.commit()
+
+    @classmethod
+    def get_adc_counter(cls):
+        return cache.get(cls.KEY2) or cls.flush_adc_counter()
+
+    @classmethod
+    def flush_sub_counter(cls):
+        result = dict(type_id2users=defaultdict(list))
+
+        types = db.session.query(PreferenceShowAttributes.type_id,
+                                 PreferenceShowAttributes.uid, PreferenceShowAttributes.created_at).filter(
+            PreferenceShowAttributes.deleted.is_(False)).group_by(
+            PreferenceShowAttributes.uid, PreferenceShowAttributes.type_id)
+        for i in types:
+            result['type_id2users'][i.type_id].append(i.uid)
+
+        types = PreferenceTreeView.get_by(to_dict=False)
+        for i in types:
+
+            if i.uid not in result['type_id2users'][i.type_id]:
+                result['type_id2users'][i.type_id].append(i.uid)
+
+        cache.set(cls.KEY3, result, timeout=0)
+
+        return result
+
+    @classmethod
+    def get_sub_counter(cls):
+        return cache.get(cls.KEY3) or cls.flush_sub_counter()
+
+
+class AutoDiscoveryMappingCache(object):
+    PREFIX = 'CMDB::AutoDiscovery::Mapping::{}'
+
+    @classmethod
+    def get(cls, name):
+        res = cache.get(cls.PREFIX.format(name)) or {}
+        if not res:
+            path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
+                                "auto_discovery/mapping/{}.yaml".format(name))
+            if os.path.exists(path):
+                with open(path, 'r') as f:
+                    mapping = yaml.safe_load(f)
+                    res = mapping.get('mapping') or {}
+                    res and cache.set(cls.PREFIX.format(name), res, timeout=0)
+
+        return res

cmdb-api/api/lib/cmdb/const.py

@@ -1,6 +1,8 @@
 # -*- coding:utf-8 -*- 
 
 
+from flask_babel import lazy_gettext as _l
+
 from api.lib.utils import BaseEnum
 
 
@@ -12,6 +14,10 @@ class ValueTypeEnum(BaseEnum):
     DATE = "4"
     TIME = "5"
     JSON = "6"
+    PASSWORD = TEXT
+    LINK = TEXT
+    BOOL = "7"
+    REFERENCE = INT
 
 
 class ConstraintEnum(BaseEnum):
@@ -39,20 +45,23 @@ class OperateType(BaseEnum):
 
 
 class CITypeOperateType(BaseEnum):
-    ADD = "0"  # 新增模型
+    ADD = "0"  # add CIType
-    UPDATE = "1"  # 修改模型
+    UPDATE = "1"  # update CIType
-    DELETE = "2"  # 删除模型
+    DELETE = "2"  # delete CIType
-    ADD_ATTRIBUTE = "3"  # 新增属性
+    ADD_ATTRIBUTE = "3"
-    UPDATE_ATTRIBUTE = "4"  # 修改属性
+    UPDATE_ATTRIBUTE = "4"
-    DELETE_ATTRIBUTE = "5"  # 删除属性
+    DELETE_ATTRIBUTE = "5"
-    ADD_TRIGGER = "6"  # 新增触发器
+    ADD_TRIGGER = "6"
-    UPDATE_TRIGGER = "7"  # 修改触发器
+    UPDATE_TRIGGER = "7"
-    DELETE_TRIGGER = "8"  # 删除触发器
+    DELETE_TRIGGER = "8"
-    ADD_UNIQUE_CONSTRAINT = "9"  # 新增联合唯一
+    ADD_UNIQUE_CONSTRAINT = "9"
-    UPDATE_UNIQUE_CONSTRAINT = "10"  # 修改联合唯一
+    UPDATE_UNIQUE_CONSTRAINT = "10"
-    DELETE_UNIQUE_CONSTRAINT = "11"  # 删除联合唯一
+    DELETE_UNIQUE_CONSTRAINT = "11"
-    ADD_RELATION = "12"  # 新增关系
+    ADD_RELATION = "12"
-    DELETE_RELATION = "13"  # 删除关系
+    DELETE_RELATION = "13"
+    ADD_RECONCILIATION = "14"
+    UPDATE_RECONCILIATION = "15"
+    DELETE_RECONCILIATION = "16"
 
 
 class RetKey(BaseEnum):
@@ -67,6 +76,8 @@ class ResourceTypeEnum(BaseEnum):
     CI_TYPE_RELATION = "CITypeRelation"  # create/delete/grant
     RELATION_VIEW = "RelationView"  # read/update/delete/grant
     CI_FILTER = "CIFilter"  # read
+    PAGE = "page"  # read
+    TOPOLOGY_VIEW = "TopologyView"  # read/update/delete/grant
 
 
 class PermEnum(BaseEnum):
@@ -86,7 +97,8 @@ class RoleEnum(BaseEnum):
 class AutoDiscoveryType(BaseEnum):
     AGENT = "agent"
     SNMP = "snmp"
-    HTTP = "http"
+    HTTP = "http"  # cloud
+    COMPONENTS = "components"
 
 
 class AttributeDefaultValueEnum(BaseEnum):
@@ -95,9 +107,52 @@ class AttributeDefaultValueEnum(BaseEnum):
     AUTO_INC_ID = "$auto_inc_id"
 
 
+class ExecuteStatusEnum(BaseEnum):
+    COMPLETED = '0'
+    FAILED = '1'
+    RUNNING = '2'
+
+
+class RelationSourceEnum(BaseEnum):
+    ATTRIBUTE_VALUES = "0"
+    AUTO_DISCOVERY = "1"
+
+
+class BuiltinModelEnum(BaseEnum):
+    IPAM_SUBNET = "ipam_subnet"
+    IPAM_ADDRESS = "ipam_address"
+    IPAM_SCOPE = "ipam_scope"
+
+    DCIM_REGION = "dcim_region"
+    DCIM_IDC = "dcim_idc"
+    DCIM_SERVER_ROOM = "dcim_server_room"
+    DCIM_RACK = "dcim_rack"
+
+
+BUILTIN_ATTRIBUTES = {
+    "_updated_at": _l("Update Time"),
+    "_updated_by": _l("Updated By"),
+}
+
 CMDB_QUEUE = "one_cmdb_async"
 REDIS_PREFIX_CI = "ONE_CMDB"
 REDIS_PREFIX_CI_RELATION = "CMDB_CI_RELATION"
+REDIS_PREFIX_CI_RELATION2 = "CMDB_CI_RELATION2"
+
+BUILTIN_KEYWORDS = {'id', '_id', 'ci_id', 'type', '_type', 'ci_type', 'ticket_id', *BUILTIN_ATTRIBUTES.keys()}
+
+
+class SysComputedAttributes(object):
+    from api.lib.cmdb.ipam.const import SubnetBuiltinAttributes
+    type2attr = {
+        BuiltinModelEnum.IPAM_SUBNET: {
+            SubnetBuiltinAttributes.HOSTS_COUNT,
+            SubnetBuiltinAttributes.ASSIGN_COUNT,
+            SubnetBuiltinAttributes.USED_COUNT,
+            SubnetBuiltinAttributes.FREE_COUNT
+        }
+    }
+
 
 L_TYPE = None
 L_CI = None

cmdb-api/api/lib/cmdb/custom_dashboard.py

@@ -14,6 +14,14 @@ class CustomDashboardManager(object):
     def get():
         return sorted(CustomDashboard.get_by(to_dict=True), key=lambda x: (x["category"], x['order']))
 
+    @staticmethod
+    def preview(**kwargs):
+        from api.lib.cmdb.cache import CMDBCounterCache
+
+        res = CMDBCounterCache.update(kwargs, flush=False)
+
+        return res
+
     @staticmethod
     def add(**kwargs):
         from api.lib.cmdb.cache import CMDBCounterCache
@@ -23,9 +31,9 @@ class CustomDashboardManager(object):
 
         new = CustomDashboard.create(**kwargs)
 
-        CMDBCounterCache.update(new.to_dict())
+        res = CMDBCounterCache.update(new.to_dict())
 
-        return new
+        return new, res
 
     @staticmethod
     def update(_id, **kwargs):
@@ -35,9 +43,9 @@ class CustomDashboardManager(object):
 
         new = existed.update(**kwargs)
 
-        CMDBCounterCache.update(new.to_dict())
+        res = CMDBCounterCache.update(new.to_dict())
 
-        return new
+        return new, res
 
     @staticmethod
     def batch_update(id2options):

cmdb-api/api/lib/cmdb/dcim/__init__.py

@@ -0,0 +1 @@
+# -*- coding:utf-8 -*-

cmdb-api/api/lib/cmdb/dcim/base.py

@@ -0,0 +1,33 @@
+# -*- coding:utf-8 -*-
+
+from api.lib.cmdb.ci import CIManager
+from api.lib.cmdb.ci import CIRelationManager
+from api.lib.cmdb.const import ExistPolicy
+
+
+class DCIMBase(object):
+    def __init__(self):
+        self.type_id = None
+
+    @staticmethod
+    def add_relation(parent_id, child_id):
+        if not parent_id or not child_id:
+            return
+
+        CIRelationManager().add(parent_id, child_id, valid=False, apply_async=False)
+
+    def add(self, parent_id, **kwargs):
+        ci_id = CIManager().add(self.type_id, exist_policy=ExistPolicy.REJECT, **kwargs)
+
+        if parent_id:
+            self.add_relation(parent_id, ci_id)
+
+        return ci_id
+
+    @classmethod
+    def update(cls, _id, **kwargs):
+        CIManager().update(_id, **kwargs)
+
+    @classmethod
+    def delete(cls, _id):
+        CIManager().delete(_id)

cmdb-api/api/lib/cmdb/dcim/const.py

@@ -0,0 +1,17 @@
+# -*- coding:utf-8 -*-
+
+
+from api.lib.utils import BaseEnum
+
+
+class RackBuiltinAttributes(BaseEnum):
+    U_COUNT = 'u_count'
+    U_START = 'u_start'
+    FREE_U_COUNT = 'free_u_count'
+    U_SLOT_ABNORMAL = 'u_slot_abnormal'
+
+
+class OperateTypeEnum(BaseEnum):
+    ADD_DEVICE = "0"
+    REMOVE_DEVICE = "1"
+    MOVE_DEVICE = "2"

cmdb-api/api/lib/cmdb/dcim/history.py

@@ -0,0 +1,40 @@
+from flask_login import current_user
+
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.cache import CITypeCache
+from api.lib.cmdb.ci import CIManager
+from api.lib.mixin import DBMixin
+from api.models.cmdb import DCIMOperationHistory
+
+
+class OperateHistoryManager(DBMixin):
+    cls = DCIMOperationHistory
+
+    @classmethod
+    def search(cls, page, page_size, fl=None, only_query=False, reverse=False, count_query=False,
+               last_size=None, **kwargs):
+        numfound, result = super(OperateHistoryManager, cls).search(page, page_size, fl, only_query, reverse,
+                                                                    count_query, last_size, **kwargs)
+
+        ci_ids = [i['ci_id'] for i in result]
+        id2ci = {i['_id']: i for i in (CIManager.get_cis_by_ids(ci_ids) or []) if i}
+        type2show_key = dict()
+        for i in id2ci.values():
+            if i.get('_type') not in type2show_key:
+                ci_type = CITypeCache.get(i.get('_type'))
+                if ci_type:
+                    show_key = AttributeCache.get(ci_type.show_id or ci_type.unique_id)
+                    type2show_key[i['_type']] = show_key and show_key.name
+
+        return numfound, result, id2ci, type2show_key
+
+    def _can_add(self, **kwargs):
+        kwargs['uid'] = current_user.uid
+
+        return kwargs
+
+    def _can_update(self, **kwargs):
+        pass
+
+    def _can_delete(self, **kwargs):
+        pass

cmdb-api/api/lib/cmdb/dcim/idc.py

@@ -0,0 +1,19 @@
+# -*- coding:utf-8 -*-
+
+
+from flask import abort
+
+from api.lib.cmdb.cache import CITypeCache
+from api.lib.cmdb.const import BuiltinModelEnum
+from api.lib.cmdb.dcim.base import DCIMBase
+from api.lib.cmdb.resp_format import ErrFormat
+
+
+class IDCManager(DCIMBase):
+    def __init__(self):
+        super(IDCManager, self).__init__()
+
+        self.ci_type = CITypeCache.get(BuiltinModelEnum.DCIM_IDC) or abort(
+            404, ErrFormat.dcim_builtin_model_not_found.format(BuiltinModelEnum.DCIM_IDC))
+
+        self.type_id = self.ci_type.id

cmdb-api/api/lib/cmdb/dcim/rack.py

@@ -0,0 +1,182 @@
+# -*- coding:utf-8 -*-
+
+import itertools
+import redis_lock
+from flask import abort
+
+from api.extensions import rd
+from api.lib.cmdb.cache import CITypeCache
+from api.lib.cmdb.ci import CIManager
+from api.lib.cmdb.ci import CIRelationManager
+from api.lib.cmdb.const import BuiltinModelEnum
+from api.lib.cmdb.dcim.base import DCIMBase
+from api.lib.cmdb.dcim.const import OperateTypeEnum
+from api.lib.cmdb.dcim.const import RackBuiltinAttributes
+from api.lib.cmdb.dcim.history import OperateHistoryManager
+from api.lib.cmdb.resp_format import ErrFormat
+from api.lib.cmdb.search.ci.db.search import Search as SearchFromDB
+from api.lib.cmdb.search.ci_relation.search import Search as RelationSearch
+
+
+class RackManager(DCIMBase):
+    def __init__(self):
+        super(RackManager, self).__init__()
+
+        self.ci_type = CITypeCache.get(BuiltinModelEnum.DCIM_RACK) or abort(
+            404, ErrFormat.dcim_builtin_model_not_found.format(BuiltinModelEnum.DCIM_RACK))
+
+        self.type_id = self.ci_type.id
+
+    @classmethod
+    def update(cls, _id, **kwargs):
+        if RackBuiltinAttributes.U_COUNT in kwargs:
+            devices, _, _, _, _, _ = RelationSearch(
+                [_id],
+                level=[1],
+                fl=[RackBuiltinAttributes.U_COUNT, RackBuiltinAttributes.U_START],
+                count=1000000).search()
+            for device in devices:
+                u_start = device.get(RackBuiltinAttributes.U_START)
+                u_count = device.get(RackBuiltinAttributes.U_COUNT) or 2
+                if u_start and u_start + u_count - 1 > kwargs[RackBuiltinAttributes.U_COUNT]:
+                    return abort(400, ErrFormat.dcim_rack_u_count_invalid)
+
+        CIManager().update(_id, _sync=True, **kwargs)
+
+        if RackBuiltinAttributes.U_COUNT in kwargs:
+            payload = {RackBuiltinAttributes.FREE_U_COUNT: cls.calc_u_free_count(_id)}
+
+            CIManager().update(_id, _sync=True, **payload)
+
+    def delete(self, _id):
+        super(RackManager, self).delete(_id)
+
+        payload = {RackBuiltinAttributes.U_START: None}
+        _, _, second_cis = CIRelationManager.get_second_cis(_id, per_page='all')
+        for ci in second_cis:
+            CIManager().update(ci['_id'], **payload)
+
+    @staticmethod
+    def calc_u_free_count(rack_id, device_id=None, u_start=None, u_count=None):
+        rack = CIManager.get_ci_by_id(rack_id, need_children=False)
+        if not rack.get(RackBuiltinAttributes.U_COUNT):
+            return 0
+
+        if device_id is not None and u_count is None:
+            ci = CIManager().get_ci_by_id(device_id, need_children=False)
+            u_count = ci.get(RackBuiltinAttributes.U_COUNT) or 2
+
+        if u_start and u_start + u_count - 1 > rack.get(RackBuiltinAttributes.U_COUNT):
+            return abort(400, ErrFormat.dcim_rack_u_slot_invalid)
+
+        devices, _, _, _, _, _ = RelationSearch(
+            [rack_id],
+            level=[1],
+            fl=[RackBuiltinAttributes.U_COUNT, RackBuiltinAttributes.U_START],
+            count=1000000).search()
+
+        u_count_sum = 0
+        for device in devices:
+            u_count_sum += (device.get(RackBuiltinAttributes.U_COUNT) or 2)
+            if device_id is not None:
+                _u_start = device.get(RackBuiltinAttributes.U_START)
+                _u_count = device.get(RackBuiltinAttributes.U_COUNT) or 2
+                if not _u_start:
+                    continue
+
+                if device.get('_id') != device_id and set(range(u_start, u_start + u_count)) & set(
+                        range(_u_start, _u_start + _u_count)):
+                    return abort(400, ErrFormat.dcim_rack_u_slot_invalid)
+
+        return rack[RackBuiltinAttributes.U_COUNT] - u_count_sum
+
+    def check_u_slot(self):
+        racks, _, _, _, _, _ = SearchFromDB(
+            "_type:{}".format(self.type_id),
+            count=10000000,
+            fl=[RackBuiltinAttributes.U_START, RackBuiltinAttributes.U_COUNT, RackBuiltinAttributes.U_SLOT_ABNORMAL],
+            parent_node_perm_passed=True).search()
+
+        for rack in racks:
+            devices, _, _, _, _, _ = RelationSearch(
+                [rack['_id']],
+                level=[1],
+                fl=[RackBuiltinAttributes.U_COUNT, RackBuiltinAttributes.U_START],
+                count=1000000).search()
+
+            u_slot_sets = []
+            for device in devices:
+                u_start = device.get(RackBuiltinAttributes.U_START)
+                u_count = device.get(RackBuiltinAttributes.U_COUNT) or 2
+                if u_start is not None and str(u_start).isdigit():
+                    u_slot_sets.append(set(range(u_start, u_start + u_count)))
+
+            if len(u_slot_sets) > 1:
+                u_slot_abnormal = False
+                for a, b in itertools.combinations(u_slot_sets, 2):
+                    if a.intersection(b):
+                        u_slot_abnormal = True
+                        break
+                if u_slot_abnormal != rack.get(RackBuiltinAttributes.U_SLOT_ABNORMAL):
+                    payload = {RackBuiltinAttributes.U_SLOT_ABNORMAL: u_slot_abnormal}
+                    CIManager().update(rack['_id'], **payload)
+
+    def add_device(self, rack_id, device_id, u_start, u_count=None):
+        with (redis_lock.Lock(rd.r, "DCIM_RACK_OPERATE_{}".format(rack_id), expire=10)):
+            self.calc_u_free_count(rack_id, device_id, u_start, u_count)
+
+            self.add_relation(rack_id, device_id)
+
+            payload = {RackBuiltinAttributes.U_START: u_start}
+            if u_count:
+                payload[RackBuiltinAttributes.U_COUNT] = u_count
+            CIManager().update(device_id, _sync=True, **payload)
+
+            payload = {
+                RackBuiltinAttributes.FREE_U_COUNT: self.calc_u_free_count(rack_id, device_id, u_start, u_count)}
+            CIManager().update(rack_id, _sync=True, **payload)
+
+        OperateHistoryManager().add(operate_type=OperateTypeEnum.ADD_DEVICE, rack_id=rack_id, ci_id=device_id)
+
+    def remove_device(self, rack_id, device_id):
+        with (redis_lock.Lock(rd.r, "DCIM_RACK_OPERATE_{}".format(rack_id), expire=10)):
+            CIRelationManager.delete_3(rack_id, device_id, apply_async=False, valid=False)
+
+            payload = {RackBuiltinAttributes.FREE_U_COUNT: self.calc_u_free_count(rack_id)}
+            CIManager().update(rack_id, _sync=True, **payload)
+
+            payload = {RackBuiltinAttributes.U_START: None}
+            CIManager().update(device_id, _sync=True, **payload)
+
+        OperateHistoryManager().add(operate_type=OperateTypeEnum.REMOVE_DEVICE, rack_id=rack_id, ci_id=device_id)
+
+    def move_device(self, rack_id, device_id, to_u_start):
+        with (redis_lock.Lock(rd.r, "DCIM_RACK_OPERATE_{}".format(rack_id), expire=10)):
+            payload = {RackBuiltinAttributes.FREE_U_COUNT: self.calc_u_free_count(rack_id, device_id, to_u_start)}
+            CIManager().update(rack_id, _sync=True, **payload)
+
+            CIManager().update(device_id, _sync=True, **{RackBuiltinAttributes.U_START: to_u_start})
+
+        OperateHistoryManager().add(operate_type=OperateTypeEnum.MOVE_DEVICE, rack_id=rack_id, ci_id=device_id)
+
+    def migrate_device(self, rack_id, device_id, to_rack_id, to_u_start):
+        with (redis_lock.Lock(rd.r, "DCIM_RACK_OPERATE_{}".format(rack_id), expire=10)):
+            self.calc_u_free_count(to_rack_id, device_id, to_u_start)
+
+            if rack_id != to_rack_id:
+                CIRelationManager.delete_3(rack_id, device_id, apply_async=False, valid=False)
+
+                self.add_relation(to_rack_id, device_id)
+
+                payload = {
+                    RackBuiltinAttributes.FREE_U_COUNT: self.calc_u_free_count(to_rack_id, device_id, to_u_start)}
+                CIManager().update(to_rack_id, _sync=True, **payload)
+
+            CIManager().update(device_id, _sync=True, **{RackBuiltinAttributes.U_START: to_u_start})
+
+            if rack_id != to_rack_id:
+                payload = {RackBuiltinAttributes.FREE_U_COUNT: self.calc_u_free_count(rack_id)}
+                CIManager().update(rack_id, _sync=True, **payload)
+
+        OperateHistoryManager().add(operate_type=OperateTypeEnum.REMOVE_DEVICE, rack_id=rack_id, ci_id=device_id)
+        OperateHistoryManager().add(operate_type=OperateTypeEnum.ADD_DEVICE, rack_id=to_rack_id, ci_id=device_id)

cmdb-api/api/lib/cmdb/dcim/region.py

@@ -0,0 +1,29 @@
+# -*- coding:utf-8 -*-
+
+
+from flask import abort
+
+from api.lib.cmdb.cache import CITypeCache
+from api.lib.cmdb.ci import CIManager
+from api.lib.cmdb.const import BuiltinModelEnum
+from api.lib.cmdb.const import ExistPolicy
+from api.lib.cmdb.resp_format import ErrFormat
+
+
+class RegionManager(object):
+    def __init__(self):
+        self.ci_type = CITypeCache.get(BuiltinModelEnum.DCIM_REGION) or abort(
+            404, ErrFormat.dcim_builtin_model_not_found.format(BuiltinModelEnum.DCIM_REGION))
+
+        self.type_id = self.ci_type.id
+
+    def add(self, **kwargs):
+        return CIManager().add(self.type_id, exist_policy=ExistPolicy.REJECT, **kwargs)
+
+    @classmethod
+    def update(cls, _id, **kwargs):
+        CIManager().update(_id, **kwargs)
+
+    @classmethod
+    def delete(cls, _id):
+        CIManager().delete(_id)

cmdb-api/api/lib/cmdb/dcim/server_room.py

@@ -0,0 +1,56 @@
+# -*- coding:utf-8 -*-
+
+
+from flask import abort
+
+from api.lib.cmdb.cache import CITypeCache
+from api.lib.cmdb.const import BuiltinModelEnum
+from api.lib.cmdb.dcim.base import DCIMBase
+from api.lib.cmdb.dcim.const import RackBuiltinAttributes
+from api.lib.cmdb.resp_format import ErrFormat
+from api.lib.cmdb.search.ci.db.search import Search as SearchFromDB
+from api.models.cmdb import CI
+from api.models.cmdb import CIRelation
+
+
+class ServerRoomManager(DCIMBase):
+    def __init__(self):
+        super(ServerRoomManager, self).__init__()
+
+        self.ci_type = CITypeCache.get(BuiltinModelEnum.DCIM_SERVER_ROOM) or abort(
+            404, ErrFormat.dcim_builtin_model_not_found.format(BuiltinModelEnum.DCIM_SERVER_ROOM))
+        self.type_id = self.ci_type.id
+
+    @staticmethod
+    def get_racks(_id, q=None):
+        rack_type = CITypeCache.get(BuiltinModelEnum.DCIM_RACK) or abort(
+            404, ErrFormat.dcim_builtin_model_not_found.format(BuiltinModelEnum.DCIM_RACK))
+
+        relations = CIRelation.get_by(first_ci_id=_id, only_query=True).join(
+            CI, CI.id == CIRelation.second_ci_id).filter(CI.type_id == rack_type.id)
+        rack_ids = [i.second_ci_id for i in relations]
+
+        q = "_type:{}".format(rack_type.id) if not q else "_type:{},{}".format(rack_type.id, q)
+        if rack_ids:
+            response, _, _, _, numfound, _ = SearchFromDB(
+                q,
+                ci_ids=list(rack_ids),
+                count=1000000,
+                parent_node_perm_passed=True).search()
+        else:
+            response, numfound = [], 0
+
+        counter = dict(rack_count=numfound)
+        u_count = 0
+        free_u_count = 0
+        for i in response:
+            _u_count = i.get(RackBuiltinAttributes.U_COUNT) or 0
+            u_count += _u_count
+            free_u_count += (_u_count if i.get(RackBuiltinAttributes.FREE_U_COUNT) is None else
+                             i.get(RackBuiltinAttributes.FREE_U_COUNT))
+        counter["u_count"] = u_count
+        counter["u_used_count"] = u_count - free_u_count
+        counter["device_count"] = CIRelation.get_by(only_query=True).filter(
+            CIRelation.first_ci_id.in_(rack_ids)).count()
+
+        return counter, response

cmdb-api/api/lib/cmdb/dcim/tree_view.py

@@ -0,0 +1,85 @@
+# -*- coding:utf-8 -*-
+
+from collections import defaultdict
+
+from flask import abort
+
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.cache import CITypeCache
+from api.lib.cmdb.const import BuiltinModelEnum
+from api.lib.cmdb.resp_format import ErrFormat
+from api.lib.cmdb.search.ci.db.search import Search as SearchFromDB
+from api.models.cmdb import CI
+from api.models.cmdb import CIRelation
+
+
+class TreeViewManager(object):
+    @classmethod
+    def get(cls):
+        region_type = CITypeCache.get(BuiltinModelEnum.DCIM_REGION) or abort(
+            404, ErrFormat.dcim_builtin_model_not_found.format(BuiltinModelEnum.DCIM_REGION))
+
+        idc_type = CITypeCache.get(BuiltinModelEnum.DCIM_IDC) or abort(
+            404, ErrFormat.dcim_builtin_model_not_found.format(BuiltinModelEnum.DCIM_IDC))
+
+        server_room_type = CITypeCache.get(BuiltinModelEnum.DCIM_SERVER_ROOM) or abort(
+            404, ErrFormat.dcim_builtin_model_not_found.format(BuiltinModelEnum.DCIM_SERVER_ROOM))
+
+        rack_type = CITypeCache.get(BuiltinModelEnum.DCIM_RACK) or abort(
+            404, ErrFormat.dcim_builtin_model_not_found.format(BuiltinModelEnum.DCIM_RACK))
+
+        relations = defaultdict(set)
+        ids = set()
+        has_parent_ids = set()
+
+        for i in CIRelation.get_by(only_query=True).join(CI, CI.id == CIRelation.first_ci_id).filter(
+                CI.type_id.in_([region_type.id, idc_type.id])):
+            relations[i.first_ci_id].add(i.second_ci_id)
+            ids.add(i.first_ci_id)
+            ids.add(i.second_ci_id)
+            has_parent_ids.add(i.second_ci_id)
+
+        for i in CIRelation.get_by(only_query=True).join(
+                CI, CI.id == CIRelation.second_ci_id).filter(CI.type_id.in_([idc_type.id, server_room_type.id])):
+            relations[i.first_ci_id].add(i.second_ci_id)
+            ids.add(i.first_ci_id)
+            ids.add(i.second_ci_id)
+            has_parent_ids.add(i.second_ci_id)
+
+        for i in CI.get_by(only_query=True).filter(CI.type_id.in_([region_type.id, idc_type.id])):
+            ids.add(i.id)
+
+        for _id in ids:
+            if _id not in has_parent_ids:
+                relations[None].add(_id)
+
+        type2name = dict()
+        type2name[region_type.id] = AttributeCache.get(region_type.show_id or region_type.unique_id).name
+        type2name[idc_type.id] = AttributeCache.get(idc_type.show_id or idc_type.unique_id).name
+        type2name[server_room_type.id] = AttributeCache.get(server_room_type.show_id or server_room_type.unique_id).name
+
+        response, _, _, _, _, _ = SearchFromDB(
+            "_type:({})".format(";".join(map(str, [region_type.id, idc_type.id, server_room_type.id]))),
+            ci_ids=list(ids),
+            count=1000000,
+            fl=list(type2name.values()),
+            parent_node_perm_passed=True).search()
+        id2ci = {i['_id']: i for i in response}
+
+        def _build_tree(_tree, parent_id=None):
+            tree = []
+            for child_id in _tree.get(parent_id, []):
+                children = sorted(_build_tree(_tree, child_id), key=lambda x: x['_id'])
+                if not id2ci.get(child_id):
+                    continue
+                ci = id2ci[child_id]
+                if ci['ci_type'] == BuiltinModelEnum.DCIM_SERVER_ROOM:
+                    ci['rack_count'] = CIRelation.get_by(first_ci_id=child_id, only_query=True).join(
+                        CI, CI.id == CIRelation.second_ci_id).filter(CI.type_id == rack_type.id).count()
+
+                tree.append({'children': children, **ci})
+            return tree
+
+        result = sorted(_build_tree(relations), key=lambda x: x['_id'])
+
+        return result, type2name

cmdb-api/api/lib/cmdb/history.py

@@ -4,28 +4,32 @@
 import json
 
 from flask import abort
-from flask import g
+from flask_login import current_user
 
 from api.extensions import db
 from api.lib.cmdb.cache import AttributeCache
 from api.lib.cmdb.cache import RelationTypeCache
 from api.lib.cmdb.const import OperateType
+from api.lib.cmdb.cache import CITypeCache
 from api.lib.cmdb.perms import CIFilterPermsCRUD
 from api.lib.cmdb.resp_format import ErrFormat
 from api.lib.perm.acl.cache import UserCache
+from api.models.cmdb import CI
 from api.models.cmdb import Attribute
 from api.models.cmdb import AttributeHistory
 from api.models.cmdb import CIRelationHistory
+from api.models.cmdb import CITriggerHistory
 from api.models.cmdb import CITypeHistory
 from api.models.cmdb import CITypeTrigger
 from api.models.cmdb import CITypeUniqueConstraint
 from api.models.cmdb import OperationRecord
+from api.lib.cmdb.utils import TableMap
 
 
 class AttributeHistoryManger(object):
     @staticmethod
     def get_records_for_attributes(start, end, username, page, page_size, operate_type, type_id,
-                                   ci_id=None, attr_id=None):
+                                   ci_id=None, attr_id=None, ci_ids=None, more=False):
 
         records = db.session.query(OperationRecord, AttributeHistory).join(
             AttributeHistory, OperationRecord.id == AttributeHistory.record_id)
@@ -47,6 +51,9 @@ class AttributeHistoryManger(object):
         if ci_id is not None:
             records = records.filter(AttributeHistory.ci_id == ci_id)
 
+        if ci_ids and isinstance(ci_ids, list):
+            records = records.filter(AttributeHistory.ci_id.in_(ci_ids))
+
         if attr_id is not None:
             records = records.filter(AttributeHistory.attr_id == attr_id)
 
@@ -54,17 +61,39 @@ class AttributeHistoryManger(object):
         total = len(records)
 
         res = {}
+        show_attr_set = {}
+        show_attr_cache = {}
         for record in records:
             record_id = record.OperationRecord.id
+            type_id = record.OperationRecord.type_id
+            ci_id = record.AttributeHistory.ci_id
+            show_attr_set[ci_id] = None
+            show_attr = show_attr_cache.setdefault(
+                type_id, 
+                AttributeCache.get(
+                    CITypeCache.get(type_id).show_id or CITypeCache.get(type_id).unique_id) if CITypeCache.get(type_id) else None
+            )
+            if show_attr:
+                attr_table = TableMap(attr=show_attr).table
+                attr_record = attr_table.get_by(attr_id=show_attr.id, ci_id=ci_id, first=True, to_dict=False)
+                show_attr_set[ci_id] = attr_record.value if attr_record else None
+    
             attr_hist = record.AttributeHistory.to_dict()
             attr_hist['attr'] = AttributeCache.get(attr_hist['attr_id'])
             if attr_hist['attr']:
                 attr_hist['attr_name'] = attr_hist['attr'].name
                 attr_hist['attr_alias'] = attr_hist['attr'].alias
+                if more:
+                    attr_hist['is_list'] = attr_hist['attr'].is_list
+                    attr_hist['is_computed'] = attr_hist['attr'].is_computed
+                    attr_hist['is_password'] = attr_hist['attr'].is_password
+                    attr_hist['default'] = attr_hist['attr'].default
+                attr_hist['value_type'] = attr_hist['attr'].value_type
                 attr_hist.pop("attr")
 
             if record_id not in res:
                 record_dict = record.OperationRecord.to_dict()
+                record_dict['show_attr_value'] = show_attr_set.get(ci_id)
                 record_dict["user"] = UserCache.get(record_dict.get("uid"))
                 if record_dict["user"]:
                     record_dict['user'] = record_dict['user'].nickname
@@ -134,7 +163,7 @@ class AttributeHistoryManger(object):
         from api.lib.cmdb.ci import CIManager
         cis = CIManager().get_cis_by_ids(list(ci_ids),
                                          unique_required=True)
-        cis = {i['_id']: i for i in cis}
+        cis = {i['_id']: i for i in cis if i}
 
         return total, res, cis
 
@@ -160,12 +189,14 @@ class AttributeHistoryManger(object):
             record = i.OperationRecord
             item = dict(attr_name=attr.name,
                         attr_alias=attr.alias,
+                        value_type=attr.value_type,
                         operate_type=hist.operate_type,
                         username=user and user.nickname,
                         old=hist.old,
                         new=hist.new,
                         created_at=record.created_at.strftime('%Y-%m-%d %H:%M:%S'),
                         record_id=record.id,
+                        ticket_id=record.ticket_id,
                         hid=hist.id
                         )
             result.append(item)
@@ -176,8 +207,8 @@ class AttributeHistoryManger(object):
     def get_record_detail(record_id):
         from api.lib.cmdb.ci import CIManager
 
-        record = OperationRecord.get_by_id(record_id) or \
+        record = (OperationRecord.get_by_id(record_id) or
-                 abort(404, ErrFormat.record_not_found.format("id={}".format(record_id)))
+                  abort(404, ErrFormat.record_not_found.format("id={}".format(record_id))))
 
         username = UserCache.get(record.uid).nickname or UserCache.get(record.uid).username
         timestamp = record.created_at.strftime("%Y-%m-%d %H:%M:%S")
@@ -199,9 +230,9 @@ class AttributeHistoryManger(object):
         return username, timestamp, attr_dict, rel_dict
 
     @staticmethod
-    def add(record_id, ci_id, history_list, type_id=None, flush=False, commit=True):
+    def add(record_id, ci_id, history_list, type_id=None, ticket_id=None, flush=False, commit=True):
         if record_id is None:
-            record = OperationRecord.create(uid=g.user.uid, type_id=type_id)
+            record = OperationRecord.create(uid=current_user.uid, type_id=type_id, ticket_id=ticket_id)
             record_id = record.id
 
         for attr_id, operate_type, old, new in history_list or []:
@@ -219,8 +250,8 @@ class AttributeHistoryManger(object):
 
 class CIRelationHistoryManager(object):
     @staticmethod
-    def add(rel_obj, operate_type=OperateType.ADD):
+    def add(rel_obj, operate_type=OperateType.ADD, uid=None):
-        record = OperationRecord.create(uid=g.user.uid)
+        record = OperationRecord.create(uid=uid or current_user.uid)
 
         CIRelationHistory.create(relation_id=rel_obj.id,
                                  record_id=record.id,
@@ -269,7 +300,7 @@ class CITypeHistoryManager(object):
         return numfound, result
 
     @staticmethod
-    def add(operate_type, type_id, attr_id=None, trigger_id=None, unique_constraint_id=None, change=None):
+    def add(operate_type, type_id, attr_id=None, trigger_id=None, unique_constraint_id=None, change=None, rc_id=None):
         if type_id is None and attr_id is not None:
             from api.models.cmdb import CITypeAttribute
             type_ids = [i.type_id for i in CITypeAttribute.get_by(attr_id=attr_id, to_dict=False)]
@@ -279,10 +310,76 @@ class CITypeHistoryManager(object):
         for _type_id in type_ids:
             payload = dict(operate_type=operate_type,
                            type_id=_type_id,
-                           uid=g.user.uid,
+                           uid=current_user.uid,
                            attr_id=attr_id,
                            trigger_id=trigger_id,
+                           rc_id=rc_id,
                            unique_constraint_id=unique_constraint_id,
                            change=change)
 
             CITypeHistory.create(**payload)
+
+
+class CITriggerHistoryManager(object):
+    @staticmethod
+    def get(page, page_size, type_id=None, trigger_id=None, operate_type=None):
+        query = CITriggerHistory.get_by(only_query=True)
+        if type_id:
+            query = query.join(CI, CI.id == CITriggerHistory.ci_id).filter(CI.type_id == type_id)
+
+        if trigger_id:
+            query = query.filter(CITriggerHistory.trigger_id == trigger_id)
+
+        if operate_type:
+            query = query.filter(CITriggerHistory.operate_type == operate_type)
+
+        numfound = query.count()
+
+        query = query.order_by(CITriggerHistory.id.desc())
+        result = query.offset((page - 1) * page_size).limit(page_size)
+        result = [i.to_dict() for i in result]
+        for res in result:
+            if res.get('trigger_id'):
+                trigger = CITypeTrigger.get_by_id(res['trigger_id'])
+                res['trigger'] = trigger and trigger.to_dict()
+
+        return numfound, result
+
+    @staticmethod
+    def get_by_ci_id(ci_id):
+        res = db.session.query(CITriggerHistory, CITypeTrigger).join(
+            CITypeTrigger, CITypeTrigger.id == CITriggerHistory.trigger_id).filter(
+            CITriggerHistory.ci_id == ci_id).order_by(CITriggerHistory.id.desc())
+
+        result = []
+        id2trigger = dict()
+        for i in res:
+            hist = i.CITriggerHistory
+            item = dict(is_ok=hist.is_ok,
+                        operate_type=hist.operate_type,
+                        notify=hist.notify,
+                        trigger_id=hist.trigger_id,
+                        trigger_name=hist.trigger_name,
+                        webhook=hist.webhook,
+                        created_at=hist.created_at.strftime('%Y-%m-%d %H:%M:%S'),
+                        record_id=hist.record_id,
+                        hid=hist.id
+                        )
+            if i.CITypeTrigger.id not in id2trigger:
+                id2trigger[i.CITypeTrigger.id] = i.CITypeTrigger.to_dict()
+
+            result.append(item)
+
+        return dict(items=result, id2trigger=id2trigger)
+
+    @staticmethod
+    def add(operate_type, record_id, ci_id, trigger_id, trigger_name, is_ok=False, notify=None, webhook=None):
+
+        CITriggerHistory.create(operate_type=operate_type,
+                                record_id=record_id,
+                                ci_id=ci_id,
+                                trigger_id=trigger_id,
+                                trigger_name=trigger_name,
+                                is_ok=is_ok,
+                                notify=notify,
+                                webhook=webhook)

cmdb-api/api/lib/cmdb/ipam/__init__.py

@@ -0,0 +1 @@
+# -*- coding:utf-8 -*-

cmdb-api/api/lib/cmdb/ipam/address.py

@@ -0,0 +1,132 @@
+# -*- coding:utf-8 -*-
+
+import redis_lock
+from flask import abort
+
+from api.extensions import rd
+from api.lib.cmdb.cache import CITypeCache
+from api.lib.cmdb.ci import CIManager
+from api.lib.cmdb.ci import CIRelationManager
+from api.lib.cmdb.const import BuiltinModelEnum
+from api.lib.cmdb.ipam.const import IPAddressAssignStatus
+from api.lib.cmdb.ipam.const import IPAddressBuiltinAttributes
+from api.lib.cmdb.ipam.const import OperateTypeEnum
+from api.lib.cmdb.ipam.const import SubnetBuiltinAttributes
+from api.lib.cmdb.ipam.history import OperateHistoryManager
+from api.lib.cmdb.resp_format import ErrFormat
+from api.lib.cmdb.search.ci.db.search import Search as SearchFromDB
+from api.lib.cmdb.search.ci_relation.search import Search as RelationSearch
+
+
+class IpAddressManager(object):
+    def __init__(self):
+        self.ci_type = CITypeCache.get(BuiltinModelEnum.IPAM_ADDRESS) or abort(
+            404, ErrFormat.ipam_address_model_not_found.format(BuiltinModelEnum.IPAM_ADDRESS))
+
+        self.type_id = self.ci_type.id
+
+    @staticmethod
+    def list_ip_address(parent_id):
+        numfound, _, result = CIRelationManager.get_second_cis(parent_id, per_page="all")
+
+        return numfound, result
+
+    def _get_cis(self, subnet_id, ips):
+
+        q = "_type:{},{}:({})".format(self.type_id, IPAddressBuiltinAttributes.IP, ";".join(ips or []))
+
+        response, _, _, _, _, _ = RelationSearch([subnet_id], level=[1], query=q, count=1000000).search()
+
+        return response
+
+    @staticmethod
+    def _add_relation(parent_id, child_id):
+        if not parent_id or not child_id:
+            return
+
+        CIRelationManager().add(parent_id, child_id, valid=False, apply_async=False)
+
+    @staticmethod
+    def calc_used_count(subnet_id):
+        q = "{}:(0;2),-{}:true".format(IPAddressBuiltinAttributes.ASSIGN_STATUS, IPAddressBuiltinAttributes.IS_USED)
+
+        return len(set(RelationSearch([subnet_id], level=[1], query=q, count=1000000).search(only_ids=True) or []))
+
+    @staticmethod
+    def _calc_assign_count(subnet_id):
+        q = "{}:(0;2)".format(IPAddressBuiltinAttributes.ASSIGN_STATUS)
+
+        return len(set(RelationSearch([subnet_id], level=[1], query=q, count=1000000).search(only_ids=True) or []))
+
+    def _update_subnet_count(self, subnet_id, assign_count_computed, used_count=None):
+        payload = {}
+
+        cur = CIManager.get_ci_by_id(subnet_id, need_children=False)
+        if assign_count_computed:
+            payload[SubnetBuiltinAttributes.ASSIGN_COUNT] = self._calc_assign_count(subnet_id)
+        if used_count is not None:
+            payload[SubnetBuiltinAttributes.USED_COUNT] = used_count
+
+        payload[SubnetBuiltinAttributes.FREE_COUNT] = (cur[SubnetBuiltinAttributes.HOSTS_COUNT] -
+                                                       self.calc_used_count(subnet_id))
+        CIManager().update(subnet_id, **payload)
+
+    def assign_ips(self, ips, subnet_id, cidr, **kwargs):
+        """
+
+        :param ips: ip list
+        :param subnet_id: subnet id
+        :param cidr: subnet cidr
+        :param kwargs: other attributes for ip address
+        :return:
+        """
+        if subnet_id is not None:
+            subnet = CIManager.get_ci_by_id(subnet_id)
+        else:
+            cis, _, _, _, _, _ = SearchFromDB("_type:{},{}:{}".format(
+                BuiltinModelEnum.IPAM_SUBNET, SubnetBuiltinAttributes.CIDR, cidr),
+                parent_node_perm_passed=True).search()
+            if cis:
+                subnet = cis[0]
+                subnet_id = subnet['_id']
+            else:
+                return abort(400, ErrFormat.ipam_address_model_not_found)
+
+        with (redis_lock.Lock(rd.r, "IPAM_ASSIGN_ADDRESS_{}".format(subnet_id), expire=10)):
+            cis = self._get_cis(subnet_id, ips)
+            ip2ci = {ci[IPAddressBuiltinAttributes.IP]: ci for ci in cis}
+
+            ci_ids = []
+            for ip in ips:
+                kwargs['name'] = ip
+                kwargs[IPAddressBuiltinAttributes.IP] = ip
+                if ip not in ip2ci:
+                    ci_id = CIManager.add(self.type_id, _sync=True, **kwargs)
+                else:
+                    ci_id = ip2ci[ip]['_id']
+                    CIManager().update(ci_id, _sync=True, **kwargs)
+                ci_ids.append(ci_id)
+
+                self._add_relation(subnet_id, ci_id)
+
+            if ips and IPAddressBuiltinAttributes.ASSIGN_STATUS in kwargs:
+                self._update_subnet_count(subnet_id, True)
+
+            if ips and IPAddressBuiltinAttributes.IS_USED in kwargs:
+                q = "{}:true".format(IPAddressBuiltinAttributes.IS_USED)
+                cur_used_ids = RelationSearch([subnet_id], level=[1], query=q).search(only_ids=True)
+                for _id in set(cur_used_ids) - set(ci_ids):
+                    CIManager().update(_id, **{IPAddressBuiltinAttributes.IS_USED: False})
+
+                self._update_subnet_count(subnet_id, False, used_count=len(ips))
+
+        if kwargs.get(IPAddressBuiltinAttributes.ASSIGN_STATUS) in (
+                IPAddressAssignStatus.ASSIGNED, IPAddressAssignStatus.RESERVED):
+            OperateHistoryManager().add(operate_type=OperateTypeEnum.ASSIGN_ADDRESS,
+                                        cidr=subnet.get(SubnetBuiltinAttributes.CIDR),
+                                        description=" | ".join(ips))
+
+        elif kwargs.get(IPAddressBuiltinAttributes.ASSIGN_STATUS) == IPAddressAssignStatus.UNASSIGNED:
+            OperateHistoryManager().add(operate_type=OperateTypeEnum.REVOKE_ADDRESS,
+                                        cidr=subnet.get(SubnetBuiltinAttributes.CIDR),
+                                        description=" | ".join(ips))

cmdb-api/api/lib/cmdb/ipam/const.py

@@ -0,0 +1,35 @@
+# -*- coding:utf-8 -*-
+
+from api.lib.utils import BaseEnum
+
+
+class IPAddressAssignStatus(BaseEnum):
+    ASSIGNED = 0
+    UNASSIGNED = 1
+    RESERVED = 2
+
+
+class OperateTypeEnum(BaseEnum):
+    ADD_SCOPE = "0"
+    UPDATE_SCOPE = "1"
+    DELETE_SCOPE = "2"
+    ADD_SUBNET = "3"
+    UPDATE_SUBNET = "4"
+    DELETE_SUBNET = "5"
+    ASSIGN_ADDRESS = "6"
+    REVOKE_ADDRESS = "7"
+
+
+class SubnetBuiltinAttributes(BaseEnum):
+    NAME = 'name'
+    CIDR = 'cidr'
+    HOSTS_COUNT = 'hosts_count'
+    ASSIGN_COUNT = 'assign_count'
+    USED_COUNT = 'used_count'
+    FREE_COUNT = 'free_count'
+
+
+class IPAddressBuiltinAttributes(BaseEnum):
+    IP = 'ip'
+    ASSIGN_STATUS = 'assign_status'  # enum: 0 - assigned   1 - unassigned   2 - reserved
+    IS_USED = 'is_used'  # bool

cmdb-api/api/lib/cmdb/ipam/history.py

@@ -0,0 +1,61 @@
+# -*- coding:utf-8 -*-
+
+from flask_login import current_user
+
+from api.lib.cmdb.ipam.const import IPAddressBuiltinAttributes
+from api.lib.mixin import DBMixin
+from api.models.cmdb import IPAMOperationHistory
+from api.models.cmdb import IPAMSubnetScan
+from api.models.cmdb import IPAMSubnetScanHistory
+
+
+class OperateHistoryManager(DBMixin):
+    cls = IPAMOperationHistory
+
+    def _can_add(self, **kwargs):
+        kwargs['uid'] = current_user.uid
+
+        return kwargs
+
+    def _can_update(self, **kwargs):
+        pass
+
+    def _can_delete(self, **kwargs):
+        pass
+
+
+class ScanHistoryManager(DBMixin):
+    cls = IPAMSubnetScanHistory
+
+    def _can_add(self, **kwargs):
+        return kwargs
+
+    def add(self, **kwargs):
+        kwargs.pop('_key', None)
+        kwargs.pop('_secret', None)
+        ci_id = kwargs.pop('ci_id', None)
+
+        existed = self.cls.get_by(exec_id=kwargs['exec_id'], first=True, to_dict=False)
+        if existed is None:
+            self.cls.create(**kwargs)
+        else:
+            existed.update(**kwargs)
+
+        if kwargs.get('ips'):
+            from api.lib.cmdb.ipam.address import IpAddressManager
+            IpAddressManager().assign_ips(kwargs['ips'], ci_id, kwargs.get('cidr'),
+                                          **{IPAddressBuiltinAttributes.IS_USED: 1})
+
+            scan_rule = IPAMSubnetScan.get_by(ci_id=ci_id, first=True, to_dict=False)
+            if scan_rule is not None:
+                scan_rule.update(last_scan_time=kwargs.get('start_at'))
+
+        for i in self.cls.get_by(subnet_scan_id=kwargs.get('subnet_scan_id'), only_query=True).order_by(
+                self.cls.id.desc()).offset(100):
+            i.delete()
+
+    def _can_update(self, **kwargs):
+        pass
+
+    def _can_delete(self, **kwargs):
+        pass

cmdb-api/api/lib/cmdb/ipam/stats.py

@@ -0,0 +1,104 @@
+# -*- coding:utf-8 -*-
+
+
+import json
+from flask import abort
+
+from api.extensions import rd
+from api.lib.cmdb.cache import CITypeCache
+from api.lib.cmdb.ci import CIManager
+from api.lib.cmdb.const import BuiltinModelEnum
+from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
+from api.lib.cmdb.resp_format import ErrFormat
+from api.lib.cmdb.search.ci.db.search import Search as SearchFromDB
+from api.models.cmdb import CI
+from api.models.cmdb import CIRelation
+from api.models.cmdb import IPAMSubnetScan
+
+
+class Stats(object):
+    def __init__(self):
+        self.address_type = CITypeCache.get(BuiltinModelEnum.IPAM_ADDRESS) or abort(
+            404, ErrFormat.ipam_address_model_not_found.format(BuiltinModelEnum.IPAM_ADDRESS))
+
+        self.address_type_id = self.address_type.id
+
+        self.subnet_type = CITypeCache.get(BuiltinModelEnum.IPAM_SUBNET) or abort(
+            404, ErrFormat.ipam_address_model_not_found.format(BuiltinModelEnum.IPAM_ADDRESS))
+
+        self.subnet_type_id = self.subnet_type.id
+
+    def leaf_nodes(self, parent_id):
+        if str(parent_id) == '0':  # all
+            ci_ids = [i.id for i in CI.get_by(type_id=self.subnet_type_id, to_dict=False)]
+            has_children_ci_ids = [i.first_ci_id for i in CIRelation.get_by(
+                only_query=True).join(CI, CIRelation.second_ci_id == CI.id).filter(
+                CIRelation.first_ci_id.in_(ci_ids)).filter(CI.type_id == self.subnet_type_id)]
+
+            return list(set(ci_ids) - set(has_children_ci_ids))
+
+        else:
+            _type = CIManager().get_by_id(parent_id)
+            if not _type:
+                return abort(404, ErrFormat.ipam_subnet_not_found)
+            key = [(str(parent_id), _type.type_id)]
+            result = []
+            while True:
+                res = [json.loads(x).items() for x in [i or '{}' for i in rd.get(
+                    [i[0] for i in key], REDIS_PREFIX_CI_RELATION) or []]]
+
+                for idx, i in enumerate(res):
+                    if (not i or list(i)[0][1] == self.address_type_id) and key[idx][1] == self.subnet_type_id:
+                        result.append(int(key[idx][0]))
+
+                res = [j for i in res for j in i]  # [(id, type_id)]
+
+                if not res:
+                    return result
+
+                key = res
+
+    def statistic_subnets(self, subnet_ids):
+        if subnet_ids:
+            response, _, _, _, _, _ = SearchFromDB(
+                "_type:{}".format(self.subnet_type_id),
+                ci_ids=subnet_ids,
+                count=1000000,
+                parent_node_perm_passed=True,
+            ).search()
+        else:
+            response = []
+
+        scans = IPAMSubnetScan.get_by(only_query=True).filter(IPAMSubnetScan.ci_id.in_(list(map(int, subnet_ids))))
+        id2scan = {i.ci_id: i for i in scans}
+
+        address_num, address_free_num, address_assign_num, address_used_num = 0, 0, 0, 0
+        for subnet in response:
+            address_num += (subnet.get('hosts_count') or 0)
+            address_free_num += (subnet.get('free_count') or 0)
+            address_assign_num += (subnet.get('assign_count') or 0)
+            address_used_num += (subnet.get('used_count') or 0)
+
+            if id2scan.get(subnet['_id']):
+                subnet['scan_enabled'] = id2scan[subnet['_id']].scan_enabled
+                subnet['last_scan_time'] = id2scan[subnet['_id']].last_scan_time
+            else:
+                subnet['scan_enabled'] = False
+                subnet['last_scan_time'] = None
+
+        return response, address_num, address_free_num, address_assign_num, address_used_num
+
+    def summary(self, parent_id):
+        subnet_ids = self.leaf_nodes(parent_id)
+
+        subnets, address_num, address_free_num, address_assign_num, address_used_num = (
+            self.statistic_subnets(subnet_ids))
+
+        return dict(subnet_num=len(subnets),
+                    address_num=address_num,
+                    address_free_num=address_free_num,
+                    address_assign_num=address_assign_num,
+                    address_unassign_num=address_num - address_assign_num,
+                    address_used_num=address_used_num,
+                    address_used_free_num=address_num - address_used_num,
+                    subnets=subnets)

cmdb-api/api/lib/cmdb/ipam/subnet.py

@@ -0,0 +1,355 @@
+# -*- coding:utf-8 -*-
+
+from collections import defaultdict
+
+import datetime
+import ipaddress
+from flask import abort
+
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.cache import CITypeCache
+from api.lib.cmdb.ci import CIManager
+from api.lib.cmdb.ci import CIRelationManager
+from api.lib.cmdb.const import BuiltinModelEnum
+from api.lib.cmdb.ipam.const import OperateTypeEnum
+from api.lib.cmdb.ipam.const import SubnetBuiltinAttributes
+from api.lib.cmdb.ipam.history import OperateHistoryManager
+from api.lib.cmdb.resp_format import ErrFormat
+from api.lib.cmdb.search.ci.db.search import Search as SearchFromDB
+from api.models.cmdb import CI
+from api.models.cmdb import CIRelation
+from api.models.cmdb import IPAMSubnetScan
+
+
+class SubnetManager(object):
+    def __init__(self):
+        self.ci_type = CITypeCache.get(BuiltinModelEnum.IPAM_SUBNET) or abort(
+            404, ErrFormat.ipam_subnet_model_not_found.format(BuiltinModelEnum.IPAM_SUBNET))
+
+        self.type_id = self.ci_type.id
+
+    def scan_rules(self, oneagent_id, last_update_at=None):
+        result = []
+        rules = IPAMSubnetScan.get_by(agent_id=oneagent_id, to_dict=True)
+        ci_ids = [i['ci_id'] for i in rules]
+        if ci_ids:
+            response, _, _, _, _, _ = SearchFromDB("_type:{}".format(self.type_id),
+                                                   ci_ids=list(ci_ids),
+                                                   count=1000000,
+                                                   fl=[SubnetBuiltinAttributes.CIDR],
+                                                   parent_node_perm_passed=True).search()
+            id2ci = {i['_id']: i for i in response}
+
+            for rule in rules:
+                if rule['ci_id'] in id2ci:
+                    rule[SubnetBuiltinAttributes.CIDR] = id2ci[rule['ci_id']][SubnetBuiltinAttributes.CIDR]
+                    result.append(rule)
+
+        new_last_update_at = ""
+        for i in result:
+            __last_update_at = max([i['rule_updated_at'] or "", i['created_at'] or ""])
+            if new_last_update_at < __last_update_at:
+                new_last_update_at = __last_update_at
+
+        if not last_update_at or new_last_update_at > last_update_at:
+            return result, new_last_update_at
+        else:
+            return [], new_last_update_at
+
+    @staticmethod
+    def get_hosts(cidr):
+        try:
+            return list(map(str, ipaddress.ip_network(cidr).hosts()))
+        except ValueError:
+            return []
+
+    def get_by_id(self, subnet_id):
+        response, _, _, _, _, _ = SearchFromDB("_type:{}".format(self.type_id),
+                                               ci_ids=[subnet_id],
+                                               parent_node_perm_passed=True).search()
+        scan_rule = IPAMSubnetScan.get_by(ci_id=subnet_id, first=True, to_dict=True)
+        if scan_rule and response:
+            scan_rule.update(response[0])
+
+        return scan_rule
+
+    def tree_view(self):
+        scope = CITypeCache.get(BuiltinModelEnum.IPAM_SCOPE)
+        ci_types = scope and [scope.id, self.type_id] or [self.type_id]
+
+        relations = defaultdict(set)
+        ids = set()
+        has_parent_ids = set()
+        for i in CIRelation.get_by(only_query=True).join(
+                CI, CI.id == CIRelation.first_ci_id).filter(CI.type_id.in_(ci_types)):
+            relations[i.first_ci_id].add(i.second_ci_id)
+            ids.add(i.first_ci_id)
+            ids.add(i.second_ci_id)
+            has_parent_ids.add(i.second_ci_id)
+        for i in CIRelation.get_by(only_query=True).join(
+                CI, CI.id == CIRelation.second_ci_id).filter(CI.type_id.in_(ci_types)):
+            relations[i.first_ci_id].add(i.second_ci_id)
+            ids.add(i.first_ci_id)
+            ids.add(i.second_ci_id)
+            has_parent_ids.add(i.second_ci_id)
+
+        for i in CI.get_by(only_query=True).filter(CI.type_id.in_(ci_types)):
+            ids.add(i.id)
+
+        for _id in ids:
+            if _id not in has_parent_ids:
+                relations[None].add(_id)
+
+        type2name = dict()
+        type2name[self.type_id] = AttributeCache.get(self.ci_type.show_id or self.ci_type.unique_id).name
+
+        fl = [type2name[self.type_id]]
+        if scope:
+            type2name[scope.id] = AttributeCache.get(scope.show_id or scope.unique_id).name
+            fl.append(type2name[scope.id])
+
+        response, _, _, _, _, _ = SearchFromDB("_type:({})".format(";".join(map(str, ci_types))),
+                                               ci_ids=list(ids),
+                                               count=1000000,
+                                               fl=list(set(fl + [SubnetBuiltinAttributes.CIDR])),
+                                               parent_node_perm_passed=True).search()
+        id2ci = {i['_id']: i for i in response}
+
+        def _build_tree(_tree, parent_id=None):
+            tree = []
+            for child_id in _tree.get(parent_id, []):
+                children = sorted(_build_tree(_tree, child_id), key=lambda x: x['_id'])
+                if not id2ci.get(child_id):
+                    continue
+                tree.append({'children': children, **id2ci[child_id]})
+            return tree
+
+        result = sorted(_build_tree(relations), key=lambda x: x['_id'])
+
+        return result, type2name
+
+    @staticmethod
+    def _is_valid_cidr(cidr):
+        try:
+            cidr = ipaddress.ip_network(cidr)
+            if not (8 <= cidr.prefixlen <= 31):
+                raise ValueError
+
+            return str(cidr)
+        except ValueError:
+            return abort(400, ErrFormat.ipam_cidr_invalid_notation.format(cidr))
+
+    def _check_root_node_is_overlapping(self, cidr, _id=None):
+        none_root_nodes = [i.id for i in CI.get_by(only_query=True).join(
+            CIRelation, CIRelation.second_ci_id == CI.id).filter(CI.type_id == self.type_id)]
+        all_nodes = [i.id for i in CI.get_by(type_id=self.type_id, to_dict=False, fl=['id'])]
+
+        root_nodes = set(all_nodes) - set(none_root_nodes) - set(_id and [_id] or [])
+        response, _, _, _, _, _ = SearchFromDB("_type:{}".format(self.type_id),
+                                               ci_ids=list(root_nodes),
+                                               count=1000000,
+                                               parent_node_perm_passed=True).search()
+
+        cur_subnet = ipaddress.ip_network(cidr)
+        for item in response:
+            if item['_id'] == _id:
+                continue
+
+            if cur_subnet.overlaps(ipaddress.ip_network(item.get(SubnetBuiltinAttributes.CIDR))):
+                return abort(400, ErrFormat.ipam_subnet_overlapped.format(cidr, item.get(SubnetBuiltinAttributes.CIDR)))
+
+        return cidr
+
+    def _check_child_node_is_overlapping(self, parent_id, cidr, _id=None):
+        child_nodes = [i.second_ci_id for i in CIRelation.get_by(
+            first_ci_id=parent_id, to_dict=False, fl=['second_ci_id']) if i.second_ci_id != _id]
+        if not child_nodes:
+            return
+
+        response, _, _, _, _, _ = SearchFromDB("_type:{}".format(self.type_id),
+                                               ci_ids=list(child_nodes),
+                                               count=1000000,
+                                               parent_node_perm_passed=True).search()
+
+        cur_subnet = ipaddress.ip_network(cidr)
+        for item in response:
+            if item['_id'] == _id:
+                continue
+
+            if cur_subnet.overlaps(ipaddress.ip_network(item.get(SubnetBuiltinAttributes.CIDR))):
+                return abort(400, ErrFormat.ipam_subnet_overlapped.format(cidr, item.get(SubnetBuiltinAttributes.CIDR)))
+
+    def validate_cidr(self, parent_id, cidr, _id=None):
+        cidr = self._is_valid_cidr(cidr)
+
+        if not parent_id:
+            return self._check_root_node_is_overlapping(cidr, _id)
+
+        parent_subnet = CIManager().get_ci_by_id(parent_id, need_children=False)
+        if parent_subnet['ci_type'] == BuiltinModelEnum.IPAM_SUBNET:
+            if parent_subnet.get(SubnetBuiltinAttributes.CIDR):
+                prefix = int(cidr.split('/')[1])
+                if int(parent_subnet[SubnetBuiltinAttributes.CIDR].split('/')[1]) >= prefix:
+                    return abort(400, ErrFormat.ipam_subnet_prefix_length_invalid.format(prefix))
+
+                valid_subnets = [str(i) for i in
+                                 ipaddress.ip_network(parent_subnet[SubnetBuiltinAttributes.CIDR]).subnets(
+                                     new_prefix=prefix)]
+                if cidr not in valid_subnets:
+                    return abort(400, ErrFormat.ipam_cidr_invalid_subnet.format(cidr, valid_subnets))
+            else:
+                return abort(400, ErrFormat.ipam_parent_subnet_node_cidr_cannot_empty)
+
+        self._check_child_node_is_overlapping(parent_id, cidr, _id)
+
+        return cidr
+
+    def _add_subnet(self, cidr, **kwargs):
+        kwargs[SubnetBuiltinAttributes.HOSTS_COUNT] = len(list(ipaddress.ip_network(cidr).hosts()))
+        kwargs[SubnetBuiltinAttributes.USED_COUNT] = 0
+        kwargs[SubnetBuiltinAttributes.ASSIGN_COUNT] = 0
+        kwargs[SubnetBuiltinAttributes.FREE_COUNT] = kwargs[SubnetBuiltinAttributes.HOSTS_COUNT]
+
+        return CIManager().add(self.type_id, cidr=cidr, **kwargs)
+
+    @staticmethod
+    def _add_scan_rule(ci_id, agent_id, cron, scan_enabled=True):
+        IPAMSubnetScan.create(ci_id=ci_id, agent_id=agent_id, cron=cron, scan_enabled=scan_enabled)
+
+    @staticmethod
+    def _add_relation(parent_id, child_id):
+        if not parent_id or not child_id:
+            return
+
+        CIRelationManager().add(parent_id, child_id, valid=False)
+
+    def add(self, cidr, parent_id, agent_id, cron, scan_enabled=True, **kwargs):
+        cidr = self.validate_cidr(parent_id, cidr)
+
+        ci_id = self._add_subnet(cidr, **kwargs)
+
+        self._add_scan_rule(ci_id, agent_id, cron, scan_enabled)
+
+        self._add_relation(parent_id, ci_id)
+
+        OperateHistoryManager().add(operate_type=OperateTypeEnum.ADD_SUBNET,
+                                    cidr=cidr,
+                                    description=cidr)
+
+        return ci_id
+
+    @staticmethod
+    def _update_subnet(_id, **kwargs):
+        return CIManager().update(_id, **kwargs)
+
+    @staticmethod
+    def _update_scan_rule(ci_id, agent_id, cron, scan_enabled=True):
+        existed = IPAMSubnetScan.get_by(ci_id=ci_id, first=True, to_dict=False)
+        if existed is not None:
+            existed.update(ci_id=ci_id, agent_id=agent_id, cron=cron, scan_enabled=scan_enabled,
+                           rule_updated_at=datetime.datetime.now())
+        else:
+            IPAMSubnetScan.create(ci_id=ci_id, agent_id=agent_id, cron=cron, scan_enabled=scan_enabled)
+
+    def update(self, _id, **kwargs):
+        kwargs[SubnetBuiltinAttributes.CIDR] = self.validate_cidr(kwargs.pop('parent_id', None),
+                                                                  kwargs.get(SubnetBuiltinAttributes.CIDR), _id)
+
+        agent_id = kwargs.pop('agent_id', None)
+        cron = kwargs.pop('cron', None)
+        scan_enabled = kwargs.pop('scan_enabled', True)
+
+        cur = CIManager.get_ci_by_id(_id, need_children=False)
+
+        self._update_subnet(_id, **kwargs)
+
+        self._update_scan_rule(_id, agent_id, cron, scan_enabled)
+
+        OperateHistoryManager().add(operate_type=OperateTypeEnum.UPDATE_SUBNET,
+                                    cidr=cur.get(SubnetBuiltinAttributes.CIDR),
+                                    description="{} -> {}".format(cur.get(SubnetBuiltinAttributes.CIDR),
+                                                                  kwargs.get(SubnetBuiltinAttributes.CIDR)))
+
+        return _id
+
+    @classmethod
+    def delete(cls, _id):
+        if CIRelation.get_by(only_query=True).filter(CIRelation.first_ci_id == _id).first():
+            return abort(400, ErrFormat.ipam_subnet_cannot_delete)
+
+        existed = IPAMSubnetScan.get_by(ci_id=_id, first=True, to_dict=False)
+        existed and existed.delete()
+
+        delete_ci_ids = []
+        for i in CIRelation.get_by(first_ci_id=_id, to_dict=False):
+            delete_ci_ids.append(i.second_ci_id)
+            i.delete()
+
+        cur = CIManager.get_ci_by_id(_id, need_children=False)
+
+        CIManager().delete(_id)
+
+        OperateHistoryManager().add(operate_type=OperateTypeEnum.DELETE_SUBNET,
+                                    cidr=cur.get(SubnetBuiltinAttributes.CIDR),
+                                    description=cur.get(SubnetBuiltinAttributes.CIDR))
+
+        # batch_delete_ci.apply_async(args=(delete_ci_ids,))
+
+        return _id
+
+
+class SubnetScopeManager(object):
+    def __init__(self):
+        self.ci_type = CITypeCache.get(BuiltinModelEnum.IPAM_SCOPE)
+        not self.ci_type and abort(400, ErrFormat.ipam_subnet_model_not_found.format(
+            BuiltinModelEnum.IPAM_SCOPE))
+
+        self.type_id = self.ci_type.id
+
+    def _add_scope(self, name):
+        return CIManager().add(self.type_id, name=name)
+
+    @staticmethod
+    def _add_relation(parent_id, child_id):
+        if not parent_id or not child_id:
+            return
+
+        CIRelationManager().add(parent_id, child_id, valid=False)
+
+    def add(self, parent_id, name):
+        ci_id = self._add_scope(name)
+
+        self._add_relation(parent_id, ci_id)
+
+        OperateHistoryManager().add(operate_type=OperateTypeEnum.ADD_SCOPE,
+                                    description=name)
+
+        return ci_id
+
+    @staticmethod
+    def _update_scope(_id, name):
+        return CIManager().update(_id, name=name)
+
+    def update(self, _id, name):
+        cur = CIManager.get_ci_by_id(_id, need_children=False)
+
+        res = self._update_scope(_id, name)
+
+        OperateHistoryManager().add(operate_type=OperateTypeEnum.UPDATE_SCOPE,
+                                    description="{} -> {}".format(cur.get('name'), name))
+
+        return res
+
+    @staticmethod
+    def delete(_id):
+        if CIRelation.get_by(first_ci_id=_id, first=True, to_dict=False):
+            return abort(400, ErrFormat.ipam_scope_cannot_delete)
+
+        cur = CIManager.get_ci_by_id(_id, need_children=False)
+
+        CIManager().delete(_id)
+
+        OperateHistoryManager().add(operate_type=OperateTypeEnum.DELETE_SCOPE,
+                                    description=cur.get('name'))
+
+        return _id

cmdb-api/api/lib/cmdb/perms.py

@@ -1,12 +1,15 @@
 # -*- coding:utf-8 -*-
-
+import copy
 import functools
-
+import redis_lock
 from flask import abort
 from flask import current_app
-from flask import g
 from flask import request
+from flask_login import current_user
 
+from api.extensions import db
+from api.extensions import rd
+from api.lib.cmdb.const import BUILTIN_ATTRIBUTES
 from api.lib.cmdb.const import ResourceTypeEnum
 from api.lib.cmdb.resp_format import ErrFormat
 from api.lib.mixin import DBMixin
@@ -24,7 +27,7 @@ class CIFilterPermsCRUD(DBMixin):
         result = {}
         for i in res:
             if i['attr_filter']:
-                i['attr_filter'] = i['attr_filter'].split(',')
+                i['attr_filter'] = i['attr_filter'].split(',') + list(BUILTIN_ATTRIBUTES.keys())
 
             if i['rid'] not in result:
                 result[i['rid']] = i
@@ -40,6 +43,11 @@ class CIFilterPermsCRUD(DBMixin):
                         result[i['rid']]['ci_filter'] = ""
                     result[i['rid']]['ci_filter'] += (i['ci_filter'] or "")
 
+                if i['id_filter']:
+                    if not result[i['rid']]['id_filter']:
+                        result[i['rid']]['id_filter'] = {}
+                    result[i['rid']]['id_filter'].update(i['id_filter'] or {})
+
         return result
 
     def get_by_ids(self, _ids, type_id=None):
@@ -54,7 +62,7 @@ class CIFilterPermsCRUD(DBMixin):
         result = {}
         for i in res:
             if i['attr_filter']:
-                i['attr_filter'] = i['attr_filter'].split(',')
+                i['attr_filter'] = i['attr_filter'].split(',') + list(BUILTIN_ATTRIBUTES.keys())
 
             if i['type_id'] not in result:
                 result[i['type_id']] = i
@@ -70,11 +78,16 @@ class CIFilterPermsCRUD(DBMixin):
                         result[i['type_id']]['ci_filter'] = ""
                     result[i['type_id']]['ci_filter'] += (i['ci_filter'] or "")
 
+                if i['id_filter']:
+                    if not result[i['type_id']]['id_filter']:
+                        result[i['type_id']]['id_filter'] = {}
+                    result[i['type_id']]['id_filter'].update(i['id_filter'] or {})
+
         return result
 
     @classmethod
     def get_attr_filter(cls, type_id):
-        if is_app_admin('cmdb') or g.user.username in ('worker', 'cmdb_agent'):
+        if is_app_admin('cmdb') or current_user.username in ('worker', 'cmdb_agent'):
             return []
 
         res2 = ACLManager('cmdb').get_resources(ResourceTypeEnum.CI_FILTER)
@@ -82,6 +95,54 @@ class CIFilterPermsCRUD(DBMixin):
             type2filter_perms = cls().get_by_ids(list(map(int, [i['name'] for i in res2])), type_id=type_id)
             return type2filter_perms.get(type_id, {}).get('attr_filter') or []
 
+    def _revoke_children(self, rid, id_filter, rebuild=True):
+        items = self.cls.get_by(rid=rid, ci_filter=None, attr_filter=None, to_dict=False)
+        for item in items:
+            changed, item_id_filter = False, copy.deepcopy(item.id_filter)
+            for prefix in id_filter:
+                for k, v in copy.deepcopy((item.id_filter or {})).items():
+                    if k.startswith(prefix) and k != prefix:
+                        item_id_filter.pop(k)
+                        changed = True
+
+            if not item_id_filter and current_app.config.get('USE_ACL'):
+                item.soft_delete(commit=False)
+                ACLManager().del_resource(str(item.id), ResourceTypeEnum.CI_FILTER, rebuild=rebuild)
+            elif changed:
+                item.update(id_filter=item_id_filter, commit=False)
+
+        db.session.commit()
+
+    def _revoke_parent(self, rid, parent_path, rebuild=True):
+        parent_path = [i for i in parent_path.split(',') if i] or []
+        revoke_nodes = [','.join(parent_path[:i]) for i in range(len(parent_path), 0, -1)]
+        for node_path in revoke_nodes:
+            delete_item, can_deleted = None, True
+            items = self.cls.get_by(rid=rid, ci_filter=None, attr_filter=None, to_dict=False)
+            for item in items:
+                if node_path in item.id_filter:
+                    delete_item = item
+                if any(filter(lambda x: x.startswith(node_path) and x != node_path, item.id_filter.keys())):
+                    can_deleted = False
+                    break
+
+            if can_deleted and delete_item:
+                id_filter = copy.deepcopy(delete_item.id_filter)
+                id_filter.pop(node_path)
+                delete_item = delete_item.update(id_filter=id_filter, filter_none=False)
+
+                if current_app.config.get('USE_ACL') and not id_filter:
+                    ACLManager().del_resource(str(delete_item.id), ResourceTypeEnum.CI_FILTER, rebuild=False)
+                    delete_item.soft_delete()
+                    items.remove(delete_item)
+
+        if rebuild:
+            from api.tasks.acl import role_rebuild
+            from api.lib.perm.acl.const import ACL_QUEUE
+            from api.lib.perm.acl.cache import AppCache
+
+            role_rebuild.apply_async(args=(rid, AppCache.get('cmdb').id), queue=ACL_QUEUE)
+
     def _can_add(self, **kwargs):
         ci_filter = kwargs.get('ci_filter')
         attr_filter = kwargs.get('attr_filter') or ""
@@ -102,34 +163,67 @@ class CIFilterPermsCRUD(DBMixin):
 
     def add(self, **kwargs):
         kwargs = self._can_add(**kwargs) or kwargs
+        with redis_lock.Lock(rd.r, 'CMDB_FILTER_{}_{}'.format(kwargs['type_id'], kwargs['rid']), expire=10):
+            request_id_filter = {}
+            if kwargs.get('id_filter'):
+                obj = self.cls.get_by(type_id=kwargs.get('type_id'),
+                                      rid=kwargs.get('rid'),
+                                      ci_filter=None,
+                                      attr_filter=None,
+                                      first=True, to_dict=False)
 
-        obj = self.cls.get_by(type_id=kwargs.get('type_id'),
+                for _id, v in (kwargs.get('id_filter') or {}).items():
-                              rid=kwargs.get('rid'),
+                    key = ",".join(([v['parent_path']] if v.get('parent_path') else []) + [str(_id)])
-                              first=True, to_dict=False)
+                    request_id_filter[key] = v['name']
-        if obj is not None:
-            obj = obj.update(filter_none=False, **kwargs)
-            if not obj.attr_filter and not obj.ci_filter:
-                if current_app.config.get('USE_ACL'):
-                    ACLManager().del_resource(str(obj.id), ResourceTypeEnum.CI_FILTER)
 
-                obj.soft_delete()
+            else:
+                obj = self.cls.get_by(type_id=kwargs.get('type_id'),
+                                      rid=kwargs.get('rid'),
+                                      id_filter=None,
+                                      first=True, to_dict=False)
+
+            is_recursive = kwargs.pop('is_recursive', 0)
+            if obj is not None:
+                if obj.id_filter and isinstance(kwargs.get('id_filter'), dict):
+                    obj_id_filter = copy.deepcopy(obj.id_filter)
+
+                    for k, v in request_id_filter.items():
+                        obj_id_filter[k] = v
+
+                    kwargs['id_filter'] = obj_id_filter
+
+                obj = obj.update(filter_none=False, **kwargs)
+
+                if not obj.attr_filter and not obj.ci_filter and not obj.id_filter:
+                    if current_app.config.get('USE_ACL'):
+                        ACLManager().del_resource(str(obj.id), ResourceTypeEnum.CI_FILTER, rebuild=False)
+
+                    obj.soft_delete()
+
+                if not is_recursive and request_id_filter:
+                    self._revoke_children(obj.rid, request_id_filter, rebuild=False)
 
-        else:
-            if not kwargs.get('ci_filter') and not kwargs.get('attr_filter'):
                 return
 
-            obj = self.cls.create(**kwargs)
+            else:
+                if not kwargs.get('ci_filter') and not kwargs.get('attr_filter') and not kwargs.get('id_filter'):
+                    return
 
-        if current_app.config.get('USE_ACL'):
+                if request_id_filter:
-            try:
+                    kwargs['id_filter'] = request_id_filter
-                ACLManager().add_resource(obj.id, ResourceTypeEnum.CI_FILTER)
-            except:
-                pass
-            ACLManager().grant_resource_to_role_by_rid(obj.id,
-                                                       kwargs.get('rid'),
-                                                       ResourceTypeEnum.CI_FILTER)
 
-        return obj
+                obj = self.cls.create(**kwargs)
+
+                if current_app.config.get('USE_ACL'):  # new resource
+                    try:
+                        ACLManager().add_resource(obj.id, ResourceTypeEnum.CI_FILTER)
+                    except:
+                        pass
+                    ACLManager().grant_resource_to_role_by_rid(obj.id,
+                                                               kwargs.get('rid'),
+                                                               ResourceTypeEnum.CI_FILTER)
+
+                return obj
 
     def _can_update(self, **kwargs):
         pass
@@ -138,15 +232,83 @@ class CIFilterPermsCRUD(DBMixin):
         pass
 
     def delete(self, **kwargs):
-        obj = self.cls.get_by(type_id=kwargs.get('type_id'),
+        with redis_lock.Lock(rd.r, 'CMDB_FILTER_{}_{}'.format(kwargs['type_id'], kwargs['rid']), expire=10):
-                              rid=kwargs.get('rid'),
+            obj = self.cls.get_by(type_id=kwargs.get('type_id'),
-                              first=True, to_dict=False)
+                                  rid=kwargs.get('rid'),
+                                  id_filter=None,
+                                  first=True, to_dict=False)
 
-        if obj is not None:
+            if obj is not None:
-            if current_app.config.get('USE_ACL'):
+                resource = None
-                ACLManager().del_resource(str(obj.id), ResourceTypeEnum.CI_FILTER)
+                if current_app.config.get('USE_ACL'):
+                    resource = ACLManager().del_resource(str(obj.id), ResourceTypeEnum.CI_FILTER)
 
-            obj.soft_delete()
+                obj.soft_delete()
+
+                return resource
+
+    def delete2(self, **kwargs):
+
+        with redis_lock.Lock(rd.r, 'CMDB_FILTER_{}_{}'.format(kwargs['type_id'], kwargs['rid']), expire=10):
+            obj = self.cls.get_by(type_id=kwargs.get('type_id'),
+                                  rid=kwargs.get('rid'),
+                                  ci_filter=None,
+                                  attr_filter=None,
+                                  first=True, to_dict=False)
+
+            request_id_filter = {}
+            for _id, v in (kwargs.get('id_filter') or {}).items():
+                key = ",".join(([v['parent_path']] if v.get('parent_path') else []) + [str(_id)])
+                request_id_filter[key] = v['name']
+
+            resource = None
+            if obj is not None:
+
+                id_filter = {}
+                for k, v in copy.deepcopy(obj.id_filter or {}).items():  # important
+                    if k not in request_id_filter:
+                        id_filter[k] = v
+
+                if not id_filter and current_app.config.get('USE_ACL'):
+                    resource = ACLManager().del_resource(str(obj.id), ResourceTypeEnum.CI_FILTER, rebuild=False)
+                    obj.soft_delete()
+                    db.session.commit()
+
+                else:
+                    obj.update(id_filter=id_filter)
+
+                self._revoke_children(kwargs.get('rid'), request_id_filter, rebuild=False)
+                self._revoke_parent(kwargs.get('rid'), kwargs.get('parent_path'))
+
+            return resource
+
+    def delete_id_filter_by_ci_id(self, ci_id):
+        items = self.cls.get_by(ci_filter=None, attr_filter=None, to_dict=False)
+
+        rebuild_roles = set()
+        for item in items:
+            id_filter = copy.deepcopy(item.id_filter)
+            changed = False
+            for node_path in item.id_filter:
+                if str(ci_id) in node_path:
+                    id_filter.pop(node_path)
+                    changed = True
+
+            if changed:
+                rebuild_roles.add(item.rid)
+                if not id_filter:
+                    item.soft_delete(commit=False)
+                else:
+                    item.update(id_filter=id_filter, commit=False)
+
+        db.session.commit()
+
+        if rebuild_roles:
+            from api.tasks.acl import role_rebuild
+            from api.lib.perm.acl.const import ACL_QUEUE
+            from api.lib.perm.acl.cache import AppCache
+            for rid in rebuild_roles:
+                role_rebuild.apply_async(args=(rid, AppCache.get('cmdb').id), queue=ACL_QUEUE)
 
 
 def has_perm_for_ci(arg_name, resource_type, perm, callback=None, app=None):
@@ -160,7 +322,7 @@ def has_perm_for_ci(arg_name, resource_type, perm, callback=None, app=None):
                 resource = callback(resource)
 
             if current_app.config.get("USE_ACL") and resource:
-                if g.user.username == "worker" or g.user.username == "cmdb_agent":
+                if current_user.username == "worker" or current_user.username == "cmdb_agent":
                     request.values['__is_admin'] = True
                     return func(*args, **kwargs)
 

cmdb-api/api/lib/cmdb/preference.py

@@ -1,26 +1,35 @@
 # -*- coding:utf-8 -*-
 
+from collections import defaultdict
 
 import copy
-
 import six
 import toposort
 from flask import abort
 from flask import current_app
-from flask import g
+from flask_login import current_user
 
 from api.extensions import db
 from api.lib.cmdb.attribute import AttributeManager
 from api.lib.cmdb.cache import AttributeCache
 from api.lib.cmdb.cache import CITypeAttributesCache
 from api.lib.cmdb.cache import CITypeCache
-from api.lib.cmdb.const import PermEnum, ResourceTypeEnum, RoleEnum
+from api.lib.cmdb.cache import CMDBCounterCache
+from api.lib.cmdb.ci_type import CITypeAttributeManager
+from api.lib.cmdb.const import BUILTIN_ATTRIBUTES
+from api.lib.cmdb.const import ConstraintEnum
+from api.lib.cmdb.const import PermEnum
+from api.lib.cmdb.const import ResourceTypeEnum
+from api.lib.cmdb.const import RoleEnum
+from api.lib.cmdb.const import SysComputedAttributes
 from api.lib.cmdb.perms import CIFilterPermsCRUD
 from api.lib.cmdb.resp_format import ErrFormat
 from api.lib.exception import AbortException
 from api.lib.perm.acl.acl import ACLManager
-from api.models.cmdb import CITypeAttribute
+from api.models.cmdb import CITypeGroup
+from api.models.cmdb import CITypeGroupItem
 from api.models.cmdb import CITypeRelation
+from api.models.cmdb import PreferenceCITypeOrder
 from api.models.cmdb import PreferenceRelationView
 from api.models.cmdb import PreferenceSearchOption
 from api.models.cmdb import PreferenceShowAttributes
@@ -35,13 +44,48 @@ class PreferenceManager(object):
 
     @staticmethod
     def get_types(instance=False, tree=False):
+        ci_type_order = sorted(PreferenceCITypeOrder.get_by(uid=current_user.uid, to_dict=False), key=lambda x: x.order)
+
+        type2group = {}
+        for i in db.session.query(CITypeGroupItem, CITypeGroup).join(
+                CITypeGroup, CITypeGroup.id == CITypeGroupItem.group_id).filter(
+                CITypeGroup.deleted.is_(False)).filter(CITypeGroupItem.deleted.is_(False)):
+            type2group[i.CITypeGroupItem.type_id] = i.CITypeGroup.to_dict()
+
         types = db.session.query(PreferenceShowAttributes.type_id).filter(
-            PreferenceShowAttributes.uid == g.user.uid).filter(
+            PreferenceShowAttributes.uid == current_user.uid).filter(
-            PreferenceShowAttributes.deleted.is_(False)).group_by(PreferenceShowAttributes.type_id).all() \
+            PreferenceShowAttributes.deleted.is_(False)).group_by(
-            if instance else []
+            PreferenceShowAttributes.type_id).all() if instance else []
-        tree_types = PreferenceTreeView.get_by(uid=g.user.uid, to_dict=False) if tree else []
+        types = sorted(types, key=lambda x: {i.type_id: idx for idx, i in enumerate(
-        type_ids = list(set([i.type_id for i in types + tree_types]))
+            ci_type_order) if not i.is_tree}.get(x.type_id, 1))
-        return [CITypeCache.get(type_id).to_dict() for type_id in type_ids]
+        group_types = []
+        other_types = []
+        group2idx = {}
+        type_ids = set()
+        for ci_type in types:
+            type_id = ci_type.type_id
+            type_ids.add(type_id)
+            type_dict = CITypeCache.get(type_id).to_dict()
+            if type_id not in type2group:
+                other_types.append(type_dict)
+            else:
+                group = type2group[type_id]
+                if group['id'] not in group2idx:
+                    group_types.append(type2group[type_id])
+                    group2idx[group['id']] = len(group_types) - 1
+                group_types[group2idx[group['id']]].setdefault('ci_types', []).append(type_dict)
+        if other_types:
+            group_types.append(dict(ci_types=other_types))
+
+        tree_types = PreferenceTreeView.get_by(uid=current_user.uid, to_dict=False) if tree else []
+        tree_types = sorted(tree_types, key=lambda x: {i.type_id: idx for idx, i in enumerate(
+            ci_type_order) if i.is_tree}.get(x.type_id, 1))
+
+        tree_types = [CITypeCache.get(_type.type_id).to_dict() for _type in tree_types]
+        for _type in tree_types:
+            type_ids.add(_type['id'])
+
+        return dict(group_types=group_types, tree_types=tree_types, type_ids=list(type_ids))
 
     @staticmethod
     def get_types2(instance=False, tree=False):
@@ -54,101 +98,140 @@ class PreferenceManager(object):
         :param tree:
         :return:
         """
-        result = dict(self=dict(instance=[], tree=[], type_id2subs_time=dict()),
+        result = dict(self=dict(instance=[], tree=[], type_id2subs_time=dict()))
-                      type_id2users=dict())
+
+        result.update(CMDBCounterCache.get_sub_counter())
+
+        ci_type_order = sorted(PreferenceCITypeOrder.get_by(uid=current_user.uid, to_dict=False), key=lambda x: x.order)
         if instance:
             types = db.session.query(PreferenceShowAttributes.type_id,
                                      PreferenceShowAttributes.uid, PreferenceShowAttributes.created_at).filter(
-                PreferenceShowAttributes.deleted.is_(False)).group_by(
+                PreferenceShowAttributes.deleted.is_(False)).filter(
+                PreferenceShowAttributes.uid == current_user.uid).group_by(
                 PreferenceShowAttributes.uid, PreferenceShowAttributes.type_id)
             for i in types:
-                if i.uid == g.user.uid:
+                result['self']['instance'].append(i.type_id)
-                    result['self']['instance'].append(i.type_id)
+                if str(i.created_at) > str(result['self']['type_id2subs_time'].get(i.type_id, "")):
-                    if str(i.created_at) > str(result['self']['type_id2subs_time'].get(i.type_id, "")):
+                    result['self']['type_id2subs_time'][i.type_id] = i.created_at
-                        result['self']['type_id2subs_time'][i.type_id] = i.created_at
 
-                result['type_id2users'].setdefault(i.type_id, []).append(i.uid)
+            instance_order = [i.type_id for i in ci_type_order if not i.is_tree]
+            if len(instance_order) == len(result['self']['instance']):
+                result['self']['instance'] = instance_order
 
         if tree:
-            types = PreferenceTreeView.get_by(to_dict=False)
+            types = PreferenceTreeView.get_by(uid=current_user.uid, to_dict=False)
             for i in types:
-                if i.uid == g.user.uid:
+                result['self']['tree'].append(i.type_id)
-                    result['self']['tree'].append(i.type_id)
+                if str(i.created_at) > str(result['self']['type_id2subs_time'].get(i.type_id, "")):
-                    if str(i.created_at) > str(result['self']['type_id2subs_time'].get(i.type_id, "")):
+                    result['self']['type_id2subs_time'][i.type_id] = i.created_at
-                        result['self']['type_id2subs_time'][i.type_id] = i.created_at
 
-                result['type_id2users'].setdefault(i.type_id, [])
+            tree_order = [i.type_id for i in ci_type_order if i.is_tree]
-                if i.uid not in result['type_id2users'][i.type_id]:
+            if len(tree_order) == len(result['self']['tree']):
-                    result['type_id2users'][i.type_id].append(i.uid)
+                result['self']['tree'] = tree_order
 
         return result
 
     @staticmethod
     def get_show_attributes(type_id):
+        _type = CITypeCache.get(type_id) or abort(404, ErrFormat.ci_type_not_found)
+        type_id = _type and _type.id
+
         if not isinstance(type_id, six.integer_types):
             _type = CITypeCache.get(type_id)
             type_id = _type and _type.id
 
-        attrs = db.session.query(PreferenceShowAttributes, CITypeAttribute.order).join(
+        attrs = PreferenceShowAttributes.get_by(uid=current_user.uid, type_id=type_id, to_dict=False)
-            CITypeAttribute, CITypeAttribute.attr_id == PreferenceShowAttributes.attr_id).filter(
-            PreferenceShowAttributes.uid == g.user.uid).filter(
-            PreferenceShowAttributes.type_id == type_id).filter(
-            PreferenceShowAttributes.deleted.is_(False)).filter(CITypeAttribute.deleted.is_(False)).filter(
-            CITypeAttribute.type_id == type_id).all()
 
         result = []
-        for i in sorted(attrs, key=lambda x: x.PreferenceShowAttributes.order):
+        for i in sorted(attrs, key=lambda x: x.order):
-            item = i.PreferenceShowAttributes.attr.to_dict()
+            if i.attr_id:
-            item.update(dict(is_fixed=i.PreferenceShowAttributes.is_fixed))
+                item = i.attr.to_dict()
+            elif i.builtin_attr:
+                item = dict(name=i.builtin_attr, alias=BUILTIN_ATTRIBUTES[i.builtin_attr])
+            else:
+                item = dict(name="", alias="")
+            item.update(dict(is_fixed=i.is_fixed))
             result.append(item)
 
         is_subscribed = True
         if not attrs:
-            attrs = db.session.query(CITypeAttribute).filter(
+            result = CITypeAttributeManager.get_attributes_by_type_id(type_id,
-                CITypeAttribute.type_id == type_id).filter(
+                                                                      choice_web_hook_parse=False,
-                CITypeAttribute.deleted.is_(False)).filter(
+                                                                      choice_other_parse=False)
-                CITypeAttribute.default_show.is_(True)).order_by(CITypeAttribute.order)
+            result = [i for i in result if i['default_show']]
-            result = [i.attr.to_dict() for i in attrs]
+
+            for i in BUILTIN_ATTRIBUTES:
+                result.append(dict(name=i, alias=BUILTIN_ATTRIBUTES[i]))
+
             is_subscribed = False
 
         for i in result:
-            if i["is_choice"]:
+            if i.get("is_choice"):
                 i.update(dict(choice_value=AttributeManager.get_choice_values(
-                    i["id"], i["value_type"], i["choice_web_hook"])))
+                    i["id"], i["value_type"], i.get("choice_web_hook"), i.get("choice_other"))))
+
+                if (_type.name in SysComputedAttributes.type2attr and
+                        i['name'] in SysComputedAttributes.type2attr[_type.name]):
+                    i['sys_computed'] = True
+                else:
+                    i['sys_computed'] = False
 
         return is_subscribed, result
 
     @classmethod
     def create_or_update_show_attributes(cls, type_id, attr_order):
-        existed_all = PreferenceShowAttributes.get_by(type_id=type_id, uid=g.user.uid, to_dict=False)
+        existed_all = PreferenceShowAttributes.get_by(type_id=type_id, uid=current_user.uid, to_dict=False)
         for x, order in attr_order:
             if isinstance(x, list):
                 _attr, is_fixed = x
             else:
                 _attr, is_fixed = x, False
-            attr = AttributeCache.get(_attr) or abort(404, ErrFormat.attribute_not_found.format("id={}".format(_attr)))
+
+            if _attr in BUILTIN_ATTRIBUTES:
+                attr = None
+                builtin_attr = _attr
+            else:
+                attr = AttributeCache.get(_attr) or abort(
+                    404, ErrFormat.attribute_not_found.format("id={}".format(_attr)))
+                builtin_attr = None
             existed = PreferenceShowAttributes.get_by(type_id=type_id,
-                                                      uid=g.user.uid,
+                                                      uid=current_user.uid,
-                                                      attr_id=attr.id,
+                                                      attr_id=attr and attr.id,
+                                                      builtin_attr=builtin_attr,
                                                       first=True,
                                                       to_dict=False)
             if existed is None:
                 PreferenceShowAttributes.create(type_id=type_id,
-                                                uid=g.user.uid,
+                                                uid=current_user.uid,
-                                                attr_id=attr.id,
+                                                attr_id=attr and attr.id,
+                                                builtin_attr=builtin_attr,
                                                 order=order,
                                                 is_fixed=is_fixed)
             else:
                 existed.update(order=order, is_fixed=is_fixed)
 
-        attr_dict = {int(i[0]) if isinstance(i, list) else int(i): j for i, j in attr_order}
+        attr_dict = {(int(i[0]) if i[0].isdigit() else i[0]) if isinstance(i, list) else
+                     (int(i) if i.isdigit() else i): j for i, j in attr_order}
         for i in existed_all:
-            if i.attr_id not in attr_dict:
+            if (i.attr_id and i.attr_id not in attr_dict) or (i.builtin_attr and i.builtin_attr not in attr_dict):
                 i.soft_delete()
 
+        if not existed_all and attr_order:
+            cls.add_ci_type_order_item(type_id, is_tree=False)
+
+        elif not PreferenceShowAttributes.get_by(type_id=type_id, uid=current_user.uid, to_dict=False):
+            cls.delete_ci_type_order_item(type_id, is_tree=False)
+
     @staticmethod
     def get_tree_view():
-        res = PreferenceTreeView.get_by(uid=g.user.uid, to_dict=True)
+        ci_type_order = sorted(PreferenceCITypeOrder.get_by(uid=current_user.uid, is_tree=True, to_dict=False),
+                               key=lambda x: x.order)
+
+        res = PreferenceTreeView.get_by(uid=current_user.uid, to_dict=True)
+        if ci_type_order:
+            res = sorted(res, key=lambda x: {ii.type_id: idx for idx, ii in enumerate(
+                ci_type_order)}.get(x['type_id'], 1))
+
         for item in res:
             if item["levels"]:
                 ci_type = CITypeCache.get(item['type_id']).to_dict()
@@ -167,8 +250,8 @@ class PreferenceManager(object):
 
         return res
 
-    @staticmethod
+    @classmethod
-    def create_or_update_tree_view(type_id, levels):
+    def create_or_update_tree_view(cls, type_id, levels):
         attrs = CITypeAttributesCache.get(type_id)
         for idx, i in enumerate(levels):
             for attr in attrs:
@@ -176,14 +259,17 @@ class PreferenceManager(object):
                 if i == attr.id or i == attr.name or i == attr.alias:
                     levels[idx] = attr.id
 
-        existed = PreferenceTreeView.get_by(uid=g.user.uid, type_id=type_id, to_dict=False, first=True)
+        existed = PreferenceTreeView.get_by(uid=current_user.uid, type_id=type_id, to_dict=False, first=True)
         if existed is not None:
             if not levels:
                 existed.soft_delete()
+                cls.delete_ci_type_order_item(type_id, is_tree=True)
                 return existed
             return existed.update(levels=levels)
         elif levels:
-            return PreferenceTreeView.create(levels=levels, type_id=type_id, uid=g.user.uid)
+            cls.add_ci_type_order_item(type_id, is_tree=True)
+
+            return PreferenceTreeView.create(levels=levels, type_id=type_id, uid=current_user.uid)
 
     @staticmethod
     def get_relation_view():
@@ -201,12 +287,14 @@ class PreferenceManager(object):
         else:
             views = _views
 
-        view2cr_ids = dict()
+        view2cr_ids = defaultdict(list)
+        name2view = dict()
         result = dict()
         name2id = list()
         for view in views:
-            view2cr_ids.setdefault(view['name'], []).extend(view['cr_ids'])
+            view2cr_ids[view['name']].extend(view['cr_ids'])
             name2id.append([view['name'], view['id']])
+            name2view[view['name']] = view
 
         id2type = dict()
         for view_name in view2cr_ids:
@@ -227,15 +315,31 @@ class PreferenceManager(object):
                 if not parents:
                     return
 
-            for l in leaf:
+            for _l in leaf:
-                _find_parent(l)
+                _find_parent(_l)
 
             for node_id in node2show_types:
                 node2show_types[node_id] = [CITypeCache.get(i).to_dict() for i in set(node2show_types[node_id])]
 
+            topo_flatten = list(toposort.toposort_flatten(topo))
+            level2constraint = {}
+            for i, _ in enumerate(topo_flatten[1:]):
+                ctr = CITypeRelation.get_by(
+                    parent_id=topo_flatten[i], child_id=topo_flatten[i + 1], first=True, to_dict=False)
+                level2constraint[i + 1] = ctr and ctr.constraint
+
+            if leaf2show_types.get(topo_flatten[-1]):
+                ctr = CITypeRelation.get_by(
+                    parent_id=topo_flatten[-1],
+                    child_id=leaf2show_types[topo_flatten[-1]][0], first=True, to_dict=False)
+                level2constraint[len(topo_flatten)] = ctr and ctr.constraint
+
             result[view_name] = dict(topo=list(map(list, toposort.toposort(topo))),
-                                     topo_flatten=list(toposort.toposort_flatten(topo)),
+                                     topo_flatten=topo_flatten,
+                                     level2constraint=level2constraint,
                                      leaf=leaf,
+                                     option=name2view[view_name]['option'],
+                                     is_public=name2view[view_name]['is_public'],
                                      leaf2show_types=leaf2show_types,
                                      node2show_types=node2show_types,
                                      show_types=[CITypeCache.get(j).to_dict()
@@ -243,18 +347,26 @@ class PreferenceManager(object):
 
         for type_id in id2type:
             id2type[type_id] = CITypeCache.get(type_id).to_dict()
+            id2type[type_id]['unique_name'] = AttributeCache.get(id2type[type_id]['unique_id']).name
+            if id2type[type_id]['show_id']:
+                show_attr = AttributeCache.get(id2type[type_id]['show_id'])
+                id2type[type_id]['show_name'] = show_attr and show_attr.name
 
         return result, id2type, sorted(name2id, key=lambda x: x[1])
 
     @classmethod
-    def create_or_update_relation_view(cls, name, cr_ids, is_public=False):
+    def create_or_update_relation_view(cls, name=None, cr_ids=None, _id=None, is_public=False, option=None):
         if not cr_ids:
             return abort(400, ErrFormat.preference_relation_view_node_required)
 
-        existed = PreferenceRelationView.get_by(name=name, to_dict=False, first=True)
+        if _id is None:
+            existed = PreferenceRelationView.get_by(name=name, to_dict=False, first=True)
+        else:
+            existed = PreferenceRelationView.get_by_id(_id)
         current_app.logger.debug(existed)
         if existed is None:
-            PreferenceRelationView.create(name=name, cr_ids=cr_ids, uid=g.user.uid, is_public=is_public)
+            PreferenceRelationView.create(name=name, cr_ids=cr_ids, uid=current_user.uid,
+                                          is_public=is_public, option=option)
 
             if current_app.config.get("USE_ACL"):
                 ACLManager().add_resource(name, ResourceTypeEnum.RELATION_VIEW)
@@ -262,6 +374,11 @@ class PreferenceManager(object):
                                                     RoleEnum.CMDB_READ_ALL,
                                                     ResourceTypeEnum.RELATION_VIEW,
                                                     permissions=[PermEnum.READ])
+        else:
+            if existed.name != name and current_app.config.get("USE_ACL"):
+                ACLManager().update_resource(existed.name, name, ResourceTypeEnum.RELATION_VIEW)
+
+            existed.update(name=name, cr_ids=cr_ids, is_public=is_public, option=option)
 
         return cls.get_relation_view()
 
@@ -278,7 +395,7 @@ class PreferenceManager(object):
     @staticmethod
     def get_search_option(**kwargs):
         query = PreferenceSearchOption.get_by(only_query=True)
-        query = query.filter(PreferenceSearchOption.uid == g.user.uid)
+        query = query.filter(PreferenceSearchOption.uid == current_user.uid)
 
         for k in kwargs:
             if hasattr(PreferenceSearchOption, k) and kwargs[k]:
@@ -288,16 +405,24 @@ class PreferenceManager(object):
 
     @staticmethod
     def add_search_option(**kwargs):
-        kwargs['uid'] = g.user.uid
+        kwargs['uid'] = current_user.uid
 
-        existed = PreferenceSearchOption.get_by(uid=g.user.uid,
+        if kwargs['name'] in ('__recent__', '__favor__', '__relation_favor__'):
-                                                name=kwargs.get('name'),
+            if kwargs['name'] == '__recent__':
-                                                prv_id=kwargs.get('prv_id'),
+                for i in PreferenceSearchOption.get_by(
-                                                ptv_id=kwargs.get('ptv_id'),
+                        only_query=True, name=kwargs['name'], uid=current_user.uid).order_by(
-                                                type_id=kwargs.get('type_id'),
+                    PreferenceSearchOption.id.desc()).offset(20):
-                                                )
+                    i.delete()
-        if existed:
+
-            return abort(400, ErrFormat.preference_search_option_exists)
+        else:
+            existed = PreferenceSearchOption.get_by(uid=current_user.uid,
+                                                    name=kwargs.get('name'),
+                                                    prv_id=kwargs.get('prv_id'),
+                                                    ptv_id=kwargs.get('ptv_id'),
+                                                    type_id=kwargs.get('type_id'),
+                                                    )
+            if existed:
+                return abort(400, ErrFormat.preference_search_option_exists)
 
         return PreferenceSearchOption.create(**kwargs)
 
@@ -306,10 +431,10 @@ class PreferenceManager(object):
 
         existed = PreferenceSearchOption.get_by_id(_id) or abort(404, ErrFormat.preference_search_option_not_found)
 
-        if g.user.uid != existed.uid:
+        if current_user.uid != existed.uid:
             return abort(400, ErrFormat.no_permission2)
 
-        other = PreferenceSearchOption.get_by(uid=g.user.uid,
+        other = PreferenceSearchOption.get_by(uid=current_user.uid,
                                               name=kwargs.get('name'),
                                               prv_id=kwargs.get('prv_id'),
                                               ptv_id=kwargs.get('ptv_id'),
@@ -324,7 +449,7 @@ class PreferenceManager(object):
     def delete_search_option(_id):
         existed = PreferenceSearchOption.get_by_id(_id) or abort(404, ErrFormat.preference_search_option_not_found)
 
-        if g.user.uid != existed.uid:
+        if current_user.uid != existed.uid:
             return abort(400, ErrFormat.no_permission2)
 
         existed.soft_delete()
@@ -336,3 +461,65 @@ class PreferenceManager(object):
 
         for i in PreferenceTreeView.get_by(type_id=type_id, uid=uid, to_dict=False):
             i.soft_delete()
+
+        for i in PreferenceCITypeOrder.get_by(type_id=type_id, uid=uid, to_dict=False):
+            i.soft_delete()
+
+    @staticmethod
+    def can_edit_relation(parent_id, child_id):
+        views = PreferenceRelationView.get_by(to_dict=False)
+        for view in views:
+            has_m2m = False
+            last_node_id = None
+            for cr in view.cr_ids:
+                _rel = CITypeRelation.get_by(parent_id=cr['parent_id'], child_id=cr['child_id'],
+                                             first=True, to_dict=False)
+                if _rel and _rel.constraint == ConstraintEnum.Many2Many:
+                    has_m2m = True
+
+                    if parent_id == _rel.parent_id and child_id == _rel.child_id:
+                        return False
+
+                if _rel:
+                    last_node_id = _rel.child_id
+
+            if parent_id == last_node_id:
+                rels = CITypeRelation.get_by(parent_id=last_node_id, to_dict=False)
+                for rel in rels:
+                    if rel.child_id == child_id and has_m2m:
+                        return False
+
+        return True
+
+    @staticmethod
+    def add_ci_type_order_item(type_id, is_tree=False):
+        max_order = PreferenceCITypeOrder.get_by(
+            uid=current_user.uid, is_tree=is_tree, only_query=True).order_by(PreferenceCITypeOrder.order.desc()).first()
+        order = (max_order and max_order.order + 1) or 1
+
+        PreferenceCITypeOrder.create(type_id=type_id, is_tree=is_tree, uid=current_user.uid, order=order)
+
+    @staticmethod
+    def delete_ci_type_order_item(type_id, is_tree=False):
+        existed = PreferenceCITypeOrder.get_by(uid=current_user.uid, type_id=type_id, is_tree=is_tree,
+                                               first=True, to_dict=False)
+
+        existed and existed.soft_delete()
+
+    @staticmethod
+    def upsert_ci_type_order(type_ids, is_tree=False):
+        for idx, type_id in enumerate(type_ids):
+            order = idx + 1
+            existed = PreferenceCITypeOrder.get_by(uid=current_user.uid, type_id=type_id, is_tree=is_tree,
+                                                   to_dict=False, first=True)
+            if existed is not None:
+                existed.update(order=order, flush=True)
+            else:
+                PreferenceCITypeOrder.create(uid=current_user.uid, type_id=type_id, is_tree=is_tree, order=order,
+                                             flush=True)
+        try:
+            db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            current_app.logger.error("upsert citype order failed: {}".format(e))
+            return abort(400, ErrFormat.unknown_error)

cmdb-api/api/lib/cmdb/query_sql.py

@@ -42,7 +42,7 @@ FACET_QUERY1 = """
 
 FACET_QUERY = """
     SELECT {0}.value,
-           count({0}.ci_id)
+           count(distinct({0}.ci_id))
     FROM {0}
     INNER JOIN ({1}) AS F ON F.ci_id={0}.ci_id
     WHERE {0}.attr_id={2:d}

cmdb-api/api/lib/cmdb/relation_type.py

@@ -24,21 +24,21 @@ class RelationTypeManager(object):
 
     @staticmethod
     def add(name):
-        RelationType.get_by(name=name, first=True, to_dict=False) and \
+        RelationType.get_by(name=name, first=True, to_dict=False) and abort(
-        abort(400, ErrFormat.relation_type_exists.format(name))
+            400, ErrFormat.relation_type_exists.format(name))
 
         return RelationType.create(name=name)
 
     @staticmethod
     def update(rel_id, name):
-        existed = RelationType.get_by_id(rel_id) or \
+        existed = RelationType.get_by_id(rel_id) or abort(
-                  abort(404, ErrFormat.relation_type_not_found.format("id={}".format(rel_id)))
+            404, ErrFormat.relation_type_not_found.format("id={}".format(rel_id)))
 
         return existed.update(name=name)
 
     @staticmethod
     def delete(rel_id):
-        existed = RelationType.get_by_id(rel_id) or \
+        existed = RelationType.get_by_id(rel_id) or abort(
-                  abort(404, ErrFormat.relation_type_not_found.format("id={}".format(rel_id)))
+            404, ErrFormat.relation_type_not_found.format("id={}".format(rel_id)))
 
         existed.soft_delete()

cmdb-api/api/lib/cmdb/resp_format.py

@@ -1,93 +1,178 @@
 # -*- coding:utf-8 -*-
 
+from flask_babel import lazy_gettext as _l
+
 from api.lib.resp_format import CommonErrFormat
 
 
 class ErrFormat(CommonErrFormat):
-    invalid_relation_type = "无效的关系类型: {}"
+    ci_type_config = _l("CI Model")  # 模型配置
-    ci_type_not_found = "模型不存在!"
-    argument_attributes_must_be_list = "参数 attributes 类型必须是列表"
-    argument_file_not_found = "文件似乎并未上传"
 
-    attribute_not_found = "属性 {} 不存在!"
+    invalid_relation_type = _l("Invalid relation type: {}")  # 无效的关系类型: {}
-    attribute_value_type_cannot_change = "属性的值类型不允许修改!"
+    ci_type_not_found = _l("CIType is not found")  # 模型不存在!
-    attribute_list_value_cannot_change = "多值不被允许修改!"
-    attribute_index_cannot_change = "修改索引 非管理员不被允许!"
-    attribute_index_change_failed = "索引切换失败!"
-    invalid_choice_values = "预定义值的类型不对！"
-    attribute_name_duplicate = "重复的属性名 {}"
-    add_attribute_failed = "创建属性 {} 失败!"
-    update_attribute_failed = "修改属性 {} 失败!"
-    cannot_edit_attribute = "您没有权限修改该属性!"
-    cannot_delete_attribute = "您没有权限删除该属性!"
-    attribute_name_cannot_be_builtin = "属性字段名不能是内置字段: id, _id, ci_id, type, _type, ci_type"
 
-    ci_not_found = "CI {} 不存在"
+    # 参数 attributes 类型必须是列表
-    unique_constraint = "多属性联合唯一校验不通过: {}"
+    argument_attributes_must_be_list = _l("The type of parameter attributes must be a list")
-    unique_value_not_found = "模型的主键 {} 不存在!"
+    argument_file_not_found = _l("The file doesn't seem to be uploaded")  # 文件似乎并未上传
-    unique_key_required = "主键字段 {} 缺失"
-    ci_is_already_existed = "CI 已经存在!"
-    relation_constraint = "关系约束: {}, 校验失败 "
-    relation_not_found = "CI关系: {} 不存在"
-    ci_search_Parentheses_invalid = "搜索表达式里小括号前不支持: 或、非"
 
-    ci_type_not_found2 = "模型 {} 不存在"
+    attribute_not_found = _l("Attribute {} does not exist!")  # 属性 {} 不存在!
-    ci_type_is_already_existed = "模型 {} 已经存在"
+    # 该属性是模型的唯一标识，不能被删除!
-    unique_key_not_define = "主键未定义或者已被删除"
+    attribute_is_unique_id = _l(
-    only_owner_can_delete = "只有创建人才能删除它!"
+        "This attribute is the unique identifier of the model and cannot be deleted!")  
-    ci_exists_and_cannot_delete_type = "因为CI已经存在，不能删除模型"
+    attribute_is_ref_by_type = _l(
-    ci_type_group_not_found = "模型分组 {} 不存在"
+        "This attribute is referenced by model {} and cannot be deleted!")  # 该属性被模型 {} 引用, 不能删除!
-    ci_type_group_exists = "模型分组 {} 已经存在"
+    attribute_value_type_cannot_change = _l(
-    ci_type_relation_not_found = "模型关系 {} 不存在"
+        "The value type of the attribute is not allowed to be modified!")  # 属性的值类型不允许修改!
-    ci_type_attribute_group_duplicate = "属性分组 {} 已存在"
+    attribute_list_value_cannot_change = _l("Multiple values are not allowed to be modified!")  # 多值不被允许修改!
-    ci_type_attribute_group_not_found = "属性分组 {} 不存在"
+    # 修改索引 非管理员不被允许!
-    ci_type_group_attribute_not_found = "属性组<{0}> - 属性<{1}> 不存在"
+    attribute_index_cannot_change = _l("Modifying the index is not allowed for non-administrators!")
-    unique_constraint_duplicate = "唯一约束已经存在!"
+    attribute_index_change_failed = _l("Index switching failed!")  # 索引切换失败!
-    unique_constraint_invalid = "唯一约束的属性不能是 JSON 和 多值"
+    invalid_choice_values = _l("The predefined value is of the wrong type!")  # 预定义值的类型不对！
-    ci_type_trigger_duplicate = "重复的触发器"
+    attribute_name_duplicate = _l("Duplicate attribute name {}")  # 重复的属性名 {}
-    ci_type_trigger_not_found = "触发器 {} 不存在"
+    add_attribute_failed = _l("Failed to create attribute {}!")  # 创建属性 {} 失败!
+    update_attribute_failed = _l("Modify attribute {} failed!")  # 修改属性 {} 失败!
+    cannot_edit_attribute = _l("You do not have permission to modify this attribute!")  # 您没有权限修改该属性!
+    cannot_delete_attribute = _l(
+        "Only creators and administrators are allowed to delete attributes!")  # 目前只允许 属性创建人、管理员 删除属性!
+    # 属性字段名不能是内置字段: id, _id, ci_id, type, _type, ci_type
+    attribute_name_cannot_be_builtin = _l(
+        "Attribute field names cannot be built-in fields: id, _id, ci_id, type, _type, ci_type, ticket_id")
+    attribute_choice_other_invalid = _l(
+        "Predefined value: Other model request parameters are illegal!")  # 预定义值: 其他模型请求参数不合法!
 
-    record_not_found = "操作记录 {} 不存在"
+    ci_not_found = _l("CI {} does not exist")  # CI {} 不存在
-    cannot_delete_unique = "不能删除唯一标识"
+    unique_constraint = _l("Multiple attribute joint unique verification failed: {}")  # 多属性联合唯一校验不通过: {}
-    cannot_delete_default_order_attr = "不能删除默认排序的属性"
+    unique_value_not_found = _l("The model's primary key {} does not exist!")  # 模型的主键 {} 不存在!
+    unique_key_required = _l("Primary key {} is missing")  # 主键字段 {} 缺失
+    ci_is_already_existed = _l("CI already exists!")  # CI 已经存在!
+    ci_reference_not_found = _l("{}: CI reference {} does not exist!")  # {}: CI引用 {} 不存在!
+    ci_reference_invalid = _l("{}: CI reference {} is illegal!")  # {}, CI引用 {} 不合法!
+    relation_constraint = _l("Relationship constraint: {}, verification failed")  # 关系约束: {}, 校验失败
+    # 多对多关系 限制: 模型 {} <-> {} 已经存在多对多关系!
+    m2m_relation_constraint = _l(
+        "Many-to-many relationship constraint: Model {} <-> {} already has a many-to-many relationship!")
 
-    preference_relation_view_node_required = "没有选择节点"
+    relation_not_found = _l("CI relationship: {} does not exist")  # CI关系: {} 不存在
-    preference_search_option_not_found = "该搜索选项不存在!"
-    preference_search_option_exists = "该搜索选项命名重复!"
 
-    relation_type_exists = "关系类型 {} 已经存在"
+    # 搜索表达式里小括号前不支持: 或、非
-    relation_type_not_found = "关系类型 {} 不存在"
+    ci_search_Parentheses_invalid = _l("In search expressions, not supported before parentheses: or, not")
 
-    attribute_value_invalid = "无效的属性值: {}"
+    ci_type_not_found2 = _l("Model {} does not exist")  # 模型 {} 不存在
-    attribute_value_invalid2 = "{} 无效的值: {}"
+    ci_type_is_already_existed = _l("Model {} already exists")  # 模型 {} 已经存在
-    not_in_choice_values = "{} 不在预定义值里"
+    unique_key_not_define = _l("The primary key is undefined or has been deleted")  # 主键未定义或者已被删除
-    attribute_value_unique_required = "属性 {} 的值必须是唯一的, 当前值 {} 已存在"
+    only_owner_can_delete = _l("Only the creator can delete it!")  # 只有创建人才能删除它!
-    attribute_value_required = "属性 {} 值必须存在"
+    ci_exists_and_cannot_delete_type = _l(
-    attribute_value_unknown_error = "新增或者修改属性值未知错误: {}"
+        "The model cannot be deleted because the CI already exists")  # 因为CI已经存在，不能删除模型
+    ci_exists_and_cannot_delete_inheritance = _l(
+        "The inheritance cannot be deleted because the CI already exists")  # 因为CI已经存在，不能删除继承关系
+    ci_type_inheritance_cannot_delete = _l("The model is inherited and cannot be deleted")  # 该模型被继承, 不能删除
+    ci_type_referenced_cannot_delete = _l(
+        "The model is referenced by attribute {} and cannot be deleted")  # 该模型被属性 {} 引用, 不能删除
 
-    custom_name_duplicate = "订制名重复"
+    # 因为关系视图 {} 引用了该模型，不能删除模型
+    ci_relation_view_exists_and_cannot_delete_type = _l(
+        "The model cannot be deleted because the model is referenced by the relational view {}")
+    ci_type_group_not_found = _l("Model group {} does not exist")  # 模型分组 {} 不存在
+    ci_type_group_exists = _l("Model group {} already exists")  # 模型分组 {} 已经存在
+    ci_type_relation_not_found = _l("Model relationship {} does not exist")  # 模型关系 {} 不存在
+    ci_type_attribute_group_duplicate = _l("Attribute group {} already exists")  # 属性分组 {} 已存在
+    ci_type_attribute_group_not_found = _l("Attribute group {} does not exist")  # 属性分组 {} 不存在
+    # 属性组<{0}> - 属性<{1}> 不存在
+    ci_type_group_attribute_not_found = _l("Attribute group <{0}> - attribute <{1}> does not exist")
+    unique_constraint_duplicate = _l("The unique constraint already exists!")  # 唯一约束已经存在!
+    # 唯一约束的属性不能是 JSON 和 多值
+    unique_constraint_invalid = _l("Uniquely constrained attributes cannot be JSON and multi-valued")
+    ci_type_trigger_duplicate = _l("Duplicated trigger")  # 重复的触发器
+    ci_type_trigger_not_found = _l("Trigger {} does not exist")  # 触发器 {} 不存在
+    ci_type_reconciliation_duplicate = _l("Duplicated reconciliation rule")  # 重复的校验规则
+    ci_type_reconciliation_not_found = _l("Reconciliation rule {} does not exist")  # 规则 {} 不存在
 
-    limit_ci_type = "模型数超过限制: {}"
+    record_not_found = _l("Operation record {} does not exist")  # 操作记录 {} 不存在
-    limit_ci = "CI数超过限制: {}"
+    cannot_delete_unique = _l("Unique identifier cannot be deleted")  # 不能删除唯一标识
+    cannot_delete_default_order_attr = _l("Cannot delete default sorted attributes")  # 不能删除默认排序的属性
 
-    adr_duplicate = "自动发现规则: {} 已经存在!"
+    preference_relation_view_node_required = _l("No node selected")  # 没有选择节点
-    adr_not_found = "自动发现规则: {} 不存在!"
+    preference_search_option_not_found = _l("This search option does not exist!")  # 该搜索选项不存在!
-    adr_referenced = "该自动发现规则被模型引用, 不能删除!"
+    preference_search_option_exists = _l("This search option has a duplicate name!")  # 该搜索选项命名重复!
-    ad_duplicate = "自动发现规则的应用不能重复定义!"
-    ad_not_found = "您要修改的自动发现: {} 不存在!"
-    ad_not_unique_key = "属性字段没有包括唯一标识: {}"
-    adc_not_found = "自动发现的实例不存在!"
-    adt_not_found = "模型并未关联该自动发现!"
-    adt_secret_no_permission = "只有创建人才能修改Secret!"
-    cannot_delete_adt = "该规则已经有自动发现的实例, 不能被删除!"
-    adr_default_ref_once = "该默认的自动发现规则 已经被模型 {} 引用!"
-    adr_unique_key_required = "unique_key方法必须返回非空字符串!"
-    adr_plugin_attributes_list_required = "attributes方法必须返回的是list"
-    adr_plugin_attributes_list_no_empty = "attributes方法返回的list不能为空!"
-    adt_target_all_no_permission = "只有管理员才可以定义执行机器为: 所有节点!"
-    adt_target_expr_no_permission = "执行机器权限检查不通过: {}"
 
-    ci_filter_name_cannot_be_empty = "CI过滤授权 必须命名!"
+    relation_type_exists = _l("Relationship type {} already exists")  # 关系类型 {} 已经存在
-    ci_filter_perm_cannot_or_query = "CI过滤授权 暂时不支持 或 查询"
+    relation_type_not_found = _l("Relationship type {} does not exist")  # 关系类型 {} 不存在
-    ci_filter_perm_attr_no_permission = "您没有属性 {} 的操作权限!"
+
-    ci_filter_perm_ci_no_permission = "您没有该CI的操作权限!"
+    attribute_value_invalid = _l("Invalid attribute value: {}")  # 无效的属性值: {}
+    attribute_value_invalid2 = _l("{} Invalid value: {}")  # {} 无效的值: {}
+    not_in_choice_values = _l("{} is not in the predefined values")  # {} 不在预定义值里
+    # 属性 {} 的值必须是唯一的, 当前值 {} 已存在
+    attribute_value_unique_required = _l("The value of attribute {} must be unique, {} already exists")
+    attribute_value_required = _l("Attribute {} value must exist")  # 属性 {} 值必须存在
+    attribute_value_out_of_range = _l("Out of range value, the maximum value is 2147483647")
+    # 新增或者修改属性值未知错误: {}
+    attribute_value_unknown_error = _l("Unknown error when adding or modifying attribute value: {}")
+
+    custom_name_duplicate = _l("Duplicate custom name")  # 订制名重复
+
+    limit_ci_type = _l("Number of models exceeds limit: {}")  # 模型数超过限制: {}
+    limit_ci = _l("The number of CIs exceeds the limit: {}")  # CI数超过限制: {}
+
+    adr_duplicate = _l("Auto-discovery rule: {} already exists!")  # 自动发现规则: {} 已经存在!
+    adr_not_found = _l("Auto-discovery rule: {} does not exist!")  # 自动发现规则: {} 不存在!
+    # 该自动发现规则被模型引用, 不能删除!
+    adr_referenced = _l("This auto-discovery rule is referenced by the model and cannot be deleted!")
+    # 自动发现规则的应用不能重复定义!
+    ad_duplicate = _l("The application of auto-discovery rules cannot be defined repeatedly!")
+    ad_not_found = _l("The auto-discovery you want to modify: {} does not exist!")  # 您要修改的自动发现: {} 不存在!
+    ad_not_unique_key = _l("Attribute does not include unique identifier: {}")  # 属性字段没有包括唯一标识: {}
+    adc_not_found = _l("The auto-discovery instance does not exist!")  # 自动发现的实例不存在!
+    adt_not_found = _l("The model is not associated with this auto-discovery!")  # 模型并未关联该自动发现!
+    adt_secret_no_permission = _l("Only the creator can modify the Secret!")  # 只有创建人才能修改Secret!
+    # 该规则已经有自动发现的实例, 不能被删除!
+    cannot_delete_adt = _l("This rule already has auto-discovery instances and cannot be deleted!")
+    # 该默认的自动发现规则 已经被模型 {} 引用!
+    adr_default_ref_once = _l("The default auto-discovery rule is already referenced by model {}!")
+    # unique_key方法必须返回非空字符串!
+    adr_unique_key_required = _l("The unique_key method must return a non-empty string!")
+    # attributes方法必须返回的是list
+    adr_plugin_attributes_list_required = _l("The attributes method must return a list")  
+    # attributes方法返回的list不能为空!
+    adr_plugin_attributes_list_no_empty = _l("The list returned by the attributes method cannot be empty!")
+    # 只有管理员才可以定义执行机器为: 所有节点!
+    adt_target_all_no_permission = _l("Only administrators can define execution targets as: all nodes!")
+    adt_target_expr_no_permission = _l("Execute targets permission check failed: {}")  # 执行机器权限检查不通过: {}
+
+    ci_filter_name_cannot_be_empty = _l("CI filter authorization must be named!")  # CI过滤授权 必须命名!
+    ci_filter_perm_cannot_or_query = _l(
+        "CI filter authorization is currently not supported or query")  # CI过滤授权 暂时不支持 或 查询
+    # 您没有属性 {} 的操作权限!
+    ci_filter_perm_attr_no_permission = _l("You do not have permission to operate attribute {}!")
+    ci_filter_perm_ci_no_permission = _l("You do not have permission to operate this CI!")  # 您没有该CI的操作权限!
+
+    password_save_failed = _l("Failed to save password: {}")  # 保存密码失败: {}
+    password_load_failed = _l("Failed to get password: {}")  # 获取密码失败: {}
+
+    cron_time_format_invalid = _l("Scheduling time format error")  # 调度时间格式错误
+    reconciliation_title = _l("CMDB data reconciliation results")  # CMDB数据合规检查结果
+    reconciliation_body = _l("Number of {} illegal: {}")  # "{} 不合规数: {}"
+
+    topology_exists = _l("Topology view {} already exists")  # 拓扑视图 {} 已经存在
+    topology_group_exists = _l("Topology group {} already exists")  # 拓扑视图分组 {} 已经存在
+    # 因为该分组下定义了拓扑视图，不能删除
+    topo_view_exists_cannot_delete_group = _l("The group cannot be deleted because the topology view already exists")
+
+    relation_path_search_src_target_required = _l("Both the source model and the target model must be selected")
+
+    builtin_type_cannot_update_name = _l("The names of built-in models cannot be changed")
+    # # IPAM
+    ipam_subnet_model_not_found = _l("The subnet model {} does not exist")
+    ipam_address_model_not_found = _l("The IP Address model {} does not exist")
+    ipam_cidr_invalid_notation = _l("CIDR {} is an invalid notation")
+    ipam_cidr_invalid_subnet = _l("Invalid CIDR: {}, available subnets: {}")
+    ipam_subnet_prefix_length_invalid = _l("Invalid subnet prefix length: {}")
+    ipam_parent_subnet_node_cidr_cannot_empty = _l("parent node cidr must be required")
+    ipam_subnet_overlapped = _l("{} and {} overlap")
+    ipam_subnet_cannot_delete = _l("Cannot delete because child nodes exist")
+    ipam_subnet_not_found = _l("Subnet is not found")
+    ipam_scope_cannot_delete = _l("Cannot delete because child nodes exist")
+
+    # # DCIM
+    dcim_builtin_model_not_found = _l("The dcim model {} does not exist")
+    dcim_rack_u_slot_invalid = _l("Irregularities in Rack Units")
+    dcim_rack_u_count_invalid = _l("The device's position is greater than the rack unit height")

cmdb-api/api/lib/cmdb/search/ci/__init__.py

@@ -16,10 +16,13 @@ def search(query=None,
            ret_key=RetKey.NAME,
            count=1,
            sort=None,
-           excludes=None):
+           excludes=None,
+           use_id_filter=False,
+           use_ci_filter=True):
     if current_app.config.get("USE_ES"):
         s = SearchFromES(query, fl, facet, page, ret_key, count, sort)
     else:
-        s = SearchFromDB(query, fl, facet, page, ret_key, count, sort, excludes=excludes)
+        s = SearchFromDB(query, fl, facet, page, ret_key, count, sort, excludes=excludes,
+                         use_id_filter=use_id_filter, use_ci_filter=use_ci_filter)
 
     return s

cmdb-api/api/lib/cmdb/search/ci/db/query_sql.py

@@ -7,6 +7,7 @@ QUERY_CIS_BY_VALUE_TABLE = """
           attr.alias AS attr_alias,
           attr.value_type,
           attr.is_list,
+          attr.is_password,
           c_cis.type_id,
           {0}.ci_id,
           {0}.attr_id,
@@ -26,7 +27,8 @@ QUERY_CIS_BY_IDS = """
            A.attr_alias,
            A.value,
            A.value_type,
-           A.is_list
+           A.is_list,
+           A.is_password
     FROM
       ({1}) AS A {0}
        ORDER BY A.ci_id;
@@ -43,7 +45,7 @@ FACET_QUERY1 = """
 
 FACET_QUERY = """
     SELECT {0}.value,
-           count({0}.ci_id)
+           count(distinct {0}.ci_id)
     FROM {0}
     INNER JOIN ({1}) AS F ON F.ci_id={0}.ci_id
     WHERE {0}.attr_id={2:d}
@@ -54,13 +56,13 @@ QUERY_CI_BY_ATTR_NAME = """
     SELECT {0}.ci_id
     FROM {0}
     WHERE {0}.attr_id={1:d}
-      AND {0}.value {2}
+      AND ({0}.value {2})
 """
 
 QUERY_CI_BY_ID = """
     SELECT c_cis.id as ci_id
     FROM c_cis
-    WHERE c_cis.id={}
+    WHERE c_cis.id {}
 """
 
 QUERY_CI_BY_TYPE = """
@@ -105,3 +107,12 @@ FROM
     WHERE c_value_index_datetime.value LIKE "{0}") AS {1}
 GROUP BY  {1}.ci_id
 """
+
+QUERY_CI_BY_NO_ATTR_IN = """
+SELECT *
+FROM 
+    (SELECT c_value_index_texts.ci_id
+    FROM c_value_index_texts
+    WHERE c_value_index_texts.value in ({0})) AS {1}
+GROUP BY {1}.ci_id
+"""

cmdb-api/api/lib/cmdb/search/ci/db/search.py

@@ -1,18 +1,22 @@
-# -*- coding:utf-8 -*- 
+# -*- coding:utf-8 -*-
 
 
 from __future__ import unicode_literals
 
 import copy
+import six
 import time
-
+from flask import abort
 from flask import current_app
-from flask import g
+from flask_login import current_user
 from jinja2 import Template
+from sqlalchemy import text
+
 from api.extensions import db
 from api.lib.cmdb.cache import AttributeCache
 from api.lib.cmdb.cache import CITypeCache
 from api.lib.cmdb.ci import CIManager
+from api.lib.cmdb.const import BUILTIN_ATTRIBUTES
 from api.lib.cmdb.const import PermEnum
 from api.lib.cmdb.const import ResourceTypeEnum
 from api.lib.cmdb.const import RetKey
@@ -24,9 +28,11 @@ from api.lib.cmdb.search.ci.db.query_sql import FACET_QUERY
 from api.lib.cmdb.search.ci.db.query_sql import QUERY_CI_BY_ATTR_NAME
 from api.lib.cmdb.search.ci.db.query_sql import QUERY_CI_BY_ID
 from api.lib.cmdb.search.ci.db.query_sql import QUERY_CI_BY_NO_ATTR
+from api.lib.cmdb.search.ci.db.query_sql import QUERY_CI_BY_NO_ATTR_IN
 from api.lib.cmdb.search.ci.db.query_sql import QUERY_CI_BY_TYPE
 from api.lib.cmdb.search.ci.db.query_sql import QUERY_UNION_CI_ATTRIBUTE_IS_NULL
 from api.lib.cmdb.utils import TableMap
+from api.lib.cmdb.utils import ValueTypeMap
 from api.lib.perm.acl.acl import ACLManager
 from api.lib.perm.acl.acl import is_app_admin
 from api.lib.utils import handle_arg_list
@@ -41,7 +47,11 @@ class Search(object):
                  count=1,
                  sort=None,
                  ci_ids=None,
-                 excludes=None):
+                 excludes=None,
+                 parent_node_perm_passed=False,
+                 use_id_filter=False,
+                 use_ci_filter=True,
+                 only_ids=False):
         self.orig_query = query
         self.fl = fl or []
         self.excludes = excludes or []
@@ -51,12 +61,20 @@ class Search(object):
         self.count = count
         self.sort = sort
         self.ci_ids = ci_ids or []
+        self.raw_ci_ids = copy.deepcopy(self.ci_ids)
         self.query_sql = ""
         self.type_id_list = []
         self.only_type_query = False
+        self.parent_node_perm_passed = parent_node_perm_passed
+        self.use_id_filter = use_id_filter
+        self.use_ci_filter = use_ci_filter
+        self.only_ids = only_ids
+        self.multi_type_has_ci_filter = False
 
         self.valid_type_names = []
         self.type2filter_perms = dict()
+        self.is_app_admin = is_app_admin('cmdb') or current_user.username == "worker"
+        self.is_app_admin = self.is_app_admin or (not self.use_ci_filter and not self.use_id_filter)
 
     @staticmethod
     def _operator_proc(key):
@@ -90,81 +108,137 @@ class Search(object):
         else:
             raise SearchError(ErrFormat.attribute_not_found.format(key))
 
-    def _type_query_handler(self, v, queries):
+    def _type_query_handler(self, v, queries, is_sub=False):
         new_v = v[1:-1].split(";") if v.startswith("(") and v.endswith(")") else [v]
+        type_num = len(new_v)
+        type_id_list = []
         for _v in new_v:
             ci_type = CITypeCache.get(_v)
 
-            if len(new_v) == 1 and not self.sort and ci_type and ci_type.default_order_attr:
+            if type_num == 1 and not self.sort and ci_type and ci_type.default_order_attr:
                 self.sort = ci_type.default_order_attr
 
             if ci_type is not None:
                 if self.valid_type_names == "ALL" or ci_type.name in self.valid_type_names:
-                    self.type_id_list.append(str(ci_type.id))
+                    if not is_sub:
-                    if ci_type.id in self.type2filter_perms:
+                        self.type_id_list.append(str(ci_type.id))
+                    type_id_list.append(str(ci_type.id))
+                    if ci_type.id in self.type2filter_perms and not is_sub:
                         ci_filter = self.type2filter_perms[ci_type.id].get('ci_filter')
-                        if ci_filter:
+                        if ci_filter and self.use_ci_filter and not self.use_id_filter:
                             sub = []
-                            ci_filter = Template(ci_filter).render(user=g.user)
+                            ci_filter = Template(ci_filter).render(user=current_user)
                             for i in ci_filter.split(','):
-                                if i.startswith("~") and not sub:
+                                if type_num == 1:
-                                    queries.append(i)
+                                    if i.startswith("~") and not sub:
+                                        queries.append(i)
+                                    else:
+                                        sub.append(i)
                                 else:
                                     sub.append(i)
                             if sub:
-                                queries.append(dict(operator="&", queries=sub))
+                                if type_num == 1:
+                                    queries.append(dict(operator="&", queries=sub))
+                                else:
+                                    if str(ci_type.id) in self.type_id_list:
+                                        self.type_id_list.remove(str(ci_type.id))
+                                    type_id_list.remove(str(ci_type.id))
+                                    sub.extend([i for i in queries[1:] if isinstance(i, (six.string_types, list))])
 
+                                    sub.insert(0, "_type:{}".format(ci_type.id))
+                                    queries.append(dict(operator="|", queries=sub))
+                                    self.multi_type_has_ci_filter = True
                         if self.type2filter_perms[ci_type.id].get('attr_filter'):
-                            if not self.fl:
+                            if type_num == 1:
-                                self.fl = set(self.type2filter_perms[ci_type.id]['attr_filter'])
+                                if not self.fl:
+                                    self.fl = set(self.type2filter_perms[ci_type.id]['attr_filter'])
+                                else:
+                                    fl = set(self.fl) & set(self.type2filter_perms[ci_type.id]['attr_filter'])
+                                    not fl and abort(400, ErrFormat.ci_filter_perm_attr_no_permission.format(self.fl))
+                                    self.fl = fl
                             else:
-                                self.fl = set(self.fl) & set(self.type2filter_perms[ci_type.id]['attr_filter'])
+                                self.fl = self.fl or {}
+                                if not self.fl or isinstance(self.fl, dict):
+                                    self.fl[ci_type.id] = set(self.type2filter_perms[ci_type.id]['attr_filter'])
+
+                        if self.type2filter_perms[ci_type.id].get('id_filter') and self.use_id_filter:
+
+                            if not self.raw_ci_ids:
+                                self.ci_ids = list(self.type2filter_perms[ci_type.id]['id_filter'].keys())
+
+                    if self.use_id_filter and not self.ci_ids and not self.is_app_admin:
+                        self.raw_ci_ids = [0]
                 else:
                     raise SearchError(ErrFormat.no_permission.format(ci_type.alias, PermEnum.READ))
             else:
                 raise SearchError(ErrFormat.ci_type_not_found2.format(_v))
 
-        if self.type_id_list:
+        if type_num != len(self.type_id_list) and queries and queries[0].startswith('_type') and not is_sub:
-            type_ids = ",".join(self.type_id_list)
+            queries[0] = "_type:({})".format(";".join(self.type_id_list))
+
+        if type_id_list:
+            type_ids = ",".join(type_id_list)
             _query_sql = QUERY_CI_BY_TYPE.format(type_ids)
-            if self.only_type_query:
+            if self.only_type_query or self.multi_type_has_ci_filter:
                 return _query_sql
-            else:
+        elif type_num > 1:  # there must be instance-level access control
-                return ""
+            return "select c_cis.id as ci_id from c_cis where c_cis.id=0"
+
         return ""
 
     @staticmethod
     def _id_query_handler(v):
-        return QUERY_CI_BY_ID.format(v)
+        if ";" in v:
+            return QUERY_CI_BY_ID.format("in {}".format(v.replace(';', ',')))
+        else:
+            return QUERY_CI_BY_ID.format("= {}".format(v))
 
     @staticmethod
     def _in_query_handler(attr, v, is_not):
         new_v = v[1:-1].split(";")
+
+        if attr.value_type == ValueTypeEnum.DATE:
+            new_v = ["{} 00:00:00".format(i) for i in new_v if len(i) == 10]
+
         table_name = TableMap(attr=attr).table_name
         in_query = " OR {0}.value ".format(table_name).join(['{0} "{1}"'.format(
             "NOT LIKE" if is_not else "LIKE",
             _v.replace("*", "%")) for _v in new_v])
         _query_sql = QUERY_CI_BY_ATTR_NAME.format(table_name, attr.id, in_query)
+
         return _query_sql
 
     @staticmethod
     def _range_query_handler(attr, v, is_not):
         start, end = [x.strip() for x in v[1:-1].split("_TO_")]
+
+        if attr.value_type == ValueTypeEnum.DATE:
+            start = "{} 00:00:00".format(start) if len(start) == 10 else start
+            end = "{} 00:00:00".format(end) if len(end) == 10 else end
+
         table_name = TableMap(attr=attr).table_name
         range_query = "{0} '{1}' AND '{2}'".format(
             "NOT BETWEEN" if is_not else "BETWEEN",
             start.replace("*", "%"), end.replace("*", "%"))
         _query_sql = QUERY_CI_BY_ATTR_NAME.format(table_name, attr.id, range_query)
+
         return _query_sql
 
     @staticmethod
     def _comparison_query_handler(attr, v):
         table_name = TableMap(attr=attr).table_name
         if v.startswith(">=") or v.startswith("<="):
+            if attr.value_type == ValueTypeEnum.DATE and len(v[2:]) == 10:
+                v = "{} 00:00:00".format(v)
+
             comparison_query = "{0} '{1}'".format(v[:2], v[2:].replace("*", "%"))
         else:
+            if attr.value_type == ValueTypeEnum.DATE and len(v[1:]) == 10:
+                v = "{} 00:00:00".format(v)
+
             comparison_query = "{0} '{1}'".format(v[0], v[1:].replace("*", "%"))
         _query_sql = QUERY_CI_BY_ATTR_NAME.format(table_name, attr.id, comparison_query)
+
         return _query_sql
 
     @staticmethod
@@ -176,6 +250,7 @@ class Search(object):
         elif field.startswith("-"):
             field = field[1:]
             sort_type = "DESC"
+
         return field, sort_type
 
     def __sort_by_id(self, sort_type, query_sql):
@@ -185,7 +260,7 @@ class Search(object):
             return ret_sql.format(query_sql, "ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format(
                 (self.page - 1) * self.count, sort_type, self.count))
 
-        elif self.type_id_list:
+        elif self.type_id_list and not self.multi_type_has_ci_filter:
             self.query_sql = "SELECT B.ci_id FROM ({0}) AS B {1}".format(
                 query_sql,
                 "INNER JOIN c_cis on c_cis.id=B.ci_id WHERE c_cis.type_id IN ({0}) ".format(
@@ -210,7 +285,7 @@ class Search(object):
     def __sort_by_type(self, sort_type, query_sql):
         ret_sql = "SELECT SQL_CALC_FOUND_ROWS DISTINCT B.ci_id FROM ({0}) AS B {1}"
 
-        if self.type_id_list:
+        if self.type_id_list and not self.multi_type_has_ci_filter:
             self.query_sql = "SELECT B.ci_id FROM ({0}) AS B {1}".format(
                 query_sql,
                 "INNER JOIN c_cis on c_cis.id=B.ci_id WHERE c_cis.type_id IN ({0}) ".format(
@@ -234,20 +309,25 @@ class Search(object):
                     (self.page - 1) * self.count, sort_type, self.count))
 
     def __sort_by_field(self, field, sort_type, query_sql):
-        attr = AttributeCache.get(field)
+        if field not in BUILTIN_ATTRIBUTES:
-        attr_id = attr.id
 
-        table_name = TableMap(attr=attr).table_name
+            attr = AttributeCache.get(field)
-        _v_query_sql = """SELECT {0}.ci_id, {1}.value 
+            attr_id = attr.id
-                          FROM ({2}) AS {0} INNER JOIN {1} ON {1}.ci_id = {0}.ci_id
-                          WHERE {1}.attr_id = {3}""".format("ALIAS", table_name, query_sql, attr_id)
-        new_table = _v_query_sql
 
-        if self.only_type_query or not self.type_id_list:
+            table_name = TableMap(attr=attr).table_name
-            return "SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id " \
+            _v_query_sql = """SELECT ALIAS.ci_id, {0}.value
-                   "FROM ({0}) AS C " \
+                              FROM ({1}) AS ALIAS INNER JOIN {0} ON {0}.ci_id = ALIAS.ci_id
-                   "ORDER BY C.value {2} " \
+                              WHERE {0}.attr_id = {2}""".format(table_name, query_sql, attr_id)
-                   "LIMIT {1:d}, {3};".format(new_table, (self.page - 1) * self.count, sort_type, self.count)
+            new_table = _v_query_sql
+        else:
+            _v_query_sql = """SELECT c_cis.id AS ci_id, c_cis.{0} AS value
+                                          FROM c_cis  INNER JOIN ({1}) AS ALIAS ON ALIAS.ci_id = c_cis.id""".format(
+                field[1:], query_sql)
+            new_table = _v_query_sql
+
+        if self.only_type_query or not self.type_id_list or self.multi_type_has_ci_filter:
+            return ("SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id FROM ({0}) AS C ORDER BY C.value {2} "
+                    "LIMIT {1:d}, {3};".format(new_table, (self.page - 1) * self.count, sort_type, self.count))
 
         elif self.type_id_list:
             self.query_sql = """SELECT C.ci_id
@@ -283,10 +363,12 @@ class Search(object):
                            INNER JOIN ({2}) as {3} USING(ci_id)""".format(query_sql, alias, _query_sql, alias + "A")
 
         elif operator == "|" or operator == "|~":
-            query_sql = "SELECT * FROM ({0}) as {1} UNION ALL ({2})".format(query_sql, alias, _query_sql)
+            query_sql = "SELECT * FROM ({0}) as {1} UNION ALL SELECT * FROM ({2}) as {3}".format(query_sql, alias,
+                                                                                                 _query_sql,
+                                                                                                 alias + "A")
 
         elif operator == "~":
-            query_sql = """SELECT * FROM ({0}) as {1} LEFT JOIN ({2}) as {3} USING(ci_id) 
+            query_sql = """SELECT * FROM ({0}) as {1} LEFT JOIN ({2}) as {3} USING(ci_id)
                            WHERE {3}.ci_id is NULL""".format(query_sql, alias, _query_sql, alias + "A")
 
         return query_sql
@@ -296,8 +378,8 @@ class Search(object):
 
         start = time.time()
         execute = db.session.execute
-        current_app.logger.debug(v_query_sql)
+        # current_app.logger.debug(v_query_sql)
-        res = execute(v_query_sql).fetchall()
+        res = execute(text(v_query_sql)).fetchall()
         end_time = time.time()
         current_app.logger.debug("query ci ids time is: {0}".format(end_time - start))
 
@@ -306,6 +388,11 @@ class Search(object):
 
         return numfound, res
 
+    def __get_type2filter_perms(self):
+        res2 = ACLManager('cmdb').get_resources(ResourceTypeEnum.CI_FILTER)
+        if res2:
+            self.type2filter_perms = CIFilterPermsCRUD().get_by_ids(list(map(int, [i['name'] for i in res2])))
+
     def __get_types_has_read(self):
         """
         :return: _type:(type1;type2)
@@ -315,14 +402,23 @@ class Search(object):
 
         self.valid_type_names = {i['name'] for i in res if PermEnum.READ in i['permissions']}
 
-        res2 = acl.get_resources(ResourceTypeEnum.CI_FILTER)
+        self.__get_type2filter_perms()
-        if res2:
+
-            self.type2filter_perms = CIFilterPermsCRUD().get_by_ids(list(map(int, [i['name'] for i in res2])))
+        for type_id in self.type2filter_perms:
+            ci_type = CITypeCache.get(type_id)
+            if ci_type:
+                if self.type2filter_perms[type_id].get('id_filter'):
+                    if self.use_id_filter:
+                        self.valid_type_names.add(ci_type.name)
+                elif self.type2filter_perms[type_id].get('ci_filter'):
+                    if self.use_ci_filter:
+                        self.valid_type_names.add(ci_type.name)
+                else:
+                    self.valid_type_names.add(ci_type.name)
 
         return "_type:({})".format(";".join(self.valid_type_names))
 
     def __confirm_type_first(self, queries):
-
         has_type = False
 
         result = []
@@ -341,11 +437,14 @@ class Search(object):
                 if not q.startswith("("):
                     raise SearchError(ErrFormat.ci_search_Parentheses_invalid)
 
-                operator, q = self._operator_proc(q)
+                if ":" not in q:  # multi-line search
-                if q.endswith(")"):
+                    result.append(q[1:-1].split(';'))
-                    result.append(dict(operator=operator, queries=[q[1:-1]]))
+                else:
+                    operator, q = self._operator_proc(q)
+                    if q.endswith(")"):
+                        result.append(dict(operator=operator, queries=[q[1:-1]]))
 
-                sub = dict(operator=operator, queries=[q[1:]])
+                    sub = dict(operator=operator, queries=[q[1:]])
             elif q.endswith(")") and sub:
                 sub['queries'].append(q[:-1])
                 result.append(copy.deepcopy(sub))
@@ -355,8 +454,10 @@ class Search(object):
             else:
                 result.append(q)
 
-        _is_app_admin = is_app_admin('cmdb') or g.user.username == "worker"
+        if self.parent_node_perm_passed:
-        if result and not has_type and not _is_app_admin:
+            self.__get_type2filter_perms()
+            self.valid_type_names = "ALL"
+        elif result and not has_type and not self.is_app_admin:
             type_q = self.__get_types_has_read()
             if id_query:
                 ci = CIManager.get_by_id(id_query)
@@ -365,23 +466,21 @@ class Search(object):
                 result.insert(0, "_type:{}".format(ci.type_id))
             else:
                 result.insert(0, type_q)
-        elif _is_app_admin:
+        elif self.is_app_admin:
             self.valid_type_names = "ALL"
         else:
             self.__get_types_has_read()
 
-        current_app.logger.warning(result)
-
         return result
 
-    def __query_by_attr(self, q, queries, alias):
+    def __query_by_attr(self, q, queries, alias, is_sub=False):
         k = q.split(":")[0].strip()
         v = "\:".join(q.split(":")[1:]).strip()
         v = v.replace("'", "\\'")
         v = v.replace('"', '\\"')
         field, field_type, operator, attr = self._attr_name_proc(k)
         if field == "_type":
-            _query_sql = self._type_query_handler(v, queries)
+            _query_sql = self._type_query_handler(v, queries, is_sub)
 
         elif field == "_id":
             _query_sql = self._id_query_handler(v)
@@ -392,6 +491,12 @@ class Search(object):
 
             is_not = True if operator == "|~" else False
 
+            if field_type == ValueTypeEnum.DATE and len(v) == 10:
+                v = "{} 00:00:00".format(v)
+
+            if field_type == ValueTypeEnum.BOOL and "*" not in str(v):
+                v = str(int(v in current_app.config.get('BOOL_TRUE')))
+
             # in query
             if v.startswith("(") and v.endswith(")"):
                 _query_sql = self._in_query_handler(attr, v, is_not)
@@ -422,26 +527,36 @@ class Search(object):
 
         return alias, _query_sql, operator
 
-    def __query_build_by_field(self, queries, is_first=True, only_type_query_special=True, alias='A', operator='&'):
+    def __query_build_by_field(self, queries, is_first=True, only_type_query_special=True, alias='A', operator='&',
+                               is_sub=False):
         query_sql = ""
 
         for q in queries:
+            # current_app.logger.debug(q)
             _query_sql = ""
             if isinstance(q, dict):
-                alias, _query_sql, operator = self.__query_build_by_field(q['queries'], True, True, alias)
+                if len(q['queries']) == 1 and ";" in q['queries'][0]:
-                current_app.logger.info(_query_sql)
+                    values = q['queries'][0].split(";")
-                current_app.logger.info((operator, is_first, alias))
+                    in_values = ",".join("'{0}'".format(v) for v in values)
-                operator = q['operator']
+                    _query_sql = QUERY_CI_BY_NO_ATTR_IN.format(in_values, alias)
+                    operator = q['operator']
+                else:
+                    alias, _query_sql, operator = self.__query_build_by_field(q['queries'], True, True, alias,
+                                                                              is_sub=True)
+                    operator = q['operator']
 
             elif ":" in q and not q.startswith("*"):
-                alias, _query_sql, operator = self.__query_by_attr(q, queries, alias)
+                alias, _query_sql, operator = self.__query_by_attr(q, queries, alias, is_sub)
             elif q == "*":
                 continue
             elif q:
-                q = q.replace("'", "\\'")
+                if not isinstance(q, list):
-                q = q.replace('"', '\\"')
+                    q = q.replace("'", "\\'")
-                q = q.replace("*", "%").replace('\\n', '%')
+                    q = q.replace('"', '\\"')
-                _query_sql = QUERY_CI_BY_NO_ATTR.format(q, alias)
+                    q = q.replace("*", "%").replace('\\n', '%')
+                    _query_sql = QUERY_CI_BY_NO_ATTR.format(q, alias)
+                else:
+                    _query_sql = QUERY_CI_BY_NO_ATTR_IN.format(",".join("'{0}'".format(v) for v in q), alias)
 
             if is_first and _query_sql and not self.only_type_query:
                 query_sql = "SELECT * FROM ({0}) AS {1}".format(_query_sql, alias)
@@ -460,7 +575,7 @@ class Search(object):
     def _filter_ids(self, query_sql):
         if self.ci_ids:
             return "SELECT * FROM ({0}) AS IN_QUERY WHERE IN_QUERY.ci_id IN ({1})".format(
-                query_sql, ",".join(list(map(str, self.ci_ids))))
+                query_sql, ",".join(list(set(map(str, self.ci_ids)))))
 
         return query_sql
 
@@ -485,13 +600,15 @@ class Search(object):
         queries = handle_arg_list(self.orig_query)
         queries = self._extra_handle_query_expr(queries)
         queries = self.__confirm_type_first(queries)
-        current_app.logger.debug(queries)
 
         _, query_sql, _ = self.__query_build_by_field(queries)
 
         s = time.time()
         if query_sql:
             query_sql = self._filter_ids(query_sql)
+            if self.raw_ci_ids and not self.ci_ids:
+                return 0, []
+
             self.query_sql = query_sql
             # current_app.logger.debug(query_sql)
             numfound, res = self._execute_sql(query_sql)
@@ -507,30 +624,35 @@ class Search(object):
             if k:
                 table_name = TableMap(attr=attr).table_name
                 query_sql = FACET_QUERY.format(table_name, self.query_sql, attr.id)
-                # current_app.logger.debug(query_sql)
+                result = db.session.execute(text(query_sql)).fetchall()
-                result = db.session.execute(query_sql).fetchall()
                 facet[k] = result
 
         facet_result = dict()
         for k, v in facet.items():
             if not k.startswith('_'):
-                a = getattr(AttributeCache.get(k), self.ret_key)
+                attr = AttributeCache.get(k)
-                facet_result[a] = [(f[0], f[1], a) for f in v]
+                a = getattr(attr, self.ret_key)
+                facet_result[a] = [(ValueTypeMap.serialize[attr.value_type](f[0]), f[1], a) for f in v]
 
         return facet_result
 
     def _fl_build(self):
-        _fl = list()
+        if isinstance(self.fl, list):
-        for f in self.fl:
+            _fl = list()
-            k, _, _, _ = self._attr_name_proc(f)
+            for f in self.fl:
-            if k:
+                k, _, _, _ = self._attr_name_proc(f)
-                _fl.append(k)
+                if k:
+                    _fl.append(k)
 
-        return _fl
+            return _fl
+        else:
+            return self.fl
 
     def search(self):
         numfound, ci_ids = self._query_build_raw()
         ci_ids = list(map(str, ci_ids))
+        if self.only_ids:
+            return ci_ids
 
         _fl = self._fl_build()
 
@@ -543,6 +665,8 @@ class Search(object):
         if ci_ids:
             response = CIManager.get_cis_by_ids(ci_ids, ret_key=self.ret_key, fields=_fl, excludes=self.excludes)
         for res in response:
+            if not res:
+                continue
             ci_type = res.get("ci_type")
             if ci_type not in counter.keys():
                 counter[ci_type] = 0
@@ -550,3 +674,8 @@ class Search(object):
         total = len(response)
 
         return response, counter, total, self.page, numfound, facet
+
+    def get_ci_ids(self):
+        _, ci_ids = self._query_build_raw()
+
+        return ci_ids

cmdb-api/api/lib/cmdb/search/ci/es/search.py

@@ -297,8 +297,8 @@ class Search(object):
             if not attr:
                 raise SearchError(ErrFormat.attribute_not_found.format(field))
 
-            sort_by = "{0}.keyword".format(field) \
+            sort_by = ("{0}.keyword".format(field)
-                if attr.value_type not in (ValueTypeEnum.INT, ValueTypeEnum.FLOAT) else field
+                       if attr.value_type not in (ValueTypeEnum.INT, ValueTypeEnum.FLOAT) else field)
             sorts.append({sort_by: {"order": sort_type}})
 
         self.query.update(dict(sort=sorts))

cmdb-api/api/lib/cmdb/search/ci_relation/search.py

@@ -1,24 +1,40 @@
 # -*- coding:utf-8 -*-
-
-
-import json
 from collections import Counter
+from collections import defaultdict
 
+import copy
+import json
+import networkx as nx
+import sys
 from flask import abort
 from flask import current_app
+from flask_login import current_user
 
 from api.extensions import rd
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.cache import CITypeCache
 from api.lib.cmdb.ci import CIRelationManager
 from api.lib.cmdb.ci_type import CITypeRelationManager
+from api.lib.cmdb.const import ConstraintEnum
+from api.lib.cmdb.const import PermEnum
 from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
+from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION2
+from api.lib.cmdb.const import ResourceTypeEnum
+from api.lib.cmdb.perms import CIFilterPermsCRUD
 from api.lib.cmdb.resp_format import ErrFormat
 from api.lib.cmdb.search.ci.db.search import Search as SearchFromDB
 from api.lib.cmdb.search.ci.es.search import Search as SearchFromES
+from api.lib.cmdb.utils import TableMap
+from api.lib.cmdb.utils import ValueTypeMap
+from api.lib.perm.acl.acl import ACLManager
+from api.lib.perm.acl.acl import is_app_admin
 from api.models.cmdb import CI
+from api.models.cmdb import CITypeRelation
+from api.models.cmdb import RelationType
 
 
 class Search(object):
-    def __init__(self, root_id,
+    def __init__(self, root_id=None,
                  level=None,
                  query=None,
                  fl=None,
@@ -26,7 +42,11 @@ class Search(object):
                  page=1,
                  count=None,
                  sort=None,
-                 reverse=False):
+                 reverse=False,
+                 ancestor_ids=None,
+                 descendant_ids=None,
+                 has_m2m=None,
+                 root_parent_path=None):
         self.orig_query = query
         self.fl = fl
         self.facet_field = facet_field
@@ -35,36 +55,116 @@ class Search(object):
         self.sort = sort or ("ci_id" if current_app.config.get("USE_ES") else None)
 
         self.root_id = root_id
-        self.level = level
+        self.level = level or 0
         self.reverse = reverse
 
-    def _get_ids(self):
+        self.level2constraint = CITypeRelationManager.get_level2constraint(
+            root_id[0] if root_id and isinstance(root_id, list) else root_id,
+            level[0] if isinstance(level, list) and level else level)
+
+        self.ancestor_ids = ancestor_ids
+        self.descendant_ids = descendant_ids
+        self.root_parent_path = root_parent_path or []
+        self.has_m2m = has_m2m or False
+        if not self.has_m2m:
+            if self.ancestor_ids:
+                self.has_m2m = True
+            else:
+                level = level[0] if isinstance(level, list) and level else level
+                for _l, c in self.level2constraint.items():
+                    if _l < int(level) and c == ConstraintEnum.Many2Many:
+                        self.has_m2m = True
+
+        self.type2filter_perms = {}
+
+        self.is_app_admin = is_app_admin('cmdb') or current_user.username == "worker"
+
+    def _get_ids(self, ids):
+
         merge_ids = []
-        ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
+        key = []
+        _tmp = []
         for level in range(1, sorted(self.level)[-1] + 1):
-            _tmp = list(map(lambda x: list(json.loads(x).keys()),
+            if len(self.descendant_ids or []) >= level and self.type2filter_perms.get(self.descendant_ids[level - 1]):
-                            filter(lambda x: x is not None, rd.get(ids, REDIS_PREFIX_CI_RELATION) or [])))
+                id_filter_limit, _ = self._get_ci_filter(self.type2filter_perms[self.descendant_ids[level - 1]])
+            else:
+                id_filter_limit = {}
+
+            if not self.has_m2m:
+                key, prefix = list(map(str, ids)), REDIS_PREFIX_CI_RELATION
+
+            else:
+                if not self.ancestor_ids:
+                    if level == 1:
+                        key, prefix = list(map(str, ids)), REDIS_PREFIX_CI_RELATION
+                    else:
+                        key = list(set(["{},{}".format(i, j) for idx, i in enumerate(key) for j in _tmp[idx]]))
+                        prefix = REDIS_PREFIX_CI_RELATION2
+                else:
+                    if level == 1:
+                        key, prefix = ["{},{}".format(self.ancestor_ids, i) for i in ids], REDIS_PREFIX_CI_RELATION2
+                    else:
+                        key = list(set(["{},{}".format(i, j) for idx, i in enumerate(key) for j in _tmp[idx]]))
+                        prefix = REDIS_PREFIX_CI_RELATION2
+
+            if not key or id_filter_limit is None:
+                return []
+
+            res = [json.loads(x).items() for x in [i or '{}' for i in rd.get(key, prefix) or []]]
+            _tmp = [[i[0] for i in x if (not id_filter_limit or (
+                    key[idx] not in id_filter_limit or int(i[0]) in id_filter_limit[key[idx]]) or
+                                         int(i[0]) in id_filter_limit)] for idx, x in enumerate(res)]
+
             ids = [j for i in _tmp for j in i]
+
             if level in self.level:
                 merge_ids.extend(ids)
 
         return merge_ids
 
-    def _get_reverse_ids(self):
+    def _get_reverse_ids(self, ids):
         merge_ids = []
-        ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
+        level2ids = {}
         for level in range(1, sorted(self.level)[-1] + 1):
-            ids = CIRelationManager.get_ancestor_ids(ids, 1)
+            ids, _level2ids = CIRelationManager.get_ancestor_ids(ids, 1)
+
+            if _level2ids.get(2):
+                level2ids[level + 1] = _level2ids[2]
+
             if level in self.level:
-                merge_ids.extend(ids)
+                if level in level2ids and level2ids[level]:
+                    merge_ids.extend(set(ids) & set(level2ids[level]))
+                else:
+                    merge_ids.extend(ids)
 
         return merge_ids
 
-    def search(self):
+    def _has_read_perm_from_parent_nodes(self):
+        self.root_parent_path = list(map(str, self.root_parent_path))
+        if str(self.root_id).isdigit() and str(self.root_id) not in self.root_parent_path:
+            self.root_parent_path.append(str(self.root_id))
+        self.root_parent_path = set(self.root_parent_path)
+
+        if self.is_app_admin:
+            self.type2filter_perms = {}
+            return True
+
+        res = ACLManager().get_resources(ResourceTypeEnum.CI_FILTER) or {}
+        self.type2filter_perms = CIFilterPermsCRUD().get_by_ids(list(map(int, [i['name'] for i in res]))) or {}
+        for _, filters in self.type2filter_perms.items():
+            if set((filters.get('id_filter') or {}).keys()) & self.root_parent_path:
+                return True
+
+        return True
+
+    def search(self, only_ids=False):
+        use_ci_filter = len(self.descendant_ids or []) == self.level[0] - 1
+        parent_node_perm_passed = not self.is_app_admin and self._has_read_perm_from_parent_nodes()
+
         ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
         cis = [CI.get_by_id(_id) or abort(404, ErrFormat.ci_not_found.format("id={}".format(_id))) for _id in ids]
 
-        merge_ids = self._get_ids() if not self.reverse else self._get_reverse_ids()
+        merge_ids = self._get_ids(ids) if not self.reverse else self._get_reverse_ids(ids)
 
         if not self.orig_query or ("_type:" not in self.orig_query
                                    and "type_id:" not in self.orig_query
@@ -76,11 +176,11 @@ class Search(object):
                         type_ids.extend(CITypeRelationManager.get_child_type_ids(ci.type_id, level))
                     else:
                         type_ids.extend(CITypeRelationManager.get_parent_type_ids(ci.type_id, level))
-            type_ids = list(set(type_ids))
+            type_ids = set(type_ids)
             if self.orig_query:
-                self.orig_query = "_type:({0}),{1}".format(";".join(list(map(str, type_ids))), self.orig_query)
+                self.orig_query = "_type:({0}),{1}".format(";".join(map(str, type_ids)), self.orig_query)
             else:
-                self.orig_query = "_type:({0})".format(";".join(list(map(str, type_ids))))
+                self.orig_query = "_type:({0})".format(";".join(map(str, type_ids)))
 
         if not merge_ids:
             # cis, counter, total, self.page, numfound, facet_
@@ -101,33 +201,142 @@ class Search(object):
                                 page=self.page,
                                 count=self.count,
                                 sort=self.sort,
-                                ci_ids=merge_ids).search()
+                                ci_ids=merge_ids,
+                                parent_node_perm_passed=parent_node_perm_passed,
+                                use_ci_filter=use_ci_filter,
+                                only_ids=only_ids).search()
 
-    def statistics(self, type_ids):
+    def _get_ci_filter(self, filter_perms, ci_filters=None):
-        _tmp = []
+        ci_filters = ci_filters or []
-        ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
+        if ci_filters:
-        for l in range(0, int(self.level)):
+            result = {}
-            if not l:
+            for item in ci_filters:
-                _tmp = list(map(lambda x: list(json.loads(x).items()),
+                res = SearchFromDB('_type:{},{}'.format(item['type_id'], item['ci_filter']),
-                                [i or '{}' for i in rd.get(ids, REDIS_PREFIX_CI_RELATION) or []]))
+                                   count=sys.maxsize, parent_node_perm_passed=True).get_ci_ids()
+                if res:
+                    result[item['type_id']] = set(res)
+
+            return {}, result if result else None
+
+        result = dict()
+        if filter_perms.get('id_filter'):
+            for k in filter_perms['id_filter']:
+                node_path = k.split(',')
+                if len(node_path) == 1:
+                    result[int(node_path[0])] = 1
+                elif not self.has_m2m:
+                    result.setdefault(node_path[-2], set()).add(int(node_path[-1]))
+                else:
+                    result.setdefault(','.join(node_path[:-1]), set()).add(int(node_path[-1]))
+            if result:
+                return result, None
             else:
+                return None, None
+
+        return {}, None
+
+    def statistics(self, type_ids, need_filter=True):
+        self.level = int(self.level)
+
+        acl = ACLManager('cmdb')
+
+        type2filter_perms = dict()
+        if not self.is_app_admin:
+            res2 = acl.get_resources(ResourceTypeEnum.CI_FILTER)
+            if res2:
+                type2filter_perms = CIFilterPermsCRUD().get_by_ids(list(map(int, [i['name'] for i in res2])))
+
+        ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
+        _tmp, tmp_res = [], []
+        level2ids = {}
+        for lv in range(1, self.level + 1):
+            level2ids[lv] = []
+
+            if need_filter:
+                id_filter_limit, ci_filter_limit = None, None
+                if len(self.descendant_ids or []) >= lv and type2filter_perms.get(self.descendant_ids[lv - 1]):
+                    id_filter_limit, _ = self._get_ci_filter(type2filter_perms[self.descendant_ids[lv - 1]])
+                elif type_ids and self.level == lv:
+                    ci_filters = [type2filter_perms[type_id] for type_id in type_ids if type_id in type2filter_perms]
+                    if ci_filters:
+                        id_filter_limit, ci_filter_limit = self._get_ci_filter({}, ci_filters=ci_filters)
+                    else:
+                        id_filter_limit = {}
+                else:
+                    id_filter_limit = {}
+            else:
+                id_filter_limit, ci_filter_limit = {}, {}
+
+            if lv == 1:
+                if not self.has_m2m:
+                    key, prefix = [str(i) for i in ids], REDIS_PREFIX_CI_RELATION
+                else:
+                    key = ["{},{}".format(self.ancestor_ids, _id) for _id in ids]
+                    if not self.ancestor_ids:
+                        key, prefix = [str(i) for i in ids], REDIS_PREFIX_CI_RELATION
+                    else:
+                        prefix = REDIS_PREFIX_CI_RELATION2
+
+                    level2ids[lv] = [[i] for i in key]
+
+                if not key or id_filter_limit is None:
+                    _tmp = [[]] * len(ids)
+                    continue
+
+                res = [json.loads(x).items() for x in [i or '{}' for i in rd.get(key, prefix) or []]]
+                _tmp = []
+                if type_ids and lv == self.level:
+                    _tmp = [[i for i in x if i[1] in type_ids and
+                             (not id_filter_limit or (key[idx] not in id_filter_limit or
+                                                      int(i[0]) in id_filter_limit[key[idx]]) or
+                              int(i[0]) in id_filter_limit)] for idx, x in enumerate(res)]
+                else:
+                    _tmp = [[i for i in x if (not id_filter_limit or (key[idx] not in id_filter_limit or
+                                                                      int(i[0]) in id_filter_limit[key[idx]]) or
+                                              int(i[0]) in id_filter_limit)] for idx, x in enumerate(res)]
+
+                if ci_filter_limit:
+                    _tmp = [[j for j in i if j[1] not in ci_filter_limit or int(j[0]) in ci_filter_limit[j[1]]]
+                            for i in _tmp]
+
+            else:
+
                 for idx, item in enumerate(_tmp):
                     if item:
-                        if type_ids and l == self.level - 1:
+                        if not self.has_m2m:
-                            __tmp = list(
+                            key, prefix = [i[0] for i in item], REDIS_PREFIX_CI_RELATION
-                                map(lambda x: [(_id, type_id) for _id, type_id in json.loads(x).items()
-                                               if type_id in type_ids],
-                                    filter(lambda x: x is not None,
-                                           rd.get([i[0] for i in item], REDIS_PREFIX_CI_RELATION) or [])))
                         else:
+                            key = list(set(['{},{}'.format(j, i[0]) for i in item for j in level2ids[lv - 1][idx]]))
+                            prefix = REDIS_PREFIX_CI_RELATION2
 
-                            __tmp = list(map(lambda x: list(json.loads(x).items()),
+                            level2ids[lv].append(key)
-                                             filter(lambda x: x is not None,
-                                                    rd.get([i[0] for i in item], REDIS_PREFIX_CI_RELATION) or [])))
 
-                        _tmp[idx] = [j for i in __tmp for j in i]
+                        if key:
+                            res = [json.loads(x).items() for x in [i or '{}' for i in rd.get(key, prefix) or []]]
+                            if type_ids and lv == self.level:
+                                tmp_res = [[i for i in x if i[1] in type_ids and
+                                            (not id_filter_limit or (
+                                                    key[idx] not in id_filter_limit or
+                                                    int(i[0]) in id_filter_limit[key[idx]]) or
+                                             int(i[0]) in id_filter_limit)] for idx, x in enumerate(res)]
+                            else:
+                                tmp_res = [[i for i in x if (not id_filter_limit or (
+                                        key[idx] not in id_filter_limit or
+                                        int(i[0]) in id_filter_limit[key[idx]]) or
+                                                             int(i[0]) in id_filter_limit)] for idx, x in
+                                           enumerate(res)]
+
+                            if ci_filter_limit:
+                                tmp_res = [[j for j in i if j[1] not in ci_filter_limit or
+                                            int(j[0]) in ci_filter_limit[j[1]]] for i in tmp_res]
+                        else:
+                            tmp_res = []
+
+                        if tmp_res:
+                            _tmp[idx] = [j for i in tmp_res for j in i]
                     else:
                         _tmp[idx] = []
+                        level2ids[lv].append([])
 
         result = {str(_id): len(_tmp[idx]) for idx, _id in enumerate(ids)}
 
@@ -135,3 +344,251 @@ class Search(object):
             detail={str(_id): dict(Counter([i[1] for i in _tmp[idx]]).items()) for idx, _id in enumerate(ids)})
 
         return result
+
+    def search_full(self, type_ids):
+        def _get_id2name(_type_id):
+            ci_type = CITypeCache.get(_type_id)
+
+            attr = AttributeCache.get(ci_type.unique_id)
+            value_table = TableMap(attr=attr).table
+            serializer = ValueTypeMap.serialize[attr.value_type]
+            unique_value = {i.ci_id: serializer(i.value) for i in value_table.get_by(attr_id=attr.id, to_dict=False)}
+
+            attr = AttributeCache.get(ci_type.show_id)
+            if attr:
+                value_table = TableMap(attr=attr).table
+                serializer = ValueTypeMap.serialize[attr.value_type]
+                show_value = {i.ci_id: serializer(i.value) for i in value_table.get_by(attr_id=attr.id, to_dict=False)}
+            else:
+                show_value = unique_value
+
+            return show_value, unique_value
+
+        self.level = int(self.level)
+
+        acl = ACLManager('cmdb')
+
+        type2filter_perms = dict()
+        if not self.is_app_admin:
+            res2 = acl.get_resources(ResourceTypeEnum.CI_FILTER)
+            if res2:
+                type2filter_perms = CIFilterPermsCRUD().get_by_ids(list(map(int, [i['name'] for i in res2])))
+
+        ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
+
+        level_ids = [str(i) for i in ids]
+        result = []
+        id2children = {}
+        id2name = _get_id2name(type_ids[0])
+        for i in level_ids:
+            item = dict(id=int(i),
+                        type_id=type_ids[0],
+                        isLeaf=False,
+                        title=id2name[0].get(int(i)),
+                        uniqueValue=id2name[1].get(int(i)),
+                        children=[])
+            result.append(item)
+            id2children[str(i)] = item['children']
+
+        for lv in range(1, self.level):
+            type_id = type_ids[lv]
+
+            if len(type_ids or []) >= lv and type2filter_perms.get(type_id):
+                id_filter_limit, _ = self._get_ci_filter(type2filter_perms[type_id])
+            else:
+                id_filter_limit = {}
+
+            if self.has_m2m and lv != 1:
+                key, prefix = [i for i in level_ids], REDIS_PREFIX_CI_RELATION2
+            else:
+                key, prefix = [i.split(',')[-1] for i in level_ids], REDIS_PREFIX_CI_RELATION
+            
+            res = [json.loads(x).items() for x in [i or '{}' for i in rd.get(key, prefix) or []]]
+            res = [[i for i in x if i[1] == type_id and (not id_filter_limit or (key[idx] not in id_filter_limit or
+                                                             int(i[0]) in id_filter_limit[key[idx]]) or
+                                     int(i[0]) in id_filter_limit)] for idx, x in enumerate(res)]
+            _level_ids = []
+            id2name = _get_id2name(type_id)
+            for idx, node_path in enumerate(level_ids):
+                for child_id, _ in (res[idx] or []):
+                    item = dict(id=int(child_id),
+                                type_id=type_id,
+                                isLeaf=True if lv == self.level - 1 else False,
+                                title=id2name[0].get(int(child_id)),
+                                uniqueValue=id2name[1].get(int(child_id)),
+                                children=[])
+                    id2children[node_path].append(item)
+
+                    _node_path = "{},{}".format(node_path, child_id)
+                    _level_ids.append(_node_path)
+                    id2children[_node_path] = item['children']
+
+            level_ids = _level_ids
+
+        return result
+
+    @staticmethod
+    def _get_src_ids(src):
+        q = src.get('q') or ''
+        if not q.startswith('_type:'):
+            q = "_type:{},{}".format(src['type_id'], q)
+
+        return SearchFromDB(q, use_ci_filter=True, only_ids=True, count=100000).search()
+
+    @staticmethod
+    def _filter_target_ids(target_ids, type_ids, q):
+        if not q.startswith('_type:'):
+            q = "_type:({}),{}".format(";".join(map(str, type_ids)), q)
+
+        ci_ids = SearchFromDB(q, ci_ids=target_ids, use_ci_filter=True, only_ids=True, count=100000).search()
+        cis = CI.get_by(fl=['id', 'type_id'], only_query=True).filter(CI.id.in_(ci_ids))
+
+        return [(str(i.id), i.type_id) for i in cis]
+
+    @staticmethod
+    def _path2level(src_type_id, target_type_ids, path):
+        if not src_type_id or not target_type_ids:
+            return abort(400, ErrFormat.relation_path_search_src_target_required)
+
+        graph = nx.DiGraph()
+        graph.add_edges_from([(n, _path[idx + 1]) for _path in path for idx, n in enumerate(_path[:-1])])
+        relation_types = defaultdict(dict)
+        level2type = defaultdict(set)
+        type2show_key = dict()
+        for _path in path:
+            for idx, node in enumerate(_path[1:]):
+                level2type[idx + 1].add(node)
+
+                src = CITypeCache.get(_path[idx])
+                target = CITypeCache.get(node)
+                relation_type = RelationType.get_by(only_query=True).join(
+                    CITypeRelation, CITypeRelation.relation_type_id == RelationType.id).filter(
+                    CITypeRelation.parent_id == src.id).filter(CITypeRelation.child_id == target.id).first()
+                relation_types[src.alias].update({target.alias: relation_type.name})
+
+                if src.id not in type2show_key:
+                    type2show_key[src.id] = AttributeCache.get(src.show_id or src.unique_id).name
+                if target.id not in type2show_key:
+                    type2show_key[target.id] = AttributeCache.get(target.show_id or target.unique_id).name
+
+        nodes = graph.nodes()
+
+        return level2type, list(nodes), relation_types, type2show_key
+
+    def _build_graph(self, source_ids, source_type_id, level2type, target_type_ids, acl):
+        type2filter_perms = dict()
+        if not self.is_app_admin:
+            res2 = acl.get_resources(ResourceTypeEnum.CI_FILTER)
+            if res2:
+                type2filter_perms = CIFilterPermsCRUD().get_by_ids(list(map(int, [i['name'] for i in res2])))
+
+        target_type_ids = set(target_type_ids)
+        graph = nx.DiGraph()
+        target_ids = []
+        key = [(str(i), source_type_id) for i in source_ids]
+        graph.add_nodes_from(key)
+        for level in level2type:
+            filter_type_ids = level2type[level]
+            id_filter_limit = dict()
+            for _type_id in filter_type_ids:
+                if type2filter_perms.get(_type_id):
+                    _id_filter_limit, _ = self._get_ci_filter(type2filter_perms[_type_id])
+                    id_filter_limit.update(_id_filter_limit)
+
+            has_target = filter_type_ids & target_type_ids
+
+            res = [json.loads(x).items() for x in [i or '{}' for i in rd.get([i[0] for i in key],
+                                                                             REDIS_PREFIX_CI_RELATION) or []]]
+            _key = []
+            for idx, _id in enumerate(key):
+                valid_targets = [i for i in res[idx] if i[1] in filter_type_ids and
+                                 (not id_filter_limit or int(i[0]) in id_filter_limit)]
+                _key.extend(valid_targets)
+                graph.add_edges_from(zip([_id] * len(valid_targets), valid_targets))
+
+            if has_target:
+                target_ids.extend([j[0] for i in res for j in i if j[1] in target_type_ids])
+
+            key = copy.deepcopy(_key)
+
+        return graph, target_ids
+
+    @staticmethod
+    def _find_paths(graph, source_ids, source_type_id, target_ids, valid_path, max_depth=6):
+        paths = []
+        for source_id in source_ids:
+            _paths = nx.all_simple_paths(graph,
+                                         source=(source_id, source_type_id),
+                                         target=target_ids,
+                                         cutoff=max_depth)
+            for __path in _paths:
+                if tuple([i[1] for i in __path]) in valid_path:
+                    paths.append([i[0] for i in __path])
+
+        return paths
+
+    @staticmethod
+    def _wrap_path_result(paths, types, valid_path, target_types, type2show_key):
+        ci_ids = [j for i in paths for j in i]
+
+        response, _, _, _, _, _ = SearchFromDB("_type:({})".format(";".join(map(str, types))),
+                                               use_ci_filter=False,
+                                               ci_ids=list(map(int, ci_ids)),
+                                               count=1000000).search()
+        id2ci = {str(i.get('_id')): i if i['_type'] in target_types else {
+            type2show_key[i['_type']]: i[type2show_key[i['_type']]],
+            "ci_type_alias": i["ci_type_alias"],
+            "_type": i["_type"],
+        } for i in response}
+
+        result = defaultdict(list)
+        counter = defaultdict(int)
+
+        for path in paths:
+            key = "-".join([id2ci.get(i, {}).get('ci_type_alias') or '' for i in path])
+            if tuple([id2ci.get(i, {}).get('_type') for i in path]) in valid_path:
+                counter[key] += 1
+                result[key].append(path)
+
+        return result, counter, id2ci
+
+    def search_by_path(self, source, target, path):
+        """
+
+        :param source: {type_id: id, q: expr}
+        :param target: {type_ids: [id], q: expr}
+        :param path: [source_type_id, ..., target_type_id], use type id
+        :return:
+        """
+        acl = ACLManager('cmdb')
+        if not self.is_app_admin:
+            res = {i['name'] for i in acl.get_resources(ResourceTypeEnum.CI_TYPE)}
+            for type_id in (source.get('type_id') and [source['type_id']] or []) + (target.get('type_ids') or []):
+                _type = CITypeCache.get(type_id)
+                if _type and _type.name not in res:
+                    return abort(403, ErrFormat.no_permission.format(_type.alias, PermEnum.READ))
+
+        target['type_ids'] = [i[-1] for i in path]
+        level2type, types, relation_types, type2show_key = self._path2level(
+            source.get('type_id'), target.get('type_ids'), path)
+        if not level2type:
+            return [], {}, 0, self.page, 0, {}, {}
+
+        source_ids = self._get_src_ids(source)
+
+        graph, target_ids = self._build_graph(source_ids, source['type_id'], level2type, target['type_ids'], acl)
+        target_ids = self._filter_target_ids(target_ids, target['type_ids'], target.get('q') or '')
+        paths = self._find_paths(graph,
+                                 source_ids,
+                                 source['type_id'],
+                                 set(target_ids),
+                                 {tuple(i): 1 for i in path})
+
+        numfound = len(paths)
+        paths = paths[(self.page - 1) * self.count:self.page * self.count]
+        response, counter, id2ci = self._wrap_path_result(paths,
+                                                          types,
+                                                          {tuple(i): 1 for i in path},
+                                                          set(target.get('type_ids') or []),
+                                                          type2show_key)
+        return response, counter, len(paths), self.page, numfound, id2ci, relation_types, type2show_key

cmdb-api/api/lib/cmdb/topology.py

@@ -0,0 +1,251 @@
+# -*- coding:utf-8 -*-
+
+import json
+
+from flask import abort
+from flask import current_app
+from flask_login import current_user
+from werkzeug.exceptions import BadRequest
+
+from api.extensions import rd
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.cache import CITypeCache
+from api.lib.cmdb.ci import CIRelationManager
+from api.lib.cmdb.ci_type import CITypeRelationManager
+from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
+from api.lib.cmdb.const import ResourceTypeEnum
+from api.lib.cmdb.resp_format import ErrFormat
+from api.lib.cmdb.search import SearchError
+from api.lib.cmdb.search.ci import search as ci_search
+from api.lib.perm.acl.acl import ACLManager
+from api.lib.perm.acl.acl import is_app_admin
+from api.models.cmdb import TopologyView
+from api.models.cmdb import TopologyViewGroup
+
+
+class TopologyViewManager(object):
+    group_cls = TopologyViewGroup
+    cls = TopologyView
+
+    @classmethod
+    def get_name_by_id(cls, _id):
+        res = cls.cls.get_by_id(_id)
+        return res and res.name
+
+    def get_view_by_id(self, _id):
+        res = self.cls.get_by_id(_id)
+
+        return res and res.to_dict() or {}
+
+    @classmethod
+    def add_group(cls, name, order):
+        if order is None:
+            cur_max_order = cls.group_cls.get_by(only_query=True).order_by(cls.group_cls.order.desc()).first()
+            cur_max_order = cur_max_order and cur_max_order.order or 0
+            order = cur_max_order + 1
+
+        cls.group_cls.get_by(name=name, first=True, to_dict=False) and abort(
+            400, ErrFormat.topology_group_exists.format(name))
+
+        return cls.group_cls.create(name=name, order=order)
+
+    def update_group(self, group_id, name, view_ids):
+        existed = self.group_cls.get_by_id(group_id) or abort(404, ErrFormat.not_found)
+        if name is not None and name != existed.name:
+            existed.update(name=name)
+
+        for idx, view_id in enumerate(view_ids):
+            view = self.cls.get_by_id(view_id)
+            if view is not None:
+                view.update(group_id=group_id, order=idx)
+
+        return existed.to_dict()
+
+    @classmethod
+    def delete_group(cls, _id):
+        existed = cls.group_cls.get_by_id(_id) or abort(404, ErrFormat.not_found)
+
+        if cls.cls.get_by(group_id=_id, first=True):
+            return abort(400, ErrFormat.topo_view_exists_cannot_delete_group)
+
+        existed.soft_delete()
+
+    @classmethod
+    def group_order(cls, group_ids):
+        for idx, group_id in enumerate(group_ids):
+            group = cls.group_cls.get_by_id(group_id)
+            group.update(order=idx + 1)
+
+    @classmethod
+    def add(cls, name, group_id, option, order=None, **kwargs):
+        cls.cls.get_by(name=name, first=True) and abort(400, ErrFormat.topology_exists.format(name))
+        if order is None:
+            cur_max_order = cls.cls.get_by(group_id=group_id, only_query=True).order_by(
+                cls.cls.order.desc()).first()
+            cur_max_order = cur_max_order and cur_max_order.order or 0
+            order = cur_max_order + 1
+
+        inst = cls.cls.create(name=name, group_id=group_id, option=option, order=order, **kwargs).to_dict()
+        if current_app.config.get('USE_ACL'):
+            try:
+                ACLManager().add_resource(name, ResourceTypeEnum.TOPOLOGY_VIEW)
+            except BadRequest:
+                pass
+
+            ACLManager().grant_resource_to_role(name,
+                                                current_user.username,
+                                                ResourceTypeEnum.TOPOLOGY_VIEW)
+
+        return inst
+
+    @classmethod
+    def update(cls, _id, **kwargs):
+        existed = cls.cls.get_by_id(_id) or abort(404, ErrFormat.not_found)
+        existed_name = existed.name
+
+        inst = existed.update(filter_none=False, **kwargs).to_dict()
+        if current_app.config.get('USE_ACL') and existed_name != kwargs.get('name') and kwargs.get('name'):
+            try:
+                ACLManager().update_resource(existed_name, kwargs['name'], ResourceTypeEnum.TOPOLOGY_VIEW)
+            except BadRequest:
+                pass
+
+        return inst
+
+    @classmethod
+    def delete(cls, _id):
+        existed = cls.cls.get_by_id(_id) or abort(404, ErrFormat.not_found)
+
+        existed.soft_delete()
+        if current_app.config.get("USE_ACL"):
+            ACLManager().del_resource(existed.name, ResourceTypeEnum.TOPOLOGY_VIEW)
+
+    @classmethod
+    def group_inner_order(cls, _ids):
+        for idx, _id in enumerate(_ids):
+            topology = cls.cls.get_by_id(_id)
+            topology.update(order=idx + 1)
+
+    @classmethod
+    def get_all(cls):
+        resources = None
+        if current_app.config.get('USE_ACL') and not is_app_admin('cmdb'):
+            resources = set([i.get('name') for i in ACLManager().get_resources(ResourceTypeEnum.TOPOLOGY_VIEW)])
+
+        groups = cls.group_cls.get_by(to_dict=True)
+        groups = sorted(groups, key=lambda x: x['order'])
+        group2pos = {group['id']: idx for idx, group in enumerate(groups)}
+
+        topo_views = sorted(cls.cls.get_by(to_dict=True), key=lambda x: x['order'])
+        other_group = dict(views=[])
+        for view in topo_views:
+            if resources is not None and view['name'] not in resources:
+                continue
+
+            if view['group_id']:
+                groups[group2pos[view['group_id']]].setdefault('views', []).append(view)
+            else:
+                other_group['views'].append(view)
+
+        if other_group['views']:
+            groups.append(other_group)
+
+        return groups
+
+    @staticmethod
+    def relation_from_ci_type(type_id):
+        nodes, edges = CITypeRelationManager.get_relations_by_type_id(type_id)
+
+        return dict(nodes=nodes, edges=edges)
+
+    def topology_view(self, view_id=None, preview=None):
+        if view_id is not None:
+            view = self.cls.get_by_id(view_id) or abort(404, ErrFormat.not_found)
+            central_node_type, central_node_instances, path = (view.central_node_type,
+                                                               view.central_node_instances, view.path)
+        else:
+            central_node_type = preview.get('central_node_type')
+            central_node_instances = preview.get('central_node_instances')
+            path = preview.get('path')
+
+        nodes, links = [], []
+        _type = CITypeCache.get(central_node_type)
+        if not _type:
+            return dict(nodes=nodes, links=links)
+        type2meta = {_type.id: _type.icon}
+        root_ids = []
+        show_key = AttributeCache.get(_type.show_id or _type.unique_id)
+
+        q = (central_node_instances[2:] if central_node_instances.startswith('q=') else
+             central_node_instances)
+        s = ci_search(q, fl=['_id', show_key.name], use_id_filter=False, use_ci_filter=False, count=1000000)
+        try:
+            response, _, _, _, _, _ = s.search()
+        except SearchError as e:
+            current_app.logger.info(e)
+            return dict(nodes=nodes, links=links)
+        for i in response:
+            root_ids.append(i['_id'])
+            nodes.append(dict(id=str(i['_id']), name=i[show_key.name], type_id=central_node_type))
+        if not root_ids:
+            return dict(nodes=nodes, links=links)
+
+        prefix = REDIS_PREFIX_CI_RELATION
+        key = list(map(str, root_ids))
+        id2node = {}
+        for level in sorted([i for i in path.keys() if int(i) > 0]):
+            type_ids = {int(i) for i in path[level]}
+
+            res = [json.loads(x).items() for x in [i or '{}' for i in rd.get(key, prefix) or []]]
+            new_key = []
+            for idx, from_id in enumerate(key):
+                for to_id, type_id in res[idx]:
+                    if type_id in type_ids:
+                        links.append({'from': from_id, 'to': to_id})
+                        id2node[to_id] = {'id': to_id, 'type_id': type_id}
+                        new_key.append(to_id)
+                        if type_id not in type2meta:
+                            type2meta[type_id] = CITypeCache.get(type_id).icon
+
+            key = new_key
+
+        ci_ids = list(map(int, root_ids))
+        for level in sorted([i for i in path.keys() if int(i) < 0]):
+            type_ids = {int(i) for i in path[level]}
+            res = CIRelationManager.get_parent_ids(ci_ids)
+            _ci_ids = []
+            for to_id in res:
+                for from_id, type_id in res[to_id]:
+                    if type_id in type_ids:
+                        from_id, to_id = str(from_id), str(to_id)
+                        links.append({'from': from_id, 'to': to_id})
+                        id2node[from_id] = {'id': str(from_id), 'type_id': type_id}
+                        _ci_ids.append(from_id)
+                        if type_id not in type2meta:
+                            type2meta[type_id] = CITypeCache.get(type_id).icon
+
+            ci_ids = _ci_ids
+
+        fl = set()
+        type_ids = {t for lv in path if lv != '0' for t in path[lv]}
+        type2show = {}
+        for type_id in type_ids:
+            ci_type = CITypeCache.get(type_id)
+            if ci_type:
+                attr = AttributeCache.get(ci_type.show_id or ci_type.unique_id)
+                if attr:
+                    fl.add(attr.name)
+                    type2show[type_id] = attr.name
+
+        if id2node:
+            s = ci_search("_id:({})".format(';'.join(id2node.keys())), fl=list(fl),
+                       use_id_filter=False, use_ci_filter=False, count=1000000)
+            try:
+                response, _, _, _, _, _ = s.search()
+            except SearchError:
+                return dict(nodes=nodes, links=links)
+            for i in response:
+                id2node[str(i['_id'])]['name'] = i[type2show[str(i['_type'])]]
+            nodes.extend(id2node.values())
+
+        return dict(nodes=nodes, links=links, type2meta=type2meta)

cmdb-api/api/lib/cmdb/utils.py

@@ -4,37 +4,67 @@ from __future__ import unicode_literals
 
 import datetime
 import json
+import re
 
 import six
-from markupsafe import escape
+from flask import current_app
 
 import api.models.cmdb as model
 from api.lib.cmdb.cache import AttributeCache
 from api.lib.cmdb.const import ValueTypeEnum
+from api.lib.cmdb.resp_format import ErrFormat
+
+TIME_RE = re.compile(r'(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d')
+
+
+class ValueDeserializeError(Exception):
+    pass
 
 
 def string2int(x):
-    return int(float(x))
+    v = int(float(x))
+    if v > 2147483647:
+        raise ValueDeserializeError(ErrFormat.attribute_value_out_of_range)
+
+    return v
 
 
-def str2datetime(x):
+def str2date(x):
+
     try:
-        return datetime.datetime.strptime(x, "%Y-%m-%d")
+        return datetime.datetime.strptime(x, "%Y-%m-%d").date()
     except ValueError:
         pass
 
-    return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")
+    try:
+        return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S").date()
+    except ValueError:
+        pass
+
+
+def str2datetime(x):
+
+    x = x.replace('T', ' ')
+    x = x.replace('Z', '')
+
+    try:
+        return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")
+    except ValueError:
+        pass
+
+    return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M")
 
 
 class ValueTypeMap(object):
     deserialize = {
         ValueTypeEnum.INT: string2int,
         ValueTypeEnum.FLOAT: float,
-        ValueTypeEnum.TEXT: lambda x: escape(x).encode('utf-8').decode('utf-8'),
+        ValueTypeEnum.TEXT: lambda x: x,
-        ValueTypeEnum.TIME: lambda x: escape(x).encode('utf-8').decode('utf-8'),
+        ValueTypeEnum.TIME: lambda x: TIME_RE.findall(x)[0],
         ValueTypeEnum.DATETIME: str2datetime,
-        ValueTypeEnum.DATE: str2datetime,
+        ValueTypeEnum.DATE: str2date,
         ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x,
+        ValueTypeEnum.BOOL: lambda x: x in current_app.config.get('BOOL_TRUE'),
     }
 
     serialize = {
@@ -42,9 +72,10 @@ class ValueTypeMap(object):
         ValueTypeEnum.FLOAT: float,
         ValueTypeEnum.TEXT: lambda x: x if isinstance(x, six.string_types) else str(x),
         ValueTypeEnum.TIME: lambda x: x if isinstance(x, six.string_types) else str(x),
-        ValueTypeEnum.DATE: lambda x: x.strftime("%Y-%m-%d"),
+        ValueTypeEnum.DATE: lambda x: x.strftime("%Y-%m-%d") if not isinstance(x, six.string_types) else x,
-        ValueTypeEnum.DATETIME: lambda x: x.strftime("%Y-%m-%d %H:%M:%S"),
+        ValueTypeEnum.DATETIME: lambda x: x.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(x, six.string_types) else x,
         ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x,
+        ValueTypeEnum.BOOL: lambda x: x in current_app.config.get('BOOL_TRUE'),
     }
 
     serialize2 = {
@@ -55,21 +86,20 @@ class ValueTypeMap(object):
         ValueTypeEnum.DATE: lambda x: (x.decode() if not isinstance(x, six.string_types) else x).split()[0],
         ValueTypeEnum.DATETIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
         ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x,
+        ValueTypeEnum.BOOL: lambda x: x in current_app.config.get('BOOL_TRUE'),
     }
 
     choice = {
         ValueTypeEnum.INT: model.IntegerChoice,
         ValueTypeEnum.FLOAT: model.FloatChoice,
         ValueTypeEnum.TEXT: model.TextChoice,
+        ValueTypeEnum.TIME: model.TextChoice,
+        ValueTypeEnum.DATE: model.TextChoice,
+        ValueTypeEnum.DATETIME: model.TextChoice,
     }
 
     table = {
-        ValueTypeEnum.INT: model.CIValueInteger,
         ValueTypeEnum.TEXT: model.CIValueText,
-        ValueTypeEnum.DATETIME: model.CIValueDateTime,
-        ValueTypeEnum.DATE: model.CIValueDateTime,
-        ValueTypeEnum.TIME: model.CIValueText,
-        ValueTypeEnum.FLOAT: model.CIValueFloat,
         ValueTypeEnum.JSON: model.CIValueJson,
         'index_{0}'.format(ValueTypeEnum.INT): model.CIIndexValueInteger,
         'index_{0}'.format(ValueTypeEnum.TEXT): model.CIIndexValueText,
@@ -78,15 +108,11 @@ class ValueTypeMap(object):
         'index_{0}'.format(ValueTypeEnum.TIME): model.CIIndexValueText,
         'index_{0}'.format(ValueTypeEnum.FLOAT): model.CIIndexValueFloat,
         'index_{0}'.format(ValueTypeEnum.JSON): model.CIValueJson,
+        'index_{0}'.format(ValueTypeEnum.BOOL): model.CIIndexValueInteger,
     }
 
     table_name = {
-        ValueTypeEnum.INT: 'c_value_integers',
         ValueTypeEnum.TEXT: 'c_value_texts',
-        ValueTypeEnum.DATETIME: 'c_value_datetime',
-        ValueTypeEnum.DATE: 'c_value_datetime',
-        ValueTypeEnum.TIME: 'c_value_texts',
-        ValueTypeEnum.FLOAT: 'c_value_floats',
         ValueTypeEnum.JSON: 'c_value_json',
         'index_{0}'.format(ValueTypeEnum.INT): 'c_value_index_integers',
         'index_{0}'.format(ValueTypeEnum.TEXT): 'c_value_index_texts',
@@ -95,6 +121,7 @@ class ValueTypeMap(object):
         'index_{0}'.format(ValueTypeEnum.TIME): 'c_value_index_texts',
         'index_{0}'.format(ValueTypeEnum.FLOAT): 'c_value_index_floats',
         'index_{0}'.format(ValueTypeEnum.JSON): 'c_value_json',
+        'index_{0}'.format(ValueTypeEnum.BOOL): 'c_value_index_integers',
     }
 
     es_type = {
@@ -104,7 +131,7 @@ class ValueTypeMap(object):
         ValueTypeEnum.DATE: 'text',
         ValueTypeEnum.TIME: 'text',
         ValueTypeEnum.FLOAT: 'float',
-        ValueTypeEnum.JSON: 'object'
+        ValueTypeEnum.JSON: 'object',
     }
 
 
@@ -117,8 +144,13 @@ class TableMap(object):
     @property
     def table(self):
         attr = AttributeCache.get(self.attr_name) if not self.attr else self.attr
-        if self.is_index is None:
+        if attr.is_password or attr.is_link:
+            self.is_index = False
+        elif attr.value_type not in {ValueTypeEnum.TEXT, ValueTypeEnum.JSON}:
+            self.is_index = True
+        elif self.is_index is None:
             self.is_index = attr.is_index
+
         i = "index_{0}".format(attr.value_type) if self.is_index else attr.value_type
 
         return ValueTypeMap.table.get(i)
@@ -126,8 +158,13 @@ class TableMap(object):
     @property
     def table_name(self):
         attr = AttributeCache.get(self.attr_name) if not self.attr else self.attr
-        if self.is_index is None:
+        if attr.is_password or attr.is_link:
+            self.is_index = False
+        elif attr.value_type not in {ValueTypeEnum.TEXT, ValueTypeEnum.JSON}:
+            self.is_index = True
+        elif self.is_index is None:
             self.is_index = attr.is_index
+
         i = "index_{0}".format(attr.value_type) if self.is_index else attr.value_type
 
         return ValueTypeMap.table_name.get(i)

cmdb-api/api/lib/cmdb/value.py

@@ -3,27 +3,29 @@
 
 from __future__ import unicode_literals
 
-import copy
+import importlib.util
-import imp
-import os
-import tempfile
 
+import copy
 import jinja2
+import os
+import re
+import tempfile
 from flask import abort
 from flask import current_app
 from jinja2schema import infer
 from jinja2schema import to_json_schema
+from werkzeug.exceptions import BadRequest
 
 from api.extensions import db
 from api.lib.cmdb.attribute import AttributeManager
 from api.lib.cmdb.cache import AttributeCache
 from api.lib.cmdb.cache import CITypeAttributeCache
-from api.lib.cmdb.const import ExistPolicy
 from api.lib.cmdb.const import OperateType
 from api.lib.cmdb.const import ValueTypeEnum
 from api.lib.cmdb.history import AttributeHistoryManger
 from api.lib.cmdb.resp_format import ErrFormat
 from api.lib.cmdb.utils import TableMap
+from api.lib.cmdb.utils import ValueDeserializeError
 from api.lib.cmdb.utils import ValueTypeMap
 from api.lib.utils import handle_arg_list
 from api.models.cmdb import CI
@@ -45,7 +47,7 @@ class AttributeValueManager(object):
         """
         return AttributeCache.get(key)
 
-    def get_attr_values(self, fields, ci_id, ret_key="name", unique_key=None, use_master=False):
+    def get_attr_values(self, fields, ci_id, ret_key="name", unique_key=None, use_master=False, enum_map=None):
         """
 
         :param fields:
@@ -53,6 +55,7 @@ class AttributeValueManager(object):
         :param ret_key: It can be name or alias
         :param unique_key: primary attribute
         :param use_master: Only for master-slave read-write separation
+        :param enum_map:
         :return:
         """
         res = dict()
@@ -67,12 +70,19 @@ class AttributeValueManager(object):
                                     use_master=use_master,
                                     to_dict=False)
             field_name = getattr(attr, ret_key)
-
             if attr.is_list:
                 res[field_name] = [ValueTypeMap.serialize[attr.value_type](i.value) for i in rs]
+            elif attr.is_password and rs:
+                res[field_name] = '******' if rs[0].value else ''
             else:
                 res[field_name] = ValueTypeMap.serialize[attr.value_type](rs[0].value) if rs else None
 
+            if enum_map and field_name in enum_map:
+                if attr.is_list:
+                    res[field_name] = [enum_map[field_name].get(i, i) for i in res[field_name]]
+                else:
+                    res[field_name] = enum_map[field_name].get(res[field_name], res[field_name])
+
             if unique_key is not None and attr.id == unique_key.id and rs:
                 res['unique'] = unique_key.name
                 res['unique_alias'] = unique_key.alias
@@ -80,24 +90,34 @@ class AttributeValueManager(object):
         return res
 
     @staticmethod
-    def __deserialize_value(value_type, value):
+    def _deserialize_value(alias, value_type, value):
         if not value:
             return value
+
         deserialize = ValueTypeMap.deserialize[value_type]
         try:
             v = deserialize(value)
+            if value_type in (ValueTypeEnum.DATE, ValueTypeEnum.DATETIME):
+                return str(v)
             return v
+        except ValueDeserializeError as e:
+            return abort(400, ErrFormat.attribute_value_invalid2.format(alias, e))
         except ValueError:
-            return abort(400, ErrFormat.attribute_value_invalid.format(value))
+            return abort(400, ErrFormat.attribute_value_invalid2.format(alias, value))
 
     @staticmethod
-    def __check_is_choice(attr, value_type, value):
+    def _check_is_choice(attr, value_type, value):
-        choice_values = AttributeManager.get_choice_values(attr.id, value_type, attr.choice_web_hook)
+        choice_values = AttributeManager.get_choice_values(attr.id, value_type, attr.choice_web_hook, attr.choice_other)
-        if str(value) not in list(map(str, [i[0] for i in choice_values])):
+        if value_type == ValueTypeEnum.FLOAT:
-            return abort(400, ErrFormat.not_in_choice_values.format(value))
+            if float(value) not in list(map(float, [i[0] for i in choice_values])):
+                return abort(400, ErrFormat.not_in_choice_values.format(value))
+
+        else:
+            if str(value) not in list(map(str, [i[0] for i in choice_values])):
+                return abort(400, ErrFormat.not_in_choice_values.format(value))
 
     @staticmethod
-    def __check_is_unique(value_table, attr, ci_id, type_id, value):
+    def _check_is_unique(value_table, attr, ci_id, type_id, value):
         existed = db.session.query(value_table.attr_id).join(CI, CI.id == value_table.ci_id).filter(
             CI.type_id == type_id).filter(
             value_table.attr_id == attr.id).filter(value_table.deleted.is_(False)).filter(
@@ -106,24 +126,38 @@ class AttributeValueManager(object):
         existed and abort(400, ErrFormat.attribute_value_unique_required.format(attr.alias, value))
 
     @staticmethod
-    def __check_is_required(type_id, attr, value, type_attr=None):
+    def _check_is_required(type_id, attr, value, type_attr=None):
         type_attr = type_attr or CITypeAttributeCache.get(type_id, attr.id)
         if type_attr and type_attr.is_required and not value and value != 0:
             return abort(400, ErrFormat.attribute_value_required.format(attr.alias))
 
-    def _validate(self, attr, value, value_table, ci=None, type_id=None, ci_id=None, type_attr=None):
+    @staticmethod
-        ci = ci or {}
+    def check_re(expr, alias, value):
-        v = self.__deserialize_value(attr.value_type, value)
+        if not re.compile(expr).match(str(value)):
+            return abort(400, ErrFormat.attribute_value_invalid2.format(alias, value))
 
-        attr.is_choice and value and self.__check_is_choice(attr, attr.value_type, v)
+    def _validate(self, attr, value, value_table, ci=None, type_id=None, ci_id=None, type_attr=None, unique_name=None):
-        attr.is_unique and self.__check_is_unique(
+        if not attr.is_reference:
+            ci = ci or {}
+            v = self._deserialize_value(attr.alias, attr.value_type, value)
+
+            attr.is_choice and value and self._check_is_choice(attr, attr.value_type, v)
+
+        else:
+            v = value or None
+
+        (attr.is_unique or attr.name == unique_name) and self._check_is_unique(
             value_table, attr, ci and ci.id or ci_id, ci and ci.type_id or type_id, v)
-
+        self._check_is_required(ci and ci.type_id or type_id, attr, v, type_attr=type_attr)
-        self.__check_is_required(ci and ci.type_id or type_id, attr, v, type_attr=type_attr)
+        if attr.is_reference:
+            return v
 
         if v == "" and attr.value_type not in (ValueTypeEnum.TEXT,):
             v = None
 
+        if attr.re_check and value:
+            self.check_re(attr.re_check, attr.alias, value)
+
         return v
 
     @staticmethod
@@ -131,20 +165,21 @@ class AttributeValueManager(object):
         return AttributeHistoryManger.add(record_id, ci_id, [(attr_id, operate_type, old, new)], type_id)
 
     @staticmethod
-    def _write_change2(changed):
+    def write_change2(changed, record_id=None, ticket_id=None):
-        record_id = None
         for ci_id, attr_id, operate_type, old, new, type_id in changed:
             record_id = AttributeHistoryManger.add(record_id, ci_id, [(attr_id, operate_type, old, new)], type_id,
+                                                   ticket_id=ticket_id,
                                                    commit=False, flush=False)
         try:
             db.session.commit()
         except Exception as e:
+            db.session.rollback()
             current_app.logger.error("write change failed: {}".format(str(e)))
 
         return record_id
 
     @staticmethod
-    def __compute_attr_value_from_expr(expr, ci_dict):
+    def _compute_attr_value_from_expr(expr, ci_dict):
         t = jinja2.Template(expr).render(ci_dict)
 
         try:
@@ -154,7 +189,7 @@ class AttributeValueManager(object):
             return t
 
     @staticmethod
-    def __compute_attr_value_from_script(script, ci_dict):
+    def _compute_attr_value_from_script(script, ci_dict):
         script = jinja2.Template(script).render(ci_dict)
 
         script_f = tempfile.NamedTemporaryFile(delete=False, suffix=".py")
@@ -163,11 +198,11 @@ class AttributeValueManager(object):
 
         try:
             path = script_f.name
-            dir_name, name = os.path.dirname(path), os.path.basename(path)[:-3]
+            name = os.path.basename(path)[:-3]
 
-            fp, path, desc = imp.find_module(name, [dir_name])
+            spec = importlib.util.spec_from_file_location(name, path)
-
+            mod = importlib.util.module_from_spec(spec)
-            mod = imp.load_module(name, fp, path, desc)
+            spec.loader.exec_module(mod)
 
             if hasattr(mod, 'computed'):
                 return mod.computed()
@@ -183,26 +218,29 @@ class AttributeValueManager(object):
 
         return [var for var in schema.get("properties")]
 
-    def _compute_attr_value(self, attr, payload, ci):
+    def _compute_attr_value(self, attr, payload, ci_id):
-        attrs = self._jinja2_parse(attr['compute_expr']) if attr.get('compute_expr') else \
+        attrs = (self._jinja2_parse(attr['compute_expr']) if attr.get('compute_expr')
-            self._jinja2_parse(attr['compute_script'])
+                 else self._jinja2_parse(attr['compute_script']))
         not_existed = [i for i in attrs if i not in payload]
-        if ci is not None:
+        if ci_id is not None:
-            payload.update(self.get_attr_values(not_existed, ci.id))
+            payload.update(self.get_attr_values(not_existed, ci_id))
 
         if attr['compute_expr']:
-            return self.__compute_attr_value_from_expr(attr['compute_expr'], payload)
+            return self._compute_attr_value_from_expr(attr['compute_expr'], payload)
         elif attr['compute_script']:
-            return self.__compute_attr_value_from_script(attr['compute_script'], payload)
+            return self._compute_attr_value_from_script(attr['compute_script'], payload)
 
     def handle_ci_compute_attributes(self, ci_dict, computed_attrs, ci):
         payload = copy.deepcopy(ci_dict)
         for attr in computed_attrs:
-            computed_value = self._compute_attr_value(attr, payload, ci)
+            computed_value = self._compute_attr_value(attr, payload, ci and ci.id)
             if computed_value is not None:
                 ci_dict[attr['name']] = computed_value
 
-    def valid_attr_value(self, ci_dict, type_id, ci_id, name2attr, alias2attr=None, ci_attr2type_attr=None):
+    def valid_attr_value(self, ci_dict, type_id, ci_id, name2attr,
+                         alias2attr=None,
+                         ci_attr2type_attr=None,
+                         unique_name=None):
         key2attr = dict()
         alias2attr = alias2attr or {}
         ci_attr2type_attr = ci_attr2type_attr or {}
@@ -215,17 +253,29 @@ class AttributeValueManager(object):
 
             try:
                 if attr.is_list:
+                    if isinstance(value, dict):
+                        if value.get('op') == "delete":
+                            value['v'] = [ValueTypeMap.serialize[attr.value_type](
+                                self._deserialize_value(attr.alias, attr.value_type, i))
+                                for i in handle_arg_list(value['v'])]
+                            continue
+                        _value = value.get('v') or []
+                    else:
+                        _value = value
                     value_list = [self._validate(attr, i, value_table, ci=None, type_id=type_id, ci_id=ci_id,
                                                  type_attr=ci_attr2type_attr.get(attr.id))
-                                  for i in handle_arg_list(value)]
+                                  for i in handle_arg_list(_value)]
-                    ci_dict[key] = value_list
+                    ci_dict[key] = value_list if not isinstance(value, dict) else dict(op=value.get('op'), v=value_list)
                     if not value_list:
-                        self.__check_is_required(type_id, attr, '')
+                        self._check_is_required(type_id, attr, '')
 
                 else:
                     value = self._validate(attr, value, value_table, ci=None, type_id=type_id, ci_id=ci_id,
-                                           type_attr=ci_attr2type_attr.get(attr.id))
+                                           type_attr=ci_attr2type_attr.get(attr.id),
+                                           unique_name=unique_name)
                     ci_dict[key] = value
+            except BadRequest as e:
+                raise
             except Exception as e:
                 current_app.logger.warning(str(e))
 
@@ -234,15 +284,17 @@ class AttributeValueManager(object):
 
         return key2attr
 
-    def create_or_update_attr_value2(self, ci, ci_dict, key2attr):
+    def create_or_update_attr_value(self, ci, ci_dict, key2attr, ticket_id=None):
         """
         add or update attribute value, then write history
         :param ci: instance object
         :param ci_dict: attribute dict
         :param key2attr: attr key to attr
+        :param ticket_id:
         :return:
         """
         changed = []
+        has_dynamic = False
         for key, value in ci_dict.items():
             attr = key2attr.get(key)
             if not attr:
@@ -251,106 +303,90 @@ class AttributeValueManager(object):
 
             if attr.is_list:
                 existed_attrs = value_table.get_by(attr_id=attr.id, ci_id=ci.id, to_dict=False)
-                existed_values = [i.value for i in existed_attrs]
+                existed_values = [(ValueTypeMap.serialize[attr.value_type](i.value) if
-                added = set(value) - set(existed_values)
+                                   i.value or i.value == 0 else i.value) for i in existed_attrs]
-                deleted = set(existed_values) - set(value)
-                for v in added:
-                    value_table.create(ci_id=ci.id, attr_id=attr.id, value=v, flush=False, commit=False)
-                    changed.append((ci.id, attr.id, OperateType.ADD, None, v, ci.type_id))
 
-                for v in deleted:
+                if isinstance(value, dict):
-                    existed_attr = existed_attrs[existed_values.index(v)]
+                    if value.get('op') == "add":
-                    existed_attr.delete(flush=False, commit=False)
+                        for v in (value.get('v') or []):
-                    changed.append((ci.id, attr.id, OperateType.DELETE, v, None, ci.type_id))
+                            if v not in existed_values:
+                                value_table.create(ci_id=ci.id, attr_id=attr.id, value=v, flush=False, commit=False)
+                                if not attr.is_dynamic:
+                                    changed.append((ci.id, attr.id, OperateType.ADD, None, v, ci.type_id))
+                                else:
+                                    has_dynamic = True
+
+                    elif value.get('op') == "delete":
+                        for v in (value.get('v') or []):
+                            if v in existed_values:
+                                existed_attrs[existed_values.index(v)].delete(flush=False, commit=False)
+                                if not attr.is_dynamic:
+                                    changed.append((ci.id, attr.id, OperateType.DELETE, v, None, ci.type_id))
+                                else:
+                                    has_dynamic = True
+                else:
+                    # Comparison array starts from which position changes
+                    min_len = min(len(value), len(existed_values))
+                    index = 0
+                    while index < min_len:
+                        if value[index] != existed_values[index]:
+                            break
+                        index += 1
+
+                    # Delete first and then add to ensure id sorting
+                    for idx in range(index, len(existed_attrs)):
+                        existed_attr = existed_attrs[idx]
+                        existed_attr.delete(flush=False, commit=False)
+                        if not attr.is_dynamic:
+                            changed.append((ci.id, attr.id, OperateType.DELETE, existed_values[idx], None, ci.type_id))
+                        else:
+                            has_dynamic = True
+                    for idx in range(index, len(value)):
+                        value_table.create(ci_id=ci.id, attr_id=attr.id, value=value[idx], flush=False, commit=False)
+                        if not attr.is_dynamic:
+                            changed.append((ci.id, attr.id, OperateType.ADD, None, value[idx], ci.type_id))
+                        else:
+                            has_dynamic = True
             else:
                 existed_attr = value_table.get_by(attr_id=attr.id, ci_id=ci.id, first=True, to_dict=False)
                 existed_value = existed_attr and existed_attr.value
+                existed_value = (ValueTypeMap.serialize[attr.value_type](existed_value) if
+                                 existed_value or existed_value == 0 else existed_value)
                 if existed_value is None and value is not None:
                     value_table.create(ci_id=ci.id, attr_id=attr.id, value=value, flush=False, commit=False)
 
-                    changed.append((ci.id, attr.id, OperateType.ADD, None, value, ci.type_id))
+                    if not attr.is_dynamic:
+                        changed.append((ci.id, attr.id, OperateType.ADD, None, value, ci.type_id))
+                    else:
+                        has_dynamic = True
                 else:
-                    if existed_value != value:
+                    if existed_value != value and existed_attr:
                         if value is None:
                             existed_attr.delete(flush=False, commit=False)
                         else:
                             existed_attr.update(value=value, flush=False, commit=False)
 
-                        changed.append((ci.id, attr.id, OperateType.UPDATE, existed_value, value, ci.type_id))
+                        if not attr.is_dynamic:
-
+                            changed.append((ci.id, attr.id, OperateType.UPDATE, existed_value, value, ci.type_id))
-        try:
-            db.session.commit()
-        except Exception as e:
-            db.session.rollback()
-            current_app.logger.warning(str(e))
-            return abort(400, ErrFormat.attribute_value_unknown_error.format(str(e)))
-
-        return self._write_change2(changed)
-
-    def create_or_update_attr_value(self, key, value, ci, _no_attribute_policy=ExistPolicy.IGNORE, record_id=None):
-        """
-        add or update attribute value, then write history
-        :param key: id, name or alias
-        :param value:
-        :param ci: instance object
-        :param _no_attribute_policy: ignore or reject
-        :param record_id: op record
-        :return:
-        """
-        attr = self._get_attr(key)
-        if attr is None:
-            if _no_attribute_policy == ExistPolicy.IGNORE:
-                return
-            if _no_attribute_policy == ExistPolicy.REJECT:
-                return abort(400, ErrFormat.attribute_not_found.format(key))
-
-        value_table = TableMap(attr=attr).table
-
-        try:
-            if attr.is_list:
-                value_list = [self._validate(attr, i, value_table, ci) for i in handle_arg_list(value)]
-                if not value_list:
-                    self.__check_is_required(ci.type_id, attr, '')
-
-                existed_attrs = value_table.get_by(attr_id=attr.id, ci_id=ci.id, to_dict=False)
-                existed_values = [i.value for i in existed_attrs]
-                added = set(value_list) - set(existed_values)
-                deleted = set(existed_values) - set(value_list)
-                for v in added:
-                    value_table.create(ci_id=ci.id, attr_id=attr.id, value=v)
-                    record_id = self._write_change(ci.id, attr.id, OperateType.ADD, None, v, record_id, ci.type_id)
-
-                for v in deleted:
-                    existed_attr = existed_attrs[existed_values.index(v)]
-                    existed_attr.delete()
-                    record_id = self._write_change(ci.id, attr.id, OperateType.DELETE, v, None, record_id, ci.type_id)
-            else:
-                value = self._validate(attr, value, value_table, ci)
-                existed_attr = value_table.get_by(attr_id=attr.id, ci_id=ci.id, first=True, to_dict=False)
-                existed_value = existed_attr and existed_attr.value
-                if existed_value is None and value is not None:
-                    value_table.create(ci_id=ci.id, attr_id=attr.id, value=value)
-
-                    record_id = self._write_change(ci.id, attr.id, OperateType.ADD, None, value, record_id, ci.type_id)
-                else:
-                    if existed_value != value:
-                        if value is None:
-                            existed_attr.delete()
                         else:
-                            existed_attr.update(value=value)
+                            has_dynamic = True
 
-                        record_id = self._write_change(ci.id, attr.id, OperateType.UPDATE,
+        if changed or has_dynamic:
-                                                       existed_value, value, record_id, ci.type_id)
+            try:
+                db.session.commit()
+            except Exception as e:
+                db.session.rollback()
+                current_app.logger.warning(str(e))
+                return abort(400, ErrFormat.attribute_value_unknown_error.format(e.args[0]))
 
-            return record_id
+            return self.write_change2(changed, ticket_id=ticket_id), has_dynamic
-        except Exception as e:
+        else:
-            current_app.logger.warning(str(e))
+            return None, has_dynamic
-            return abort(400, ErrFormat.attribute_value_invalid2.format("{}({})".format(attr.alias, attr.name), value))
 
     @staticmethod
-    def delete_attr_value(attr_id, ci_id):
+    def delete_attr_value(attr_id, ci_id, commit=True):
         attr = AttributeCache.get(attr_id)
         if attr is not None:
             value_table = TableMap(attr=attr).table
             for item in value_table.get_by(attr_id=attr.id, ci_id=ci_id, to_dict=False):
-                item.delete()
+                item.delete(commit=commit)

cmdb-api/api/lib/common_setting/acl.py

@@ -1,13 +1,20 @@
 # -*- coding:utf-8 -*-
-from flask import abort
 from flask import current_app
 
 from api.lib.common_setting.resp_format import ErrFormat
+from api.lib.perm.acl.app import AppCRUD
 from api.lib.perm.acl.cache import RoleCache, AppCache
+from api.lib.perm.acl.permission import PermissionCRUD
+from api.lib.perm.acl.resource import ResourceTypeCRUD, ResourceCRUD
 from api.lib.perm.acl.role import RoleCRUD, RoleRelationCRUD
 from api.lib.perm.acl.user import UserCRUD
 
 
+def validate_app(app_id):
+    app = AppCache.get(app_id)
+    return app.id if app else None
+
+
 class ACLManager(object):
     def __init__(self, app_name='acl', uid=None):
         self.log = current_app.logger
@@ -78,19 +85,69 @@ class ACLManager(object):
         return role.to_dict()
 
     @staticmethod
-    def delete_role(_id, payload):
+    def delete_role(_id):
         RoleCRUD.delete_role(_id)
         return dict(rid=_id)
 
     def get_user_info(self, username):
         from api.lib.perm.acl.acl import ACLManager as ACL
         user_info = ACL().get_user_info(username, self.app_name)
-        result = dict(name=user_info.get('nickname') or username,
+        result = dict(
-                      username=user_info.get('username') or username,
+            name=user_info.get('nickname') or username,
-                      email=user_info.get('email'),
+            username=user_info.get('username') or username,
-                      uid=user_info.get('uid'),
+            email=user_info.get('email'),
-                      rid=user_info.get('rid'),
+            uid=user_info.get('uid'),
-                      role=dict(permissions=user_info.get('parents')),
+            rid=user_info.get('rid'),
-                      avatar=user_info.get('avatar'))
+            role=dict(permissions=user_info.get('parents')),
+            avatar=user_info.get('avatar')
+        )
 
         return result
+
+    def validate_app(self):
+        return AppCache.get(self.app_name)
+
+    def get_all_resources_types(self, q=None, page=1, page_size=999999):
+        app_id = self.validate_app().id
+        numfound, res, id2perms = ResourceTypeCRUD.search(q, app_id, page, page_size)
+
+        return dict(
+            numfound=numfound,
+            groups=[i.to_dict() for i in res],
+            id2perms=id2perms
+        )
+
+    def create_resources_type(self, payload):
+        payload['app_id'] = self.validate_app().id
+        rt = ResourceTypeCRUD.add(**payload)
+
+        return rt.to_dict()
+
+    def update_resources_type(self, _id, payload):
+        rt = ResourceTypeCRUD.update(_id, **payload)
+
+        return rt.to_dict()
+
+    def create_resource(self, payload):
+        payload['app_id'] = self.validate_app().id
+        resource = ResourceCRUD.add(**payload)
+
+        return resource.to_dict()
+
+    def get_resource_by_type(self, q, u, rt_id, page=1, page_size=999999):
+        numfound, res = ResourceCRUD.search(q, u, self.validate_app().id, rt_id, page, page_size)
+        return res
+
+    @staticmethod
+    def grant_resource(rid, resource_id, perms):
+        PermissionCRUD.grant(rid, perms, resource_id=resource_id, group_id=None)
+
+    @staticmethod
+    def create_app(payload):
+        rt = AppCRUD.add(**payload)
+
+        return rt.to_dict()
+
+    def role_has_perms(self, rid, resource_name, resource_type_name, perm):
+        app_id = validate_app(self.app_name)
+        return RoleCRUD.has_permission(rid, resource_name, resource_type_name, app_id, perm)

cmdb-api/api/lib/common_setting/common_data.py

@@ -0,0 +1,282 @@
+import copy
+import json
+
+from flask import abort, current_app
+from ldap3 import Connection
+from ldap3 import Server
+from ldap3.core.exceptions import LDAPBindError, LDAPSocketOpenError
+from ldap3 import AUTO_BIND_NO_TLS
+
+from api.extensions import db
+from api.lib.common_setting.resp_format import ErrFormat
+from api.models.common_setting import CommonData
+from api.lib.utils import AESCrypto
+from api.lib.common_setting.const import AuthCommonConfig, AuthenticateType, AuthCommonConfigAutoRedirect, TestType
+
+
+class CommonDataCRUD(object):
+
+    @staticmethod
+    def get_data_by_type(data_type):
+        CommonDataCRUD.check_auth_type(data_type)
+        return CommonData.get_by(data_type=data_type)
+
+    @staticmethod
+    def get_data_by_id(_id, to_dict=True):
+        return CommonData.get_by(first=True, id=_id, to_dict=to_dict)
+
+    @staticmethod
+    def create_new_data(data_type, **kwargs):
+        try:
+            CommonDataCRUD.check_auth_type(data_type)
+
+            return CommonData.create(data_type=data_type, **kwargs)
+        except Exception as e:
+            db.session.rollback()
+            abort(400, str(e))
+
+    @staticmethod
+    def update_data(_id, **kwargs):
+        existed = CommonDataCRUD.get_data_by_id(_id, to_dict=False)
+        if not existed:
+            abort(404, ErrFormat.common_data_not_found.format(_id))
+        try:
+            CommonDataCRUD.check_auth_type(existed.data_type)
+            return existed.update(**kwargs)
+        except Exception as e:
+            db.session.rollback()
+            abort(400, str(e))
+
+    @staticmethod
+    def delete(_id):
+        existed = CommonDataCRUD.get_data_by_id(_id, to_dict=False)
+        if not existed:
+            abort(404, ErrFormat.common_data_not_found.format(_id))
+        try:
+            CommonDataCRUD.check_auth_type(existed.data_type)
+            existed.soft_delete()
+        except Exception as e:
+            db.session.rollback()
+            abort(400, str(e))
+
+    @staticmethod
+    def check_auth_type(data_type):
+        if data_type in list(AuthenticateType.all()) + [AuthCommonConfig]:
+            abort(400, ErrFormat.common_data_not_support_auth_type.format(data_type))
+
+    @staticmethod
+    def set_auth_auto_redirect_enable(_value: int):
+        existed = CommonData.get_by(first=True, data_type=AuthCommonConfig, to_dict=False)
+        if not existed:
+            CommonDataCRUD.create_new_data(AuthCommonConfig, data={AuthCommonConfigAutoRedirect: _value})
+        else:
+            data = existed.data
+            data = copy.deepcopy(existed.data) if data else {}
+            data[AuthCommonConfigAutoRedirect] = _value
+            CommonDataCRUD.update_data(existed.id, data=data)
+        return True
+
+    @staticmethod
+    def get_auth_auto_redirect_enable():
+        existed = CommonData.get_by(first=True, data_type=AuthCommonConfig)
+        if not existed:
+            return 0
+        data = existed.get('data', {})
+        if not data:
+            return 0
+        return data.get(AuthCommonConfigAutoRedirect, 0)
+
+
+class AuthenticateDataCRUD(object):
+    common_type_list = [AuthCommonConfig]
+
+    def __init__(self, _type):
+        self._type = _type
+        self.record = None
+        self.decrypt_data = {}
+
+    def get_support_type_list(self):
+        return list(AuthenticateType.all()) + self.common_type_list
+
+    def get(self):
+        if not self.decrypt_data:
+            self.decrypt_data = self.get_decrypt_data()
+
+        return self.decrypt_data
+
+    def get_by_key(self, _key):
+        if not self.decrypt_data:
+            self.decrypt_data = self.get_decrypt_data()
+
+        return self.decrypt_data.get(_key, None)
+
+    def get_record(self, to_dict=False) -> CommonData:
+        return CommonData.get_by(first=True, data_type=self._type, to_dict=to_dict)
+
+    def get_record_with_decrypt(self) -> dict:
+        record = CommonData.get_by(first=True, data_type=self._type, to_dict=True)
+        if not record:
+            return {}
+        data = self.get_decrypt_dict(record.get('data', ''))
+        record['data'] = data
+        return record
+
+    def get_decrypt_dict(self, data):
+        decrypt_str = self.decrypt(data)
+        try:
+            return json.loads(decrypt_str)
+        except Exception as e:
+            abort(400, str(e))
+
+    def get_decrypt_data(self) -> dict:
+        self.record = self.get_record()
+        if not self.record:
+            return self.get_from_config()
+        return self.get_decrypt_dict(self.record.data)
+
+    def get_from_config(self):
+        return current_app.config.get(self._type, {})
+
+    def check_by_type(self) -> None:
+        existed = self.get_record()
+        if existed:
+            abort(400, ErrFormat.common_data_already_existed.format(self._type))
+
+    def create(self, data) -> CommonData:
+        self.check_by_type()
+        encrypt = data.pop('encrypt', None)
+        if encrypt is False:
+            return CommonData.create(data_type=self._type, data=data)
+        encrypted_data = self.encrypt(data)
+        try:
+            return CommonData.create(data_type=self._type, data=encrypted_data)
+        except Exception as e:
+            db.session.rollback()
+            abort(400, str(e))
+
+    def update_by_record(self, record, data) -> CommonData:
+        encrypt = data.pop('encrypt', None)
+        if encrypt is False:
+            return record.update(data=data)
+        encrypted_data = self.encrypt(data)
+        try:
+            return record.update(data=encrypted_data)
+        except Exception as e:
+            db.session.rollback()
+            abort(400, str(e))
+
+    def update(self, _id, data) -> CommonData:
+        existed = CommonData.get_by(first=True, to_dict=False, id=_id)
+        if not existed:
+            abort(404, ErrFormat.common_data_not_found.format(_id))
+
+        return self.update_by_record(existed, data)
+
+    @staticmethod
+    def delete(_id) -> None:
+        existed = CommonData.get_by(first=True, to_dict=False, id=_id)
+        if not existed:
+            abort(404, ErrFormat.common_data_not_found.format(_id))
+        try:
+            existed.soft_delete()
+        except Exception as e:
+            db.session.rollback()
+            abort(400, str(e))
+
+    @staticmethod
+    def encrypt(data) -> str:
+        if type(data) is dict:
+            try:
+                data = json.dumps(data)
+            except Exception as e:
+                abort(400, str(e))
+        return AESCrypto().encrypt(data)
+
+    @staticmethod
+    def decrypt(data) -> str:
+        return AESCrypto().decrypt(data)
+
+    @staticmethod
+    def get_enable_list():
+        all_records = CommonData.query.filter(
+            CommonData.data_type.in_(AuthenticateType.all()),
+            CommonData.deleted == 0
+        ).all()
+        enable_list = []
+        for auth_type in AuthenticateType.all():
+            record = list(filter(lambda x: x.data_type == auth_type, all_records))
+            if not record:
+                config = current_app.config.get(auth_type, None)
+                if not config:
+                    continue
+
+                if config.get('enable', False):
+                    enable_list.append(dict(
+                        auth_type=auth_type,
+                    ))
+
+                continue
+
+            try:
+                decrypt_data = json.loads(AuthenticateDataCRUD.decrypt(record[0].data))
+            except Exception as e:
+                current_app.logger.error(e)
+                continue
+
+            if decrypt_data.get('enable', 0) == 1:
+                enable_list.append(dict(
+                    auth_type=auth_type,
+                ))
+
+        auth_auto_redirect = CommonDataCRUD.get_auth_auto_redirect_enable()
+
+        return dict(
+            enable_list=enable_list,
+            auth_auto_redirect=auth_auto_redirect,
+        )
+
+    def test(self, test_type, data):
+        type_lower = self._type.lower()
+        func_name = f'test_{type_lower}'
+        if hasattr(self, func_name):
+            try:
+                return getattr(self, f'test_{type_lower}')(test_type, data)
+            except Exception as e:
+                abort(400, str(e))
+        abort(400, ErrFormat.not_support_test.format(self._type))
+
+    @staticmethod
+    def test_ldap(test_type, data):
+        ldap_server = data.get('ldap_server')
+        ldap_user_dn = data.get('ldap_user_dn', '{}')
+
+        server = Server(ldap_server, connect_timeout=2)
+        if not server.check_availability():
+            raise Exception(ErrFormat.ldap_server_connect_not_available)
+        else:
+            if test_type == TestType.Connect:
+                return True
+
+        username = data.get('username', None)
+        if not username:
+            raise Exception(ErrFormat.ldap_test_username_required)
+        user = ldap_user_dn.format(username)
+        password = data.get('password', None)
+
+        try:
+            Connection(server, user=user, password=password, auto_bind=AUTO_BIND_NO_TLS)
+        except LDAPBindError:
+            ldap_domain = data.get('ldap_domain')
+            user_with_domain = f"{username}@{ldap_domain}"
+            try:
+                Connection(server, user=user_with_domain, password=password, auto_bind=AUTO_BIND_NO_TLS)
+            except Exception as e:
+                raise Exception(ErrFormat.ldap_test_unknown_error.format(str(e)))
+
+        except LDAPSocketOpenError:
+            raise Exception(ErrFormat.ldap_server_connect_timeout)
+
+        except Exception as e:
+            raise Exception(ErrFormat.ldap_test_unknown_error.format(str(e)))
+
+        return True

cmdb-api/api/lib/common_setting/company_info.py

@@ -1,5 +1,7 @@
 # -*- coding:utf-8 -*-
+from urllib.parse import urlparse
 
+from api.extensions import cache
 from api.models.common_setting import CompanyInfo
 
 
@@ -11,14 +13,51 @@ class CompanyInfoCRUD(object):
 
     @staticmethod
     def create(**kwargs):
-        return CompanyInfo.create(**kwargs)
+        CompanyInfoCRUD.check_data(**kwargs)
+        res = CompanyInfo.create(**kwargs)
+        CompanyInfoCache.refresh(res.info)
+        return res
 
     @staticmethod
     def update(_id, **kwargs):
         kwargs.pop('id', None)
         existed = CompanyInfo.get_by_id(_id)
         if not existed:
-            return CompanyInfoCRUD.create(**kwargs)
+            existed = CompanyInfoCRUD.create(**kwargs)
         else:
+            CompanyInfoCRUD.check_data(**kwargs)
             existed = existed.update(**kwargs)
-            return existed
+        CompanyInfoCache.refresh(existed.info)
+        return existed
+
+    @staticmethod
+    def check_data(**kwargs):
+        info = kwargs.get('info', {})
+        info['messenger'] = CompanyInfoCRUD.check_messenger(info.get('messenger', None))
+
+        kwargs['info'] = info
+
+    @staticmethod
+    def check_messenger(messenger):
+        if not messenger:
+            return messenger
+
+        parsed_url = urlparse(messenger)
+        return f"{parsed_url.scheme}://{parsed_url.netloc}"
+
+
+class CompanyInfoCache(object):
+    key = 'CompanyInfoCache::'
+
+    @classmethod
+    def get(cls):
+        info = cache.get(cls.key)
+        if not info:
+            res = CompanyInfo.get_by(first=True) or {}
+            info = res.get('info', {})
+            cache.set(cls.key, info)
+        return info
+
+    @classmethod
+    def refresh(cls, info):
+        cache.set(cls.key, info)

cmdb-api/api/lib/common_setting/const.py

@@ -4,11 +4,63 @@ COMMON_SETTING_QUEUE = "common_setting_async"
 
 
 class OperatorType(BaseEnum):
-    EQUAL = 1  # 等于
+    EQUAL = 1
-    NOT_EQUAL = 2  # 不等于
+    NOT_EQUAL = 2
-    IN = 3  # 包含
+    IN = 3
-    NOT_IN = 4  # 不包含
+    NOT_IN = 4
-    GREATER_THAN = 5  # 大于
+    GREATER_THAN = 5
-    LESS_THAN = 6  # 小于
+    LESS_THAN = 6
-    IS_EMPTY = 7  # 为空
+    IS_EMPTY = 7
-    IS_NOT_EMPTY = 8  # 不为空
+    IS_NOT_EMPTY = 8
+
+
+BotNameMap = {
+    'wechatApp': 'wechatBot',
+    'feishuApp': 'feishuBot',
+    'dingdingApp': 'dingdingBot',
+}
+
+
+class AuthenticateType(BaseEnum):
+    CAS = 'CAS'
+    OAUTH2 = 'OAUTH2'
+    OIDC = 'OIDC'
+    LDAP = 'LDAP'
+
+
+AuthCommonConfig = 'AuthCommonConfig'
+AuthCommonConfigAutoRedirect = 'auto_redirect'
+
+
+class TestType(BaseEnum):
+    Connect = 'connect'
+    Login = 'login'
+
+
+MIMEExtMap = {
+    'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx',
+    'application/msword': '.doc',
+    'application/vnd.ms-word.document.macroEnabled.12': '.docm',
+    'application/vnd.ms-excel': '.xls',
+    'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': '.xlsx',
+    'application/vnd.ms-excel.sheet.macroEnabled.12': '.xlsm',
+    'application/vnd.ms-powerpoint': '.ppt',
+    'application/vnd.openxmlformats-officedocument.presentationml.presentation': '.pptx',
+    'application/vnd.ms-powerpoint.presentation.macroEnabled.12': '.pptm',
+    'application/zip': '.zip',
+    'application/x-7z-compressed': '.7z',
+    'application/json': '.json',
+    'application/pdf': '.pdf',
+    'image/png': '.png',
+    'image/bmp': '.bmp',
+    'image/prs.btif': '.btif',
+    'image/gif': '.gif',
+    'image/jpeg': '.jpg',
+    'image/tiff': '.tif',
+    'image/vnd.microsoft.icon': '.ico',
+    'image/webp': '.webp',
+    'image/svg+xml': '.svg',
+    'image/vnd.adobe.photoshop': '.psd',
+    'text/plain': '.txt',
+    'text/csv': '.csv',
+}

cmdb-api/api/lib/common_setting/decorator.py

@@ -0,0 +1,39 @@
+import functools
+
+from flask import abort, session, current_app
+from api.lib.common_setting.acl import ACLManager
+from api.lib.common_setting.resp_format import ErrFormat
+from api.lib.perm.acl.acl import is_app_admin
+
+
+def perms_role_required(app_name, resource_type_name, resource_name, perm, role_name=None):
+    def decorator_perms_role_required(func):
+        @functools.wraps(func)
+        def wrapper_required(*args, **kwargs):
+            acl = ACLManager(app_name)
+            has_perms = False
+            try:
+                has_perms = acl.role_has_perms(session["acl"]['rid'], resource_name, resource_type_name, perm)
+            except Exception as e:
+                current_app.logger.error(f"acl role_has_perms err: {e}")
+                # resource_type not exist, continue check role
+                if role_name:
+                    if role_name not in session.get("acl", {}).get("parentRoles", []) and not is_app_admin(app_name):
+                        abort(403, ErrFormat.role_required.format(role_name))
+
+                    return func(*args, **kwargs)
+                else:
+                    abort(403, ErrFormat.resource_no_permission.format(resource_name, perm))
+
+            if not has_perms:
+                if role_name:
+                    if role_name not in session.get("acl", {}).get("parentRoles", []) and not is_app_admin(app_name):
+                        abort(403, ErrFormat.role_required.format(role_name))
+                else:
+                    abort(403, ErrFormat.resource_no_permission.format(resource_name, perm))
+
+            return func(*args, **kwargs)
+
+        return wrapper_required
+
+    return decorator_perms_role_required

cmdb-api/api/lib/common_setting/department.py

@@ -1,47 +1,41 @@
 # -*- coding:utf-8 -*-
 
-from flask import abort
+from flask import abort, current_app
 from treelib import Tree
 from wtforms import Form
 from wtforms import IntegerField
 from wtforms import StringField
 from wtforms import validators
 
+from api.extensions import db
 from api.lib.common_setting.resp_format import ErrFormat
-from api.lib.common_setting.utils import get_df_from_read_sql
+from api.lib.common_setting.acl import ACLManager
 from api.lib.perm.acl.role import RoleCRUD
 from api.models.common_setting import Department, Employee
 
 sub_departments_column_name = 'sub_departments'
 
 
-def drop_ts_column(df):
+def get_all_department_list(to_dict=True):
-    columns = list(df.columns)
-    remove_columns = []
-    for column in ['created_at', 'updated_at', 'deleted_at', 'last_login']:
-        targets = list(filter(lambda c: c.startswith(column), columns))
-        if targets:
-            remove_columns.extend(targets)
-
-    remove_columns = list(set(remove_columns))
-
-    return df.drop(remove_columns, axis=1) if len(remove_columns) > 0 else df
-
-
-def get_department_df():
     criterion = [
         Department.deleted == 0,
     ]
     query = Department.query.filter(
         *criterion
-    )
+    ).order_by(Department.department_id.asc())
-    df = get_df_from_read_sql(query)
+    results = query.all()
-    if df.empty:
+    if to_dict:
-        return
+        datas = []
-    return drop_ts_column(df)
+        for r in results:
+            d = r.to_dict()
+            if r.department_id == 0:
+                d['department_name'] = ErrFormat.company_wide
+            datas.append(d)
+        return datas
+    return results
 
 
-def get_all_employee_df(block=0):
+def get_all_employee_list(block=0, to_dict=True):
     criterion = [
         Employee.deleted == 0,
     ]
@@ -50,112 +44,107 @@ def get_all_employee_df(block=0):
             Employee.block == block
         )
 
-    entities = [getattr(Employee, c) for c in Employee.get_columns(
+    results = db.session.query(Employee).filter(*criterion).all()
-    ).keys() if c not in ['deleted', 'deleted_at']]
+
-    query = Employee.query.with_entities(
+    DepartmentTreeEmployeeColumns = [
-        *entities
+        'acl_rid',
-    ).filter(
+        'employee_id',
-        *criterion
+        'username',
-    )
+        'nickname',
-    df = get_df_from_read_sql(query)
+        'email',
-    if df.empty:
+        'mobile',
-        return df
+        'direct_supervisor_id',
-    return drop_ts_column(df)
+        'block',
+        'department_id',
+    ]
+
+    def format_columns(e):
+        return {column: getattr(e, column) for column in DepartmentTreeEmployeeColumns}
+
+    return [format_columns(r) for r in results] if to_dict else results
 
 
 class DepartmentTree(object):
     def __init__(self, append_employee=False, block=-1):
         self.append_employee = append_employee
         self.block = block
-        self.d_df = get_department_df()
+        self.all_department_list = get_all_department_list()
-        self.employee_df = get_all_employee_df(
+        self.all_employee_list = get_all_employee_list(
             block) if append_employee else None
 
     def prepare(self):
         pass
 
     def get_employees_by_d_id(self, d_id):
-        _df = self.employee_df[
+        block = self.block
-            self.employee_df['department_id'].eq(d_id)
+
-        ].sort_values(by=['direct_supervisor_id'], ascending=True)
+        def filter_department_id(e):
-        if _df.empty:
+            if self.block != -1:
+                return e['department_id'] == d_id and e['block'] == block
+            return e.department_id == d_id
+
+        results = list(filter(lambda e: filter_department_id(e), self.all_employee_list))
+
+        return results
+
+    def get_department_by_parent_id(self, parent_id):
+        results = list(filter(lambda d: d['department_parent_id'] == parent_id, self.all_department_list))
+        if not results:
             return []
-
+        return results
-        if self.block != -1:
-            _df = _df[
-                _df['block'].eq(self.block)
-            ]
-
-        return _df.to_dict('records')
 
     def get_tree_departments(self):
         # 一级部门
-        top_df = self.d_df[self.d_df['department_parent_id'].eq(-1)]
+        top_departments = self.get_department_by_parent_id(-1)
-        if top_df.empty:
+        if len(top_departments) == 0:
             return []
 
         d_list = []
 
-        for index in top_df.index:
+        for top_d in top_departments:
-            top_d = top_df.loc[index].to_dict()
-
             department_id = top_d['department_id']
-
+            sub_deps = self.get_department_by_parent_id(department_id)
-            # 检查 department_id 是否作为其他部门的 parent
-            sub_df = self.d_df[
-                self.d_df['department_parent_id'].eq(department_id)
-            ].sort_values(by=['sort_value'], ascending=True)
-
             employees = []
-
             if self.append_employee:
-                # 要包含员工
                 employees = self.get_employees_by_d_id(department_id)
 
             top_d['employees'] = employees
-
+            top_d['department_name'] = ErrFormat.company_wide
-            if sub_df.empty:
+            if len(sub_deps) == 0:
                 top_d[sub_departments_column_name] = []
                 d_list.append(top_d)
                 continue
 
-            self.parse_sub_department(sub_df, top_d)
+            self.parse_sub_department(sub_deps, top_d)
             d_list.append(top_d)
 
         return d_list
 
     def get_all_departments(self, is_tree=1):
-        if self.d_df.empty:
+        if len(self.all_department_list) == 0:
             return []
 
         if is_tree != 1:
-            return self.d_df.to_dict('records')
+            return self.all_department_list
 
         return self.get_tree_departments()
 
-    def parse_sub_department(self, df, top_d):
+    def parse_sub_department(self, deps, top_d):
         sub_departments = []
-        for s_index in df.index:
+        for d in deps:
-            d = df.loc[s_index].to_dict()
+            sub_deps = self.get_department_by_parent_id(d['department_id'])
-            sub_df = self.d_df[
-                self.d_df['department_parent_id'].eq(
-                    df.at[s_index, 'department_id'])
-            ].sort_values(by=['sort_value'], ascending=True)
             employees = []
-
             if self.append_employee:
-                # 要包含员工
+                employees = self.get_employees_by_d_id(d['department_id'])
-                employees = self.get_employees_by_d_id(
-                    df.at[s_index, 'department_id'])
 
             d['employees'] = employees
 
-            if sub_df.empty:
+            if len(sub_deps) == 0:
                 d[sub_departments_column_name] = []
                 sub_departments.append(d)
                 continue
 
-            self.parse_sub_department(sub_df, d)
+            self.parse_sub_department(sub_deps, d)
             sub_departments.append(d)
 
         top_d[sub_departments_column_name] = sub_departments
@@ -173,6 +162,10 @@ class DepartmentForm(Form):
 
 class DepartmentCRUD(object):
 
+    @staticmethod
+    def get_department_by_id(d_id, to_dict=True):
+        return Department.get_by(first=True, department_id=d_id, to_dict=to_dict)
+
     @staticmethod
     def add(**kwargs):
         DepartmentCRUD.check_department_name_unique(kwargs['department_name'])
@@ -202,16 +195,16 @@ class DepartmentCRUD(object):
     def check_department_parent_id_allow(d_id, department_parent_id):
         if department_parent_id == 0:
             return
-        # 检查 department_parent_id 是否在许可范围内
         allow_p_d_id_list = DepartmentCRUD.get_allow_parent_d_id_by(d_id)
         target = list(
             filter(lambda d: d['department_id'] == department_parent_id, allow_p_d_id_list))
         if len(target) == 0:
             try:
-                d = Department.get_by(
+                dep = Department.get_by(
                     first=True, to_dict=False, department_id=department_parent_id)
-                name = d.department_name if d else ErrFormat.department_id_not_found.format(department_parent_id)
+                name = dep.department_name if dep else ErrFormat.department_id_not_found.format(department_parent_id)
             except Exception as e:
+                current_app.logger.error(str(e))
                 name = ErrFormat.department_id_not_found.format(department_parent_id)
             abort(400, ErrFormat.cannot_to_be_parent_department.format(name))
 
@@ -262,7 +255,7 @@ class DepartmentCRUD(object):
             return abort(400, ErrFormat.acl_update_role_failed.format(str(e)))
 
         try:
-            existed.update(**kwargs)
+            return existed.update(**kwargs)
         except Exception as e:
             return abort(400, str(e))
 
@@ -275,15 +268,12 @@ class DepartmentCRUD(object):
         try:
             RoleCRUD.delete_role(existed.acl_rid)
         except Exception as e:
-            pass
+            current_app.logger.error(str(e))
 
         return existed.soft_delete()
 
     @staticmethod
     def get_allow_parent_d_id_by(department_id):
-        """
-        获取可以成为 department_id 的 department_parent_id 的 list
-        """
         tree_list = DepartmentCRUD.get_department_tree_list()
 
         allow_d_id_list = []
@@ -293,7 +283,7 @@ class DepartmentCRUD(object):
                 try:
                     tree.remove_subtree(department_id)
                 except Exception as e:
-                    pass
+                    current_app.logger.error(str(e))
 
             [allow_d_id_list.append({'department_id': int(n.identifier), 'department_name': n.tag}) for n in
              tree.all_nodes()]
@@ -321,58 +311,58 @@ class DepartmentCRUD(object):
 
     @staticmethod
     def get_department_tree_list():
-        df = get_department_df()
+        all_deps = get_all_department_list()
-        if df.empty:
+        if len(all_deps) == 0:
             return []
 
-        # 一级部门
+        top_deps = list(filter(lambda d: d['department_parent_id'] == -1, all_deps))
-        top_df = df[df['department_parent_id'].eq(-1)]
+        if len(top_deps) == 0:
-        if top_df.empty:
             return []
 
         tree_list = []
 
-        for index in top_df.index:
+        for top_d in top_deps:
+            top_d['department_name'] = ErrFormat.company_wide
             tree = Tree()
-            identifier_root = top_df.at[index, 'department_id']
+            identifier_root = top_d['department_id']
             tree.create_node(
-                top_df.at[index, 'department_name'],
+                top_d['department_name'],
                 identifier_root
             )
-
+            sub_ds = list(filter(lambda d: d['department_parent_id'] == identifier_root, all_deps))
-            # 检查 department_id 是否作为其他部门的 parent
+            if len(sub_ds) == 0:
-            sub_df = df[
-                df['department_parent_id'].eq(identifier_root)
-            ]
-            if sub_df.empty:
                 tree_list.append(tree)
                 continue
 
             DepartmentCRUD.parse_sub_department_node(
-                sub_df, df, tree, identifier_root)
+                sub_ds, all_deps, tree, identifier_root)
 
             tree_list.append(tree)
 
         return tree_list
 
     @staticmethod
-    def parse_sub_department_node(df, all_df, tree, parent_id):
+    def parse_sub_department_node(sub_ds, all_ds, tree, parent_id):
-        for s_index in df.index:
+        for d in sub_ds:
             tree.create_node(
-                df.at[s_index, 'department_name'],
+                d['department_name'],
-                df.at[s_index, 'department_id'],
+                d['department_id'],
                 parent=parent_id
             )
 
-            sub_df = all_df[
+            next_sub_ds = list(filter(lambda item_d: item_d['department_parent_id'] == d['department_id'], all_ds))
-                all_df['department_parent_id'].eq(
+            if len(next_sub_ds) == 0:
-                    df.at[s_index, 'department_id'])
-            ]
-            if sub_df.empty:
                 continue
 
             DepartmentCRUD.parse_sub_department_node(
-                sub_df, all_df, tree, df.at[s_index, 'department_id'])
+                next_sub_ds, all_ds, tree, d['department_id'])
+
+    @staticmethod
+    def get_department_by_query(query, to_dict=True):
+        results = query.all()
+        if not results:
+            return []
+        return results if not to_dict else [r.to_dict() for r in results]
 
     @staticmethod
     def get_departments_and_ids(department_parent_id, block):
@@ -380,44 +370,33 @@ class DepartmentCRUD(object):
             Department.department_parent_id == department_parent_id,
             Department.deleted == 0,
         ).order_by(Department.sort_value.asc())
-        df = get_df_from_read_sql(query)
+        all_departments = DepartmentCRUD.get_department_by_query(query)
-        if df.empty:
+        if len(all_departments) == 0:
             return [], []
 
         tree_list = DepartmentCRUD.get_department_tree_list()
-        employee_df = get_all_employee_df(block)
+        all_employee_list = get_all_employee_list(block)
 
-        department_id_list = list(df['department_id'].values)
+        department_id_list = [d['department_id'] for d in all_departments]
         query = Department.query.filter(
             Department.department_parent_id.in_(department_id_list),
             Department.deleted == 0,
         ).order_by(Department.sort_value.asc()).group_by(Department.department_id)
-        sub_df = get_df_from_read_sql(query)
+        sub_deps = DepartmentCRUD.get_department_by_query(query)
-        if sub_df.empty:
-            df['has_sub'] = 0
 
-            def handle_row_employee_count(row):
+        sub_map = {d['department_parent_id']: 1 for d in sub_deps}
-                return len(employee_df[employee_df['department_id'] == row['department_id']])
 
-            df['employee_count'] = df.apply(
+        for d in all_departments:
-                lambda row: handle_row_employee_count(row), axis=1)
+            d['has_sub'] = sub_map.get(d['department_id'], 0)
 
-        else:
+            d_ids = DepartmentCRUD.get_department_id_list_by_root(d['department_id'], tree_list)
-            sub_map = {d['department_parent_id']: 1 for d in sub_df.to_dict('records')}
 
-            def handle_row(row):
+            d['employee_count'] = len(list(filter(lambda e: e['department_id'] in d_ids, all_employee_list)))
-                d_ids = DepartmentCRUD.get_department_id_list_by_root(
-                    row['department_id'], tree_list)
-                row['employee_count'] = len(
-                    employee_df[employee_df['department_id'].isin(d_ids)])
 
-                row['has_sub'] = sub_map.get(row['department_id'], 0)
+            if int(department_parent_id) == -1:
+                d['department_name'] = ErrFormat.company_wide
 
-                return row
+        return all_departments, department_id_list
-
-            df = df.apply(lambda row: handle_row(row), axis=1)
-
-        return df.to_dict('records'), department_id_list
 
     @staticmethod
     def get_department_id_list_by_root(root_department_id, tree_list=None):
@@ -430,6 +409,151 @@ class DepartmentCRUD(object):
                 [id_list.append(int(n.identifier))
                  for n in tmp_tree.all_nodes()]
             except Exception as e:
-                pass
+                current_app.logger.error(str(e))
 
         return id_list
+
+
+class EditDepartmentInACL(object):
+
+    @staticmethod
+    def add_department_to_acl(department_id, op_uid):
+        db_department = DepartmentCRUD.get_department_by_id(department_id, to_dict=False)
+        if not db_department:
+            return
+
+        from api.models.acl import Role
+        role = Role.get_by(first=True, name=db_department.department_name, app_id=None)
+
+        acl = ACLManager('acl', str(op_uid))
+        if role is None:
+            payload = {
+                'app_id': 'acl',
+                'name': db_department.department_name,
+            }
+            role = acl.create_role(payload)
+
+        acl_rid = role.get('id') if role else 0
+
+        db_department.update(
+            acl_rid=acl_rid
+        )
+        info = f"add_department_to_acl, acl_rid: {acl_rid}"
+        current_app.logger.info(info)
+        return info
+
+    @staticmethod
+    def delete_department_from_acl(department_rids, op_uid):
+        acl = ACLManager('acl', str(op_uid))
+
+        result = []
+
+        for rid in department_rids:
+            try:
+                acl.delete_role(rid)
+            except Exception as e:
+                result.append(f"delete_department_in_acl, rid: {rid}, error: {e}")
+                continue
+
+        return result
+
+    @staticmethod
+    def edit_department_name_in_acl(d_rid: int, d_name: str, op_uid: int):
+        acl = ACLManager('acl', str(op_uid))
+        payload = {
+            'name': d_name
+        }
+        try:
+            acl.edit_role(d_rid, payload)
+        except Exception as e:
+            return f"edit_department_name_in_acl, rid: {d_rid}, error: {e}"
+
+        return f"edit_department_name_in_acl, rid: {d_rid}, success"
+
+    @classmethod
+    def remove_from_old_department_role(cls, e_list, acl):
+        result = []
+        for employee in e_list:
+            employee_acl_rid = employee.get('e_acl_rid')
+            if employee_acl_rid == 0:
+                result.append("employee_acl_rid == 0")
+                continue
+            cls.remove_single_employee_from_old_department(acl, employee, result)
+
+    @staticmethod
+    def remove_single_employee_from_old_department(acl, employee, result):
+        from api.models.acl import Role
+        old_department = DepartmentCRUD.get_department_by_id(employee.get('department_id'), False)
+        if not old_department:
+            return False
+
+        old_role = Role.get_by(first=True, name=old_department.department_name, app_id=None)
+        old_d_rid_in_acl = old_role.get('id') if old_role else 0
+        if old_d_rid_in_acl == 0:
+            return False
+
+        d_acl_rid = old_department.acl_rid if old_d_rid_in_acl == old_department.acl_rid else old_d_rid_in_acl
+        payload = {
+            'app_id': 'acl',
+            'parent_id': d_acl_rid,
+        }
+        try:
+            acl.remove_user_from_role(employee.get('e_acl_rid'), payload)
+            current_app.logger.info(f"remove {employee.get('e_acl_rid')} from {d_acl_rid}")
+        except Exception as e:
+            err = f"remove_user_from_role e_acl_rid: {employee.get('e_acl_rid')}, parent_id: {d_acl_rid}, err: {e}"
+            result.append(err)
+
+        return True
+
+    @staticmethod
+    def add_employee_to_new_department(acl, employee_acl_rid, new_department_acl_rid, result):
+        payload = {
+            'app_id': 'acl',
+            'child_ids': [employee_acl_rid],
+        }
+        try:
+            acl.add_user_to_role(new_department_acl_rid, payload)
+            current_app.logger.info(f"add {employee_acl_rid} to {new_department_acl_rid}")
+        except Exception as e:
+            result.append(
+                f"add_user_to_role employee_acl_rid: {employee_acl_rid}, parent_id: {new_department_acl_rid}, \
+                             err: {e}")
+
+    @classmethod
+    def edit_employee_department_in_acl(cls, e_list: list, new_d_id: int, op_uid: int):
+        result = []
+        new_department = DepartmentCRUD.get_department_by_id(new_d_id, False)
+        if not new_department:
+            result.append(f"{new_d_id} new_department is None")
+            return result
+
+        from api.models.acl import Role
+        new_role = Role.get_by(first=True, name=new_department.department_name, app_id=None)
+        new_d_rid_in_acl = new_role.get('id') if new_role else 0
+        acl = ACLManager('acl', str(op_uid))
+
+        if new_d_rid_in_acl == 0:
+            # only remove from old department role
+            cls.remove_from_old_department_role(e_list, acl)
+            return
+
+        if new_d_rid_in_acl != new_department.acl_rid:
+            new_department.update(
+                acl_rid=new_d_rid_in_acl
+            )
+        new_department_acl_rid = new_department.acl_rid if new_d_rid_in_acl == new_department.acl_rid else \
+            new_d_rid_in_acl
+
+        for employee in e_list:
+            employee_acl_rid = employee.get('e_acl_rid')
+            if employee_acl_rid == 0:
+                result.append("employee_acl_rid == 0")
+                continue
+
+            cls.remove_single_employee_from_old_department(acl, employee, result)
+
+            # 在新部门中添加员工
+            cls.add_employee_to_new_department(acl, employee_acl_rid, new_department_acl_rid, result)
+
+        return result

cmdb-api/api/lib/common_setting/employee.py

@@ -1,10 +1,10 @@
 # -*- coding:utf-8 -*-
-
+import copy
 import traceback
 from datetime import datetime
 
-import pandas as pd
+import requests
-from flask import abort
+from flask import abort, current_app
 from flask_login import current_user
 from sqlalchemy import or_, literal_column, func, not_, and_
 from werkzeug.datastructures import MultiDict
@@ -15,11 +15,25 @@ from wtforms import validators
 
 from api.extensions import db
 from api.lib.common_setting.acl import ACLManager
-from api.lib.common_setting.const import COMMON_SETTING_QUEUE, OperatorType
+from api.lib.common_setting.const import OperatorType
+from api.lib.perm.acl.const import ACL_QUEUE
 from api.lib.common_setting.resp_format import ErrFormat
-from api.lib.common_setting.utils import get_df_from_read_sql
 from api.models.common_setting import Employee, Department
 
+from api.tasks.common_setting import refresh_employee_acl_info, edit_employee_department_in_acl
+
+acl_user_columns = [
+    'email',
+    'mobile',
+    'nickname',
+    'username',
+    'password',
+    'block',
+    'avatar',
+]
+employee_pop_columns = ['password']
+can_not_edit_columns = ['email']
+
 
 def edit_acl_user(uid, **kwargs):
     user_data = {column: kwargs.get(
@@ -70,9 +84,6 @@ class EmployeeCRUD(object):
 
     @staticmethod
     def get_employee_by_uid_with_create(_uid):
-        """
-        根据 uid 获取员工信息，不存在则创建
-        """
         try:
             return EmployeeCRUD.get_employee_by_uid(_uid).to_dict()
         except Exception as e:
@@ -102,7 +113,6 @@ class EmployeeCRUD(object):
                 acl_uid=user_info['uid'],
             )
             return existed.to_dict()
-        # 创建员工
         if not user_info.get('nickname', None):
             user_info['nickname'] = user_info['name']
 
@@ -114,10 +124,25 @@ class EmployeeCRUD(object):
         employee = CreateEmployee().create_single(**data)
         return employee.to_dict()
 
+    @staticmethod
+    def add_employee_from_acl_created(**kwargs):
+        try:
+            kwargs['acl_uid'] = kwargs.pop('uid')
+            kwargs['acl_rid'] = kwargs.pop('rid')
+            kwargs['department_id'] = 0
+
+            Employee.create(
+                **kwargs
+            )
+        except Exception as e:
+            abort(400, str(e))
+
     @staticmethod
     def add(**kwargs):
         try:
-            return CreateEmployee().create_single(**kwargs)
+            res = CreateEmployee().create_single(**kwargs)
+            refresh_employee_acl_info.apply_async(args=(res.employee_id,), queue=ACL_QUEUE)
+            return res
         except Exception as e:
             abort(400, str(e))
 
@@ -144,13 +169,9 @@ class EmployeeCRUD(object):
             existed.update(**kwargs)
 
             if len(e_list) > 0:
-                from api.tasks.common_setting import edit_employee_department_in_acl
-                # fixme: comment next line
-                # edit_employee_department_in_acl(e_list, new_department_id, current_user.uid)
-
                 edit_employee_department_in_acl.apply_async(
                     args=(e_list, new_department_id, current_user.uid),
-                    queue=COMMON_SETTING_QUEUE
+                    queue=ACL_QUEUE
                 )
 
             return existed
@@ -161,7 +182,7 @@ class EmployeeCRUD(object):
     def edit_employee_by_uid(_uid, **kwargs):
         existed = EmployeeCRUD.get_employee_by_uid(_uid)
         try:
-            user = edit_acl_user(_uid, **kwargs)
+            edit_acl_user(_uid, **kwargs)
 
             for column in employee_pop_columns:
                 if kwargs.get(column):
@@ -173,9 +194,9 @@ class EmployeeCRUD(object):
 
     @staticmethod
     def change_password_by_uid(_uid, password):
-        existed = EmployeeCRUD.get_employee_by_uid(_uid)
+        EmployeeCRUD.get_employee_by_uid(_uid)
         try:
-            user = edit_acl_user(_uid, password=password)
+            edit_acl_user(_uid, password=password)
         except Exception as e:
             return abort(400, str(e))
 
@@ -209,173 +230,6 @@ class EmployeeCRUD(object):
             *criterion
         ).count()
 
-    @staticmethod
-    def import_employee(employee_list):
-        return CreateEmployee().batch_create(employee_list)
-
-    @staticmethod
-    def get_export_employee_df(block_status):
-        criterion = [
-            Employee.deleted == 0
-        ]
-        if block_status >= 0:
-            criterion.append(
-                Employee.block == block_status
-            )
-
-        query = Employee.query.with_entities(
-            Employee.employee_id,
-            Employee.nickname,
-            Employee.email,
-            Employee.sex,
-            Employee.mobile,
-            Employee.position_name,
-            Employee.last_login,
-            Employee.department_id,
-            Employee.direct_supervisor_id,
-        ).filter(*criterion)
-        df = get_df_from_read_sql(query)
-        if df.empty:
-            return df
-
-        query = Department.query.filter(
-            *criterion
-        )
-        department_df = get_df_from_read_sql(query)
-
-        def find_name(row):
-            department_id = row['department_id']
-            _df = department_df[department_df['department_id']
-                                == department_id]
-            row['department_name'] = '' if _df.empty else _df.iloc[0]['department_name']
-
-            direct_supervisor_id = row['direct_supervisor_id']
-            _df = df[df['employee_id'] == direct_supervisor_id]
-            row['nickname_direct_supervisor'] = '' if _df.empty else _df.iloc[0]['nickname']
-
-            if isinstance(row['last_login'], pd.Timestamp):
-                try:
-                    row['last_login'] = str(row['last_login'])
-                except:
-                    row['last_login'] = ''
-            else:
-                row['last_login'] = ''
-
-            return row
-
-        df = df.apply(find_name, axis=1)
-        df.drop(['department_id', 'direct_supervisor_id',
-                 'employee_id'], axis=1, inplace=True)
-        return df
-
-    @staticmethod
-    def batch_employee(column_name, column_value, employee_id_list):
-        if not column_value:
-            abort(400, ErrFormat.value_is_required)
-        if column_name in ['password', 'block']:
-            return EmployeeCRUD.batch_edit_password_or_block_column(column_name, employee_id_list, column_value, True)
-
-        elif column_name in ['department_id']:
-            return EmployeeCRUD.batch_edit_employee_department(employee_id_list, column_value)
-
-        elif column_name in [
-            'direct_supervisor_id', 'position_name'
-        ]:
-            return EmployeeCRUD.batch_edit_column(column_name, employee_id_list, column_value, False)
-
-        else:
-            abort(400, ErrFormat.column_name_not_support)
-
-    @staticmethod
-    def batch_edit_employee_department(employee_id_list, column_value):
-        err_list = []
-        employee_list = []
-        for _id in employee_id_list:
-            try:
-                existed = EmployeeCRUD.get_employee_by_id(_id)
-                employee = dict(
-                    e_acl_rid=existed.acl_rid,
-                    department_id=existed.department_id
-                )
-                employee_list.append(employee)
-                existed.update(department_id=column_value)
-
-            except Exception as e:
-                err_list.append({
-                    'employee_id': _id,
-                    'err': str(e),
-                })
-        from api.tasks.common_setting import edit_employee_department_in_acl
-        edit_employee_department_in_acl.apply_async(
-            args=(employee_list, column_value, current_user.uid),
-            queue=COMMON_SETTING_QUEUE
-        )
-        return err_list
-
-    @staticmethod
-    def batch_edit_password_or_block_column(column_name, employee_id_list, column_value, is_acl=False):
-        if column_name == 'block':
-            err_list = []
-            success_list = []
-            for _id in employee_id_list:
-                try:
-                    employee = EmployeeCRUD.edit_employee_block_column(
-                        _id, is_acl, **{column_name: column_value})
-                    success_list.append(employee)
-                except Exception as e:
-                    err_list.append({
-                        'employee_id': _id,
-                        'err': str(e),
-                    })
-            return err_list
-        else:
-            return EmployeeCRUD.batch_edit_column(column_name, employee_id_list, column_value, is_acl)
-
-    @staticmethod
-    def batch_edit_column(column_name, employee_id_list, column_value, is_acl=False):
-        err_list = []
-        for _id in employee_id_list:
-            try:
-                EmployeeCRUD.edit_employee_single_column(
-                    _id, is_acl, **{column_name: column_value})
-            except Exception as e:
-                err_list.append({
-                    'employee_id': _id,
-                    'err': str(e),
-                })
-
-        return err_list
-
-    @staticmethod
-    def edit_employee_single_column(_id, is_acl=False, **kwargs):
-        existed = EmployeeCRUD.get_employee_by_id(_id)
-
-        if is_acl:
-            return edit_acl_user(existed.acl_uid, **kwargs)
-
-        try:
-            for column in employee_pop_columns:
-                if kwargs.get(column):
-                    kwargs.pop(column)
-
-            return existed.update(**kwargs)
-        except Exception as e:
-            return abort(400, str(e))
-
-    @staticmethod
-    def edit_employee_block_column(_id, is_acl=False, **kwargs):
-        existed = EmployeeCRUD.get_employee_by_id(_id)
-        value = get_block_value(kwargs.get('block'))
-        if value is True:
-            # 判断该用户是否为 部门负责人，或者员工的直接上级
-            check_department_director_id_or_direct_supervisor_id(_id)
-
-        if is_acl:
-            kwargs['block'] = value
-            edit_acl_user(existed.acl_uid, **kwargs)
-        data = existed.to_dict()
-        return data
-
     @staticmethod
     def check_email_unique(email, _id=0):
         criterion = [
@@ -395,7 +249,7 @@ class EmployeeCRUD(object):
             raise Exception(err)
 
     @staticmethod
-    def get_employee_list_by_body(department_id, block_status, search='', order='', conditions=[], page=1,
+    def get_employee_list_by_body(department_id, block_status, search='', order='', conditions=None, page=1,
                                   page_size=10):
         criterion = [
             Employee.deleted == 0
@@ -441,7 +295,9 @@ class EmployeeCRUD(object):
         employees = []
         for r in pagination.items:
             d = r.Employee.to_dict()
-            d['department_name'] = r.Department.department_name
+            d['department_name'] = r.Department.department_name if r.Department else ''
+            if r.Employee.department_id == 0:
+                d['department_name'] = ErrFormat.company_wide
             employees.append(d)
 
         return {
@@ -461,7 +317,7 @@ class EmployeeCRUD(object):
     @staticmethod
     def get_expr_by_condition(column, operator, value, relation):
         """
-        根据conditions返回expr: (and_list, or_list)
+        get expr: (and_list, or_list)
         """
         attr = EmployeeCRUD.get_attr_by_column(column)
         # 根据operator生成条件表达式
@@ -481,7 +337,7 @@ class EmployeeCRUD(object):
             if value:
                 abort(400, ErrFormat.query_column_none_keep_value_empty.format(column))
             expr = [attr.is_(None)]
-            if column not in ["entry_date", "leave_date", "dfc_entry_date", "last_login"]:
+            if column not in ["last_login"]:
                 expr += [attr == '']
                 expr = [or_(*expr)]
         elif operator == OperatorType.IS_NOT_EMPTY:
@@ -495,7 +351,6 @@ class EmployeeCRUD(object):
         else:
             abort(400, ErrFormat.not_support_operator.format(operator))
 
-        # 根据relation生成复合条件
         if relation == "&":
             return expr, []
         elif relation == "|":
@@ -505,15 +360,16 @@ class EmployeeCRUD(object):
 
     @staticmethod
     def check_condition(column, operator, value, relation):
-        # 对于condition中column为空的，报错
         if column is None or operator is None or relation is None:
             return abort(400, ErrFormat.conditions_field_missing)
 
         if value and column == "last_login":
             try:
-                value = datetime.strptime(value, "%Y-%m-%d %H:%M:%S")
+                return datetime.strptime(value, "%Y-%m-%d %H:%M:%S")
             except Exception as e:
-                abort(400, ErrFormat.datetime_format_error.format(column))
+                err = f"{ErrFormat.datetime_format_error.format(column)}: {str(e)}"
+                abort(400, err)
+        return value
 
     @staticmethod
     def get_attr_by_column(column):
@@ -534,7 +390,7 @@ class EmployeeCRUD(object):
             relation = condition.get("relation", None)
             value = condition.get("value", None)
 
-            EmployeeCRUD.check_condition(column, operator, value, relation)
+            value = EmployeeCRUD.check_condition(column, operator, value, relation)
             a, o = EmployeeCRUD.get_expr_by_condition(
                 column, operator, value, relation)
             and_list += a
@@ -587,7 +443,7 @@ class EmployeeCRUD(object):
         employees = []
         for r in pagination.items:
             d = r.Employee.to_dict()
-            d['department_name'] = r.Department.department_name
+            d['department_name'] = r.Department.department_name if r.Department else ''
             employees.append(d)
 
         return {
@@ -622,7 +478,7 @@ class EmployeeCRUD(object):
             Employee.deleted == 0,
             Employee.block == block,
         ]
-        if type(department_id) == list:
+        if isinstance(department_id, list):
             if len(department_id) == 0:
                 return []
             else:
@@ -640,6 +496,226 @@ class EmployeeCRUD(object):
 
         return [r.to_dict() for r in results]
 
+    @staticmethod
+    def remove_bind_notice_by_uid(_platform, _uid):
+        existed = EmployeeCRUD.get_employee_by_uid(_uid)
+        employee_data = existed.to_dict()
+
+        notice_info = employee_data.get('notice_info', {})
+        notice_info = copy.deepcopy(notice_info) if notice_info else {}
+
+        notice_info[_platform] = ''
+
+        existed.update(
+            notice_info=notice_info
+        )
+        return ErrFormat.notice_remove_bind_success
+
+    @staticmethod
+    def bind_notice_by_uid(_platform, _uid):
+        existed = EmployeeCRUD.get_employee_by_uid(_uid)
+        mobile = existed.mobile
+        if not mobile or len(mobile) == 0:
+            abort(400, ErrFormat.notice_bind_err_with_empty_mobile)
+
+        from api.lib.common_setting.notice_config import NoticeConfigCRUD
+        messenger = NoticeConfigCRUD.get_messenger_url()
+        if not messenger or len(messenger) == 0:
+            abort(400, ErrFormat.notice_please_config_messenger_first)
+
+        url = f"{messenger}/v1/uid/getbyphone"
+        try:
+            payload = dict(
+                phone=mobile,
+                sender=_platform
+            )
+            res = requests.post(url, json=payload)
+            result = res.json()
+            if res.status_code != 200:
+                raise Exception(result.get('msg', ''))
+            target_id = result.get('uid', '')
+
+            employee_data = existed.to_dict()
+
+            notice_info = employee_data.get('notice_info', {})
+            notice_info = copy.deepcopy(notice_info) if notice_info else {}
+
+            notice_info[_platform] = '' if not target_id else target_id
+
+            existed.update(
+                notice_info=notice_info
+            )
+            return ErrFormat.notice_bind_success
+
+        except Exception as e:
+            return abort(400, ErrFormat.notice_bind_failed.format(str(e)))
+
+    @staticmethod
+    def get_employee_notice_by_ids(employee_ids):
+        criterion = [
+            Employee.employee_id.in_(employee_ids),
+            Employee.deleted == 0,
+        ]
+        direct_columns = ['email', 'mobile']
+        employees = Employee.query.filter(
+            *criterion
+        ).all()
+        results = []
+        for employee in employees:
+            d = employee.to_dict()
+            tmp = dict(
+                employee_id=employee.employee_id,
+            )
+            for column in direct_columns:
+                tmp[column] = d.get(column, '')
+            notice_info = d.get('notice_info', {})
+            notice_info = copy.deepcopy(notice_info) if notice_info else {}
+            tmp.update(**notice_info)
+            results.append(tmp)
+        return results
+
+    @staticmethod
+    def import_employee(employee_list):
+        res = CreateEmployee().batch_create(employee_list)
+        return res
+
+    @staticmethod
+    def batch_edit_employee_department(employee_id_list, column_value):
+        err_list = []
+        employee_list = []
+        for _id in employee_id_list:
+            try:
+                existed = EmployeeCRUD.get_employee_by_id(_id)
+                employee = dict(
+                    e_acl_rid=existed.acl_rid,
+                    department_id=existed.department_id
+                )
+                employee_list.append(employee)
+                existed.update(department_id=column_value)
+
+            except Exception as e:
+                err_list.append({
+                    'employee_id': _id,
+                    'err': str(e),
+                })
+        from api.lib.common_setting.department import EditDepartmentInACL
+        EditDepartmentInACL.edit_employee_department_in_acl(
+            employee_list, column_value, current_user.uid
+        )
+        return err_list
+
+    @staticmethod
+    def batch_edit_password_or_block_column(column_name, employee_id_list, column_value, is_acl=False):
+        if column_name == 'block':
+            err_list = []
+            success_list = []
+            for _id in employee_id_list:
+                try:
+                    employee = EmployeeCRUD.edit_employee_block_column(
+                        _id, is_acl, **{column_name: column_value})
+                    success_list.append(employee)
+                except Exception as e:
+                    err_list.append({
+                        'employee_id': _id,
+                        'err': str(e),
+                    })
+            return err_list
+        else:
+            return EmployeeCRUD.batch_edit_column(column_name, employee_id_list, column_value, is_acl)
+
+    @staticmethod
+    def batch_edit_column(column_name, employee_id_list, column_value, is_acl=False):
+        err_list = []
+        for _id in employee_id_list:
+            try:
+                EmployeeCRUD.edit_employee_single_column(
+                    _id, is_acl, **{column_name: column_value})
+            except Exception as e:
+                err_list.append({
+                    'employee_id': _id,
+                    'err': str(e),
+                })
+
+        return err_list
+
+    @staticmethod
+    def edit_employee_single_column(_id, is_acl=False, **kwargs):
+        existed = EmployeeCRUD.get_employee_by_id(_id)
+        if 'direct_supervisor_id' in kwargs.keys():
+            if kwargs['direct_supervisor_id'] == existed.direct_supervisor_id:
+                raise Exception(ErrFormat.direct_supervisor_is_not_self)
+
+        if is_acl:
+            return edit_acl_user(existed.acl_uid, **kwargs)
+
+        try:
+            for column in employee_pop_columns:
+                if kwargs.get(column):
+                    kwargs.pop(column)
+
+            return existed.update(**kwargs)
+        except Exception as e:
+            return abort(400, str(e))
+
+    @staticmethod
+    def edit_employee_block_column(_id, is_acl=False, **kwargs):
+        existed = EmployeeCRUD.get_employee_by_id(_id)
+        value = get_block_value(kwargs.get('block'))
+        if value is True:
+            check_department_director_id_or_direct_supervisor_id(_id)
+            value = 1
+        else:
+            value = 0
+
+        if is_acl:
+            kwargs['block'] = value
+            edit_acl_user(existed.acl_uid, **kwargs)
+
+        existed.update(block=value)
+        data = existed.to_dict()
+        return data
+
+    @staticmethod
+    def batch_employee(column_name, column_value, employee_id_list):
+        if column_value is None:
+            abort(400, ErrFormat.value_is_required)
+        if column_name in ['password', 'block']:
+            return EmployeeCRUD.batch_edit_password_or_block_column(column_name, employee_id_list, column_value, True)
+
+        elif column_name in ['department_id']:
+            return EmployeeCRUD.batch_edit_employee_department(employee_id_list, column_value)
+
+        elif column_name in [
+            'direct_supervisor_id', 'position_name'
+        ]:
+            return EmployeeCRUD.batch_edit_column(column_name, employee_id_list, column_value, False)
+
+        else:
+            abort(400, ErrFormat.column_name_not_support)
+
+    @staticmethod
+    def update_last_login_by_uid(uid, last_login=None):
+        employee = Employee.get_by(acl_uid=uid, first=True, to_dict=False)
+        if not employee:
+            return
+        if last_login:
+            try:
+                last_login = datetime.strptime(last_login, '%Y-%m-%d %H:%M:%S')
+            except Exception as e:
+                current_app.logger.error(f"strptime {last_login} err: {e}")
+                last_login = datetime.now()
+        else:
+            last_login = datetime.now()
+
+        try:
+            employee.update(
+                last_login=last_login
+            )
+            return last_login
+        except Exception as e:
+            current_app.logger.error(f"update last_login err: {e}")
+            return
+
 
 def get_user_map(key='uid', acl=None):
     """
@@ -654,19 +730,6 @@ def get_user_map(key='uid', acl=None):
     return data
 
 
-acl_user_columns = [
-    'email',
-    'mobile',
-    'nickname',
-    'username',
-    'password',
-    'block',
-    'avatar',
-]
-employee_pop_columns = ['password']
-can_not_edit_columns = ['email']
-
-
 def format_params(params):
     for k in ['_key', '_secret']:
         params.pop(k, None)
@@ -676,20 +739,24 @@ def format_params(params):
 class CreateEmployee(object):
     def __init__(self):
         self.acl = ACLManager()
-        self.useremail_map = {}
+        self.all_acl_users = self.acl.get_all_users()
 
-    def check_acl_user(self, email):
+    def check_acl_user(self, user_data):
-        user_info = self.useremail_map.get(email, None)
+        target_email = list(filter(lambda x: x['email'] == user_data['email'], self.all_acl_users))
-        if user_info:
+        if target_email:
-            return user_info
+            return target_email[0]
-        return None
+
+        target_username = list(filter(lambda x: x['username'] == user_data['username'], self.all_acl_users))
+        if target_username:
+            return target_username[0]
 
     def add_acl_user(self, **kwargs):
         user_data = {column: kwargs.get(
             column, '') for column in acl_user_columns if kwargs.get(column, '')}
         try:
-            existed = self.check_acl_user(user_data['email'])
+            existed = self.check_acl_user(user_data)
             if not existed:
+                user_data['add_from'] = 'common'
                 return self.acl.create_user(user_data)
             return existed
         except Exception as e:
@@ -697,8 +764,6 @@ class CreateEmployee(object):
 
     def create_single(self, **kwargs):
         EmployeeCRUD.check_email_unique(kwargs['email'])
-        self.useremail_map = self.useremail_map if self.useremail_map else get_user_map(
-            'email', self.acl)
         user = self.add_acl_user(**kwargs)
         kwargs['acl_uid'] = user['uid']
         kwargs['last_login'] = user['last_login']
@@ -711,8 +776,6 @@ class CreateEmployee(object):
         )
 
     def create_single_with_import(self, **kwargs):
-        self.useremail_map = self.useremail_map if self.useremail_map else get_user_map(
-            'email', self.acl)
         user = self.add_acl_user(**kwargs)
         kwargs['acl_uid'] = user['uid']
         kwargs['last_login'] = user['last_login']
@@ -726,11 +789,14 @@ class CreateEmployee(object):
         if existed:
             return existed
 
-        return Employee.create(
+        res = Employee.create(
             **kwargs
         )
+        refresh_employee_acl_info.apply_async(args=(res.employee_id,), queue=ACL_QUEUE)
+        return res
 
-    def get_department_by_name(self, d_name):
+    @staticmethod
+    def get_department_by_name(d_name):
         return Department.get_by(first=True, department_name=d_name)
 
     def get_end_department_id(self, department_name_list, department_name_map):
@@ -755,9 +821,6 @@ class CreateEmployee(object):
         return end_d_id
 
     def format_department_id(self, employee):
-        """
-        部门名称转化为ID，不存在则创建
-        """
         department_name_map = {}
         try:
             department_name = employee.get('department_name', '')
@@ -774,16 +837,13 @@ class CreateEmployee(object):
 
     def batch_create(self, employee_list):
         err_list = []
-        self.useremail_map = get_user_map('email', self.acl)
 
         for employee in employee_list:
             try:
-                # 获取username
                 username = employee.get('username', None)
                 if username is None:
                     employee['username'] = employee['email']
 
-                # 校验通过后获取department_id
                 employee = self.format_department_id(employee)
                 err = employee.get('err', None)
                 if err:
@@ -795,7 +855,7 @@ class CreateEmployee(object):
                     raise Exception(
                         ','.join(['{}: {}'.format(filed, ','.join(msg)) for filed, msg in form.errors.items()]))
 
-                data = self.create_single_with_import(**form.data)
+                self.create_single_with_import(**form.data)
             except Exception as e:
                 err_list.append({
                     'email': employee.get('email', ''),
@@ -809,12 +869,12 @@ class CreateEmployee(object):
 
 class EmployeeAddForm(Form):
     username = StringField(validators=[
-        validators.DataRequired(message="username不能为空"),
+        validators.DataRequired(message=ErrFormat.username_is_required),
         validators.Length(max=255),
     ])
     email = StringField(validators=[
-        validators.DataRequired(message="邮箱不能为空"),
+        validators.DataRequired(message=ErrFormat.email_is_required),
-        validators.Email(message="邮箱格式不正确"),
+        validators.Email(message=ErrFormat.email_format_error),
         validators.Length(max=255),
     ])
     password = StringField(validators=[
@@ -823,7 +883,7 @@ class EmployeeAddForm(Form):
     position_name = StringField(validators=[])
 
     nickname = StringField(validators=[
-        validators.DataRequired(message="用户名不能为空"),
+        validators.DataRequired(message=ErrFormat.nickname_is_required),
         validators.Length(max=255),
     ])
     sex = StringField(validators=[])
@@ -834,9 +894,81 @@ class EmployeeAddForm(Form):
 
 class EmployeeUpdateByUidForm(Form):
     nickname = StringField(validators=[
-        validators.DataRequired(message="用户名不能为空"),
+        validators.DataRequired(message=ErrFormat.nickname_is_required),
         validators.Length(max=255),
     ])
     avatar = StringField(validators=[])
     sex = StringField(validators=[])
     mobile = StringField(validators=[])
+
+
+class GrantEmployeeACLPerm(object):
+    """
+    Grant ACL Permission After Create New Employee
+    """
+
+    def __init__(self, acl=None):
+        self.perms_by_create_resources_type = ['read', 'grant', 'delete', 'update']
+        self.perms_by_common_grant = ['read']
+        self.resource_name_list = ['公司信息', '公司架构', '通知设置']
+
+        self.acl = acl if acl else self.check_app('backend')
+        self.resources_types = self.acl.get_all_resources_types()
+        self.resources_type = self.get_resources_type()
+        self.resource_list = self.acl.get_resource_by_type(None, None, self.resources_type['id'])
+
+    @staticmethod
+    def check_app(app_name):
+        acl = ACLManager(app_name)
+        payload = dict(
+            name=app_name,
+            description=app_name
+        )
+        app = acl.validate_app()
+        if not app:
+            acl.create_app(payload)
+        return acl
+
+    def get_resources_type(self):
+        results = list(filter(lambda t: t['name'] == '操作权限', self.resources_types['groups']))
+        if len(results) == 0:
+            payload = dict(
+                app_id=self.acl.app_name,
+                name='操作权限',
+                description='',
+                perms=self.perms_by_create_resources_type
+            )
+            resource_type = self.acl.create_resources_type(payload)
+        else:
+            resource_type = results[0]
+            resource_type_id = resource_type['id']
+            existed_perms = self.resources_types.get('id2perms', {}).get(resource_type_id, [])
+            existed_perms = [p['name'] for p in existed_perms]
+            new_perms = []
+            for perm in self.perms_by_create_resources_type:
+                if perm not in existed_perms:
+                    new_perms.append(perm)
+            if len(new_perms) > 0:
+                resource_type['perms'] = existed_perms + new_perms
+                self.acl.update_resources_type(resource_type_id, resource_type)
+
+        return resource_type
+
+    def grant(self, rid_list):
+        [self.grant_by_rid(rid) for rid in rid_list if rid > 0]
+
+    def grant_by_rid(self, rid, is_admin=False):
+        for name in self.resource_name_list:
+            resource = list(filter(lambda r: r['name'] == name, self.resource_list))
+            if len(resource) == 0:
+                payload = dict(
+                    type_id=self.resources_type['id'],
+                    app_id=self.acl.app_name,
+                    name=name,
+                )
+                resource = self.acl.create_resource(payload)
+            else:
+                resource = resource[0]
+
+            perms = self.perms_by_create_resources_type if is_admin else self.perms_by_common_grant
+            self.acl.grant_resource(rid, resource['id'], perms)

cmdb-api/api/lib/common_setting/notice_config.py

@@ -0,0 +1,165 @@
+import requests
+
+from api.lib.common_setting.const import BotNameMap
+from api.lib.common_setting.resp_format import ErrFormat
+from api.models.common_setting import NoticeConfig
+from wtforms import Form
+from wtforms import StringField
+from wtforms import validators
+from flask import abort, current_app
+
+
+class NoticeConfigCRUD(object):
+
+    @staticmethod
+    def add_notice_config(**kwargs):
+        platform = kwargs.get('platform')
+        NoticeConfigCRUD.check_platform(platform)
+        info = kwargs.get('info', {})
+        if 'name' not in info:
+            info['name'] = platform
+        kwargs['info'] = info
+        try:
+            NoticeConfigCRUD.update_messenger_config(**info)
+            res = NoticeConfig.create(
+                **kwargs
+            )
+            return res
+
+        except Exception as e:
+            return abort(400, str(e))
+
+    @staticmethod
+    def check_platform(platform):
+        NoticeConfig.get_by(first=True, to_dict=False, platform=platform) and \
+        abort(400, ErrFormat.notice_platform_existed.format(platform))
+
+    @staticmethod
+    def edit_notice_config(_id, **kwargs):
+        existed = NoticeConfigCRUD.get_notice_config_by_id(_id)
+        try:
+            info = kwargs.get('info', {})
+            if 'name' not in info:
+                info['name'] = existed.platform
+            kwargs['info'] = info
+            NoticeConfigCRUD.update_messenger_config(**info)
+
+            res = existed.update(**kwargs)
+            return res
+        except Exception as e:
+            return abort(400, str(e))
+
+    @staticmethod
+    def get_messenger_url():
+        from api.lib.common_setting.company_info import CompanyInfoCache
+        com_info = CompanyInfoCache.get()
+        if not com_info:
+            return
+        messenger = com_info.get('messenger', '')
+        if len(messenger) == 0:
+            return
+        if messenger[-1] == '/':
+            messenger = messenger[:-1]
+        return messenger
+
+    @staticmethod
+    def update_messenger_config(**kwargs):
+        try:
+            messenger = NoticeConfigCRUD.get_messenger_url()
+            if not messenger or len(messenger) == 0:
+                raise Exception(ErrFormat.notice_please_config_messenger_first)
+
+            url = f"{messenger}/v1/senders"
+            name = kwargs.get('name')
+            bot_list = kwargs.pop('bot', None)
+            for k, v in kwargs.items():
+                if isinstance(v, bool):
+                    kwargs[k] = 'true' if v else 'false'
+                else:
+                    kwargs[k] = str(v)
+
+            payload = {name: [kwargs]}
+            current_app.logger.info(f"update_messenger_config: {url}, {payload}")
+            res = requests.put(url, json=payload, timeout=2)
+            current_app.logger.info(f"update_messenger_config: {res.status_code}, {res.text}")
+
+            if not bot_list or len(bot_list) == 0:
+                return
+            bot_name = BotNameMap.get(name)
+            payload = {bot_name: bot_list}
+            current_app.logger.info(f"update_messenger_config: {url}, {payload}")
+            bot_res = requests.put(url, json=payload, timeout=2)
+            current_app.logger.info(f"update_messenger_config: {bot_res.status_code}, {bot_res.text}")
+
+        except Exception as e:
+            return abort(400, str(e))
+
+    @staticmethod
+    def get_notice_config_by_id(_id):
+        return NoticeConfig.get_by(first=True, to_dict=False, id=_id) or \
+            abort(400,
+                  ErrFormat.notice_not_existed.format(_id))
+
+    @staticmethod
+    def get_all():
+        return NoticeConfig.get_by(to_dict=True)
+
+    @staticmethod
+    def test_send_email(receive_address, **kwargs):
+        messenger = NoticeConfigCRUD.get_messenger_url()
+        if not messenger or len(messenger) == 0:
+            abort(400, ErrFormat.notice_please_config_messenger_first)
+        url = f"{messenger}/v1/message"
+
+        recipient_email = receive_address
+
+        subject = 'Test Email'
+        body = 'This is a test email'
+        payload = {
+            "sender": 'email',
+            "msgtype": "text/plain",
+            "title": subject,
+            "content": body,
+            "tos": [recipient_email],
+        }
+        current_app.logger.info(f"test_send_email: {url}, {payload}")
+        response = requests.post(url, json=payload)
+        if response.status_code != 200:
+            abort(400, response.text)
+
+        return 1
+
+    @staticmethod
+    def get_app_bot():
+        result = []
+        for notice_app in NoticeConfig.get_by(to_dict=False):
+            if notice_app.platform in ['email']:
+                continue
+            info = notice_app.info
+            name = info.get('name', '')
+            if name not in BotNameMap:
+                continue
+            result.append(dict(
+                name=info.get('name', ''),
+                label=info.get('label', ''),
+                bot=info.get('bot', []),
+            ))
+        return result
+
+
+class NoticeConfigForm(Form):
+    platform = StringField(validators=[
+        validators.DataRequired(message="平台 不能为空"),
+        validators.Length(max=255),
+    ])
+    info = StringField(validators=[
+        validators.DataRequired(message="信息 不能为空"),
+        validators.Length(max=255),
+    ])
+
+
+class NoticeConfigUpdateForm(Form):
+    info = StringField(validators=[
+        validators.DataRequired(message="信息 不能为空"),
+        validators.Length(max=255),
+    ])

cmdb-api/api/lib/common_setting/resp_format.py

@@ -1,51 +1,84 @@
 # -*- coding:utf-8 -*-
+from flask_babel import lazy_gettext as _l
 
 from api.lib.resp_format import CommonErrFormat
 
 
 class ErrFormat(CommonErrFormat):
-    company_info_is_already_existed = "公司信息已存在！无法创建"
+    company_info_is_already_existed = _l("Company info already existed")  # 公司信息已存在！无法创建
 
-    no_file_part = "没有文件部分"
+    no_file_part = _l("No file part")  # 没有文件部分
-    file_is_required = "文件是必须的"
+    file_is_required = _l("File is required")  # 文件是必须的
+    file_not_found = _l("File not found")  # 文件不存在
+    file_type_not_allowed = _l("File type not allowed")  # 文件类型不允许
+    upload_failed = _l("Upload failed: {}")  # 上传失败: {}
 
-    direct_supervisor_is_not_self = "直属上级不能是自己"
+    direct_supervisor_is_not_self = _l("Direct supervisor is not self")  # 直属上级不能是自己
-    parent_department_is_not_self = "上级部门不能是自己"
+    parent_department_is_not_self = _l("Parent department is not self")  # 上级部门不能是自己
-    employee_list_is_empty = "员工列表为空"
+    employee_list_is_empty = _l("Employee list is empty")  # 员工列表为空
 
-    column_name_not_support = "不支持的列名"
+    column_name_not_support = _l("Column name not support")  # 不支持的列名
-    password_is_required = "密码不能为空"
+    password_is_required = _l("Password is required")  # 密码是必须的
-    employee_acl_rid_is_zero = "员工ACL角色ID不能为0"
+    employee_acl_rid_is_zero = _l("Employee acl rid is zero")  # 员工ACL角色ID不能为0
 
-    generate_excel_failed = "生成excel失败: {}"
+    generate_excel_failed = _l("Generate excel failed: {}")  # 生成excel失败: {}
-    rename_columns_failed = "字段转换为中文失败: {}"
+    rename_columns_failed = _l("Rename columns failed: {}")  # 重命名字段失败: {}
-    cannot_block_this_employee_is_other_direct_supervisor = "该员工是其他员工的直属上级, 不能禁用"
+    cannot_block_this_employee_is_other_direct_supervisor = _l(
-    cannot_block_this_employee_is_department_manager = "该员工是部门负责人, 不能禁用"
+        "Cannot block this employee is other direct supervisor")  # 该员工是其他员工的直属上级, 不能禁用
-    employee_id_not_found = "员工ID [{}] 不存在"
+    cannot_block_this_employee_is_department_manager = _l(
-    value_is_required = "值是必须的"
+        "Cannot block this employee is department manager")  # 该员工是部门负责人, 不能禁用
-    email_already_exists = "邮箱 [{}] 已存在"
+    employee_id_not_found = _l("Employee id [{}] not found")  # 员工ID [{}] 不存在
-    query_column_none_keep_value_empty = "查询 {} 空值时请保持value为空"
+    value_is_required = _l("Value is required")  # 值是必须的
-    not_support_operator = "不支持的操作符: {}"
+    email_already_exists = _l("Email already exists")  # 邮箱已存在
-    not_support_relation = "不支持的关系: {}"
+    query_column_none_keep_value_empty = _l("Query {} none keep value empty")  # 查询 {} 空值时请保持value为空"
-    conditions_field_missing = "conditions内元素字段缺失，请检查！"
+    not_support_operator = _l("Not support operator: {}")  # 不支持的操作符: {}
-    datetime_format_error = "{} 格式错误，应该为：%Y-%m-%d %H:%M:%S"
+    not_support_relation = _l("Not support relation: {}")  # 不支持的关系: {}
-    department_level_relation_error = "部门层级关系不正确"
+    conditions_field_missing = _l("Conditions field missing")  # conditions内元素字段缺失，请检查！
-    delete_reserved_department_name = "保留部门，无法删除！"
+    datetime_format_error = _l("Datetime format error: {}")  # {} 格式错误，应该为：%Y-%m-%d %H:%M:%S
-    department_id_is_required = "部门ID是必须的"
+    department_level_relation_error = _l("Department level relation error")  # 部门层级关系不正确
-    department_list_is_required = "部门列表是必须的"
+    delete_reserved_department_name = _l("Delete reserved department name")  # 保留部门，无法删除！
-    cannot_to_be_parent_department = "{} 不能设置为上级部门"
+    department_id_is_required = _l("Department id is required")  # 部门ID是必须的
-    department_id_not_found = "部门ID [{}] 不存在"
+    department_list_is_required = _l("Department list is required")  # 部门列表是必须的
-    parent_department_id_must_more_than_zero = "上级部门ID必须大于0"
+    cannot_to_be_parent_department = _l("{} Cannot to be parent department")  # 不能设置为上级部门
-    department_name_already_exists = "部门名称 [{}] 已存在"
+    department_id_not_found = _l("Department id [{}] not found")  # 部门ID [{}] 不存在
-    new_department_is_none = "新部门是空的"
+    parent_department_id_must_more_than_zero = _l("Parent department id must more than zero")  # 上级部门ID必须大于0
+    department_name_already_exists = _l("Department name [{}] already exists")  # 部门名称 [{}] 已存在
+    new_department_is_none = _l("New department is none")  # 新部门是空的
 
-    acl_edit_user_failed = "ACL 修改用户失败: {}"
+    acl_edit_user_failed = _l("ACL edit user failed: {}")  # ACL 修改用户失败: {}
-    acl_uid_not_found = "ACL 用户UID [{}] 不存在"
+    acl_uid_not_found = _l("ACL uid not found: {}")  # ACL 用户UID [{}] 不存在
-    acl_add_user_failed = "ACL 添加用户失败: {}"
+    acl_add_user_failed = _l("ACL add user failed: {}")  # ACL 添加用户失败: {}
-    acl_add_role_failed = "ACL 添加角色失败: {}"
+    acl_add_role_failed = _l("ACL add role failed: {}")  # ACL 添加角色失败: {}
-    acl_update_role_failed = "ACL 更新角色失败: {}"
+    acl_update_role_failed = _l("ACL update role failed: {}")  # ACL 更新角色失败: {}
-    acl_get_all_users_failed = "ACL 获取所有用户失败: {}"
+    acl_get_all_users_failed = _l("ACL get all users failed: {}")  # ACL 获取所有用户失败: {}
-    acl_remove_user_from_role_failed = "ACL 从角色中移除用户失败: {}"
+    acl_remove_user_from_role_failed = _l("ACL remove user from role failed: {}")  # ACL 从角色中移除用户失败: {}
-    acl_add_user_to_role_failed = "ACL 添加用户到角色失败: {}"
+    acl_add_user_to_role_failed = _l("ACL add user to role failed: {}")  # ACL 添加用户到角色失败: {}
-    acl_import_user_failed = "ACL 导入用户[{}]失败: {}"
+    acl_import_user_failed = _l("ACL import user failed: {}")  # ACL 导入用户失败: {}
 
+    nickname_is_required = _l("Nickname is required")  # 昵称不能为空
+    username_is_required = _l("Username is required")  # 用户名不能为空
+    email_is_required = _l("Email is required")  # 邮箱不能为空
+    email_format_error = _l("Email format error")  # 邮箱格式错误
+    email_send_timeout = _l("Email send timeout")  # 邮件发送超时
+
+    common_data_not_found = _l("Common data not found {} ")  # ID {} 找不到记录
+    common_data_already_existed = _l("Common data {} already existed")  # {} 已存在
+    notice_platform_existed = _l("Notice platform {} existed")  # {} 已存在
+    notice_not_existed = _l("Notice {} not existed")  # {} 配置项不存在
+    notice_please_config_messenger_first = _l("Notice please config messenger first")  # 请先配置messenger URL
+    notice_bind_err_with_empty_mobile = _l("Notice bind err with empty mobile")  # 绑定错误，手机号为空
+    notice_bind_failed = _l("Notice bind failed: {}")  # 绑定失败: {}
+    notice_bind_success = _l("Notice bind success")  # 绑定成功
+    notice_remove_bind_success = _l("Notice remove bind success")  # 解绑成功
+
+    not_support_test = _l("Not support test type: {}")  # 不支持的测试类型: {}
+    not_support_auth_type = _l("Not support auth type: {}")  # 不支持的认证类型: {}
+    ldap_server_connect_timeout = _l("LDAP server connect timeout")  # LDAP服务器连接超时
+    ldap_server_connect_not_available = _l("LDAP server connect not available")  # LDAP服务器连接不可用
+    ldap_test_unknown_error = _l("LDAP test unknown error: {}")  # LDAP测试未知错误: {}
+    common_data_not_support_auth_type = _l("Common data not support auth type: {}")  # 通用数据不支持auth类型: {}
+    ldap_test_username_required = _l("LDAP test username required")  # LDAP测试用户名必填
+
+    company_wide = _l("Company wide")  # 全公司
+
+    resource_no_permission = _l("No permission to access resource {}, perm {} ")  # 没有权限访问 {} 资源的 {} 权限"

cmdb-api/api/lib/common_setting/role_perm_base.py

@@ -0,0 +1,68 @@
+class OperationPermission(object):
+
+    def __init__(self, resource_perms):
+        for _r in resource_perms:
+            setattr(self, _r['page'], _r['page'])
+            for _p in _r['perms']:
+                setattr(self, _p, _p)
+
+
+class BaseApp(object):
+    resource_type_name = 'OperationPermission'
+    all_resource_perms = []
+
+    def __init__(self):
+        self.admin_name = None
+        self.roles = []
+        self.app_name = 'acl'
+        self.require_create_resource_type = self.resource_type_name
+        self.extra_create_resource_type_list = []
+
+        self.op = None
+
+    @staticmethod
+    def format_role(role_name, role_type, acl_rid, resource_perms, description=''):
+        return dict(
+            role_name=role_name,
+            role_type=role_type,
+            acl_rid=acl_rid,
+            description=description,
+            resource_perms=resource_perms,
+        )
+
+
+class CMDBApp(BaseApp):
+    all_resource_perms = [
+        {"page": "Big_Screen", "page_cn": "大屏", "perms": ["read"]},
+        {"page": "Dashboard", "page_cn": "仪表盘", "perms": ["read"]},
+        {"page": "Resource_Search", "page_cn": "资源搜索", "perms": ["read"]},
+        {"page": "Auto_Discovery_Pool", "page_cn": "自动发现池", "perms": ["read"]},
+        {"page": "My_Subscriptions", "page_cn": "我的订阅", "perms": ["read"]},
+        {"page": "Bulk_Import", "page_cn": "批量导入", "perms": ["read"]},
+        {"page": "Model_Configuration", "page_cn": "模型配置",
+         "perms": ["read", "create_CIType", "create_CIType_group", "update_CIType_group",
+                   "delete_CIType_group", "download_CIType"]},
+        {"page": "Backend_Management", "page_cn": "后台管理", "perms": ["read"]},
+        {"page": "Customized_Dashboard", "page_cn": "定制仪表盘", "perms": ["read"]},
+        {"page": "Service_Tree_Definition", "page_cn": "服务树定义", "perms": ["read"]},
+        {"page": "Model_Relationships", "page_cn": "模型关系", "perms": ["read"]},
+        {"page": "Operation_Audit", "page_cn": "操作审计", "perms": ["read"]},
+        {"page": "Relationship_Types", "page_cn": "关系类型", "perms": ["read"]},
+        {"page": "Auto_Discovery", "page_cn": "自动发现",
+         "perms": ["read", "create_plugin", "update_plugin", "delete_plugin"]
+        },
+        {"page": "TopologyView", "page_cn": "拓扑视图",
+         "perms": ["read", "create_topology_group", "update_topology_group", "delete_topology_group",
+                   "create_topology_view"],
+         },
+        {"page": "IPAM", "page_cn": "IPAM", "perms": ["read"]},
+        {"page": "DCIM", "page_cn": "数据中心", "perms": ["read"]},
+    ]
+
+    def __init__(self):
+        super().__init__()
+
+        self.admin_name = 'cmdb_admin'
+        self.app_name = 'cmdb'
+
+        self.op = OperationPermission(self.all_resource_perms)

cmdb-api/api/lib/common_setting/upload_file.py

@@ -1,11 +1,18 @@
+import base64
 import uuid
+import os
+from io import BytesIO
+
+from flask import abort, current_app
+import lz4.frame
 
 from api.lib.common_setting.utils import get_cur_time_str
+from api.models.common_setting import CommonFile
+from api.lib.common_setting.resp_format import ErrFormat
 
 
 def allowed_file(filename, allowed_extensions):
-    return '.' in filename and \
+    return '.' in filename and filename.rsplit('.', 1)[1].lower() in allowed_extensions
-           filename.rsplit('.', 1)[1].lower() in allowed_extensions
 
 
 def generate_new_file_name(name):
@@ -13,4 +20,75 @@ def generate_new_file_name(name):
     prev_name = ''.join(name.split(f".{ext}")[:-1])
     uid = str(uuid.uuid4())
     cur_str = get_cur_time_str('_')
+
     return f"{prev_name}_{cur_str}_{uid}.{ext}"
+
+
+class CommonFileCRUD:
+    @staticmethod
+    def add_file(**kwargs):
+        return CommonFile.create(**kwargs)
+
+    @staticmethod
+    def get_file(file_name, to_str=False):
+        existed = CommonFile.get_by(file_name=file_name, first=True, to_dict=False)
+        if not existed:
+            abort(400, ErrFormat.file_not_found)
+
+        uncompressed_data = lz4.frame.decompress(existed.binary)
+
+        return base64.b64encode(uncompressed_data).decode('utf-8') if to_str else BytesIO(uncompressed_data)
+
+    @staticmethod
+    def sync_file_to_db():
+        for p in ['UPLOAD_DIRECTORY_FULL']:
+            upload_path = current_app.config.get(p, None)
+            if not upload_path:
+                continue
+            for root, dirs, files in os.walk(upload_path):
+                for file in files:
+                    file_path = os.path.join(root, file)
+                    if not os.path.isfile(file_path):
+                        continue
+
+                    existed = CommonFile.get_by(file_name=file, first=True, to_dict=False)
+                    if existed:
+                        continue
+                    with open(file_path, 'rb') as f:
+                        data = f.read()
+                    compressed_data = lz4.frame.compress(data)
+                    try:
+                        CommonFileCRUD.add_file(
+                            origin_name=file,
+                            file_name=file,
+                            binary=compressed_data
+                        )
+
+                        current_app.logger.info(f'sync file {file} to db')
+                    except Exception as e:
+                        current_app.logger.error(f'sync file {file} to db error: {e}')
+
+    def get_file_binary_str(self, file_name):
+        return self.get_file(file_name, True)
+
+    def save_str_to_file(self, file_name, str_data):
+        try:
+            self.get_file(file_name)
+            current_app.logger.info(f'file {file_name} already exists')
+            return
+        except Exception as e:
+            # file not found
+            pass
+
+        bytes_data = base64.b64decode(str_data)
+        compressed_data = lz4.frame.compress(bytes_data)
+
+        try:
+            self.add_file(
+                origin_name=file_name,
+                file_name=file_name,
+                binary=compressed_data
+            )
+            current_app.logger.info(f'save_str_to_file {file_name} success')
+        except Exception as e:
+            current_app.logger.error(f"save_str_to_file error: {e}")

cmdb-api/api/lib/common_setting/utils.py

@@ -1,24 +1,12 @@
 # -*- coding:utf-8 -*-
 from datetime import datetime
-
+from flask import current_app
-import pandas as pd
+from sqlalchemy import inspect, text
-from sqlalchemy import text
+from sqlalchemy.dialects.mysql import ENUM
 
 from api.extensions import db
 
 
-def get_df_from_read_sql(query, to_dict=False):
-    bind = query.session.bind
-    query = query.statement.compile(dialect=bind.dialect if bind else None,
-                                    compile_kwargs={"literal_binds": True}).string
-    a = db.engine
-    df = pd.read_sql(sql=text(query), con=a.connect())
-
-    if to_dict:
-        return df.to_dict('records')
-    return df
-
-
 def get_cur_time_str(split_flag='-'):
     f = f"%Y{split_flag}%m{split_flag}%d{split_flag}%H{split_flag}%M{split_flag}%S{split_flag}%f"
     return datetime.now().strftime(f)[:-3]
@@ -40,3 +28,115 @@ class BaseEnum(object):
                 if not attr.startswith("_") and not callable(getattr(cls, attr))
             }
         return cls._ALL_
+
+
+class CheckNewColumn(object):
+
+    def __init__(self):
+        self.engine = db.get_engine()
+        self.inspector = inspect(self.engine)
+        self.table_names = self.inspector.get_table_names()
+
+    @staticmethod
+    def get_model_by_table_name(_table_name):
+        registry = getattr(db.Model, 'registry', None)
+        class_registry = getattr(registry, '_class_registry', None)
+        for _model in class_registry.values():
+            if hasattr(_model, '__tablename__') and _model.__tablename__ == _table_name:
+                return _model
+        return None
+
+    def run(self):
+        for table_name in self.table_names:
+            self.check_by_table(table_name)
+
+    def check_by_table(self, table_name):
+        existed_columns = self.inspector.get_columns(table_name)
+        enum_columns = []
+        existed_column_name_list = []
+        for c in existed_columns:
+            if isinstance(c['type'], ENUM):
+                enum_columns.append(c['name'])
+            existed_column_name_list.append(c['name'])
+
+        model = self.get_model_by_table_name(table_name)
+        if model is None:
+            return
+        model_columns = getattr(getattr(getattr(model, '__table__'), 'columns'), '_all_columns')
+        for column in model_columns:
+            if column.name not in existed_column_name_list:
+                add_res = self.add_new_column(table_name, column)
+                if not add_res:
+                    continue
+
+                current_app.logger.info(f"add new column [{column.name}] in table [{table_name}] success.")
+
+                if column.name in enum_columns:
+                    enum_columns.remove(column.name)
+
+                self.add_new_index(table_name, column)
+
+        if len(enum_columns) > 0:
+            self.check_enum_column(enum_columns, existed_columns, model_columns, table_name)
+
+    def add_new_column(self, target_table_name, new_column):
+        try:
+            column_type = new_column.type.compile(self.engine.dialect)
+            default_value = new_column.default.arg if new_column.default else None
+
+            sql = "ALTER TABLE " + target_table_name + " ADD COLUMN " + f"`{new_column.name}`" + " " + column_type
+            if new_column.comment:
+                sql += f" comment '{new_column.comment}'"
+
+            if column_type == 'JSON':
+                pass
+            elif default_value:
+                if column_type.startswith('VAR') or column_type.startswith('Text'):
+                    if default_value is None or len(default_value) == 0:
+                        pass
+                else:
+                    sql += f" DEFAULT {default_value}"
+
+            sql = text(sql)
+            db.session.execute(sql)
+            return True
+        except Exception as e:
+            err = f"add_new_column [{new_column.name}] to table [{target_table_name}] err: {e}"
+            current_app.logger.error(err)
+            return False
+
+    @staticmethod
+    def add_new_index(target_table_name, new_column):
+        try:
+            if new_column.index:
+                index_name = f"{target_table_name}_{new_column.name}"
+                sql = "CREATE INDEX " + f"{index_name}" + " ON " + target_table_name + " (" + new_column.name + ")"
+                db.session.execute(sql)
+                current_app.logger.info(f"add new index [{index_name}] in table [{target_table_name}] success.")
+
+            return True
+        except Exception as e:
+            err = f"add_new_index [{new_column.name}] to table [{target_table_name}] err: {e}"
+            current_app.logger.error(err)
+            return False
+
+    @staticmethod
+    def check_enum_column(enum_columns, existed_columns, model_columns, table_name):
+        for column_name in enum_columns:
+            try:
+                enum_column = list(filter(lambda x: x['name'] == column_name, existed_columns))[0]
+                old_enum_value = enum_column.get('type', {}).enums
+                target_column = list(filter(lambda x: x.name == column_name, model_columns))[0]
+                new_enum_value = target_column.type.enums
+
+                if set(old_enum_value) == set(new_enum_value):
+                    continue
+
+                enum_values_str = ','.join(["'{}'".format(value) for value in new_enum_value])
+                sql = f"ALTER TABLE {table_name} MODIFY COLUMN" + f"`{column_name}`" + f" enum({enum_values_str})"
+                db.session.execute(sql)
+                current_app.logger.info(
+                    f"modify column [{column_name}] ENUM: {new_enum_value} in table [{table_name}] success.")
+            except Exception as e:
+                current_app.logger.error(
+                    f"modify column  ENUM [{column_name}] in table [{table_name}] err: {e}")

cmdb-api/api/lib/database.py

@@ -10,14 +10,18 @@ from api.lib.exception import CommitException
 
 class FormatMixin(object):
     def to_dict(self):
-        res = dict([(k, getattr(self, k) if not isinstance(
+        res = dict()
-            getattr(self, k), (datetime.datetime, datetime.date, datetime.time)) else str(
+        for k in getattr(self, "__mapper__").c.keys():
-            getattr(self, k))) for k in getattr(self, "__mapper__").c.keys()])
+            if k in {'password', '_password', 'secret', '_secret'}:
-        # FIXME: getattr(cls, "__table__").columns  k.name
+                continue
 
-        res.pop('password', None)
+            if k.startswith('_'):
-        res.pop('_password', None)
+                k = k[1:]
-        res.pop('secret', None)
+
+            if not isinstance(getattr(self, k), (datetime.datetime, datetime.date, datetime.time)):
+                res[k] = getattr(self, k)
+            else:
+                res[k] = str(getattr(self, k))
 
         return res
     
@@ -80,17 +84,17 @@ class CRUDMixin(FormatMixin):
             db.session.rollback()
             raise CommitException(str(e))
 
-    def soft_delete(self, flush=False):
+    def soft_delete(self, flush=False, commit=True):
         setattr(self, "deleted", True)
         setattr(self, "deleted_at", datetime.datetime.now())
-        self.save(flush=flush)
+        self.save(flush=flush, commit=commit)
 
     @classmethod
     def get_by_id(cls, _id):
         if any((isinstance(_id, six.string_types) and _id.isdigit(),
                 isinstance(_id, (six.integer_types, float))), ):
             obj = getattr(cls, "query").get(int(_id))
-            if obj and not obj.deleted:
+            if obj and not getattr(obj, 'deleted', False):
                 return obj
 
     @classmethod
@@ -138,8 +142,11 @@ class CRUDMixin(FormatMixin):
         return result[0] if first and result else (None if first else result)
 
     @classmethod
-    def get_by_like(cls, to_dict=True, **kwargs):
+    def get_by_like(cls, to_dict=True, deleted=False, **kwargs):
         query = db.session.query(cls)
+        if hasattr(cls, "deleted") and deleted is not None:
+            query = query.filter(cls.deleted.is_(deleted))
+
         for k, v in kwargs.items():
             query = query.filter(getattr(cls, k).ilike('%{0}%'.format(v)))
         return [i.to_dict() if to_dict else i for i in query]

cmdb-api/api/lib/decorator.py

@@ -4,8 +4,14 @@
 from functools import wraps
 
 from flask import abort
+from flask import current_app
 from flask import request
+from sqlalchemy.exc import InvalidRequestError
+from sqlalchemy.exc import OperationalError
+from sqlalchemy.exc import PendingRollbackError
+from sqlalchemy.exc import StatementError
 
+from api.extensions import db
 from api.lib.resp_format import CommonErrFormat
 
 
@@ -55,8 +61,8 @@ def args_validate(model_cls, exclude_args=None):
                     if exclude_args and arg in exclude_args:
                         continue
 
-                    if attr.type.python_type == str and attr.type.length and \
+                    if attr.type.python_type == str and attr.type.length and (
-                            len(request.values[arg] or '') > attr.type.length:
+                            len(request.values[arg] or '') > attr.type.length):
 
                         return abort(400, CommonErrFormat.argument_str_length_limit.format(arg, attr.type.length))
                     elif attr.type.python_type in (int, float) and request.values[arg]:
@@ -70,3 +76,43 @@ def args_validate(model_cls, exclude_args=None):
         return wrapper
 
     return decorate
+
+
+def reconnect_db(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except (StatementError, OperationalError, InvalidRequestError) as e:
+            error_msg = str(e)
+            if 'Lost connection' in error_msg or 'reconnect until invalid transaction' in error_msg or \
+                    'can be emitted within this transaction' in error_msg:
+                current_app.logger.info('[reconnect_db] lost connect rollback then retry')
+                db.session.rollback()
+                return func(*args, **kwargs)
+            else:
+                raise e
+        except Exception as e:
+            raise e
+
+    return wrapper
+
+
+def _flush_db():
+    try:
+        db.session.commit()
+    except (StatementError, OperationalError, InvalidRequestError, PendingRollbackError):
+        db.session.rollback()
+
+
+def flush_db(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        _flush_db()
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+def run_flush_db():
+    _flush_db()

cmdb-api/api/lib/http_cli.py

@@ -4,21 +4,22 @@
 import hashlib
 
 import requests
-from future.moves.urllib.parse import urlparse
 from flask import abort
-from flask import g
 from flask import current_app
+from flask_login import current_user
+from future.moves.urllib.parse import urlparse
 
 
 def build_api_key(path, params):
-    g.user is not None or abort(403, u"您得登陆才能进行该操作")
+    current_user is not None or abort(403, u"您得登陆才能进行该操作")
-    key = g.user.key
+    key = current_user.key
-    secret = g.user.secret
+    secret = current_user.secret
     values = "".join([str(params[k]) for k in sorted(params.keys())
                       if params[k] is not None]) if params.keys() else ""
     _secret = "".join([path, secret, values]).encode("utf-8")
     params["_secret"] = hashlib.sha1(_secret).hexdigest()
     params["_key"] = key
+
     return params
 
 

cmdb-api/api/lib/mixin.py

@@ -1,7 +1,7 @@
 # -*- coding:utf-8 -*-
 
 
-from flask import abort
+from flask import current_app
 from sqlalchemy import func
 
 from api.extensions import db
@@ -13,25 +13,41 @@ class DBMixin(object):
     cls = None
 
     @classmethod
-    def search(cls, page, page_size, fl=None, only_query=False, reverse=False, count_query=False, **kwargs):
+    def search(cls, page, page_size, fl=None, only_query=False, reverse=False, count_query=False,
+               last_size=None, **kwargs):
         page = get_page(page)
         page_size = get_page_size(page_size)
         if fl is None:
-            query = db.session.query(cls.cls).filter(cls.cls.deleted.is_(False))
+            query = db.session.query(cls.cls)
         else:
-            query = db.session.query(*[getattr(cls.cls, i) for i in fl]).filter(cls.cls.deleted.is_(False))
+            query = db.session.query(*[getattr(cls.cls, i) for i in fl])
 
         _query = None
         if count_query:
-            _query = db.session.query(func.count(cls.cls.id)).filter(cls.cls.deleted.is_(False))
+            _query = db.session.query(func.count(cls.cls.id))
+
+        if hasattr(cls.cls, 'deleted'):
+            query = query.filter(cls.cls.deleted.is_(False))
+            if _query:
+                _query = _query.filter(cls.cls.deleted.is_(False))
 
         for k in kwargs:
             if hasattr(cls.cls, k):
-                query = query.filter(getattr(cls.cls, k) == kwargs[k])
+                if isinstance(kwargs[k], list):
-                if count_query:
+                    query = query.filter(getattr(cls.cls, k).in_(kwargs[k]))
-                    _query = _query.filter(getattr(cls.cls, k) == kwargs[k])
+                    if count_query:
+                        _query = _query.filter(getattr(cls.cls, k).in_(kwargs[k]))
+                else:
+                    if "*" in str(kwargs[k]):
+                        query = query.filter(getattr(cls.cls, k).ilike(kwargs[k].replace('*', '%')))
+                        if count_query:
+                            _query = _query.filter(getattr(cls.cls, k).ilike(kwargs[k].replace('*', '%')))
+                    else:
+                        query = query.filter(getattr(cls.cls, k) == kwargs[k])
+                        if count_query:
+                            _query = _query.filter(getattr(cls.cls, k) == kwargs[k])
 
-        if reverse:
+        if reverse in current_app.config.get('BOOL_TRUE'):
             query = query.order_by(cls.cls.id.desc())
 
         if only_query and not count_query:
@@ -40,14 +56,15 @@ class DBMixin(object):
             return _query, query
 
         numfound = query.count()
-        return numfound, [i.to_dict() if fl is None else getattr(i, '_asdict')()
+        if not last_size:
-                          for i in query.offset((page - 1) * page_size).limit(page_size)]
+            return numfound, [i.to_dict() if fl is None else getattr(i, '_asdict')()
-
+                              for i in query.offset((page - 1) * page_size).limit(page_size)]
-    def _must_be_required(self, _id):
+        else:
-        existed = self.cls.get_by_id(_id)
+            offset = numfound - last_size
-        existed or abort(404, "Factor [{}] does not exist".format(_id))
+            if offset < 0:
-
+                offset = 0
-        return existed
+            return numfound, [i.to_dict() if fl is None else getattr(i, '_asdict')()
+                              for i in query.offset(offset).limit(last_size)]
 
     def _can_add(self, **kwargs):
         raise NotImplementedError

cmdb-api/api/lib/notify.py

@@ -0,0 +1,72 @@
+# -*- coding:utf-8 -*-
+
+import json
+
+import requests
+import six
+from flask import current_app
+from jinja2 import Template
+from markdownify import markdownify as md
+
+from api.lib.common_setting.notice_config import NoticeConfigCRUD
+from api.lib.mail import send_mail
+
+
+def _request_messenger(subject, body, tos, sender, payload):
+    params = dict(sender=sender, title=subject,
+                  tos=[to[sender] for to in tos if to.get(sender)])
+
+    if not params['tos']:
+        raise Exception("no receivers")
+
+    flat_tos = []
+    for i in params['tos']:
+        if i.strip():
+            to = Template(i).render(payload)
+            if isinstance(to, list):
+                flat_tos.extend(to)
+            elif isinstance(to, six.string_types):
+                flat_tos.append(to)
+    params['tos'] = flat_tos
+
+    if sender == "email":
+        params['msgtype'] = 'text/html'
+        params['content'] = body
+    else:
+        params['msgtype'] = 'markdown'
+        try:
+            content = md("{}\n{}".format(subject or '', body or ''))
+        except Exception as e:
+            current_app.logger.warning("html2markdown failed: {}".format(e))
+            content = "{}\n{}".format(subject or '', body or '')
+
+        params['content'] = json.dumps(dict(content=content))
+
+    url = current_app.config.get('MESSENGER_URL') or NoticeConfigCRUD.get_messenger_url()
+    if not url:
+        raise Exception("no messenger url")
+
+    if not url.endswith("message"):
+        url = "{}/v1/message".format(url)
+
+    resp = requests.post(url, json=params)
+    if resp.status_code != 200:
+        raise Exception(resp.text)
+
+    return resp.text
+
+
+def notify_send(subject, body, methods, tos, payload=None):
+    payload = payload or {}
+    payload = {k: '' if v is None else v for k, v in payload.items()}
+    subject = Template(subject).render(payload)
+    body = Template(body).render(payload)
+
+    res = ''
+    for method in methods:
+        if method == "email" and not current_app.config.get('USE_MESSENGER', True):
+            send_mail(None, [Template(to.get('email')).render(payload) for to in tos], subject, body)
+
+        res += (_request_messenger(subject, body, tos, method, payload) + "\n")
+
+    return res

cmdb-api/api/lib/perm/acl/__init__.py

@@ -6,7 +6,7 @@ from functools import wraps
 from flask import abort
 from flask import request
 
-from api.lib.perm.acl.cache import AppCache, AppAccessTokenCache
+from api.lib.perm.acl.cache import AppCache
 from api.lib.perm.acl.resp_format import ErrFormat
 
 

cmdb-api/api/lib/perm/acl/acl.py

@@ -5,8 +5,11 @@ import hashlib
 
 import requests
 import six
-from flask import current_app, g, request
+from flask import abort
-from flask import session, abort
+from flask import current_app
+from flask import request
+from flask import session
+from flask_login import current_user
 
 from api.extensions import cache
 from api.lib.perm.acl.audit import AuditCRUD
@@ -84,8 +87,8 @@ class ACLManager(object):
         if user:
             return Role.get_by(name=name, uid=user.uid, first=True, to_dict=False)
 
-        return Role.get_by(name=name, app_id=self.app_id, first=True, to_dict=False) or \
+        return (Role.get_by(name=name, app_id=self.app_id, first=True, to_dict=False) or
-               Role.get_by(name=name, first=True, to_dict=False)
+                Role.get_by(name=name, first=True, to_dict=False))
 
     def add_resource(self, name, resource_type_name=None):
         resource_type = ResourceType.get_by(name=resource_type_name, first=True, to_dict=False)
@@ -114,15 +117,15 @@ class ACLManager(object):
             if group:
                 PermissionCRUD.grant(role.id, permissions, group_id=group.id)
 
-    def grant_resource_to_role_by_rid(self, name, rid, resource_type_name=None, permissions=None):
+    def grant_resource_to_role_by_rid(self, name, rid, resource_type_name=None, permissions=None, rebuild=True):
         resource = self._get_resource(name, resource_type_name)
 
         if resource:
-            PermissionCRUD.grant(rid, permissions, resource_id=resource.id)
+            PermissionCRUD.grant(rid, permissions, resource_id=resource.id, rebuild=rebuild)
         else:
             group = self._get_resource_group(name)
             if group:
-                PermissionCRUD.grant(rid, permissions, group_id=group.id)
+                PermissionCRUD.grant(rid, permissions, group_id=group.id, rebuild=rebuild)
 
     def revoke_resource_from_role(self, name, role, resource_type_name=None, permissions=None):
         resource = self._get_resource(name, resource_type_name)
@@ -135,28 +138,28 @@ class ACLManager(object):
             if group:
                 PermissionCRUD.revoke(role.id, permissions, group_id=group.id)
 
-    def revoke_resource_from_role_by_rid(self, name, rid, resource_type_name=None, permissions=None):
+    def revoke_resource_from_role_by_rid(self, name, rid, resource_type_name=None, permissions=None, rebuild=True):
         resource = self._get_resource(name, resource_type_name)
 
         if resource:
-            PermissionCRUD.revoke(rid, permissions, resource_id=resource.id)
+            PermissionCRUD.revoke(rid, permissions, resource_id=resource.id, rebuild=rebuild)
         else:
             group = self._get_resource_group(name)
             if group:
-                PermissionCRUD.revoke(rid, permissions, group_id=group.id)
+                PermissionCRUD.revoke(rid, permissions, group_id=group.id, rebuild=rebuild)
 
-    def del_resource(self, name, resource_type_name=None):
+    def del_resource(self, name, resource_type_name=None, rebuild=True):
         resource = self._get_resource(name, resource_type_name)
         if resource:
-            ResourceCRUD.delete(resource.id)
+            return ResourceCRUD.delete(resource.id, rebuild=rebuild)
 
-    def has_permission(self, resource_name, resource_type, perm, resource_id=None):
+    def has_permission(self, resource_name, resource_type, perm, resource_id=None, rid=None):
         if is_app_admin(self.app_id):
             return True
 
-        role = self._get_role(g.user.username)
+        role = self._get_role(current_user.username) if rid is None else RoleCache.get(rid)
 
-        role or abort(404, ErrFormat.role_not_found.format(g.user.username))
+        role or abort(404, ErrFormat.role_not_found.format(current_user.username))
 
         return RoleCRUD.has_permission(role.id, resource_name, resource_type, self.app_id, perm,
                                        resource_id=resource_id)
@@ -193,9 +196,9 @@ class ACLManager(object):
         return user
 
     def get_resources(self, resource_type_name=None):
-        role = self._get_role(g.user.username)
+        role = self._get_role(current_user.username)
 
-        role or abort(404, ErrFormat.role_not_found.format(g.user.username))
+        role or abort(404, ErrFormat.role_not_found.format(current_user.username))
         rid = role.id
 
         return RoleCRUD.recursive_resources(rid, self.app_id, resource_type_name).get('resources')
@@ -215,7 +218,7 @@ def validate_permission(resources, resource_type, perm, app=None):
         return
 
     if current_app.config.get("USE_ACL"):
-        if g.user.username == "worker":
+        if current_user.username == "worker":
             return
 
         resources = [resources] if isinstance(resources, six.string_types) else resources
@@ -313,7 +316,7 @@ def role_required(role_name, app=None):
                 return
 
             if current_app.config.get("USE_ACL"):
-                if getattr(g.user, 'username', None) == "worker":
+                if getattr(current_user, 'username', None) == "worker":
                     return func(*args, **kwargs)
 
                 if role_name not in session.get("acl", {}).get("parentRoles", []) and not is_app_admin(app):

cmdb-api/api/lib/perm/acl/app.py

@@ -8,7 +8,9 @@ from flask import abort
 from flask import current_app
 
 from api.extensions import db
-from api.lib.perm.acl.audit import AuditCRUD, AuditOperateType, AuditScope
+from api.lib.perm.acl.audit import AuditCRUD
+from api.lib.perm.acl.audit import AuditOperateType
+from api.lib.perm.acl.audit import AuditScope
 from api.lib.perm.acl.resp_format import ErrFormat
 from api.models.acl import App
 

cmdb-api/api/lib/perm/acl/audit.py

@@ -1,16 +1,29 @@
 # -*- coding:utf-8 -*-
+
+import datetime
 import itertools
 import json
 from enum import Enum
 from typing import List
 
-from flask import g, has_request_context, request
+from flask import has_request_context
+from flask import request
 from flask_login import current_user
 from sqlalchemy import func
 
+from api.extensions import db
 from api.lib.perm.acl import AppCache
-from api.models.acl import AuditRoleLog, AuditResourceLog, AuditPermissionLog, AuditTriggerLog, RolePermission, \
+from api.models.acl import AuditLoginLog
-    Resource, ResourceGroup, Permission, Role, ResourceType
+from api.models.acl import AuditPermissionLog
+from api.models.acl import AuditResourceLog
+from api.models.acl import AuditRoleLog
+from api.models.acl import AuditTriggerLog
+from api.models.acl import Permission
+from api.models.acl import Resource
+from api.models.acl import ResourceGroup
+from api.models.acl import ResourceType
+from api.models.acl import Role
+from api.models.acl import RolePermission
 
 
 class AuditScope(str, Enum):
@@ -49,9 +62,7 @@ class AuditCRUD(object):
 
     @staticmethod
     def get_current_operate_uid(uid=None):
-
+        user_id = uid or (getattr(current_user, 'uid', None)) or getattr(current_user, 'user_id', None)
-        user_id = uid or (hasattr(g, 'user') and getattr(g.user, 'uid', None)) \
-                  or getattr(current_user, 'user_id', None)
 
         if has_request_context() and request.headers.get('X-User-Id'):
             _user_id = request.headers['X-User-Id']
@@ -93,11 +104,8 @@ class AuditCRUD(object):
                 criterion.append(AuditPermissionLog.operate_type == v)
 
         records = AuditPermissionLog.query.filter(
-            AuditPermissionLog.deleted == 0,
+            AuditPermissionLog.deleted == 0, *criterion).order_by(
-            *criterion) \
+            AuditPermissionLog.id.desc()).offset((page - 1) * page_size).limit(page_size).all()
-            .order_by(AuditPermissionLog.id.desc()) \
-            .offset((page - 1) * page_size) \
-            .limit(page_size).all()
 
         data = {
             'data': [r.to_dict() for r in records],
@@ -160,10 +168,8 @@ class AuditCRUD(object):
             elif k == 'operate_type':
                 criterion.append(AuditRoleLog.operate_type == v)
 
-        records = AuditRoleLog.query.filter(AuditRoleLog.deleted == 0, *criterion) \
+        records = AuditRoleLog.query.filter(AuditRoleLog.deleted == 0, *criterion).order_by(
-            .order_by(AuditRoleLog.id.desc()) \
+            AuditRoleLog.id.desc()).offset((page - 1) * page_size).limit(page_size).all()
-            .offset((page - 1) * page_size) \
-            .limit(page_size).all()
 
         data = {
             'data': [r.to_dict() for r in records],
@@ -225,11 +231,8 @@ class AuditCRUD(object):
                 criterion.append(AuditResourceLog.operate_type == v)
 
         records = AuditResourceLog.query.filter(
-            AuditResourceLog.deleted == 0,
+            AuditResourceLog.deleted == 0, *criterion).order_by(
-            *criterion) \
+            AuditResourceLog.id.desc()).offset((page - 1) * page_size).limit(page_size).all()
-            .order_by(AuditResourceLog.id.desc()) \
-            .offset((page - 1) * page_size) \
-            .limit(page_size).all()
 
         data = {
             'data': [r.to_dict() for r in records],
@@ -259,11 +262,8 @@ class AuditCRUD(object):
                 criterion.append(AuditTriggerLog.operate_type == v)
 
         records = AuditTriggerLog.query.filter(
-            AuditTriggerLog.deleted == 0,
+            AuditTriggerLog.deleted == 0, *criterion).order_by(
-            *criterion) \
+            AuditTriggerLog.id.desc()).offset((page - 1) * page_size).limit(page_size).all()
-            .order_by(AuditTriggerLog.id.desc()) \
-            .offset((page - 1) * page_size) \
-            .limit(page_size).all()
 
         data = {
             'data': [r.to_dict() for r in records],
@@ -288,6 +288,27 @@ class AuditCRUD(object):
 
         return data
 
+    @staticmethod
+    def search_login(_, q=None, page=1, page_size=10, start=None, end=None):
+        query = db.session.query(AuditLoginLog)
+
+        if start:
+            query = query.filter(AuditLoginLog.login_at >= start)
+        if end:
+            query = query.filter(AuditLoginLog.login_at <= end)
+
+        if q:
+            query = query.filter(AuditLoginLog.username == q)
+
+        records = query.order_by(
+            AuditLoginLog.id.desc()).offset((page - 1) * page_size).limit(page_size).all()
+
+        data = {
+            'data': [r.to_dict() for r in records],
+        }
+
+        return data
+
     @classmethod
     def add_role_log(cls, app_id, operate_type: AuditOperateType,
                      scope: AuditScope, link_id: int, origin: dict, current: dict, extra: dict,
@@ -353,3 +374,32 @@ class AuditCRUD(object):
         AuditTriggerLog.create(app_id=app_id, trigger_id=trigger_id, operate_uid=user_id,
                                operate_type=operate_type.value,
                                origin=origin, current=current, extra=extra, source=source.value)
+
+    @classmethod
+    def add_login_log(cls, username, is_ok, description, _id=None, logout_at=None, ip=None, browser=None):
+        if _id is not None:
+            existed = AuditLoginLog.get_by_id(_id)
+            if existed is not None:
+                existed.update(logout_at=logout_at)
+                return
+
+        payload = dict(username=username,
+                       is_ok=is_ok,
+                       description=description,
+                       logout_at=logout_at,
+                       ip=(ip or request.headers.get('X-Forwarded-For') or
+                           request.headers.get('X-Real-IP') or request.remote_addr or '').split(',')[0],
+                       browser=browser or request.headers.get('User-Agent'),
+                       channel=request.values.get('channel', 'web'),
+                       )
+
+        if logout_at is None:
+            payload['login_at'] = datetime.datetime.now()
+
+        try:
+            from api.lib.common_setting.employee import EmployeeCRUD
+            EmployeeCRUD.update_last_login_by_uid(current_user.uid)
+        except:
+            pass
+
+        return AuditLoginLog.create(**payload).id

cmdb-api/api/lib/perm/acl/cache.py

@@ -2,10 +2,12 @@
 
 
 import msgpack
+import redis_lock
 
 from api.extensions import cache
 from api.extensions import db
-from api.lib.utils import Lock
+from api.extensions import rd
+from api.lib.decorator import flush_db
 from api.models.acl import App
 from api.models.acl import Permission
 from api.models.acl import Resource
@@ -60,15 +62,15 @@ class UserCache(object):
 
     @classmethod
     def get(cls, key):
-        user = cache.get(cls.PREFIX_ID.format(key)) or \
+        user = (cache.get(cls.PREFIX_ID.format(key)) or
-               cache.get(cls.PREFIX_NAME.format(key)) or \
+                cache.get(cls.PREFIX_NAME.format(key)) or
-               cache.get(cls.PREFIX_NICK.format(key)) or \
+                cache.get(cls.PREFIX_NICK.format(key)) or
-               cache.get(cls.PREFIX_WXID.format(key))
+                cache.get(cls.PREFIX_WXID.format(key)))
         if not user:
-            user = User.query.get(key) or \
+            user = (User.query.get(key) or
-                   User.query.get_by_username(key) or \
+                    User.query.get_by_username(key) or
-                   User.query.get_by_nickname(key) or \
+                    User.query.get_by_nickname(key) or
-                   User.query.get_by_wxid(key)
+                    User.query.get_by_wxid(key))
         if user:
             cls.set(user)
 
@@ -136,14 +138,14 @@ class HasResourceRoleCache(object):
 
     @classmethod
     def add(cls, rid, app_id):
-        with Lock('HasResourceRoleCache'):
+        with redis_lock.Lock(rd.r, 'HasResourceRoleCache', expire=10):
             c = cls.get(app_id)
             c[rid] = 1
             cache.set(cls.PREFIX_KEY.format(app_id), c, timeout=0)
 
     @classmethod
     def remove(cls, rid, app_id):
-        with Lock('HasResourceRoleCache'):
+        with redis_lock.Lock(rd.r, 'HasResourceRoleCache', expire=10):
             c = cls.get(app_id)
             c.pop(rid, None)
             cache.set(cls.PREFIX_KEY.format(app_id), c, timeout=0)
@@ -156,9 +158,10 @@ class RoleRelationCache(object):
     PREFIX_RESOURCES2 = "RoleRelationResources2::id::{0}::AppId::{1}"
 
     @classmethod
-    def get_parent_ids(cls, rid, app_id):
+    def get_parent_ids(cls, rid, app_id, force=False):
         parent_ids = cache.get(cls.PREFIX_PARENT.format(rid, app_id))
-        if not parent_ids:
+        if not parent_ids or force:
+            db.session.commit()
             from api.lib.perm.acl.role import RoleRelationCRUD
             parent_ids = RoleRelationCRUD.get_parent_ids(rid, app_id)
             cache.set(cls.PREFIX_PARENT.format(rid, app_id), parent_ids, timeout=0)
@@ -166,9 +169,10 @@ class RoleRelationCache(object):
         return parent_ids
 
     @classmethod
-    def get_child_ids(cls, rid, app_id):
+    def get_child_ids(cls, rid, app_id, force=False):
         child_ids = cache.get(cls.PREFIX_CHILDREN.format(rid, app_id))
-        if not child_ids:
+        if not child_ids or force:
+            db.session.commit()
             from api.lib.perm.acl.role import RoleRelationCRUD
             child_ids = RoleRelationCRUD.get_child_ids(rid, app_id)
             cache.set(cls.PREFIX_CHILDREN.format(rid, app_id), child_ids, timeout=0)
@@ -176,14 +180,16 @@ class RoleRelationCache(object):
         return child_ids
 
     @classmethod
-    def get_resources(cls, rid, app_id):
+    def get_resources(cls, rid, app_id, force=False):
         """
         :param rid: 
         :param app_id: 
+        :param force:
         :return: {id2perms: {resource_id: [perm,]}, group2perms: {group_id: [perm, ]}}
         """
         resources = cache.get(cls.PREFIX_RESOURCES.format(rid, app_id))
-        if not resources:
+        if not resources or force:
+            db.session.commit()
             from api.lib.perm.acl.role import RoleCRUD
             resources = RoleCRUD.get_resources(rid, app_id)
             if resources['id2perms'] or resources['group2perms']:
@@ -192,9 +198,10 @@ class RoleRelationCache(object):
         return resources or {}
 
     @classmethod
-    def get_resources2(cls, rid, app_id):
+    def get_resources2(cls, rid, app_id, force=False):
         r_g = cache.get(cls.PREFIX_RESOURCES2.format(rid, app_id))
-        if not r_g:
+        if not r_g or force:
+            db.session.commit()
             res = cls.get_resources(rid, app_id)
             id2perms = res['id2perms']
             group2perms = res['group2perms']
@@ -221,24 +228,30 @@ class RoleRelationCache(object):
         return msgpack.loads(r_g, raw=False)
 
     @classmethod
+    @flush_db
     def rebuild(cls, rid, app_id):
-        cls.clean(rid, app_id)
+        if app_id is None:
-        db.session.remove()
+            app_ids = [None] + [i.id for i in App.get_by(to_dict=False)]
-
-        cls.get_parent_ids(rid, app_id)
-        cls.get_child_ids(rid, app_id)
-        resources = cls.get_resources(rid, app_id)
-        if resources.get('id2perms') or resources.get('group2perms'):
-            HasResourceRoleCache.add(rid, app_id)
         else:
-            HasResourceRoleCache.remove(rid, app_id)
+            app_ids = [app_id]
-        cls.get_resources2(rid, app_id)
+
+        for _app_id in app_ids:
+            cls.clean(rid, _app_id)
+
+            cls.get_parent_ids(rid, _app_id, force=True)
+            cls.get_child_ids(rid, _app_id, force=True)
+            resources = cls.get_resources(rid, _app_id, force=True)
+            if resources.get('id2perms') or resources.get('group2perms'):
+                HasResourceRoleCache.add(rid, _app_id)
+            else:
+                HasResourceRoleCache.remove(rid, _app_id)
+            cls.get_resources2(rid, _app_id, force=True)
 
     @classmethod
+    @flush_db
     def rebuild2(cls, rid, app_id):
         cache.delete(cls.PREFIX_RESOURCES2.format(rid, app_id))
-        db.session.remove()
+        cls.get_resources2(rid, app_id, force=True)
-        cls.get_resources2(rid, app_id)
 
     @classmethod
     def clean(cls, rid, app_id):

cmdb-api/api/lib/perm/acl/permission.py

@@ -4,7 +4,9 @@ import datetime
 from flask import abort
 
 from api.extensions import db
-from api.lib.perm.acl.audit import AuditCRUD, AuditOperateType, AuditOperateSource
+from api.lib.perm.acl.audit import AuditCRUD
+from api.lib.perm.acl.audit import AuditOperateSource
+from api.lib.perm.acl.audit import AuditOperateType
 from api.lib.perm.acl.cache import PermissionCache
 from api.lib.perm.acl.cache import RoleCache
 from api.lib.perm.acl.cache import UserCache
@@ -69,7 +71,7 @@ class PermissionCRUD(object):
 
     @classmethod
     def get_all2(cls, resource_name, resource_type_name, app_id):
-        rt = ResourceType.get_by(name=resource_type_name, first=True, to_dict=False)
+        rt = ResourceType.get_by(name=resource_type_name, app_id=app_id, first=True, to_dict=False)
         rt or abort(404, ErrFormat.resource_type_not_found.format(resource_type_name))
 
         r = Resource.get_by(name=resource_name, resource_type_id=rt.id, app_id=app_id, first=True, to_dict=False)
@@ -77,7 +79,8 @@ class PermissionCRUD(object):
         return r and cls.get_all(r.id)
 
     @staticmethod
-    def grant(rid, perms, resource_id=None, group_id=None, rebuild=True, source=AuditOperateSource.acl):
+    def grant(rid, perms, resource_id=None, group_id=None, rebuild=True,
+              source=AuditOperateSource.acl, force_update=False):
         app_id = None
         rt_id = None
 
@@ -97,15 +100,30 @@ class PermissionCRUD(object):
         elif group_id is not None:
             from api.models.acl import ResourceGroup
 
-            group = ResourceGroup.get_by_id(group_id) or \
+            group = ResourceGroup.get_by_id(group_id) or abort(
-                    abort(404, ErrFormat.resource_group_not_found.format("id={}".format(group_id)))
+                404, ErrFormat.resource_group_not_found.format("id={}".format(group_id)))
             app_id = group.app_id
             rt_id = group.resource_type_id
             if not perms:
                 perms = [i.get('name') for i in ResourceTypeCRUD.get_perms(group.resource_type_id)]
 
-        _role_permissions = []
+        if force_update:
+            revoke_role_permissions = []
+            existed_perms = RolePermission.get_by(rid=rid,
+                                                  app_id=app_id,
+                                                  group_id=group_id,
+                                                  resource_id=resource_id,
+                                                  to_dict=False)
+            for role_perm in existed_perms:
+                perm = PermissionCache.get(role_perm.perm_id, rt_id)
+                if perm and perm.name not in perms:
+                    role_perm.soft_delete()
+                    revoke_role_permissions.append(role_perm)
 
+            AuditCRUD.add_permission_log(app_id, AuditOperateType.revoke, rid, rt_id,
+                                         revoke_role_permissions, source=source)
+
+        _role_permissions = []
         for _perm in set(perms):
             perm = PermissionCache.get(_perm, rt_id)
             if not perm:
@@ -206,8 +224,8 @@ class PermissionCRUD(object):
         if resource_id is not None:
             from api.models.acl import Resource
 
-            resource = Resource.get_by_id(resource_id) or \
+            resource = Resource.get_by_id(resource_id) or abort(
-                       abort(404, ErrFormat.resource_not_found.format("id={}".format(resource_id)))
+                404, ErrFormat.resource_not_found.format("id={}".format(resource_id)))
             app_id = resource.app_id
             rt_id = resource.resource_type_id
             if not perms:
@@ -216,8 +234,8 @@ class PermissionCRUD(object):
         elif group_id is not None:
             from api.models.acl import ResourceGroup
 
-            group = ResourceGroup.get_by_id(group_id) or \
+            group = ResourceGroup.get_by_id(group_id) or abort(
-                    abort(404, ErrFormat.resource_group_not_found.format("id={}".format(group_id)))
+                404, ErrFormat.resource_group_not_found.format("id={}".format(group_id)))
             app_id = group.app_id
 
             rt_id = group.resource_type_id
@@ -272,12 +290,14 @@ class PermissionCRUD(object):
                     perm2resource.setdefault(_perm, []).append(resource_id)
             for _perm in perm2resource:
                 perm = PermissionCache.get(_perm, resource_type_id)
-                existeds = RolePermission.get_by(rid=rid,
+                if perm is None:
-                                                 app_id=app_id,
+                    continue
-                                                 perm_id=perm.id,
+                exists = RolePermission.get_by(rid=rid,
-                                                 __func_in___key_resource_id=perm2resource[_perm],
+                                               app_id=app_id,
-                                                 to_dict=False)
+                                               perm_id=perm.id,
-                for existed in existeds:
+                                               __func_in___key_resource_id=perm2resource[_perm],
+                                               to_dict=False)
+                for existed in exists:
                     existed.deleted = True
                     existed.deleted_at = datetime.datetime.now()
                     db.session.add(existed)

cmdb-api/api/lib/perm/acl/resource.py

@@ -2,10 +2,11 @@
 
 
 from flask import abort
-from flask import current_app
 
 from api.extensions import db
-from api.lib.perm.acl.audit import AuditCRUD, AuditOperateType, AuditScope
+from api.lib.perm.acl.audit import AuditCRUD
+from api.lib.perm.acl.audit import AuditOperateType
+from api.lib.perm.acl.audit import AuditScope
 from api.lib.perm.acl.cache import ResourceCache
 from api.lib.perm.acl.cache import ResourceGroupCache
 from api.lib.perm.acl.cache import UserCache
@@ -102,8 +103,8 @@ class ResourceTypeCRUD(object):
 
     @classmethod
     def delete(cls, rt_id):
-        rt = ResourceType.get_by_id(rt_id) or \
+        rt = ResourceType.get_by_id(rt_id) or abort(
-             abort(404, ErrFormat.resource_type_not_found.format("id={}".format(rt_id)))
+            404, ErrFormat.resource_type_not_found.format("id={}".format(rt_id)))
 
         Resource.get_by(resource_type_id=rt_id) and abort(400, ErrFormat.resource_type_cannot_delete)
 
@@ -125,11 +126,18 @@ class ResourceTypeCRUD(object):
         existed_ids = [i.id for i in existed]
         current_ids = []
 
+        rebuild_rids = set()
         for i in existed:
             if i.name not in perms:
-                i.soft_delete()
+                i.soft_delete(commit=False)
+                for rp in RolePermission.get_by(perm_id=i.id, to_dict=False):
+                    rp.soft_delete(commit=False)
+                    rebuild_rids.add((rp.app_id, rp.rid))
             else:
                 current_ids.append(i.id)
+        db.session.commit()
+        for _app_id, _rid in rebuild_rids:
+            role_rebuild.apply_async(args=(_rid, _app_id), queue=ACL_QUEUE)
 
         for i in perms:
             if i not in existed_names:
@@ -165,8 +173,8 @@ class ResourceGroupCRUD(object):
 
     @staticmethod
     def add(name, type_id, app_id, uid=None):
-        ResourceGroup.get_by(name=name, resource_type_id=type_id, app_id=app_id) and \
+        ResourceGroup.get_by(name=name, resource_type_id=type_id, app_id=app_id) and abort(
-        abort(400, ErrFormat.resource_group_exists.format(name))
+            400, ErrFormat.resource_group_exists.format(name))
         rg = ResourceGroup.create(name=name, resource_type_id=type_id, app_id=app_id, uid=uid)
 
         AuditCRUD.add_resource_log(app_id, AuditOperateType.create,
@@ -175,8 +183,8 @@ class ResourceGroupCRUD(object):
 
     @staticmethod
     def update(rg_id, items):
-        rg = ResourceGroup.get_by_id(rg_id) or \
+        rg = ResourceGroup.get_by_id(rg_id) or abort(
-             abort(404, ErrFormat.resource_group_not_found.format("id={}".format(rg_id)))
+            404, ErrFormat.resource_group_not_found.format("id={}".format(rg_id)))
 
         existed = ResourceGroupItems.get_by(group_id=rg_id, to_dict=False)
         existed_ids = [i.resource_id for i in existed]
@@ -196,8 +204,8 @@ class ResourceGroupCRUD(object):
 
     @staticmethod
     def delete(rg_id):
-        rg = ResourceGroup.get_by_id(rg_id) or \
+        rg = ResourceGroup.get_by_id(rg_id) or abort(
-             abort(404, ErrFormat.resource_group_not_found.format("id={}".format(rg_id)))
+            404, ErrFormat.resource_group_not_found.format("id={}".format(rg_id)))
 
         origin = rg.to_dict()
         rg.soft_delete()
@@ -258,7 +266,8 @@ class ResourceCRUD(object):
         numfound = query.count()
         res = [i.to_dict() for i in query.offset((page - 1) * page_size).limit(page_size)]
         for i in res:
-            i['user'] = UserCache.get(i['uid']).nickname if i['uid'] else ''
+            user = UserCache.get(i['uid']) if i['uid'] else ''
+            i['user'] = user and user.nickname
 
         return numfound, res
 
@@ -266,14 +275,13 @@ class ResourceCRUD(object):
     def add(cls, name, type_id, app_id, uid=None):
         type_id = cls._parse_resource_type_id(type_id, app_id)
 
-        Resource.get_by(name=name, resource_type_id=type_id, app_id=app_id) and \
+        Resource.get_by(name=name, resource_type_id=type_id, app_id=app_id) and abort(
-        abort(400, ErrFormat.resource_exists.format(name))
+            400, ErrFormat.resource_exists.format(name))
 
         r = Resource.create(name=name, resource_type_id=type_id, app_id=app_id, uid=uid)
 
         from api.tasks.acl import apply_trigger
         triggers = TriggerCRUD.match_triggers(app_id, r.name, r.resource_type_id, uid)
-        current_app.logger.info(triggers)
         for trigger in triggers:
             # auto trigger should be no uid
             apply_trigger.apply_async(args=(trigger.id,),
@@ -307,9 +315,12 @@ class ResourceCRUD(object):
         return resource
 
     @staticmethod
-    def delete(_id):
+    def delete(_id, rebuild=True, app_id=None):
         resource = Resource.get_by_id(_id) or abort(404, ErrFormat.resource_not_found.format("id={}".format(_id)))
 
+        if app_id is not None and resource.app_id != app_id:
+            return abort(404, ErrFormat.resource_not_found.format("id={}".format(_id)))
+
         origin = resource.to_dict()
         resource.soft_delete()
 
@@ -320,12 +331,15 @@ class ResourceCRUD(object):
             i.soft_delete()
             rebuilds.append((i.rid, i.app_id))
 
-        for rid, app_id in set(rebuilds):
+        if rebuild:
-            role_rebuild.apply_async(args=(rid, app_id), queue=ACL_QUEUE)
+            for rid, app_id in set(rebuilds):
+                role_rebuild.apply_async(args=(rid, app_id), queue=ACL_QUEUE)
 
         AuditCRUD.add_resource_log(resource.app_id, AuditOperateType.delete,
                                    AuditScope.resource, resource.id, origin, {}, {})
 
+        return rebuilds
+
     @classmethod
     def delete_by_name(cls, name, type_id, app_id):
         resource = Resource.get_by(name=name, resource_type_id=type_id, app_id=app_id) or abort(

cmdb-api/api/lib/perm/acl/resp_format.py

@@ -1,42 +1,50 @@
 # -*- coding:utf-8 -*-
 
+from flask_babel import lazy_gettext as _l
+
 from api.lib.resp_format import CommonErrFormat
 
 
 class ErrFormat(CommonErrFormat):
-    auth_only_with_app_token_failed = "应用 Token验证失败"
+    login_succeed = _l("login successful")  # 登录成功
-    session_invalid = "您不是应用管理员 或者 session失效(尝试一下退出重新登录)"
+    ldap_connection_failed = _l("Failed to connect to LDAP service")  # 连接LDAP服务失败
+    invalid_password = _l("Password verification failed")  # 密码验证失败
+    auth_only_with_app_token_failed = _l("Application Token verification failed")  # 应用 Token验证失败
+    # 您不是应用管理员 或者 session失效(尝试一下退出重新登录)
+    session_invalid = _l(
+        "You are not the application administrator or the session has expired (try logging out and logging in again)")
 
-    resource_type_not_found = "资源类型 {} 不存在!"
+    resource_type_not_found = _l("Resource type {} does not exist!")  # 资源类型 {} 不存在!
-    resource_type_exists = "资源类型 {} 已经存在!"
+    resource_type_exists = _l("Resource type {} already exists!")  # 资源类型 {} 已经存在!
-    resource_type_cannot_delete = "因为该类型下有资源的存在, 不能删除!"
+    # 因为该类型下有资源的存在, 不能删除!
+    resource_type_cannot_delete = _l("Because there are resources under this type, they cannot be deleted!")
 
-    user_not_found = "用户 {} 不存在!"
+    user_not_found = _l("User {} does not exist!")  # 用户 {} 不存在!
-    user_exists = "用户 {} 已经存在!"
+    user_exists = _l("User {} already exists!")  # 用户 {} 已经存在!
-    role_not_found = "角色 {} 不存在!"
+    role_not_found = _l("Role {} does not exist!")  # 角色 {} 不存在!
-    role_exists = "角色 {} 已经存在!"
+    role_exists = _l("Role {} already exists!")  # 角色 {} 已经存在!
-    global_role_not_found = "全局角色 {} 不存在!"
+    global_role_not_found = _l("Global role {} does not exist!")  # 全局角色 {} 不存在!
-    global_role_exists = "全局角色 {} 已经存在!"
+    global_role_exists = _l("Global role {} already exists!")  # 全局角色 {} 已经存在!
 
-    resource_no_permission = "您没有资源: {} 的 {} 权限"
+    resource_no_permission = _l("You do not have {} permission on resource: {}")  # 您没有资源: {} 的 {} 权限
-    admin_required = "需要管理员权限"
+    admin_required = _l("Requires administrator permissions")  # 需要管理员权限
-    role_required = "需要角色: {}"
+    role_required = _l("Requires role: {}")  # 需要角色: {}
+    # 删除用户角色, 请在 用户管理 页面操作!
+    user_role_delete_invalid = _l("To delete a user role, please operate on the User Management page!")
 
-    app_is_ready_existed = "应用 {} 已经存在"
+    app_is_ready_existed = _l("Application {} already exists")  # 应用 {} 已经存在
-    app_not_found = "应用 {} 不存在!"
+    app_not_found = _l("Application {} does not exist!")  # 应用 {} 不存在!
-    app_secret_invalid = "应用的Secret无效"
+    app_secret_invalid = _l("The Secret is invalid")  # 应用的Secret无效
 
-    resource_not_found = "资源 {} 不存在!"
+    resource_not_found = _l("Resource {} does not exist!")  # 资源 {} 不存在!
-    resource_exists = "资源 {} 已经存在!"
+    resource_exists = _l("Resource {} already exists!")  # 资源 {} 已经存在!
 
-    resource_group_not_found = "资源组 {} 不存在!"
+    resource_group_not_found = _l("Resource group {} does not exist!")  # 资源组 {} 不存在!
-    resource_group_exists = "资源组 {} 已经存在!"
+    resource_group_exists = _l("Resource group {} already exists!")  # 资源组 {} 已经存在!
 
-    inheritance_dead_loop = "继承检测到了死循环"
+    inheritance_dead_loop = _l("Inheritance detected infinite loop")  # 继承检测到了死循环
-    role_relation_not_found = "角色关系 {} 不存在!"
+    role_relation_not_found = _l("Role relationship {} does not exist!")  # 角色关系 {} 不存在!
 
-    trigger_not_found = "触发器 {} 不存在!"
+    trigger_not_found = _l("Trigger {} does not exist!")  # 触发器 {} 不存在!
-    trigger_exists = "触发器 {} 已经存在!"
+    trigger_exists = _l("Trigger {} already exists!")  # 触发器 {} 已经存在!
-    trigger_disabled = "触发器 {} 已经被禁用!"
+    trigger_disabled = _l("Trigger {} has been disabled!")  # Trigger {} has been disabled!
-
-    invalid_password = "密码不正确!"

cmdb-api/api/lib/perm/acl/role.py

@@ -1,13 +1,13 @@
 # -*- coding:utf-8 -*-
 
-
+import redis_lock
-import time
-
 import six
 from flask import abort
 from flask import current_app
+from sqlalchemy import or_
 
 from api.extensions import db
+from api.extensions import rd
 from api.lib.perm.acl.app import AppCRUD
 from api.lib.perm.acl.audit import AuditCRUD, AuditOperateType, AuditScope
 from api.lib.perm.acl.cache import AppCache
@@ -61,7 +61,9 @@ class RoleRelationCRUD(object):
 
         id2parents = {}
         for i in res:
-            id2parents.setdefault(rid2uid.get(i.child_id, i.child_id), []).append(RoleCache.get(i.parent_id).to_dict())
+            parent = RoleCache.get(i.parent_id)
+            if parent:
+                id2parents.setdefault(rid2uid.get(i.child_id, i.child_id), []).append(parent.to_dict())
 
         return id2parents
 
@@ -140,24 +142,27 @@ class RoleRelationCRUD(object):
 
     @classmethod
     def add(cls, role, parent_id, child_ids, app_id):
-        result = []
+        with redis_lock.Lock(rd.r, "ROLE_RELATION_ADD", expire=10):
-        for child_id in child_ids:
+            db.session.commit()
-            existed = RoleRelation.get_by(parent_id=parent_id, child_id=child_id, app_id=app_id)
-            if existed:
-                continue
 
-            RoleRelationCache.clean(parent_id, app_id)
+            result = []
-            RoleRelationCache.clean(child_id, app_id)
+            for child_id in child_ids:
+                existed = RoleRelation.get_by(parent_id=parent_id, child_id=child_id, app_id=app_id)
+                if existed:
+                    continue
 
-            if parent_id in cls.recursive_child_ids(child_id, app_id):
+                if parent_id in cls.recursive_child_ids(child_id, app_id):
-                return abort(400, ErrFormat.inheritance_dead_loop)
+                    return abort(400, ErrFormat.inheritance_dead_loop)
 
-            if app_id is None:
+                result.append(RoleRelation.create(parent_id=parent_id, child_id=child_id, app_id=app_id).to_dict())
-                for app in AppCRUD.get_all():
-                    if app.name != "acl":
-                        RoleRelationCache.clean(child_id, app.id)
 
-            result.append(RoleRelation.create(parent_id=parent_id, child_id=child_id, app_id=app_id).to_dict())
+                RoleRelationCache.clean(parent_id, app_id)
+                RoleRelationCache.clean(child_id, app_id)
+
+                if app_id is None:
+                    for app in AppCRUD.get_all():
+                        if app.name != "acl":
+                            RoleRelationCache.clean(child_id, app.id)
 
         AuditCRUD.add_role_log(app_id, AuditOperateType.role_relation_add,
                                AuditScope.role_relation, role.id, {}, {},
@@ -212,18 +217,15 @@ class RoleCRUD(object):
 
     @staticmethod
     def search(q, app_id, page=1, page_size=None, user_role=True, is_all=False, user_only=False):
-        query = db.session.query(Role).filter(Role.deleted.is_(False))
+        if user_only:  # only user role
-        query1 = query.filter(Role.app_id == app_id).filter(Role.uid.is_(None))
-        query2 = query.filter(Role.app_id.is_(None)).filter(Role.uid.is_(None))
-        query = query1.union(query2)
-
-        if user_role:
-            query1 = db.session.query(Role).filter(Role.deleted.is_(False)).filter(Role.uid.isnot(None))
-            query = query.union(query1)
-
-        if user_only:
             query = db.session.query(Role).filter(Role.deleted.is_(False)).filter(Role.uid.isnot(None))
 
+        else:
+            query = db.session.query(Role).filter(Role.deleted.is_(False)).filter(
+                or_(Role.app_id == app_id, Role.app_id.is_(None)))
+            if not user_role:  # only virtual role
+                query = query.filter(Role.uid.is_(None))
+
         if not is_all:
             role_ids = list(HasResourceRoleCache.get(app_id).keys())
             query = query.filter(Role.id.in_(role_ids))
@@ -272,6 +274,13 @@ class RoleCRUD(object):
         RoleCache.clean(rid)
 
         role = role.update(**kwargs)
+
+        if origin['uid'] and kwargs.get('name') and kwargs.get('name') != origin['name']:
+            from api.models.acl import User
+            user = User.get_by(uid=origin['uid'], first=True, to_dict=False)
+            if user:
+                user.update(username=kwargs['name'])
+
         AuditCRUD.add_role_log(role.app_id, AuditOperateType.update,
                                AuditScope.role, role.id, origin, role.to_dict(), {},
                                )
@@ -286,14 +295,15 @@ class RoleCRUD(object):
         return role
 
     @classmethod
-    def delete_role(cls, rid):
+    def delete_role(cls, rid, force=False):
         from api.lib.perm.acl.acl import is_admin
 
         role = Role.get_by_id(rid) or abort(404, ErrFormat.role_not_found.format("rid={}".format(rid)))
-
         if not role.app_id and not is_admin():
             return abort(403, ErrFormat.admin_required)
 
+        not force and role.uid and abort(400, ErrFormat.user_role_delete_invalid)
+
         origin = role.to_dict()
 
         child_ids = []
@@ -366,16 +376,16 @@ class RoleCRUD(object):
             resource_type_id = resource_type and resource_type.id
 
         result = dict(resources=dict(), groups=dict())
-        s = time.time()
+        # s = time.time()
         parent_ids = RoleRelationCRUD.recursive_parent_ids(rid, app_id)
-        current_app.logger.info('parent ids {0}: {1}'.format(parent_ids, time.time() - s))
+        # current_app.logger.info('parent ids {0}: {1}'.format(parent_ids, time.time() - s))
         for parent_id in parent_ids:
 
             _resources, _groups = cls._extend_resources(parent_id, resource_type_id, app_id)
-            current_app.logger.info('middle1: {0}'.format(time.time() - s))
+            # current_app.logger.info('middle1: {0}'.format(time.time() - s))
             _merge(result['resources'], _resources)
-            current_app.logger.info('middle2: {0}'.format(time.time() - s))
+            # current_app.logger.info('middle2: {0}'.format(time.time() - s))
-            current_app.logger.info(len(_groups))
+            # current_app.logger.info(len(_groups))
             if not group_flat:
                 _merge(result['groups'], _groups)
             else:
@@ -386,7 +396,7 @@ class RoleCRUD(object):
                             item.setdefault('permissions', [])
                             item['permissions'] = list(set(item['permissions'] + _groups[rg_id]['permissions']))
                             result['resources'][item['id']] = item
-        current_app.logger.info('End: {0}'.format(time.time() - s))
+        # current_app.logger.info('End: {0}'.format(time.time() - s))
 
         result['resources'] = list(result['resources'].values())
         result['groups'] = list(result['groups'].values())

cmdb-api/api/lib/perm/acl/trigger.py

@@ -6,9 +6,10 @@ import json
 import re
 from fnmatch import fnmatch
 
-from flask import abort, current_app
+from flask import abort
 
-from api.lib.perm.acl.audit import AuditCRUD, AuditOperateType
+from api.lib.perm.acl.audit import AuditCRUD
+from api.lib.perm.acl.audit import AuditOperateType
 from api.lib.perm.acl.cache import UserCache
 from api.lib.perm.acl.const import ACL_QUEUE
 from api.lib.perm.acl.resp_format import ErrFormat

cmdb-api/api/lib/perm/acl/user.py

@@ -6,10 +6,12 @@ import string
 import uuid
 
 from flask import abort
-from flask import g
+from flask_login import current_user
 
 from api.extensions import db
-from api.lib.perm.acl.audit import AuditCRUD, AuditOperateType, AuditScope
+from api.lib.perm.acl.audit import AuditCRUD
+from api.lib.perm.acl.audit import AuditOperateType
+from api.lib.perm.acl.audit import AuditScope
 from api.lib.perm.acl.cache import UserCache
 from api.lib.perm.acl.resp_format import ErrFormat
 from api.lib.perm.acl.role import RoleCRUD
@@ -39,27 +41,35 @@ class UserCRUD(object):
 
     @classmethod
     def add(cls, **kwargs):
-        existed = User.get_by(username=kwargs['username'], email=kwargs['email'])
+        add_from = kwargs.pop('add_from', None)
+        existed = User.get_by(username=kwargs['username'])
         existed and abort(400, ErrFormat.user_exists.format(kwargs['username']))
 
+        existed = User.get_by(username=kwargs['email'])
+        existed and abort(400, ErrFormat.user_exists.format(kwargs['email']))
+
         kwargs['nickname'] = kwargs.get('nickname') or kwargs['username']
         kwargs['block'] = 0
         kwargs['key'], kwargs['secret'] = cls.gen_key_secret()
 
-        user_employee = db.session.query(User).filter(User.deleted.is_(False)).order_by(
+        user_employee = db.session.query(User).filter(User.deleted.is_(False)).order_by(User.employee_id.desc()).first()
-            User.employee_id.desc()).first()
 
-        biggest_employee_id = int(float(user_employee.employee_id)) \
+        biggest_employee_id = int(float(user_employee.employee_id)) if user_employee is not None else 0
-            if user_employee is not None else 0
 
         kwargs['employee_id'] = '{0:04d}'.format(biggest_employee_id + 1)
         user = User.create(**kwargs)
 
-        RoleCRUD.add_role(user.username, uid=user.uid)
+        role = RoleCRUD.add_role(user.username, uid=user.uid)
         AuditCRUD.add_role_log(None, AuditOperateType.create,
                                AuditScope.user, user.uid, {}, user.to_dict(), {}, {}
                                )
 
+        if add_from != 'common':
+            from api.lib.common_setting.employee import EmployeeCRUD
+            payload = {column: getattr(user, column) for column in ['uid', 'username', 'nickname', 'email', 'block']}
+            payload['rid'] = role.id
+            EmployeeCRUD.add_employee_from_acl_created(**payload)
+
         return user
 
     @staticmethod
@@ -90,9 +100,9 @@ class UserCRUD(object):
     @classmethod
     def reset_key_secret(cls):
         key, secret = cls.gen_key_secret()
-        g.user.update(key=key, secret=secret)
+        current_user.update(key=key, secret=secret)
 
-        UserCache.clean(g.user)
+        UserCache.clean(current_user)
 
         return key, secret
 
@@ -103,10 +113,14 @@ class UserCRUD(object):
 
         origin = user.to_dict()
 
-        user.soft_delete()
+        user.delete()
 
         UserCache.clean(user)
 
+        role = RoleCRUD.get_by_name(user.username, app_id=None)
+        if role:
+            RoleCRUD.delete_role(role[0]['id'], force=True)
+
         AuditCRUD.add_role_log(None, AuditOperateType.delete,
                                AuditScope.user, user.uid, origin, {}, {}, {})
 

cmdb-api/api/lib/perm/auth.py

@@ -7,7 +7,6 @@ from functools import wraps
 import jwt
 from flask import abort
 from flask import current_app
-from flask import g
 from flask import request
 from flask import session
 from flask_login import login_user
@@ -52,24 +51,22 @@ def _auth_with_key():
     user, authenticated = User.query.authenticate_with_key(key, secret, req_args, path)
     if user and authenticated:
         login_user(user)
-        reset_session(user)
+        # reset_session(user)
         return True
 
     role, authenticated = Role.query.authenticate_with_key(key, secret, req_args, path)
     if role and authenticated:
-        reset_session(None, role=role.name)
+        # reset_session(None, role=role.name)
         return True
 
     return False
 
 
 def _auth_with_session():
-    if isinstance(getattr(g, 'user', None), User):
-        login_user(g.user)
-        return True
     if "acl" in session and "userName" in (session["acl"] or {}):
         login_user(UserCache.get(session["acl"]["userName"]))
         return True
+
     return False
 
 
@@ -96,6 +93,9 @@ def _auth_with_token():
 
 
 def _auth_with_ip_white_list():
+    if request.url.endswith("acl/users/info"):
+        return False
+    
     ip = request.headers.get('X-Real-IP') or request.remote_addr
     key = request.values.get('_key')
     secret = request.values.get('_secret')
@@ -108,7 +108,7 @@ def _auth_with_ip_white_list():
 
 
 def _auth_with_app_token():
-    if _auth_with_session():
+    if _auth_with_session() or _auth_with_token():
         if not is_app_admin(request.values.get('app_id')) and request.method != "GET":
             return False
         elif is_app_admin(request.values.get('app_id')):
@@ -157,7 +157,7 @@ def _auth_with_acl_token():
 
 
 def auth_required(func):
-    if request.json is not None:
+    if request.get_json(silent=True) is not None:
         setattr(request, 'values', request.json)
     else:
         setattr(request, 'values', request.values.to_dict())

cmdb-api/api/lib/perm/authentication/__init__.py

@@ -0,0 +1 @@
+# -*- coding:utf-8 -*-

cmdb-api/api/lib/perm/authentication/cas/__init__.py

@@ -15,7 +15,7 @@ try:
 except ImportError:
     from flask import _request_ctx_stack as stack
 
-from api.flask_cas import routing
+from . import routing
 
 
 class CAS(object):

cmdb-api/api/lib/perm/authentication/cas/cas_urls.py

@@ -119,4 +119,4 @@ def create_cas_validate_url(cas_url, cas_route, service, ticket,
         ('service', service),
         ('ticket', ticket),
         ('renew', renew),
-        )
+        )

cmdb-api/api/lib/perm/authentication/cas/routing.py

@@ -1,14 +1,24 @@
 # -*- coding:utf-8 -*-
-
+import datetime
-import json
+import uuid
 
 import bs4
 from flask import Blueprint
-from flask import current_app, session, request, url_for, redirect
+from flask import current_app
-from flask_login import login_user, logout_user
+from flask import redirect
+from flask import request
+from flask import session
+from flask import url_for
+from flask_login import login_user
+from flask_login import logout_user
+from six.moves.urllib.parse import urlparse
 from six.moves.urllib_request import urlopen
 
+from api.lib.common_setting.common_data import AuthenticateDataCRUD
+from api.lib.common_setting.const import AuthenticateType
+from api.lib.perm.acl.audit import AuditCRUD
 from api.lib.perm.acl.cache import UserCache
+from api.lib.perm.acl.resp_format import ErrFormat
 from .cas_urls import create_cas_login_url
 from .cas_urls import create_cas_logout_url
 from .cas_urls import create_cas_validate_url
@@ -16,6 +26,7 @@ from .cas_urls import create_cas_validate_url
 blueprint = Blueprint('cas', __name__)
 
 
+@blueprint.route('/api/cas/login')
 @blueprint.route('/api/sso/login')
 def login():
     """
@@ -29,16 +40,20 @@ def login():
     If validation was successful the logged in username is saved in
     the user's session under the key `CAS_USERNAME_SESSION_KEY`.
     """
+    config = AuthenticateDataCRUD(AuthenticateType.CAS).get()
 
     cas_token_session_key = current_app.config['CAS_TOKEN_SESSION_KEY']
     if request.values.get("next"):
         session["next"] = request.values.get("next")
 
-    _service = url_for('cas.login', _external=True, next=session["next"]) \
+    # _service = url_for('cas.login', _external=True)
-        if session.get("next") else url_for('cas.login', _external=True)
+    _service = "{}://{}{}".format(urlparse(request.referrer).scheme,
+                                  urlparse(request.referrer).netloc,
+                                  url_for('cas.login'))
+
     redirect_url = create_cas_login_url(
-        current_app.config['CAS_SERVER'],
+        config['cas_server'],
-        current_app.config['CAS_LOGIN_ROUTE'],
+        config['cas_login_route'],
         _service)
 
     if 'ticket' in request.args:
@@ -47,30 +62,38 @@ def login():
     if request.args.get('ticket'):
 
         if validate(request.args['ticket']):
-            redirect_url = session.get("next") or \
+            redirect_url = session.get("next") or config.get("cas_after_login") or "/"
-                           current_app.config.get("CAS_AFTER_LOGIN")
             username = session.get("CAS_USERNAME")
             user = UserCache.get(username)
             login_user(user)
 
             session.permanent = True
 
+            _id = AuditCRUD.add_login_log(username, True, ErrFormat.login_succeed)
+            session['LOGIN_ID'] = _id
+
         else:
             del session[cas_token_session_key]
             redirect_url = create_cas_login_url(
-                current_app.config['CAS_SERVER'],
+                config['cas_server'],
-                current_app.config['CAS_LOGIN_ROUTE'],
+                config['cas_login_route'],
                 url_for('cas.login', _external=True),
                 renew=True)
+
+            AuditCRUD.add_login_log(session.get("CAS_USERNAME"), False, ErrFormat.invalid_password)
+
     current_app.logger.info("redirect to: {0}".format(redirect_url))
     return redirect(redirect_url)
 
 
+@blueprint.route('/api/cas/logout')
 @blueprint.route('/api/sso/logout')
 def logout():
     """
     When the user accesses this route they are logged out.
     """
+    config = AuthenticateDataCRUD(AuthenticateType.CAS).get()
+    current_app.logger.info(config)
 
     cas_username_session_key = current_app.config['CAS_USERNAME_SESSION_KEY']
     cas_token_session_key = current_app.config['CAS_TOKEN_SESSION_KEY']
@@ -82,12 +105,14 @@ def logout():
     "next" in session and session.pop("next")
 
     redirect_url = create_cas_logout_url(
-        current_app.config['CAS_SERVER'],
+        config['cas_server'],
-        current_app.config['CAS_LOGOUT_ROUTE'],
+        config['cas_logout_route'],
         url_for('cas.login', _external=True, next=request.referrer))
 
     logout_user()
 
+    AuditCRUD.add_login_log(None, None, None, _id=session.get('LOGIN_ID'), logout_at=datetime.datetime.now())
+
     current_app.logger.debug('Redirecting to: {0}'.format(redirect_url))
 
     return redirect(redirect_url)
@@ -100,14 +125,15 @@ def validate(ticket):
     and the validated username is saved in the session under the
     key `CAS_USERNAME_SESSION_KEY`.
     """
+    config = AuthenticateDataCRUD(AuthenticateType.CAS).get()
 
     cas_username_session_key = current_app.config['CAS_USERNAME_SESSION_KEY']
 
     current_app.logger.debug("validating token {0}".format(ticket))
 
     cas_validate_url = create_cas_validate_url(
-        current_app.config['CAS_VALIDATE_SERVER'],
+        config['cas_validate_server'],
-        current_app.config['CAS_VALIDATE_ROUTE'],
+        config['cas_validate_route'],
         url_for('cas.login', _external=True),
         ticket)
 
@@ -115,23 +141,35 @@ def validate(ticket):
 
     try:
         response = urlopen(cas_validate_url).read()
-        ticketid = _parse_tag(response, "cas:user")
+        ticket_id = _parse_tag(response, "cas:user")
-        strs = [s.strip() for s in ticketid.split('|') if s.strip()]
+        strs = [s.strip() for s in ticket_id.split('|') if s.strip()]
         username, is_valid = None, False
         if len(strs) == 1:
             username = strs[0]
             is_valid = True
-        user_info = json.loads(_parse_tag(response, "cas:other"))
-        current_app.logger.info(user_info)
     except ValueError:
         current_app.logger.error("CAS returned unexpected result")
         is_valid = False
         return is_valid
 
     if is_valid:
-        current_app.logger.debug("valid")
+        current_app.logger.debug("{}: {}".format(cas_username_session_key, username))
         session[cas_username_session_key] = username
         user = UserCache.get(username)
+        if user is None:
+            current_app.logger.info("create user: {}".format(username))
+            from api.lib.perm.acl.user import UserCRUD
+            soup = bs4.BeautifulSoup(response)
+            cas_user_map = config.get('cas_user_map')
+            user_dict = dict()
+            for k in cas_user_map:
+                v = soup.find(cas_user_map[k]['tag'], cas_user_map[k].get('attrs', {}))
+                user_dict[k] = v and v.text or None
+            user_dict['password'] = uuid.uuid4().hex
+            if "email" not in user_dict:
+                user_dict['email'] = username
+
+            UserCRUD.add(**user_dict)
 
         from api.lib.perm.acl.acl import ACLManager
         user_info = ACLManager.get_user_info(username)
@@ -156,7 +194,7 @@ def validate(ticket):
 
 def _parse_tag(string, tag):
     """
-    Used for parsing xml.  Search string for the first occurence of
+    Used for parsing xml.  Search string for the first occurrence of
     <tag>.....</tag> and return text (stripped of leading and tailing
     whitespace) between tags.  Return "" if tag not found.
     """
@@ -164,4 +202,5 @@ def _parse_tag(string, tag):
 
     if soup.find(tag) is None:
         return ''
+
     return soup.find(tag).string.strip()