chore: release v2.4.13

feat(api): relation path search

.env

@@ -0,0 +1,6 @@
+ MYSQL_ROOT_PASSWORD='123456'
+ MYSQL_HOST='mysql'
+ MYSQL_PORT=3306
+ MYSQL_USER='cmdb'
+ MYSQL_DATABASE='cmdb'
+ MYSQL_PASSWORD='123456'

.github/workflows/docker-build-and-release.yaml

@@ -0,0 +1,79 @@
+name: docker-images-build-and-release
+
+on:
+  push:
+    branches:
+      - master
+    tags: ["v*"]
+#  pull_request:
+#    branches:
+#      - master
+
+env:
+  # Use docker.io for Docker Hub if empty
+  REGISTRY_SERVER_ADDRESS: ghcr.io/veops
+  TAG: ${{ github.sha }}
+
+jobs:
+  setup-environment:
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+  release-api-images:
+    runs-on: ubuntu-latest
+    needs: [setup-environment]
+    permissions:
+      contents: read
+      packages: write
+    timeout-minutes: 90
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+      - name: Login to GitHub Package Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and push CMDB-API Docker image
+        uses: docker/build-push-action@v6
+        with:
+          file: docker/Dockerfile-API
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ env.REGISTRY_SERVER_ADDRESS }}/cmdb-api:${{ env.TAG }}
+#  release-ui-images:
+#    runs-on: ubuntu-latest
+#    needs: [setup-environment]
+#    permissions:
+#      contents: read
+#      packages: write
+#    timeout-minutes: 90
+#    steps:
+#      - name: Checkout Repo
+#        uses: actions/checkout@v4
+#      - name: Login to GitHub Package Registry
+#        uses: docker/login-action@v2
+#        with:
+#          registry: ghcr.io
+#          username: ${{ github.repository_owner }}
+#          password: ${{ secrets.GITHUB_TOKEN }}
+#      - name: Set up QEMU
+#        uses: docker/setup-qemu-action@v3
+#      - name: Set up Docker Buildx
+#        uses: docker/setup-buildx-action@v3
+#      - name: Build and push CMDB-UI Docker image
+#        uses: docker/build-push-action@v6
+#        with:
+#          file: docker/Dockerfile-UI
+#          context: .
+#          platforms: linux/amd64,linux/arm64
+#          push: true
+#          tags: ${{ env.REGISTRY_SERVER_ADDRESS }}/cmdb-ui:${{ env.TAG }}

.gitignore

@@ -78,3 +78,4 @@ cmdb-ui/npm-debug.log*
 cmdb-ui/yarn-debug.log*
 cmdb-ui/yarn-error.log*
 cmdb-ui/package-lock.json
+start.sh

Makefile

@@ -1,6 +1,4 @@
-MYSQL_ROOT_PASSWORD ?= root
-MYSQL_PORT ?= 3306
-REDIS_PORT ?= 6379
+include ./Makefile.variable
 
 default: help
 help:  ## display this help
@@ -50,3 +48,25 @@ clean: ## remove unwanted files like .pyc's
 lint: ## check style with flake8
 	flake8 --exclude=env .
 .PHONY: lint
+
+api-docker-build:
+	export DOCKER_CLI_EXPERIMENTAL=enabled ;\
+	! ( docker buildx ls | grep multi-platform-builder ) && docker buildx create --use --platform=$(BUILD_ARCH) --name multi-platform-builder ;\
+	docker buildx build \
+    			--builder multi-platform-builder \
+    			--platform=$(BUILD_ARCH) \
+    			--tag $(REGISTRY)/cmdb-api:$(CMDB_DOCKER_VERSION)  \
+    			--tag $(REGISTRY)/cmdb-api:latest  \
+    			-f docker/Dockerfile-API \
+    			.
+
+ui-docker-build:
+	export DOCKER_CLI_EXPERIMENTAL=enabled ;\
+	! ( docker buildx ls | grep multi-platform-builder ) && docker buildx create --use --platform=$(BUILD_ARCH) --name multi-platform-builder ;\
+	docker buildx build \
+    			--builder multi-platform-builder \
+    			--platform=$(BUILD_ARCH) \
+    			--tag $(REGISTRY)/cmdb-ui:$(CMDB_DOCKER_VERSION)  \
+    			--tag $(REGISTRY)/cmdb-ui:latest  \
+    			-f docker/Dockerfile-UI \
+    			.

Makefile.variable

@@ -0,0 +1,21 @@
+SHELL := /bin/bash -o pipefail
+
+MYSQL_ROOT_PASSWORD ?= root
+MYSQL_PORT ?= 3306
+REDIS_PORT ?= 6379
+
+LATEST_TAG_DIFF:=$(shell git describe --tags --abbrev=8)
+LATEST_COMMIT:=$(VERSION)-dev-$(shell git rev-parse --short=8 HEAD)
+BUILD_ARCH ?= linux/amd64,linux/arm64
+
+# Set your version by env or using latest tags from git
+CMDB_VERSION?=$(LATEST_TAG_DIFF)
+ifeq ($(CMDB_VERSION),)
+ 	#fall back to last commit
+	CMDB_VERSION=$(LATEST_COMMIT)
+endif
+COMMIT_VERSION:=$(LATEST_COMMIT)
+CMDB_DOCKER_VERSION:=${CMDB_VERSION}
+CMDB_CHART_VERSION:=$(shell echo ${CMDB_VERSION} | sed  's/^v//g' )
+
+REGISTRY ?= local

README.md

@@ -73,7 +73,8 @@
 ## 安装
 
 ### Docker 一键快速构建
-> 方法一
+
+[//]: # (> 方法一)
 - 第一步: 先安装 Docker 环境, 以及Docker Compose (v2)
 - 第二步: 拷贝项目
 ```shell 
@@ -83,13 +84,20 @@ git clone https://github.com/veops/cmdb.git
 ```
 docker compose up -d
 ```
-> 方法二, 该方法适用于linux系统
-- 第一步： 先安装 Docker 环境, 以及Docker Compose (v2)
-- 第二步： 直接使用项目根目录下的install.sh 文件进行 `安装`、`启动`、`暂停`、`查状态`、`删除`、`卸载`
-```shell
-curl -so install.sh https://raw.githubusercontent.com/veops/cmdb/deploy_on_kylin_docker/install.sh
-sh install.sh install
-```
+
+[//]: # (> 方法二, 该方法适用于linux系统)
+
+[//]: # (- 第一步： 先安装 Docker 环境, 以及Docker Compose (v2))
+
+[//]: # (- 第二步： 直接使用项目根目录下的install.sh 文件进行 `安装`、`启动`、`暂停`、`查状态`、`删除`、`卸载`)
+
+[//]: # (```shell)
+
+[//]: # (curl -so install.sh https://raw.githubusercontent.com/veops/cmdb/deploy_on_kylin_docker/install.sh)
+
+[//]: # (sh install.sh install)
+
+[//]: # (```)
 
 ### [本地开发环境搭建](docs/local.md)
 
@@ -105,4 +113,7 @@ sh install.sh install
 
 _**欢迎关注公众号(维易科技OneOps)，关注后可加入微信群，进行产品和技术交流。**_
 
-![公众号: 维易科技OneOps](docs/images/wechat.png)
+
+<p align="center">
+  <img src="docs/images/wechat.png" alt="公众号: 维易科技OneOps" />
+</p>

cmdb-api/Pipfile

@@ -36,7 +36,7 @@ Flask-Caching = ">=1.0.0"
 environs = "==4.2.0"
 marshmallow = "==2.20.2"
 # async tasks
-celery = ">=5.3.1"
+celery = "==5.3.1"
 celery_once = "==3.0.1"
 more-itertools = "==5.0.0"
 kombu = ">=5.3.1"
@@ -67,6 +67,8 @@ colorama = ">=0.4.6"
 pycryptodomex = ">=3.19.0"
 lz4 = ">=4.3.2"
 python-magic = "==0.4.27"
+jsonpath = "==0.82.2"
+networkx = ">=3.1"
 
 [dev-packages]
 # Testing

cmdb-api/api/commands/click_cmdb.py

@@ -128,7 +128,7 @@ def cmdb_init_acl():
                 perms = [PermEnum.READ]
             elif resource_type == ResourceTypeEnum.CI_TYPE_RELATION:
                 perms = [PermEnum.ADD, PermEnum.DELETE, PermEnum.GRANT]
-            elif resource_type == ResourceTypeEnum.RELATION_VIEW:
+            elif resource_type in (ResourceTypeEnum.RELATION_VIEW, ResourceTypeEnum.TOPOLOGY_VIEW):
                 perms = [PermEnum.READ, PermEnum.UPDATE, PermEnum.DELETE, PermEnum.GRANT]
 
             ResourceTypeCRUD.add(app_id, resource_type, '', perms)
@@ -190,6 +190,7 @@ def cmdb_counter():
     login_user(UserCache.get('worker'))
 
     i = 0
+    today = datetime.date.today()
     while True:
         try:
             db.session.remove()
@@ -200,6 +201,10 @@ def cmdb_counter():
                 CMDBCounterCache.flush_adc_counter()
                 i = 0
 
+            if datetime.date.today() != today:
+                CMDBCounterCache.clear_ad_exec_history()
+                today = datetime.date.today()
+
             CMDBCounterCache.flush_sub_counter()
 
             i += 1
@@ -326,7 +331,7 @@ def cmdb_inner_secrets_init(address):
     """
     init inner secrets for password feature
     """
-    res, ok = KeyManage(backend=InnerKVManger).init()
+    res, ok = KeyManage(backend=InnerKVManger()).init()
     if not ok:
         if res.get("status") == "failed":
             KeyManage.print_response(res)
@@ -493,3 +498,64 @@ def cmdb_agent_init():
 
     click.echo("Key   : {}".format(click.style(user.key, bg='red')))
     click.echo("Secret: {}".format(click.style(user.secret, bg='red')))
+
+
+@click.command()
+@click.option(
+    '-v',
+    '--version',
+    help='input cmdb version, e.g. 2.4.6',
+    required=True,
+)
+@with_appcontext
+def cmdb_patch(version):
+    """
+    CMDB upgrade patch
+    """
+
+    version = version[1:] if version.lower().startswith("v") else version
+
+    try:
+        if version >= '2.4.6':
+
+            from api.models.cmdb import CITypeRelation
+            for cr in CITypeRelation.get_by(to_dict=False):
+                if hasattr(cr, 'parent_attr_id') and cr.parent_attr_id and not cr.parent_attr_ids:
+                    parent_attr_ids, child_attr_ids = [cr.parent_attr_id], [cr.child_attr_id]
+                    cr.update(parent_attr_ids=parent_attr_ids, child_attr_ids=child_attr_ids, commit=False)
+            db.session.commit()
+
+            from api.models.cmdb import AutoDiscoveryCIType, AutoDiscoveryCITypeRelation
+            from api.lib.cmdb.cache import CITypeCache, AttributeCache
+            for adt in AutoDiscoveryCIType.get_by(to_dict=False):
+                if adt.relation:
+                    if not AutoDiscoveryCITypeRelation.get_by(ad_type_id=adt.type_id):
+                        peer_type = CITypeCache.get(list(adt.relation.values())[0]['type_name'])
+                        peer_type_id = peer_type and peer_type.id
+                        peer_attr = AttributeCache.get(list(adt.relation.values())[0]['attr_name'])
+                        peer_attr_id = peer_attr and peer_attr.id
+                        if peer_type_id and peer_attr_id:
+                            AutoDiscoveryCITypeRelation.create(ad_type_id=adt.type_id,
+                                                               ad_key=list(adt.relation.keys())[0],
+                                                               peer_type_id=peer_type_id,
+                                                               peer_attr_id=peer_attr_id,
+                                                               commit=False)
+                if hasattr(adt, 'interval') and adt.interval and not adt.cron:
+                    adt.cron = "*/{} * * * *".format(adt.interval // 60 or 1)
+
+            db.session.commit()
+
+        if version >= "2.4.7":
+            from api.lib.cmdb.auto_discovery.const import DEFAULT_INNER
+            from api.models.cmdb import AutoDiscoveryRule
+            for i in DEFAULT_INNER:
+                existed = AutoDiscoveryRule.get_by(name=i['name'], first=True, to_dict=False)
+                if existed is not None:
+                    if "en" in i['option'] and 'en' not in (existed.option or {}):
+                        option = copy.deepcopy(existed.option)
+                        option['en'] = i['option']['en']
+                        existed.update(option=option, commit=False)
+
+            db.session.commit()
+    except Exception as e:
+        print("cmdb patch failed: {}".format(e))

cmdb-api/api/lib/cmdb/attribute.py

@@ -108,7 +108,8 @@ class AttributeManager(object):
             return []
         choice_values = choice_table.get_by(fl=["value", "option"], attr_id=attr_id)
 
-        return [[ValueTypeMap.serialize[value_type](choice_value['value']), choice_value['option']]
+        return [[ValueTypeMap.serialize[value_type](choice_value['value']), choice_value['option'] or
+                 {"label": ValueTypeMap.serialize[value_type](choice_value['value'])}]
                 for choice_value in choice_values]
 
     @staticmethod
@@ -135,6 +136,15 @@ class AttributeManager(object):
         choice_table and choice_table.get_by(attr_id=_id, only_query=True).delete()
         db.session.flush()
 
+    @classmethod
+    def get_enum_map(cls, _attr_id, _attr=None):
+        attr = AttributeCache.get(_attr_id) if _attr_id else _attr
+        if attr and attr.is_choice:
+            choice_values = cls.get_choice_values(attr.id, attr.value_type, None, None)
+            return {i[0]: i[1]['label'] for i in choice_values if i[1] and i[1].get('label')}
+
+        return {}
+
     @classmethod
     def search_attributes(cls, name=None, alias=None, page=1, page_size=None):
         """
@@ -167,24 +177,30 @@ class AttributeManager(object):
     def get_attribute_by_name(self, name):
         attr = Attribute.get_by(name=name, first=True)
         if attr.get("is_choice"):
-            attr["choice_value"] = self.get_choice_values(attr["id"], attr["value_type"],
-                                                          attr["choice_web_hook"], attr.get("choice_other"))
+            attr["choice_value"] = self.get_choice_values(attr["id"],
+                                                          attr["value_type"],
+                                                          attr["choice_web_hook"],
+                                                          attr.get("choice_other"))
 
         return attr
 
     def get_attribute_by_alias(self, alias):
         attr = Attribute.get_by(alias=alias, first=True)
         if attr.get("is_choice"):
-            attr["choice_value"] = self.get_choice_values(attr["id"], attr["value_type"],
-                                                          attr["choice_web_hook"], attr.get("choice_other"))
+            attr["choice_value"] = self.get_choice_values(attr["id"],
+                                                          attr["value_type"],
+                                                          attr["choice_web_hook"],
+                                                          attr.get("choice_other"))
 
         return attr
 
     def get_attribute_by_id(self, _id):
         attr = Attribute.get_by_id(_id).to_dict()
         if attr.get("is_choice"):
-            attr["choice_value"] = self.get_choice_values(attr["id"], attr["value_type"],
-                                                          attr["choice_web_hook"], attr.get("choice_other"))
+            attr["choice_value"] = self.get_choice_values(attr["id"],
+                                                          attr["value_type"],
+                                                          attr["choice_web_hook"],
+                                                          attr.get("choice_other"))
 
         return attr
 
@@ -229,7 +245,7 @@ class AttributeManager(object):
         is_choice = True if choice_value or kwargs.get('choice_web_hook') or kwargs.get('choice_other') else False
 
         name = kwargs.pop("name")
-        if name in BUILTIN_KEYWORDS:
+        if name in BUILTIN_KEYWORDS or kwargs.get('alias') in BUILTIN_KEYWORDS:
             return abort(400, ErrFormat.attribute_name_cannot_be_builtin)
 
         while kwargs.get('choice_other'):

cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py

@@ -1,34 +1,51 @@
 # -*- coding:utf-8 -*-
+import copy
 import datetime
 import json
+import jsonpath
 import os
-
-from api.extensions import db
-from api.lib.cmdb.auto_discovery.const import ClOUD_MAP
-from api.lib.cmdb.cache import CITypeAttributeCache
-from api.lib.cmdb.cache import CITypeCache
-from api.lib.cmdb.ci import CIManager
-from api.lib.cmdb.ci import CIRelationManager
-from api.lib.cmdb.ci_type import CITypeGroupManager
-from api.lib.cmdb.const import AutoDiscoveryType
-from api.lib.cmdb.const import PermEnum
-from api.lib.cmdb.const import ResourceTypeEnum
-from api.lib.cmdb.resp_format import ErrFormat
-from api.lib.cmdb.search import SearchError
-from api.lib.cmdb.search.ci import search
-from api.lib.mixin import DBMixin
-from api.lib.perm.acl.acl import is_app_admin
-from api.lib.perm.acl.acl import validate_permission
-from api.lib.utils import AESCrypto
-from api.models.cmdb import AutoDiscoveryCI
-from api.models.cmdb import AutoDiscoveryCIType
-from api.models.cmdb import AutoDiscoveryRule
 from flask import abort
 from flask import current_app
 from flask_login import current_user
 from sqlalchemy import func
 
+from api.extensions import db
+from api.lib.cmdb.auto_discovery.const import CLOUD_MAP
+from api.lib.cmdb.auto_discovery.const import DEFAULT_INNER
+from api.lib.cmdb.auto_discovery.const import PRIVILEGED_USERS
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.cache import AutoDiscoveryMappingCache
+from api.lib.cmdb.cache import CITypeAttributeCache
+from api.lib.cmdb.cache import CITypeCache
+from api.lib.cmdb.ci import CIManager
+from api.lib.cmdb.ci_type import CITypeGroupManager
+from api.lib.cmdb.const import AutoDiscoveryType
+from api.lib.cmdb.const import CMDB_QUEUE
+from api.lib.cmdb.const import PermEnum
+from api.lib.cmdb.const import ResourceTypeEnum
+from api.lib.cmdb.custom_dashboard import SystemConfigManager
+from api.lib.cmdb.resp_format import ErrFormat
+from api.lib.cmdb.search import SearchError
+from api.lib.cmdb.search.ci import search as ci_search
+from api.lib.common_setting.role_perm_base import CMDBApp
+from api.lib.mixin import DBMixin
+from api.lib.perm.acl.acl import ACLManager
+from api.lib.perm.acl.acl import is_app_admin
+from api.lib.perm.acl.acl import validate_permission
+from api.lib.utils import AESCrypto
+from api.models.cmdb import AutoDiscoveryAccount
+from api.models.cmdb import AutoDiscoveryCI
+from api.models.cmdb import AutoDiscoveryCIType
+from api.models.cmdb import AutoDiscoveryCITypeRelation
+from api.models.cmdb import AutoDiscoveryCounter
+from api.models.cmdb import AutoDiscoveryExecHistory
+from api.models.cmdb import AutoDiscoveryRule
+from api.models.cmdb import AutoDiscoveryRuleSyncHistory
+from api.tasks.cmdb import build_relations_for_ad_accept
+from api.tasks.cmdb import write_ad_rule_sync_history
+
 PWD = os.path.abspath(os.path.dirname(__file__))
+app_cli = CMDBApp()
 
 
 def parse_plugin_script(script):
@@ -96,14 +113,30 @@ class AutoDiscoveryRuleCRUD(DBMixin):
             else:
                 self.cls.create(**rule)
 
-    def _can_add(self, **kwargs):
+    def _can_add(self, valid=True, **kwargs):
         self.cls.get_by(name=kwargs['name']) and abort(400, ErrFormat.adr_duplicate.format(kwargs['name']))
-        if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
+        if kwargs.get('is_plugin') and kwargs.get('plugin_script') and valid:
             kwargs = check_plugin_script(**kwargs)
+            acl = ACLManager(app_cli.app_name)
+            has_perm = True
+            try:
+                if not acl.has_permission(app_cli.op.Auto_Discovery,
+                                          app_cli.resource_type_name,
+                                          app_cli.op.create_plugin) and not is_app_admin(app_cli.app_name):
+                    has_perm = False
+            except Exception:
+                if not is_app_admin(app_cli.app_name):
+                    return abort(403, ErrFormat.role_required.format(app_cli.admin_name))
+
+            if not has_perm:
+                return abort(403, ErrFormat.no_permission.format(
+                    app_cli.op.Auto_Discovery, app_cli.op.create_plugin))
+
+        kwargs['owner'] = current_user.uid
 
         return kwargs
 
-    def _can_update(self, **kwargs):
+    def _can_update(self, valid=True, **kwargs):
         existed = self.cls.get_by_id(kwargs['_id']) or abort(
             404, ErrFormat.adr_not_found.format("id={}".format(kwargs['_id'])))
 
@@ -115,6 +148,22 @@ class AutoDiscoveryRuleCRUD(DBMixin):
             if other and other.id != existed.id:
                 return abort(400, ErrFormat.adr_duplicate.format(kwargs['name']))
 
+        if existed.is_plugin and valid:
+            acl = ACLManager(app_cli.app_name)
+            has_perm = True
+            try:
+                if not acl.has_permission(app_cli.op.Auto_Discovery,
+                                          app_cli.resource_type_name,
+                                          app_cli.op.update_plugin) and not is_app_admin(app_cli.app_name):
+                    has_perm = False
+            except Exception:
+                if not is_app_admin(app_cli.app_name):
+                    return abort(403, ErrFormat.role_required.format(app_cli.admin_name))
+
+            if not has_perm:
+                return abort(403, ErrFormat.no_permission.format(
+                    app_cli.op.Auto_Discovery, app_cli.op.update_plugin))
+
         return existed
 
     def update(self, _id, **kwargs):
@@ -122,21 +171,44 @@ class AutoDiscoveryRuleCRUD(DBMixin):
         if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
             kwargs = check_plugin_script(**kwargs)
 
+            for item in AutoDiscoveryCIType.get_by(adr_id=_id, to_dict=False):
+                item.update(updated_at=datetime.datetime.now())
+
         return super(AutoDiscoveryRuleCRUD, self).update(_id, filter_none=False, **kwargs)
 
     def _can_delete(self, **kwargs):
         if AutoDiscoveryCIType.get_by(adr_id=kwargs['_id'], first=True):
             return abort(400, ErrFormat.adr_referenced)
 
-        return self._can_update(**kwargs)
+        existed = self.cls.get_by_id(kwargs['_id']) or abort(
+            404, ErrFormat.adr_not_found.format("id={}".format(kwargs['_id'])))
+
+        if existed.is_plugin:
+            acl = ACLManager(app_cli.app_name)
+            has_perm = True
+            try:
+                if not acl.has_permission(app_cli.op.Auto_Discovery,
+                                          app_cli.resource_type_name,
+                                          app_cli.op.delete_plugin) and not is_app_admin(app_cli.app_name):
+                    has_perm = False
+            except Exception:
+                if not is_app_admin(app_cli.app_name):
+                    return abort(403, ErrFormat.role_required.format(app_cli.admin_name))
+
+            if not has_perm:
+                return abort(403, ErrFormat.no_permission.format(
+                    app_cli.op.Auto_Discovery, app_cli.op.delete_plugin))
+
+        return existed
 
 
 class AutoDiscoveryCITypeCRUD(DBMixin):
     cls = AutoDiscoveryCIType
 
     @classmethod
-    def get_all(cls):
-        return cls.cls.get_by(to_dict=False)
+    def get_all(cls, type_ids=None):
+        res = cls.cls.get_by(to_dict=False)
+        return [i for i in res if type_ids is None or i.type_id in type_ids]
 
     @classmethod
     def get_by_id(cls, _id):
@@ -147,25 +219,59 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
         return cls.cls.get_by(type_id=type_id, to_dict=False)
 
     @classmethod
-    def get(cls, ci_id, oneagent_id, last_update_at=None):
+    def get_ad_attributes(cls, type_id):
+        result = []
+        adts = cls.get_by_type_id(type_id)
+        for adt in adts:
+            adr = AutoDiscoveryRuleCRUD.get_by_id(adt.adr_id)
+            if not adr:
+                continue
+            if adr.type == AutoDiscoveryType.HTTP:
+                for i in DEFAULT_INNER:
+                    if adr.name == i['name']:
+                        attrs = AutoDiscoveryHTTPManager.get_attributes(
+                            i['en'], (adt.extra_option or {}).get('category')) or []
+                        result.extend([i.get('name') for i in attrs])
+                        break
+            elif adr.type == AutoDiscoveryType.SNMP:
+                attributes = AutoDiscoverySNMPManager.get_attributes()
+                result.extend([i.get('name') for i in (attributes or [])])
+            else:
+                result.extend([i.get('name') for i in (adr.attributes or [])])
+
+        return sorted(list(set(result)))
+
+    @classmethod
+    def get(cls, ci_id, oneagent_id, oneagent_name, last_update_at=None):
+        """
+        OneAgent sync rules
+        :param ci_id:
+        :param oneagent_id:
+        :param oneagent_name:
+        :param last_update_at:
+        :return:
+        """
         result = []
         rules = cls.cls.get_by(to_dict=True)
 
         for rule in rules:
-            if rule.get('relation'):
+            if not rule['enabled']:
                 continue
 
-            if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('secret'):
-                if not (current_user.username == "cmdb_agent" or current_user.uid == rule['uid']):
+            if isinstance(rule.get("extra_option"), dict):
+                decrypt_account(rule['extra_option'], rule['uid'])
+
+                if rule['extra_option'].get('_reference'):
+                    rule['extra_option'].pop('password', None)
                     rule['extra_option'].pop('secret', None)
-                else:
-                    rule['extra_option']['secret'] = AESCrypto.decrypt(rule['extra_option']['secret'])
+                    rule['extra_option'].update(
+                        AutoDiscoveryAccountCRUD().get_config_by_id(rule['extra_option']['_reference']))
 
             if oneagent_id and rule['agent_id'] == oneagent_id:
                 result.append(rule)
             elif rule['query_expr']:
                 query = rule['query_expr'].lstrip('q').lstrip('=')
-                s = search(query, fl=['_id'], count=1000000)
+                s = ci_search(query, fl=['_id'], count=1000000)
                 try:
                     response, _, _, _, _, _ = s.search()
                 except SearchError as e:
@@ -176,25 +282,32 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
                         result.append(rule)
                         break
             elif not rule['agent_id'] and not rule['query_expr'] and rule['adr_id']:
+                try:
+                    if not int(oneagent_id, 16):  # excludes master
+                        continue
+                except Exception:
+                    pass
+
                 adr = AutoDiscoveryRuleCRUD.get_by_id(rule['adr_id'])
                 if not adr:
                     continue
                 if adr.type in (AutoDiscoveryType.SNMP, AutoDiscoveryType.HTTP):
                     continue
 
-                if not rule['updated_at']:
-                    continue
-
                 result.append(rule)
 
+        ad_rules_updated_at = (SystemConfigManager.get('ad_rules_updated_at') or {}).get('option', {}).get('v') or ""
         new_last_update_at = ""
         for i in result:
             i['adr'] = AutoDiscoveryRule.get_by_id(i['adr_id']).to_dict()
+            i['adr'].pop("attributes", None)
             __last_update_at = max([i['updated_at'] or "", i['created_at'] or "",
-                                    i['adr']['created_at'] or "", i['adr']['updated_at'] or ""])
+                                    i['adr']['created_at'] or "", i['adr']['updated_at'] or "", ad_rules_updated_at])
             if new_last_update_at < __last_update_at:
                 new_last_update_at = __last_update_at
 
+        write_ad_rule_sync_history.apply_async(args=(result, oneagent_id, oneagent_name, datetime.datetime.now()),
+                                               queue=CMDB_QUEUE)
         if not last_update_at or new_last_update_at > last_update_at:
             return result, new_last_update_at
         else:
@@ -213,7 +326,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
             agent_id = agent_id.strip()
             q = "op_duty:{0},-rd_duty:{0},oneagent_id:{1}"
 
-            s = search(q.format(current_user.username, agent_id.strip()))
+            s = ci_search(q.format(current_user.username, agent_id.strip()))
             try:
                 response, _, _, _, _, _ = s.search()
                 if response:
@@ -222,7 +335,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
                 current_app.logger.warning(e)
                 return abort(400, str(e))
 
-            s = search(q.format(current_user.nickname, agent_id.strip()))
+            s = ci_search(q.format(current_user.nickname, agent_id.strip()))
             try:
                 response, _, _, _, _, _ = s.search()
                 if response:
@@ -236,7 +349,7 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
             if query_expr.startswith('q='):
                 query_expr = query_expr[2:]
 
-            s = search(query_expr, count=1000000)
+            s = ci_search(query_expr, count=1000000)
             try:
                 response, _, _, _, _, _ = s.search()
                 for i in response:
@@ -254,19 +367,39 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
     def _can_add(**kwargs):
 
         if kwargs.get('adr_id'):
-            AutoDiscoveryRule.get_by_id(kwargs['adr_id']) or abort(
+            adr = AutoDiscoveryRule.get_by_id(kwargs['adr_id']) or abort(
                 404, ErrFormat.adr_not_found.format("id={}".format(kwargs['adr_id'])))
-            # if not adr.is_plugin:
-            #     other = self.cls.get_by(adr_id=adr.id, first=True, to_dict=False)
-            #     if other:
-            #         ci_type = CITypeCache.get(other.type_id)
-            #         return abort(400, ErrFormat.adr_default_ref_once.format(ci_type.alias))
+            if adr.type == AutoDiscoveryType.HTTP:
+                kwargs.setdefault('extra_option', dict())
+                en_name = None
+                for i in DEFAULT_INNER:
+                    if i['name'] == adr.name:
+                        en_name = i['en']
+                        break
+                if en_name and kwargs['extra_option'].get('category'):
+                    for item in CLOUD_MAP[en_name]:
+                        if item["collect_key_map"].get(kwargs['extra_option']['category']):
+                            kwargs["extra_option"]["collect_key"] = item["collect_key_map"][
+                                kwargs['extra_option']['category']]
+                            kwargs["extra_option"]["provider"] = en_name
+                            break
+
+            if adr.type == AutoDiscoveryType.COMPONENTS and kwargs.get('extra_option'):
+                for i in DEFAULT_INNER:
+                    if i['name'] == adr.name:
+                        kwargs['extra_option']['collect_key'] = i['option'].get('collect_key')
+                        break
 
         if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
             kwargs = check_plugin_script(**kwargs)
 
-        if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'):
-            kwargs['extra_option']['secret'] = AESCrypto.encrypt(kwargs['extra_option']['secret'])
+        encrypt_account(kwargs.get('extra_option'))
+
+        ci_type = CITypeCache.get(kwargs['type_id'])
+        unique = AttributeCache.get(ci_type.unique_id)
+        if unique and unique.name not in (kwargs.get('attributes') or {}).values():
+            current_app.logger.warning((unique.name, kwargs.get('attributes'), ci_type.alias))
+            return abort(400, ErrFormat.ad_not_unique_key.format(unique.name))
 
         kwargs['uid'] = current_user.uid
 
@@ -276,11 +409,44 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
         existed = self.cls.get_by_id(kwargs['_id']) or abort(
             404, ErrFormat.ad_not_found.format("id={}".format(kwargs['_id'])))
 
-        self.__valid_exec_target(kwargs.get('agent_id'), kwargs.get('query_expr'))
+        adr = AutoDiscoveryRule.get_by_id(existed.adr_id) or abort(
+            404, ErrFormat.adr_not_found.format("id={}".format(existed.adr_id)))
+        if adr.type == AutoDiscoveryType.HTTP:
+            kwargs.setdefault('extra_option', dict())
+            en_name = None
+            for i in DEFAULT_INNER:
+                if i['name'] == adr.name:
+                    en_name = i['en']
+                    break
+            if en_name and kwargs['extra_option'].get('category'):
+                for item in CLOUD_MAP[en_name]:
+                    if item["collect_key_map"].get(kwargs['extra_option']['category']):
+                        kwargs["extra_option"]["collect_key"] = item["collect_key_map"][
+                            kwargs['extra_option']['category']]
+                        kwargs["extra_option"]["provider"] = en_name
+                        break
+
+        if adr.type == AutoDiscoveryType.COMPONENTS and kwargs.get('extra_option'):
+            for i in DEFAULT_INNER:
+                if i['name'] == adr.name:
+                    kwargs['extra_option']['collect_key'] = i['option'].get('collect_key')
+                    break
+
+        if 'attributes' in kwargs:
+            self.__valid_exec_target(kwargs.get('agent_id'), kwargs.get('query_expr'))
+
+            ci_type = CITypeCache.get(existed.type_id)
+            unique = AttributeCache.get(ci_type.unique_id)
+            if unique and unique.name not in (kwargs.get('attributes') or {}).values():
+                current_app.logger.warning((unique.name, kwargs.get('attributes'), ci_type.alias))
+                return abort(400, ErrFormat.ad_not_unique_key.format(unique.name))
 
         if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'):
             if current_user.uid != existed.uid:
                 return abort(403, ErrFormat.adt_secret_no_permission)
+        if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('password'):
+            if current_user.uid != existed.uid:
+                return abort(403, ErrFormat.adt_secret_no_permission)
 
         return existed
 
@@ -289,10 +455,22 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
         if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
             kwargs = check_plugin_script(**kwargs)
 
-        if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'):
-            kwargs['extra_option']['secret'] = AESCrypto.encrypt(kwargs['extra_option']['secret'])
+        encrypt_account(kwargs.get('extra_option'))
 
-        return super(AutoDiscoveryCITypeCRUD, self).update(_id, filter_none=False, **kwargs)
+        inst = self._can_update(_id=_id, **kwargs)
+        if len(kwargs) == 1 and 'enabled' in kwargs:  # enable or disable
+            pass
+        elif inst.agent_id != kwargs.get('agent_id') or inst.query_expr != kwargs.get('query_expr'):
+            for item in AutoDiscoveryRuleSyncHistory.get_by(adt_id=inst.id, to_dict=False):
+                item.delete(commit=False)
+            db.session.commit()
+
+            SystemConfigManager.create_or_update("ad_rules_updated_at",
+                                                 dict(v=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
+
+        obj = inst.update(_id=_id, filter_none=False, **kwargs)
+
+        return obj
 
     def _can_delete(self, **kwargs):
         if AutoDiscoveryCICRUD.get_by_adt_id(kwargs['_id']):
@@ -303,6 +481,61 @@ class AutoDiscoveryCITypeCRUD(DBMixin):
 
         return existed
 
+    def delete(self, _id):
+        inst = self._can_delete(_id=_id)
+
+        inst.soft_delete()
+
+        for item in AutoDiscoveryRuleSyncHistory.get_by(adt_id=inst.id, to_dict=False):
+            item.delete(commit=False)
+        db.session.commit()
+
+        attributes = self.get_ad_attributes(inst.type_id)
+        for item in AutoDiscoveryCITypeRelationCRUD.get_by_type_id(inst.type_id):
+            if item.ad_key not in attributes:
+                item.soft_delete()
+
+        return inst
+
+
+class AutoDiscoveryCITypeRelationCRUD(DBMixin):
+    cls = AutoDiscoveryCITypeRelation
+
+    @classmethod
+    def get_all(cls, type_ids=None):
+        res = cls.cls.get_by(to_dict=False)
+        return [i for i in res if type_ids is None or i.ad_type_id in type_ids]
+
+    @classmethod
+    def get_by_type_id(cls, type_id, to_dict=False):
+        return cls.cls.get_by(ad_type_id=type_id, to_dict=to_dict)
+
+    def upsert(self, ad_type_id, relations):
+        existed = self.cls.get_by(ad_type_id=ad_type_id, to_dict=False)
+        existed = {(i.ad_key, i.peer_type_id, i.peer_attr_id): i for i in existed}
+
+        new = []
+        for r in relations:
+            k = (r.get('ad_key'), r.get('peer_type_id'), r.get('peer_attr_id'))
+            if len(list(filter(lambda x: x, k))) == 3 and k not in existed:
+                self.cls.create(ad_type_id=ad_type_id, **r)
+
+            new.append(k)
+
+        for deleted in set(existed.keys()) - set(new):
+            existed[deleted].soft_delete()
+
+        return self.get_by_type_id(ad_type_id, to_dict=True)
+
+    def _can_add(self, **kwargs):
+        pass
+
+    def _can_update(self, **kwargs):
+        pass
+
+    def _can_delete(self, **kwargs):
+        pass
+
 
 class AutoDiscoveryCICRUD(DBMixin):
     cls = AutoDiscoveryCI
@@ -337,7 +570,6 @@ class AutoDiscoveryCICRUD(DBMixin):
         adts = AutoDiscoveryCITypeCRUD.get_by_type_id(type_id)
         for adt in adts:
             attr_names |= set((adt.attributes or {}).values())
-
         return [attr for attr in attributes if attr['name'] in attr_names]
 
     @classmethod
@@ -391,16 +623,24 @@ class AutoDiscoveryCICRUD(DBMixin):
         changed = False
         if existed is not None:
             if existed.instance != kwargs['instance']:
+                instance = copy.deepcopy(existed.instance) or {}
+                instance.update(kwargs['instance'])
+                kwargs['instance'] = instance
                 existed.update(filter_none=False, **kwargs)
+                AutoDiscoveryExecHistoryCRUD().add(type_id=adt.type_id,
+                                                   stdout="update resource: {}".format(kwargs.get('unique_value')))
                 changed = True
         else:
             existed = self.cls.create(**kwargs)
+            AutoDiscoveryExecHistoryCRUD().add(type_id=adt.type_id,
+                                               stdout="add resource: {}".format(kwargs.get('unique_value')))
             changed = True
 
         if adt.auto_accept and changed:
             try:
                 self.accept(existed)
             except Exception as e:
+                current_app.logger.error(e)
                 return abort(400, str(e))
         elif changed:
             existed.update(is_accept=False, accept_time=None, accept_by=None, filter_none=False)
@@ -420,6 +660,13 @@ class AutoDiscoveryCICRUD(DBMixin):
 
         inst.delete()
 
+        adt = AutoDiscoveryCIType.get_by_id(inst.adt_id)
+        if adt:
+            adt.update(updated_at=datetime.datetime.now())
+
+        AutoDiscoveryExecHistoryCRUD().add(type_id=inst.type_id,
+                                           stdout="delete resource: {}".format(inst.unique_value))
+
         self._after_delete(inst)
 
         return inst
@@ -435,6 +682,13 @@ class AutoDiscoveryCICRUD(DBMixin):
             not is_app_admin("cmdb") and validate_permission(ci_type.name, ResourceTypeEnum.CI, PermEnum.DELETE, "cmdb")
 
         existed.delete()
+
+        adt = AutoDiscoveryCIType.get_by_id(existed.adt_id)
+        if adt:
+            adt.update(updated_at=datetime.datetime.now())
+
+        AutoDiscoveryExecHistoryCRUD().add(type_id=type_id,
+                                           stdout="delete resource: {}".format(unique_value))
         # TODO: delete ci
 
     @classmethod
@@ -445,36 +699,24 @@ class AutoDiscoveryCICRUD(DBMixin):
         adt = AutoDiscoveryCITypeCRUD.get_by_id(adc.adt_id) or abort(404, ErrFormat.adt_not_found)
 
         ci_id = None
-        if adt.attributes:
-            ci_dict = {adt.attributes[k]: v for k, v in adc.instance.items() if k in adt.attributes}
-            ci_id = CIManager.add(adc.type_id, is_auto_discovery=True, **ci_dict)
 
-        relation_adts = AutoDiscoveryCIType.get_by(type_id=adt.type_id, adr_id=None, to_dict=False)
-        for r_adt in relation_adts:
-            if not r_adt.relation or ci_id is None:
-                continue
-            for ad_key in r_adt.relation:
-                if not adc.instance.get(ad_key):
-                    continue
-                cmdb_key = r_adt.relation[ad_key]
-                query = "_type:{},{}:{}".format(cmdb_key.get('type_name'), cmdb_key.get('attr_name'),
-                                                adc.instance.get(ad_key))
-                s = search(query)
-                try:
-                    response, _, _, _, _, _ = s.search()
-                except SearchError as e:
-                    current_app.logger.warning(e)
-                    return abort(400, str(e))
+        ad_key2attr = adt.attributes or {}
+        if ad_key2attr:
+            ci_dict = {ad_key2attr[k]: None if not v and isinstance(v, (list, dict)) else v
+                       for k, v in adc.instance.items() if k in ad_key2attr}
+            extra_option = adt.extra_option or {}
+            mapping, path_mapping = AutoDiscoveryHTTPManager.get_predefined_value_mapping(
+                extra_option.get('provider'), extra_option.get('category'))
+            if mapping:
+                ci_dict = {k: (mapping.get(k) or {}).get(str(v), v) for k, v in ci_dict.items()}
+            if path_mapping:
+                ci_dict = {k: jsonpath.jsonpath(v, path_mapping[k]) if k in path_mapping else v
+                           for k, v in ci_dict.items()}
+            ci_id = CIManager.add(adc.type_id, is_auto_discovery=True, _is_admin=True, **ci_dict)
+            AutoDiscoveryExecHistoryCRUD().add(type_id=adt.type_id,
+                                               stdout="accept resource: {}".format(adc.unique_value))
 
-                relation_ci_id = response and response[0]['_id']
-                if relation_ci_id:
-                    try:
-                        CIRelationManager.add(ci_id, relation_ci_id)
-                    except:
-                        try:
-                            CIRelationManager.add(relation_ci_id, ci_id)
-                        except:
-                            pass
+        build_relations_for_ad_accept.apply_async(args=(adc.to_dict(), ci_id, ad_key2attr), queue=CMDB_QUEUE)
 
         adc.update(is_accept=True,
                    accept_by=nickname or current_user.nickname,
@@ -485,17 +727,75 @@ class AutoDiscoveryCICRUD(DBMixin):
 class AutoDiscoveryHTTPManager(object):
     @staticmethod
     def get_categories(name):
-        return (ClOUD_MAP.get(name) or {}).get('categories') or []
+        categories = (CLOUD_MAP.get(name) or {}) or []
+        for item in copy.deepcopy(categories):
+            item.pop('map', None)
+            item.pop('collect_key_map', None)
 
-    @staticmethod
-    def get_attributes(name, category):
-        tpt = ((ClOUD_MAP.get(name) or {}).get('map') or {}).get(category)
-        if tpt and os.path.exists(os.path.join(PWD, tpt)):
-            with open(os.path.join(PWD, tpt)) as f:
-                return json.loads(f.read())
+        return categories
+
+    def get_resources(self, name):
+        en_name = None
+        for i in DEFAULT_INNER:
+            if i['name'] == name:
+                en_name = i['en']
+                break
+
+        if en_name:
+            categories = self.get_categories(en_name)
+
+            return [j for i in categories for j in i['items']]
 
         return []
 
+    @staticmethod
+    def get_attributes(provider, resource):
+        for item in (CLOUD_MAP.get(provider) or {}):
+            for _resource in (item.get('map') or {}):
+                if _resource == resource:
+                    tpt = item['map'][_resource]
+                    if isinstance(tpt, dict):
+                        tpt = tpt.get('template')
+                    if tpt and os.path.exists(os.path.join(PWD, tpt)):
+                        with open(os.path.join(PWD, tpt)) as f:
+                            return json.loads(f.read())
+
+        return []
+
+    @staticmethod
+    def get_mapping(provider, resource):
+        for item in (CLOUD_MAP.get(provider) or {}):
+            for _resource in (item.get('map') or {}):
+                if _resource == resource:
+                    mapping = item['map'][_resource]
+                    if not isinstance(mapping, dict):
+                        return {}
+                    name = mapping.get('mapping')
+                    mapping = AutoDiscoveryMappingCache.get(name)
+                    if isinstance(mapping, dict):
+                        return {mapping[key][provider]['key'].split('.')[0]: key for key in mapping if
+                                (mapping[key].get(provider) or {}).get('key')}
+
+        return {}
+
+    @staticmethod
+    def get_predefined_value_mapping(provider, resource):
+        for item in (CLOUD_MAP.get(provider) or {}):
+            for _resource in (item.get('map') or {}):
+                if _resource == resource:
+                    mapping = item['map'][_resource]
+                    if not isinstance(mapping, dict):
+                        return {}, {}
+                    name = mapping.get('mapping')
+                    mapping = AutoDiscoveryMappingCache.get(name)
+                    if isinstance(mapping, dict):
+                        return ({key: mapping[key][provider].get('map') for key in mapping if
+                                 mapping[key].get(provider, {}).get('map')},
+                                {key: mapping[key][provider]['key'].split('.', 1)[1] for key in mapping if
+                                 ((mapping[key].get(provider) or {}).get('key') or '').split('.')[1:]})
+
+        return {}, {}
+
 
 class AutoDiscoverySNMPManager(object):
 
@@ -506,3 +806,191 @@ class AutoDiscoverySNMPManager(object):
                 return json.loads(f.read())
 
         return []
+
+
+class AutoDiscoveryComponentsManager(object):
+
+    @staticmethod
+    def get_attributes(name):
+        if os.path.exists(os.path.join(PWD, "templates/{}.json".format(name))):
+            with open(os.path.join(PWD, "templates/{}.json".format(name))) as f:
+                return json.loads(f.read())
+
+        return []
+
+
+class AutoDiscoveryRuleSyncHistoryCRUD(DBMixin):
+    cls = AutoDiscoveryRuleSyncHistory
+
+    def _can_add(self, **kwargs):
+        pass
+
+    def _can_update(self, **kwargs):
+        pass
+
+    def _can_delete(self, **kwargs):
+        pass
+
+    def upsert(self, **kwargs):
+        existed = self.cls.get_by(adt_id=kwargs.get('adt_id'),
+                                  oneagent_id=kwargs.get('oneagent_id'),
+                                  oneagent_name=kwargs.get('oneagent_name'),
+                                  first=True,
+                                  to_dict=False)
+
+        if existed is not None:
+            existed.update(**kwargs)
+        else:
+            self.cls.create(**kwargs)
+
+
+class AutoDiscoveryExecHistoryCRUD(DBMixin):
+    cls = AutoDiscoveryExecHistory
+
+    def _can_add(self, **kwargs):
+        pass
+
+    def _can_update(self, **kwargs):
+        pass
+
+    def _can_delete(self, **kwargs):
+        pass
+
+
+class AutoDiscoveryCounterCRUD(DBMixin):
+    cls = AutoDiscoveryCounter
+
+    def get(self, type_id):
+        res = self.cls.get_by(type_id=type_id, first=True, to_dict=True)
+        if res is None:
+            return dict(rule_count=0, exec_target_count=0, instance_count=0, accept_count=0,
+                        this_month_count=0, this_week_count=0, last_month_count=0, last_week_count=0)
+
+        return res
+
+    def _can_add(self, **kwargs):
+        pass
+
+    def _can_update(self, **kwargs):
+        pass
+
+    def _can_delete(self, **kwargs):
+        pass
+
+
+def encrypt_account(config):
+    if isinstance(config, dict):
+        if config.get('secret'):
+            config['secret'] = AESCrypto.encrypt(config['secret'])
+        if config.get('password'):
+            config['password'] = AESCrypto.encrypt(config['password'])
+
+
+def decrypt_account(config, uid):
+    if isinstance(config, dict):
+        if config.get('password'):
+            if not (current_user.username in PRIVILEGED_USERS or current_user.uid == uid):
+                config.pop('password', None)
+            else:
+                try:
+                    config['password'] = AESCrypto.decrypt(config['password'])
+                except Exception as e:
+                    current_app.logger.error('decrypt account failed: {}'.format(e))
+
+        if config.get('secret'):
+            if not (current_user.username in PRIVILEGED_USERS or current_user.uid == uid):
+                config.pop('secret', None)
+            else:
+                try:
+                    config['secret'] = AESCrypto.decrypt(config['secret'])
+                except Exception as e:
+                    current_app.logger.error('decrypt account failed: {}'.format(e))
+
+
+class AutoDiscoveryAccountCRUD(DBMixin):
+    cls = AutoDiscoveryAccount
+
+    def get(self, adr_id):
+        res = self.cls.get_by(adr_id=adr_id, to_dict=True)
+
+        for i in res:
+            decrypt_account(i.get('config'), i['uid'])
+
+        return res
+
+    def get_config_by_id(self, _id):
+        res = self.cls.get_by_id(_id)
+        if not res:
+            return {}
+
+        config = res.to_dict().get('config') or {}
+
+        decrypt_account(config, res.uid)
+
+        return config
+
+    def _can_add(self, **kwargs):
+        encrypt_account(kwargs.get('config'))
+
+        kwargs['uid'] = current_user.uid
+
+        return kwargs
+
+    def upsert(self, adr_id, accounts):
+        existed_all = self.cls.get_by(adr_id=adr_id, to_dict=False)
+        account_names = {i['name'] for i in accounts}
+
+        name_changed = dict()
+        for account in accounts:
+            existed = None
+            if account.get('id'):
+                existed = self.cls.get_by_id(account.get('id'))
+                if existed is None:
+                    continue
+
+                account.pop('id')
+                name_changed[existed.name] = account.get('name')
+            else:
+                account = self._can_add(**account)
+
+            if existed is not None:
+                if current_user.uid == existed.uid:
+                    config = copy.deepcopy(existed.config) or {}
+                    config.update(account.get('config') or {})
+                    account['config'] = config
+                    existed.update(**account)
+            else:
+                self.cls.create(adr_id=adr_id, **account)
+
+        for item in existed_all:
+            if name_changed.get(item.name, item.name) not in account_names:
+                if current_user.uid == item.uid:
+                    item.soft_delete()
+
+    def _can_update(self, **kwargs):
+        existed = self.cls.get_by_id(kwargs['_id']) or abort(404, ErrFormat.not_found)
+
+        if isinstance(kwargs.get('config'), dict) and kwargs['config'].get('secret'):
+            if current_user.uid != existed.uid:
+                return abort(403, ErrFormat.adt_secret_no_permission)
+        if isinstance(kwargs.get('config'), dict) and kwargs['config'].get('password'):
+            if current_user.uid != existed.uid:
+                return abort(403, ErrFormat.adt_secret_no_permission)
+
+        return existed
+
+    def update(self, _id, **kwargs):
+
+        if kwargs.get('is_plugin') and kwargs.get('plugin_script'):
+            kwargs = check_plugin_script(**kwargs)
+
+        encrypt_account(kwargs.get('config'))
+
+        inst = self._can_update(_id=_id, **kwargs)
+
+        obj = inst.update(_id=_id, filter_none=False, **kwargs)
+
+        return obj
+
+    def _can_delete(self, **kwargs):
+        pass

cmdb-api/api/lib/cmdb/auto_discovery/const.py

@@ -2,15 +2,38 @@
 
 from api.lib.cmdb.const import AutoDiscoveryType
 
-DEFAULT_HTTP = [
-    dict(name="阿里云", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
-         option={'icon': {'name': 'caise-aliyun'}}),
-    dict(name="腾讯云", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
-         option={'icon': {'name': 'caise-tengxunyun'}}),
-    dict(name="华为云", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
-         option={'icon': {'name': 'caise-huaweiyun'}}),
-    dict(name="AWS", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
-         option={'icon': {'name': 'caise-aws'}}),
+PRIVILEGED_USERS = ("cmdb_agent", "worker", "admin")
+
+DEFAULT_INNER = [
+    dict(name="阿里云", en="aliyun", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-aliyun'}, "en": "aliyun"}),
+    dict(name="腾讯云", en="tencentcloud", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-tengxunyun'}, "en": "tencentcloud"}),
+    dict(name="华为云", en="huaweicloud", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-huaweiyun'}, "en": "huaweicloud"}),
+    dict(name="AWS", en="aws", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-aws'}, "en": "aws"}),
+
+    dict(name="VCenter", en="vcenter", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'cmdb-vcenter'}, "category": "private_cloud", "en": "vcenter"}),
+    dict(name="KVM", en="kvm", type=AutoDiscoveryType.HTTP, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'ops-KVM'}, "category": "private_cloud", "en": "kvm"}),
+
+
+    dict(name="Nginx", en="nginx", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-nginx'}, "en": "nginx", "collect_key": "nginx"}),
+    dict(name="Apache", en="apache", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-apache'}, "en": "apache", "collect_key": "apache"}),
+    dict(name="Tomcat", en="tomcat", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-tomcat'}, "en": "tomcat", "collect_key": "tomcat"}),
+    dict(name="MySQL", en="mysql", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-mySQL'}, "en": "mysql", "collect_key": "mysql"}),
+    dict(name="MSSQL", en="mssql", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+             option={'icon': {'name': 'caise-SQLServer'}, "en": "mssql", "collect_key": "sqlserver"}),
+    dict(name="Oracle", en="oracle", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-oracle'}, "en": "oracle", "collect_key": "oracle"}),
+    dict(name="Redis", en="redis", type=AutoDiscoveryType.COMPONENTS, is_inner=True, is_plugin=False,
+         option={'icon': {'name': 'caise-redis'}, "en": "redis", "collect_key": "redis"}),
 
     dict(name="交换机", type=AutoDiscoveryType.SNMP, is_inner=True, is_plugin=False,
          option={'icon': {'name': 'caise-jiaohuanji'}}),
@@ -22,32 +45,307 @@ DEFAULT_HTTP = [
          option={'icon': {'name': 'caise-dayinji'}}),
 ]
 
-ClOUD_MAP = {
-    "aliyun": {
-        "categories": ["云服务器 ECS"],
-        "map": {
-            "云服务器 ECS": "templates/aliyun_ecs.json",
-        }
-    },
-
-    "tencentcloud": {
-        "categories": ["云服务器 CVM"],
-        "map": {
-            "云服务器 CVM": "templates/tencent_cvm.json",
-        }
-    },
-
-    "huaweicloud": {
-        "categories": ["云服务器 ECS"],
-        "map": {
-            "云服务器 ECS": "templates/huaweicloud_ecs.json",
-        }
-    },
-
-    "aws": {
-        "categories": ["云服务器 EC2"],
-        "map": {
-            "云服务器 EC2": "templates/aws_ec2.json",
-        }
-    },
+CLOUD_MAP = {
+    "aliyun": [
+        {
+            "category": "计算",
+            "items": ["云服务器 ECS", "云服务器 Disk"],
+            "map": {
+                "云服务器 ECS": {"template": "templates/aliyun_ecs.json", "mapping": "ecs"},
+                "云服务器 Disk": {"template": "templates/aliyun_ecs_disk.json", "mapping": "evs"},
+            },
+            "collect_key_map": {
+                "云服务器 ECS": "ali.ecs",
+                "云服务器 Disk": "ali.ecs_disk",
+            },
+        },
+        {
+            "category": "网络与CDN",
+            "items": [
+                "内容分发CDN",
+                "负载均衡SLB",
+                "专有网络VPC",
+                "交换机Switch",
+            ],
+            "map": {
+                "内容分发CDN": {"template": "templates/aliyun_cdn.json", "mapping": "CDN"},
+                "负载均衡SLB": {"template": "templates/aliyun_slb.json", "mapping": "loadbalancer"},
+                "专有网络VPC": {"template": "templates/aliyun_vpc.json", "mapping": "vpc"},
+                "交换机Switch": {"template": "templates/aliyun_switch.json", "mapping": "vswitch"},
+            },
+            "collect_key_map": {
+                "内容分发CDN": "ali.cdn",
+                "负载均衡SLB": "ali.slb",
+                "专有网络VPC": "ali.vpc",
+                "交换机Switch": "ali.switch",
+            },
+        },
+        {
+            "category": "存储",
+            "items": ["块存储EBS", "对象存储OSS"],
+            "map": {
+                "块存储EBS": {"template": "templates/aliyun_ebs.json", "mapping": "evs"},
+                "对象存储OSS": {"template": "templates/aliyun_oss.json", "mapping": "objectStorage"},
+            },
+            "collect_key_map": {
+                "块存储EBS": "ali.ebs",
+                "对象存储OSS": "ali.oss",
+            },
+        },
+        {
+            "category": "数据库",
+            "items": ["云数据库RDS MySQL", "云数据库RDS PostgreSQL", "云数据库 Redis"],
+            "map": {
+                "云数据库RDS MySQL": {"template": "templates/aliyun_rds_mysql.json", "mapping": "mysql"},
+                "云数据库RDS PostgreSQL": {"template": "templates/aliyun_rds_postgre.json", "mapping": "postgresql"},
+                "云数据库 Redis": {"template": "templates/aliyun_redis.json", "mapping": "redis"},
+            },
+            "collect_key_map": {
+                "云数据库RDS MySQL": "ali.rds_mysql",
+                "云数据库RDS PostgreSQL": "ali.rds_postgre",
+                "云数据库 Redis": "ali.redis",
+            },
+        },
+    ],
+    "tencentcloud": [
+        {
+            "category": "计算",
+            "items": ["云服务器 CVM"],
+            "map": {
+                "云服务器 CVM": {"template": "templates/tencent_cvm.json", "mapping": "ecs"},
+            },
+            "collect_key_map": {
+                "云服务器 CVM": "tencent.cvm",
+            },
+        },
+        {
+            "category": "CDN与边缘",
+            "items": ["内容分发CDN"],
+            "map": {
+                "内容分发CDN": {"template": "templates/tencent_cdn.json", "mapping": "CDN"},
+            },
+            "collect_key_map": {
+                "内容分发CDN": "tencent.cdn",
+            },
+        },
+        {
+            "category": "网络",
+            "items": ["负载均衡CLB", "私有网络VPC", "子网"],
+            "map": {
+                "负载均衡CLB": {"template": "templates/tencent_clb.json", "mapping": "loadbalancer"},
+                "私有网络VPC": {"template": "templates/tencent_vpc.json", "mapping": "vpc"},
+                "子网": {"template": "templates/tencent_subnet.json", "mapping": "vswitch"},
+            },
+            "collect_key_map": {
+                "负载均衡CLB": "tencent.clb",
+                "私有网络VPC": "tencent.vpc",
+                "子网": "tencent.subnet",
+            },
+        },
+        {
+            "category": "存储",
+            "items": ["云硬盘CBS", "对象存储COS"],
+            "map": {
+                "云硬盘CBS": {"template": "templates/tencent_cbs.json", "mapping": "evs"},
+                "对象存储COS": {"template": "templates/tencent_cos.json", "mapping": "objectStorage"},
+            },
+            "collect_key_map": {
+                "云硬盘CBS": "tencent.cbs",
+                "对象存储COS": "tencent.cos",
+            },
+        },
+        {
+            "category": "数据库",
+            "items": ["云数据库 MySQL", "云数据库 PostgreSQL", "云数据库 Redis"],
+            "map": {
+                "云数据库 MySQL": {"template": "templates/tencent_rdb.json", "mapping": "mysql"},
+                "云数据库 PostgreSQL": {"template": "templates/tencent_postgres.json", "mapping": "postgresql"},
+                "云数据库 Redis": {"template": "templates/tencent_redis.json", "mapping": "redis"},
+            },
+            "collect_key_map": {
+                "云数据库 MySQL": "tencent.rdb",
+                "云数据库 PostgreSQL": "tencent.rds_postgres",
+                "云数据库 Redis": "tencent.redis",
+            },
+        },
+    ],
+    "huaweicloud": [
+        {
+            "category": "计算",
+            "items": ["云服务器 ECS"],
+            "map": {
+                "云服务器 ECS": {"template": "templates/huaweicloud_ecs.json", "mapping": "ecs"},
+            },
+            "collect_key_map": {
+                "云服务器 ECS": "huawei.ecs",
+            },
+        },
+        {
+            "category": "CDN与智能边缘",
+            "items": ["内容分发网络CDN"],
+            "map": {
+                "内容分发网络CDN": {"template": "templates/huawei_cdn.json", "mapping": "CDN"},
+            },
+            "collect_key_map": {
+                "内容分发网络CDN": "huawei.cdn",
+            },
+        },
+        {
+            "category": "网络",
+            "items": ["弹性负载均衡ELB", "虚拟私有云VPC", "子网"],
+            "map": {
+                "弹性负载均衡ELB": {"template": "templates/huawei_elb.json", "mapping": "loadbalancer"},
+                "虚拟私有云VPC": {"template": "templates/huawei_vpc.json", "mapping": "vpc"},
+                "子网": {"template": "templates/huawei_subnet.json", "mapping": "vswitch"},
+            },
+            "collect_key_map": {
+                "弹性负载均衡ELB": "huawei.elb",
+                "虚拟私有云VPC": "huawei.vpc",
+                "子网": "huawei.subnet",
+            },
+        },
+        {
+            "category": "存储",
+            "items": ["云硬盘EVS", "对象存储OBS"],
+            "map": {
+                "云硬盘EVS": {"template": "templates/huawei_evs.json", "mapping": "evs"},
+                "对象存储OBS": {"template": "templates/huawei_obs.json", "mapping": "objectStorage"},
+            },
+            "collect_key_map": {
+                "云硬盘EVS": "huawei.evs",
+                "对象存储OBS": "huawei.obs",
+            },
+        },
+        {
+            "category": "数据库",
+            "items": ["云数据库RDS MySQL", "云数据库RDS PostgreSQL"],
+            "map": {
+                "云数据库RDS MySQL": {"template": "templates/huawei_rds_mysql.json", "mapping": "mysql"},
+                "云数据库RDS PostgreSQL": {"template": "templates/huawei_rds_postgre.json", "mapping": "postgresql"},
+            },
+            "collect_key_map": {
+                "云数据库RDS MySQL": "huawei.rds_mysql",
+                "云数据库RDS PostgreSQL": "huawei.rds_postgre",
+            },
+        },
+        {
+            "category": "应用中间件",
+            "items": ["分布式缓存Redis"],
+            "map": {
+                "分布式缓存Redis": {"template": "templates/huawei_dcs.json", "mapping": "redis"},
+            },
+            "collect_key_map": {
+                "分布式缓存Redis": "huawei.dcs",
+            },
+        },
+    ],
+    "aws": [
+        {
+            "category": "计算",
+            "items": ["云服务器 EC2"],
+            "map": {
+                "云服务器 EC2": {"template": "templates/aws_ec2.json", "mapping": "ecs"},
+            },
+            "collect_key_map": {
+                "云服务器 EC2": "aws.ec2",
+            },
+        },
+        {"category": "网络与CDN", "items": [], "map": {}, "collect_key_map": {}},
+    ],
+    "vcenter": [
+        {
+            "category": "计算",
+            "items": [
+                "主机",
+                "虚拟机",
+                "主机集群"
+            ],
+            "map": {
+                "主机": "templates/vsphere_host.json",
+                "虚拟机": "templates/vsphere_vm.json",
+                "主机集群": "templates/vsphere_cluster.json",
+            },
+            "collect_key_map": {
+                "主机": "vsphere.host",
+                "虚拟机": "vsphere.vm",
+                "主机集群": "vsphere.cluster",
+            },
+        },
+        {
+            "category": "网络",
+            "items": [
+                "网络",
+                "标准交换机",
+                "分布式交换机",
+            ],
+            "map": {
+                "网络": "templates/vsphere_network.json",
+                "标准交换机": "templates/vsphere_standard_switch.json",
+                "分布式交换机": "templates/vsphere_distributed_switch.json",
+            },
+            "collect_key_map": {
+                "网络": "vsphere.network",
+                "标准交换机": "vsphere.standard_switch",
+                "分布式交换机": "vsphere.distributed_switch",
+            },
+        },
+        {
+            "category": "存储",
+            "items": ["数据存储", "数据存储集群"],
+            "map": {
+                "数据存储": "templates/vsphere_datastore.json",
+                "数据存储集群": "templates/vsphere_storage_pod.json",
+            },
+            "collect_key_map": {
+                "数据存储": "vsphere.datastore",
+                "数据存储集群": "vsphere.storage_pod",
+            },
+        },
+        {
+            "category": "其他",
+            "items": ["资源池", "数据中心", "文件夹"],
+            "map": {
+                "资源池": "templates/vsphere_pool.json",
+                "数据中心": "templates/vsphere_datacenter.json",
+                "文件夹": "templates/vsphere_folder.json",
+            },
+            "collect_key_map": {
+                "资源池": "vsphere.pool",
+                "数据中心": "vsphere.datacenter",
+                "文件夹": "vsphere.folder",
+            },
+        },
+    ],
+    "kvm": [
+        {
+            "category": "计算",
+            "items": ["虚拟机"],
+            "map": {
+                "虚拟机": "templates/kvm_vm.json",
+            },
+            "collect_key_map": {
+                "虚拟机": "kvm.vm",
+            },
+        },
+        {
+            "category": "存储",
+            "items": ["存储"],
+            "map": {
+                "存储": "templates/kvm_storage.json",
+            },
+            "collect_key_map": {
+                "存储": "kvm.storage",
+            },
+        },
+        {
+            "category": "network",
+            "items": ["网络"],
+            "map": {
+                "网络": "templates/kvm_network.json",
+            },
+            "collect_key_map": {
+                "网络": "kvm.network",
+            },
+        },
+    ],
 }

cmdb-api/api/lib/cmdb/cache.py

@@ -2,12 +2,22 @@
 
 from __future__ import unicode_literals
 
-from flask import current_app
+from collections import defaultdict
 
+import datetime
+import os
+import yaml
+from flask import current_app
+import json
 from api.extensions import cache
 from api.extensions import db
 from api.lib.cmdb.custom_dashboard import CustomDashboardManager
-from api.models.cmdb import Attribute
+from api.models.cmdb import Attribute, AutoDiscoveryExecHistory
+from api.models.cmdb import AutoDiscoveryCI
+from api.models.cmdb import AutoDiscoveryCIType
+from api.models.cmdb import AutoDiscoveryCITypeRelation
+from api.models.cmdb import AutoDiscoveryCounter
+from api.models.cmdb import AutoDiscoveryRuleSyncHistory
 from api.models.cmdb import CI
 from api.models.cmdb import CIType
 from api.models.cmdb import CITypeAttribute
@@ -245,7 +255,7 @@ class CMDBCounterCache(object):
 
     @classmethod
     def set(cls, result):
-        cache.set(cls.KEY, result, timeout=0)
+        cache.set(cls.KEY, json.loads(json.dumps(result)), timeout=0)
 
     @classmethod
     def reset(cls):
@@ -267,7 +277,7 @@ class CMDBCounterCache(object):
 
         cls.set(result)
 
-        return result
+        return json.loads(json.dumps(result))
 
     @classmethod
     def update(cls, custom, flush=True):
@@ -289,25 +299,36 @@ class CMDBCounterCache(object):
             result[custom['id']] = res
             cls.set(result)
 
-        return res
+        return json.loads(json.dumps(res))
 
-    @staticmethod
-    def relation_counter(type_id, level, other_filer, type_ids):
+    @classmethod
+    def relation_counter(cls, type_id, level, other_filer, type_ids):
         from api.lib.cmdb.search.ci_relation.search import Search as RelSearch
         from api.lib.cmdb.search import SearchError
         from api.lib.cmdb.search.ci import search
+        from api.lib.cmdb.attribute import AttributeManager
 
         query = "_type:{}".format(type_id)
+        if other_filer:
+            query = "{},{}".format(query, other_filer)
         s = search(query, count=1000000)
         try:
             type_names, _, _, _, _, _ = s.search()
         except SearchError as e:
             current_app.logger.error(e)
             return
+        root_type = CITypeCache.get(type_id)
+        show_attr_id = root_type and root_type.show_id
+        show_attr = AttributeCache.get(show_attr_id)
 
-        type_id_names = [(str(i.get('_id')), i.get(i.get('unique'))) for i in type_names]
+        type_id_names = []
+        for i in type_names:
+            attr_value = i.get(show_attr and show_attr.name) or i.get(i.get('unique'))
+            enum_map = AttributeManager.get_enum_map(show_attr_id or i.get('unique'))
 
-        s = RelSearch([i[0] for i in type_id_names], level, other_filer or '')
+            type_id_names.append((str(i.get('_id')), enum_map.get(attr_value, attr_value)))
+
+        s = RelSearch([i[0] for i in type_id_names], level)
         try:
             stats = s.statistics(type_ids, need_filter=False)
         except SearchError as e:
@@ -337,11 +358,12 @@ class CMDBCounterCache(object):
 
         return result
 
-    @staticmethod
-    def attribute_counter(custom):
+    @classmethod
+    def attribute_counter(cls, custom):
         from api.lib.cmdb.search import SearchError
         from api.lib.cmdb.search.ci import search
         from api.lib.cmdb.utils import ValueTypeMap
+        from api.lib.cmdb.attribute import AttributeManager
 
         custom.setdefault('options', {})
         type_id = custom.get('type_id')
@@ -357,16 +379,24 @@ class CMDBCounterCache(object):
         other_filter = "{}".format(other_filter) if other_filter else ''
 
         if custom['options'].get('ret') == 'cis':
+            enum_map = {}
+            for _attr_id in attr_ids:
+                _attr = AttributeCache.get(_attr_id)
+                if _attr:
+                    enum_map[_attr.alias] = AttributeManager.get_enum_map(_attr_id)
+
             query = "_type:({}),{}".format(";".join(map(str, type_ids)), other_filter)
             s = search(query, fl=attr_ids, ret_key='alias', count=100)
             try:
                 cis, _, _, _, _, _ = s.search()
+                cis = [{k: (enum_map.get(k) or {}).get(v, v) for k, v in ci.items()} for ci in cis]
             except SearchError as e:
                 current_app.logger.error(e)
                 return
 
             return cis
 
+        origin_result = dict()
         result = dict()
         # level = 1
         query = "_type:({}),{}".format(";".join(map(str, type_ids)), other_filter)
@@ -376,13 +406,18 @@ class CMDBCounterCache(object):
         except SearchError as e:
             current_app.logger.error(e)
             return
+
+        enum_map1 = AttributeManager.get_enum_map(attr_ids[0])
         for i in (list(facet.values()) or [[]])[0]:
-            result[ValueTypeMap.serialize2[attr2value_type[0]](str(i[0]))] = i[1]
+            k = ValueTypeMap.serialize2[attr2value_type[0]](str(i[0]))
+            result[enum_map1.get(k, k)] = i[1]
+            origin_result[k] = i[1]
         if len(attr_ids) == 1:
             return result
 
         # level = 2
-        for v in result:
+        enum_map2 = AttributeManager.get_enum_map(attr_ids[1])
+        for v in origin_result:
             query = "_type:({}),{},{}:{}".format(";".join(map(str, type_ids)), other_filter, attr_ids[0], v)
             s = search(query, fl=attr_ids, facet=[attr_ids[1]], count=1)
             try:
@@ -390,18 +425,22 @@ class CMDBCounterCache(object):
             except SearchError as e:
                 current_app.logger.error(e)
                 return
-            result[v] = dict()
+            result[enum_map1.get(v, v)] = dict()
+            origin_result[v] = dict()
             for i in (list(facet.values()) or [[]])[0]:
-                result[v][ValueTypeMap.serialize2[attr2value_type[1]](str(i[0]))] = i[1]
+                k = ValueTypeMap.serialize2[attr2value_type[1]](str(i[0]))
+                result[enum_map1.get(v, v)][enum_map2.get(k, k)] = i[1]
+                origin_result[v][k] = i[1]
 
         if len(attr_ids) == 2:
             return result
 
         # level = 3
-        for v1 in result:
-            if not isinstance(result[v1], dict):
+        enum_map3 = AttributeManager.get_enum_map(attr_ids[2])
+        for v1 in origin_result:
+            if not isinstance(result[enum_map1.get(v1, v1)], dict):
                 continue
-            for v2 in result[v1]:
+            for v2 in origin_result[v1]:
                 query = "_type:({}),{},{}:{},{}:{}".format(";".join(map(str, type_ids)), other_filter,
                                                            attr_ids[0], v1, attr_ids[1], v2)
                 s = search(query, fl=attr_ids, facet=[attr_ids[2]], count=1)
@@ -410,9 +449,10 @@ class CMDBCounterCache(object):
                 except SearchError as e:
                     current_app.logger.error(e)
                     return
-                result[v1][v2] = dict()
+                result[enum_map1.get(v1, v1)][enum_map2.get(v2, v2)] = dict()
                 for i in (list(facet.values()) or [[]])[0]:
-                    result[v1][v2][ValueTypeMap.serialize2[attr2value_type[2]](str(i[0]))] = i[1]
+                    k = ValueTypeMap.serialize2[attr2value_type[2]](str(i[0]))
+                    result[enum_map1.get(v1, v1)][enum_map2.get(v2, v2)][enum_map3.get(k, k)] = i[1]
 
         return result
 
@@ -448,7 +488,67 @@ class CMDBCounterCache(object):
 
         cache.set(cls.KEY2, result, timeout=0)
 
-        return result
+        res = db.session.query(AutoDiscoveryCI.created_at,
+                               AutoDiscoveryCI.updated_at,
+                               AutoDiscoveryCI.adt_id,
+                               AutoDiscoveryCI.type_id,
+                               AutoDiscoveryCI.is_accept).filter(AutoDiscoveryCI.deleted.is_(False))
+
+        today = datetime.datetime.today()
+        this_month = datetime.datetime(today.year, today.month, 1)
+        last_month = this_month - datetime.timedelta(days=1)
+        last_month = datetime.datetime(last_month.year, last_month.month, 1)
+        this_week = today - datetime.timedelta(days=datetime.date.weekday(today))
+        this_week = datetime.datetime(this_week.year, this_week.month, this_week.day)
+        last_week = this_week - datetime.timedelta(days=7)
+        last_week = datetime.datetime(last_week.year, last_week.month, last_week.day)
+        result = dict()
+        for i in res:
+            if i.type_id not in result:
+                result[i.type_id] = dict(instance_count=0, accept_count=0,
+                                         this_month_count=0, this_week_count=0, last_month_count=0, last_week_count=0)
+
+                adts = AutoDiscoveryCIType.get_by(type_id=i.type_id, to_dict=False)
+                result[i.type_id]['rule_count'] = len(adts) + AutoDiscoveryCITypeRelation.get_by(
+                    ad_type_id=i.type_id, only_query=True).count()
+                result[i.type_id]['exec_target_count'] = len(
+                    set([i.oneagent_id for adt in adts for i in db.session.query(
+                        AutoDiscoveryRuleSyncHistory.oneagent_id).filter(
+                        AutoDiscoveryRuleSyncHistory.adt_id == adt.id)]))
+
+            result[i.type_id]['instance_count'] += 1
+            if i.is_accept:
+                result[i.type_id]['accept_count'] += 1
+
+            if last_month <= i.created_at < this_month:
+                result[i.type_id]['last_month_count'] += 1
+            elif i.created_at >= this_month:
+                result[i.type_id]['this_month_count'] += 1
+
+            if last_week <= i.created_at < this_week:
+                result[i.type_id]['last_week_count'] += 1
+            elif i.created_at >= this_week:
+                result[i.type_id]['this_week_count'] += 1
+
+        for type_id in result:
+            existed = AutoDiscoveryCounter.get_by(type_id=type_id, first=True, to_dict=False)
+            if existed is None:
+                AutoDiscoveryCounter.create(type_id=type_id, **result[type_id])
+            else:
+                existed.update(**result[type_id])
+
+        for i in AutoDiscoveryCounter.get_by(to_dict=False):
+            if i.type_id not in result:
+                i.delete()
+
+    @classmethod
+    def clear_ad_exec_history(cls):
+        ci_types = CIType.get_by(to_dict=False)
+        for ci_type in ci_types:
+            for i in AutoDiscoveryExecHistory.get_by(type_id=ci_type.id, only_query=True).order_by(
+                    AutoDiscoveryExecHistory.id.desc()).offset(50000):
+                i.delete(commit=False)
+            db.session.commit()
 
     @classmethod
     def get_adc_counter(cls):
@@ -456,19 +556,18 @@ class CMDBCounterCache(object):
 
     @classmethod
     def flush_sub_counter(cls):
-        result = dict(type_id2users=dict())
+        result = dict(type_id2users=defaultdict(list))
 
         types = db.session.query(PreferenceShowAttributes.type_id,
                                  PreferenceShowAttributes.uid, PreferenceShowAttributes.created_at).filter(
             PreferenceShowAttributes.deleted.is_(False)).group_by(
             PreferenceShowAttributes.uid, PreferenceShowAttributes.type_id)
         for i in types:
-            result['type_id2users'].setdefault(i.type_id, []).append(i.uid)
+            result['type_id2users'][i.type_id].append(i.uid)
 
         types = PreferenceTreeView.get_by(to_dict=False)
         for i in types:
 
-            result['type_id2users'].setdefault(i.type_id, [])
             if i.uid not in result['type_id2users'][i.type_id]:
                 result['type_id2users'][i.type_id].append(i.uid)
 
@@ -479,3 +578,21 @@ class CMDBCounterCache(object):
     @classmethod
     def get_sub_counter(cls):
         return cache.get(cls.KEY3) or cls.flush_sub_counter()
+
+
+class AutoDiscoveryMappingCache(object):
+    PREFIX = 'CMDB::AutoDiscovery::Mapping::{}'
+
+    @classmethod
+    def get(cls, name):
+        res = cache.get(cls.PREFIX.format(name)) or {}
+        if not res:
+            path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
+                                "auto_discovery/mapping/{}.yaml".format(name))
+            if os.path.exists(path):
+                with open(path, 'r') as f:
+                    mapping = yaml.safe_load(f)
+                    res = mapping.get('mapping') or {}
+                    res and cache.set(cls.PREFIX.format(name), res, timeout=0)
+
+        return res

cmdb-api/api/lib/cmdb/ci.py

@@ -4,12 +4,12 @@
 import copy
 import datetime
 import json
-import threading
-
 import redis_lock
+import threading
 from flask import abort
 from flask import current_app
 from flask_login import current_user
+from sqlalchemy.orm import aliased
 from werkzeug.exceptions import BadRequest
 
 from api.extensions import db
@@ -28,6 +28,7 @@ from api.lib.cmdb.const import ExistPolicy
 from api.lib.cmdb.const import OperateType
 from api.lib.cmdb.const import PermEnum
 from api.lib.cmdb.const import REDIS_PREFIX_CI
+from api.lib.cmdb.const import RelationSourceEnum
 from api.lib.cmdb.const import ResourceTypeEnum
 from api.lib.cmdb.const import RetKey
 from api.lib.cmdb.const import ValueTypeEnum
@@ -160,7 +161,7 @@ class CIManager(object):
 
     @classmethod
     def get_ci_by_id_from_db(cls, ci_id, ret_key=RetKey.NAME, fields=None, need_children=True, use_master=False,
-                             valid=False):
+                             valid=False, enum_use_label=False):
         """
 
         :param ci_id:
@@ -169,6 +170,7 @@ class CIManager(object):
         :param need_children:
         :param use_master: whether to use master db
         :param valid:
+        :param enum_use_label:
         :return:
         """
 
@@ -186,13 +188,19 @@ class CIManager(object):
 
         res["ci_type"] = ci_type.name
 
-        fields = CITypeAttributeManager.get_attr_names_by_type_id(ci.type_id) if not fields else fields
+        enum_map = dict()
+        if not enum_use_label:
+            fields = CITypeAttributeManager.get_attr_names_by_type_id(ci.type_id) if not fields else fields
+        else:
+            fields, enum_map = CITypeAttributeManager.get_attr_names_label_enum(
+                ci.type_id) if not fields else (fields, {})
         unique_key = AttributeCache.get(ci_type.unique_id)
         _res = AttributeValueManager().get_attr_values(fields,
                                                        ci_id,
                                                        ret_key=ret_key,
                                                        unique_key=unique_key,
-                                                       use_master=use_master)
+                                                       use_master=use_master,
+                                                       enum_map=enum_map)
         res.update(_res)
 
         res['_type'] = ci_type.id
@@ -217,13 +225,13 @@ class CIManager(object):
 
     @classmethod
     def get_ad_statistics(cls):
-        return CMDBCounterCache.get_adc_counter()
+        return CMDBCounterCache.get_adc_counter() or {}
 
     @staticmethod
     def ci_is_exist(unique_key, unique_value, type_id):
         """
 
-        :param unique_key: is a attribute
+        :param unique_key: is an attribute
         :param unique_value:
         :param type_id:
         :return:
@@ -264,9 +272,11 @@ class CIManager(object):
             for attr_id in constraint.attr_ids:
                 value_table = TableMap(attr_name=id2name[attr_id]).table
 
-                _ci_ids = set([i.ci_id for i in value_table.get_by(attr_id=attr_id,
-                                                                   to_dict=False,
-                                                                   value=ci_dict.get(id2name[attr_id]) or None)])
+                values = value_table.get_by(attr_id=attr_id,
+                                            value=ci_dict.get(id2name[attr_id]),
+                                            only_query=True).join(
+                    CI, CI.id == value_table.ci_id).filter(CI.type_id == type_id)
+                _ci_ids = set([i.ci_id for i in values])
                 if ci_ids is None:
                     ci_ids = _ci_ids
                 else:
@@ -289,6 +299,53 @@ class CIManager(object):
 
         return 1
 
+    @staticmethod
+    def _reference_to_ci_id(attr, payload):
+        def __unique_value2id(_type, _v):
+            value_table = TableMap(attr_name=_type.unique_id).table
+            ci = value_table.get_by(attr_id=attr.id, value=_v)
+            if ci is not None:
+                return ci.ci_id
+
+            return abort(400, ErrFormat.ci_reference_invalid.format(attr.alias, _v))
+
+        def __valid_reference_id_existed(_id, _type_id):
+            ci = CI.get_by_id(_id) or abort(404, ErrFormat.ci_reference_not_found.format(attr.alias, _id))
+
+            if ci.type_id != _type_id:
+                return abort(400, ErrFormat.ci_reference_invalid.format(attr.alias, _id))
+
+        if attr.name in payload:
+            k, reference_value = attr.name, payload[attr.name]
+        elif attr.alias in payload:
+            k, reference_value = attr.alias, payload[attr.alias]
+        else:
+            return
+        if not reference_value:
+            return
+
+        reference_type = None
+        if isinstance(reference_value, list):
+            for idx, v in enumerate(reference_value):
+                if isinstance(v, dict) and v.get('unique'):
+                    if reference_type is None:
+                        reference_type = CITypeCache.get(attr.reference_type_id)
+                    if reference_type is not None:
+                        reference_value[idx] = __unique_value2id(reference_type, v)
+                else:
+                    __valid_reference_id_existed(v, attr.reference_type_id)
+
+        elif isinstance(reference_value, dict) and reference_value.get('unique'):
+            if reference_type is None:
+                reference_type = CITypeCache.get(attr.reference_type_id)
+            if reference_type is not None:
+                reference_value = __unique_value2id(reference_type, reference_value)
+        elif str(reference_value).isdigit():
+            reference_value = int(reference_value)
+            __valid_reference_id_existed(reference_value, attr.reference_type_id)
+
+        payload[k] = reference_value
+
     @classmethod
     def add(cls, ci_type_name,
             exist_policy=ExistPolicy.REPLACE,
@@ -316,8 +373,8 @@ class CIManager(object):
             400, ErrFormat.unique_value_not_found.format("unique_id={}".format(ci_type.unique_id)))
 
         unique_value = None
-        if not (unique_key.default and unique_key.default.get('default') == AttributeDefaultValueEnum.AUTO_INC_ID and
-                not ci_dict.get(unique_key.name)):  # primary key is not auto inc id
+        # primary key is not auto inc id
+        if not (unique_key.default and unique_key.default.get('default') == AttributeDefaultValueEnum.AUTO_INC_ID):
             unique_value = ci_dict.get(unique_key.name) or ci_dict.get(unique_key.alias) or ci_dict.get(unique_key.id)
             unique_value = unique_value or abort(400, ErrFormat.unique_key_required.format(unique_key.name))
 
@@ -325,6 +382,7 @@ class CIManager(object):
         ci_type_attrs_name = {attr.name: attr for _, attr in attrs}
         ci_type_attrs_alias = {attr.alias: attr for _, attr in attrs}
         ci_attr2type_attr = {type_attr.attr_id: type_attr for type_attr, _ in attrs}
+        ci_type_attrs_name_alias = {**ci_type_attrs_name, **ci_type_attrs_alias}
 
         ci = None
         record_id = None
@@ -379,17 +437,17 @@ class CIManager(object):
             for _, attr in attrs:
                 if attr.is_computed:
                     computed_attrs.append(attr.to_dict())
+                    ci_dict[attr.name] = None
                 elif attr.is_password:
                     if attr.name in ci_dict:
-                        password_dict[attr.id] = ci_dict.pop(attr.name)
+                        password_dict[attr.id] = (ci_dict.pop(attr.name), attr.is_dynamic)
                     elif attr.alias in ci_dict:
-                        password_dict[attr.id] = ci_dict.pop(attr.alias)
+                        password_dict[attr.id] = (ci_dict.pop(attr.alias), attr.is_dynamic)
 
                     if attr.re_check and password_dict.get(attr.id):
-                        value_manager.check_re(attr.re_check, password_dict[attr.id])
-
-            if computed_attrs:
-                value_manager.handle_ci_compute_attributes(ci_dict, computed_attrs, ci)
+                        value_manager.check_re(attr.re_check, attr.alias, password_dict[attr.id][0])
+                elif attr.is_reference:
+                    cls._reference_to_ci_id(attr, ci_dict)
 
             cls._valid_unique_constraint(ci_type.id, ci_dict, ci and ci.id)
 
@@ -411,15 +469,19 @@ class CIManager(object):
                     else:
                         ci_dict.pop(k)
 
-            ci_dict = {k: v for k, v in ci_dict.items() if k in ci_type_attrs_name or k in ci_type_attrs_alias}
+            ci_dict = {ci_type_attrs_name_alias[k].name: v for k, v in ci_dict.items() if k in ci_type_attrs_name_alias}
 
             key2attr = value_manager.valid_attr_value(ci_dict, ci_type.id, ci and ci.id,
                                                       ci_type_attrs_name, ci_type_attrs_alias, ci_attr2type_attr)
 
+            if computed_attrs:
+                value_manager.handle_ci_compute_attributes(ci_dict, computed_attrs, ci)
+
             operate_type = OperateType.UPDATE if ci is not None else OperateType.ADD
             try:
                 ci = ci or CI.create(type_id=ci_type.id, is_auto_discovery=is_auto_discovery)
-                record_id = value_manager.create_or_update_attr_value(ci, ci_dict, key2attr, ticket_id=ticket_id)
+                record_id, has_dynamic = value_manager.create_or_update_attr_value(
+                    ci, ci_dict, key2attr, ticket_id=ticket_id)
             except BadRequest as e:
                 if existed is None:
                     cls.delete(ci.id)
@@ -429,7 +491,7 @@ class CIManager(object):
             for attr_id in password_dict:
                 record_id = cls.save_password(ci.id, attr_id, password_dict[attr_id], record_id, ci_type.id)
 
-        if record_id:  # has change
+        if record_id or has_dynamic:  # has changed
             ci_cache.apply_async(args=(ci.id, operate_type, record_id), queue=CMDB_QUEUE)
 
         if ref_ci_dict:  # add relations
@@ -437,14 +499,18 @@ class CIManager(object):
 
         return ci.id
 
-    def update(self, ci_id, _is_admin=False, ticket_id=None, **ci_dict):
+    def update(self, ci_id, _is_admin=False, ticket_id=None, _sync=False, **ci_dict):
         now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
         ci = self.confirm_ci_existed(ci_id)
-
-        raw_dict = copy.deepcopy(ci_dict)
+        ci_type = ci.ci_type
 
         attrs = CITypeAttributeManager.get_all_attributes(ci.type_id)
         ci_type_attrs_name = {attr.name: attr for _, attr in attrs}
+        ci_type_attrs_alias2name = {attr.alias: attr.name for _, attr in attrs}
+        ci_dict = {ci_type_attrs_alias2name[k] if k in ci_type_attrs_alias2name else k: v for k, v in ci_dict.items()}
+
+        raw_dict = copy.deepcopy(ci_dict)
+
         ci_attr2type_attr = {type_attr.attr_id: type_attr for type_attr, _ in attrs}
         for _, attr in attrs:
             if attr.default and attr.default.get('default') == AttributeDefaultValueEnum.UPDATED_AT:
@@ -457,22 +523,22 @@ class CIManager(object):
         for _, attr in attrs:
             if attr.is_computed:
                 computed_attrs.append(attr.to_dict())
+                ci_dict[attr.name] = None
             elif attr.is_password:
                 if attr.name in ci_dict:
-                    password_dict[attr.id] = ci_dict.pop(attr.name)
+                    password_dict[attr.id] = (ci_dict.pop(attr.name), attr.is_dynamic)
                 elif attr.alias in ci_dict:
-                    password_dict[attr.id] = ci_dict.pop(attr.alias)
+                    password_dict[attr.id] = (ci_dict.pop(attr.alias), attr.is_dynamic)
 
                 if attr.re_check and password_dict.get(attr.id):
-                    value_manager.check_re(attr.re_check, password_dict[attr.id])
-
-        if computed_attrs:
-            value_manager.handle_ci_compute_attributes(ci_dict, computed_attrs, ci)
+                    value_manager.check_re(attr.re_check, attr.alias, password_dict[attr.id][0])
+            elif attr.is_reference:
+                self._reference_to_ci_id(attr, ci_dict)
 
         limit_attrs = self._valid_ci_for_no_read(ci) if not _is_admin else {}
 
         record_id = None
-        with redis_lock.Lock(rd.r, ci.ci_type.name):
+        with redis_lock.Lock(rd.r, ci_type.name):
             db.session.commit()
 
             self._valid_unique_constraint(ci.type_id, ci_dict, ci_id)
@@ -480,6 +546,10 @@ class CIManager(object):
             ci_dict = {k: v for k, v in ci_dict.items() if k in ci_type_attrs_name}
             key2attr = value_manager.valid_attr_value(ci_dict, ci.type_id, ci.id, ci_type_attrs_name,
                                                       ci_attr2type_attr=ci_attr2type_attr)
+
+            if computed_attrs:
+                value_manager.handle_ci_compute_attributes(ci_dict, computed_attrs, ci)
+
             if limit_attrs:
                 for k in copy.deepcopy(ci_dict):
                     if k not in limit_attrs:
@@ -489,7 +559,8 @@ class CIManager(object):
                             ci_dict.pop(k)
 
             try:
-                record_id = value_manager.create_or_update_attr_value(ci, ci_dict, key2attr, ticket_id=ticket_id)
+                record_id, has_dynamic = value_manager.create_or_update_attr_value(
+                    ci, ci_dict, key2attr, ticket_id=ticket_id)
             except BadRequest as e:
                 raise e
 
@@ -497,19 +568,25 @@ class CIManager(object):
             for attr_id in password_dict:
                 record_id = self.save_password(ci.id, attr_id, password_dict[attr_id], record_id, ci.type_id)
 
-        if record_id:  # has change
-            ci_cache.apply_async(args=(ci_id, OperateType.UPDATE, record_id), queue=CMDB_QUEUE)
+        if record_id or has_dynamic:  # has changed
+            if not _sync:
+                ci_cache.apply_async(args=(ci_id, OperateType.UPDATE, record_id), queue=CMDB_QUEUE)
+            else:
+                ci_cache(ci_id, OperateType.UPDATE, record_id)
 
         ref_ci_dict = {k: v for k, v in ci_dict.items() if k.startswith("$") and "." in k}
         if ref_ci_dict:
-            ci_relation_add.apply_async(args=(ref_ci_dict, ci.id), queue=CMDB_QUEUE)
+            if not _sync:
+                ci_relation_add.apply_async(args=(ref_ci_dict, ci.id), queue=CMDB_QUEUE)
+            else:
+                ci_relation_add(ref_ci_dict, ci.id)
 
     @staticmethod
     def update_unique_value(ci_id, unique_name, unique_value):
         ci = CI.get_by_id(ci_id) or abort(404, ErrFormat.ci_not_found.format("id={}".format(ci_id)))
 
         key2attr = {unique_name: AttributeCache.get(unique_name)}
-        record_id = AttributeValueManager().create_or_update_attr_value(ci, {unique_name: unique_value}, key2attr)
+        record_id, _ = AttributeValueManager().create_or_update_attr_value(ci, {unique_name: unique_value}, key2attr)
 
         ci_cache.apply_async(args=(ci_id, OperateType.UPDATE, record_id), queue=CMDB_QUEUE)
 
@@ -560,7 +637,7 @@ class CIManager(object):
         if ci_dict:
             AttributeHistoryManger.add(None, ci_id, [(None, OperateType.DELETE, ci_dict, None)], ci.type_id)
 
-        ci_delete.apply_async(args=(ci_id,), queue=CMDB_QUEUE)
+        ci_delete.apply_async(args=(ci_id, ci.type_id), queue=CMDB_QUEUE)
         delete_id_filter.apply_async(args=(ci_id,), queue=CMDB_QUEUE)
 
         return ci_id
@@ -632,13 +709,18 @@ class CIManager(object):
             elif fields:
                 _res = []
                 for d in res:
+                    if isinstance(fields, dict) and d.get("_type") not in fields:
+                        _res.append(d)
+                        continue
+
                     _d = dict()
                     _d["_id"], _d["_type"] = d.get("_id"), d.get("_type")
                     _d["ci_type"] = d.get("ci_type")
                     if unique_required:
                         _d[d.get('unique')] = d.get(d.get('unique'))
 
-                    for field in fields + ['ci_type_alias', 'unique', 'unique_alias']:
+                    _fields = list(fields.get(_d['_type']) or [] if isinstance(fields, dict) else fields)
+                    for field in _fields + ['ci_type_alias', 'unique', 'unique_alias']:
                         _d[field] = d.get(field)
                     _res.append(_d)
                 return _res
@@ -655,9 +737,8 @@ class CIManager(object):
         from api.lib.cmdb.search.ci.db.query_sql import QUERY_CIS_BY_IDS
         from api.lib.cmdb.search.ci.db.query_sql import QUERY_CIS_BY_VALUE_TABLE
 
-        if not fields:
-            filter_fields_sql = ""
-        else:
+        filter_fields_sql = ""
+        if fields and isinstance(fields, list):
             _fields = list()
             for field in fields:
                 attr = AttributeCache.get(field)
@@ -699,6 +780,10 @@ class CIManager(object):
                 ci_set.add(ci_id)
                 res[ci2pos[ci_id]] = ci_dict
 
+            if isinstance(fields, dict) and fields.get(type_id):
+                if attr_name not in fields[type_id]:
+                    continue
+
             if ret_key == RetKey.NAME:
                 attr_key = attr_name
             elif ret_key == RetKey.ALIAS:
@@ -724,7 +809,7 @@ class CIManager(object):
                        fields=None, value_tables=None, unique_required=False, excludes=None):
         """
 
-        :param ci_ids: list of CI instance ID, eg. ['1', '2']
+        :param ci_ids: list of CI instance ID, e.g. ['1', '2']
         :param ret_key: name, id or alias
         :param fields:
         :param value_tables:
@@ -736,7 +821,7 @@ class CIManager(object):
         if not ci_ids:
             return []
 
-        fields = [] if fields is None or not isinstance(fields, list) else fields
+        fields = [] if not fields else fields
 
         ci_id_tuple = tuple(map(int, ci_ids))
         res = cls._get_cis_from_cache(ci_id_tuple, ret_key, fields, unique_required, excludes=excludes)
@@ -749,12 +834,13 @@ class CIManager(object):
 
     @classmethod
     def save_password(cls, ci_id, attr_id, value, record_id, type_id):
+        value, is_dynamic = value
         changed = None
         encrypt_value = None
         value_table = ValueTypeMap.table[ValueTypeEnum.PASSWORD]
         if current_app.config.get('SECRETS_ENGINE') == 'inner':
             if value:
-                encrypt_value, status = InnerCrypt().encrypt(value)
+                encrypt_value, status = InnerCrypt().encrypt(str(value))
                 if not status:
                     current_app.logger.error('save password failed: {}'.format(encrypt_value))
                     return abort(400, ErrFormat.password_save_failed.format(encrypt_value))
@@ -765,20 +851,24 @@ class CIManager(object):
         if existed is None:
             if value:
                 value_table.create(ci_id=ci_id, attr_id=attr_id, value=encrypt_value)
-                changed = [(ci_id, attr_id, OperateType.ADD, '', PASSWORD_DEFAULT_SHOW, type_id)]
+                if not is_dynamic:
+                    changed = [(ci_id, attr_id, OperateType.ADD, '', PASSWORD_DEFAULT_SHOW, type_id)]
         elif existed.value != encrypt_value:
             if value:
                 existed.update(ci_id=ci_id, attr_id=attr_id, value=encrypt_value)
-                changed = [(ci_id, attr_id, OperateType.UPDATE, PASSWORD_DEFAULT_SHOW, PASSWORD_DEFAULT_SHOW, type_id)]
+                if not is_dynamic:
+                    changed = [(ci_id, attr_id, OperateType.UPDATE, PASSWORD_DEFAULT_SHOW,
+                                PASSWORD_DEFAULT_SHOW, type_id)]
             else:
                 existed.delete()
-                changed = [(ci_id, attr_id, OperateType.DELETE, PASSWORD_DEFAULT_SHOW, '', type_id)]
+                if not is_dynamic:
+                    changed = [(ci_id, attr_id, OperateType.DELETE, PASSWORD_DEFAULT_SHOW, '', type_id)]
 
         if current_app.config.get('SECRETS_ENGINE') == 'vault':
             vault = VaultClient(current_app.config.get('VAULT_URL'), current_app.config.get('VAULT_TOKEN'))
             if value:
                 try:
-                    vault.update("/{}/{}".format(ci_id, attr_id), dict(v=value))
+                    vault.update("/{}/{}".format(ci_id, attr_id), dict(v=str(value)))
                 except Exception as e:
                     current_app.logger.error('save password to vault failed: {}'.format(e))
                     return abort(400, ErrFormat.password_save_failed.format('write vault failed'))
@@ -825,6 +915,149 @@ class CIManager(object):
 
             return data.get('v')
 
+    def baseline(self, ci_ids, before_date):
+        """
+        return CI changes
+        :param ci_ids:
+        :param before_date:
+        :return:
+        """
+        ci_list = self.get_cis_by_ids(ci_ids, ret_key=RetKey.ALIAS)
+        if not ci_list:
+            return dict()
+
+        ci2changed = dict()
+        changed = AttributeHistoryManger.get_records_for_attributes(
+            before_date, None, None, 1, 100000, None, None, ci_ids=ci_ids, more=True)[1]
+        for records in changed:
+            for change in records[1]:
+                if change['is_computed'] or change['is_password']:
+                    continue
+
+                if change.get('default') and change['default'].get('default') == AttributeDefaultValueEnum.UPDATED_AT:
+                    continue
+
+                ci2changed.setdefault(change['ci_id'], {})
+                item = (change['old'],
+                        change['new'],
+                        change.get('is_list'),
+                        change.get('value_type'),
+                        change['operate_type'])
+                if change.get('is_list'):
+                    ci2changed[change['ci_id']].setdefault(change.get('attr_alias'), []).append(item)
+                else:
+                    ci2changed[change['ci_id']].update({change.get('attr_alias'): item})
+
+        type2show_name = {}
+        result = []
+        for ci in ci_list:
+            list_attr2item = {}
+            for alias_name, v in (ci2changed.get(ci['_id']) or {}).items():
+                if not alias_name:
+                    continue
+                if alias_name == ci.get('unique_alias'):
+                    continue
+
+                if ci.get('_type') not in type2show_name:
+                    ci_type = CITypeCache.get(ci.get('_type'))
+                    show_id = ci_type.show_id or ci_type.unique_id
+                    type2show_name[ci['_type']] = AttributeCache.get(show_id).alias
+
+                if isinstance(v, list):
+                    for old, new, is_list, value_type, operate_type in v:
+                        if alias_name not in list_attr2item:
+                            list_attr2item[alias_name] = dict(instance=ci.get(type2show_name[ci['_type']]),
+                                                              attr_name=alias_name,
+                                                              value_type=value_type,
+                                                              is_list=is_list,
+                                                              ci_type=ci.get('ci_type'),
+                                                              unique_alias=ci.get('unique_alias'),
+                                                              unique_value=ci.get(ci['unique_alias']),
+                                                              cur=copy.deepcopy(ci.get(alias_name)),
+                                                              to=ci.get(alias_name) or [])
+
+                        old = ValueTypeMap.deserialize[value_type](old) if old else old
+                        new = ValueTypeMap.deserialize[value_type](new) if new else new
+                        if operate_type == OperateType.ADD:
+                            list_attr2item[alias_name]['to'].remove(new)
+                        elif operate_type == OperateType.DELETE and old not in list_attr2item[alias_name]['to']:
+                            list_attr2item[alias_name]['to'].append(old)
+                    continue
+
+                old, value_type = v[0], v[3]
+                old = ValueTypeMap.deserialize[value_type](old) if old else old
+                if isinstance(old, (datetime.datetime, datetime.date)):
+                    old = str(old)
+                if ci.get(alias_name) != old:
+                    item = dict(instance=ci.get(type2show_name[ci['_type']]),
+                                attr_name=alias_name,
+                                value_type=value_type,
+                                ci_type=ci.get('ci_type'),
+                                unique_alias=ci.get('unique_alias'),
+                                unique_value=ci.get(ci['unique_alias']),
+                                cur=ci.get(alias_name),
+                                to=old)
+                    result.append(item)
+
+            for alias_name, item in list_attr2item.items():
+                if sorted(item['cur'] or []) != sorted(item['to'] or []):
+                    result.append(item)
+
+        return result
+
+    def baseline_cis(self, ci_ids, before_date, fl=None):
+        """
+               return CI changes
+               :param ci_ids:
+               :param before_date:
+               :param fl:
+               :return:
+               """
+        ci_list = self.get_cis_by_ids(ci_ids, fields=fl)
+        if not ci_list:
+            return []
+
+        id2ci = {ci['_id']: ci for ci in ci_list}
+        changed = AttributeHistoryManger.get_records_for_attributes(
+            before_date, None, None, 1, 100000, None, None, ci_ids=ci_ids, more=True)[1]
+        for records in changed:
+            for change in records[1]:
+                if change['is_computed'] or change['is_password']:
+                    continue
+
+                if change.get('default') and change['default'].get('default') == AttributeDefaultValueEnum.UPDATED_AT:
+                    continue
+
+                if change['is_list']:
+                    old, new, value_type, operate_type, ci_id, attr_name = (
+                        change['old'], change['new'], change['value_type'], change['operate_type'],
+                        change['ci_id'], change['attr_name'])
+                    old = ValueTypeMap.deserialize[value_type](old) if old else old
+                    new = ValueTypeMap.deserialize[value_type](new) if new else new
+                    if operate_type == OperateType.ADD and new in (id2ci[ci_id][attr_name] or []):
+                        id2ci[ci_id][attr_name].remove(new)
+                    elif operate_type == OperateType.DELETE and old not in id2ci[ci_id][attr_name]:
+                        id2ci[ci_id][attr_name].append(old)
+                else:
+                    id2ci[change['ci_id']][change['attr_name']] = change['old']
+
+        return list(id2ci.values())
+
+    def rollback(self, ci_id, before_date):
+        baseline_ci = self.baseline([ci_id], before_date)
+
+        payload = dict()
+        for item in baseline_ci:
+            payload[item.get('attr_name')] = item.get('to')
+
+        if payload:
+            payload['ci_type'] = baseline_ci[0]['ci_type']
+            payload[baseline_ci[0]['unique_alias']] = baseline_ci[0]['unique_value']
+
+            self.update(ci_id, **payload)
+
+        return payload
+
 
 class CIRelationManager(object):
     """
@@ -935,6 +1168,18 @@ class CIRelationManager(object):
 
         return ci_ids, level2ids
 
+    @classmethod
+    def get_parent_ids(cls, ci_ids):
+        cis = db.session.query(CIRelation.first_ci_id, CIRelation.second_ci_id, CI.type_id).join(
+            CI, CI.id == CIRelation.first_ci_id).filter(
+            CIRelation.second_ci_id.in_(ci_ids)).filter(CIRelation.deleted.is_(False))
+
+        result = {}
+        for ci in cis:
+            result.setdefault(ci.second_ci_id, []).append((ci.first_ci_id, ci.type_id))
+
+        return result
+
     @staticmethod
     def _check_constraint(first_ci_id, first_type_id, second_ci_id, second_type_id, type_relation):
         db.session.commit()
@@ -960,7 +1205,14 @@ class CIRelationManager(object):
                     return abort(400, ErrFormat.relation_constraint.format("1-N"))
 
     @classmethod
-    def add(cls, first_ci_id, second_ci_id, more=None, relation_type_id=None, ancestor_ids=None, valid=True):
+    def add(cls, first_ci_id, second_ci_id,
+            more=None,
+            relation_type_id=None,
+            ancestor_ids=None,
+            valid=True,
+            apply_async=True,
+            source=None,
+            uid=None):
 
         first_ci = CIManager.confirm_ci_existed(first_ci_id)
         second_ci = CIManager.confirm_ci_existed(second_ci_id)
@@ -972,9 +1224,10 @@ class CIRelationManager(object):
                                     first=True)
         if existed is not None:
             if existed.relation_type_id != relation_type_id and relation_type_id is not None:
-                existed.update(relation_type_id=relation_type_id)
+                source = existed.source or source
+                existed.update(relation_type_id=relation_type_id, source=source)
 
-                CIRelationHistoryManager().add(existed, OperateType.UPDATE)
+                CIRelationHistoryManager().add(existed, OperateType.UPDATE, uid=uid)
         else:
             if relation_type_id is None:
                 type_relation = CITypeRelation.get_by(parent_id=first_ci.type_id,
@@ -985,7 +1238,7 @@ class CIRelationManager(object):
                 relation_type_id or abort(404, ErrFormat.relation_not_found.format("{} -> {}".format(
                     first_ci.ci_type.name, second_ci.ci_type.name)))
 
-                if current_app.config.get('USE_ACL') and valid:
+                if current_app.config.get('USE_ACL') and valid and current_user.username != 'worker':
                     resource_name = CITypeRelationManager.acl_resource_name(first_ci.ci_type.name,
                                                                             second_ci.ci_type.name)
                     if not ACLManager().has_permission(
@@ -1004,11 +1257,13 @@ class CIRelationManager(object):
                 existed = CIRelation.create(first_ci_id=first_ci_id,
                                             second_ci_id=second_ci_id,
                                             relation_type_id=relation_type_id,
-                                            ancestor_ids=ancestor_ids)
-
-                CIRelationHistoryManager().add(existed, OperateType.ADD)
-
-                ci_relation_cache.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE)
+                                            ancestor_ids=ancestor_ids,
+                                            source=source)
+                CIRelationHistoryManager().add(existed, OperateType.ADD, uid=uid)
+                if apply_async:
+                    ci_relation_cache.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE)
+                else:
+                    ci_relation_cache(first_ci_id, second_ci_id, ancestor_ids)
 
         if more is not None:
             existed.upadte(more=more)
@@ -1016,10 +1271,10 @@ class CIRelationManager(object):
         return existed.id
 
     @staticmethod
-    def delete(cr_id):
+    def delete(cr_id, apply_async=True):
         cr = CIRelation.get_by_id(cr_id) or abort(404, ErrFormat.relation_not_found.format("id={}".format(cr_id)))
 
-        if current_app.config.get('USE_ACL'):
+        if current_app.config.get('USE_ACL') and current_user.username != 'worker':
             resource_name = CITypeRelationManager.acl_resource_name(cr.first_ci.ci_type.name, cr.second_ci.ci_type.name)
             if not ACLManager().has_permission(
                     resource_name,
@@ -1032,8 +1287,12 @@ class CIRelationManager(object):
         his_manager = CIRelationHistoryManager()
         his_manager.add(cr, operate_type=OperateType.DELETE)
 
-        ci_relation_delete.apply_async(args=(cr.first_ci_id, cr.second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE)
-        delete_id_filter.apply_async(args=(cr.second_ci_id,), queue=CMDB_QUEUE)
+        if apply_async:
+            ci_relation_delete.apply_async(args=(cr.first_ci_id, cr.second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE)
+            delete_id_filter.apply_async(args=(cr.second_ci_id,), queue=CMDB_QUEUE)
+        else:
+            ci_relation_delete(cr.first_ci_id, cr.second_ci_id, cr.ancestor_ids)
+            delete_id_filter(cr.second_ci_id)
 
         return cr_id
 
@@ -1048,8 +1307,23 @@ class CIRelationManager(object):
         if cr is not None:
             cls.delete(cr.id)
 
-        ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE)
-        delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE)
+        # ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, ancestor_ids), queue=CMDB_QUEUE)
+        # delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE)
+
+        return cr
+
+    @classmethod
+    def delete_3(cls, first_ci_id, second_ci_id, apply_async=True):
+        cr = CIRelation.get_by(first_ci_id=first_ci_id,
+                               second_ci_id=second_ci_id,
+                               to_dict=False,
+                               first=True)
+
+        if cr is not None:
+            # ci_relation_delete.apply_async(args=(first_ci_id, second_ci_id, cr.ancestor_ids), queue=CMDB_QUEUE)
+            # delete_id_filter.apply_async(args=(second_ci_id,), queue=CMDB_QUEUE)
+
+            cls.delete(cr.id, apply_async=apply_async)
 
         return cr
 
@@ -1088,56 +1362,140 @@ class CIRelationManager(object):
                 for ci_id in ci_ids:
                     cls.delete_2(parent_id, ci_id, ancestor_ids=ancestor_ids)
 
+    @classmethod
+    def delete_relations_by_source(cls, source,
+                                   first_ci_id=None, second_ci_type_id=None,
+                                   second_ci_id=None, first_ci_type_id=None,
+                                   added=None):
+        existed = []
+        if first_ci_id is not None and second_ci_type_id is not None:
+            existed = [(i.first_ci_id, i.second_ci_id) for i in CIRelation.get_by(
+                source=source, first_ci_id=first_ci_id, only_query=True).join(
+                CI, CIRelation.second_ci_id == CI.id).filter(CI.type_id == second_ci_type_id)]
+
+        if second_ci_id is not None and first_ci_type_id is not None:
+            existed = [(i.first_ci_id, i.second_ci_id) for i in CIRelation.get_by(
+                source=source, second_ci_id=second_ci_id, only_query=True).join(
+                CI, CIRelation.first_ci_id == CI.id).filter(CI.type_id == first_ci_type_id)]
+
+        deleted = set(existed) - set(added or [])
+
+        for first, second in deleted:
+            cls.delete_3(first, second, apply_async=False)
+
     @classmethod
     def build_by_attribute(cls, ci_dict):
         type_id = ci_dict['_type']
         child_items = CITypeRelation.get_by(parent_id=type_id, only_query=True).filter(
-            CITypeRelation.parent_attr_id.isnot(None))
+            CITypeRelation.parent_attr_ids.isnot(None))
         for item in child_items:
-            parent_attr = AttributeCache.get(item.parent_attr_id)
-            child_attr = AttributeCache.get(item.child_attr_id)
-            attr_value = ci_dict.get(parent_attr.name)
-            value_table = TableMap(attr=child_attr).table
-            for child in value_table.get_by(attr_id=child_attr.id, value=attr_value, only_query=True).join(
-                    CI, CI.id == value_table.ci_id).filter(CI.type_id == item.child_id):
-                CIRelationManager.add(ci_dict['_id'], child.ci_id, valid=False)
+            relations = None
+            for parent_attr_id, child_attr_id in zip(item.parent_attr_ids, item.child_attr_ids):
+                _relations = set()
+                parent_attr = AttributeCache.get(parent_attr_id)
+                child_attr = AttributeCache.get(child_attr_id)
+                attr_value = ci_dict.get(parent_attr.name)
+                value_table = TableMap(attr=child_attr).table
+                for child in value_table.get_by(attr_id=child_attr.id, value=attr_value, only_query=True).join(
+                        CI, CI.id == value_table.ci_id).filter(CI.type_id == item.child_id):
+                    _relations.add((ci_dict['_id'], child.ci_id))
+                if relations is None:
+                    relations = _relations
+                else:
+                    relations &= _relations
+
+            cls.delete_relations_by_source(RelationSourceEnum.ATTRIBUTE_VALUES,
+                                           first_ci_id=ci_dict['_id'],
+                                           second_ci_type_id=item.child_id,
+                                           added=relations)
+            for parent_ci_id, child_ci_id in (relations or []):
+                cls.add(parent_ci_id, child_ci_id,
+                        valid=False,
+                        source=RelationSourceEnum.ATTRIBUTE_VALUES)
 
         parent_items = CITypeRelation.get_by(child_id=type_id, only_query=True).filter(
-            CITypeRelation.child_attr_id.isnot(None))
+            CITypeRelation.child_attr_ids.isnot(None))
         for item in parent_items:
-            parent_attr = AttributeCache.get(item.parent_attr_id)
-            child_attr = AttributeCache.get(item.child_attr_id)
-            attr_value = ci_dict.get(child_attr.name)
-            value_table = TableMap(attr=parent_attr).table
-            for parent in value_table.get_by(attr_id=parent_attr.id, value=attr_value, only_query=True).join(
-                    CI, CI.id == value_table.ci_id).filter(CI.type_id == item.parent_id):
-                CIRelationManager.add(parent.ci_id, ci_dict['_id'], valid=False)
+            relations = None
+            for parent_attr_id, child_attr_id in zip(item.parent_attr_ids, item.child_attr_ids):
+                _relations = set()
+                parent_attr = AttributeCache.get(parent_attr_id)
+                child_attr = AttributeCache.get(child_attr_id)
+                attr_value = ci_dict.get(child_attr.name)
+                value_table = TableMap(attr=parent_attr).table
+                for parent in value_table.get_by(attr_id=parent_attr.id, value=attr_value, only_query=True).join(
+                        CI, CI.id == value_table.ci_id).filter(CI.type_id == item.parent_id):
+                    _relations.add((parent.ci_id, ci_dict['_id']))
+                if relations is None:
+                    relations = _relations
+                else:
+                    relations &= _relations
+
+            cls.delete_relations_by_source(RelationSourceEnum.ATTRIBUTE_VALUES,
+                                           second_ci_id=ci_dict['_id'],
+                                           first_ci_type_id=item.parent_id,
+                                           added=relations)
+            for parent_ci_id, child_ci_id in (relations or []):
+                cls.add(parent_ci_id, child_ci_id,
+                        valid=False,
+                        source=RelationSourceEnum.ATTRIBUTE_VALUES)
 
     @classmethod
-    def rebuild_all_by_attribute(cls, ci_type_relation):
-        parent_attr = AttributeCache.get(ci_type_relation['parent_attr_id'])
-        child_attr = AttributeCache.get(ci_type_relation['child_attr_id'])
-        if not parent_attr or not child_attr:
-            return
+    def rebuild_all_by_attribute(cls, ci_type_relation, uid):
+        relations = None
+        for parent_attr_id, child_attr_id in zip(ci_type_relation['parent_attr_ids'] or [],
+                                                 ci_type_relation['child_attr_ids'] or []):
 
-        parent_value_table = TableMap(attr=parent_attr).table
-        child_value_table = TableMap(attr=child_attr).table
+            _relations = set()
+            parent_attr = AttributeCache.get(parent_attr_id)
+            child_attr = AttributeCache.get(child_attr_id)
+            if not parent_attr or not child_attr:
+                continue
 
-        parent_values = parent_value_table.get_by(attr_id=parent_attr.id, only_query=True).join(
-            CI, CI.id == parent_value_table.ci_id).filter(CI.type_id == ci_type_relation['parent_id'])
-        child_values = child_value_table.get_by(attr_id=child_attr.id, only_query=True).join(
-            CI, CI.id == child_value_table.ci_id).filter(CI.type_id == ci_type_relation['child_id'])
+            parent_value_table = TableMap(attr=parent_attr).table
+            child_value_table = TableMap(attr=child_attr).table
 
-        child_value2ci_ids = {}
-        for child in child_values:
-            child_value2ci_ids.setdefault(child.value, []).append(child.ci_id)
+            parent_values = parent_value_table.get_by(attr_id=parent_attr.id, only_query=True).join(
+                CI, CI.id == parent_value_table.ci_id).filter(CI.type_id == ci_type_relation['parent_id'])
+            child_values = child_value_table.get_by(attr_id=child_attr.id, only_query=True).join(
+                CI, CI.id == child_value_table.ci_id).filter(CI.type_id == ci_type_relation['child_id'])
 
-        for parent in parent_values:
-            for child_ci_id in child_value2ci_ids.get(parent.value, []):
-                try:
-                    cls.add(parent.ci_id, child_ci_id, valid=False)
-                except:
-                    pass
+            child_value2ci_ids = {}
+            for child in child_values:
+                child_value2ci_ids.setdefault(child.value, []).append(child.ci_id)
+
+            for parent in parent_values:
+                for child_ci_id in child_value2ci_ids.get(parent.value, []):
+                    _relations.add((parent.ci_id, child_ci_id))
+
+            if relations is None:
+                relations = _relations
+            else:
+                relations &= _relations
+
+        t1 = aliased(CI)
+        t2 = aliased(CI)
+        query = db.session.query(CIRelation).join(t1, t1.id == CIRelation.first_ci_id).join(
+            t2, t2.id == CIRelation.second_ci_id).filter(t1.type_id == ci_type_relation['parent_id']).filter(
+            t2.type_id == ci_type_relation['child_id'])
+        for i in query:
+            db.session.delete(i)
+            ci_relation_delete(i.first_ci_id, i.second_ci_id, i.ancestor_ids)
+        try:
+            db.session.commit()
+        except Exception as e:
+            current_app.logger.error(e)
+            db.session.rollback()
+
+        for parent_ci_id, child_ci_id in (relations or []):
+            try:
+                cls.add(parent_ci_id, child_ci_id,
+                        valid=False,
+                        apply_async=False,
+                        source=RelationSourceEnum.ATTRIBUTE_VALUES,
+                        uid=uid)
+            except Exception as e:
+                current_app.logger.error(e)
 
 
 class CITriggerManager(object):
@@ -1166,7 +1524,8 @@ class CITriggerManager(object):
                 cls._update_old_attr_value(record_id, ci_dict)
 
             if ci_id is not None:
-                ci_dict = CIManager().get_ci_by_id_from_db(ci_id, need_children=False, use_master=False)
+                ci_dict = CIManager().get_ci_by_id_from_db(
+                    ci_id, need_children=False, use_master=False, enum_use_label=True)
 
             try:
                 response = webhook_request(webhook, ci_dict).text
@@ -1193,7 +1552,8 @@ class CITriggerManager(object):
         with app.app_context():
 
             if ci_id is not None:
-                ci_dict = CIManager().get_ci_by_id_from_db(ci_id, need_children=False, use_master=False)
+                ci_dict = CIManager().get_ci_by_id_from_db(
+                    ci_id, need_children=False, use_master=False, enum_use_label=True)
 
             if operate_type == OperateType.UPDATE:
                 cls._update_old_attr_value(record_id, ci_dict)

cmdb-api/api/lib/cmdb/ci_type.py

@@ -1,11 +1,12 @@
 # -*- coding:utf-8 -*-
 
-import copy
+from collections import defaultdict
 
+import copy
+import networkx as nx
 import toposort
 from flask import abort
 from flask import current_app
-from flask import session
 from flask_login import current_user
 from toposort import toposort_flatten
 from werkzeug.exceptions import BadRequest
@@ -28,12 +29,15 @@ from api.lib.cmdb.perms import CIFilterPermsCRUD
 from api.lib.cmdb.relation_type import RelationTypeManager
 from api.lib.cmdb.resp_format import ErrFormat
 from api.lib.cmdb.value import AttributeValueManager
+from api.lib.common_setting.role_perm_base import CMDBApp
 from api.lib.decorator import kwargs_required
 from api.lib.perm.acl.acl import ACLManager
 from api.lib.perm.acl.acl import is_app_admin
+from api.lib.perm.acl.acl import validate_permission
 from api.models.cmdb import Attribute
 from api.models.cmdb import AutoDiscoveryCI
 from api.models.cmdb import AutoDiscoveryCIType
+from api.models.cmdb import AutoDiscoveryCITypeRelation
 from api.models.cmdb import CI
 from api.models.cmdb import CIFilterPerms
 from api.models.cmdb import CIType
@@ -53,6 +57,7 @@ from api.models.cmdb import PreferenceSearchOption
 from api.models.cmdb import PreferenceShowAttributes
 from api.models.cmdb import PreferenceTreeView
 from api.models.cmdb import RelationType
+from api.models.cmdb import TopologyView
 
 
 class CITypeManager(object):
@@ -81,7 +86,7 @@ class CITypeManager(object):
             self.cls.id, self.cls.icon, self.cls.name).filter(self.cls.deleted.is_(False))}
 
     @staticmethod
-    def get_ci_types(type_name=None, like=True):
+    def get_ci_types(type_name=None, like=True, type_ids=None):
         resources = None
         if current_app.config.get('USE_ACL') and not is_app_admin('cmdb'):
             resources = set([i.get('name') for i in ACLManager().get_resources(ResourceTypeEnum.CI_TYPE)])
@@ -90,6 +95,9 @@ class CITypeManager(object):
             CIType.get_by_like(name=type_name) if like else CIType.get_by(name=type_name))
         res = list()
         for type_dict in ci_types:
+            if type_ids is not None and type_dict['id'] not in type_ids:
+                continue
+
             attr = AttributeCache.get(type_dict["unique_id"])
             type_dict["unique_key"] = attr and attr.name
             if type_dict.get('show_id'):
@@ -130,7 +138,9 @@ class CITypeManager(object):
     def add(cls, **kwargs):
         if current_app.config.get('USE_ACL') and not is_app_admin('cmdb'):
             if ErrFormat.ci_type_config not in {i['name'] for i in ACLManager().get_resources(ResourceTypeEnum.PAGE)}:
-                return abort(403, ErrFormat.no_permission2)
+                app_cli = CMDBApp()
+                validate_permission(app_cli.op.Model_Configuration, app_cli.resource_type_name,
+                                    app_cli.op.create_CIType, app_cli.app_name)
 
         unique_key = kwargs.pop("unique_key", None) or kwargs.pop("unique_id", None)
         unique_key = AttributeCache.get(unique_key) or abort(404, ErrFormat.unique_key_not_define)
@@ -138,7 +148,7 @@ class CITypeManager(object):
         kwargs["alias"] = kwargs["name"] if not kwargs.get("alias") else kwargs["alias"]
 
         cls._validate_unique(name=kwargs['name'])
-        cls._validate_unique(alias=kwargs['alias'])
+        # cls._validate_unique(alias=kwargs['alias'])
 
         kwargs["unique_id"] = unique_key.id
         kwargs['uid'] = current_user.uid
@@ -177,7 +187,7 @@ class CITypeManager(object):
         ci_type = cls.check_is_existed(type_id)
 
         cls._validate_unique(type_id=type_id, name=kwargs.get('name'))
-        cls._validate_unique(type_id=type_id, alias=kwargs.get('alias') or kwargs.get('name'))
+        # cls._validate_unique(type_id=type_id, alias=kwargs.get('alias') or kwargs.get('name'))
 
         unique_key = kwargs.pop("unique_key", None)
         unique_key = AttributeCache.get(unique_key)
@@ -224,6 +234,13 @@ class CITypeManager(object):
         if CI.get_by(type_id=type_id, first=True, to_dict=False) is not None:
             return abort(400, ErrFormat.ci_exists_and_cannot_delete_type)
 
+        if CITypeInheritance.get_by(parent_id=type_id, first=True):
+            return abort(400, ErrFormat.ci_type_inheritance_cannot_delete)
+
+        reference = Attribute.get_by(reference_type_id=type_id, first=True, to_dict=False)
+        if reference is not None:
+            return abort(400, ErrFormat.ci_type_referenced_cannot_delete.format(reference.alias))
+
         relation_views = PreferenceRelationView.get_by(to_dict=False)
         for rv in relation_views:
             for item in (rv.cr_ids or []):
@@ -247,10 +264,26 @@ class CITypeManager(object):
         for item in AutoDiscoveryCI.get_by(type_id=type_id, to_dict=False):
             item.delete(commit=False)
 
+        for item in AutoDiscoveryCITypeRelation.get_by(ad_type_id=type_id, to_dict=False):
+            item.soft_delete(commit=False)
+
+        for item in AutoDiscoveryCITypeRelation.get_by(peer_type_id=type_id, to_dict=False):
+            item.soft_delete(commit=False)
+
         for item in CITypeInheritance.get_by(parent_id=type_id, to_dict=False):
-            item.delete(commit=False)
+            item.soft_delete(commit=False)
 
         for item in CITypeInheritance.get_by(child_id=type_id, to_dict=False):
+            item.soft_delete(commit=False)
+
+        try:
+            from api.models.cmdb import CITypeReconciliation
+            for item in CITypeReconciliation.get_by(type_id=type_id, to_dict=False):
+                item.soft_delete(commit=False)
+        except Exception:
+            pass
+
+        for item in TopologyView.get_by(central_node_type=type_id, to_dict=False):
             item.delete(commit=False)
 
         db.session.commit()
@@ -268,6 +301,12 @@ class CITypeManager(object):
 class CITypeInheritanceManager(object):
     cls = CITypeInheritance
 
+    @classmethod
+    def get_all(cls, type_ids=None):
+        res = cls.cls.get_by(to_dict=True)
+
+        return [i for i in res if type_ids is None or (i['parent_id'] in type_ids and i['child_id'] in type_ids)]
+
     @classmethod
     def get_parents(cls, type_id):
         return [i.parent_id for i in cls.cls.get_by(child_id=type_id, to_dict=False)]
@@ -311,9 +350,9 @@ class CITypeInheritanceManager(object):
     @classmethod
     def add(cls, parent_ids, child_id):
 
-        rels = {}
+        rels = defaultdict(set)
         for i in cls.cls.get_by(to_dict=False):
-            rels.setdefault(i.child_id, set()).add(i.parent_id)
+            rels[i.child_id].add(i.parent_id)
 
         try:
             toposort_flatten(rels)
@@ -327,7 +366,7 @@ class CITypeInheritanceManager(object):
 
             existed = cls.cls.get_by(parent_id=parent_id, child_id=child_id, first=True, to_dict=False)
             if existed is None:
-                rels.setdefault(child_id, set()).add(parent_id)
+                rels[child_id].add(parent_id)
                 try:
                     toposort_flatten(rels)
                 except toposort.CircularDependencyError as e:
@@ -363,7 +402,7 @@ class CITypeGroupManager(object):
     cls = CITypeGroup
 
     @staticmethod
-    def get(need_other=None, config_required=True):
+    def get(need_other=None, config_required=True, type_ids=None, ci_types=None):
         resources = None
         if current_app.config.get('USE_ACL'):
             resources = ACLManager('cmdb').get_resources(ResourceTypeEnum.CI)
@@ -377,6 +416,8 @@ class CITypeGroupManager(object):
         for group in groups:
             for t in sorted(CITypeGroupItem.get_by(group_id=group['id']), key=lambda x: x['order'] or 0):
                 ci_type = CITypeCache.get(t['type_id']).to_dict()
+                if type_ids is not None and ci_type['id'] not in type_ids:
+                    continue
                 if resources is None or (ci_type and ci_type['name'] in resources):
                     ci_type['permissions'] = resources[ci_type['name']] if resources is not None else None
                     ci_type['inherited'] = True if CITypeInheritanceManager.get_parents(ci_type['id']) else False
@@ -384,7 +425,7 @@ class CITypeGroupManager(object):
                     group_types.add(t["type_id"])
 
         if need_other:
-            ci_types = CITypeManager.get_ci_types()
+            ci_types = CITypeManager.get_ci_types(type_ids=type_ids) if ci_types is None else ci_types
             other_types = dict(ci_types=[])
             for ci_type in ci_types:
                 if ci_type["id"] not in group_types and (resources is None or ci_type['name'] in resources):
@@ -414,9 +455,6 @@ class CITypeGroupManager(object):
         existed = CITypeGroup.get_by_id(gid) or abort(
             404, ErrFormat.ci_type_group_not_found.format("id={}".format(gid)))
         if name is not None and name != existed.name:
-            if RoleEnum.CONFIG not in session.get("acl", {}).get("parentRoles", []) and not is_app_admin("cmdb"):
-                return abort(403, ErrFormat.role_required.format(RoleEnum.CONFIG))
-
             existed.update(name=name)
 
         max_order = max([i.order or 0 for i in CITypeGroupItem.get_by(group_id=gid, to_dict=False)] or [0])
@@ -469,14 +507,13 @@ class CITypeAttributeManager(object):
     def __init__(self):
         pass
 
-    @staticmethod
-    def get_attr_name(ci_type_name, key):
+    @classmethod
+    def get_attr_name(cls, ci_type_name, key):
         ci_type = CITypeCache.get(ci_type_name)
         if ci_type is None:
             return
 
-        for i in CITypeAttributesCache.get(ci_type.id):
-            attr = AttributeCache.get(i.attr_id)
+        for _, attr in cls.get_all_attributes(ci_type.id):
             if attr and (attr.name == key or attr.alias == key):
                 return attr.name
 
@@ -488,12 +525,31 @@ class CITypeAttributeManager(object):
         for _type_id in parent_ids + [type_id]:
             result.extend(CITypeAttributesCache.get2(_type_id))
 
-        return result
+        attr_ids = set()
+        result2 = []
+        for i in result:
+            if i[1].id not in attr_ids:
+                result2.append(i)
+                attr_ids.add(i[1].id)
+
+        return result2
 
     @classmethod
     def get_attr_names_by_type_id(cls, type_id):
         return [attr.name for _, attr in cls.get_all_attributes(type_id)]
 
+    @classmethod
+    def get_attr_names_label_enum(cls, type_id):
+        attr_names, enum_map = list(), defaultdict(dict)
+        for _, attr in cls.get_all_attributes(type_id):
+            attr_names.append(attr.name)
+            if attr.is_choice and not attr.choice_other and not attr.choice_web_hook:
+                _map = AttributeManager.get_enum_map(attr.id)
+                if _map:
+                    enum_map[attr.name].update(_map)
+
+        return attr_names, enum_map
+
     @staticmethod
     def get_attributes_by_type_id(type_id, choice_web_hook_parse=True, choice_other_parse=True):
         has_config_perm = ACLManager('cmdb').has_permission(
@@ -508,6 +564,8 @@ class CITypeAttributeManager(object):
             attrs = CITypeAttributesCache.get(_type_id)
             for attr in sorted(attrs, key=lambda x: (x.order, x.id)):
                 attr_dict = AttributeManager().get_attribute(attr.attr_id, choice_web_hook_parse, choice_other_parse)
+                if not attr_dict:
+                    continue
                 attr_dict["is_required"] = attr.is_required
                 attr_dict["order"] = attr.order
                 attr_dict["default_show"] = attr.default_show
@@ -516,7 +574,6 @@ class CITypeAttributeManager(object):
                 if not has_config_perm:
                     attr_dict.pop('choice_web_hook', None)
                     attr_dict.pop('choice_other', None)
-
                 if attr_dict['id'] not in id2pos:
                     id2pos[attr_dict['id']] = len(result)
                     result.append(attr_dict)
@@ -533,10 +590,10 @@ class CITypeAttributeManager(object):
                 CITypeManager.get_name_by_id(type_id), ResourceTypeEnum.CI, PermEnum.CONFIG)
 
         result = {type_id: [i for _, i in cls.get_all_attributes(type_id)] for type_id in type_ids}
-        attr2types = {}
+        attr2types = defaultdict(list)
         for type_id in result:
             for i in result[type_id]:
-                attr2types.setdefault(i.id, []).append(type_id)
+                attr2types[i.id].append(type_id)
 
         attrs = []
         for attr_id in attr2types:
@@ -581,7 +638,6 @@ class CITypeAttributeManager(object):
             if existed is not None:
                 continue
 
-            current_app.logger.debug(attr_id)
             CITypeAttribute.create(type_id=type_id, attr_id=attr_id, **kwargs)
 
             attr = AttributeCache.get(attr_id)
@@ -675,9 +731,19 @@ class CITypeAttributeManager(object):
                     item = CITypeTrigger.get_by(type_id=_type_id, attr_id=attr_id, to_dict=False, first=True)
                     item and item.soft_delete(commit=False)
 
-                for item in (CITypeRelation.get_by(parent_id=type_id, parent_attr_id=attr_id, to_dict=False) +
-                             CITypeRelation.get_by(child_id=type_id, child_attr_id=attr_id, to_dict=False)):
-                    item.soft_delete(commit=False)
+                for item in (CITypeRelation.get_by(parent_id=type_id, to_dict=False) +
+                             CITypeRelation.get_by(child_id=type_id, to_dict=False)):
+                    if item.parent_id == type_id and attr_id in (item.parent_attr_ids or []):
+                        item_dict = item.to_dict()
+                        pop_idx = item.parent_attr_ids.index(attr_id)
+                    elif item.child_id == type_id and attr_id in (item.child_attr_ids or []):
+                        item_dict = item.to_dict()
+                        pop_idx = item.child_attr_ids.index(attr_id)
+                    else:
+                        continue
+                    item.update(parent_attr_ids=item_dict['parent_attr_ids'].pop(pop_idx),
+                                child_attr_ids=item_dict['child_attr_ids'].pop(pop_idx),
+                                commit=False)
 
                 db.session.commit()
 
@@ -738,21 +804,29 @@ class CITypeRelationManager(object):
     """
 
     @staticmethod
-    def get():
+    def get(type_ids=None):
         res = CITypeRelation.get_by(to_dict=False)
         type2attributes = dict()
+        result = []
         for idx, item in enumerate(res):
             _item = item.to_dict()
+            if type_ids is not None and _item['parent_id'] not in type_ids and _item['child_id'] not in type_ids:
+                continue
+
             res[idx] = _item
             res[idx]['parent'] = item.parent.to_dict()
             if item.parent_id not in type2attributes:
-                type2attributes[item.parent_id] = [i[1].to_dict() for i in CITypeAttributesCache.get2(item.parent_id)]
+                type2attributes[item.parent_id] = [i[1].to_dict() for i in
+                                                   CITypeAttributeManager.get_all_attributes(item.parent_id)]
             res[idx]['child'] = item.child.to_dict()
             if item.child_id not in type2attributes:
-                type2attributes[item.child_id] = [i[1].to_dict() for i in CITypeAttributesCache.get2(item.child_id)]
+                type2attributes[item.child_id] = [i[1].to_dict() for i in
+                                                  CITypeAttributeManager.get_all_attributes(item.child_id)]
             res[idx]['relation_type'] = item.relation_type.to_dict()
 
-        return res, type2attributes
+            result.append(res[idx])
+
+        return result, type2attributes
 
     @staticmethod
     def get_child_type_ids(type_id, level):
@@ -772,6 +846,29 @@ class CITypeRelationManager(object):
 
         return ids
 
+    @staticmethod
+    def find_path(source_type_id, target_type_ids):
+        source_type_id = int(source_type_id)
+        target_type_ids = map(int, target_type_ids)
+
+        graph = nx.DiGraph()
+
+        def get_children(_id):
+            children = CITypeRelation.get_by(parent_id=_id, to_dict=False)
+
+            for i in children:
+                if i.child_id != _id:
+                    graph.add_edge(i.parent_id, i.child_id)
+                    get_children(i.child_id)
+
+        get_children(source_type_id)
+
+        paths = list(nx.all_simple_paths(graph, source_type_id, target_type_ids))
+
+        del graph
+
+        return paths
+
     @staticmethod
     def _wrap_relation_type_dict(type_id, relation_inst):
         ci_type_dict = CITypeCache.get(type_id).to_dict()
@@ -784,8 +881,8 @@ class CITypeRelationManager(object):
 
         ci_type_dict["relation_type"] = relation_inst.relation_type.name
         ci_type_dict["constraint"] = relation_inst.constraint
-        ci_type_dict["parent_attr_id"] = relation_inst.parent_attr_id
-        ci_type_dict["child_attr_id"] = relation_inst.child_attr_id
+        ci_type_dict["parent_attr_ids"] = relation_inst.parent_attr_ids
+        ci_type_dict["child_attr_ids"] = relation_inst.child_attr_ids
 
         return ci_type_dict
 
@@ -797,12 +894,12 @@ class CITypeRelationManager(object):
 
     @classmethod
     def recursive_level2children(cls, parent_id):
-        result = dict()
+        result = defaultdict(list)
 
         def get_children(_id, level):
             children = CITypeRelation.get_by(parent_id=_id, to_dict=False)
             if children:
-                result.setdefault(level + 1, []).extend([i.child.to_dict() for i in children])
+                result[level + 1].extend([i.child.to_dict() for i in children])
 
             for i in children:
                 if i.child_id != _id:
@@ -818,6 +915,72 @@ class CITypeRelationManager(object):
 
         return [cls._wrap_relation_type_dict(parent.parent_id, parent) for parent in parents]
 
+    @staticmethod
+    def get_relations_by_type_id(type_id):
+        nodes, edges = [], []
+        node_ids, edge_tuples = set(), set()
+        ci_type = CITypeCache.get(type_id)
+        if ci_type is None:
+            return nodes, edges
+
+        ci_type_dict = ci_type.to_dict()
+        ci_type_dict.setdefault('level', [0])
+        nodes.append(ci_type_dict)
+        node_ids.add(ci_type.id)
+
+        def _find(_id, lv):
+            lv += 1
+            for i in CITypeRelation.get_by(parent_id=_id, to_dict=False):
+                if i.child_id not in node_ids:
+                    node_ids.add(i.child_id)
+                    node = i.child.to_dict()
+                    node.setdefault('level', []).append(lv)
+                    nodes.append(node)
+
+                    edges.append(dict(from_id=i.parent_id, to_id=i.child_id, text=i.relation_type.name, reverse=False))
+                    edge_tuples.add((i.parent_id, i.child_id))
+
+                    _find(i.child_id, lv)
+                    continue
+                elif (i.parent_id, i.child_id) not in edge_tuples:
+                    edges.append(dict(from_id=i.parent_id, to_id=i.child_id, text=i.relation_type.name, reverse=False))
+                    edge_tuples.add((i.parent_id, i.child_id))
+                    _find(i.child_id, lv)
+
+                for _node in nodes:
+                    if _node['id'] == i.child_id and lv not in _node['level']:
+                        _node['level'].append(lv)
+
+        def _reverse_find(_id, lv):
+            lv -= 1
+            for i in CITypeRelation.get_by(child_id=_id, to_dict=False):
+                if i.parent_id not in node_ids:
+                    node_ids.add(i.parent_id)
+                    node = i.parent.to_dict()
+                    node.setdefault('level', []).append(lv)
+                    nodes.append(node)
+
+                    edges.append(dict(from_id=i.parent_id, to_id=i.child_id, text=i.relation_type.name, reverse=True))
+                    edge_tuples.add((i.parent_id, i.child_id))
+
+                    _reverse_find(i.parent_id, lv)
+                    continue
+                elif (i.parent_id, i.child_id) not in edge_tuples:
+                    edges.append(dict(from_id=i.parent_id, to_id=i.child_id, text=i.relation_type.name, reverse=True))
+                    edge_tuples.add((i.parent_id, i.child_id))
+                    _reverse_find(i.parent_id, lv)
+
+                for _node in nodes:
+                    if _node['id'] == i.child_id and lv not in _node['level']:
+                        _node['level'].append(lv)
+
+        level = 0
+        _reverse_find(ci_type.id, level)
+        level = 0
+        _find(ci_type.id, level)
+
+        return nodes, edges
+
     @staticmethod
     def _get(parent_id, child_id):
         return CITypeRelation.get_by(parent_id=parent_id,
@@ -831,14 +994,14 @@ class CITypeRelationManager(object):
 
     @classmethod
     def add(cls, parent, child, relation_type_id, constraint=ConstraintEnum.One2Many,
-            parent_attr_id=None, child_attr_id=None):
+            parent_attr_ids=None, child_attr_ids=None):
         p = CITypeManager.check_is_existed(parent)
         c = CITypeManager.check_is_existed(child)
 
-        rels = {}
+        rels = defaultdict(set)
         for i in CITypeRelation.get_by(to_dict=False):
-            rels.setdefault(i.child_id, set()).add(i.parent_id)
-        rels.setdefault(c.id, set()).add(p.id)
+            rels[i.child_id].add(i.parent_id)
+        rels[c.id].add(p.id)
 
         try:
             toposort_flatten(rels)
@@ -846,21 +1009,22 @@ class CITypeRelationManager(object):
             current_app.logger.warning(str(e))
             return abort(400, ErrFormat.circular_dependency_error)
 
-        old_parent_attr_id = None
+        old_parent_attr_ids, old_child_attr_ids = None, None
         existed = cls._get(p.id, c.id)
         if existed is not None:
-            old_parent_attr_id = existed.parent_attr_id
+            old_parent_attr_ids = copy.deepcopy(existed.parent_attr_ids)
+            old_child_attr_ids = copy.deepcopy(existed.child_attr_ids)
             existed = existed.update(relation_type_id=relation_type_id,
                                      constraint=constraint,
-                                     parent_attr_id=parent_attr_id,
-                                     child_attr_id=child_attr_id,
+                                     parent_attr_ids=parent_attr_ids,
+                                     child_attr_ids=child_attr_ids,
                                      filter_none=False)
         else:
             existed = CITypeRelation.create(parent_id=p.id,
                                             child_id=c.id,
                                             relation_type_id=relation_type_id,
-                                            parent_attr_id=parent_attr_id,
-                                            child_attr_id=child_attr_id,
+                                            parent_attr_ids=parent_attr_ids,
+                                            child_attr_ids=child_attr_ids,
                                             constraint=constraint)
 
             if current_app.config.get("USE_ACL"):
@@ -874,10 +1038,10 @@ class CITypeRelationManager(object):
                                                     current_user.username,
                                                     ResourceTypeEnum.CI_TYPE_RELATION)
 
-        if parent_attr_id and parent_attr_id != old_parent_attr_id:
-            if parent_attr_id and parent_attr_id != existed.parent_attr_id:
-                from api.tasks.cmdb import rebuild_relation_for_attribute_changed
-                rebuild_relation_for_attribute_changed.apply_async(args=(existed.to_dict()))
+        if ((parent_attr_ids and parent_attr_ids != old_parent_attr_ids) or
+                (child_attr_ids and child_attr_ids != old_child_attr_ids)):
+            from api.tasks.cmdb import rebuild_relation_for_attribute_changed
+            rebuild_relation_for_attribute_changed.apply_async(args=(existed.to_dict(), current_user.uid))
 
         CITypeHistoryManager.add(CITypeOperateType.ADD_RELATION, p.id,
                                  change=dict(parent=p.to_dict(), child=c.to_dict(), relation_type_id=relation_type_id))
@@ -934,7 +1098,8 @@ class CITypeAttributeGroupManager(object):
         parent_ids = CITypeInheritanceManager.base(type_id)
 
         groups = []
-        id2type = {i: CITypeCache.get(i).alias for i in parent_ids}
+        id2type = {i: CITypeCache.get(i) for i in parent_ids}
+        id2type = {k: v.alias for k, v in id2type.items() if v}
         for _type_id in parent_ids + [type_id]:
             _groups = CITypeAttributeGroup.get_by(type_id=_type_id)
             _groups = sorted(_groups, key=lambda x: x["order"] or 0)
@@ -1127,17 +1292,16 @@ class CITypeAttributeGroupManager(object):
         if isinstance(_from, int):
             from_group = CITypeAttributeGroup.get_by_id(_from)
         else:
-            from_group = CITypeAttributeGroup.get_by(name=_from, first=True, to_dict=False)
+            from_group = CITypeAttributeGroup.get_by(name=_from, type_id=type_id, first=True, to_dict=False)
         from_group or abort(404, ErrFormat.ci_type_attribute_group_not_found.format("id={}".format(_from)))
 
         if isinstance(_to, int):
             to_group = CITypeAttributeGroup.get_by_id(_to)
         else:
-            to_group = CITypeAttributeGroup.get_by(name=_to, first=True, to_dict=False)
+            to_group = CITypeAttributeGroup.get_by(name=_to, type_id=type_id, first=True, to_dict=False)
         to_group or abort(404, ErrFormat.ci_type_attribute_group_not_found.format("id={}".format(_to)))
 
         from_order, to_order = from_group.order, to_group.order
-
         from_group.update(order=to_order)
         to_group.update(order=from_order)
 
@@ -1171,8 +1335,8 @@ class CITypeTemplateManager(object):
                                                  id2obj_dicts[added_id].get('child_id'),
                                                  id2obj_dicts[added_id].get('relation_type_id'),
                                                  id2obj_dicts[added_id].get('constraint'),
-                                                 id2obj_dicts[added_id].get('parent_attr_id'),
-                                                 id2obj_dicts[added_id].get('child_attr_id'),
+                                                 id2obj_dicts[added_id].get('parent_attr_ids'),
+                                                 id2obj_dicts[added_id].get('child_attr_ids'),
                                                  )
             else:
                 obj = cls.create(flush=True, **id2obj_dicts[added_id])
@@ -1207,19 +1371,25 @@ class CITypeTemplateManager(object):
     def _import_attributes(self, type2attributes):
         attributes = [attr for type_id in type2attributes for attr in type2attributes[type_id]]
         attrs = []
+        references = []
         for i in copy.deepcopy(attributes):
+            if i.pop('inherited', None):
+                continue
             i.pop('default_show', None)
             i.pop('is_required', None)
             i.pop('order', None)
             i.pop('choice_web_hook', None)
             i.pop('choice_other', None)
             i.pop('order', None)
-            i.pop('inherited', None)
             i.pop('inherited_from', None)
             choice_value = i.pop('choice_value', None)
             if not choice_value:
                 i['is_choice'] = False
 
+            if i.get('reference_type_id'):
+                references.append(copy.deepcopy(i))
+                i.pop('reference_type_id')
+
             attrs.append((i, choice_value))
 
         attr_id_map = self.__import(Attribute, [i[0] for i in copy.deepcopy(attrs)])
@@ -1228,12 +1398,13 @@ class CITypeTemplateManager(object):
             if choice_value and not i.get('choice_web_hook') and not i.get('choice_other'):
                 AttributeManager.add_choice_values(attr_id_map.get(i['id'], i['id']), i['value_type'], choice_value)
 
-        return attr_id_map
+        return attr_id_map, references
 
     def _import_ci_types(self, ci_types, attr_id_map):
         for i in ci_types:
             i.pop("unique_key", None)
             i.pop("show_name", None)
+            i.pop("parent_ids", None)
             i['unique_id'] = attr_id_map.get(i['unique_id'], i['unique_id'])
             if i.get('show_id'):
                 i['show_id'] = attr_id_map.get(i['show_id'], i['show_id'])
@@ -1241,6 +1412,11 @@ class CITypeTemplateManager(object):
 
         return self.__import(CIType, ci_types)
 
+    def _import_reference_attributes(self, attrs, type_id_map):
+        for attr in attrs:
+            attr['reference_type_id'] = type_id_map.get(attr['reference_type_id'], attr['reference_type_id'])
+        self.__import(Attribute, attrs)
+
     def _import_ci_type_groups(self, ci_type_groups, type_id_map):
         _ci_type_groups = copy.deepcopy(ci_type_groups)
         for i in _ci_type_groups:
@@ -1254,6 +1430,10 @@ class CITypeTemplateManager(object):
                 payload = dict(group_id=group_id_map.get(group['id'], group['id']),
                                type_id=type_id_map.get(ci_type['id'], ci_type['id']),
                                order=order)
+                for i in CITypeGroupItem.get_by(type_id=payload['type_id'], to_dict=False):
+                    if i.group_id != payload['group_id']:
+                        i.soft_delete(flush=True)
+
                 existed = CITypeGroupItem.get_by(group_id=payload['group_id'], type_id=payload['type_id'],
                                                  first=True, to_dict=False)
                 if existed is None:
@@ -1271,7 +1451,7 @@ class CITypeTemplateManager(object):
         return self.__import(RelationType, relation_types)
 
     @staticmethod
-    def _import_ci_type_relations(ci_type_relations, type_id_map, relation_type_id_map):
+    def _import_ci_type_relations(ci_type_relations, type_id_map, relation_type_id_map, attr_id_map):
         for i in ci_type_relations:
             i.pop('parent', None)
             i.pop('child', None)
@@ -1281,15 +1461,32 @@ class CITypeTemplateManager(object):
             i['child_id'] = type_id_map.get(i['child_id'], i['child_id'])
             i['relation_type_id'] = relation_type_id_map.get(i['relation_type_id'], i['relation_type_id'])
 
+            i['parent_attr_ids'] = [attr_id_map.get(attr_id, attr_id) for attr_id in i.get('parent_attr_ids') or []]
+            i['child_attr_ids'] = [attr_id_map.get(attr_id, attr_id) for attr_id in i.get('child_attr_ids') or []]
             try:
                 CITypeRelationManager.add(i.get('parent_id'),
                                           i.get('child_id'),
                                           i.get('relation_type_id'),
                                           i.get('constraint'),
+                                          parent_attr_ids=i.get('parent_attr_ids', []),
+                                          child_attr_ids=i.get('child_attr_ids', []),
                                           )
-            except BadRequest:
+            except Exception:
                 pass
 
+    @staticmethod
+    def _import_ci_type_inheritance(ci_type_inheritance, type_id_map):
+
+        for i in ci_type_inheritance:
+            i['parent_id'] = type_id_map.get(i['parent_id'])
+            i['child_id'] = type_id_map.get(i['child_id'])
+
+            if i['parent_id'] and i['child_id']:
+                try:
+                    CITypeInheritanceManager.add([i.get('parent_id')], i.get('child_id'))
+                except BadRequest:
+                    pass
+
     @staticmethod
     def _import_type_attributes(type2attributes, type_id_map, attr_id_map):
         for type_id in type2attributes:
@@ -1301,6 +1498,9 @@ class CITypeTemplateManager(object):
 
             handled = set()
             for attr in type2attributes[type_id]:
+                if attr.get('inherited'):
+                    continue
+
                 payload = dict(type_id=type_id_map.get(int(type_id), type_id),
                                attr_id=attr_id_map.get(attr['id'], attr['id']),
                                default_show=attr['default_show'],
@@ -1364,6 +1564,9 @@ class CITypeTemplateManager(object):
 
         for rule in rules:
             ci_type = CITypeCache.get(rule.pop('type_name', None))
+            if ci_type is None:
+                continue
+
             adr = rule.pop('adr', {}) or {}
 
             if ci_type:
@@ -1376,10 +1579,10 @@ class CITypeTemplateManager(object):
 
                 if ad_rule:
                     rule['adr_id'] = ad_rule.id
-                    ad_rule.update(**adr)
+                    ad_rule.update(valid=False, **adr)
 
                 elif adr:
-                    ad_rule = AutoDiscoveryRuleCRUD().add(**adr)
+                    ad_rule = AutoDiscoveryRuleCRUD().add(valid=False, **adr)
                     rule['adr_id'] = ad_rule.id
                 else:
                     continue
@@ -1399,6 +1602,9 @@ class CITypeTemplateManager(object):
                 if ((i.extra_option or {}).get('alias') or None) == (
                         (rule.get('extra_option') or {}).get('alias') or None):
                     existed = True
+                    rule.pop('extra_option', None)
+                    rule.pop('enabled', None)
+                    rule.pop('cron', None)
                     AutoDiscoveryCITypeCRUD().update(i.id, **rule)
                     break
 
@@ -1408,6 +1614,23 @@ class CITypeTemplateManager(object):
                 except Exception as e:
                     current_app.logger.warning("import auto discovery rules failed: {}".format(e))
 
+    @staticmethod
+    def _import_auto_discovery_relation_rules(rules):
+        from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeRelationCRUD
+
+        for rule in rules:
+            ad_ci_type = CITypeCache.get(rule.pop('ad_type_name', None))
+            peer_ci_type = CITypeCache.get(rule.pop('peer_type_name', None))
+            peer_attr = AttributeCache.get(rule.pop('peer_attr_name', None))
+            if ad_ci_type and peer_attr and peer_ci_type:
+                if not AutoDiscoveryCITypeRelation.get_by(
+                        ad_type_id=ad_ci_type.id, ad_key=rule.get('ad_key'),
+                        peer_attr_id=peer_attr.id, peer_type_id=peer_ci_type.id):
+                    AutoDiscoveryCITypeRelationCRUD().add(ad_type_id=ad_ci_type.id,
+                                                          ad_key=rule.get('ad_key'),
+                                                          peer_attr_id=peer_attr.id,
+                                                          peer_type_id=peer_ci_type.id)
+
     @staticmethod
     def _import_icons(icons):
         from api.lib.common_setting.upload_file import CommonFileCRUD
@@ -1419,15 +1642,19 @@ class CITypeTemplateManager(object):
                     current_app.logger.warning("save icon failed: {}".format(e))
 
     def import_template(self, tpt):
+        db.session.commit()
+
         import time
         s = time.time()
-        attr_id_map = self._import_attributes(tpt.get('type2attributes') or {})
+        attr_id_map, references = self._import_attributes(tpt.get('type2attributes') or {})
         current_app.logger.info('import attributes cost: {}'.format(time.time() - s))
 
         s = time.time()
         ci_type_id_map = self._import_ci_types(tpt.get('ci_types') or [], attr_id_map)
         current_app.logger.info('import ci_types cost: {}'.format(time.time() - s))
 
+        self._import_reference_attributes(references, ci_type_id_map)
+
         s = time.time()
         self._import_ci_type_groups(tpt.get('ci_type_groups') or [], ci_type_id_map)
         current_app.logger.info('import ci_type_groups cost: {}'.format(time.time() - s))
@@ -1437,9 +1664,14 @@ class CITypeTemplateManager(object):
         current_app.logger.info('import relation_types cost: {}'.format(time.time() - s))
 
         s = time.time()
-        self._import_ci_type_relations(tpt.get('ci_type_relations') or [], ci_type_id_map, relation_type_id_map)
+        self._import_ci_type_relations(tpt.get('ci_type_relations') or [],
+                                       ci_type_id_map, relation_type_id_map, attr_id_map)
         current_app.logger.info('import ci_type_relations cost: {}'.format(time.time() - s))
 
+        s = time.time()
+        self._import_ci_type_inheritance(tpt.get('ci_type_inheritance') or [], ci_type_id_map)
+        current_app.logger.info('import ci_type_inheritance cost: {}'.format(time.time() - s))
+
         s = time.time()
         self._import_type_attributes(tpt.get('type2attributes') or {}, ci_type_id_map, attr_id_map)
         current_app.logger.info('import type2attributes cost: {}'.format(time.time() - s))
@@ -1452,26 +1684,84 @@ class CITypeTemplateManager(object):
         self._import_auto_discovery_rules(tpt.get('ci_type_auto_discovery_rules') or [])
         current_app.logger.info('import ci_type_auto_discovery_rules cost: {}'.format(time.time() - s))
 
+        s = time.time()
+        self._import_auto_discovery_relation_rules(tpt.get('ci_type_auto_discovery_relation_rules') or [])
+        current_app.logger.info('import ci_type_auto_discovery_relation_rules cost: {}'.format(time.time() - s))
+
         s = time.time()
         self._import_icons(tpt.get('icons') or {})
         current_app.logger.info('import icons cost: {}'.format(time.time() - s))
 
     @staticmethod
-    def export_template():
+    def export_template(type_ids=None):
         from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeCRUD
+        from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeRelationCRUD
         from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryRuleCRUD
         from api.lib.common_setting.upload_file import CommonFileCRUD
 
+        ci_types = CITypeManager.get_ci_types(type_ids=type_ids)
+        extend_type_ids = []
+        for ci_type in ci_types:
+            if ci_type.get('parent_ids'):
+                extend_type_ids.extend(CITypeInheritanceManager.base(ci_type['id']))
+        extend_type_ids = list(set(extend_type_ids) - set(type_ids))
+
+        ci_type_relations = CITypeRelationManager.get(type_ids=type_ids)[0]
+        for i in ci_type_relations:
+            if i['parent_id'] not in type_ids:
+                extend_type_ids.append(i['parent_id'])
+            if i['child_id'] not in type_ids:
+                extend_type_ids.append(i['child_id'])
+
+        ad_relation_rules = AutoDiscoveryCITypeRelationCRUD.get_all(type_ids=type_ids)
+        rules = []
+        for r in ad_relation_rules:
+            if r.peer_type_id not in type_ids:
+                extend_type_ids.append(r.peer_type_id)
+
+            r = r.to_dict()
+            r['ad_type_name'] = CITypeCache.get(r.pop('ad_type_id')).name
+            peer_type_id = r.pop("peer_type_id")
+            peer_type_name = CITypeCache.get(peer_type_id).name
+            if not peer_type_name:
+                peer_type = CITypeCache.get(peer_type_id)
+                peer_type_name = peer_type and peer_type.name
+            r['peer_type_name'] = peer_type_name
+            peer_attr_id = r.pop("peer_attr_id")
+            peer_attr = AttributeCache.get(peer_attr_id)
+            r['peer_attr_name'] = peer_attr and peer_attr.name
+
+            rules.append(r)
+        ci_type_auto_discovery_relation_rules = rules
+
+        if extend_type_ids:
+            extend_type_ids = list(set(extend_type_ids))
+            type_ids.extend(extend_type_ids)
+            ci_types.extend(CITypeManager.get_ci_types(type_ids=extend_type_ids))
+
+        # handle reference type
+        references = Attribute.get_by(only_query=True).join(
+            CITypeAttribute, CITypeAttribute.attr_id == Attribute.id).filter(
+            CITypeAttribute.type_id.in_(type_ids)).filter(CITypeAttribute.deleted.is_(False)).filter(
+            Attribute.reference_type_id.isnot(None))
+        reference_type_ids = list(set([i.reference_type_id for i in references if i.reference_type_id]))
+        if reference_type_ids:
+            type_ids.extend(reference_type_ids)
+            ci_types.extend(CITypeManager.get_ci_types(type_ids=reference_type_ids))
+
         tpt = dict(
-            ci_types=CITypeManager.get_ci_types(),
-            ci_type_groups=CITypeGroupManager.get(),
+            ci_types=ci_types,
             relation_types=[i.to_dict() for i in RelationTypeManager.get_all()],
-            ci_type_relations=CITypeRelationManager.get()[0],
+            ci_type_relations=ci_type_relations,
+            ci_type_inheritance=CITypeInheritanceManager.get_all(type_ids=type_ids),
             ci_type_auto_discovery_rules=list(),
+            ci_type_auto_discovery_relation_rules=ci_type_auto_discovery_relation_rules,
             type2attributes=dict(),
             type2attribute_group=dict(),
             icons=dict()
         )
+        tpt['ci_type_groups'] = CITypeGroupManager.get(ci_types=tpt['ci_types'], type_ids=type_ids)
+        tpt['ci_type_groups'] = [i for i in tpt['ci_type_groups'] if i.get('ci_types')]
 
         def get_icon_value(icon):
             try:
@@ -1479,12 +1769,16 @@ class CITypeTemplateManager(object):
             except:
                 return ""
 
-        ad_rules = AutoDiscoveryCITypeCRUD.get_all()
+        type_id2name = {i['id']: i['name'] for i in tpt['ci_types']}
+        ad_rules = AutoDiscoveryCITypeCRUD.get_all(type_ids=type_ids)
         rules = []
         for r in ad_rules:
             r = r.to_dict()
-            ci_type = CITypeCache.get(r.pop('type_id'))
-            r['type_name'] = ci_type and ci_type.name
+
+            if r.get('extra_option') and '_reference' in r['extra_option']:
+                r['extra_option'].pop('_reference')
+
+            r['type_name'] = type_id2name.get(r.pop('type_id'))
             if r.get('adr_id'):
                 adr = AutoDiscoveryRuleCRUD.get_by_id(r.pop('adr_id'))
                 r['adr_name'] = adr and adr.name
@@ -1518,65 +1812,6 @@ class CITypeTemplateManager(object):
 
         return tpt
 
-    @staticmethod
-    def export_template_by_type(type_id):
-        ci_type = CITypeCache.get(type_id) or abort(404, ErrFormat.ci_type_not_found2.format("id={}".format(type_id)))
-
-        from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeCRUD
-        from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryRuleCRUD
-        from api.lib.common_setting.upload_file import CommonFileCRUD
-
-        tpt = dict(
-            ci_types=CITypeManager.get_ci_types(type_name=ci_type.name, like=False),
-            ci_type_auto_discovery_rules=list(),
-            type2attributes=dict(),
-            type2attribute_group=dict(),
-            icons=dict()
-        )
-
-        def get_icon_value(icon):
-            try:
-                return CommonFileCRUD().get_file_binary_str(icon)
-            except:
-                return ""
-
-        ad_rules = AutoDiscoveryCITypeCRUD.get_by_type_id(ci_type.id)
-        rules = []
-        for r in ad_rules:
-            r = r.to_dict()
-            r['type_name'] = ci_type and ci_type.name
-            if r.get('adr_id'):
-                adr = AutoDiscoveryRuleCRUD.get_by_id(r.pop('adr_id'))
-                r['adr_name'] = adr and adr.name
-                r['adr'] = adr and adr.to_dict() or {}
-
-                icon_url = r['adr'].get('option', {}).get('icon', {}).get('url')
-                if icon_url and icon_url not in tpt['icons']:
-                    tpt['icons'][icon_url] = get_icon_value(icon_url)
-
-            rules.append(r)
-        tpt['ci_type_auto_discovery_rules'] = rules
-
-        for ci_type in tpt['ci_types']:
-            if ci_type['icon'] and len(ci_type['icon'].split('$$')) > 3:
-                icon_url = ci_type['icon'].split('$$')[3]
-                if icon_url not in tpt['icons']:
-                    tpt['icons'][icon_url] = get_icon_value(icon_url)
-
-            tpt['type2attributes'][ci_type['id']] = CITypeAttributeManager.get_attributes_by_type_id(
-                ci_type['id'], choice_web_hook_parse=False, choice_other_parse=False)
-
-            for attr in tpt['type2attributes'][ci_type['id']]:
-                for i in (attr.get('choice_value') or []):
-                    if (i[1] or {}).get('icon', {}).get('url') and len(i[1]['icon']['url'].split('$$')) > 3:
-                        icon_url = i[1]['icon']['url'].split('$$')[3]
-                        if icon_url not in tpt['icons']:
-                            tpt['icons'][icon_url] = get_icon_value(icon_url)
-
-            tpt['type2attribute_group'][ci_type['id']] = CITypeAttributeGroupManager.get_by_type_id(ci_type['id'])
-
-        return tpt
-
 
 class CITypeUniqueConstraintManager(object):
     @staticmethod

cmdb-api/api/lib/cmdb/const.py

@@ -14,6 +14,8 @@ class ValueTypeEnum(BaseEnum):
     JSON = "6"
     PASSWORD = TEXT
     LINK = TEXT
+    BOOL = "7"
+    REFERENCE = INT
 
 
 class ConstraintEnum(BaseEnum):
@@ -41,20 +43,23 @@ class OperateType(BaseEnum):
 
 
 class CITypeOperateType(BaseEnum):
-    ADD = "0"  # 新增模型
-    UPDATE = "1"  # 修改模型
-    DELETE = "2"  # 删除模型
-    ADD_ATTRIBUTE = "3"  # 新增属性
-    UPDATE_ATTRIBUTE = "4"  # 修改属性
-    DELETE_ATTRIBUTE = "5"  # 删除属性
-    ADD_TRIGGER = "6"  # 新增触发器
-    UPDATE_TRIGGER = "7"  # 修改触发器
-    DELETE_TRIGGER = "8"  # 删除触发器
-    ADD_UNIQUE_CONSTRAINT = "9"  # 新增联合唯一
-    UPDATE_UNIQUE_CONSTRAINT = "10"  # 修改联合唯一
-    DELETE_UNIQUE_CONSTRAINT = "11"  # 删除联合唯一
-    ADD_RELATION = "12"  # 新增关系
-    DELETE_RELATION = "13"  # 删除关系
+    ADD = "0"  # add CIType
+    UPDATE = "1"  # update CIType
+    DELETE = "2"  # delete CIType
+    ADD_ATTRIBUTE = "3"
+    UPDATE_ATTRIBUTE = "4"
+    DELETE_ATTRIBUTE = "5"
+    ADD_TRIGGER = "6"
+    UPDATE_TRIGGER = "7"
+    DELETE_TRIGGER = "8"
+    ADD_UNIQUE_CONSTRAINT = "9"
+    UPDATE_UNIQUE_CONSTRAINT = "10"
+    DELETE_UNIQUE_CONSTRAINT = "11"
+    ADD_RELATION = "12"
+    DELETE_RELATION = "13"
+    ADD_RECONCILIATION = "14"
+    UPDATE_RECONCILIATION = "15"
+    DELETE_RECONCILIATION = "16"
 
 
 class RetKey(BaseEnum):
@@ -70,6 +75,7 @@ class ResourceTypeEnum(BaseEnum):
     RELATION_VIEW = "RelationView"  # read/update/delete/grant
     CI_FILTER = "CIFilter"  # read
     PAGE = "page"  # read
+    TOPOLOGY_VIEW = "TopologyView"  # read/update/delete/grant
 
 
 class PermEnum(BaseEnum):
@@ -89,7 +95,8 @@ class RoleEnum(BaseEnum):
 class AutoDiscoveryType(BaseEnum):
     AGENT = "agent"
     SNMP = "snmp"
-    HTTP = "http"
+    HTTP = "http"  # cloud
+    COMPONENTS = "components"
 
 
 class AttributeDefaultValueEnum(BaseEnum):
@@ -98,12 +105,22 @@ class AttributeDefaultValueEnum(BaseEnum):
     AUTO_INC_ID = "$auto_inc_id"
 
 
+class ExecuteStatusEnum(BaseEnum):
+    COMPLETED = '0'
+    FAILED = '1'
+    RUNNING = '2'
+
+class RelationSourceEnum(BaseEnum):
+    ATTRIBUTE_VALUES = "0"
+    AUTO_DISCOVERY = "1"
+
+
 CMDB_QUEUE = "one_cmdb_async"
 REDIS_PREFIX_CI = "ONE_CMDB"
 REDIS_PREFIX_CI_RELATION = "CMDB_CI_RELATION"
 REDIS_PREFIX_CI_RELATION2 = "CMDB_CI_RELATION2"
 
-BUILTIN_KEYWORDS = {'id', '_id', 'ci_id', 'type', '_type', 'ci_type'}
+BUILTIN_KEYWORDS = {'id', '_id', 'ci_id', 'type', '_type', 'ci_type', 'ticket_id'}
 
 L_TYPE = None
 L_CI = None

cmdb-api/api/lib/cmdb/history.py

@@ -10,9 +10,11 @@ from api.extensions import db
 from api.lib.cmdb.cache import AttributeCache
 from api.lib.cmdb.cache import RelationTypeCache
 from api.lib.cmdb.const import OperateType
+from api.lib.cmdb.cache import CITypeCache
 from api.lib.cmdb.perms import CIFilterPermsCRUD
 from api.lib.cmdb.resp_format import ErrFormat
 from api.lib.perm.acl.cache import UserCache
+from api.models.cmdb import CI
 from api.models.cmdb import Attribute
 from api.models.cmdb import AttributeHistory
 from api.models.cmdb import CIRelationHistory
@@ -21,12 +23,13 @@ from api.models.cmdb import CITypeHistory
 from api.models.cmdb import CITypeTrigger
 from api.models.cmdb import CITypeUniqueConstraint
 from api.models.cmdb import OperationRecord
+from api.lib.cmdb.utils import TableMap
 
 
 class AttributeHistoryManger(object):
     @staticmethod
     def get_records_for_attributes(start, end, username, page, page_size, operate_type, type_id,
-                                   ci_id=None, attr_id=None):
+                                   ci_id=None, attr_id=None, ci_ids=None, more=False):
 
         records = db.session.query(OperationRecord, AttributeHistory).join(
             AttributeHistory, OperationRecord.id == AttributeHistory.record_id)
@@ -48,6 +51,9 @@ class AttributeHistoryManger(object):
         if ci_id is not None:
             records = records.filter(AttributeHistory.ci_id == ci_id)
 
+        if ci_ids and isinstance(ci_ids, list):
+            records = records.filter(AttributeHistory.ci_id.in_(ci_ids))
+
         if attr_id is not None:
             records = records.filter(AttributeHistory.attr_id == attr_id)
 
@@ -55,17 +61,39 @@ class AttributeHistoryManger(object):
         total = len(records)
 
         res = {}
+        show_attr_set = {}
+        show_attr_cache = {}
         for record in records:
             record_id = record.OperationRecord.id
+            type_id = record.OperationRecord.type_id
+            ci_id = record.AttributeHistory.ci_id
+            show_attr_set[ci_id] = None
+            show_attr = show_attr_cache.setdefault(
+                type_id, 
+                AttributeCache.get(
+                    CITypeCache.get(type_id).show_id or CITypeCache.get(type_id).unique_id) if CITypeCache.get(type_id) else None
+            )
+            if show_attr:
+                attr_table = TableMap(attr=show_attr).table
+                attr_record = attr_table.get_by(attr_id=show_attr.id, ci_id=ci_id, first=True, to_dict=False)
+                show_attr_set[ci_id] = attr_record.value if attr_record else None
+    
             attr_hist = record.AttributeHistory.to_dict()
             attr_hist['attr'] = AttributeCache.get(attr_hist['attr_id'])
             if attr_hist['attr']:
                 attr_hist['attr_name'] = attr_hist['attr'].name
                 attr_hist['attr_alias'] = attr_hist['attr'].alias
+                if more:
+                    attr_hist['is_list'] = attr_hist['attr'].is_list
+                    attr_hist['is_computed'] = attr_hist['attr'].is_computed
+                    attr_hist['is_password'] = attr_hist['attr'].is_password
+                    attr_hist['default'] = attr_hist['attr'].default
+                attr_hist['value_type'] = attr_hist['attr'].value_type
                 attr_hist.pop("attr")
 
             if record_id not in res:
                 record_dict = record.OperationRecord.to_dict()
+                record_dict['show_attr_value'] = show_attr_set.get(ci_id)
                 record_dict["user"] = UserCache.get(record_dict.get("uid"))
                 if record_dict["user"]:
                     record_dict['user'] = record_dict['user'].nickname
@@ -161,6 +189,7 @@ class AttributeHistoryManger(object):
             record = i.OperationRecord
             item = dict(attr_name=attr.name,
                         attr_alias=attr.alias,
+                        value_type=attr.value_type,
                         operate_type=hist.operate_type,
                         username=user and user.nickname,
                         old=hist.old,
@@ -221,8 +250,8 @@ class AttributeHistoryManger(object):
 
 class CIRelationHistoryManager(object):
     @staticmethod
-    def add(rel_obj, operate_type=OperateType.ADD):
-        record = OperationRecord.create(uid=current_user.uid)
+    def add(rel_obj, operate_type=OperateType.ADD, uid=None):
+        record = OperationRecord.create(uid=uid or current_user.uid)
 
         CIRelationHistory.create(relation_id=rel_obj.id,
                                  record_id=record.id,
@@ -271,7 +300,7 @@ class CITypeHistoryManager(object):
         return numfound, result
 
     @staticmethod
-    def add(operate_type, type_id, attr_id=None, trigger_id=None, unique_constraint_id=None, change=None):
+    def add(operate_type, type_id, attr_id=None, trigger_id=None, unique_constraint_id=None, change=None, rc_id=None):
         if type_id is None and attr_id is not None:
             from api.models.cmdb import CITypeAttribute
             type_ids = [i.type_id for i in CITypeAttribute.get_by(attr_id=attr_id, to_dict=False)]
@@ -284,6 +313,7 @@ class CITypeHistoryManager(object):
                            uid=current_user.uid,
                            attr_id=attr_id,
                            trigger_id=trigger_id,
+                           rc_id=rc_id,
                            unique_constraint_id=unique_constraint_id,
                            change=change)
 
@@ -295,7 +325,7 @@ class CITriggerHistoryManager(object):
     def get(page, page_size, type_id=None, trigger_id=None, operate_type=None):
         query = CITriggerHistory.get_by(only_query=True)
         if type_id:
-            query = query.filter(CITriggerHistory.type_id == type_id)
+            query = query.join(CI, CI.id == CITriggerHistory.ci_id).filter(CI.type_id == type_id)
 
         if trigger_id:
             query = query.filter(CITriggerHistory.trigger_id == trigger_id)

cmdb-api/api/lib/cmdb/preference.py

@@ -1,8 +1,9 @@
 # -*- coding:utf-8 -*-
 
 
-import copy
+from collections import defaultdict
 
+import copy
 import six
 import toposort
 from flask import abort
@@ -25,6 +26,8 @@ from api.lib.cmdb.resp_format import ErrFormat
 from api.lib.exception import AbortException
 from api.lib.perm.acl.acl import ACLManager
 from api.models.cmdb import CITypeAttribute
+from api.models.cmdb import CITypeGroup
+from api.models.cmdb import CITypeGroupItem
 from api.models.cmdb import CITypeRelation
 from api.models.cmdb import PreferenceCITypeOrder
 from api.models.cmdb import PreferenceRelationView
@@ -43,22 +46,46 @@ class PreferenceManager(object):
     def get_types(instance=False, tree=False):
         ci_type_order = sorted(PreferenceCITypeOrder.get_by(uid=current_user.uid, to_dict=False), key=lambda x: x.order)
 
+        type2group = {}
+        for i in db.session.query(CITypeGroupItem, CITypeGroup).join(
+                CITypeGroup, CITypeGroup.id == CITypeGroupItem.group_id).filter(
+            CITypeGroup.deleted.is_(False)).filter(CITypeGroupItem.deleted.is_(False)):
+            type2group[i.CITypeGroupItem.type_id] = i.CITypeGroup.to_dict()
+
         types = db.session.query(PreferenceShowAttributes.type_id).filter(
             PreferenceShowAttributes.uid == current_user.uid).filter(
             PreferenceShowAttributes.deleted.is_(False)).group_by(
             PreferenceShowAttributes.type_id).all() if instance else []
         types = sorted(types, key=lambda x: {i.type_id: idx for idx, i in enumerate(
             ci_type_order) if not i.is_tree}.get(x.type_id, 1))
+        group_types = []
+        other_types = []
+        group2idx = {}
+        type_ids = set()
+        for ci_type in types:
+            type_id = ci_type.type_id
+            type_ids.add(type_id)
+            type_dict = CITypeCache.get(type_id).to_dict()
+            if type_id not in type2group:
+                other_types.append(type_dict)
+            else:
+                group = type2group[type_id]
+                if group['id'] not in group2idx:
+                    group_types.append(type2group[type_id])
+                    group2idx[group['id']] = len(group_types) - 1
+                group_types[group2idx[group['id']]].setdefault('ci_types', []).append(type_dict)
+        if other_types:
+            group_types.append(dict(ci_types=other_types))
 
         tree_types = PreferenceTreeView.get_by(uid=current_user.uid, to_dict=False) if tree else []
         tree_types = sorted(tree_types, key=lambda x: {i.type_id: idx for idx, i in enumerate(
             ci_type_order) if i.is_tree}.get(x.type_id, 1))
 
-        type_ids = [i.type_id for i in types + tree_types]
-        if types and tree_types:
-            type_ids = set(type_ids)
+        tree_types = [CITypeCache.get(_type.type_id).to_dict() for _type in tree_types]
+        for _type in tree_types:
+            type_ids.add(_type['id'])
 
-        return [CITypeCache.get(type_id).to_dict() for type_id in type_ids]
+        return dict(group_types=group_types, tree_types=tree_types, type_ids=list(type_ids))
 
     @staticmethod
     def get_types2(instance=False, tree=False):
@@ -237,12 +264,12 @@ class PreferenceManager(object):
         else:
             views = _views
 
-        view2cr_ids = dict()
+        view2cr_ids = defaultdict(list)
         name2view = dict()
         result = dict()
         name2id = list()
         for view in views:
-            view2cr_ids.setdefault(view['name'], []).extend(view['cr_ids'])
+            view2cr_ids[view['name']].extend(view['cr_ids'])
             name2id.append([view['name'], view['id']])
             name2view[view['name']] = view
 
@@ -357,14 +384,22 @@ class PreferenceManager(object):
     def add_search_option(**kwargs):
         kwargs['uid'] = current_user.uid
 
-        existed = PreferenceSearchOption.get_by(uid=current_user.uid,
-                                                name=kwargs.get('name'),
-                                                prv_id=kwargs.get('prv_id'),
-                                                ptv_id=kwargs.get('ptv_id'),
-                                                type_id=kwargs.get('type_id'),
-                                                )
-        if existed:
-            return abort(400, ErrFormat.preference_search_option_exists)
+        if kwargs['name'] in ('__recent__', '__favor__'):
+            if kwargs['name'] == '__recent__':
+                for i in PreferenceSearchOption.get_by(
+                        only_query=True, name=kwargs['name'], uid=current_user.uid).order_by(
+                    PreferenceSearchOption.id.desc()).offset(20):
+                    i.delete()
+
+        else:
+            existed = PreferenceSearchOption.get_by(uid=current_user.uid,
+                                                    name=kwargs.get('name'),
+                                                    prv_id=kwargs.get('prv_id'),
+                                                    ptv_id=kwargs.get('ptv_id'),
+                                                    type_id=kwargs.get('type_id'),
+                                                    )
+            if existed:
+                return abort(400, ErrFormat.preference_search_option_exists)
 
         return PreferenceSearchOption.create(**kwargs)
 

cmdb-api/api/lib/cmdb/resp_format.py

@@ -35,7 +35,7 @@ class ErrFormat(CommonErrFormat):
         "Only creators and administrators are allowed to delete attributes!")  # 目前只允许 属性创建人、管理员 删除属性!
     # 属性字段名不能是内置字段: id, _id, ci_id, type, _type, ci_type
     attribute_name_cannot_be_builtin = _l(
-        "Attribute field names cannot be built-in fields: id, _id, ci_id, type, _type, ci_type")
+        "Attribute field names cannot be built-in fields: id, _id, ci_id, type, _type, ci_type, ticket_id")
     attribute_choice_other_invalid = _l(
         "Predefined value: Other model request parameters are illegal!")  # 预定义值: 其他模型请求参数不合法!
 
@@ -44,6 +44,8 @@ class ErrFormat(CommonErrFormat):
     unique_value_not_found = _l("The model's primary key {} does not exist!")  # 模型的主键 {} 不存在!
     unique_key_required = _l("Primary key {} is missing")  # 主键字段 {} 缺失
     ci_is_already_existed = _l("CI already exists!")  # CI 已经存在!
+    ci_reference_not_found = _l("{}: CI reference {} does not exist!")  # {}: CI引用 {} 不存在!
+    ci_reference_invalid = _l("{}: CI reference {} is illegal!")  # {}, CI引用 {} 不合法!
     relation_constraint = _l("Relationship constraint: {}, verification failed")  # 关系约束: {}, 校验失败
     # 多对多关系 限制: 模型 {} <-> {} 已经存在多对多关系!
     m2m_relation_constraint = _l(
@@ -62,6 +64,9 @@ class ErrFormat(CommonErrFormat):
         "The model cannot be deleted because the CI already exists")  # 因为CI已经存在，不能删除模型
     ci_exists_and_cannot_delete_inheritance = _l(
         "The inheritance cannot be deleted because the CI already exists")  # 因为CI已经存在，不能删除继承关系
+    ci_type_inheritance_cannot_delete = _l("The model is inherited and cannot be deleted")  # 该模型被继承, 不能删除
+    ci_type_referenced_cannot_delete = _l(
+        "The model is referenced by attribute {} and cannot be deleted")  # 该模型被属性 {} 引用, 不能删除
 
     # 因为关系视图 {} 引用了该模型，不能删除模型
     ci_relation_view_exists_and_cannot_delete_type = _l(
@@ -78,6 +83,8 @@ class ErrFormat(CommonErrFormat):
     unique_constraint_invalid = _l("Uniquely constrained attributes cannot be JSON and multi-valued")
     ci_type_trigger_duplicate = _l("Duplicated trigger")  # 重复的触发器
     ci_type_trigger_not_found = _l("Trigger {} does not exist")  # 触发器 {} 不存在
+    ci_type_reconciliation_duplicate = _l("Duplicated reconciliation rule")  # 重复的校验规则
+    ci_type_reconciliation_not_found = _l("Reconciliation rule {} does not exist")  # 规则 {} 不存在
 
     record_not_found = _l("Operation record {} does not exist")  # 操作记录 {} 不存在
     cannot_delete_unique = _l("Unique identifier cannot be deleted")  # 不能删除唯一标识
@@ -138,3 +145,14 @@ class ErrFormat(CommonErrFormat):
 
     password_save_failed = _l("Failed to save password: {}")  # 保存密码失败: {}
     password_load_failed = _l("Failed to get password: {}")  # 获取密码失败: {}
+
+    cron_time_format_invalid = _l("Scheduling time format error")  # 调度时间格式错误
+    reconciliation_title = _l("CMDB data reconciliation results")  # CMDB数据合规检查结果
+    reconciliation_body = _l("Number of {} illegal: {}")  # "{} 不合规数: {}"
+
+    topology_exists = _l("Topology view {} already exists")  # 拓扑视图 {} 已经存在
+    topology_group_exists = _l("Topology group {} already exists")  # 拓扑视图分组 {} 已经存在
+    # 因为该分组下定义了拓扑视图，不能删除
+    topo_view_exists_cannot_delete_group = _l("The group cannot be deleted because the topology view already exists")
+
+    relation_path_search_src_target_required = _l("Both the source model and the target model must be selected")

cmdb-api/api/lib/cmdb/search/ci/__init__.py

@@ -17,10 +17,12 @@ def search(query=None,
            count=1,
            sort=None,
            excludes=None,
-           use_id_filter=False):
+           use_id_filter=False,
+           use_ci_filter=True):
     if current_app.config.get("USE_ES"):
         s = SearchFromES(query, fl, facet, page, ret_key, count, sort)
     else:
-        s = SearchFromDB(query, fl, facet, page, ret_key, count, sort, excludes=excludes, use_id_filter=use_id_filter)
+        s = SearchFromDB(query, fl, facet, page, ret_key, count, sort, excludes=excludes,
+                         use_id_filter=use_id_filter, use_ci_filter=use_ci_filter)
 
     return s

cmdb-api/api/lib/cmdb/search/ci/db/query_sql.py

@@ -56,7 +56,7 @@ QUERY_CI_BY_ATTR_NAME = """
     SELECT {0}.ci_id
     FROM {0}
     WHERE {0}.attr_id={1:d}
-      AND {0}.value {2}
+      AND ({0}.value {2})
 """
 
 QUERY_CI_BY_ID = """

cmdb-api/api/lib/cmdb/search/ci/db/search.py

@@ -4,8 +4,8 @@
 from __future__ import unicode_literals
 
 import copy
+import six
 import time
-
 from flask import current_app
 from flask_login import current_user
 from jinja2 import Template
@@ -47,7 +47,8 @@ class Search(object):
                  excludes=None,
                  parent_node_perm_passed=False,
                  use_id_filter=False,
-                 use_ci_filter=True):
+                 use_ci_filter=True,
+                 only_ids=False):
         self.orig_query = query
         self.fl = fl or []
         self.excludes = excludes or []
@@ -64,10 +65,13 @@ class Search(object):
         self.parent_node_perm_passed = parent_node_perm_passed
         self.use_id_filter = use_id_filter
         self.use_ci_filter = use_ci_filter
+        self.only_ids = only_ids
+        self.multi_type_has_ci_filter = False
 
         self.valid_type_names = []
         self.type2filter_perms = dict()
         self.is_app_admin = is_app_admin('cmdb') or current_user.username == "worker"
+        self.is_app_admin = self.is_app_admin or (not self.use_ci_filter and not self.use_id_filter)
 
     @staticmethod
     def _operator_proc(key):
@@ -101,35 +105,56 @@ class Search(object):
         else:
             raise SearchError(ErrFormat.attribute_not_found.format(key))
 
-    def _type_query_handler(self, v, queries):
+    def _type_query_handler(self, v, queries, is_sub=False):
         new_v = v[1:-1].split(";") if v.startswith("(") and v.endswith(")") else [v]
+        type_num = len(new_v)
+        type_id_list = []
         for _v in new_v:
             ci_type = CITypeCache.get(_v)
 
-            if len(new_v) == 1 and not self.sort and ci_type and ci_type.default_order_attr:
+            if type_num == 1 and not self.sort and ci_type and ci_type.default_order_attr:
                 self.sort = ci_type.default_order_attr
 
             if ci_type is not None:
                 if self.valid_type_names == "ALL" or ci_type.name in self.valid_type_names:
-                    self.type_id_list.append(str(ci_type.id))
-                    if ci_type.id in self.type2filter_perms:
+                    if not is_sub:
+                        self.type_id_list.append(str(ci_type.id))
+                    type_id_list.append(str(ci_type.id))
+                    if ci_type.id in self.type2filter_perms and not is_sub:
                         ci_filter = self.type2filter_perms[ci_type.id].get('ci_filter')
                         if ci_filter and self.use_ci_filter and not self.use_id_filter:
                             sub = []
                             ci_filter = Template(ci_filter).render(user=current_user)
                             for i in ci_filter.split(','):
-                                if i.startswith("~") and not sub:
-                                    queries.append(i)
+                                if type_num == 1:
+                                    if i.startswith("~") and not sub:
+                                        queries.append(i)
+                                    else:
+                                        sub.append(i)
                                 else:
                                     sub.append(i)
                             if sub:
-                                queries.append(dict(operator="&", queries=sub))
+                                if type_num == 1:
+                                    queries.append(dict(operator="&", queries=sub))
+                                else:
+                                    if str(ci_type.id) in self.type_id_list:
+                                        self.type_id_list.remove(str(ci_type.id))
+                                    type_id_list.remove(str(ci_type.id))
+                                    sub.extend([i for i in queries[1:] if isinstance(i, six.string_types)])
 
+                                    sub.insert(0, "_type:{}".format(ci_type.id))
+                                    queries.append(dict(operator="|", queries=sub))
+                                    self.multi_type_has_ci_filter = True
                         if self.type2filter_perms[ci_type.id].get('attr_filter'):
-                            if not self.fl:
-                                self.fl = set(self.type2filter_perms[ci_type.id]['attr_filter'])
+                            if type_num == 1:
+                                if not self.fl:
+                                    self.fl = set(self.type2filter_perms[ci_type.id]['attr_filter'])
+                                else:
+                                    self.fl = set(self.fl) & set(self.type2filter_perms[ci_type.id]['attr_filter'])
                             else:
-                                self.fl = set(self.fl) & set(self.type2filter_perms[ci_type.id]['attr_filter'])
+                                self.fl = self.fl or {}
+                                if not self.fl or isinstance(self.fl, dict):
+                                    self.fl[ci_type.id] = set(self.type2filter_perms[ci_type.id]['attr_filter'])
 
                         if self.type2filter_perms[ci_type.id].get('id_filter') and self.use_id_filter:
 
@@ -143,13 +168,17 @@ class Search(object):
             else:
                 raise SearchError(ErrFormat.ci_type_not_found2.format(_v))
 
-        if self.type_id_list:
-            type_ids = ",".join(self.type_id_list)
+        if type_num != len(self.type_id_list) and queries and queries[0].startswith('_type') and not is_sub:
+            queries[0] = "_type:({})".format(";".join(self.type_id_list))
+
+        if type_id_list:
+            type_ids = ",".join(type_id_list)
             _query_sql = QUERY_CI_BY_TYPE.format(type_ids)
-            if self.only_type_query:
+            if self.only_type_query or self.multi_type_has_ci_filter:
                 return _query_sql
-            else:
-                return ""
+        elif type_num > 1:  # there must be instance-level access control
+            return "select c_cis.id as ci_id from c_cis where c_cis.id=0"
+
         return ""
 
     @staticmethod
@@ -226,7 +255,7 @@ class Search(object):
             return ret_sql.format(query_sql, "ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format(
                 (self.page - 1) * self.count, sort_type, self.count))
 
-        elif self.type_id_list:
+        elif self.type_id_list and not self.multi_type_has_ci_filter:
             self.query_sql = "SELECT B.ci_id FROM ({0}) AS B {1}".format(
                 query_sql,
                 "INNER JOIN c_cis on c_cis.id=B.ci_id WHERE c_cis.type_id IN ({0}) ".format(
@@ -251,7 +280,7 @@ class Search(object):
     def __sort_by_type(self, sort_type, query_sql):
         ret_sql = "SELECT SQL_CALC_FOUND_ROWS DISTINCT B.ci_id FROM ({0}) AS B {1}"
 
-        if self.type_id_list:
+        if self.type_id_list and not self.multi_type_has_ci_filter:
             self.query_sql = "SELECT B.ci_id FROM ({0}) AS B {1}".format(
                 query_sql,
                 "INNER JOIN c_cis on c_cis.id=B.ci_id WHERE c_cis.type_id IN ({0}) ".format(
@@ -284,7 +313,7 @@ class Search(object):
                           WHERE {1}.attr_id = {3}""".format("ALIAS", table_name, query_sql, attr_id)
         new_table = _v_query_sql
 
-        if self.only_type_query or not self.type_id_list:
+        if self.only_type_query or not self.type_id_list or self.multi_type_has_ci_filter:
             return ("SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id FROM ({0}) AS C ORDER BY C.value {2} "
                     "LIMIT {1:d}, {3};".format(new_table, (self.page - 1) * self.count, sort_type, self.count))
 
@@ -322,7 +351,9 @@ class Search(object):
                            INNER JOIN ({2}) as {3} USING(ci_id)""".format(query_sql, alias, _query_sql, alias + "A")
 
         elif operator == "|" or operator == "|~":
-            query_sql = "SELECT * FROM ({0}) as {1} UNION ALL ({2})".format(query_sql, alias, _query_sql)
+            query_sql = "SELECT * FROM ({0}) as {1} UNION ALL SELECT * FROM ({2}) as {3}".format(query_sql, alias,
+                                                                                                 _query_sql,
+                                                                                                 alias + "A")
 
         elif operator == "~":
             query_sql = """SELECT * FROM ({0}) as {1} LEFT JOIN ({2}) as {3} USING(ci_id)
@@ -427,14 +458,14 @@ class Search(object):
 
         return result
 
-    def __query_by_attr(self, q, queries, alias):
+    def __query_by_attr(self, q, queries, alias, is_sub=False):
         k = q.split(":")[0].strip()
         v = "\:".join(q.split(":")[1:]).strip()
         v = v.replace("'", "\\'")
         v = v.replace('"', '\\"')
         field, field_type, operator, attr = self._attr_name_proc(k)
         if field == "_type":
-            _query_sql = self._type_query_handler(v, queries)
+            _query_sql = self._type_query_handler(v, queries, is_sub)
 
         elif field == "_id":
             _query_sql = self._id_query_handler(v)
@@ -448,6 +479,9 @@ class Search(object):
             if field_type == ValueTypeEnum.DATE and len(v) == 10:
                 v = "{} 00:00:00".format(v)
 
+            if field_type == ValueTypeEnum.BOOL and "*" not in str(v):
+                v = str(int(v in current_app.config.get('BOOL_TRUE')))
+
             # in query
             if v.startswith("(") and v.endswith(")"):
                 _query_sql = self._in_query_handler(attr, v, is_not)
@@ -478,19 +512,20 @@ class Search(object):
 
         return alias, _query_sql, operator
 
-    def __query_build_by_field(self, queries, is_first=True, only_type_query_special=True, alias='A', operator='&'):
+    def __query_build_by_field(self, queries, is_first=True, only_type_query_special=True, alias='A', operator='&',
+                               is_sub=False):
         query_sql = ""
 
         for q in queries:
             _query_sql = ""
             if isinstance(q, dict):
-                alias, _query_sql, operator = self.__query_build_by_field(q['queries'], True, True, alias)
-                current_app.logger.info(_query_sql)
-                current_app.logger.info((operator, is_first, alias))
+                alias, _query_sql, operator = self.__query_build_by_field(q['queries'], True, True, alias, is_sub=True)
+                # current_app.logger.info(_query_sql)
+                # current_app.logger.info((operator, is_first, alias))
                 operator = q['operator']
 
             elif ":" in q and not q.startswith("*"):
-                alias, _query_sql, operator = self.__query_by_attr(q, queries, alias)
+                alias, _query_sql, operator = self.__query_by_attr(q, queries, alias, is_sub)
             elif q == "*":
                 continue
             elif q:
@@ -541,7 +576,6 @@ class Search(object):
         queries = handle_arg_list(self.orig_query)
         queries = self._extra_handle_query_expr(queries)
         queries = self.__confirm_type_first(queries)
-        current_app.logger.debug(queries)
 
         _, query_sql, _ = self.__query_build_by_field(queries)
 
@@ -579,17 +613,22 @@ class Search(object):
         return facet_result
 
     def _fl_build(self):
-        _fl = list()
-        for f in self.fl:
-            k, _, _, _ = self._attr_name_proc(f)
-            if k:
-                _fl.append(k)
+        if isinstance(self.fl, list):
+            _fl = list()
+            for f in self.fl:
+                k, _, _, _ = self._attr_name_proc(f)
+                if k:
+                    _fl.append(k)
 
-        return _fl
+            return _fl
+        else:
+            return self.fl
 
     def search(self):
         numfound, ci_ids = self._query_build_raw()
         ci_ids = list(map(str, ci_ids))
+        if self.only_ids:
+            return ci_ids
 
         _fl = self._fl_build()
 
@@ -602,6 +641,8 @@ class Search(object):
         if ci_ids:
             response = CIManager.get_cis_by_ids(ci_ids, ret_key=self.ret_key, fields=_fl, excludes=self.excludes)
         for res in response:
+            if not res:
+                continue
             ci_type = res.get("ci_type")
             if ci_type not in counter.keys():
                 counter[ci_type] = 0

cmdb-api/api/lib/cmdb/search/ci_relation/search.py

@@ -1,8 +1,11 @@
 # -*- coding:utf-8 -*-
-import json
-import sys
 from collections import Counter
+from collections import defaultdict
 
+import copy
+import json
+import networkx as nx
+import sys
 from flask import abort
 from flask import current_app
 from flask_login import current_user
@@ -13,6 +16,7 @@ from api.lib.cmdb.cache import CITypeCache
 from api.lib.cmdb.ci import CIRelationManager
 from api.lib.cmdb.ci_type import CITypeRelationManager
 from api.lib.cmdb.const import ConstraintEnum
+from api.lib.cmdb.const import PermEnum
 from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
 from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION2
 from api.lib.cmdb.const import ResourceTypeEnum
@@ -25,10 +29,12 @@ from api.lib.cmdb.utils import ValueTypeMap
 from api.lib.perm.acl.acl import ACLManager
 from api.lib.perm.acl.acl import is_app_admin
 from api.models.cmdb import CI
+from api.models.cmdb import CITypeRelation
+from api.models.cmdb import RelationType
 
 
 class Search(object):
-    def __init__(self, root_id,
+    def __init__(self, root_id=None,
                  level=None,
                  query=None,
                  fl=None,
@@ -69,7 +75,7 @@ class Search(object):
                     if _l < int(level) and c == ConstraintEnum.Many2Many:
                         self.has_m2m = True
 
-        self.type2filter_perms = None
+        self.type2filter_perms = {}
 
         self.is_app_admin = is_app_admin('cmdb') or current_user.username == "worker"
 
@@ -79,7 +85,7 @@ class Search(object):
         key = []
         _tmp = []
         for level in range(1, sorted(self.level)[-1] + 1):
-            if len(self.descendant_ids) >= level and self.type2filter_perms.get(self.descendant_ids[level - 1]):
+            if len(self.descendant_ids or []) >= level and self.type2filter_perms.get(self.descendant_ids[level - 1]):
                 id_filter_limit, _ = self._get_ci_filter(self.type2filter_perms[self.descendant_ids[level - 1]])
             else:
                 id_filter_limit = {}
@@ -151,9 +157,9 @@ class Search(object):
 
         return True
 
-    def search(self):
-        use_ci_filter = len(self.descendant_ids) == self.level[0] - 1
-        parent_node_perm_passed = self._has_read_perm_from_parent_nodes()
+    def search(self, only_ids=False):
+        use_ci_filter = len(self.descendant_ids or []) == self.level[0] - 1
+        parent_node_perm_passed = not self.is_app_admin and self._has_read_perm_from_parent_nodes()
 
         ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
         cis = [CI.get_by_id(_id) or abort(404, ErrFormat.ci_not_found.format("id={}".format(_id))) for _id in ids]
@@ -197,7 +203,8 @@ class Search(object):
                                 sort=self.sort,
                                 ci_ids=merge_ids,
                                 parent_node_perm_passed=parent_node_perm_passed,
-                                use_ci_filter=use_ci_filter).search()
+                                use_ci_filter=use_ci_filter,
+                                only_ids=only_ids).search()
 
     def _get_ci_filter(self, filter_perms, ci_filters=None):
         ci_filters = ci_filters or []
@@ -418,3 +425,169 @@ class Search(object):
             level_ids = _level_ids
 
         return result
+
+    @staticmethod
+    def _get_src_ids(src):
+        q = src.get('q') or ''
+        if not q.startswith('_type:'):
+            q = "_type:{},{}".format(src['type_id'], q)
+
+        return SearchFromDB(q, use_ci_filter=True, only_ids=True, count=100000).search()
+
+    @staticmethod
+    def _filter_target_ids(target_ids, type_ids, q):
+        if not q.startswith('_type:'):
+            q = "_type:({}),{}".format(";".join(map(str, type_ids)), q)
+
+        ci_ids = SearchFromDB(q, ci_ids=target_ids, use_ci_filter=True, only_ids=True, count=100000).search()
+        cis = CI.get_by(fl=['id', 'type_id'], only_query=True).filter(CI.id.in_(ci_ids))
+
+        return [(str(i.id), i.type_id) for i in cis]
+
+    @staticmethod
+    def _path2level(src_type_id, target_type_ids, path):
+        if not src_type_id or not target_type_ids:
+            return abort(400, ErrFormat.relation_path_search_src_target_required)
+
+        graph = nx.DiGraph()
+        graph.add_edges_from([(n, _path[idx + 1]) for _path in path for idx, n in enumerate(_path[:-1])])
+        relation_types = defaultdict(dict)
+        level2type = defaultdict(set)
+        type2show_key = dict()
+        for _path in path:
+            for idx, node in enumerate(_path[1:]):
+                level2type[idx + 1].add(node)
+
+                src = CITypeCache.get(_path[idx])
+                target = CITypeCache.get(node)
+                relation_type = RelationType.get_by(only_query=True).join(
+                    CITypeRelation, CITypeRelation.relation_type_id == RelationType.id).filter(
+                    CITypeRelation.parent_id == src.id).filter(CITypeRelation.child_id == target.id).first()
+                relation_types[src.alias].update({target.alias: relation_type.name})
+
+                if src.id not in type2show_key:
+                    type2show_key[src.id] = AttributeCache.get(src.show_id or src.unique_id).name
+                if target.id not in type2show_key:
+                    type2show_key[target.id] = AttributeCache.get(target.show_id or target.unique_id).name
+
+        nodes = graph.nodes()
+
+        return level2type, list(nodes), relation_types, type2show_key
+
+    def _build_graph(self, source_ids, source_type_id, level2type, target_type_ids, acl):
+        type2filter_perms = dict()
+        if not self.is_app_admin:
+            res2 = acl.get_resources(ResourceTypeEnum.CI_FILTER)
+            if res2:
+                type2filter_perms = CIFilterPermsCRUD().get_by_ids(list(map(int, [i['name'] for i in res2])))
+
+        target_type_ids = set(target_type_ids)
+        graph = nx.DiGraph()
+        target_ids = []
+        key = [(str(i), source_type_id) for i in source_ids]
+        graph.add_nodes_from(key)
+        for level in level2type:
+            filter_type_ids = level2type[level]
+            id_filter_limit = dict()
+            for _type_id in filter_type_ids:
+                if type2filter_perms.get(_type_id):
+                    _id_filter_limit, _ = self._get_ci_filter(type2filter_perms[_type_id])
+                    id_filter_limit.update(_id_filter_limit)
+
+            has_target = filter_type_ids & target_type_ids
+
+            res = [json.loads(x).items() for x in [i or '{}' for i in rd.get([i[0] for i in key],
+                                                                             REDIS_PREFIX_CI_RELATION) or []]]
+            _key = []
+            for idx, _id in enumerate(key):
+                valid_targets = [i for i in res[idx] if i[1] in filter_type_ids and
+                                 (not id_filter_limit or int(i[0]) in id_filter_limit)]
+                _key.extend(valid_targets)
+                graph.add_edges_from(zip([_id] * len(valid_targets), valid_targets))
+
+            if has_target:
+                target_ids.extend([j[0] for i in res for j in i if j[1] in target_type_ids])
+
+            key = copy.deepcopy(_key)
+
+        return graph, target_ids
+
+    @staticmethod
+    def _find_paths(graph, source_ids, source_type_id, target_ids, valid_path, max_depth=6):
+        paths = []
+        for source_id in source_ids:
+            _paths = nx.all_simple_paths(graph,
+                                         source=(source_id, source_type_id),
+                                         target=target_ids,
+                                         cutoff=max_depth)
+            for __path in _paths:
+                if tuple([i[1] for i in __path]) in valid_path:
+                    paths.append([i[0] for i in __path])
+
+        return paths
+
+    @staticmethod
+    def _wrap_path_result(paths, types, valid_path, target_types, type2show_key):
+        ci_ids = [j for i in paths for j in i]
+
+        response, _, _, _, _, _ = SearchFromDB("_type:({})".format(";".join(map(str, types))),
+                                               use_ci_filter=False,
+                                               ci_ids=list(map(int, ci_ids)),
+                                               count=1000000).search()
+        id2ci = {str(i.get('_id')): i if i['_type'] in target_types else {
+            type2show_key[i['_type']]: i[type2show_key[i['_type']]],
+            "ci_type_alias": i["ci_type_alias"],
+            "_type": i["_type"],
+        } for i in response}
+
+        result = defaultdict(list)
+        counter = defaultdict(int)
+
+        for path in paths:
+            key = "-".join([id2ci.get(i, {}).get('ci_type_alias') or '' for i in path])
+            if tuple([id2ci.get(i, {}).get('_type') for i in path]) in valid_path:
+                counter[key] += 1
+                result[key].append(path)
+
+        return result, counter, id2ci
+
+    def search_by_path(self, source, target, path):
+        """
+
+        :param source: {type_id: id, q: expr}
+        :param target: {type_ids: [id], q: expr}
+        :param path: [source_type_id, ..., target_type_id], use type id
+        :return:
+        """
+        acl = ACLManager('cmdb')
+        if not self.is_app_admin:
+            res = {i['name'] for i in acl.get_resources(ResourceTypeEnum.CI_TYPE)}
+            for type_id in (source.get('type_id') and [source['type_id']] or []) + (target.get('type_ids') or []):
+                _type = CITypeCache.get(type_id)
+                if _type and _type.name not in res:
+                    return abort(403, ErrFormat.no_permission.format(_type.alias, PermEnum.READ))
+
+        target['type_ids'] = [i[-1] for i in path]
+        level2type, types, relation_types, type2show_key = self._path2level(
+            source.get('type_id'), target.get('type_ids'), path)
+        if not level2type:
+            return [], {}, 0, self.page, 0, {}, {}
+
+        source_ids = self._get_src_ids(source)
+
+        graph, target_ids = self._build_graph(source_ids, source['type_id'], level2type, target['type_ids'], acl)
+        target_ids = self._filter_target_ids(target_ids, target['type_ids'], target.get('q') or '')
+        paths = self._find_paths(graph,
+                                 source_ids,
+                                 source['type_id'],
+                                 set(target_ids),
+                                 {tuple(i): 1 for i in path})
+
+        numfound = len(paths)
+        paths = paths[(self.page - 1) * self.count:self.page * self.count]
+        response, counter, id2ci = self._wrap_path_result(paths,
+                                                          types,
+                                                          {tuple(i): 1 for i in path},
+                                                          set(target.get('type_ids') or []),
+                                                          type2show_key)
+        return response, counter, len(paths), self.page, numfound, id2ci, relation_types, type2show_key

cmdb-api/api/lib/cmdb/topology.py

@@ -0,0 +1,251 @@
+# -*- coding:utf-8 -*-
+
+import json
+
+from flask import abort
+from flask import current_app
+from flask_login import current_user
+from werkzeug.exceptions import BadRequest
+
+from api.extensions import rd
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.cache import CITypeCache
+from api.lib.cmdb.ci import CIRelationManager
+from api.lib.cmdb.ci_type import CITypeRelationManager
+from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
+from api.lib.cmdb.const import ResourceTypeEnum
+from api.lib.cmdb.resp_format import ErrFormat
+from api.lib.cmdb.search import SearchError
+from api.lib.cmdb.search.ci import search as ci_search
+from api.lib.perm.acl.acl import ACLManager
+from api.lib.perm.acl.acl import is_app_admin
+from api.models.cmdb import TopologyView
+from api.models.cmdb import TopologyViewGroup
+
+
+class TopologyViewManager(object):
+    group_cls = TopologyViewGroup
+    cls = TopologyView
+
+    @classmethod
+    def get_name_by_id(cls, _id):
+        res = cls.cls.get_by_id(_id)
+        return res and res.name
+
+    def get_view_by_id(self, _id):
+        res = self.cls.get_by_id(_id)
+
+        return res and res.to_dict() or {}
+
+    @classmethod
+    def add_group(cls, name, order):
+        if order is None:
+            cur_max_order = cls.group_cls.get_by(only_query=True).order_by(cls.group_cls.order.desc()).first()
+            cur_max_order = cur_max_order and cur_max_order.order or 0
+            order = cur_max_order + 1
+
+        cls.group_cls.get_by(name=name, first=True, to_dict=False) and abort(
+            400, ErrFormat.topology_group_exists.format(name))
+
+        return cls.group_cls.create(name=name, order=order)
+
+    def update_group(self, group_id, name, view_ids):
+        existed = self.group_cls.get_by_id(group_id) or abort(404, ErrFormat.not_found)
+        if name is not None and name != existed.name:
+            existed.update(name=name)
+
+        for idx, view_id in enumerate(view_ids):
+            view = self.cls.get_by_id(view_id)
+            if view is not None:
+                view.update(group_id=group_id, order=idx)
+
+        return existed.to_dict()
+
+    @classmethod
+    def delete_group(cls, _id):
+        existed = cls.group_cls.get_by_id(_id) or abort(404, ErrFormat.not_found)
+
+        if cls.cls.get_by(group_id=_id, first=True):
+            return abort(400, ErrFormat.topo_view_exists_cannot_delete_group)
+
+        existed.soft_delete()
+
+    @classmethod
+    def group_order(cls, group_ids):
+        for idx, group_id in enumerate(group_ids):
+            group = cls.group_cls.get_by_id(group_id)
+            group.update(order=idx + 1)
+
+    @classmethod
+    def add(cls, name, group_id, option, order=None, **kwargs):
+        cls.cls.get_by(name=name, first=True) and abort(400, ErrFormat.topology_exists.format(name))
+        if order is None:
+            cur_max_order = cls.cls.get_by(group_id=group_id, only_query=True).order_by(
+                cls.cls.order.desc()).first()
+            cur_max_order = cur_max_order and cur_max_order.order or 0
+            order = cur_max_order + 1
+
+        inst = cls.cls.create(name=name, group_id=group_id, option=option, order=order, **kwargs).to_dict()
+        if current_app.config.get('USE_ACL'):
+            try:
+                ACLManager().add_resource(name, ResourceTypeEnum.TOPOLOGY_VIEW)
+            except BadRequest:
+                pass
+
+            ACLManager().grant_resource_to_role(name,
+                                                current_user.username,
+                                                ResourceTypeEnum.TOPOLOGY_VIEW)
+
+        return inst
+
+    @classmethod
+    def update(cls, _id, **kwargs):
+        existed = cls.cls.get_by_id(_id) or abort(404, ErrFormat.not_found)
+        existed_name = existed.name
+
+        inst = existed.update(filter_none=False, **kwargs).to_dict()
+        if current_app.config.get('USE_ACL') and existed_name != kwargs.get('name') and kwargs.get('name'):
+            try:
+                ACLManager().update_resource(existed_name, kwargs['name'], ResourceTypeEnum.TOPOLOGY_VIEW)
+            except BadRequest:
+                pass
+
+        return inst
+
+    @classmethod
+    def delete(cls, _id):
+        existed = cls.cls.get_by_id(_id) or abort(404, ErrFormat.not_found)
+
+        existed.soft_delete()
+        if current_app.config.get("USE_ACL"):
+            ACLManager().del_resource(existed.name, ResourceTypeEnum.TOPOLOGY_VIEW)
+
+    @classmethod
+    def group_inner_order(cls, _ids):
+        for idx, _id in enumerate(_ids):
+            topology = cls.cls.get_by_id(_id)
+            topology.update(order=idx + 1)
+
+    @classmethod
+    def get_all(cls):
+        resources = None
+        if current_app.config.get('USE_ACL') and not is_app_admin('cmdb'):
+            resources = set([i.get('name') for i in ACLManager().get_resources(ResourceTypeEnum.TOPOLOGY_VIEW)])
+
+        groups = cls.group_cls.get_by(to_dict=True)
+        groups = sorted(groups, key=lambda x: x['order'])
+        group2pos = {group['id']: idx for idx, group in enumerate(groups)}
+
+        topo_views = sorted(cls.cls.get_by(to_dict=True), key=lambda x: x['order'])
+        other_group = dict(views=[])
+        for view in topo_views:
+            if resources is not None and view['name'] not in resources:
+                continue
+
+            if view['group_id']:
+                groups[group2pos[view['group_id']]].setdefault('views', []).append(view)
+            else:
+                other_group['views'].append(view)
+
+        if other_group['views']:
+            groups.append(other_group)
+
+        return groups
+
+    @staticmethod
+    def relation_from_ci_type(type_id):
+        nodes, edges = CITypeRelationManager.get_relations_by_type_id(type_id)
+
+        return dict(nodes=nodes, edges=edges)
+
+    def topology_view(self, view_id=None, preview=None):
+        if view_id is not None:
+            view = self.cls.get_by_id(view_id) or abort(404, ErrFormat.not_found)
+            central_node_type, central_node_instances, path = (view.central_node_type,
+                                                               view.central_node_instances, view.path)
+        else:
+            central_node_type = preview.get('central_node_type')
+            central_node_instances = preview.get('central_node_instances')
+            path = preview.get('path')
+
+        nodes, links = [], []
+        _type = CITypeCache.get(central_node_type)
+        if not _type:
+            return dict(nodes=nodes, links=links)
+        type2meta = {_type.id: _type.icon}
+        root_ids = []
+        show_key = AttributeCache.get(_type.show_id or _type.unique_id)
+
+        q = (central_node_instances[2:] if central_node_instances.startswith('q=') else
+             central_node_instances)
+        s = ci_search(q, fl=['_id', show_key.name], use_id_filter=False, use_ci_filter=False, count=1000000)
+        try:
+            response, _, _, _, _, _ = s.search()
+        except SearchError as e:
+            current_app.logger.info(e)
+            return dict(nodes=nodes, links=links)
+        for i in response:
+            root_ids.append(i['_id'])
+            nodes.append(dict(id=str(i['_id']), name=i[show_key.name], type_id=central_node_type))
+        if not root_ids:
+            return dict(nodes=nodes, links=links)
+
+        prefix = REDIS_PREFIX_CI_RELATION
+        key = list(map(str, root_ids))
+        id2node = {}
+        for level in sorted([i for i in path.keys() if int(i) > 0]):
+            type_ids = {int(i) for i in path[level]}
+
+            res = [json.loads(x).items() for x in [i or '{}' for i in rd.get(key, prefix) or []]]
+            new_key = []
+            for idx, from_id in enumerate(key):
+                for to_id, type_id in res[idx]:
+                    if type_id in type_ids:
+                        links.append({'from': from_id, 'to': to_id})
+                        id2node[to_id] = {'id': to_id, 'type_id': type_id}
+                        new_key.append(to_id)
+                        if type_id not in type2meta:
+                            type2meta[type_id] = CITypeCache.get(type_id).icon
+
+            key = new_key
+
+        ci_ids = list(map(int, root_ids))
+        for level in sorted([i for i in path.keys() if int(i) < 0]):
+            type_ids = {int(i) for i in path[level]}
+            res = CIRelationManager.get_parent_ids(ci_ids)
+            _ci_ids = []
+            for to_id in res:
+                for from_id, type_id in res[to_id]:
+                    if type_id in type_ids:
+                        from_id, to_id = str(from_id), str(to_id)
+                        links.append({'from': from_id, 'to': to_id})
+                        id2node[from_id] = {'id': str(from_id), 'type_id': type_id}
+                        _ci_ids.append(from_id)
+                        if type_id not in type2meta:
+                            type2meta[type_id] = CITypeCache.get(type_id).icon
+
+            ci_ids = _ci_ids
+
+        fl = set()
+        type_ids = {t for lv in path if lv != '0' for t in path[lv]}
+        type2show = {}
+        for type_id in type_ids:
+            ci_type = CITypeCache.get(type_id)
+            if ci_type:
+                attr = AttributeCache.get(ci_type.show_id or ci_type.unique_id)
+                if attr:
+                    fl.add(attr.name)
+                    type2show[type_id] = attr.name
+
+        if id2node:
+            s = ci_search("_id:({})".format(';'.join(id2node.keys())), fl=list(fl),
+                       use_id_filter=False, use_ci_filter=False, count=1000000)
+            try:
+                response, _, _, _, _, _ = s.search()
+            except SearchError:
+                return dict(nodes=nodes, links=links)
+            for i in response:
+                id2node[str(i['_id'])]['name'] = i[type2show[str(i['_type'])]]
+            nodes.extend(id2node.values())
+
+        return dict(nodes=nodes, links=links, type2meta=type2meta)

cmdb-api/api/lib/cmdb/utils.py

@@ -7,6 +7,7 @@ import json
 import re
 
 import six
+from flask import current_app
 
 import api.models.cmdb as model
 from api.lib.cmdb.cache import AttributeCache
@@ -28,13 +29,31 @@ def string2int(x):
     return v
 
 
-def str2datetime(x):
+def str2date(x):
+
     try:
         return datetime.datetime.strptime(x, "%Y-%m-%d").date()
     except ValueError:
         pass
 
-    return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")
+    try:
+        return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S").date()
+    except ValueError:
+        pass
+
+
+def str2datetime(x):
+
+    x = x.replace('T', ' ')
+    x = x.replace('Z', '')
+
+    try:
+        return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S")
+    except ValueError:
+        pass
+
+    return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M")
+
 
 
 class ValueTypeMap(object):
@@ -44,8 +63,9 @@ class ValueTypeMap(object):
         ValueTypeEnum.TEXT: lambda x: x,
         ValueTypeEnum.TIME: lambda x: TIME_RE.findall(x)[0],
         ValueTypeEnum.DATETIME: str2datetime,
-        ValueTypeEnum.DATE: str2datetime,
+        ValueTypeEnum.DATE: str2date,
         ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x,
+        ValueTypeEnum.BOOL: lambda x: x in current_app.config.get('BOOL_TRUE'),
     }
 
     serialize = {
@@ -56,6 +76,7 @@ class ValueTypeMap(object):
         ValueTypeEnum.DATE: lambda x: x.strftime("%Y-%m-%d") if not isinstance(x, six.string_types) else x,
         ValueTypeEnum.DATETIME: lambda x: x.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(x, six.string_types) else x,
         ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x,
+        ValueTypeEnum.BOOL: lambda x: x in current_app.config.get('BOOL_TRUE'),
     }
 
     serialize2 = {
@@ -66,6 +87,7 @@ class ValueTypeMap(object):
         ValueTypeEnum.DATE: lambda x: (x.decode() if not isinstance(x, six.string_types) else x).split()[0],
         ValueTypeEnum.DATETIME: lambda x: x.decode() if not isinstance(x, six.string_types) else x,
         ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x,
+        ValueTypeEnum.BOOL: lambda x: x in current_app.config.get('BOOL_TRUE'),
     }
 
     choice = {
@@ -87,6 +109,7 @@ class ValueTypeMap(object):
         'index_{0}'.format(ValueTypeEnum.TIME): model.CIIndexValueText,
         'index_{0}'.format(ValueTypeEnum.FLOAT): model.CIIndexValueFloat,
         'index_{0}'.format(ValueTypeEnum.JSON): model.CIValueJson,
+        'index_{0}'.format(ValueTypeEnum.BOOL): model.CIIndexValueInteger,
     }
 
     table_name = {
@@ -99,6 +122,7 @@ class ValueTypeMap(object):
         'index_{0}'.format(ValueTypeEnum.TIME): 'c_value_index_texts',
         'index_{0}'.format(ValueTypeEnum.FLOAT): 'c_value_index_floats',
         'index_{0}'.format(ValueTypeEnum.JSON): 'c_value_json',
+        'index_{0}'.format(ValueTypeEnum.BOOL): 'c_value_index_integers',
     }
 
     es_type = {

cmdb-api/api/lib/cmdb/value.py

@@ -3,13 +3,13 @@
 
 from __future__ import unicode_literals
 
-import copy
 import imp
+
+import copy
+import jinja2
 import os
 import re
 import tempfile
-
-import jinja2
 from flask import abort
 from flask import current_app
 from jinja2schema import infer
@@ -47,7 +47,7 @@ class AttributeValueManager(object):
         """
         return AttributeCache.get(key)
 
-    def get_attr_values(self, fields, ci_id, ret_key="name", unique_key=None, use_master=False):
+    def get_attr_values(self, fields, ci_id, ret_key="name", unique_key=None, use_master=False, enum_map=None):
         """
 
         :param fields:
@@ -55,6 +55,7 @@ class AttributeValueManager(object):
         :param ret_key: It can be name or alias
         :param unique_key: primary attribute
         :param use_master: Only for master-slave read-write separation
+        :param enum_map:
         :return:
         """
         res = dict()
@@ -76,6 +77,12 @@ class AttributeValueManager(object):
             else:
                 res[field_name] = ValueTypeMap.serialize[attr.value_type](rs[0].value) if rs else None
 
+            if enum_map and field_name in enum_map:
+                if attr.is_list:
+                    res[field_name] = [enum_map[field_name].get(i, i) for i in res[field_name]]
+                else:
+                    res[field_name] = enum_map[field_name].get(res[field_name], res[field_name])
+
             if unique_key is not None and attr.id == unique_key.id and rs:
                 res['unique'] = unique_key.name
                 res['unique_alias'] = unique_key.alias
@@ -90,11 +97,13 @@ class AttributeValueManager(object):
         deserialize = ValueTypeMap.deserialize[value_type]
         try:
             v = deserialize(value)
+            if value_type in (ValueTypeEnum.DATE, ValueTypeEnum.DATETIME):
+                return str(v)
             return v
         except ValueDeserializeError as e:
             return abort(400, ErrFormat.attribute_value_invalid2.format(alias, e))
         except ValueError:
-            return abort(400, ErrFormat.attribute_value_invalid.format(value))
+            return abort(400, ErrFormat.attribute_value_invalid2.format(alias, value))
 
     @staticmethod
     def _check_is_choice(attr, value_type, value):
@@ -123,25 +132,31 @@ class AttributeValueManager(object):
             return abort(400, ErrFormat.attribute_value_required.format(attr.alias))
 
     @staticmethod
-    def check_re(expr, value):
+    def check_re(expr, alias, value):
         if not re.compile(expr).match(str(value)):
-            return abort(400, ErrFormat.attribute_value_invalid.format(value))
+            return abort(400, ErrFormat.attribute_value_invalid2.format(alias, value))
 
     def _validate(self, attr, value, value_table, ci=None, type_id=None, ci_id=None, type_attr=None):
-        ci = ci or {}
-        v = self._deserialize_value(attr.alias, attr.value_type, value)
+        if not attr.is_reference:
+            ci = ci or {}
+            v = self._deserialize_value(attr.alias, attr.value_type, value)
+
+            attr.is_choice and value and self._check_is_choice(attr, attr.value_type, v)
+
+        else:
+            v = value or None
 
-        attr.is_choice and value and self._check_is_choice(attr, attr.value_type, v)
         attr.is_unique and self._check_is_unique(
             value_table, attr, ci and ci.id or ci_id, ci and ci.type_id or type_id, v)
-
         self._check_is_required(ci and ci.type_id or type_id, attr, v, type_attr=type_attr)
+        if attr.is_reference:
+            return v
 
         if v == "" and attr.value_type not in (ValueTypeEnum.TEXT,):
             v = None
 
         if attr.re_check and value:
-            self.check_re(attr.re_check, value)
+            self.check_re(attr.re_check, attr.alias, value)
 
         return v
 
@@ -235,10 +250,19 @@ class AttributeValueManager(object):
 
             try:
                 if attr.is_list:
+                    if isinstance(value, dict):
+                        if value.get('op') == "delete":
+                            value['v'] = [ValueTypeMap.serialize[attr.value_type](
+                                self._deserialize_value(attr.alias, attr.value_type, i))
+                                for i in handle_arg_list(value['v'])]
+                            continue
+                        _value = value.get('v') or []
+                    else:
+                        _value = value
                     value_list = [self._validate(attr, i, value_table, ci=None, type_id=type_id, ci_id=ci_id,
                                                  type_attr=ci_attr2type_attr.get(attr.id))
-                                  for i in handle_arg_list(value)]
-                    ci_dict[key] = value_list
+                                  for i in handle_arg_list(_value)]
+                    ci_dict[key] = value_list if not isinstance(value, dict) else dict(op=value.get('op'), v=value_list)
                     if not value_list:
                         self._check_is_required(type_id, attr, '')
 
@@ -266,6 +290,7 @@ class AttributeValueManager(object):
         :return:
         """
         changed = []
+        has_dynamic = False
         for key, value in ci_dict.items():
             attr = key2attr.get(key)
             if not attr:
@@ -274,41 +299,85 @@ class AttributeValueManager(object):
 
             if attr.is_list:
                 existed_attrs = value_table.get_by(attr_id=attr.id, ci_id=ci.id, to_dict=False)
-                existed_values = [i.value for i in existed_attrs]
-                added = set(value) - set(existed_values)
-                deleted = set(existed_values) - set(value)
-                for v in added:
-                    value_table.create(ci_id=ci.id, attr_id=attr.id, value=v, flush=False, commit=False)
-                    changed.append((ci.id, attr.id, OperateType.ADD, None, v, ci.type_id))
+                existed_values = [(ValueTypeMap.serialize[attr.value_type](i.value) if
+                                   i.value or i.value == 0 else i.value) for i in existed_attrs]
 
-                for v in deleted:
-                    existed_attr = existed_attrs[existed_values.index(v)]
-                    existed_attr.delete(flush=False, commit=False)
-                    changed.append((ci.id, attr.id, OperateType.DELETE, v, None, ci.type_id))
+                if isinstance(value, dict):
+                    if value.get('op') == "add":
+                        for v in (value.get('v') or []):
+                            if v not in existed_values:
+                                value_table.create(ci_id=ci.id, attr_id=attr.id, value=v, flush=False, commit=False)
+                                if not attr.is_dynamic:
+                                    changed.append((ci.id, attr.id, OperateType.ADD, None, v, ci.type_id))
+                                else:
+                                    has_dynamic = True
+
+                    elif value.get('op') == "delete":
+                        for v in (value.get('v') or []):
+                            if v in existed_values:
+                                existed_attrs[existed_values.index(v)].delete(flush=False, commit=False)
+                                if not attr.is_dynamic:
+                                    changed.append((ci.id, attr.id, OperateType.DELETE, v, None, ci.type_id))
+                                else:
+                                    has_dynamic = True
+                else:
+                    # Comparison array starts from which position changes
+                    min_len = min(len(value), len(existed_values))
+                    index = 0
+                    while index < min_len:
+                        if value[index] != existed_values[index]:
+                            break
+                        index += 1
+
+                    # Delete first and then add to ensure id sorting
+                    for idx in range(index, len(existed_attrs)):
+                        existed_attr = existed_attrs[idx]
+                        existed_attr.delete(flush=False, commit=False)
+                        if not attr.is_dynamic:
+                            changed.append((ci.id, attr.id, OperateType.DELETE, existed_values[idx], None, ci.type_id))
+                        else:
+                            has_dynamic = True
+                    for idx in range(index, len(value)):
+                        value_table.create(ci_id=ci.id, attr_id=attr.id, value=value[idx], flush=False, commit=False)
+                        if not attr.is_dynamic:
+                            changed.append((ci.id, attr.id, OperateType.ADD, None, value[idx], ci.type_id))
+                        else:
+                            has_dynamic = True
             else:
                 existed_attr = value_table.get_by(attr_id=attr.id, ci_id=ci.id, first=True, to_dict=False)
                 existed_value = existed_attr and existed_attr.value
+                existed_value = (ValueTypeMap.serialize[attr.value_type](existed_value) if
+                                 existed_value or existed_value == 0 else existed_value)
                 if existed_value is None and value is not None:
                     value_table.create(ci_id=ci.id, attr_id=attr.id, value=value, flush=False, commit=False)
 
-                    changed.append((ci.id, attr.id, OperateType.ADD, None, value, ci.type_id))
+                    if not attr.is_dynamic:
+                        changed.append((ci.id, attr.id, OperateType.ADD, None, value, ci.type_id))
+                    else:
+                        has_dynamic = True
                 else:
-                    if existed_value != value:
+                    if existed_value != value and existed_attr:
                         if value is None:
                             existed_attr.delete(flush=False, commit=False)
                         else:
                             existed_attr.update(value=value, flush=False, commit=False)
 
-                        changed.append((ci.id, attr.id, OperateType.UPDATE, existed_value, value, ci.type_id))
+                        if not attr.is_dynamic:
+                            changed.append((ci.id, attr.id, OperateType.UPDATE, existed_value, value, ci.type_id))
+                        else:
+                            has_dynamic = True
 
-        try:
-            db.session.commit()
-        except Exception as e:
-            db.session.rollback()
-            current_app.logger.warning(str(e))
-            return abort(400, ErrFormat.attribute_value_unknown_error.format(e.args[0]))
+        if changed or has_dynamic:
+            try:
+                db.session.commit()
+            except Exception as e:
+                db.session.rollback()
+                current_app.logger.warning(str(e))
+                return abort(400, ErrFormat.attribute_value_unknown_error.format(e.args[0]))
 
-        return self.write_change2(changed, ticket_id=ticket_id)
+            return self.write_change2(changed, ticket_id=ticket_id), has_dynamic
+        else:
+            return None, has_dynamic
 
     @staticmethod
     def delete_attr_value(attr_id, ci_id, commit=True):

cmdb-api/api/lib/common_setting/acl.py

@@ -10,6 +10,11 @@ from api.lib.perm.acl.role import RoleCRUD, RoleRelationCRUD
 from api.lib.perm.acl.user import UserCRUD
 
 
+def validate_app(app_id):
+    app = AppCache.get(app_id)
+    return app.id if app else None
+
+
 class ACLManager(object):
     def __init__(self, app_name='acl', uid=None):
         self.log = current_app.logger
@@ -133,7 +138,8 @@ class ACLManager(object):
         numfound, res = ResourceCRUD.search(q, u, self.validate_app().id, rt_id, page, page_size)
         return res
 
-    def grant_resource(self, rid, resource_id, perms):
+    @staticmethod
+    def grant_resource(rid, resource_id, perms):
         PermissionCRUD.grant(rid, perms, resource_id=resource_id, group_id=None)
 
     @staticmethod
@@ -141,3 +147,7 @@ class ACLManager(object):
         rt = AppCRUD.add(**payload)
 
         return rt.to_dict()
+
+    def role_has_perms(self, rid, resource_name, resource_type_name, perm):
+        app_id = validate_app(self.app_name)
+        return RoleCRUD.has_permission(rid, resource_name, resource_type_name, app_id, perm)

cmdb-api/api/lib/common_setting/decorator.py

@@ -0,0 +1,38 @@
+import functools
+
+from flask import abort, session
+from api.lib.common_setting.acl import ACLManager
+from api.lib.common_setting.resp_format import ErrFormat
+from api.lib.perm.acl.acl import is_app_admin
+
+
+def perms_role_required(app_name, resource_type_name, resource_name, perm, role_name=None):
+    def decorator_perms_role_required(func):
+        @functools.wraps(func)
+        def wrapper_required(*args, **kwargs):
+            acl = ACLManager(app_name)
+            has_perms = False
+            try:
+                has_perms = acl.role_has_perms(session["acl"]['rid'], resource_name, resource_type_name, perm)
+            except Exception as e:
+                # resource_type not exist, continue check role
+                if role_name:
+                    if role_name not in session.get("acl", {}).get("parentRoles", []) and not is_app_admin(app_name):
+                        abort(403, ErrFormat.role_required.format(role_name))
+
+                    return func(*args, **kwargs)
+                else:
+                    abort(403, ErrFormat.resource_no_permission.format(resource_name, perm))
+
+            if not has_perms:
+                if role_name:
+                    if role_name not in session.get("acl", {}).get("parentRoles", []) and not is_app_admin(app_name):
+                        abort(403, ErrFormat.role_required.format(role_name))
+                else:
+                    abort(403, ErrFormat.resource_no_permission.format(resource_name, perm))
+
+            return func(*args, **kwargs)
+
+        return wrapper_required
+
+    return decorator_perms_role_required

cmdb-api/api/lib/common_setting/resp_format.py

@@ -80,3 +80,5 @@ class ErrFormat(CommonErrFormat):
     ldap_test_username_required = _l("LDAP test username required")  # LDAP测试用户名必填
 
     company_wide = _l("Company wide")  # 全公司
+
+    resource_no_permission = _l("No permission to access resource {}, perm {} ")  # 没有权限访问 {} 资源的 {} 权限"

cmdb-api/api/lib/common_setting/role_perm_base.py

@@ -0,0 +1,64 @@
+class OperationPermission(object):
+
+    def __init__(self, resource_perms):
+        for _r in resource_perms:
+            setattr(self, _r['page'], _r['page'])
+            for _p in _r['perms']:
+                setattr(self, _p, _p)
+
+
+class BaseApp(object):
+    resource_type_name = 'OperationPermission'
+    all_resource_perms = []
+
+    def __init__(self):
+        self.admin_name = None
+        self.roles = []
+        self.app_name = 'acl'
+        self.require_create_resource_type = self.resource_type_name
+        self.extra_create_resource_type_list = []
+
+        self.op = None
+
+    @staticmethod
+    def format_role(role_name, role_type, acl_rid, resource_perms, description=''):
+        return dict(
+            role_name=role_name,
+            role_type=role_type,
+            acl_rid=acl_rid,
+            description=description,
+            resource_perms=resource_perms,
+        )
+
+
+class CMDBApp(BaseApp):
+    all_resource_perms = [
+        {"page": "Big_Screen", "page_cn": "大屏", "perms": ["read"]},
+        {"page": "Dashboard", "page_cn": "仪表盘", "perms": ["read"]},
+        {"page": "Resource_Search", "page_cn": "资源搜索", "perms": ["read"]},
+        {"page": "Auto_Discovery_Pool", "page_cn": "自动发现池", "perms": ["read"]},
+        {"page": "My_Subscriptions", "page_cn": "我的订阅", "perms": ["read"]},
+        {"page": "Bulk_Import", "page_cn": "批量导入", "perms": ["read"]},
+        {"page": "Model_Configuration", "page_cn": "模型配置",
+         "perms": ["read", "create_CIType", "create_CIType_group", "update_CIType_group",
+                   "delete_CIType_group", "download_CIType"]},
+        {"page": "Backend_Management", "page_cn": "后台管理", "perms": ["read"]},
+        {"page": "Customized_Dashboard", "page_cn": "定制仪表盘", "perms": ["read"]},
+        {"page": "Service_Tree_Definition", "page_cn": "服务树定义", "perms": ["read"]},
+        {"page": "Model_Relationships", "page_cn": "模型关系", "perms": ["read"]},
+        {"page": "Operation_Audit", "page_cn": "操作审计", "perms": ["read"]},
+        {"page": "Relationship_Types", "page_cn": "关系类型", "perms": ["read"]},
+        {"page": "Auto_Discovery", "page_cn": "自动发现", "perms": ["read", "create_plugin", "update_plugin", "delete_plugin"]},
+        {"page": "TopologyView", "page_cn": "拓扑视图",
+         "perms": ["read", "create_topology_group", "update_topology_group", "delete_topology_group",
+                   "create_topology_view"],
+         },
+    ]
+
+    def __init__(self):
+        super().__init__()
+
+        self.admin_name = 'cmdb_admin'
+        self.app_name = 'cmdb'
+
+        self.op = OperationPermission(self.all_resource_perms)

cmdb-api/api/lib/mixin.py

@@ -1,7 +1,6 @@
 # -*- coding:utf-8 -*-
 
 
-from flask import abort
 from sqlalchemy import func
 
 from api.extensions import db
@@ -13,17 +12,23 @@ class DBMixin(object):
     cls = None
 
     @classmethod
-    def search(cls, page, page_size, fl=None, only_query=False, reverse=False, count_query=False, **kwargs):
+    def search(cls, page, page_size, fl=None, only_query=False, reverse=False, count_query=False,
+               last_size=None, **kwargs):
         page = get_page(page)
         page_size = get_page_size(page_size)
         if fl is None:
-            query = db.session.query(cls.cls).filter(cls.cls.deleted.is_(False))
+            query = db.session.query(cls.cls)
         else:
-            query = db.session.query(*[getattr(cls.cls, i) for i in fl]).filter(cls.cls.deleted.is_(False))
+            query = db.session.query(*[getattr(cls.cls, i) for i in fl])
 
         _query = None
         if count_query:
-            _query = db.session.query(func.count(cls.cls.id)).filter(cls.cls.deleted.is_(False))
+            _query = db.session.query(func.count(cls.cls.id))
+
+        if hasattr(cls.cls, 'deleted'):
+            query = query.filter(cls.cls.deleted.is_(False))
+            if _query:
+                _query = _query.filter(cls.cls.deleted.is_(False))
 
         for k in kwargs:
             if hasattr(cls.cls, k):
@@ -40,14 +45,15 @@ class DBMixin(object):
             return _query, query
 
         numfound = query.count()
-        return numfound, [i.to_dict() if fl is None else getattr(i, '_asdict')()
-                          for i in query.offset((page - 1) * page_size).limit(page_size)]
-
-    def _must_be_required(self, _id):
-        existed = self.cls.get_by_id(_id)
-        existed or abort(404, "Factor [{}] does not exist".format(_id))
-
-        return existed
+        if not last_size:
+            return numfound, [i.to_dict() if fl is None else getattr(i, '_asdict')()
+                              for i in query.offset((page - 1) * page_size).limit(page_size)]
+        else:
+            offset = numfound - last_size
+            if offset < 0:
+                offset = 0
+            return numfound, [i.to_dict() if fl is None else getattr(i, '_asdict')()
+                              for i in query.offset(offset).limit(last_size)]
 
     def _can_add(self, **kwargs):
         raise NotImplementedError

cmdb-api/api/lib/notify.py

@@ -58,7 +58,7 @@ def _request_messenger(subject, body, tos, sender, payload):
 
 def notify_send(subject, body, methods, tos, payload=None):
     payload = payload or {}
-    payload = {k: v or '' for k, v in payload.items()}
+    payload = {k: '' if v is None else v for k, v in payload.items()}
     subject = Template(subject).render(payload)
     body = Template(body).render(payload)
 

cmdb-api/api/lib/perm/acl/acl.py

@@ -153,11 +153,11 @@ class ACLManager(object):
         if resource:
             return ResourceCRUD.delete(resource.id, rebuild=rebuild)
 
-    def has_permission(self, resource_name, resource_type, perm, resource_id=None):
+    def has_permission(self, resource_name, resource_type, perm, resource_id=None, rid=None):
         if is_app_admin(self.app_id):
             return True
 
-        role = self._get_role(current_user.username)
+        role = self._get_role(current_user.username) if rid is None else RoleCache.get(rid)
 
         role or abort(404, ErrFormat.role_not_found.format(current_user.username))
 

cmdb-api/api/lib/perm/acl/audit.py

@@ -376,7 +376,7 @@ class AuditCRUD(object):
                                origin=origin, current=current, extra=extra, source=source.value)
 
     @classmethod
-    def add_login_log(cls, username, is_ok, description, _id=None, logout_at=None):
+    def add_login_log(cls, username, is_ok, description, _id=None, logout_at=None, ip=None, browser=None):
         if _id is not None:
             existed = AuditLoginLog.get_by_id(_id)
             if existed is not None:
@@ -387,8 +387,9 @@ class AuditCRUD(object):
                        is_ok=is_ok,
                        description=description,
                        logout_at=logout_at,
-                       ip=request.headers.get('X-Real-IP') or request.remote_addr,
-                       browser=request.headers.get('User-Agent'),
+                       ip=(ip or request.headers.get('X-Forwarded-For') or
+                           request.headers.get('X-Real-IP') or request.remote_addr or '').split(',')[0],
+                       browser=browser or request.headers.get('User-Agent'),
                        channel=request.values.get('channel', 'web'),
                        )
 

cmdb-api/api/lib/perm/acl/cache.py

@@ -5,6 +5,7 @@ import msgpack
 import redis_lock
 
 from api.extensions import cache
+from api.extensions import db
 from api.extensions import rd
 from api.lib.decorator import flush_db
 from api.models.acl import App
@@ -157,9 +158,10 @@ class RoleRelationCache(object):
     PREFIX_RESOURCES2 = "RoleRelationResources2::id::{0}::AppId::{1}"
 
     @classmethod
-    def get_parent_ids(cls, rid, app_id):
+    def get_parent_ids(cls, rid, app_id, force=False):
         parent_ids = cache.get(cls.PREFIX_PARENT.format(rid, app_id))
-        if not parent_ids:
+        if not parent_ids or force:
+            db.session.commit()
             from api.lib.perm.acl.role import RoleRelationCRUD
             parent_ids = RoleRelationCRUD.get_parent_ids(rid, app_id)
             cache.set(cls.PREFIX_PARENT.format(rid, app_id), parent_ids, timeout=0)
@@ -167,9 +169,10 @@ class RoleRelationCache(object):
         return parent_ids
 
     @classmethod
-    def get_child_ids(cls, rid, app_id):
+    def get_child_ids(cls, rid, app_id, force=False):
         child_ids = cache.get(cls.PREFIX_CHILDREN.format(rid, app_id))
-        if not child_ids:
+        if not child_ids or force:
+            db.session.commit()
             from api.lib.perm.acl.role import RoleRelationCRUD
             child_ids = RoleRelationCRUD.get_child_ids(rid, app_id)
             cache.set(cls.PREFIX_CHILDREN.format(rid, app_id), child_ids, timeout=0)
@@ -177,14 +180,16 @@ class RoleRelationCache(object):
         return child_ids
 
     @classmethod
-    def get_resources(cls, rid, app_id):
+    def get_resources(cls, rid, app_id, force=False):
         """
         :param rid: 
         :param app_id: 
+        :param force:
         :return: {id2perms: {resource_id: [perm,]}, group2perms: {group_id: [perm, ]}}
         """
         resources = cache.get(cls.PREFIX_RESOURCES.format(rid, app_id))
-        if not resources:
+        if not resources or force:
+            db.session.commit()
             from api.lib.perm.acl.role import RoleCRUD
             resources = RoleCRUD.get_resources(rid, app_id)
             if resources['id2perms'] or resources['group2perms']:
@@ -193,9 +198,10 @@ class RoleRelationCache(object):
         return resources or {}
 
     @classmethod
-    def get_resources2(cls, rid, app_id):
+    def get_resources2(cls, rid, app_id, force=False):
         r_g = cache.get(cls.PREFIX_RESOURCES2.format(rid, app_id))
-        if not r_g:
+        if not r_g or force:
+            db.session.commit()
             res = cls.get_resources(rid, app_id)
             id2perms = res['id2perms']
             group2perms = res['group2perms']
@@ -224,22 +230,28 @@ class RoleRelationCache(object):
     @classmethod
     @flush_db
     def rebuild(cls, rid, app_id):
-        cls.clean(rid, app_id)
-
-        cls.get_parent_ids(rid, app_id)
-        cls.get_child_ids(rid, app_id)
-        resources = cls.get_resources(rid, app_id)
-        if resources.get('id2perms') or resources.get('group2perms'):
-            HasResourceRoleCache.add(rid, app_id)
+        if app_id is None:
+            app_ids = [None] + [i.id for i in App.get_by(to_dict=False)]
         else:
-            HasResourceRoleCache.remove(rid, app_id)
-        cls.get_resources2(rid, app_id)
+            app_ids = [app_id]
+
+        for _app_id in app_ids:
+            cls.clean(rid, _app_id)
+
+            cls.get_parent_ids(rid, _app_id, force=True)
+            cls.get_child_ids(rid, _app_id, force=True)
+            resources = cls.get_resources(rid, _app_id, force=True)
+            if resources.get('id2perms') or resources.get('group2perms'):
+                HasResourceRoleCache.add(rid, _app_id)
+            else:
+                HasResourceRoleCache.remove(rid, _app_id)
+            cls.get_resources2(rid, _app_id, force=True)
 
     @classmethod
     @flush_db
     def rebuild2(cls, rid, app_id):
         cache.delete(cls.PREFIX_RESOURCES2.format(rid, app_id))
-        cls.get_resources2(rid, app_id)
+        cls.get_resources2(rid, app_id, force=True)
 
     @classmethod
     def clean(cls, rid, app_id):

cmdb-api/api/lib/perm/acl/permission.py

@@ -79,7 +79,8 @@ class PermissionCRUD(object):
         return r and cls.get_all(r.id)
 
     @staticmethod
-    def grant(rid, perms, resource_id=None, group_id=None, rebuild=True, source=AuditOperateSource.acl):
+    def grant(rid, perms, resource_id=None, group_id=None, rebuild=True,
+              source=AuditOperateSource.acl, force_update=False):
         app_id = None
         rt_id = None
 
@@ -106,8 +107,23 @@ class PermissionCRUD(object):
             if not perms:
                 perms = [i.get('name') for i in ResourceTypeCRUD.get_perms(group.resource_type_id)]
 
-        _role_permissions = []
+        if force_update:
+            revoke_role_permissions = []
+            existed_perms = RolePermission.get_by(rid=rid,
+                                                  app_id=app_id,
+                                                  group_id=group_id,
+                                                  resource_id=resource_id,
+                                                  to_dict=False)
+            for role_perm in existed_perms:
+                perm = PermissionCache.get(role_perm.perm_id, rt_id)
+                if perm and perm.name not in perms:
+                    role_perm.soft_delete()
+                    revoke_role_permissions.append(role_perm)
 
+            AuditCRUD.add_permission_log(app_id, AuditOperateType.revoke, rid, rt_id,
+                                         revoke_role_permissions, source=source)
+
+        _role_permissions = []
         for _perm in set(perms):
             perm = PermissionCache.get(_perm, rt_id)
             if not perm:
@@ -274,12 +290,14 @@ class PermissionCRUD(object):
                     perm2resource.setdefault(_perm, []).append(resource_id)
             for _perm in perm2resource:
                 perm = PermissionCache.get(_perm, resource_type_id)
-                existeds = RolePermission.get_by(rid=rid,
-                                                 app_id=app_id,
-                                                 perm_id=perm.id,
-                                                 __func_in___key_resource_id=perm2resource[_perm],
-                                                 to_dict=False)
-                for existed in existeds:
+                if perm is None:
+                    continue
+                exists = RolePermission.get_by(rid=rid,
+                                               app_id=app_id,
+                                               perm_id=perm.id,
+                                               __func_in___key_resource_id=perm2resource[_perm],
+                                               to_dict=False)
+                for existed in exists:
                     existed.deleted = True
                     existed.deleted_at = datetime.datetime.now()
                     db.session.add(existed)

cmdb-api/api/lib/perm/acl/resource.py

@@ -2,7 +2,6 @@
 
 
 from flask import abort
-from flask import current_app
 
 from api.extensions import db
 from api.lib.perm.acl.audit import AuditCRUD
@@ -127,11 +126,18 @@ class ResourceTypeCRUD(object):
         existed_ids = [i.id for i in existed]
         current_ids = []
 
+        rebuild_rids = set()
         for i in existed:
             if i.name not in perms:
-                i.soft_delete()
+                i.soft_delete(commit=False)
+                for rp in RolePermission.get_by(perm_id=i.id, to_dict=False):
+                    rp.soft_delete(commit=False)
+                    rebuild_rids.add((rp.app_id, rp.rid))
             else:
                 current_ids.append(i.id)
+        db.session.commit()
+        for _app_id, _rid in rebuild_rids:
+            role_rebuild.apply_async(args=(_rid, _app_id), queue=ACL_QUEUE)
 
         for i in perms:
             if i not in existed_names:
@@ -309,9 +315,12 @@ class ResourceCRUD(object):
         return resource
 
     @staticmethod
-    def delete(_id, rebuild=True):
+    def delete(_id, rebuild=True, app_id=None):
         resource = Resource.get_by_id(_id) or abort(404, ErrFormat.resource_not_found.format("id={}".format(_id)))
 
+        if app_id is not None and resource.app_id != app_id:
+            return abort(404, ErrFormat.resource_not_found.format("id={}".format(_id)))
+
         origin = resource.to_dict()
         resource.soft_delete()
 

cmdb-api/api/lib/perm/acl/role.py

@@ -3,12 +3,14 @@
 
 import time
 
+import redis_lock
 import six
 from flask import abort
 from flask import current_app
 from sqlalchemy import or_
 
 from api.extensions import db
+from api.extensions import rd
 from api.lib.perm.acl.app import AppCRUD
 from api.lib.perm.acl.audit import AuditCRUD, AuditOperateType, AuditScope
 from api.lib.perm.acl.cache import AppCache
@@ -62,7 +64,9 @@ class RoleRelationCRUD(object):
 
         id2parents = {}
         for i in res:
-            id2parents.setdefault(rid2uid.get(i.child_id, i.child_id), []).append(RoleCache.get(i.parent_id).to_dict())
+            parent = RoleCache.get(i.parent_id)
+            if parent:
+                id2parents.setdefault(rid2uid.get(i.child_id, i.child_id), []).append(parent.to_dict())
 
         return id2parents
 
@@ -141,24 +145,27 @@ class RoleRelationCRUD(object):
 
     @classmethod
     def add(cls, role, parent_id, child_ids, app_id):
-        result = []
-        for child_id in child_ids:
-            existed = RoleRelation.get_by(parent_id=parent_id, child_id=child_id, app_id=app_id)
-            if existed:
-                continue
+        with redis_lock.Lock(rd.r, "ROLE_RELATION_ADD"):
+            db.session.commit()
 
-            RoleRelationCache.clean(parent_id, app_id)
-            RoleRelationCache.clean(child_id, app_id)
+            result = []
+            for child_id in child_ids:
+                existed = RoleRelation.get_by(parent_id=parent_id, child_id=child_id, app_id=app_id)
+                if existed:
+                    continue
 
-            if parent_id in cls.recursive_child_ids(child_id, app_id):
-                return abort(400, ErrFormat.inheritance_dead_loop)
+                if parent_id in cls.recursive_child_ids(child_id, app_id):
+                    return abort(400, ErrFormat.inheritance_dead_loop)
 
-            if app_id is None:
-                for app in AppCRUD.get_all():
-                    if app.name != "acl":
-                        RoleRelationCache.clean(child_id, app.id)
+                result.append(RoleRelation.create(parent_id=parent_id, child_id=child_id, app_id=app_id).to_dict())
 
-            result.append(RoleRelation.create(parent_id=parent_id, child_id=child_id, app_id=app_id).to_dict())
+                RoleRelationCache.clean(parent_id, app_id)
+                RoleRelationCache.clean(child_id, app_id)
+
+                if app_id is None:
+                    for app in AppCRUD.get_all():
+                        if app.name != "acl":
+                            RoleRelationCache.clean(child_id, app.id)
 
         AuditCRUD.add_role_log(app_id, AuditOperateType.role_relation_add,
                                AuditScope.role_relation, role.id, {}, {},
@@ -372,16 +379,16 @@ class RoleCRUD(object):
             resource_type_id = resource_type and resource_type.id
 
         result = dict(resources=dict(), groups=dict())
-        s = time.time()
+        # s = time.time()
         parent_ids = RoleRelationCRUD.recursive_parent_ids(rid, app_id)
-        current_app.logger.info('parent ids {0}: {1}'.format(parent_ids, time.time() - s))
+        # current_app.logger.info('parent ids {0}: {1}'.format(parent_ids, time.time() - s))
         for parent_id in parent_ids:
 
             _resources, _groups = cls._extend_resources(parent_id, resource_type_id, app_id)
-            current_app.logger.info('middle1: {0}'.format(time.time() - s))
+            # current_app.logger.info('middle1: {0}'.format(time.time() - s))
             _merge(result['resources'], _resources)
-            current_app.logger.info('middle2: {0}'.format(time.time() - s))
-            current_app.logger.info(len(_groups))
+            # current_app.logger.info('middle2: {0}'.format(time.time() - s))
+            # current_app.logger.info(len(_groups))
             if not group_flat:
                 _merge(result['groups'], _groups)
             else:
@@ -392,7 +399,7 @@ class RoleCRUD(object):
                             item.setdefault('permissions', [])
                             item['permissions'] = list(set(item['permissions'] + _groups[rg_id]['permissions']))
                             result['resources'][item['id']] = item
-        current_app.logger.info('End: {0}'.format(time.time() - s))
+        # current_app.logger.info('End: {0}'.format(time.time() - s))
 
         result['resources'] = list(result['resources'].values())
         result['groups'] = list(result['groups'].values())

cmdb-api/api/lib/perm/auth.py

@@ -51,12 +51,12 @@ def _auth_with_key():
     user, authenticated = User.query.authenticate_with_key(key, secret, req_args, path)
     if user and authenticated:
         login_user(user)
-        reset_session(user)
+        # reset_session(user)
         return True
 
     role, authenticated = Role.query.authenticate_with_key(key, secret, req_args, path)
     if role and authenticated:
-        reset_session(None, role=role.name)
+        # reset_session(None, role=role.name)
         return True
 
     return False

cmdb-api/api/lib/resp_format.py

@@ -29,6 +29,6 @@ class CommonErrFormat(object):
 
     role_required = _l("Role {} can only operate!")  # 角色 {} 才能操作!
     user_not_found = _l("User {} does not exist")  # 用户 {} 不存在
-    no_permission = _l("You do not have {} permission for resource: {}!")  # 您没有资源: {} 的{}权限!
+    no_permission = _l("For resource: {}, you do not have {} permission!")  # 您没有资源: {} 的{}权限!
     no_permission2 = _l("You do not have permission to operate!")  # 您没有操作权限!
     no_permission_only_owner = _l("Only the creator or administrator has permission!")  # 只有创建人或者管理员才有权限!

cmdb-api/api/lib/secrets/inner.py

@@ -3,8 +3,8 @@ import os
 import secrets
 import sys
 import threading
-
 from base64 import b64decode, b64encode
+
 from Cryptodome.Protocol.SecretSharing import Shamir
 from colorama import Back, Fore, Style, init as colorama_init
 from cryptography.hazmat.backends import default_backend
@@ -30,6 +30,7 @@ seal_status = True
 secrets_encrypt_key = ""
 secrets_root_key = ""
 
+
 def string_to_bytes(value):
     if not value:
         return ""
@@ -78,7 +79,7 @@ class KeyManage:
                 (len(sys.argv) > 1 and sys.argv[1] in ("run", "cmdb-password-data-migrate"))):
 
             self.backend = backend
-            threading.Thread(target=self.watch_root_key, args=(app,)).start()
+            threading.Thread(target=self.watch_root_key, args=(app,), daemon=True).start()
 
             self.trigger = app.config.get("INNER_TRIGGER_TOKEN")
             if not self.trigger:
@@ -412,7 +413,7 @@ class KeyManage:
 class InnerCrypt:
     def __init__(self):
         self.encrypt_key = b64decode(secrets_encrypt_key)
-        #self.encrypt_key = b64decode(secrets_encrypt_key, "".encode("utf-8"))
+        # self.encrypt_key = b64decode(secrets_encrypt_key, "".encode("utf-8"))
 
     def encrypt(self, plaintext):
         """
@@ -490,4 +491,4 @@ if __name__ == "__main__":
     t_ciphertext, status1 = c.encrypt(t_plaintext)
     print("Ciphertext:", t_ciphertext)
     decrypted_plaintext, status2 = c.decrypt(t_ciphertext)
-    print("Decrypted plaintext:", decrypted_plaintext)
+    print("Decrypted plaintext:", decrypted_plaintext)

cmdb-api/api/lib/webhook.py

@@ -88,11 +88,11 @@ def webhook_request(webhook, payload):
 
     params = webhook.get('parameters') or None
     if isinstance(params, dict):
-        params = json.loads(Template(json.dumps(params)).render(payload))
+        params = json.loads(Template(json.dumps(params)).render(payload).encode('utf-8'))
 
     headers = json.loads(Template(json.dumps(webhook.get('headers') or {})).render(payload))
 
-    data = Template(json.dumps(webhook.get('body', ''))).render(payload)
+    data = Template(json.dumps(webhook.get('body', ''))).render(payload).encode('utf-8')
     auth = _wrap_auth(**webhook.get('authorization', {}))
 
     if (webhook.get('authorization', {}).get("type") or '').lower() == 'oauth2.0':

cmdb-api/api/models/cmdb.py

@@ -2,7 +2,6 @@
 
 
 import datetime
-
 from sqlalchemy.dialects.mysql import DOUBLE
 
 from api.extensions import db
@@ -11,6 +10,7 @@ from api.lib.cmdb.const import CIStatusEnum
 from api.lib.cmdb.const import CITypeOperateType
 from api.lib.cmdb.const import ConstraintEnum
 from api.lib.cmdb.const import OperateType
+from api.lib.cmdb.const import RelationSourceEnum
 from api.lib.cmdb.const import ValueTypeEnum
 from api.lib.database import Model
 from api.lib.database import Model2
@@ -79,8 +79,11 @@ class CITypeRelation(Model):
     relation_type_id = db.Column(db.Integer, db.ForeignKey("c_relation_types.id"), nullable=False)
     constraint = db.Column(db.Enum(*ConstraintEnum.all()), default=ConstraintEnum.One2Many)
 
-    parent_attr_id = db.Column(db.Integer, db.ForeignKey("c_attributes.id"))
-    child_attr_id = db.Column(db.Integer, db.ForeignKey("c_attributes.id"))
+    parent_attr_id = db.Column(db.Integer, db.ForeignKey("c_attributes.id"))  # CMDB > 2.4.5: deprecated
+    child_attr_id = db.Column(db.Integer, db.ForeignKey("c_attributes.id"))  # CMDB > 2.4.5: deprecated
+
+    parent_attr_ids = db.Column(db.JSON)  # [parent_attr_id, ]
+    child_attr_ids = db.Column(db.JSON)  # [child_attr_id, ]
 
     parent = db.relationship("CIType", primaryjoin="CIType.id==CITypeRelation.parent_id")
     child = db.relationship("CIType", primaryjoin="CIType.id==CITypeRelation.child_id")
@@ -101,6 +104,11 @@ class Attribute(Model):
     is_link = db.Column(db.Boolean, default=False)
     is_password = db.Column(db.Boolean, default=False)
     is_sortable = db.Column(db.Boolean, default=False)
+    is_dynamic = db.Column(db.Boolean, default=False)
+    is_bool = db.Column(db.Boolean, default=False)
+
+    is_reference = db.Column(db.Boolean, default=False)
+    reference_type_id = db.Column(db.Integer, db.ForeignKey('c_ci_types.id'))
 
     default = db.Column(db.JSON)  # {"default": None}
 
@@ -209,6 +217,26 @@ class CITriggerHistory(Model):
     webhook = db.Column(db.Text)
 
 
+class TopologyViewGroup(Model):
+    __tablename__ = 'c_topology_view_groups'
+
+    name = db.Column(db.String(64), index=True)
+    order = db.Column(db.Integer, default=0)
+
+
+class TopologyView(Model):
+    __tablename__ = 'c_topology_views'
+
+    name = db.Column(db.String(64), index=True)
+    group_id = db.Column(db.Integer, db.ForeignKey('c_topology_view_groups.id'))
+    category = db.Column(db.String(32))
+    central_node_type = db.Column(db.Integer)
+    central_node_instances = db.Column(db.Text)
+    path = db.Column(db.JSON)
+    order = db.Column(db.Integer, default=0)
+    option = db.Column(db.JSON)
+
+
 class CITypeUniqueConstraint(Model):
     __tablename__ = "c_c_t_u_c"
 
@@ -236,6 +264,7 @@ class CIRelation(Model):
     second_ci_id = db.Column(db.Integer, db.ForeignKey("c_cis.id"), nullable=False)
     relation_type_id = db.Column(db.Integer, db.ForeignKey("c_relation_types.id"), nullable=False)
     more = db.Column(db.Integer, db.ForeignKey("c_cis.id"))
+    source = db.Column(db.Enum(*RelationSourceEnum.all()), name="source")
 
     ancestor_ids = db.Column(db.String(128), index=True)
 
@@ -432,6 +461,7 @@ class CITypeHistory(Model):
 
     attr_id = db.Column(db.Integer)
     trigger_id = db.Column(db.Integer)
+    rc_id = db.Column(db.Integer)
     unique_constraint_id = db.Column(db.Integer)
 
     uid = db.Column(db.Integer, index=True)
@@ -541,18 +571,28 @@ class AutoDiscoveryCIType(Model):
 
     attributes = db.Column(db.JSON)  # {ad_key: cmdb_key}
 
-    relation = db.Column(db.JSON)  # [{ad_key: {type_id: x, attr_id: x}}]
+    relation = db.Column(db.JSON)  # [{ad_key: {type_id: x, attr_id: x}}], CMDB > 2.4.5: deprecated
 
     auto_accept = db.Column(db.Boolean, default=False)
 
     agent_id = db.Column(db.String(8), index=True)
     query_expr = db.Column(db.Text)
 
-    interval = db.Column(db.Integer)  # seconds
+    interval = db.Column(db.Integer)  # seconds, > 2.4.5: deprecated
     cron = db.Column(db.String(128))
 
     extra_option = db.Column(db.JSON)
     uid = db.Column(db.Integer, index=True)
+    enabled = db.Column(db.Boolean, default=True)
+
+
+class AutoDiscoveryCITypeRelation(Model):
+    __tablename__ = "c_ad_ci_type_relations"
+
+    ad_type_id = db.Column(db.Integer, db.ForeignKey('c_ci_types.id'), nullable=False)
+    ad_key = db.Column(db.String(128))
+    peer_type_id = db.Column(db.Integer, db.ForeignKey('c_ci_types.id'), nullable=False)
+    peer_attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False)
 
 
 class AutoDiscoveryCI(Model):
@@ -570,6 +610,45 @@ class AutoDiscoveryCI(Model):
     accept_time = db.Column(db.DateTime)
 
 
+class AutoDiscoveryRuleSyncHistory(Model2):
+    __tablename__ = "c_ad_rule_sync_histories"
+
+    adt_id = db.Column(db.Integer, db.ForeignKey('c_ad_ci_types.id'))
+    oneagent_id = db.Column(db.String(8))
+    oneagent_name = db.Column(db.String(64))
+    sync_at = db.Column(db.DateTime, default=datetime.datetime.now())
+
+
+class AutoDiscoveryExecHistory(Model2):
+    __tablename__ = "c_ad_exec_histories"
+
+    type_id = db.Column(db.Integer, index=True)
+    stdout = db.Column(db.Text)
+
+
+class AutoDiscoveryCounter(Model2):
+    __tablename__ = "c_ad_counter"
+
+    type_id = db.Column(db.Integer, index=True)
+    rule_count = db.Column(db.Integer, default=0)
+    exec_target_count = db.Column(db.Integer, default=0)
+    instance_count = db.Column(db.Integer, default=0)
+    accept_count = db.Column(db.Integer, default=0)
+    this_month_count = db.Column(db.Integer, default=0)
+    this_week_count = db.Column(db.Integer, default=0)
+    last_month_count = db.Column(db.Integer, default=0)
+    last_week_count = db.Column(db.Integer, default=0)
+
+
+class AutoDiscoveryAccount(Model):
+    __tablename__ = "c_ad_accounts"
+
+    uid = db.Column(db.Integer, index=True)
+    name = db.Column(db.String(64))
+    adr_id = db.Column(db.Integer, db.ForeignKey('c_ad_rules.id'))
+    config = db.Column(db.JSON)
+
+
 class CIFilterPerms(Model):
     __tablename__ = "c_ci_filter_perms"
 

cmdb-api/api/tasks/acl.py

@@ -3,12 +3,13 @@
 import json
 import re
 
-from celery_once import QueueOnce
+import redis_lock
 from flask import current_app
 from werkzeug.exceptions import BadRequest
 from werkzeug.exceptions import NotFound
 
 from api.extensions import celery
+from api.extensions import rd
 from api.lib.decorator import flush_db
 from api.lib.decorator import reconnect_db
 from api.lib.perm.acl.audit import AuditCRUD
@@ -25,14 +26,14 @@ from api.models.acl import Role
 from api.models.acl import Trigger
 
 
-@celery.task(name="acl.role_rebuild",
-             queue=ACL_QUEUE,)
+@celery.task(name="acl.role_rebuild", queue=ACL_QUEUE, )
 @flush_db
 @reconnect_db
 def role_rebuild(rids, app_id):
     rids = rids if isinstance(rids, list) else [rids]
     for rid in rids:
-        RoleRelationCache.rebuild(rid, app_id)
+        with redis_lock.Lock(rd.r, "ROLE_REBUILD_{}_{}".format(rid, app_id)):
+            RoleRelationCache.rebuild(rid, app_id)
 
     current_app.logger.info("Role {0} App {1} rebuild..........".format(rids, app_id))
 

cmdb-api/api/tasks/cmdb.py

@@ -1,8 +1,8 @@
 # -*- coding:utf-8 -*- 
 
 
+import datetime
 import json
-
 import redis_lock
 from flask import current_app
 from flask_login import login_user
@@ -12,16 +12,23 @@ from api.extensions import celery
 from api.extensions import db
 from api.extensions import es
 from api.extensions import rd
+from api.lib.cmdb.cache import AttributeCache
 from api.lib.cmdb.cache import CITypeAttributesCache
 from api.lib.cmdb.const import CMDB_QUEUE
 from api.lib.cmdb.const import REDIS_PREFIX_CI
 from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
 from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION2
+from api.lib.cmdb.const import RelationSourceEnum
 from api.lib.cmdb.perms import CIFilterPermsCRUD
+from api.lib.cmdb.utils import TableMap
 from api.lib.decorator import flush_db
 from api.lib.decorator import reconnect_db
 from api.lib.perm.acl.cache import UserCache
 from api.lib.utils import handle_arg_list
+from api.models.cmdb import Attribute
+from api.models.cmdb import AutoDiscoveryCI
+from api.models.cmdb import AutoDiscoveryCIType
+from api.models.cmdb import AutoDiscoveryCITypeRelation
 from api.models.cmdb import CI
 from api.models.cmdb import CIRelation
 from api.models.cmdb import CITypeAttribute
@@ -33,6 +40,7 @@ from api.models.cmdb import CITypeAttribute
 def ci_cache(ci_id, operate_type, record_id):
     from api.lib.cmdb.ci import CITriggerManager
     from api.lib.cmdb.ci import CIRelationManager
+    from api.lib.cmdb.ci_type import CITypeAttributeManager
 
     m = api.lib.cmdb.ci.CIManager()
     ci_dict = m.get_ci_by_id_from_db(ci_id, need_children=False, use_master=False)
@@ -48,17 +56,27 @@ def ci_cache(ci_id, operate_type, record_id):
         current_app.test_request_context().push()
         login_user(UserCache.get('worker'))
 
-        CITriggerManager.fire(operate_type, ci_dict, record_id)
+        _, enum_map = CITypeAttributeManager.get_attr_names_label_enum(ci_dict.get('_type'))
+        payload = dict()
+        for k, v in ci_dict.items():
+            if k in enum_map:
+                if isinstance(v, list):
+                    payload[k] = [enum_map[k].get(i, i) for i in v]
+                else:
+                    payload[k] = enum_map[k].get(v, v)
+            else:
+                payload[k] = v
+        CITriggerManager.fire(operate_type, payload, record_id)
 
     ci_dict and CIRelationManager.build_by_attribute(ci_dict)
 
 
 @celery.task(name="cmdb.rebuild_relation_for_attribute_changed", queue=CMDB_QUEUE)
 @reconnect_db
-def rebuild_relation_for_attribute_changed(ci_type_relation):
+def rebuild_relation_for_attribute_changed(ci_type_relation, uid):
     from api.lib.cmdb.ci import CIRelationManager
 
-    CIRelationManager.rebuild_all_by_attribute(ci_type_relation)
+    CIRelationManager.rebuild_all_by_attribute(ci_type_relation, uid)
 
 
 @celery.task(name="cmdb.batch_ci_cache", queue=CMDB_QUEUE)
@@ -79,7 +97,7 @@ def batch_ci_cache(ci_ids, ):  # only for attribute change index
 
 @celery.task(name="cmdb.ci_delete", queue=CMDB_QUEUE)
 @reconnect_db
-def ci_delete(ci_id):
+def ci_delete(ci_id, type_id):
     current_app.logger.info(ci_id)
 
     if current_app.config.get("USE_ES"):
@@ -87,6 +105,19 @@ def ci_delete(ci_id):
     else:
         rd.delete(ci_id, REDIS_PREFIX_CI)
 
+    instance = AutoDiscoveryCI.get_by(ci_id=ci_id, to_dict=False, first=True)
+    if instance is not None:
+        adt = AutoDiscoveryCIType.get_by_id(instance.adt_id)
+        if adt:
+            adt.update(updated_at=datetime.datetime.now())
+        instance.delete()
+
+    for attr in Attribute.get_by(reference_type_id=type_id, to_dict=False):
+        table = TableMap(attr=attr).table
+        for i in getattr(table, 'get_by')(attr_id=attr.id, value=ci_id, to_dict=False):
+            i.delete()
+            ci_cache(i.ci_id, None, None)
+
     current_app.logger.info("{0} delete..........".format(ci_id))
 
 
@@ -113,18 +144,17 @@ def ci_delete_trigger(trigger, operate_type, ci_dict):
 @reconnect_db
 def ci_relation_cache(parent_id, child_id, ancestor_ids):
     with redis_lock.Lock(rd.r, "CIRelation_{}".format(parent_id)):
-        if ancestor_ids is None:
-            children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0]
-            children = json.loads(children) if children is not None else {}
+        children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0]
+        children = json.loads(children) if children is not None else {}
 
-            cr = CIRelation.get_by(first_ci_id=parent_id, second_ci_id=child_id, ancestor_ids=ancestor_ids,
-                                   first=True, to_dict=False)
-            if str(child_id) not in children:
-                children[str(child_id)] = cr.second_ci.type_id
+        cr = CIRelation.get_by(first_ci_id=parent_id, second_ci_id=child_id, ancestor_ids=ancestor_ids,
+                               first=True, to_dict=False)
+        if str(child_id) not in children:
+            children[str(child_id)] = cr.second_ci.type_id
 
-            rd.create_or_update({parent_id: json.dumps(children)}, REDIS_PREFIX_CI_RELATION)
+        rd.create_or_update({parent_id: json.dumps(children)}, REDIS_PREFIX_CI_RELATION)
 
-        else:
+        if ancestor_ids is not None:
             key = "{},{}".format(ancestor_ids, parent_id)
             grandson = rd.get([key], REDIS_PREFIX_CI_RELATION2)[0]
             grandson = json.loads(grandson) if grandson is not None else {}
@@ -177,7 +207,7 @@ def ci_relation_add(parent_dict, child_id, uid):
             for ci in response:
                 try:
                     CIRelationManager.add(ci['_id'], child_id)
-                    ci_relation_cache(ci['_id'], child_id)
+                    ci_relation_cache(ci['_id'], child_id, None)
                 except Exception as e:
                     current_app.logger.warning(e)
                 finally:
@@ -191,16 +221,15 @@ def ci_relation_add(parent_dict, child_id, uid):
 @reconnect_db
 def ci_relation_delete(parent_id, child_id, ancestor_ids):
     with redis_lock.Lock(rd.r, "CIRelation_{}".format(parent_id)):
-        if ancestor_ids is None:
-            children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0]
-            children = json.loads(children) if children is not None else {}
+        children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0]
+        children = json.loads(children) if children is not None else {}
 
-            if str(child_id) in children:
-                children.pop(str(child_id))
+        if str(child_id) in children:
+            children.pop(str(child_id))
 
-            rd.create_or_update({parent_id: json.dumps(children)}, REDIS_PREFIX_CI_RELATION)
+        rd.create_or_update({parent_id: json.dumps(children)}, REDIS_PREFIX_CI_RELATION)
 
-        else:
+        if ancestor_ids is not None:
             key = "{},{}".format(ancestor_ids, parent_id)
             grandson = rd.get([key], REDIS_PREFIX_CI_RELATION2)[0]
             grandson = json.loads(grandson) if grandson is not None else {}
@@ -251,3 +280,93 @@ def calc_computed_attribute(attr_id, uid):
         cis = CI.get_by(type_id=i.type_id, to_dict=False)
         for ci in cis:
             cim.update(ci.id, {})
+
+
+@celery.task(name="cmdb.write_ad_rule_sync_history", queue=CMDB_QUEUE)
+@reconnect_db
+def write_ad_rule_sync_history(rules, oneagent_id, oneagent_name, sync_at):
+    from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryRuleSyncHistoryCRUD
+
+    for rule in rules:
+        AutoDiscoveryRuleSyncHistoryCRUD().upsert(adt_id=rule['id'],
+                                                  oneagent_id=oneagent_id,
+                                                  oneagent_name=oneagent_name,
+                                                  sync_at=sync_at,
+                                                  commit=False)
+    try:
+        db.session.commit()
+    except Exception as e:
+        current_app.logger.error("write auto discovery rule sync history failed: {}".format(e))
+        db.session.rollback()
+
+
+@celery.task(name="cmdb.build_relations_for_ad_accept", queue=CMDB_QUEUE)
+@reconnect_db
+def build_relations_for_ad_accept(adc, ci_id, ad_key2attr):
+    from api.lib.cmdb.ci import CIRelationManager
+    from api.lib.cmdb.search import SearchError
+    from api.lib.cmdb.search.ci import search as ci_search
+
+    current_app.test_request_context().push()
+    login_user(UserCache.get('worker'))
+
+    relation_ads = AutoDiscoveryCITypeRelation.get_by(ad_type_id=adc['type_id'], to_dict=False)
+    for r_adt in relation_ads:
+        ad_key = r_adt.ad_key
+        if not adc['instance'].get(ad_key):
+            continue
+
+        ad_key_values = [adc['instance'].get(ad_key)] if not isinstance(
+            adc['instance'].get(ad_key), list) else adc['instance'].get(ad_key)
+        for ad_key_value in ad_key_values:
+            query = "_type:{},{}:{}".format(r_adt.peer_type_id, r_adt.peer_attr_id, ad_key_value)
+            s = ci_search(query, use_ci_filter=False, count=1000000)
+            try:
+                response, _, _, _, _, _ = s.search()
+            except SearchError as e:
+                current_app.logger.error("build_relations_for_ad_accept failed: {}".format(e))
+                return
+
+            for relation_ci in response:
+                relation_ci_id = relation_ci['_id']
+                try:
+                    CIRelationManager.add(ci_id, relation_ci_id,
+                                          valid=False,
+                                          source=RelationSourceEnum.AUTO_DISCOVERY)
+
+                except:
+                    try:
+                        CIRelationManager.add(relation_ci_id, ci_id,
+                                              valid=False,
+                                              source=RelationSourceEnum.AUTO_DISCOVERY)
+                    except:
+                        pass
+
+    # build relations in reverse
+    relation_ads = AutoDiscoveryCITypeRelation.get_by(peer_type_id=adc['type_id'], to_dict=False)
+    attr2ad_key = {v: k for k, v in ad_key2attr.items()}
+    for r_adt in relation_ads:
+        attr = AttributeCache.get(r_adt.peer_attr_id)
+        ad_key = attr2ad_key.get(attr and attr.name)
+        if not ad_key:
+            continue
+
+        ad_value = adc['instance'].get(ad_key)
+        peer_ad_key = r_adt.ad_key
+        peer_instances = AutoDiscoveryCI.get_by(type_id=r_adt.ad_type_id, to_dict=False)
+        for peer_instance in peer_instances:
+            peer_ad_values = peer_instance.instance.get(peer_ad_key)
+            peer_ad_values = [peer_ad_values] if not isinstance(peer_ad_values, list) else peer_ad_values
+            if ad_value in peer_ad_values and peer_instance.ci_id:
+                try:
+                    CIRelationManager.add(peer_instance.ci_id, ci_id,
+                                          valid=False,
+                                          source=RelationSourceEnum.AUTO_DISCOVERY)
+
+                except:
+                    try:
+                        CIRelationManager.add(ci_id, peer_instance.ci_id,
+                                              valid=False,
+                                              source=RelationSourceEnum.AUTO_DISCOVERY)
+                    except:
+                        pass

cmdb-api/api/translations/zh/LC_MESSAGES/messages.po

@@ -7,7 +7,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2024-03-29 10:42+0800\n"
+"POT-Creation-Date: 2024-09-26 17:57+0800\n"
 "PO-Revision-Date: 2023-12-25 20:21+0800\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language: zh\n"
@@ -16,7 +16,7 @@ msgstr ""
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=utf-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"Generated-By: Babel 2.14.0\n"
+"Generated-By: Babel 2.16.0\n"
 
 #: api/lib/resp_format.py:7
 msgid "unauthorized"
@@ -81,7 +81,7 @@ msgid "User {} does not exist"
 msgstr "用户 {} 不存在"
 
 #: api/lib/resp_format.py:32
-msgid "You do not have {} permission for resource: {}!"
+msgid "For resource: {}, you do not have {} permission!"
 msgstr "您没有资源: {} 的{}权限!"
 
 #: api/lib/resp_format.py:33
@@ -169,8 +169,8 @@ msgstr "目前只允许 属性创建人、管理员 删除属性!"
 #: api/lib/cmdb/resp_format.py:37
 msgid ""
 "Attribute field names cannot be built-in fields: id, _id, ci_id, type, "
-"_type, ci_type"
-msgstr "属性字段名不能是内置字段: id, _id, ci_id, type, _type, ci_type"
+"_type, ci_type, ticket_id"
+msgstr "属性字段名不能是内置字段: id, _id, ci_id, type, _type, ci_type, ci_type, ticket_id"
 
 #: api/lib/cmdb/resp_format.py:39
 msgid "Predefined value: Other model request parameters are illegal!"
@@ -197,253 +197,305 @@ msgid "CI already exists!"
 msgstr "CI 已经存在!"
 
 #: api/lib/cmdb/resp_format.py:47
+msgid "{}: CI reference {} does not exist!"
+msgstr "{}: CI引用 {} 不存在!"
+
+#: api/lib/cmdb/resp_format.py:48
+msgid "{}: CI reference {} is illegal!"
+msgstr "{}, CI引用 {} 不合法!"
+
+#: api/lib/cmdb/resp_format.py:49
 msgid "Relationship constraint: {}, verification failed"
 msgstr "关系约束: {}, 校验失败"
 
-#: api/lib/cmdb/resp_format.py:49
+#: api/lib/cmdb/resp_format.py:51
 msgid ""
 "Many-to-many relationship constraint: Model {} <-> {} already has a many-"
 "to-many relationship!"
 msgstr "多对多关系 限制: 模型 {} <-> {} 已经存在多对多关系!"
 
-#: api/lib/cmdb/resp_format.py:52
+#: api/lib/cmdb/resp_format.py:54
 msgid "CI relationship: {} does not exist"
 msgstr "CI关系: {} 不存在"
 
-#: api/lib/cmdb/resp_format.py:55
+#: api/lib/cmdb/resp_format.py:57
 msgid "In search expressions, not supported before parentheses: or, not"
 msgstr "搜索表达式里小括号前不支持: 或、非"
 
-#: api/lib/cmdb/resp_format.py:57
+#: api/lib/cmdb/resp_format.py:59
 msgid "Model {} does not exist"
 msgstr "模型 {} 不存在"
 
-#: api/lib/cmdb/resp_format.py:58
+#: api/lib/cmdb/resp_format.py:60
 msgid "Model {} already exists"
 msgstr "模型 {} 已经存在"
 
-#: api/lib/cmdb/resp_format.py:59
+#: api/lib/cmdb/resp_format.py:61
 msgid "The primary key is undefined or has been deleted"
 msgstr "主键未定义或者已被删除"
 
-#: api/lib/cmdb/resp_format.py:60
+#: api/lib/cmdb/resp_format.py:62
 msgid "Only the creator can delete it!"
 msgstr "只有创建人才能删除它!"
 
-#: api/lib/cmdb/resp_format.py:61
+#: api/lib/cmdb/resp_format.py:63
 msgid "The model cannot be deleted because the CI already exists"
 msgstr "因为CI已经存在，不能删除模型"
 
-#: api/lib/cmdb/resp_format.py:63
+#: api/lib/cmdb/resp_format.py:65
 msgid "The inheritance cannot be deleted because the CI already exists"
 msgstr "因为CI已经存在，不能删除继承关系"
 
 #: api/lib/cmdb/resp_format.py:67
+msgid "The model is inherited and cannot be deleted"
+msgstr "该模型被继承, 不能删除"
+
+#: api/lib/cmdb/resp_format.py:68
+msgid "The model is referenced by attribute {} and cannot be deleted"
+msgstr "该模型被属性 {} 引用, 不能删除"
+
+#: api/lib/cmdb/resp_format.py:72
 msgid ""
 "The model cannot be deleted because the model is referenced by the "
 "relational view {}"
 msgstr "因为关系视图 {} 引用了该模型，不能删除模型"
 
-#: api/lib/cmdb/resp_format.py:69
+#: api/lib/cmdb/resp_format.py:74
 msgid "Model group {} does not exist"
 msgstr "模型分组 {} 不存在"
 
-#: api/lib/cmdb/resp_format.py:70
+#: api/lib/cmdb/resp_format.py:75
 msgid "Model group {} already exists"
 msgstr "模型分组 {} 已经存在"
 
-#: api/lib/cmdb/resp_format.py:71
+#: api/lib/cmdb/resp_format.py:76
 msgid "Model relationship {} does not exist"
 msgstr "模型关系 {} 不存在"
 
-#: api/lib/cmdb/resp_format.py:72
+#: api/lib/cmdb/resp_format.py:77
 msgid "Attribute group {} already exists"
 msgstr "属性分组 {} 已存在"
 
-#: api/lib/cmdb/resp_format.py:73
+#: api/lib/cmdb/resp_format.py:78
 msgid "Attribute group {} does not exist"
 msgstr "属性分组 {} 不存在"
 
-#: api/lib/cmdb/resp_format.py:75
+#: api/lib/cmdb/resp_format.py:80
 msgid "Attribute group <{0}> - attribute <{1}> does not exist"
 msgstr "属性组<{0}> - 属性<{1}> 不存在"
 
-#: api/lib/cmdb/resp_format.py:76
+#: api/lib/cmdb/resp_format.py:81
 msgid "The unique constraint already exists!"
 msgstr "唯一约束已经存在!"
 
-#: api/lib/cmdb/resp_format.py:78
+#: api/lib/cmdb/resp_format.py:83
 msgid "Uniquely constrained attributes cannot be JSON and multi-valued"
 msgstr "唯一约束的属性不能是 JSON 和 多值"
 
-#: api/lib/cmdb/resp_format.py:79
+#: api/lib/cmdb/resp_format.py:84
 msgid "Duplicated trigger"
 msgstr "重复的触发器"
 
-#: api/lib/cmdb/resp_format.py:80
+#: api/lib/cmdb/resp_format.py:85
 msgid "Trigger {} does not exist"
 msgstr "触发器 {} 不存在"
 
-#: api/lib/cmdb/resp_format.py:82
+#: api/lib/cmdb/resp_format.py:86
+msgid "Duplicated reconciliation rule"
+msgstr ""
+
+#: api/lib/cmdb/resp_format.py:87
+msgid "Reconciliation rule {} does not exist"
+msgstr "关系类型 {} 不存在"
+
+#: api/lib/cmdb/resp_format.py:89
 msgid "Operation record {} does not exist"
 msgstr "操作记录 {} 不存在"
 
-#: api/lib/cmdb/resp_format.py:83
+#: api/lib/cmdb/resp_format.py:90
 msgid "Unique identifier cannot be deleted"
 msgstr "不能删除唯一标识"
 
-#: api/lib/cmdb/resp_format.py:84
+#: api/lib/cmdb/resp_format.py:91
 msgid "Cannot delete default sorted attributes"
 msgstr "不能删除默认排序的属性"
 
-#: api/lib/cmdb/resp_format.py:86
+#: api/lib/cmdb/resp_format.py:93
 msgid "No node selected"
 msgstr "没有选择节点"
 
-#: api/lib/cmdb/resp_format.py:87
+#: api/lib/cmdb/resp_format.py:94
 msgid "This search option does not exist!"
 msgstr "该搜索选项不存在!"
 
-#: api/lib/cmdb/resp_format.py:88
+#: api/lib/cmdb/resp_format.py:95
 msgid "This search option has a duplicate name!"
 msgstr "该搜索选项命名重复!"
 
-#: api/lib/cmdb/resp_format.py:90
+#: api/lib/cmdb/resp_format.py:97
 msgid "Relationship type {} already exists"
 msgstr "关系类型 {} 已经存在"
 
-#: api/lib/cmdb/resp_format.py:91
+#: api/lib/cmdb/resp_format.py:98
 msgid "Relationship type {} does not exist"
 msgstr "关系类型 {} 不存在"
 
-#: api/lib/cmdb/resp_format.py:93
+#: api/lib/cmdb/resp_format.py:100
 msgid "Invalid attribute value: {}"
 msgstr "无效的属性值: {}"
 
-#: api/lib/cmdb/resp_format.py:94
+#: api/lib/cmdb/resp_format.py:101
 msgid "{} Invalid value: {}"
 msgstr "{} 无效的值: {}"
 
-#: api/lib/cmdb/resp_format.py:95
+#: api/lib/cmdb/resp_format.py:102
 msgid "{} is not in the predefined values"
 msgstr "{} 不在预定义值里"
 
-#: api/lib/cmdb/resp_format.py:97
+#: api/lib/cmdb/resp_format.py:104
 msgid "The value of attribute {} must be unique, {} already exists"
 msgstr "属性 {} 的值必须是唯一的, 当前值 {} 已存在"
 
-#: api/lib/cmdb/resp_format.py:98
+#: api/lib/cmdb/resp_format.py:105
 msgid "Attribute {} value must exist"
 msgstr "属性 {} 值必须存在"
 
-#: api/lib/cmdb/resp_format.py:99
+#: api/lib/cmdb/resp_format.py:106
 msgid "Out of range value, the maximum value is 2147483647"
 msgstr "超过最大值限制, 最大值是2147483647"
 
-#: api/lib/cmdb/resp_format.py:101
+#: api/lib/cmdb/resp_format.py:108
 msgid "Unknown error when adding or modifying attribute value: {}"
 msgstr "新增或者修改属性值未知错误: {}"
 
-#: api/lib/cmdb/resp_format.py:103
+#: api/lib/cmdb/resp_format.py:110
 msgid "Duplicate custom name"
 msgstr "订制名重复"
 
-#: api/lib/cmdb/resp_format.py:105
+#: api/lib/cmdb/resp_format.py:112
 msgid "Number of models exceeds limit: {}"
 msgstr "模型数超过限制: {}"
 
-#: api/lib/cmdb/resp_format.py:106
+#: api/lib/cmdb/resp_format.py:113
 msgid "The number of CIs exceeds the limit: {}"
 msgstr "CI数超过限制: {}"
 
-#: api/lib/cmdb/resp_format.py:108
+#: api/lib/cmdb/resp_format.py:115
 msgid "Auto-discovery rule: {} already exists!"
 msgstr "自动发现规则: {} 已经存在!"
 
-#: api/lib/cmdb/resp_format.py:109
+#: api/lib/cmdb/resp_format.py:116
 msgid "Auto-discovery rule: {} does not exist!"
 msgstr "自动发现规则: {} 不存在!"
 
-#: api/lib/cmdb/resp_format.py:111
+#: api/lib/cmdb/resp_format.py:118
 msgid "This auto-discovery rule is referenced by the model and cannot be deleted!"
 msgstr "该自动发现规则被模型引用, 不能删除!"
 
-#: api/lib/cmdb/resp_format.py:113
+#: api/lib/cmdb/resp_format.py:120
 msgid "The application of auto-discovery rules cannot be defined repeatedly!"
 msgstr "自动发现规则的应用不能重复定义!"
 
-#: api/lib/cmdb/resp_format.py:114
+#: api/lib/cmdb/resp_format.py:121
 msgid "The auto-discovery you want to modify: {} does not exist!"
 msgstr "您要修改的自动发现: {} 不存在!"
 
-#: api/lib/cmdb/resp_format.py:115
+#: api/lib/cmdb/resp_format.py:122
 msgid "Attribute does not include unique identifier: {}"
 msgstr "属性字段没有包括唯一标识: {}"
 
-#: api/lib/cmdb/resp_format.py:116
+#: api/lib/cmdb/resp_format.py:123
 msgid "The auto-discovery instance does not exist!"
 msgstr "自动发现的实例不存在!"
 
-#: api/lib/cmdb/resp_format.py:117
+#: api/lib/cmdb/resp_format.py:124
 msgid "The model is not associated with this auto-discovery!"
 msgstr "模型并未关联该自动发现!"
 
-#: api/lib/cmdb/resp_format.py:118
+#: api/lib/cmdb/resp_format.py:125
 msgid "Only the creator can modify the Secret!"
 msgstr "只有创建人才能修改Secret!"
 
-#: api/lib/cmdb/resp_format.py:120
+#: api/lib/cmdb/resp_format.py:127
 msgid "This rule already has auto-discovery instances and cannot be deleted!"
 msgstr "该规则已经有自动发现的实例, 不能被删除!"
 
-#: api/lib/cmdb/resp_format.py:122
+#: api/lib/cmdb/resp_format.py:129
 msgid "The default auto-discovery rule is already referenced by model {}!"
 msgstr "该默认的自动发现规则 已经被模型 {} 引用!"
 
-#: api/lib/cmdb/resp_format.py:124
+#: api/lib/cmdb/resp_format.py:131
 msgid "The unique_key method must return a non-empty string!"
 msgstr "unique_key方法必须返回非空字符串!"
 
-#: api/lib/cmdb/resp_format.py:125
+#: api/lib/cmdb/resp_format.py:132
 msgid "The attributes method must return a list"
 msgstr "attributes方法必须返回的是list"
 
-#: api/lib/cmdb/resp_format.py:127
+#: api/lib/cmdb/resp_format.py:134
 msgid "The list returned by the attributes method cannot be empty!"
 msgstr "attributes方法返回的list不能为空!"
 
-#: api/lib/cmdb/resp_format.py:129
+#: api/lib/cmdb/resp_format.py:136
 msgid "Only administrators can define execution targets as: all nodes!"
 msgstr "只有管理员才可以定义执行机器为: 所有节点!"
 
-#: api/lib/cmdb/resp_format.py:130
+#: api/lib/cmdb/resp_format.py:137
 msgid "Execute targets permission check failed: {}"
 msgstr "执行机器权限检查不通过: {}"
 
-#: api/lib/cmdb/resp_format.py:132
+#: api/lib/cmdb/resp_format.py:139
 msgid "CI filter authorization must be named!"
 msgstr "CI过滤授权 必须命名!"
 
-#: api/lib/cmdb/resp_format.py:133
+#: api/lib/cmdb/resp_format.py:140
 msgid "CI filter authorization is currently not supported or query"
 msgstr "CI过滤授权 暂时不支持 或 查询"
 
-#: api/lib/cmdb/resp_format.py:136
+#: api/lib/cmdb/resp_format.py:143
 msgid "You do not have permission to operate attribute {}!"
 msgstr "您没有属性 {} 的操作权限!"
 
-#: api/lib/cmdb/resp_format.py:137
+#: api/lib/cmdb/resp_format.py:144
 msgid "You do not have permission to operate this CI!"
 msgstr "您没有该CI的操作权限!"
 
-#: api/lib/cmdb/resp_format.py:139
+#: api/lib/cmdb/resp_format.py:146
 msgid "Failed to save password: {}"
 msgstr "保存密码失败: {}"
 
-#: api/lib/cmdb/resp_format.py:140
+#: api/lib/cmdb/resp_format.py:147
 msgid "Failed to get password: {}"
 msgstr "获取密码失败: {}"
 
+#: api/lib/cmdb/resp_format.py:149
+msgid "Scheduling time format error"
+msgstr "{}格式错误，应该为：%Y-%m-%d %H:%M:%S"
+
+#: api/lib/cmdb/resp_format.py:150
+msgid "CMDB data reconciliation results"
+msgstr ""
+
+#: api/lib/cmdb/resp_format.py:151
+msgid "Number of {} illegal: {}"
+msgstr ""
+
+#: api/lib/cmdb/resp_format.py:153
+msgid "Topology view {} already exists"
+msgstr "拓扑视图 {} 已经存在"
+
+#: api/lib/cmdb/resp_format.py:154
+msgid "Topology group {} already exists"
+msgstr "拓扑视图分组 {} 已经存在"
+
+#: api/lib/cmdb/resp_format.py:156
+msgid "The group cannot be deleted because the topology view already exists"
+msgstr "因为该分组下定义了拓扑视图，不能删除"
+
+#: api/lib/cmdb/resp_format.py:158
+msgid "Both the source model and the target model must be selected"
+msgstr "源模型和目标模型不能为空!"
+
 #: api/lib/common_setting/resp_format.py:8
 msgid "Company info already existed"
 msgstr "公司信息已存在,无法创建!"
@@ -700,6 +752,10 @@ msgstr "LDAP测试用户名必填"
 msgid "Company wide"
 msgstr "全公司"
 
+#: api/lib/common_setting/resp_format.py:84
+msgid "No permission to access resource {}, perm {} "
+msgstr "您没有资源: {} 的 {} 权限"
+
 #: api/lib/perm/acl/resp_format.py:9
 msgid "login successful"
 msgstr "登录成功"

cmdb-api/api/views/acl/login.py

@@ -1,7 +1,6 @@
 # -*- coding:utf-8 -*-
 
 import datetime
-
 import jwt
 import six
 from flask import abort
@@ -17,10 +16,12 @@ from api.lib.decorator import args_required
 from api.lib.decorator import args_validate
 from api.lib.perm.acl.acl import ACLManager
 from api.lib.perm.acl.audit import AuditCRUD
+from api.lib.perm.acl.cache import AppCache
 from api.lib.perm.acl.cache import RoleCache
 from api.lib.perm.acl.cache import User
 from api.lib.perm.acl.cache import UserCache
 from api.lib.perm.acl.resp_format import ErrFormat
+from api.lib.perm.acl.role import RoleRelationCRUD
 from api.lib.perm.auth import auth_abandoned
 from api.lib.perm.auth import auth_with_app_token
 from api.models.acl import Role
@@ -124,10 +125,17 @@ class AuthWithKeyView(APIView):
         if not user.get('username'):
             user['username'] = user.get('name')
 
-        return self.jsonify(user=user,
-                            authenticated=authenticated,
-                            rid=role and role.id,
-                            can_proxy=can_proxy)
+        result = dict(user=user,
+                      authenticated=authenticated,
+                      rid=role and role.id,
+                      can_proxy=can_proxy)
+
+        if request.values.get('need_parentRoles') in current_app.config.get('BOOL_TRUE'):
+            app_id = AppCache.get(request.values.get('app_id'))
+            parent_ids = RoleRelationCRUD.recursive_parent_ids(role and role.id, app_id and app_id.id)
+            result['user']['parentRoles'] = [RoleCache.get(rid).name for rid in set(parent_ids) if RoleCache.get(rid)]
+
+        return self.jsonify(result)
 
 
 class AuthWithTokenView(APIView):
@@ -184,6 +192,8 @@ class LogoutView(APIView):
     def post(self):
         logout_user()
 
-        AuditCRUD.add_login_log(None, None, None, _id=session.get('LOGIN_ID'), logout_at=datetime.datetime.now())
+        AuditCRUD.add_login_log(None, None, None,
+                                _id=session.get('LOGIN_ID') or request.values.get('LOGIN_ID'),
+                                logout_at=datetime.datetime.now())
 
         self.jsonify(code=200)

cmdb-api/api/views/acl/user.py

@@ -11,6 +11,7 @@ from flask_login import current_user
 from api.lib.decorator import args_required
 from api.lib.decorator import args_validate
 from api.lib.perm.acl.acl import ACLManager
+from api.lib.perm.acl.acl import AuditCRUD
 from api.lib.perm.acl.acl import role_required
 from api.lib.perm.acl.cache import AppCache
 from api.lib.perm.acl.cache import UserCache
@@ -48,6 +49,13 @@ class GetUserInfoView(APIView):
                       role=dict(permissions=user_info.get('parents')),
                       avatar=user_info.get('avatar'))
 
+        if request.values.get('channel'):
+            _id = AuditCRUD.add_login_log(name, True, ErrFormat.login_succeed,
+                                          ip=request.values.get('ip'),
+                                          browser=request.values.get('browser'))
+            session['LOGIN_ID'] = _id
+            result['LOGIN_ID'] = _id
+
         current_app.logger.info("get user info for3: {}".format(result))
         return self.jsonify(result=result)
 

cmdb-api/api/views/cmdb/auto_discovery.py

@@ -1,24 +1,32 @@
 # -*- coding:utf-8 -*-
-
+import copy
 import json
-from io import BytesIO
-
+import uuid
 from flask import abort
 from flask import current_app
 from flask import request
 from flask_login import current_user
+from io import BytesIO
 
+from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryAccountCRUD
 from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCICRUD
 from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeCRUD
+from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeRelationCRUD
+from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryComponentsManager
+from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCounterCRUD
+from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryExecHistoryCRUD
 from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryHTTPManager
 from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryRuleCRUD
+from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryRuleSyncHistoryCRUD
 from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoverySNMPManager
-from api.lib.cmdb.auto_discovery.const import DEFAULT_HTTP
+from api.lib.cmdb.auto_discovery.const import DEFAULT_INNER
+from api.lib.cmdb.auto_discovery.const import PRIVILEGED_USERS
+from api.lib.cmdb.cache import AttributeCache
 from api.lib.cmdb.const import PermEnum
 from api.lib.cmdb.const import ResourceTypeEnum
 from api.lib.cmdb.resp_format import ErrFormat
 from api.lib.cmdb.search import SearchError
-from api.lib.cmdb.search.ci import search
+from api.lib.cmdb.search.ci import search as ci_search
 from api.lib.decorator import args_required
 from api.lib.decorator import args_validate
 from api.lib.perm.acl.acl import has_perm_from_args
@@ -37,14 +45,19 @@ class AutoDiscoveryRuleView(APIView):
 
         rebuild = False
         exists = {i['name'] for i in res}
-        for i in DEFAULT_HTTP:
+        for i in copy.deepcopy(DEFAULT_INNER):
             if i['name'] not in exists:
+                i.pop('en', None)
                 AutoDiscoveryRuleCRUD().add(**i)
                 rebuild = True
 
         if rebuild:
             _, res = AutoDiscoveryRuleCRUD.search(page=1, page_size=100000, **request.values)
 
+        for i in res:
+            if i['type'] == 'http':
+                i['resources'] = AutoDiscoveryHTTPManager().get_resources(i['name'])
+
         return self.jsonify(res)
 
     @args_required("name", value_required=True)
@@ -98,24 +111,39 @@ class AutoDiscoveryRuleTemplateFileView(APIView):
 
 
 class AutoDiscoveryRuleHTTPView(APIView):
-    url_prefix = ("/adr/http/<string:name>/categories", "/adr/http/<string:name>/attributes",
-                  "/adr/snmp/<string:name>/attributes")
+    url_prefix = ("/adr/http/<string:name>/categories",
+                  "/adr/http/<string:name>/attributes",
+                  "/adr/http/<string:name>/mapping",
+                  "/adr/snmp/<string:name>/attributes",
+                  "/adr/components/<string:name>/attributes",)
 
     def get(self, name):
         if "snmp" in request.url:
             return self.jsonify(AutoDiscoverySNMPManager.get_attributes())
 
+        if "components" in request.url:
+            return self.jsonify(AutoDiscoveryComponentsManager.get_attributes(name))
+
         if "attributes" in request.url:
-            category = request.values.get('category')
-            return self.jsonify(AutoDiscoveryHTTPManager.get_attributes(name, category))
+            resource = request.values.get('resource')
+            return self.jsonify(AutoDiscoveryHTTPManager.get_attributes(name, resource))
+
+        if "mapping" in request.url:
+            resource = request.values.get('resource')
+            return self.jsonify(AutoDiscoveryHTTPManager.get_mapping(name, resource))
 
         return self.jsonify(AutoDiscoveryHTTPManager.get_categories(name))
 
 
 class AutoDiscoveryCITypeView(APIView):
-    url_prefix = ("/adt/ci_types/<int:type_id>", "/adt/<int:adt_id>")
+    url_prefix = ("/adt/ci_types/<int:type_id>",
+                  "/adt/ci_types/<int:type_id>/attributes",
+                  "/adt/<int:adt_id>")
 
     def get(self, type_id):
+        if "attributes" in request.url:
+            return self.jsonify(AutoDiscoveryCITypeCRUD.get_ad_attributes(type_id))
+
         _, res = AutoDiscoveryCITypeCRUD.search(page=1, page_size=100000, type_id=type_id, **request.values)
         for i in res:
             if isinstance(i.get("extra_option"), dict) and i['extra_option'].get('secret'):
@@ -123,6 +151,11 @@ class AutoDiscoveryCITypeView(APIView):
                     i['extra_option'].pop('secret', None)
                 else:
                     i['extra_option']['secret'] = AESCrypto.decrypt(i['extra_option']['secret'])
+            if isinstance(i.get("extra_option"), dict) and i['extra_option'].get('password'):
+                if not (current_user.username == "cmdb_agent" or current_user.uid == i['uid']):
+                    i['extra_option'].pop('password', None)
+                else:
+                    i['extra_option']['password'] = AESCrypto.decrypt(i['extra_option']['password'])
 
         return self.jsonify(res)
 
@@ -146,6 +179,27 @@ class AutoDiscoveryCITypeView(APIView):
         return self.jsonify(adt_id=adt_id)
 
 
+class AutoDiscoveryCITypeRelationView(APIView):
+    url_prefix = ("/adt/ci_types/<int:type_id>/relations", "/adt/relations/<int:_id>")
+
+    def get(self, type_id):
+        _, res = AutoDiscoveryCITypeRelationCRUD.search(page=1, page_size=100000, ad_type_id=type_id, **request.values)
+
+        return self.jsonify(res)
+
+    @args_required("relations")
+    def post(self, type_id):
+        return self.jsonify(AutoDiscoveryCITypeRelationCRUD().upsert(type_id, request.values['relations']))
+
+    def put(self):
+        return self.post()
+
+    def delete(self, _id):
+        AutoDiscoveryCITypeRelationCRUD().delete(_id)
+
+        return self.jsonify(id=_id)
+
+
 class AutoDiscoveryCIView(APIView):
     url_prefix = ("/adc", "/adc/<int:adc_id>", "/adc/ci_types/<int:type_id>/attributes", "/adc/ci_types")
 
@@ -171,6 +225,7 @@ class AutoDiscoveryCIView(APIView):
     @args_required("type_id")
     @args_required("adt_id")
     @args_required("instance")
+    @args_required("unique_value")
     def post(self):
         request.values.pop("_key", None)
         request.values.pop("_secret", None)
@@ -213,24 +268,127 @@ class AutoDiscoveryRuleSyncView(APIView):
     url_prefix = ("/adt/sync",)
 
     def get(self):
-        if current_user.username not in ("cmdb_agent", "worker", "admin"):
+        if current_user.username not in PRIVILEGED_USERS:
             return abort(403)
 
         oneagent_name = request.values.get('oneagent_name')
         oneagent_id = request.values.get('oneagent_id')
         last_update_at = request.values.get('last_update_at')
 
-        query = "{},oneagent_id:{}".format(oneagent_name, oneagent_id)
-        current_app.logger.info(query)
-        s = search(query)
-        try:
-            response, _, _, _, _, _ = s.search()
-        except SearchError as e:
-            import traceback
-            current_app.logger.error(traceback.format_exc())
-            return abort(400, str(e))
+        response = []
+        if AttributeCache.get('oneagent_id'):
+            query = "oneagent_id:{}".format(oneagent_id)
+            s = ci_search(query)
+            try:
+                response, _, _, _, _, _ = s.search()
+            except SearchError as e:
+                import traceback
+                current_app.logger.error(traceback.format_exc())
+                return abort(400, str(e))
 
-        ci_id = response and response[0]["_id"]
-        rules, last_update_at = AutoDiscoveryCITypeCRUD.get(ci_id, oneagent_id, last_update_at)
+        for res in response:
+            if res.get('{}_name'.format(res['ci_type'])) == oneagent_name or oneagent_name == res.get('oneagent_name'):
+                ci_id = res["_id"]
+                rules, last_update_at = AutoDiscoveryCITypeCRUD.get(ci_id, oneagent_id, oneagent_name, last_update_at)
+
+                return self.jsonify(rules=rules, last_update_at=last_update_at)
+
+        rules, last_update_at = AutoDiscoveryCITypeCRUD.get(None, oneagent_id, oneagent_name, last_update_at)
 
         return self.jsonify(rules=rules, last_update_at=last_update_at)
+
+
+class AutoDiscoveryRuleSyncHistoryView(APIView):
+    url_prefix = ("/adt/<int:adt_id>/sync/histories",)
+
+    def get(self, adt_id):
+        page = get_page(request.values.pop('page', 1))
+        page_size = get_page_size(request.values.pop('page_size', None))
+        numfound, res = AutoDiscoveryRuleSyncHistoryCRUD.search(page=page,
+                                                                page_size=page_size,
+                                                                adt_id=adt_id,
+                                                                **request.values)
+
+        return self.jsonify(page=page,
+                            page_size=page_size,
+                            numfound=numfound,
+                            total=len(res),
+                            result=res)
+
+
+class AutoDiscoveryTestView(APIView):
+    url_prefix = ("/adt/<int:adt_id>/test", "/adt/test/<string:exec_id>/result")
+
+    def get(self, exec_id):
+        return self.jsonify(stdout="1\n2\n3", exec_id=exec_id)
+
+    def post(self, adt_id):
+        return self.jsonify(exec_id=uuid.uuid4().hex)
+
+
+class AutoDiscoveryExecHistoryView(APIView):
+    url_prefix = ("/adc/exec/histories",)
+
+    @args_required('type_id')
+    def get(self):
+        page = get_page(request.values.pop('page', 1))
+        page_size = get_page_size(request.values.pop('page_size', None))
+        last_size = request.values.pop('last_size', None)
+        if last_size and last_size.isdigit():
+            last_size = int(last_size)
+        numfound, res = AutoDiscoveryExecHistoryCRUD.search(page=page,
+                                                            page_size=page_size,
+                                                            last_size=last_size,
+                                                            **request.values)
+
+        return self.jsonify(page=page,
+                            page_size=page_size,
+                            numfound=numfound,
+                            total=len(res),
+                            result=res)
+
+    @args_required('type_id')
+    @args_required('stdout')
+    def post(self):
+        AutoDiscoveryExecHistoryCRUD().add(type_id=request.values.get('type_id'),
+                                           stdout=request.values.get('stdout'))
+
+        return self.jsonify(code=200)
+
+
+class AutoDiscoveryCounterView(APIView):
+    url_prefix = ("/adc/counter",)
+
+    @args_required('type_id')
+    def get(self):
+        type_id = request.values.get('type_id')
+
+        return self.jsonify(AutoDiscoveryCounterCRUD().get(type_id))
+
+
+class AutoDiscoveryAccountView(APIView):
+    url_prefix = ("/adr/accounts", "/adr/accounts/<int:account_id>")
+
+    @args_required('adr_id')
+    def get(self):
+        adr_id = request.values.get('adr_id')
+
+        return self.jsonify(AutoDiscoveryAccountCRUD().get(adr_id))
+
+    @args_required('adr_id')
+    @args_required('accounts', value_required=False)
+    def post(self):
+        AutoDiscoveryAccountCRUD().upsert(**request.values)
+
+        return self.jsonify(code=200)
+
+    @args_required('config')
+    def put(self, account_id):
+        res = AutoDiscoveryAccountCRUD().update(account_id, **request.values)
+
+        return self.jsonify(res.to_dict())
+
+    def delete(self, account_id):
+        AutoDiscoveryAccountCRUD().delete(account_id)
+
+        return self.jsonify(account_id=account_id)

cmdb-api/api/views/cmdb/ci.py

@@ -15,7 +15,8 @@ from api.lib.cmdb.const import ResourceTypeEnum, PermEnum
 from api.lib.cmdb.const import RetKey
 from api.lib.cmdb.perms import has_perm_for_ci
 from api.lib.cmdb.search import SearchError
-from api.lib.cmdb.search.ci import search
+from api.lib.cmdb.search.ci import search as ci_search
+from api.lib.decorator import args_required
 from api.lib.perm.acl.acl import has_perm_from_args
 from api.lib.utils import get_page
 from api.lib.utils import get_page_size
@@ -159,7 +160,7 @@ class CISearchView(APIView):
         use_id_filter = request.values.get("use_id_filter", False) in current_app.config.get('BOOL_TRUE')
 
         start = time.time()
-        s = search(query, fl, facet, page, ret_key, count, sort, excludes, use_id_filter=use_id_filter)
+        s = ci_search(query, fl, facet, page, ret_key, count, sort, excludes, use_id_filter=use_id_filter)
         try:
             response, counter, total, page, numfound, facet = s.search()
         except SearchError as e:
@@ -254,3 +255,24 @@ class CIPasswordView(APIView):
 
     def post(self, ci_id, attr_id):
         return self.get(ci_id, attr_id)
+
+
+class CIBaselineView(APIView):
+    url_prefix = ("/ci/baseline", "/ci/<int:ci_id>/baseline/rollback")
+
+    @args_required("before_date")
+    def get(self):
+        ci_ids = handle_arg_list(request.values.get('ci_ids'))
+        before_date = request.values.get('before_date')
+
+        return self.jsonify(CIManager().baseline(list(map(int, ci_ids)), before_date))
+
+    @args_required("before_date")
+    @has_perm_for_ci("ci_id", ResourceTypeEnum.CI, PermEnum.UPDATE, CIManager.get_type)
+    def post(self, ci_id):
+        if 'rollback' in request.url:
+            before_date = request.values.get('before_date')
+
+            return self.jsonify(**CIManager().rollback(ci_id, before_date))
+
+        return self.get(ci_id)

cmdb-api/api/views/cmdb/ci_relation.py

@@ -2,7 +2,6 @@
 
 
 import time
-
 from flask import abort
 from flask import current_app
 from flask import request
@@ -65,6 +64,42 @@ class CIRelationSearchView(APIView):
                             result=response)
 
 
+class CIRelationSearchPathView(APIView):
+    url_prefix = ("/ci_relations/path/s", "/ci_relations/path/search")
+
+    @args_required("source", "target", "path")
+    def post(self):
+        """@params: page: page number
+                    page_size | count: page size
+                    source: source CIType, e.g. {type_id: 1, q: `search expr`}
+                    target: target CIType, e.g. {type_ids: [2], q: `search expr`}
+                    path: Path from the Source CIType to the Target CIType, e.g. [1, ..., 2]
+        """
+
+        page = get_page(request.values.get("page", 1))
+        count = get_page_size(request.values.get("count") or request.values.get("page_size"))
+
+        source = request.values.get("source")
+        target = request.values.get("target")
+        path = request.values.get("path")
+
+        s = Search(page=page, count=count)
+        try:
+            (response, counter, total, page, numfound, id2ci,
+             relation_types, type2show_key) = s.search_by_path(source, target, path)
+        except SearchError as e:
+            return abort(400, str(e))
+
+        return self.jsonify(numfound=numfound,
+                            total=total,
+                            page=page,
+                            counter=counter,
+                            paths=response,
+                            id2ci=id2ci,
+                            relation_types=relation_types,
+                            type2show_key=type2show_key)
+
+
 class CIRelationStatisticsView(APIView):
     url_prefix = "/ci_relations/statistics"
 

cmdb-api/api/views/cmdb/ci_type.py

@@ -7,7 +7,6 @@ from io import BytesIO
 from flask import abort
 from flask import current_app
 from flask import request
-from flask import session
 
 from api.lib.cmdb.cache import AttributeCache
 from api.lib.cmdb.cache import CITypeCache
@@ -19,10 +18,12 @@ from api.lib.cmdb.ci_type import CITypeManager
 from api.lib.cmdb.ci_type import CITypeTemplateManager
 from api.lib.cmdb.ci_type import CITypeTriggerManager
 from api.lib.cmdb.ci_type import CITypeUniqueConstraintManager
-from api.lib.cmdb.const import PermEnum, ResourceTypeEnum, RoleEnum
+from api.lib.cmdb.const import PermEnum, ResourceTypeEnum
 from api.lib.cmdb.perms import CIFilterPermsCRUD
 from api.lib.cmdb.preference import PreferenceManager
 from api.lib.cmdb.resp_format import ErrFormat
+from api.lib.common_setting.decorator import perms_role_required
+from api.lib.common_setting.role_perm_base import CMDBApp
 from api.lib.decorator import args_required
 from api.lib.decorator import args_validate
 from api.lib.perm.acl.acl import ACLManager
@@ -36,6 +37,8 @@ from api.lib.perm.auth import auth_with_app_token
 from api.lib.utils import handle_arg_list
 from api.resource import APIView
 
+app_cli = CMDBApp()
+
 
 class CITypeView(APIView):
     url_prefix = ("/ci_types", "/ci_types/<int:type_id>", "/ci_types/<string:type_name>",
@@ -45,12 +48,21 @@ class CITypeView(APIView):
         if request.url.endswith("icons"):
             return self.jsonify(CITypeManager().get_icons())
 
-        q = request.args.get("type_name")
+        q = request.values.get("type_name")
+        type_ids = handle_arg_list(request.values.get("type_ids"))
+        type_ids = type_ids or (type_id and [type_id])
+        if type_ids:
+            ci_types = []
+            for _type_id in type_ids:
+                ci_type = CITypeCache.get(_type_id)
+                if ci_type is None:
+                    return abort(404, ErrFormat.ci_type_not_found)
 
-        if type_id is not None:
-            ci_type = CITypeCache.get(type_id).to_dict()
-            ci_type['parent_ids'] = CITypeInheritanceManager.get_parents(type_id)
-            ci_types = [ci_type]
+                ci_type = ci_type.to_dict()
+                ci_type['parent_ids'] = CITypeInheritanceManager.get_parents(_type_id)
+                ci_type['show_name'] = ci_type.get('show_id') and AttributeCache.get(ci_type['show_id']).name
+                ci_type['unique_name'] = ci_type['unique_id'] and AttributeCache.get(ci_type['unique_id']).name
+                ci_types.append(ci_type)
         elif type_name is not None:
             ci_type = CITypeCache.get(type_name).to_dict()
             ci_type['parent_ids'] = CITypeInheritanceManager.get_parents(ci_type['id'])
@@ -116,7 +128,6 @@ class CITypeInheritanceView(APIView):
 class CITypeGroupView(APIView):
     url_prefix = ("/ci_types/groups",
                   "/ci_types/groups/config",
-                  "/ci_types/groups/order",
                   "/ci_types/groups/<int:gid>")
 
     def get(self):
@@ -125,7 +136,8 @@ class CITypeGroupView(APIView):
 
         return self.jsonify(CITypeGroupManager.get(need_other, config_required))
 
-    @role_required(RoleEnum.CONFIG)
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Model_Configuration,
+                         app_cli.op.create_CIType_group, app_cli.admin_name)
     @args_required("name")
     @args_validate(CITypeGroupManager.cls)
     def post(self):
@@ -136,15 +148,6 @@ class CITypeGroupView(APIView):
 
     @args_validate(CITypeGroupManager.cls)
     def put(self, gid=None):
-        if "/order" in request.url:
-            if RoleEnum.CONFIG not in session.get("acl", {}).get("parentRoles", []) and not is_app_admin("cmdb"):
-                return abort(403, ErrFormat.role_required.format(RoleEnum.CONFIG))
-
-            group_ids = request.values.get('group_ids')
-            CITypeGroupManager.order(group_ids)
-
-            return self.jsonify(group_ids=group_ids)
-
         name = request.values.get('name') or abort(400, ErrFormat.argument_value_required.format("name"))
         type_ids = request.values.get('type_ids')
 
@@ -152,7 +155,8 @@ class CITypeGroupView(APIView):
 
         return self.jsonify(gid=gid)
 
-    @role_required(RoleEnum.CONFIG)
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Model_Configuration,
+                         app_cli.op.delete_CIType_group, app_cli.admin_name)
     def delete(self, gid):
         type_ids = request.values.get("type_ids")
         CITypeGroupManager.delete(gid, type_ids)
@@ -160,6 +164,18 @@ class CITypeGroupView(APIView):
         return self.jsonify(gid=gid)
 
 
+class CITypeGroupOrderView(APIView):
+    url_prefix = "/ci_types/groups/order"
+
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Model_Configuration,
+                         app_cli.op.update_CIType_group, app_cli.admin_name)
+    def put(self):
+        group_ids = request.values.get('group_ids')
+        CITypeGroupManager.order(group_ids)
+
+        return self.jsonify(group_ids=group_ids)
+
+
 class CITypeQueryView(APIView):
     url_prefix = "/ci_types/query"
 
@@ -350,16 +366,16 @@ class CITypeAttributeGroupView(APIView):
 
 
 class CITypeTemplateView(APIView):
-    url_prefix = ("/ci_types/template/import", "/ci_types/template/export", "/ci_types/<int:type_id>/template/export")
+    url_prefix = ("/ci_types/template/import", "/ci_types/template/export")
 
-    @role_required(RoleEnum.CONFIG)
-    def get(self, type_id=None):  # export
-        if type_id is not None:
-            return self.jsonify(dict(ci_type_template=CITypeTemplateManager.export_template_by_type(type_id)))
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Model_Configuration,
+                         app_cli.op.download_CIType, app_cli.admin_name)
+    def get(self):  # export
+        type_ids = list(map(int, handle_arg_list(request.values.get('type_ids')))) or None
+        return self.jsonify(dict(ci_type_template=CITypeTemplateManager.export_template(type_ids=type_ids)))
 
-        return self.jsonify(dict(ci_type_template=CITypeTemplateManager.export_template()))
-
-    @role_required(RoleEnum.CONFIG)
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Model_Configuration,
+                         app_cli.op.download_CIType, app_cli.admin_name)
     def post(self):  # import
         tpt = request.values.get('ci_type_template') or {}
 
@@ -379,7 +395,8 @@ class CITypeCanDefineComputed(APIView):
 class CITypeTemplateFileView(APIView):
     url_prefix = ("/ci_types/template/import/file", "/ci_types/template/export/file")
 
-    @role_required(RoleEnum.CONFIG)
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Model_Configuration,
+                         app_cli.op.download_CIType, app_cli.admin_name)
     def get(self):  # export
         tpt_json = CITypeTemplateManager.export_template()
         tpt_json = dict(ci_type_template=tpt_json)
@@ -394,7 +411,8 @@ class CITypeTemplateFileView(APIView):
                               mimetype='application/json',
                               max_age=0)
 
-    @role_required(RoleEnum.CONFIG)
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Model_Configuration,
+                         app_cli.op.download_CIType, app_cli.admin_name)
     def post(self):  # import
         f = request.files.get('file')
 

cmdb-api/api/views/cmdb/ci_type_relation.py

@@ -8,16 +8,19 @@ from api.lib.cmdb.ci_type import CITypeManager
 from api.lib.cmdb.ci_type import CITypeRelationManager
 from api.lib.cmdb.const import PermEnum
 from api.lib.cmdb.const import ResourceTypeEnum
-from api.lib.cmdb.const import RoleEnum
 from api.lib.cmdb.preference import PreferenceManager
 from api.lib.cmdb.resp_format import ErrFormat
+from api.lib.common_setting.decorator import perms_role_required
+from api.lib.common_setting.role_perm_base import CMDBApp
 from api.lib.decorator import args_required
 from api.lib.perm.acl.acl import ACLManager
 from api.lib.perm.acl.acl import has_perm_from_args
 from api.lib.perm.acl.acl import is_app_admin
-from api.lib.perm.acl.acl import role_required
+from api.lib.utils import handle_arg_list
 from api.resource import APIView
 
+app_cli = CMDBApp()
+
 
 class GetChildrenView(APIView):
     url_prefix = ("/ci_type_relations/<int:parent_id>/children",
@@ -38,10 +41,24 @@ class GetParentsView(APIView):
         return self.jsonify(parents=CITypeRelationManager.get_parents(child_id))
 
 
+class CITypeRelationPathView(APIView):
+    url_prefix = ("/ci_type_relations/path",)
+
+    @args_required("source_type_id", "target_type_ids")
+    def get(self):
+        source_type_id = request.values.get("source_type_id")
+        target_type_ids = handle_arg_list(request.values.get("target_type_ids"))
+
+        paths = CITypeRelationManager.find_path(source_type_id, target_type_ids)
+
+        return self.jsonify(paths=paths)
+
+
 class CITypeRelationView(APIView):
     url_prefix = ("/ci_type_relations", "/ci_type_relations/<int:parent_id>/<int:child_id>")
 
-    @role_required(RoleEnum.CONFIG)
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Service_Tree_Definition,
+                         app_cli.op.read, app_cli.admin_name)
     def get(self):
         res, type2attributes = CITypeRelationManager.get()
 
@@ -52,10 +69,10 @@ class CITypeRelationView(APIView):
     def post(self, parent_id, child_id):
         relation_type_id = request.values.get("relation_type_id")
         constraint = request.values.get("constraint")
-        parent_attr_id = request.values.get("parent_attr_id")
-        child_attr_id = request.values.get("child_attr_id")
+        parent_attr_ids = request.values.get("parent_attr_ids")
+        child_attr_ids = request.values.get("child_attr_ids")
         ctr_id = CITypeRelationManager.add(parent_id, child_id, relation_type_id, constraint,
-                                           parent_attr_id, child_attr_id)
+                                           parent_attr_ids, child_attr_ids)
 
         return self.jsonify(ctr_id=ctr_id)
 
@@ -69,7 +86,8 @@ class CITypeRelationView(APIView):
 class CITypeRelationDelete2View(APIView):
     url_prefix = "/ci_type_relations/<int:ctr_id>"
 
-    @role_required(RoleEnum.CONFIG)
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Model_Relationships,
+                         app_cli.op.read, app_cli.admin_name)
     def delete(self, ctr_id):
         CITypeRelationManager.delete(ctr_id)
 

cmdb-api/api/views/cmdb/custom_dashboard.py

@@ -3,14 +3,16 @@
 
 from flask import request
 
-from api.lib.cmdb.const import RoleEnum
 from api.lib.cmdb.custom_dashboard import CustomDashboardManager
 from api.lib.cmdb.custom_dashboard import SystemConfigManager
+from api.lib.common_setting.decorator import perms_role_required
+from api.lib.common_setting.role_perm_base import CMDBApp
 from api.lib.decorator import args_required
 from api.lib.decorator import args_validate
-from api.lib.perm.acl.acl import role_required
 from api.resource import APIView
 
+app_cli = CMDBApp()
+
 
 class CustomDashboardApiView(APIView):
     url_prefix = ("/custom_dashboard", "/custom_dashboard/<int:_id>", "/custom_dashboard/batch",
@@ -19,7 +21,8 @@ class CustomDashboardApiView(APIView):
     def get(self):
         return self.jsonify(CustomDashboardManager.get())
 
-    @role_required(RoleEnum.CONFIG)
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Customized_Dashboard,
+                         app_cli.op.read, app_cli.admin_name)
     @args_validate(CustomDashboardManager.cls)
     def post(self):
         if request.url.endswith("/preview"):
@@ -32,7 +35,8 @@ class CustomDashboardApiView(APIView):
 
         return self.jsonify(res)
 
-    @role_required(RoleEnum.CONFIG)
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Customized_Dashboard,
+                         app_cli.op.read, app_cli.admin_name)
     @args_validate(CustomDashboardManager.cls)
     def put(self, _id=None):
         if _id is not None:
@@ -47,7 +51,8 @@ class CustomDashboardApiView(APIView):
 
         return self.jsonify(id2options=request.values.get('id2options'))
 
-    @role_required(RoleEnum.CONFIG)
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Customized_Dashboard,
+                         app_cli.op.read, app_cli.admin_name)
     def delete(self, _id):
         CustomDashboardManager.delete(_id)
 
@@ -57,12 +62,14 @@ class CustomDashboardApiView(APIView):
 class SystemConfigApiView(APIView):
     url_prefix = ("/system_config",)
 
-    @role_required(RoleEnum.CONFIG)
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Service_Tree_Definition,
+                         app_cli.op.read, app_cli.admin_name)
     @args_required("name", value_required=True)
     def get(self):
         return self.jsonify(SystemConfigManager.get(request.values['name']))
 
-    @role_required(RoleEnum.CONFIG)
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Service_Tree_Definition,
+                         app_cli.op.read, app_cli.admin_name)
     @args_validate(SystemConfigManager.cls)
     @args_required("name", value_required=True)
     @args_required("option", value_required=True)
@@ -74,7 +81,8 @@ class SystemConfigApiView(APIView):
     def put(self, _id=None):
         return self.post()
 
-    @role_required(RoleEnum.CONFIG)
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Service_Tree_Definition,
+                         app_cli.op.read, app_cli.admin_name)
     @args_required("name")
     def delete(self):
         CustomDashboardManager.delete(request.values['name'])

cmdb-api/api/views/cmdb/history.py

@@ -5,28 +5,29 @@ import datetime
 
 from flask import abort
 from flask import request
-from flask import session
 
 from api.lib.cmdb.ci import CIManager
 from api.lib.cmdb.const import PermEnum
 from api.lib.cmdb.const import ResourceTypeEnum
-from api.lib.cmdb.const import RoleEnum
 from api.lib.cmdb.history import AttributeHistoryManger
 from api.lib.cmdb.history import CITriggerHistoryManager
 from api.lib.cmdb.history import CITypeHistoryManager
 from api.lib.cmdb.resp_format import ErrFormat
+from api.lib.common_setting.decorator import perms_role_required
+from api.lib.common_setting.role_perm_base import CMDBApp
 from api.lib.perm.acl.acl import has_perm_from_args
-from api.lib.perm.acl.acl import is_app_admin
-from api.lib.perm.acl.acl import role_required
 from api.lib.utils import get_page
 from api.lib.utils import get_page_size
 from api.resource import APIView
 
+app_cli = CMDBApp()
+
 
 class RecordView(APIView):
     url_prefix = ("/history/records/attribute", "/history/records/relation")
 
-    @role_required(RoleEnum.CONFIG)
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Operation_Audit,
+                         app_cli.op.read, app_cli.admin_name)
     def get(self):
         page = get_page(request.values.get("page", 1))
         page_size = get_page_size(request.values.get("page_size"))
@@ -80,18 +81,21 @@ class CIHistoryView(APIView):
 
 
 class CITriggerHistoryView(APIView):
-    url_prefix = ("/history/ci_triggers/<int:ci_id>", "/history/ci_triggers")
+    url_prefix = ("/history/ci_triggers/<int:ci_id>",)
 
     @has_perm_from_args("ci_id", ResourceTypeEnum.CI, PermEnum.READ, CIManager.get_type_name)
-    def get(self, ci_id=None):
-        if ci_id is not None:
-            result = CITriggerHistoryManager.get_by_ci_id(ci_id)
+    def get(self, ci_id):
+        result = CITriggerHistoryManager.get_by_ci_id(ci_id)
 
-            return self.jsonify(result)
+        return self.jsonify(result)
 
-        if RoleEnum.CONFIG not in session.get("acl", {}).get("parentRoles", []) and not is_app_admin("cmdb"):
-            return abort(403, ErrFormat.role_required.format(RoleEnum.CONFIG))
 
+class CIsTriggerHistoryView(APIView):
+    url_prefix = ("/history/ci_triggers",)
+
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Operation_Audit,
+                         app_cli.op.read, app_cli.admin_name)
+    def get(self):
         type_id = request.values.get("type_id")
         trigger_id = request.values.get("trigger_id")
         operate_type = request.values.get("operate_type")
@@ -115,7 +119,8 @@ class CITriggerHistoryView(APIView):
 class CITypeHistoryView(APIView):
     url_prefix = "/history/ci_types"
 
-    @role_required(RoleEnum.CONFIG)
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Operation_Audit,
+                         app_cli.op.read, app_cli.admin_name)
     def get(self):
         type_id = request.values.get("type_id")
         username = request.values.get("username")

cmdb-api/api/views/cmdb/preference.py

@@ -8,20 +8,22 @@ from flask import request
 from api.lib.cmdb.ci_type import CITypeManager
 from api.lib.cmdb.const import PermEnum
 from api.lib.cmdb.const import ResourceTypeEnum
-from api.lib.cmdb.const import RoleEnum
 from api.lib.cmdb.perms import CIFilterPermsCRUD
 from api.lib.cmdb.preference import PreferenceManager
 from api.lib.cmdb.resp_format import ErrFormat
+from api.lib.common_setting.decorator import perms_role_required
+from api.lib.common_setting.role_perm_base import CMDBApp
 from api.lib.decorator import args_required
 from api.lib.decorator import args_validate
 from api.lib.perm.acl.acl import ACLManager
 from api.lib.perm.acl.acl import has_perm_from_args
 from api.lib.perm.acl.acl import is_app_admin
-from api.lib.perm.acl.acl import role_required
 from api.lib.perm.acl.acl import validate_permission
 from api.lib.utils import handle_arg_list
 from api.resource import APIView
 
+app_cli = CMDBApp()
+
 
 class PreferenceShowCITypesView(APIView):
     url_prefix = ("/preference/ci_types", "/preference/ci_types2")
@@ -104,7 +106,8 @@ class PreferenceRelationApiView(APIView):
 
         return self.jsonify(views=views, id2type=id2type, name2id=name2id)
 
-    @role_required(RoleEnum.CONFIG)
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Service_Tree_Definition,
+                         app_cli.op.read, app_cli.admin_name)
     @args_required("name")
     @args_required("cr_ids")
     @args_validate(PreferenceManager.pref_rel_cls)
@@ -118,14 +121,16 @@ class PreferenceRelationApiView(APIView):
 
         return self.jsonify(views=views, id2type=id2type, name2id=name2id)
 
-    @role_required(RoleEnum.CONFIG)
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Service_Tree_Definition,
+                         app_cli.op.read, app_cli.admin_name)
     @args_required("name")
     def put(self, _id):
         views, id2type, name2id = PreferenceManager.create_or_update_relation_view(_id=_id, **request.values)
 
         return self.jsonify(views=views, id2type=id2type, name2id=name2id)
 
-    @role_required(RoleEnum.CONFIG)
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Service_Tree_Definition,
+                         app_cli.op.read, app_cli.admin_name)
     @args_required("name")
     def delete(self):
         name = request.values.get("name")

cmdb-api/api/views/cmdb/relation_type.py

@@ -4,14 +4,16 @@
 from flask import abort
 from flask import request
 
-from api.lib.cmdb.const import RoleEnum
 from api.lib.cmdb.relation_type import RelationTypeManager
 from api.lib.cmdb.resp_format import ErrFormat
+from api.lib.common_setting.decorator import perms_role_required
+from api.lib.common_setting.role_perm_base import CMDBApp
 from api.lib.decorator import args_required
 from api.lib.decorator import args_validate
-from api.lib.perm.acl.acl import role_required
 from api.resource import APIView
 
+app_cli = CMDBApp()
+
 
 class RelationTypeView(APIView):
     url_prefix = ("/relation_types", "/relation_types/<int:rel_id>")
@@ -19,7 +21,8 @@ class RelationTypeView(APIView):
     def get(self):
         return self.jsonify([i.to_dict() for i in RelationTypeManager.get_all()])
 
-    @role_required(RoleEnum.CONFIG)
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Relationship_Types,
+                         app_cli.op.read, app_cli.admin_name)
     @args_required("name")
     @args_validate(RelationTypeManager.cls)
     def post(self):
@@ -28,7 +31,8 @@ class RelationTypeView(APIView):
 
         return self.jsonify(rel.to_dict())
 
-    @role_required(RoleEnum.CONFIG)
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Relationship_Types,
+                         app_cli.op.read, app_cli.admin_name)
     @args_required("name")
     @args_validate(RelationTypeManager.cls)
     def put(self, rel_id):
@@ -37,7 +41,8 @@ class RelationTypeView(APIView):
 
         return self.jsonify(rel.to_dict())
 
-    @role_required(RoleEnum.CONFIG)
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.Relationship_Types,
+                         app_cli.op.read, app_cli.admin_name)
     def delete(self, rel_id):
         RelationTypeManager.delete(rel_id)
 

cmdb-api/api/views/cmdb/topology.py

@@ -0,0 +1,178 @@
+# -*- coding:utf-8 -*-
+
+from flask import abort
+from flask import request
+
+from api.lib.cmdb.const import PermEnum, ResourceTypeEnum
+from api.lib.cmdb.resp_format import ErrFormat
+from api.lib.cmdb.topology import TopologyViewManager
+from api.lib.common_setting.decorator import perms_role_required
+from api.lib.common_setting.role_perm_base import CMDBApp
+from api.lib.decorator import args_required
+from api.lib.decorator import args_validate
+from api.lib.perm.acl.acl import ACLManager
+from api.lib.perm.acl.acl import has_perm_from_args
+from api.lib.perm.acl.acl import is_app_admin
+from api.resource import APIView
+
+app_cli = CMDBApp()
+
+
+class TopologyGroupView(APIView):
+    url_prefix = ('/topology_views/groups', '/topology_views/groups/<int:group_id>')
+
+    @args_required('name')
+    @args_validate(TopologyViewManager.group_cls)
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.TopologyView,
+                         app_cli.op.create_topology_group, app_cli.admin_name)
+    def post(self):
+        name = request.values.get('name')
+        order = request.values.get('order')
+
+        group = TopologyViewManager.add_group(name, order)
+
+        return self.jsonify(group.to_dict())
+
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.TopologyView,
+                         app_cli.op.update_topology_group, app_cli.admin_name)
+    def put(self, group_id):
+        name = request.values.get('name')
+        view_ids = request.values.get('view_ids')
+        group = TopologyViewManager().update_group(group_id, name, view_ids)
+
+        return self.jsonify(**group)
+
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.TopologyView,
+                         app_cli.op.delete_topology_group, app_cli.admin_name)
+    def delete(self, group_id):
+        TopologyViewManager.delete_group(group_id)
+
+        return self.jsonify(group_id=group_id)
+
+
+class TopologyGroupOrderView(APIView):
+    url_prefix = ('/topology_views/groups/order',)
+
+    @args_required('group_ids')
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.TopologyView,
+                         app_cli.op.update_topology_group, app_cli.admin_name)
+    def post(self):
+        group_ids = request.values.get('group_ids')
+
+        TopologyViewManager.group_order(group_ids)
+
+        return self.jsonify(group_ids=group_ids)
+
+    def put(self):
+        return self.post()
+
+
+class TopologyView(APIView):
+    url_prefix = ('/topology_views', '/topology_views/relations/ci_types/<int:type_id>', '/topology_views/<int:_id>')
+
+    def get(self, type_id=None, _id=None):
+        if type_id is not None:
+            return self.jsonify(TopologyViewManager.relation_from_ci_type(type_id))
+
+        if _id is not None:
+            return self.jsonify(TopologyViewManager().get_view_by_id(_id))
+
+        return self.jsonify(TopologyViewManager.get_all())
+
+    @args_required('name', 'central_node_type', 'central_node_instances', 'path', 'group_id')
+    @args_validate(TopologyViewManager.cls)
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.TopologyView,
+                         app_cli.op.create_topology_view, app_cli.admin_name)
+    def post(self):
+        name = request.values.pop('name')
+        group_id = request.values.pop('group_id', None)
+        option = request.values.pop('option', None)
+        order = request.values.pop('order', None)
+
+        topo_view = TopologyViewManager.add(name, group_id, option, order, **request.values)
+
+        return self.jsonify(topo_view)
+
+    @args_validate(TopologyViewManager.cls)
+    @has_perm_from_args("_id", ResourceTypeEnum.TOPOLOGY_VIEW, PermEnum.UPDATE, TopologyViewManager.get_name_by_id)
+    def put(self, _id):
+        topo_view = TopologyViewManager.update(_id, **request.values)
+
+        return self.jsonify(topo_view)
+
+    @has_perm_from_args("_id", ResourceTypeEnum.TOPOLOGY_VIEW, PermEnum.DELETE, TopologyViewManager.get_name_by_id)
+    def delete(self, _id):
+        TopologyViewManager.delete(_id)
+
+        return self.jsonify(code=200)
+
+
+class TopologyOrderView(APIView):
+    url_prefix = ('/topology_views/order',)
+
+    @args_required('view_ids')
+    @perms_role_required(app_cli.app_name, app_cli.resource_type_name, app_cli.op.TopologyView,
+                         app_cli.op.create_topology_view, app_cli.admin_name)
+    def post(self):
+        view_ids = request.values.get('view_ids')
+
+        TopologyViewManager.group_inner_order(view_ids)
+
+        return self.jsonify(view_ids=view_ids)
+
+    def put(self):
+        return self.post()
+
+
+class TopologyViewPreview(APIView):
+    url_prefix = ('/topology_views/preview', '/topology_views/<int:_id>/view')
+
+    def get(self, _id=None):
+        if _id is not None:
+            acl = ACLManager('cmdb')
+            resource_name = TopologyViewManager.get_name_by_id(_id)
+            if (not acl.has_permission(resource_name, ResourceTypeEnum.TOPOLOGY_VIEW, PermEnum.READ) and
+                    not is_app_admin('cmdb')):
+                return abort(403, ErrFormat.no_permission.format(resource_name, PermEnum.READ))
+
+            return self.jsonify(TopologyViewManager().topology_view(view_id=_id))
+        else:
+            return self.jsonify(TopologyViewManager().topology_view(preview=request.values))
+
+    def post(self, _id=None):
+        return self.get(_id)
+
+
+class TopologyViewGrantView(APIView):
+    url_prefix = "/topology_views/<int:view_id>/roles/<int:rid>/grant"
+
+    def post(self, view_id, rid):
+        perms = request.values.pop('perms', None)
+
+        view_name = TopologyViewManager.get_name_by_id(view_id) or abort(404, ErrFormat.not_found)
+        acl = ACLManager('cmdb')
+        if not acl.has_permission(view_name, ResourceTypeEnum.TOPOLOGY_VIEW,
+                                  PermEnum.GRANT) and not is_app_admin('cmdb'):
+            return abort(403, ErrFormat.no_permission.format(view_name, PermEnum.GRANT))
+
+        acl.grant_resource_to_role_by_rid(view_name, rid, ResourceTypeEnum.TOPOLOGY_VIEW, perms, rebuild=True)
+
+        return self.jsonify(code=200)
+
+
+class TopologyViewRevokeView(APIView):
+    url_prefix = "/topology_views/<int:view_id>/roles/<int:rid>/revoke"
+
+    @args_required('perms')
+    def post(self, view_id, rid):
+        perms = request.values.pop('perms', None)
+
+        view_name = TopologyViewManager.get_name_by_id(view_id) or abort(404, ErrFormat.not_found)
+        acl = ACLManager('cmdb')
+        if not acl.has_permission(view_name, ResourceTypeEnum.TOPOLOGY_VIEW,
+                                  PermEnum.GRANT) and not is_app_admin('cmdb'):
+            return abort(403, ErrFormat.no_permission.format(view_name, PermEnum.GRANT))
+
+        acl.revoke_resource_from_role_by_rid(view_name, rid, ResourceTypeEnum.TOPOLOGY_VIEW, perms, rebuild=True)
+
+        return self.jsonify(code=200)

cmdb-api/requirements.txt

@@ -1,7 +1,7 @@
 -i https://mirrors.aliyun.com/pypi/simple
 alembic==1.7.7
 bs4==0.0.1
-celery>=5.3.1
+celery==5.3.1
 celery-once==3.0.1
 click==8.1.3
 elasticsearch==7.17.9
@@ -31,6 +31,7 @@ marshmallow==2.20.2
 more-itertools==5.0.0
 msgpack-python==0.5.6
 Pillow>=10.0.1
+pycryptodome==3.12.0
 cryptography>=41.0.2
 PyJWT==2.4.0
 PyMySQL==1.1.0
@@ -53,4 +54,6 @@ shamir~=17.12.0
 pycryptodomex>=3.19.0
 colorama>=0.4.6
 lz4>=4.3.2
-python-magic==0.4.27
+python-magic==0.4.27
+jsonpath==0.82.2
+networkx>=3.1

cmdb-api/settings.example.py

@@ -20,10 +20,16 @@ DEBUG_TB_INTERCEPT_REDIRECTS = False
 
 ERROR_CODES = [400, 401, 403, 404, 405, 500, 502]
 
+MYSQL_USER = env.str('MYSQL_USER', default='cmdb')
+MYSQL_PASSWORD = env.str('MYSQL_PASSWORD', default='123456')
+MYSQL_HOST = env.str('MYSQL_HOST', default='127.0.0.1')
+MYSQL_PORT = env.int('MYSQL_PORT', default=3306)
+MYSQL_DATABASE = env.str('MYSQL_DATABASE', default='cmdb')
 # # database
-SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://{user}:{password}@127.0.0.1:3306/{db}?charset=utf8'
+SQLALCHEMY_DATABASE_URI = f'mysql+pymysql://{MYSQL_USER}:{MYSQL_PASSWORD}@' \
+                          f'{MYSQL_HOST}:{MYSQL_PORT}/{MYSQL_DATABASE}?charset=utf8'
 SQLALCHEMY_BINDS = {
-    'user': 'mysql+pymysql://{user}:{password}@127.0.0.1:3306/{db}?charset=utf8'
+    'user': SQLALCHEMY_DATABASE_URI
 }
 SQLALCHEMY_ECHO = False
 SQLALCHEMY_TRACK_MODIFICATIONS = False
@@ -33,9 +39,9 @@ SQLALCHEMY_ENGINE_OPTIONS = {
 
 # # cache
 CACHE_TYPE = 'redis'
-CACHE_REDIS_HOST = '127.0.0.1'
-CACHE_REDIS_PORT = 6379
-CACHE_REDIS_PASSWORD = ''
+CACHE_REDIS_HOST = env.str('CACHE_REDIS_HOST', default='redis')
+CACHE_REDIS_PORT = env.str('CACHE_REDIS_PORT', default='6379')
+CACHE_REDIS_PASSWORD = env.str('CACHE_REDIS_PASSWORD', default='')
 CACHE_KEY_PREFIX = 'CMDB::'
 CACHE_DEFAULT_TIMEOUT = 3000
 

cmdb-ui/.env

@@ -3,3 +3,4 @@ VUE_APP_PREVIEW=false
 VUE_APP_API_BASE_URL=http://127.0.0.1:5000/api
 VUE_APP_BUILD_PACKAGES="ticket,calendar,acl"
 VUE_APP_IS_OUTER=true
+VUE_APP_IS_OPEN_SOURCE=true

cmdb-ui/package.json

@@ -39,7 +39,7 @@
     "md5": "^2.2.1",
     "moment": "^2.24.0",
     "nprogress": "^0.2.0",
-    "relation-graph": "^1.1.0",
+    "relation-graph": "^2.1.42",
     "snabbdom": "^3.5.1",
     "sortablejs": "1.9.0",
     "style-resources-loader": "^1.5.0",
@@ -59,7 +59,7 @@
     "vue-template-compiler": "2.6.11",
     "vuedraggable": "^2.23.0",
     "vuex": "^3.1.1",
-    "vxe-table": "3.6.9",
+    "vxe-table": "3.7.10",
     "vxe-table-plugin-export-xlsx": "2.0.0",
     "xe-utils": "3",
     "xlsx": "0.15.0",

cmdb-ui/src/App.vue

@@ -61,12 +61,12 @@ export default {
     )
 
     // 注册富文本自定义元素
-    const resume = {
-      type: 'attachment',
-      attachmentLabel: '',
-      attachmentValue: '',
-      children: [{ text: '' }], // void 元素必须有一个 children ，其中只有一个空字符串，重要！！！
-    }
+    // const resume = {
+    //   type: 'attachment',
+    //   attachmentLabel: '',
+    //   attachmentValue: '',
+    //   children: [{ text: '' }], // void 元素必须有一个 children ，其中只有一个空字符串，重要！！！
+    // }
 
     function withAttachment(editor) {
       // JS 语法

cmdb-ui/src/api/cmdb.js

@@ -0,0 +1,18 @@
+import { axios } from '@/utils/request'
+
+export function searchCI(params, isShowMessage = true) {
+  return axios({
+    url: `/v0.1/ci/s`,
+    method: 'GET',
+    params: params,
+    isShowMessage
+  })
+}
+
+export function getCIType(CITypeName, parameter) {
+  return axios({
+    url: `/v0.1/ci_types/${CITypeName}`,
+    method: 'GET',
+    params: parameter
+  })
+}

cmdb-ui/src/components/CMDBExprDrawer/index.vue

@@ -9,7 +9,7 @@
     :bodyStyle="{ padding: '24px 12px' }"
     :placement="placement"
   >
-    <ResourceSearch :fromCronJob="true" @copySuccess="copySuccess" />
+    <ResourceSearch ref="resourceSearch" :fromCronJob="true" :type="type" :typeId="typeId" @copySuccess="copySuccess" />
   </CustomDrawer>
 </template>
 
@@ -23,6 +23,14 @@ export default {
       type: String,
       default: 'right',
     },
+    type: {
+      type: String,
+      default: 'resourceSearch'
+    },
+    typeId: {
+      type: Number,
+      default: null
+    }
   },
   data() {
     return {

cmdb-ui/src/components/CMDBFilterComp/expression.vue

@@ -1,346 +1,407 @@
-<template>
-  <div>
-    <a-space :style="{ display: 'flex', marginBottom: '10px' }" v-for="(item, index) in ruleList" :key="item.id">
-      <div :style="{ width: '70px', height: '24px', position: 'relative' }">
-        <treeselect
-          v-if="index"
-          class="custom-treeselect"
-          :style="{ width: '70px', '--custom-height': '24px', position: 'absolute', top: '-17px', left: 0 }"
-          v-model="item.type"
-          :multiple="false"
-          :clearable="false"
-          searchable
-          :options="ruleTypeList"
-          :normalizer="
-            (node) => {
-              return {
-                id: node.value,
-                label: node.label,
-                children: node.children,
-              }
-            }
-          "
-          :disabled="disabled"
-        >
-        </treeselect>
-      </div>
-      <treeselect
-        class="custom-treeselect"
-        :style="{ width: '130px', '--custom-height': '24px' }"
-        v-model="item.property"
-        :multiple="false"
-        :clearable="false"
-        searchable
-        :options="canSearchPreferenceAttrList"
-        :normalizer="
-          (node) => {
-            return {
-              id: node.name,
-              label: node.alias || node.name,
-              children: node.children,
-            }
-          }
-        "
-        appendToBody
-        :zIndex="1050"
-        :disabled="disabled"
-      >
-        <div
-          :title="node.label"
-          slot="option-label"
-          slot-scope="{ node }"
-          :style="{ width: '100%', whiteSpace: 'nowrap', textOverflow: 'ellipsis', overflow: 'hidden' }"
-        >
-          <ValueTypeMapIcon :attr="node.raw" />
-          {{ node.label }}
-        </div>
-        <div
-          :style="{ width: '100%', whiteSpace: 'nowrap', textOverflow: 'ellipsis', overflow: 'hidden' }"
-          slot="value-label"
-          slot-scope="{ node }"
-        >
-          <ValueTypeMapIcon :attr="node.raw" /> {{ node.label }}
-        </div>
-      </treeselect>
-      <treeselect
-        class="custom-treeselect"
-        :style="{ width: '100px', '--custom-height': '24px' }"
-        v-model="item.exp"
-        :multiple="false"
-        :clearable="false"
-        searchable
-        :options="[...getExpListByProperty(item.property), ...advancedExpList]"
-        :normalizer="
-          (node) => {
-            return {
-              id: node.value,
-              label: node.label,
-              children: node.children,
-            }
-          }
-        "
-        @select="(value) => handleChangeExp(value, item, index)"
-        appendToBody
-        :zIndex="1050"
-        :disabled="disabled"
-      >
-      </treeselect>
-      <treeselect
-        class="custom-treeselect"
-        :style="{ width: '175px', '--custom-height': '24px' }"
-        v-model="item.value"
-        :multiple="false"
-        :clearable="false"
-        searchable
-        v-if="isChoiceByProperty(item.property) && (item.exp === 'is' || item.exp === '~is')"
-        :options="getChoiceValueByProperty(item.property)"
-        :placeholder="$t('placeholder2')"
-        :normalizer="
-          (node) => {
-            return {
-              id: node[0],
-              label: node[0],
-              children: node.children,
-            }
-          }
-        "
-        appendToBody
-        :zIndex="1050"
-        :disabled="disabled"
-      >
-        <div
-          :title="node.label"
-          slot="option-label"
-          slot-scope="{ node }"
-          :style="{ width: '100%', whiteSpace: 'nowrap', textOverflow: 'ellipsis', overflow: 'hidden' }"
-        >
-          {{ node.label }}
-        </div>
-      </treeselect>
-      <a-input-group
-        size="small"
-        compact
-        v-else-if="item.exp === 'range' || item.exp === '~range'"
-        :style="{ width: '175px' }"
-      >
-        <a-input
-          class="ops-input"
-          size="small"
-          v-model="item.min"
-          :style="{ width: '78px' }"
-          :placeholder="$t('min')"
-          :disabled="disabled"
-        />
-        ~
-        <a-input
-          class="ops-input"
-          size="small"
-          v-model="item.max"
-          :style="{ width: '78px' }"
-          :placeholder="$t('max')"
-          :disabled="disabled"
-        />
-      </a-input-group>
-      <a-input-group size="small" compact v-else-if="item.exp === 'compare'" :style="{ width: '175px' }">
-        <treeselect
-          class="custom-treeselect"
-          :style="{ width: '60px', '--custom-height': '24px' }"
-          v-model="item.compareType"
-          :multiple="false"
-          :clearable="false"
-          searchable
-          :options="compareTypeList"
-          :normalizer="
-            (node) => {
-              return {
-                id: node.value,
-                label: node.label,
-                children: node.children,
-              }
-            }
-          "
-          appendToBody
-          :zIndex="1050"
-          :disabled="disabled"
-        >
-        </treeselect>
-        <a-input class="ops-input" v-model="item.value" size="small" style="width: 113px" />
-      </a-input-group>
-      <a-input
-        v-else-if="item.exp !== 'value' && item.exp !== '~value'"
-        size="small"
-        v-model="item.value"
-        :placeholder="item.exp === 'in' || item.exp === '~in' ? $t('cmdbFilterComp.split', { separator: ';' }) : ''"
-        class="ops-input"
-        :style="{ width: '175px' }"
-        :disabled="disabled"
-      ></a-input>
-      <div v-else :style="{ width: '175px' }"></div>
-      <template v-if="!disabled">
-        <a-tooltip :title="$t('copy')">
-          <a class="operation" @click="handleCopyRule(item)"><ops-icon type="veops-copy"/></a>
-        </a-tooltip>
-        <a-tooltip :title="$t('delete')">
-          <a class="operation" @click="handleDeleteRule(item)"><ops-icon type="icon-xianxing-delete"/></a>
-        </a-tooltip>
-        <a-tooltip :title="$t('cmdbFilterComp.addHere')" v-if="needAddHere">
-          <a class="operation" @click="handleAddRuleAt(item)"><a-icon type="plus-circle"/></a>
-        </a-tooltip>
-      </template>
-    </a-space>
-    <div class="table-filter-add" v-if="!disabled">
-      <a @click="handleAddRule">+ {{ $t('new') }}</a>
-    </div>
-  </div>
-</template>
-
-<script>
-import _ from 'lodash'
-import { v4 as uuidv4 } from 'uuid'
-import { ruleTypeList, expList, advancedExpList, compareTypeList } from './constants'
-import ValueTypeMapIcon from '../CMDBValueTypeMapIcon'
-
-export default {
-  name: 'Expression',
-  components: { ValueTypeMapIcon },
-  model: {
-    prop: 'value',
-    event: 'change',
-  },
-  props: {
-    value: {
-      type: Array,
-      default: () => [],
-    },
-    canSearchPreferenceAttrList: {
-      type: Array,
-      required: true,
-      default: () => [],
-    },
-    needAddHere: {
-      type: Boolean,
-      default: false,
-    },
-    disabled: {
-      type: Boolean,
-      default: false,
-    },
-  },
-  data() {
-    return {
-      compareTypeList,
-    }
-  },
-  computed: {
-    ruleList: {
-      get() {
-        return this.value
-      },
-      set(val) {
-        this.$emit('change', val)
-        return val
-      },
-    },
-    ruleTypeList() {
-      return ruleTypeList()
-    },
-    expList() {
-      return expList()
-    },
-    advancedExpList() {
-      return advancedExpList()
-    },
-  },
-  methods: {
-    getExpListByProperty(property) {
-      if (property) {
-        const _find = this.canSearchPreferenceAttrList.find((item) => item.name === property)
-        if (_find && ['0', '1', '3', '4', '5'].includes(_find.value_type)) {
-          return [
-            { value: 'is', label: this.$t('cmdbFilterComp.is') },
-            { value: '~is', label: this.$t('cmdbFilterComp.~is') },
-            { value: '~value', label: this.$t('cmdbFilterComp.~value') }, // 为空的定义有点绕
-            { value: 'value', label: this.$t('cmdbFilterComp.value') },
-          ]
-        }
-        return this.expList
-      }
-      return this.expList
-    },
-    isChoiceByProperty(property) {
-      const _find = this.canSearchPreferenceAttrList.find((item) => item.name === property)
-      if (_find) {
-        return _find.is_choice
-      }
-      return false
-    },
-    handleAddRule() {
-      this.ruleList.push({
-        id: uuidv4(),
-        type: 'and',
-        property: this.canSearchPreferenceAttrList[0]?.name,
-        exp: 'is',
-        value: null,
-      })
-      this.$emit('change', this.ruleList)
-    },
-    handleCopyRule(item) {
-      this.ruleList.push({ ...item, id: uuidv4() })
-      this.$emit('change', this.ruleList)
-    },
-    handleDeleteRule(item) {
-      const idx = this.ruleList.findIndex((r) => r.id === item.id)
-      if (idx > -1) {
-        this.ruleList.splice(idx, 1)
-      }
-      this.$emit('change', this.ruleList)
-    },
-    handleAddRuleAt(item) {
-      const idx = this.ruleList.findIndex((r) => r.id === item.id)
-      if (idx > -1) {
-        this.ruleList.splice(idx, 0, {
-          id: uuidv4(),
-          type: 'and',
-          property: this.canSearchPreferenceAttrList[0]?.name,
-          exp: 'is',
-          value: null,
-        })
-      }
-      this.$emit('change', this.ruleList)
-    },
-    getChoiceValueByProperty(property) {
-      const _find = this.canSearchPreferenceAttrList.find((item) => item.name === property)
-      if (_find) {
-        return _find.choice_value
-      }
-      return []
-    },
-    handleChangeExp({ value }, item, index) {
-      const _ruleList = _.cloneDeep(this.ruleList)
-      if (value === 'range') {
-        _ruleList[index] = {
-          ..._ruleList[index],
-          min: '',
-          max: '',
-          exp: value,
-        }
-      } else if (value === 'compare') {
-        _ruleList[index] = {
-          ..._ruleList[index],
-          compareType: '1',
-          exp: value,
-        }
-      } else {
-        _ruleList[index] = {
-          ..._ruleList[index],
-          exp: value,
-        }
-      }
-      this.ruleList = _ruleList
-      this.$emit('change', this.ruleList)
-    },
-  },
-}
-</script>
-
-<style></style>
+<template>
+  <div>
+    <a-space :style="{ display: 'flex', marginBottom: '10px' }" v-for="(item, index) in ruleList" :key="item.id">
+      <div :style="{ width: '70px', height: '24px', position: 'relative' }">
+        <treeselect
+          v-if="index"
+          class="custom-treeselect"
+          :style="{ width: '70px', '--custom-height': '24px', position: 'absolute', top: '-17px', left: 0 }"
+          v-model="item.type"
+          :multiple="false"
+          :clearable="false"
+          searchable
+          :options="ruleTypeList"
+          :normalizer="
+            (node) => {
+              return {
+                id: node.value,
+                label: node.label,
+                children: node.children,
+              }
+            }
+          "
+          :disabled="disabled"
+        >
+        </treeselect>
+      </div>
+      <treeselect
+        class="custom-treeselect"
+        :style="{ width: '130px', '--custom-height': '24px' }"
+        v-model="item.property"
+        :multiple="false"
+        :clearable="false"
+        searchable
+        :options="canSearchPreferenceAttrList"
+        :normalizer="
+          (node) => {
+            return {
+              id: node.name,
+              label: node.alias || node.name,
+              children: node.children,
+            }
+          }
+        "
+        appendToBody
+        :zIndex="1050"
+        :disabled="disabled"
+      >
+        <div
+          :title="node.label"
+          slot="option-label"
+          slot-scope="{ node }"
+          :style="{ width: '100%', whiteSpace: 'nowrap', textOverflow: 'ellipsis', overflow: 'hidden' }"
+        >
+          <ValueTypeMapIcon :attr="node.raw" />
+          {{ node.label }}
+        </div>
+        <div
+          :style="{ width: '100%', whiteSpace: 'nowrap', textOverflow: 'ellipsis', overflow: 'hidden' }"
+          slot="value-label"
+          slot-scope="{ node }"
+        >
+          <ValueTypeMapIcon :attr="node.raw" /> {{ node.label }}
+        </div>
+      </treeselect>
+      <treeselect
+        class="custom-treeselect"
+        :style="{ width: '100px', '--custom-height': '24px' }"
+        v-model="item.exp"
+        :multiple="false"
+        :clearable="false"
+        searchable
+        :options="[...getExpListByProperty(item.property), ...advancedExpList]"
+        :normalizer="
+          (node) => {
+            return {
+              id: node.value,
+              label: node.label,
+              children: node.children,
+            }
+          }
+        "
+        @select="(value) => handleChangeExp(value, item, index)"
+        appendToBody
+        :zIndex="1050"
+        :disabled="disabled"
+      >
+        <div
+          slot="option-label"
+          slot-scope="{ node }"
+          :style="{ width: '100%', whiteSpace: 'nowrap', textOverflow: 'ellipsis', overflow: 'hidden' }"
+        >
+          <a-tooltip :title="node.label">
+            {{ node.label }}
+          </a-tooltip>
+        </div>
+        <div
+          :style="{ width: '100%', whiteSpace: 'nowrap', textOverflow: 'ellipsis', overflow: 'hidden' }"
+          slot="value-label"
+          slot-scope="{ node }"
+        >
+          <a-tooltip :title="node.label">
+            {{ node.label }}
+          </a-tooltip>
+        </div>
+      </treeselect>
+      <CIReferenceAttr
+        v-if="getAttr(item.property).is_reference && (item.exp === 'is' || item.exp === '~is')"
+        :style="{ width: '175px' }"
+        class="select-filter-component"
+        :referenceTypeId="getAttr(item.property).reference_type_id"
+        :disabled="disabled"
+        v-model="item.value"
+      />
+      <a-select
+        v-else-if="getAttr(item.property).is_bool && (item.exp === 'is' || item.exp === '~is')"
+        v-model="item.value"
+        class="select-filter-component"
+        :style="{ width: '175px' }"
+        :disabled="disabled"
+        :placeholder="$t('placeholder2')"
+      >
+        <a-select-option key="1">
+          true
+        </a-select-option>
+        <a-select-option key="0">
+          false
+        </a-select-option>
+      </a-select>
+      <treeselect
+        class="custom-treeselect"
+        :style="{ width: '175px', '--custom-height': '24px' }"
+        v-model="item.value"
+        :multiple="false"
+        :clearable="false"
+        searchable
+        v-else-if="isChoiceByProperty(item.property) && (item.exp === 'is' || item.exp === '~is')"
+        :options="getChoiceValueByProperty(item.property)"
+        :placeholder="$t('placeholder2')"
+        :normalizer="
+          (node) => {
+            return {
+              id: String(node[0] || ''),
+              label: node[1] ? node[1].label || node[0] : node[0],
+              children: node.children && node.children.length ? node.children : undefined,
+            }
+          }
+        "
+        appendToBody
+        :zIndex="1050"
+        :disabled="disabled"
+      >
+        <div
+          :title="node.label"
+          slot="option-label"
+          slot-scope="{ node }"
+          :style="{ width: '100%', whiteSpace: 'nowrap', textOverflow: 'ellipsis', overflow: 'hidden' }"
+        >
+          {{ node.label }}
+        </div>
+      </treeselect>
+      <a-input-group
+        size="small"
+        compact
+        v-else-if="item.exp === 'range' || item.exp === '~range'"
+        :style="{ width: '175px' }"
+      >
+        <a-input
+          class="ops-input"
+          size="small"
+          v-model="item.min"
+          :style="{ width: '78px' }"
+          :placeholder="$t('min')"
+          :disabled="disabled"
+        />
+        ~
+        <a-input
+          class="ops-input"
+          size="small"
+          v-model="item.max"
+          :style="{ width: '78px' }"
+          :placeholder="$t('max')"
+          :disabled="disabled"
+        />
+      </a-input-group>
+      <a-input-group size="small" compact v-else-if="item.exp === 'compare'" :style="{ width: '175px' }">
+        <treeselect
+          class="custom-treeselect"
+          :style="{ width: '60px', '--custom-height': '24px' }"
+          v-model="item.compareType"
+          :multiple="false"
+          :clearable="false"
+          searchable
+          :options="compareTypeList"
+          :normalizer="
+            (node) => {
+              return {
+                id: node.value,
+                label: node.label,
+                children: node.children,
+              }
+            }
+          "
+          appendToBody
+          :zIndex="1050"
+          :disabled="disabled"
+        >
+        </treeselect>
+        <a-input class="ops-input" v-model="item.value" size="small" style="width: 113px" />
+      </a-input-group>
+      <a-input
+        v-else-if="item.exp !== 'value' && item.exp !== '~value'"
+        size="small"
+        v-model="item.value"
+        :placeholder="item.exp === 'in' || item.exp === '~in' ? $t('cmdbFilterComp.split', { separator: ';' }) : ''"
+        class="ops-input"
+        :style="{ width: '175px' }"
+        :disabled="disabled"
+      ></a-input>
+      <div v-else :style="{ width: '175px' }"></div>
+      <template v-if="!disabled">
+        <a-tooltip :title="$t('copy')">
+          <a class="operation" @click="handleCopyRule(item)"><ops-icon type="veops-copy"/></a>
+        </a-tooltip>
+        <a-tooltip :title="$t('delete')">
+          <a class="operation" @click="handleDeleteRule(item)"><ops-icon type="icon-xianxing-delete"/></a>
+        </a-tooltip>
+        <a-tooltip :title="$t('cmdbFilterComp.addHere')" v-if="needAddHere">
+          <a class="operation" @click="handleAddRuleAt(item)"><a-icon type="plus-circle"/></a>
+        </a-tooltip>
+      </template>
+    </a-space>
+    <div class="table-filter-add" v-if="!disabled">
+      <a @click="handleAddRule">+ {{ $t('new') }}</a>
+    </div>
+  </div>
+</template>
+
+<script>
+import _ from 'lodash'
+import { v4 as uuidv4 } from 'uuid'
+import { ruleTypeList, expList, advancedExpList, compareTypeList } from './constants'
+import ValueTypeMapIcon from '../CMDBValueTypeMapIcon'
+import CIReferenceAttr from '../ciReferenceAttr/index.vue'
+
+export default {
+  name: 'Expression',
+  components: { ValueTypeMapIcon, CIReferenceAttr },
+  model: {
+    prop: 'value',
+    event: 'change',
+  },
+  props: {
+    value: {
+      type: Array,
+      default: () => [],
+    },
+    canSearchPreferenceAttrList: {
+      type: Array,
+      required: true,
+      default: () => [],
+    },
+    needAddHere: {
+      type: Boolean,
+      default: false,
+    },
+    disabled: {
+      type: Boolean,
+      default: false,
+    },
+  },
+  data() {
+    return {
+      compareTypeList,
+    }
+  },
+  computed: {
+    ruleList: {
+      get() {
+        return this.value
+      },
+      set(val) {
+        this.$emit('change', val)
+        return val
+      },
+    },
+    ruleTypeList() {
+      return ruleTypeList()
+    },
+    expList() {
+      return expList()
+    },
+    advancedExpList() {
+      return advancedExpList()
+    },
+  },
+  methods: {
+    getExpListByProperty(property) {
+      if (property) {
+        const _find = this.canSearchPreferenceAttrList.find((item) => item.name === property)
+        if (_find && (['0', '1', '3', '4', '5'].includes(_find.value_type) || _find.is_reference || _find.is_bool)) {
+          return [
+            { value: 'is', label: this.$t('cmdbFilterComp.is') },
+            { value: '~is', label: this.$t('cmdbFilterComp.~is') },
+            { value: '~value', label: this.$t('cmdbFilterComp.~value') }, // 为空的定义有点绕
+            { value: 'value', label: this.$t('cmdbFilterComp.value') },
+          ]
+        }
+        return this.expList
+      }
+      return this.expList
+    },
+    isChoiceByProperty(property) {
+      const _find = this.canSearchPreferenceAttrList.find((item) => item.name === property)
+      if (_find) {
+        return _find.is_choice
+      }
+      return false
+    },
+    handleAddRule() {
+      this.ruleList.push({
+        id: uuidv4(),
+        type: 'and',
+        property: this.canSearchPreferenceAttrList[0]?.name,
+        exp: 'is',
+        value: null,
+      })
+      this.$emit('change', this.ruleList)
+    },
+    handleCopyRule(item) {
+      this.ruleList.push({ ...item, id: uuidv4() })
+      this.$emit('change', this.ruleList)
+    },
+    handleDeleteRule(item) {
+      const idx = this.ruleList.findIndex((r) => r.id === item.id)
+      if (idx > -1) {
+        this.ruleList.splice(idx, 1)
+      }
+      this.$emit('change', this.ruleList)
+    },
+    handleAddRuleAt(item) {
+      const idx = this.ruleList.findIndex((r) => r.id === item.id)
+      if (idx > -1) {
+        this.ruleList.splice(idx, 0, {
+          id: uuidv4(),
+          type: 'and',
+          property: this.canSearchPreferenceAttrList[0]?.name,
+          exp: 'is',
+          value: null,
+        })
+      }
+      this.$emit('change', this.ruleList)
+    },
+    getChoiceValueByProperty(property) {
+      const _find = this.canSearchPreferenceAttrList.find((item) => item.name === property)
+      if (_find) {
+        return _find.choice_value
+      }
+      return []
+    },
+    getAttr(property) {
+      return this.canSearchPreferenceAttrList.find((item) => item.name === property) || {}
+    },
+    handleChangeExp({ value }, item, index) {
+      const _ruleList = _.cloneDeep(this.ruleList)
+      if (value === 'range') {
+        _ruleList[index] = {
+          ..._ruleList[index],
+          min: '',
+          max: '',
+          exp: value,
+        }
+      } else if (value === 'compare') {
+        _ruleList[index] = {
+          ..._ruleList[index],
+          compareType: '1',
+          exp: value,
+        }
+      } else {
+        _ruleList[index] = {
+          ..._ruleList[index],
+          exp: value,
+        }
+      }
+      this.ruleList = _ruleList
+      this.$emit('change', this.ruleList)
+    },
+  },
+}
+</script>
+
+<style lang="less" scoped>
+.select-filter-component {
+  height: 24px;
+
+  /deep/ .ant-select-selection {
+    height: 24px;
+    background: #f7f8fa;
+    line-height: 24px;
+    border: none;
+
+    .ant-select-selection__rendered {
+      height: 24px;
+      line-height: 24px;
+    }
+  }
+}
+</style>

cmdb-ui/src/components/CMDBValueTypeMapIcon/index.vue

@@ -1,49 +1,77 @@
-<template>
-  <span>
-    <ops-icon :type="getPropertyIcon(attr)" />
-  </span>
-</template>
-
-<script>
-export default {
-  name: 'ValueTypeIcon',
-  props: {
-    attr: {
-      type: Object,
-      default: () => {},
-    },
-  },
-  methods: {
-    getPropertyIcon(attr) {
-      switch (attr.value_type) {
-        case '0':
-          return 'duose-shishu'
-        case '1':
-          return 'duose-fudianshu'
-        case '2':
-          if (attr.is_password) {
-            return 'duose-password'
-          }
-          if (attr.is_link) {
-            return 'duose-link'
-          }
-          return 'duose-wenben'
-        case '3':
-          return 'duose-datetime'
-        case '4':
-          return 'duose-date'
-        case '5':
-          return 'duose-time'
-        case '6':
-          return 'duose-json'
-        case '7':
-          return 'duose-password'
-        case '8':
-          return 'duose-link'
-      }
-    },
-  },
-}
-</script>
-
-<style></style>
+<template>
+  <span>
+    <ops-icon :type="getPropertyIcon(attr)" />
+  </span>
+</template>
+
+<script>
+export default {
+  name: 'ValueTypeIcon',
+  props: {
+    attr: {
+      type: Object,
+      default: () => {},
+    },
+  },
+  methods: {
+    getPropertyIcon(attr) {
+      let valueType = attr.value_type
+
+      if (valueType === '2') {
+        if (attr.is_password) {
+          valueType = '7'
+        } else if (attr.is_link) {
+          valueType = '8'
+        } else if (!attr.is_index) {
+          valueType = '9'
+        }
+      }
+
+      if (
+        valueType === '7' &&
+        attr.is_bool
+      ) {
+        valueType = '10'
+      }
+
+      if (
+        valueType === '0' &&
+        attr.is_reference
+      ) {
+        valueType = '11'
+      }
+
+      switch (valueType) {
+        case '0':
+          return 'duose-shishu'
+        case '1':
+          return 'duose-fudianshu'
+        case '2':
+          return 'duose-wenben'
+        case '3':
+          return 'duose-datetime'
+        case '4':
+          return 'duose-date'
+        case '5':
+          return 'duose-time'
+        case '6':
+          return 'duose-json'
+        case '7':
+          return 'duose-password'
+        case '8':
+          return 'duose-link'
+        case '9':
+          return 'duose-changwenben1'
+        case '10':
+          return 'duose-boole'
+        case '11':
+          return 'duose-quote'
+        default:
+          return ''
+      }
+    },
+  },
+}
+</script>
+
+<style></style>

cmdb-ui/src/components/CustomDrawer/CustomDrawer.vue

@@ -73,7 +73,7 @@ export default {
 .custom-drawer-close {
   position: absolute;
   cursor: pointer;
-  background: #custom_colors[color_1];
+  background: @primary-color;
   color: white;
   text-align: center;
   transition: all 0.3s;

cmdb-ui/src/components/CustomIconSelect/constants.js

@@ -177,7 +177,7 @@ export const linearIconList = [
     }]
   }, {
     value: 'icon-xianxing-application',
-    label: '应用',
+    label: '常用组件',
     list: [{
       value: 'icon-xianxing-yilianjie',
       label: '已连接'
@@ -517,7 +517,7 @@ export const fillIconList = [
     }]
   }, {
     value: 'icon-shidi-application',
-    label: '应用',
+    label: '常用组件',
     list: [{
       value: 'icon-shidi-yilianjie',
       label: '已连接'
@@ -729,6 +729,18 @@ export const multicolorIconList = [
     value: 'database',
     label: '数据库',
     list: [{
+      value: 'caise-TIDB',
+      label: 'TIDB'
+    }, {
+      value: 'caise-dameng',
+      label: '达梦'
+    }, {
+      value: 'caise-kingbase',
+      label: 'KingBase'
+    }, {
+      value: 'caise-TDSQL',
+      label: 'TDSQL'
+    }, {
       value: 'caise-DB2',
       label: 'DB2'
     }, {
@@ -809,6 +821,9 @@ export const multicolorIconList = [
     value: 'system',
     label: '操作系统',
     list: [{
+      value: 'ciase-aix',
+      label: 'aix'
+    }, {
       value: 'caise-Windows',
       label: 'Windows'
     }, {
@@ -903,8 +918,140 @@ export const multicolorIconList = [
     }]
   }, {
     value: 'caise-application',
-    label: '应用',
+    label: '常用组件',
     list: [{
+      value: 'caise-websphere',
+      label: 'WebSphere'
+    }, {
+      value: 'caise-vps',
+      label: 'VPS'
+    }, {
+      value: 'caise-F5',
+      label: 'F5'
+    }, {
+      value: 'caise-HAProxy',
+      label: 'HAProxy'
+    }, {
+      value: 'caise-kafka',
+      label: 'kafka'
+    }, {
+      value: 'caise-dongfangtong',
+      label: '东方通'
+    }, {
+      value: 'cmdb-vcenter',
+      label: 'VCenter'
+    }, {
+      value: 'ops-KVM',
+      label: 'KVM'
+    }, {
+      value: 'caise-JBoss',
+      label: 'JBoss'
+    }, {
+      value: 'caise-weblogic',
+      label: 'WebLogic'
+    }, {
+      value: 'caise-disk_array',
+      label: '磁盘阵列'
+    }, {
+      value: 'caise-fiber',
+      label: '光纤交换机'
+    }, {
+      value: 'caise-bandwidth_line',
+      label: '带宽线路'
+    }, {
+      value: 'caise-pc',
+      label: 'PC'
+    }, {
+      value: 'caise-rack',
+      label: '机柜'
+    }, {
+      value: 'caise-computer_room',
+      label: '机房'
+    }, {
+      value: 'caise-ip_address',
+      label: 'ip地址'
+    }, {
+      value: 'caise_pool',
+      label: 'ip池'
+    }, {
+      value: 'caise-storage_volume1',
+      label: '存储卷'
+    }, {
+      value: 'caise-virtualization',
+      label: '虚拟化'
+    }, {
+      value: 'caise-business',
+      label: '业务'
+    }, {
+      value: 'caise-database',
+      label: '数据库'
+    }, {
+      value: 'caise-middleware',
+      label: '中间件'
+    }, {
+      value: 'caise-websever',
+      label: 'websever'
+    }, {
+      value: 'caise-message_queue',
+      label: '消息队列'
+    }, {
+      value: 'caise-load_balancing',
+      label: '负载均衡'
+    }, {
+      value: 'caise-storage_device',
+      label: '存储设备'
+    }, {
+      value: 'caise-network_devices',
+      label: '网络设备'
+    }, {
+      value: 'caise-computer',
+      label: '计算机'
+    }, {
+      value: 'caise-hardware',
+      label: '硬件设备'
+    }, {
+      value: 'caise-data_center2',
+      label: '数据中心'
+    }, {
+      value: 'caise-hyperV',
+      label: 'hyperV'
+    }, {
+      value: 'caise-IPAM',
+      label: 'IPAM'
+    }, {
+      value: 'caise-system',
+      label: '操作系统'
+    }, {
+      value: 'caise-public_cloud',
+      label: '公有云'
+    }, {
+      value: 'caise-data_center',
+      label: '数据中心'
+    }, {
+      value: 'caise-folder',
+      label: '文件夹'
+    }, {
+      value: 'caise-resource_pool',
+      label: '资源池'
+    }, {
+      value: 'caise-network',
+      label: '网络'
+    }, {
+      value: 'caise-distributed_switch',
+      label: '分布式交换机'
+    }, {
+      value: 'caise-standard_switch',
+      label: '标准式交换机'
+    }, {
+      value: 'caise-host_cluster',
+      label: '主机集群'
+    }, {
+      value: 'caise-storage_cluster',
+      label: '数据存储集群'
+    }, {
+      value: 'caise-data_storage',
+      label: '数据存储'
+    }, {
       value: 'caise-yilianjie',
       label: '已连接'
     }, {

cmdb-ui/src/components/EmployeeTransfer/index.vue

@@ -298,14 +298,14 @@ export default {
       width: 20px;
       height: 20px;
       border-radius: 2px;
-      background-color: #custom_colors[color_2];
-      color: #custom_colors[color_1];
+      background-color: @primary-color_5;
+      color: @primary-color;
       display: inline-flex;
       justify-content: center;
       align-items: center;
       cursor: pointer;
       &:hover {
-        background-color: #custom_colors[color_1];
+        background-color: @primary-color;
         color: #fff;
       }
     }

cmdb-ui/src/components/Menu/index.module.less

@@ -0,0 +1,24 @@
+.cmdb-side-menu-search {
+  background-color: #FFFFFF !important;
+  cursor: auto !important;
+
+  :global {
+    .ant-input-affix-wrapper {
+      max-width: 170px !important;
+      width: 170px;
+      border-radius: 30px;
+    }
+
+    .ant-input {
+      box-shadow: none;
+      border: none;
+      background-color: #F7F8FA;
+      height: 30px;
+      line-height: 30px;
+    }
+
+    .ant-input-suffix {
+      right: 0px !important;
+    }
+  }
+}

cmdb-ui/src/components/Menu/menu.js

@@ -9,6 +9,8 @@ import {
 import { searchResourceType } from '@/modules/acl/api/resource'
 import { roleHasPermissionToGrant } from '@/modules/acl/api/permission'
 import CMDBGrant from '@/modules/cmdb/components/cmdbGrant'
+import styles from './index.module.less'
+import { mapActions } from 'vuex'
 
 const { Item, SubMenu } = Menu
 
@@ -40,7 +42,8 @@ export default {
       openKeys: [],
       selectedKeys: [],
       cachedOpenKeys: [],
-      resource_type: {}
+      resource_type: {},
+      currentAppRoute: ''
     }
   },
   computed: {
@@ -64,6 +67,7 @@ export default {
     searchResourceType({ page_size: 9999, app_id: 'cmdb' }).then(res => {
       this.resource_type = { groups: res.groups, id2perms: res.id2perms }
     })
+    this.currentAppRoute = this.$route?.matched?.[0]?.name || ''
     this.updateMenu()
   },
   watch: {
@@ -75,12 +79,14 @@ export default {
         this.openKeys = this.cachedOpenKeys
       }
     },
-    $route: function () {
+    $route: function (route) {
+      this.currentAppRoute = route?.matched?.[0]?.name
       this.updateMenu()
     },
   },
   inject: ['reload'],
   methods: {
+    ...mapActions(['UpdateCMDBSEarchValue']),
     cancelAttributes(e, menu) {
       const that = this
       e.preventDefault()
@@ -182,8 +188,8 @@ export default {
           <tag {...{ props, attrs }}>
             {this.renderIcon({ icon: menu.meta.icon, customIcon: menu.meta.customIcon, name: menu.meta.name, typeId: menu.meta.typeId, routeName: menu.name, selectedIcon: menu.meta.selectedIcon, })}
             <span>
-              <span class={this.renderI18n(menu.meta.title).length > 10 ? 'scroll' : ''}>{this.renderI18n(menu.meta.title)}</span>
-              {isShowDot &&
+              <span style={menu.meta.style} class={this.renderI18n(menu.meta.title).length > 10 ? 'scroll' : ''}>{this.renderI18n(menu.meta.title)}</span>
+              {isShowDot && !menu.meta.disabled &&
                 <a-popover
                   overlayClassName="custom-menu-extra-submenu"
                   placement="rightTop"
@@ -257,7 +263,7 @@ export default {
       const props = {}
       if (this.$route.name === routeName && selectedIcon) {
         return <ops-icon type={selectedIcon}></ops-icon>
-      } else if (icon.startsWith('ops-') || icon.startsWith('icon-xianxing') || icon.startsWith('icon-shidi')) {
+      } else if (icon.startsWith('ops-') || icon.startsWith('icon-xianxing') || icon.startsWith('icon-shidi') || icon.startsWith('veops-')) {
         return <ops-icon type={icon}></ops-icon>
       } else {
         typeof (icon) === 'object' ? props.component = icon : props.type = icon
@@ -286,6 +292,47 @@ export default {
           this.$message.error(this.$t('noPermission'))
         }
       })
+    },
+
+    jumpCMDBSearch(value) {
+      this.UpdateCMDBSEarchValue(value)
+
+      if (this.$route.name !== 'cmdb_resource_search') {
+        this.$router.push({
+          name: 'cmdb_resource_search',
+        })
+      }
+    },
+
+    renderCMDBSearch() {
+      if (this.currentAppRoute !== 'cmdb' || this.collapsed) {
+        return null
+      }
+
+      return (
+        <Item class={styles['cmdb-side-menu-search']}>
+          <a-input
+            ref="cmdbSideMenuSearchInputRef"
+            class={styles['cmdb-side-menu-search-input']}
+            style={{
+              border: this.$route.name === 'cmdb_resource_search' ? 'solid 1px #B1C9FF' : ''
+            }}
+            placeholder={this.$t('cmdbSearch')}
+            onPressEnter={(e) => {
+              this.jumpCMDBSearch(e.target.value)
+            }}
+          >
+            <ops-icon
+              slot="suffix"
+              type="veops-search1"
+              onClick={() => {
+                const value = this.$refs?.cmdbSideMenuSearchInputRef?.$refs?.input?.value || ''
+                this.jumpCMDBSearch(value)
+              }}
+            />
+          </a-input>
+        </Item>
+      )
     }
   },
 
@@ -313,6 +360,7 @@ export default {
     // {...{ props, on: on }}
     return (
       <Menu class="ops-side-bar" selectedKeys={this.selectedKeys} {...{ props, on: on }}>
+        {this.renderCMDBSearch()}
         {menuTree}
       </Menu>
     )

cmdb-ui/src/components/RegexSelect/constants.js

@@ -10,10 +10,10 @@ export const regList = () => {
         { id: 'landline', label: i18n.t('regexSelect.landline'), value: '^(?:(?:\\d{3}-)?\\d{8}|^(?:\\d{4}-)?\\d{7,8})(?:-\\d+)?$', message: '请输入正确座机' },
         { id: 'zipCode', label: i18n.t('regexSelect.zipCode'), value: '^(0[1-7]|1[0-356]|2[0-7]|3[0-6]|4[0-7]|5[1-7]|6[1-7]|7[0-5]|8[013-6])\\d{4}$', message: '请输入正确邮政编码' },
         { id: 'IDCard', label: i18n.t('regexSelect.IDCard'), value: '(^[1-9]\\d{5}(18|19|([23]\\d))\\d{2}((0[1-9])|(10|11|12))(([0-2][1-9])|10|20|30|31)\\d{3}[0-9Xx]$)|(^[1-9]\\d{5}\\d{2}((0[1-9])|(10|11|12))(([0-2][1-9])|10|20|30|31)\\d{3}$)', message: '请输入正确身份证号' },
-        { id: 'ip', label: i18n.t('regexSelect.ip'), value: '^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$', message: '请输入正确IP地址' },
-        { id: 'email', label: i18n.t('regexSelect.email'), value: '^\\w+([-+.]\\w+)*@\\w+([-.]\\w+)*\.\\w+([-.]\\w+)*$', message: '请输入正确邮箱' },
-        { id: 'link', label: i18n.t('regexSelect.link'), value: '^(https:\/\/www\.|http:\/\/www\.|https:\/\/|http:\/\/)?[a-zA-Z0-9]{2,}(\.[a-zA-Z0-9]{2,})(\.[a-zA-Z0-9]{2,})?$', message: '请输入链接' },
-        { id: 'monetaryAmount', label: i18n.t('regexSelect.monetaryAmount'), value: '^-?\\d+(,\\d{3})*(\.\\d{1,2})?$', message: '请输入货币金额' },
+        { id: 'ip', label: i18n.t('regexSelect.ip'), value: '^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$', message: '请输入正确IP地址' },
+        { id: 'email', label: i18n.t('regexSelect.email'), value: '^\\w+([-+.]\\w+)*@\\w+([-.]\\w+)*\\.\\w+([-.]\\w+)*$', message: '请输入正确邮箱' },
+        { id: 'link', label: i18n.t('regexSelect.link'), value: '^(https:\/\/www\\.|http:\/\/www\\.|https:\/\/|http:\/\/)?[a-zA-Z0-9]{2,}(\\.[a-zA-Z0-9]{2,})(\\.[a-zA-Z0-9]{2,})?$', message: '请输入链接' },
+        { id: 'monetaryAmount', label: i18n.t('regexSelect.monetaryAmount'), value: '^-?\\d+(,\\d{3})*(\\.\\d{1,2})?$', message: '请输入货币金额' },
         { id: 'custom', label: i18n.t('regexSelect.custom'), value: '', message: '' }
     ]
 }

cmdb-ui/src/components/RegexSelect/regexSelect.vue

@@ -61,7 +61,13 @@
         </template>
       </div>
     </div>
-    <a-input ref="regInput" :placeholder="$t('regexSelect.placeholder')" :value="current.label" @change="changeLabel">
+    <a-input
+      ref="regInput"
+      :placeholder="$t('regexSelect.placeholder')"
+      :value="current.label"
+      :disabled="disabled"
+      @change="changeLabel"
+    >
     </a-input>
   </a-popover>
 </template>
@@ -88,6 +94,10 @@ export default {
       type: Array,
       default: () => [],
     },
+    disabled: {
+      type: Boolean,
+      default: false,
+    }
   },
   data() {
     return {
@@ -161,7 +171,7 @@ export default {
       cursor: pointer;
       &-selected,
       &:hover {
-        color: #custom_colors[color_1];
+        color: @primary-color;
       }
     }
   }
@@ -179,7 +189,7 @@ export default {
       font-weight: 400;
       font-size: 14px;
       color: #000000;
-      border-left: 2px solid #custom_colors[color_1];
+      border-left: 2px solid @primary-color;
       padding-left: 6px;
       margin-left: -6px;
     }

cmdb-ui/src/components/RoleTransfer/index.vue

@@ -185,14 +185,14 @@ export default {
       width: 20px;
       height: 20px;
       border-radius: 2px;
-      background-color: #custom_colors[color_2];
-      color: #custom_colors[color_1];
+      background-color: @primary-color_5;
+      color: @primary-color;
       display: inline-flex;
       justify-content: center;
       align-items: center;
       cursor: pointer;
       &:hover {
-        background-color: #custom_colors[color_1];
+        background-color: @primary-color;
         color: #fff;
       }
     }

cmdb-ui/src/components/SidebarList/SidebarList.vue

@@ -93,7 +93,7 @@ export default {
   background-color: transparent;
 }
 .sidebar-list-item.sidebar-list-item-selected::before {
-  background-color: #custom_colors[color_1];
+  background-color: @primary-color;
 }
 .sidebar-list-item-dotline {
   padding-left: 20px;

cmdb-ui/src/components/SplitPane/SplitPane.vue

@@ -1,183 +1,187 @@
-<template>
-  <div ref="splitPane" class="split-pane" :class="direction + ' ' + appName" :style="{ flexDirection: direction }">
-    <div class="pane pane-one" ref="one" :style="lengthType + ':' + paneLengthValue1">
-      <slot name="one"></slot>
-    </div>
-
-    <div class="spliter-wrap">
-      <a-button
-        v-show="collapsable"
-        :icon="isExpanded ? 'left' : 'right'"
-        class="collapse-btn"
-        @click="handleExpand"
-      ></a-button>
-      <div
-        class="pane-trigger"
-        @mousedown="handleMouseDown"
-        :style="{ backgroundColor: triggerColor, width: `${triggerLength}px` }"
-      ></div>
-    </div>
-
-    <div class="pane pane-two" ref="two" :style="lengthType + ':' + paneLengthValue2">
-      <slot name="two"></slot>
-    </div>
-  </div>
-</template>
-
-<script>
-export default {
-  name: 'SplitPane',
-  props: {
-    direction: {
-      type: String,
-      default: 'row',
-    },
-
-    min: {
-      type: Number,
-      default: 10,
-    },
-
-    max: {
-      type: Number,
-      default: 90,
-    },
-
-    paneLengthPixel: {
-      type: Number,
-      default: 220,
-    },
-
-    triggerLength: {
-      type: Number,
-      default: 8,
-    },
-
-    appName: {
-      type: String,
-      default: 'viewer',
-    },
-
-    collapsable: {
-      type: Boolean,
-      default: false,
-    },
-    triggerColor: {
-      type: String,
-      default: '#f7f8fa',
-    },
-  },
-  data() {
-    return {
-      triggerLeftOffset: 0, // 鼠标距滑动器左(顶)侧偏移量
-      isExpanded: localStorage.getItem(`${this.appName}-isExpanded`)
-        ? JSON.parse(localStorage.getItem(`${this.appName}-isExpanded`))
-        : false,
-      parentContainer: null,
-    }
-  },
-  computed: {
-    lengthType() {
-      return this.direction === 'row' ? 'width' : 'height'
-    },
-
-    minLengthType() {
-      return this.direction === 'row' ? 'minWidth' : 'minHeight'
-    },
-
-    paneLengthValue1() {
-      return `calc(${this.paneLengthPercent}% - ${this.triggerLength / 2 + 'px'})`
-    },
-
-    paneLengthValue2() {
-      const rest = 100 - this.paneLengthPercent
-      return `calc(${rest}% - ${this.triggerLength / 2 + 'px'})`
-    },
-
-    paneLengthPercent() {
-      const clientRectWidth = this.parentContainer
-        ? this.parentContainer.clientWidth
-        : document.documentElement.getBoundingClientRect().width
-      return (this.paneLengthPixel / clientRectWidth) * 100
-    },
-  },
-
-  watch: {
-    isExpanded(newValue) {
-      if (newValue) {
-        document.querySelector(`.${this.appName} .pane-two`).style.display = 'none'
-      } else {
-        document.querySelector(`.${this.appName} .pane-two`).style.display = ''
-      }
-    },
-  },
-
-  mounted() {
-    const paneLengthPixel = localStorage.getItem(`${this.appName}-paneLengthPixel`)
-    if (paneLengthPixel) {
-      this.$emit('update:paneLengthPixel', Number(paneLengthPixel))
-    }
-    this.parentContainer = document.querySelector(`.${this.appName}`)
-    if (this.isExpanded) {
-      document.querySelector(`.${this.appName} .pane-two`).style.display = 'none'
-    } else {
-      document.querySelector(`.${this.appName} .pane-two`).style.display = ''
-    }
-  },
-
-  methods: {
-    // 按下滑动器
-    handleMouseDown(e) {
-      document.addEventListener('mousemove', this.handleMouseMove)
-      document.addEventListener('mouseup', this.handleMouseUp)
-      if (this.direction === 'row') {
-        this.triggerLeftOffset = e.pageX - e.srcElement.getBoundingClientRect().left
-      } else {
-        this.triggerLeftOffset = e.pageY - e.srcElement.getBoundingClientRect().top
-      }
-    },
-
-    // 按下滑动器后移动鼠标
-    handleMouseMove(e) {
-      this.isExpanded = false
-      this.$emit('expand', this.isExpanded)
-      const clientRect = this.$refs.splitPane.getBoundingClientRect()
-      let paneLengthPixel = 0
-
-      if (this.direction === 'row') {
-        const offset = e.pageX - clientRect.left - this.triggerLeftOffset + this.triggerLength / 2
-        paneLengthPixel = offset
-      } else {
-        const offset = e.pageY - clientRect.top - this.triggerLeftOffset + this.triggerLength / 2
-        paneLengthPixel = offset
-      }
-
-      if (paneLengthPixel < this.min) {
-        paneLengthPixel = this.min
-      }
-      if (paneLengthPixel > this.max) {
-        paneLengthPixel = this.max
-      }
-
-      this.$emit('update:paneLengthPixel', paneLengthPixel)
-
-      localStorage.setItem(`${this.appName}-paneLengthPixel`, paneLengthPixel)
-    },
-
-    // 松开滑动器
-    handleMouseUp() {
-      document.removeEventListener('mousemove', this.handleMouseMove)
-    },
-
-    handleExpand() {
-      this.isExpanded = !this.isExpanded
-      this.$emit('expand', this.isExpanded)
-      localStorage.setItem(`${this.appName}-isExpanded`, this.isExpanded)
-    },
-  },
-}
-</script>
-
-<style scoped lang="less">
-@import './index.less';
-</style>
+<template>
+  <div ref="splitPane" class="split-pane" :class="direction + ' ' + appName" :style="{ flexDirection: direction }">
+    <div class="pane pane-one" ref="one" :style="lengthType + ':' + paneLengthValue1">
+      <slot name="one"></slot>
+    </div>
+
+    <div class="spliter-wrap">
+      <a-button
+        v-show="collapsable"
+        :icon="isExpanded ? 'left' : 'right'"
+        class="collapse-btn"
+        @click="handleExpand"
+      ></a-button>
+      <div
+        class="pane-trigger"
+        @mousedown="handleMouseDown"
+        :style="{ backgroundColor: triggerColor, width: `${triggerLength}px` }"
+      ></div>
+    </div>
+
+    <div class="pane pane-two" ref="two" :style="lengthType + ':' + paneLengthValue2">
+      <slot name="two"></slot>
+    </div>
+  </div>
+</template>
+
+<script>
+export default {
+  name: 'SplitPane',
+  props: {
+    direction: {
+      type: String,
+      default: 'row',
+    },
+
+    min: {
+      type: Number,
+      default: 10,
+    },
+
+    max: {
+      type: Number,
+      default: 90,
+    },
+
+    paneLengthPixel: {
+      type: Number,
+      default: 220,
+    },
+
+    triggerLength: {
+      type: Number,
+      default: 8,
+    },
+
+    appName: {
+      type: String,
+      default: 'viewer',
+    },
+
+    collapsable: {
+      type: Boolean,
+      default: false,
+    },
+    triggerColor: {
+      type: String,
+      default: '#f7f8fa',
+    },
+    calcBasedParent: {
+      type: Boolean,
+      defualt: false
+    }
+  },
+  data() {
+    return {
+      triggerLeftOffset: 0, // 鼠标距滑动器左(顶)侧偏移量
+      isExpanded: localStorage.getItem(`${this.appName}-isExpanded`)
+        ? JSON.parse(localStorage.getItem(`${this.appName}-isExpanded`))
+        : false,
+      parentContainer: null,
+    }
+  },
+  computed: {
+    lengthType() {
+      return this.direction === 'row' ? 'width' : 'height'
+    },
+
+    minLengthType() {
+      return this.direction === 'row' ? 'minWidth' : 'minHeight'
+    },
+
+    paneLengthValue1() {
+      return `calc(${this.paneLengthPercent}% - ${this.triggerLength / 2 + 'px'})`
+    },
+
+    paneLengthValue2() {
+      const rest = 100 - this.paneLengthPercent
+      return `calc(${rest}% - ${this.triggerLength / 2 + 'px'})`
+    },
+
+    paneLengthPercent() {
+      const clientRectWidth = this.parentContainer && this.calcBasedParent
+        ? this.parentContainer.clientWidth
+        : document.documentElement.getBoundingClientRect().width
+      return (this.paneLengthPixel / clientRectWidth) * 100
+    },
+  },
+
+  watch: {
+    isExpanded(newValue) {
+      if (newValue) {
+        document.querySelector(`.${this.appName} .pane-two`).style.display = 'none'
+      } else {
+        document.querySelector(`.${this.appName} .pane-two`).style.display = ''
+      }
+    },
+  },
+
+  mounted() {
+    const paneLengthPixel = localStorage.getItem(`${this.appName}-paneLengthPixel`)
+    if (paneLengthPixel) {
+      this.$emit('update:paneLengthPixel', Number(paneLengthPixel))
+    }
+    this.parentContainer = document.querySelector(`.${this.appName}`)
+    if (this.isExpanded) {
+      document.querySelector(`.${this.appName} .pane-two`).style.display = 'none'
+    } else {
+      document.querySelector(`.${this.appName} .pane-two`).style.display = ''
+    }
+  },
+
+  methods: {
+    // 按下滑动器
+    handleMouseDown(e) {
+      document.addEventListener('mousemove', this.handleMouseMove)
+      document.addEventListener('mouseup', this.handleMouseUp)
+      if (this.direction === 'row') {
+        this.triggerLeftOffset = e.pageX - e.srcElement.getBoundingClientRect().left
+      } else {
+        this.triggerLeftOffset = e.pageY - e.srcElement.getBoundingClientRect().top
+      }
+    },
+
+    // 按下滑动器后移动鼠标
+    handleMouseMove(e) {
+      this.isExpanded = false
+      this.$emit('expand', this.isExpanded)
+      const clientRect = this.$refs.splitPane.getBoundingClientRect()
+      let paneLengthPixel = 0
+
+      if (this.direction === 'row') {
+        const offset = e.pageX - clientRect.left - this.triggerLeftOffset + this.triggerLength / 2
+        paneLengthPixel = offset
+      } else {
+        const offset = e.pageY - clientRect.top - this.triggerLeftOffset + this.triggerLength / 2
+        paneLengthPixel = offset
+      }
+
+      if (paneLengthPixel < this.min) {
+        paneLengthPixel = this.min
+      }
+      if (paneLengthPixel > this.max) {
+        paneLengthPixel = this.max
+      }
+
+      this.$emit('update:paneLengthPixel', paneLengthPixel)
+
+      localStorage.setItem(`${this.appName}-paneLengthPixel`, paneLengthPixel)
+    },
+
+    // 松开滑动器
+    handleMouseUp() {
+      document.removeEventListener('mousemove', this.handleMouseMove)
+    },
+
+    handleExpand() {
+      this.isExpanded = !this.isExpanded
+      this.$emit('expand', this.isExpanded)
+      localStorage.setItem(`${this.appName}-isExpanded`, this.isExpanded)
+    },
+  },
+}
+</script>
+
+<style scoped lang="less">
+@import './index.less';
+</style>

cmdb-ui/src/components/TwoColumnLayout/TwoColumnLayout.vue

@@ -6,7 +6,8 @@
       :paneLengthPixel.sync="paneLengthPixel"
       :appName="appName"
       :triggerColor="triggerColor"
-      :triggerLength="18"
+      :triggerLength="triggerLength"
+      :calcBasedParent="calcBasedParent"
     >
       <template #one>
         <div class="two-column-layout-sidebar">
@@ -37,6 +38,14 @@ export default {
       type: String,
       default: '#f7f8fa',
     },
+    triggerLength: {
+      type: Number,
+      default: 18
+    },
+    calcBasedParent: {
+      type: Boolean,
+      defualt: false
+    }
   },
   data() {
     return {

cmdb-ui/src/components/ciReferenceAttr/index.vue

@@ -0,0 +1,178 @@
+<template>
+  <div class="reference-attr-select-wrap">
+    <a-select
+      v-bind="$attrs"
+      v-model="selectCIIds"
+      optionFilterProp="title"
+      :mode="isList ? 'multiple' : 'default'"
+      showSearch
+      allowClear
+      :getPopupContainer="(trigger) => trigger.parentElement"
+      class="reference-attr-select"
+      :maxTagCount="2"
+      @dropdownVisibleChange="handleDropdownVisibleChange"
+      @search="handleSearch"
+      @change="handleChange"
+    >
+      <template v-if="!isInit">
+        <a-select-option
+          v-for="(item) in initSelectOption"
+          :key="item.key"
+          :title="item.title"
+        >
+          {{ item.title }}
+        </a-select-option>
+      </template>
+      <a-select-option
+        v-for="(item) in options"
+        :key="item.key"
+        :title="item.title"
+      >
+        {{ item.title }}
+      </a-select-option>
+    </a-select>
+  </div>
+</template>
+
+<script>
+import _ from 'lodash'
+import debounce from 'lodash/debounce'
+import { searchCI, getCIType } from '@/api/cmdb'
+
+export default {
+  name: 'CIReferenceAttr',
+  props: {
+    value: {
+      type: [Number, String, Array],
+      default: () => '',
+    },
+    isList: {
+      type: Boolean,
+      default: false,
+    },
+    referenceShowAttrName: {
+      type: String,
+      default: ''
+    },
+    referenceTypeId: {
+      type: [String, Number],
+      default: ''
+    },
+    initSelectOption: {
+      type: Array,
+      default: () => []
+    }
+  },
+  model: {
+    prop: 'value',
+    event: 'change',
+  },
+  data() {
+    return {
+      isInit: false,
+      options: [],
+      innerReferenceShowAttrName: ''
+    }
+  },
+  watch: {
+    referenceTypeId: {
+      immediate: true,
+      deep: true,
+      handler() {
+        this.isInit = false
+      }
+    }
+  },
+  computed: {
+    selectCIIds: {
+      get() {
+        if (this.isList) {
+          return this.value || []
+        } else {
+          return this.value ? Number(this.value) : ''
+        }
+      },
+      set(val) {
+        this.$emit('change', val ?? (this.isList ? [] : null))
+        return val
+      },
+    },
+  },
+  methods: {
+    async handleDropdownVisibleChange(open) {
+      if (!this.isInit && open && this.referenceTypeId) {
+        this.isInit = true
+
+        if (!this.referenceShowAttrName) {
+          const res = await getCIType(this.referenceTypeId)
+          const ciType = res?.ci_types?.[0]
+          this.innerReferenceShowAttrName = ciType?.show_name || ciType?.unique_name || ''
+        }
+
+        const attrName = this.referenceShowAttrName || this.innerReferenceShowAttrName || ''
+        if (!attrName) {
+          return
+        }
+
+        const res = await searchCI({
+          q: `_type:${this.referenceTypeId}`,
+          fl: attrName,
+          count: 25,
+        })
+
+        let options = res?.result?.map((item) => {
+          return {
+            key: item._id,
+            title: String(item?.[attrName] ?? '')
+          }
+        })
+
+        options = _.uniqBy([...this.initSelectOption, ...options], 'key')
+
+        this.options = options
+      }
+    },
+
+    handleSearch: debounce(async function(v) {
+      const attrName = this.referenceShowAttrName || this.innerReferenceShowAttrName || ''
+
+      if (!attrName || !this.referenceTypeId) {
+        return
+      }
+
+      const res = await searchCI({
+        q: `_type:${this.referenceTypeId}${v ? ',*' + v + '*' : ''}`,
+        fl: attrName,
+        count: v ? 100 : 25,
+      })
+
+      this.options = res?.result?.map((item) => {
+        return {
+          key: item._id,
+          title: String(item?.[attrName] ?? '')
+        }
+      })
+    }, 300),
+
+    handleChange(v) {
+      if (Array.isArray(v) ? !v.length : !v) {
+        this.handleSearch()
+      }
+    }
+  }
+}
+</script>
+
+<style lang="less" scoped>
+.reference-attr-select-wrap {
+  width: 100%;
+
+  .reference-attr-select {
+    width: 100%;
+
+    /deep/ .ant-select-dropdown {
+      z-index: 15;
+    }
+  }
+}
+</style>

cmdb-ui/src/components/tools/DocumentLink.vue

@@ -31,7 +31,7 @@ export default {
   text-align: center;
   border-radius: 4px;
   &:hover {
-    background: linear-gradient(0deg, rgba(0, 80, 201, 0.2) 0%, rgba(174, 207, 255, 0.06) 86.76%);
+    // background: linear-gradient(0deg, rgba(0, 80, 201, 0.2) 0%, rgba(174, 207, 255, 0.06) 86.76%);
     color: @layout-header-font-selected-color;
   }
 }

cmdb-ui/src/components/tools/TopMenu.vue

@@ -5,7 +5,6 @@
       v-for="route in defaultShowRoutes"
       :key="route.name"
       @click="() => handleClick(route)"
-      :title="$t(route.meta.title)"
     >
       {{ route.meta.title }}
     </span>
@@ -81,7 +80,7 @@ export default {
         if (route.name === 'cmdb') {
           const preference = await getPreference()
           const lastTypeId = window.localStorage.getItem('ops_ci_typeid') || undefined
-          if (lastTypeId && preference.some((item) => item.id === Number(lastTypeId))) {
+          if (lastTypeId && preference.type_ids.some((item) => item === Number(lastTypeId))) {
             this.$router.push(`/cmdb/instances/types/${lastTypeId}`)
           } else {
             this.$router.push('/cmdb/dashboard')
@@ -119,7 +118,9 @@ export default {
     line-height: @layout-header-line-height;
     display: inline-block;
   }
-  > span:hover,
+  > span:hover {
+    background-color: #f0f2f5;
+  }
   .top-menu-selected {
     font-weight: bold;
     color: @layout-header-font-selected-color;

cmdb-ui/src/components/tools/UserMenu.vue

@@ -5,15 +5,30 @@
       <span
         v-if="hasBackendPermission"
         @click="handleClick"
-        class="action"
-        style="width: 40px; display: flex; justify-content: center"
+        class="common-settings-btn"
       >
-        <a-icon type="setting" />
+        <ops-icon class="common-settings-btn-icon" type="veops-setting" />
+        <span class="common-settings-btn-text">{{ $t('settings') }}</span>
       </span>
-      <span class="locale" @click="changeLang">{{ locale === 'zh' ? 'English' : '中文' }}</span>
       <a-popover
-        trigger="click"
-        :overlayStyle="{ width: '150px' }"
+        overlayClassName="lang-popover-wrap"
+        placement="bottomRight"
+        :getPopupContainer="(trigger) => trigger.parentNode"
+      >
+        <span class="locale">{{ languageList.find((lang) => lang.key === locale).title }}</span>
+        <div class="lang-menu" slot="content">
+          <a
+            v-for="(lang) in languageList"
+            :key="lang.key"
+            :class="['lang-menu-item', lang.key === locale ? 'lang-menu-item_active' : '']"
+            @click="changeLang(lang.key)"
+          >
+            {{ lang.title }}
+          </a>
+        </div>
+      </a-popover>
+      <a-popover
+        :overlayStyle="{ width: '130px' }"
         placement="bottomRight"
         overlayClassName="custom-user"
       >
@@ -29,7 +44,7 @@
             <span>{{ $t('topMenu.logout') }}</span>
           </div>
         </template>
-        <span class="action ant-dropdown-link user-dropdown-menu">
+        <span class="action ant-dropdown-link user-dropdown-menu user-info-wrap">
           <a-avatar
             v-if="avatar()"
             class="avatar"
@@ -54,10 +69,26 @@ export default {
   components: {
     DocumentLink,
   },
+  data() {
+    return {
+      languageList: [
+        {
+          title: '简中',
+          key: 'zh'
+        },
+        {
+          title: 'EN',
+          key: 'en'
+        },
+      ]
+    }
+  },
   computed: {
     ...mapState(['user', 'locale']),
     hasBackendPermission() {
-      return this.user?.detailPermissions?.backend?.length
+      const isAdmin = this?.user?.roles?.permissions?.includes('acl_admin')
+
+      return isAdmin || this.user?.detailPermissions?.backend?.length
     },
   },
   methods: {
@@ -79,14 +110,9 @@ export default {
     handleClick() {
       this.$router.push('/setting')
     },
-    changeLang() {
-      if (this.locale === 'zh') {
-        this.SET_LOCALE('en')
-        this.$i18n.locale = 'en'
-      } else {
-        this.SET_LOCALE('zh')
-        this.$i18n.locale = 'zh'
-      }
+    changeLang(lang) {
+      this.SET_LOCALE(lang)
+      this.$i18n.locale = lang
       this.$nextTick(() => {
         setDocumentTitle(`${this.$t(this.$route.meta.title)} - ${domTitle}`)
       })
@@ -97,8 +123,8 @@ export default {
 
 <style lang="less">
 .color {
-  color: #custom_colors[color_1];
-  background-color: #custom_colors[color_2];
+  color: @primary-color;
+  background-color: @primary-color_5;
 }
 .custom-user {
   .custom-user-item {
@@ -116,8 +142,88 @@ export default {
 
 .locale {
   cursor: pointer;
+  padding: 0 8px;
+
   &:hover {
-    color: #custom_colors[color_1];
+    color: @primary-color;
+  }
+}
+
+.lang-popover-wrap {
+  width: 70px;
+  padding: 0px;
+
+  .ant-popover-inner-content {
+    padding: 0px;
+  }
+}
+</style>
+
+<style lang="less" scoped>
+.user-wrapper {
+  .common-settings-btn {
+    cursor: pointer;
+    padding: 0px 18px;
+    background-color: #F0F5FF;
+    border-radius: 22px;
+    height: 26px;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    margin: 0 8px;
+
+    &-icon {
+      font-size: 12px;
+      color: #2F54EB;
+    }
+
+    &-text {
+      margin-left: 4px;
+      font-size: 12px;
+      font-weight: 400;
+      color: #4E5969;
+    }
+
+    &:hover {
+      .commen-settings-btn-text {
+        color: #2F54EB;
+      }
+    }
+  }
+
+  .lang-menu {
+    width: 100%;
+    display: flex;
+    flex-direction: column;
+
+    &-item {
+      width: 100%;
+      padding: 5px 10px;
+      cursor: pointer;
+      color: #4E5969;
+
+      &_active {
+        color: #2F54EB;
+        background-color: #f0f5ff;
+      }
+
+      &:hover {
+        color: #2F54EB;
+      }
+    }
+  }
+
+  .user-info-wrap {
+    .avatar {
+      transition: all 0.2s;
+      border: solid 1px transparent;
+    }
+
+    &:hover {
+      .avatar {
+        border-color: #2F54EB;
+      }
+    }
   }
 }
 </style>

cmdb-ui/src/core/use.js

@@ -69,6 +69,8 @@ Vue.prototype.$httpError = function (err, describe) {
 
 window.$message = Vue.prototype.$message
 
+Vue.prototype.isOpenSource = process.env.VUE_APP_IS_OPEN_SOURCE === 'true'
+
 Vue.use(Antd)
 Vue.use(Viser)
 

cmdb-ui/src/lang/en.js

@@ -10,6 +10,7 @@ export default {
         resourceType: 'Resource Types',
         trigger: 'Triggers',
     },
+    settings: 'Common Settings',
     screen: 'Big Screen',
     dashboard: 'Dashboard',
     admin: 'Admin',
@@ -107,6 +108,7 @@ export default {
     visual: 'Visual',
     default: 'default',
     tip: 'Tip',
+    cmdbSearch: 'Search',
     pagination: {
         total: '{range0}-{range1} of {total} items'
     },
@@ -160,7 +162,7 @@ export default {
         landline: 'landline',
         zipCode: 'zip code',
         IDCard: 'ID card',
-        ip: 'IP',
+        ip: 'IPv4',
         email: 'email',
         link: 'link',
         monetaryAmount: 'monetary amount',

cmdb-ui/src/lang/zh.js

@@ -10,6 +10,7 @@ export default {
         resourceType: '资源类型',
         trigger: '触发器',
     },
+    settings: '通用设置',
     screen: '大屏',
     dashboard: '仪表盘',
     admin: '管理员',
@@ -107,6 +108,7 @@ export default {
     visual: '虚拟',
     default: '默认',
     tip: '提示',
+    cmdbSearch: '搜索一下',
     pagination: {
         total: '当前展示 {range0}-{range1} 条数据， 共 {total} 条'
     },
@@ -160,7 +162,7 @@ export default {
         landline: '座机',
         zipCode: '邮政编码',
         IDCard: '身份证号',
-        ip: 'IP地址',
+        ip: 'IPv4地址',
         email: '邮箱',
         link: '链接',
         monetaryAmount: '货币金额',

cmdb-ui/src/modules/cmdb/api/CIRelation.js

@@ -81,3 +81,11 @@ export function searchCIRelationFull(params) {
     params,
   })
 }
+
+export function searchCIRelationPath(data) {
+  return axios({
+    url: `/v0.1/ci_relations/path/s`,
+    method: 'POST',
+    data,
+  })
+}

cmdb-ui/src/modules/cmdb/api/CIType.js

@@ -1,232 +1,257 @@
-import { axios } from '@/utils/request'
-
-/**
- * 获取 所有的 ci_types
- * @param parameter
- * @returns {AxiosPromise}
- */
-export function getCITypes(parameter) {
-  return axios({
-    url: '/v0.1/ci_types',
-    method: 'GET',
-    params: parameter
-  })
-}
-
-/**
- * 获取 某个 ci_types
- * @param CITypeName
- * @param parameter
- * @returns {AxiosPromise}
- */
-export function getCIType(CITypeName, parameter) {
-  return axios({
-    url: `/v0.1/ci_types/${CITypeName}`,
-    method: 'GET',
-    params: parameter
-  })
-}
-
-/**
- * 创建 ci_type
- * @param data
- * @returns {AxiosPromise}
- */
-export function createCIType(data) {
-  return axios({
-    url: '/v0.1/ci_types',
-    method: 'POST',
-    data: data
-  })
-}
-
-/**
- * 更新 ci_type
- * @param CITypeId
- * @param data
- * @returns {AxiosPromise}
- */
-export function updateCIType(CITypeId, data) {
-  return axios({
-    url: `/v0.1/ci_types/${CITypeId}`,
-    method: 'PUT',
-    data: data
-  })
-}
-
-/**
- * 删除 ci_type
- * @param CITypeId
- * @returns {AxiosPromise}
- */
-export function deleteCIType(CITypeId) {
-  return axios({
-    url: `/v0.1/ci_types/${CITypeId}`,
-    method: 'DELETE'
-  })
-}
-
-/**
- * 获取 某个 ci_type 的分组
- * @param CITypeId
- * @param data
- * @returns {AxiosPromise}
- */
-export function getCITypeGroupById(CITypeId, data) {
-  return axios({
-    url: `/v0.1/ci_types/${CITypeId}/attribute_groups`,
-    method: 'GET',
-    params: data
-  })
-}
-
-/**
- * 保存 某个 ci_type 的分组
- * @param CITypeId
- * @param data
- * @returns {AxiosPromise}
- */
-export function createCITypeGroupById(CITypeId, data) {
-  return axios({
-    url: `/v0.1/ci_types/${CITypeId}/attribute_groups`,
-    method: 'POST',
-    data: data
-  })
-}
-
-/**
- * 修改 某个 ci_type 的分组
- * @param groupId
- * @param data
- * @returns {AxiosPromise}
- */
-export function updateCITypeGroupById(groupId, data) {
-  return axios({
-    url: `/v0.1/ci_types/attribute_groups/${groupId}`,
-    method: 'PUT',
-    data: data
-  })
-}
-
-/**
- * 删除 某个 ci_type 的分组
- * @param groupId
- * @param data
- * @returns {AxiosPromise}
- */
-export function deleteCITypeGroupById(groupId, data) {
-  return axios({
-    url: `/v0.1/ci_types/attribute_groups/${groupId}`,
-    method: 'delete',
-    data: data
-  })
-}
-
-export function getUniqueConstraintList(type_id) {
-  return axios({
-    url: `/v0.1/ci_types/${type_id}/unique_constraint`,
-    method: 'get',
-  })
-}
-
-export function addUniqueConstraint(type_id, data) {
-  return axios({
-    url: `/v0.1/ci_types/${type_id}/unique_constraint`,
-    method: 'post',
-    data: data
-  })
-}
-
-export function updateUniqueConstraint(type_id, id, data) {
-  return axios({
-    url: `/v0.1/ci_types/${type_id}/unique_constraint/${id}`,
-    method: 'put',
-    data: data
-  })
-}
-
-export function deleteUniqueConstraint(type_id, id) {
-  return axios({
-    url: `/v0.1/ci_types/${type_id}/unique_constraint/${id}`,
-    method: 'delete',
-  })
-}
-
-export function getTriggerList(type_id) {
-  return axios({
-    url: `/v0.1/ci_types/${type_id}/triggers`,
-    method: 'get',
-  })
-}
-
-export function addTrigger(type_id, data) {
-  return axios({
-    url: `/v0.1/ci_types/${type_id}/triggers`,
-    method: 'post',
-    data: data
-  })
-}
-
-export function updateTrigger(type_id, id, data) {
-  return axios({
-    url: `/v0.1/ci_types/${type_id}/triggers/${id}`,
-    method: 'put',
-    data: data
-  })
-}
-
-export function deleteTrigger(type_id, id) {
-  return axios({
-    url: `/v0.1/ci_types/${type_id}/triggers/${id}`,
-    method: 'delete',
-  })
-}
-
-// CMDB的模型和实例的授权接口
-export function grantCiType(type_id, rid, data) {
-  return axios({
-    url: `/v0.1/ci_types/${type_id}/roles/${rid}/grant`,
-    method: 'post',
-    data
-  })
-}
-// CMDB的模型和实例的删除授权接口
-export function revokeCiType(type_id, rid, data) {
-  return axios({
-    url: `/v0.1/ci_types/${type_id}/roles/${rid}/revoke`,
-    method: 'post',
-    data
-  })
-}
-// CMDB的模型和实例的过滤的权限
-export function ciTypeFilterPermissions(type_id) {
-  return axios({
-    url: `/v0.1/ci_types/${type_id}/filters/permissions`,
-    method: 'get',
-  })
-}
-
-// parent_ids, child_id
-export function postCiTypeInheritance(data) {
-  return axios({
-    url: `/v0.1/ci_types/inheritance`,
-    method: 'post',
-    data
-  })
-}
-
-// parent_id, child_id
-export function deleteCiTypeInheritance(data) {
-  return axios({
-    url: `/v0.1/ci_types/inheritance`,
-    method: 'delete',
-    data
-  })
-}
-
-export function getCITypeIcons() {
-  return axios({
-    url: '/v0.1/ci_types/icons',
-    method: 'GET',
-  })
-}
+import { axios } from '@/utils/request'
+
+/**
+ * 获取 所有的 ci_types
+ * @param parameter
+ * @returns {AxiosPromise}
+ */
+export function getCITypes(parameter) {
+  return axios({
+    url: '/v0.1/ci_types',
+    method: 'GET',
+    params: parameter
+  })
+}
+
+/**
+ * 获取 某个 ci_types
+ * @param CITypeName
+ * @param parameter
+ * @returns {AxiosPromise}
+ */
+export function getCIType(CITypeName, parameter) {
+  return axios({
+    url: `/v0.1/ci_types/${CITypeName}`,
+    method: 'GET',
+    params: parameter
+  })
+}
+
+/**
+ * 创建 ci_type
+ * @param data
+ * @returns {AxiosPromise}
+ */
+export function createCIType(data) {
+  return axios({
+    url: '/v0.1/ci_types',
+    method: 'POST',
+    data: data
+  })
+}
+
+/**
+ * 更新 ci_type
+ * @param CITypeId
+ * @param data
+ * @returns {AxiosPromise}
+ */
+export function updateCIType(CITypeId, data) {
+  return axios({
+    url: `/v0.1/ci_types/${CITypeId}`,
+    method: 'PUT',
+    data: data
+  })
+}
+
+/**
+ * 删除 ci_type
+ * @param CITypeId
+ * @returns {AxiosPromise}
+ */
+export function deleteCIType(CITypeId) {
+  return axios({
+    url: `/v0.1/ci_types/${CITypeId}`,
+    method: 'DELETE'
+  })
+}
+
+/**
+ * 获取 某个 ci_type 的分组
+ * @param CITypeId
+ * @param data
+ * @returns {AxiosPromise}
+ */
+export function getCITypeGroupById(CITypeId, data) {
+  return axios({
+    url: `/v0.1/ci_types/${CITypeId}/attribute_groups`,
+    method: 'GET',
+    params: data
+  })
+}
+
+/**
+ * 保存 某个 ci_type 的分组
+ * @param CITypeId
+ * @param data
+ * @returns {AxiosPromise}
+ */
+export function createCITypeGroupById(CITypeId, data) {
+  return axios({
+    url: `/v0.1/ci_types/${CITypeId}/attribute_groups`,
+    method: 'POST',
+    data: data
+  })
+}
+
+/**
+ * 修改 某个 ci_type 的分组
+ * @param groupId
+ * @param data
+ * @returns {AxiosPromise}
+ */
+export function updateCITypeGroupById(groupId, data) {
+  return axios({
+    url: `/v0.1/ci_types/attribute_groups/${groupId}`,
+    method: 'PUT',
+    data: data
+  })
+}
+
+/**
+ * 删除 某个 ci_type 的分组
+ * @param groupId
+ * @param data
+ * @returns {AxiosPromise}
+ */
+export function deleteCITypeGroupById(groupId, data) {
+  return axios({
+    url: `/v0.1/ci_types/attribute_groups/${groupId}`,
+    method: 'delete',
+    data: data
+  })
+}
+
+/**
+ * 获取级联属性配置
+ * @param {*} typeId
+ * @returns
+ */
+export function getCITypeCascadeAttributes(typeId) {
+  return axios({
+    url: `/v0.1/cascade_attributes/ci_types/${typeId}`,
+    method: 'get'
+  })
+}
+
+/**
+ * 获取级联属性数据
+ * @param {*} typeId
+ * @returns
+ */
+export function postCITypeCascadeAttributesValues(attrId, data) {
+  return axios({
+    url: `/v0.1/cascade_attributes/${attrId}/values`,
+    method: 'post',
+    data
+  })
+}
+
+export function getUniqueConstraintList(type_id) {
+  return axios({
+    url: `/v0.1/ci_types/${type_id}/unique_constraint`,
+    method: 'get',
+  })
+}
+
+export function addUniqueConstraint(type_id, data) {
+  return axios({
+    url: `/v0.1/ci_types/${type_id}/unique_constraint`,
+    method: 'post',
+    data: data
+  })
+}
+
+export function updateUniqueConstraint(type_id, id, data) {
+  return axios({
+    url: `/v0.1/ci_types/${type_id}/unique_constraint/${id}`,
+    method: 'put',
+    data: data
+  })
+}
+
+export function deleteUniqueConstraint(type_id, id) {
+  return axios({
+    url: `/v0.1/ci_types/${type_id}/unique_constraint/${id}`,
+    method: 'delete',
+  })
+}
+
+export function getTriggerList(type_id) {
+  return axios({
+    url: `/v0.1/ci_types/${type_id}/triggers`,
+    method: 'get',
+  })
+}
+
+export function addTrigger(type_id, data) {
+  return axios({
+    url: `/v0.1/ci_types/${type_id}/triggers`,
+    method: 'post',
+    data: data
+  })
+}
+
+export function updateTrigger(type_id, id, data) {
+  return axios({
+    url: `/v0.1/ci_types/${type_id}/triggers/${id}`,
+    method: 'put',
+    data: data
+  })
+}
+
+export function deleteTrigger(type_id, id) {
+  return axios({
+    url: `/v0.1/ci_types/${type_id}/triggers/${id}`,
+    method: 'delete',
+  })
+}
+
+// CMDB的模型和实例的授权接口
+export function grantCiType(type_id, rid, data) {
+  return axios({
+    url: `/v0.1/ci_types/${type_id}/roles/${rid}/grant`,
+    method: 'post',
+    data
+  })
+}
+// CMDB的模型和实例的删除授权接口
+export function revokeCiType(type_id, rid, data) {
+  return axios({
+    url: `/v0.1/ci_types/${type_id}/roles/${rid}/revoke`,
+    method: 'post',
+    data
+  })
+}
+// CMDB的模型和实例的过滤的权限
+export function ciTypeFilterPermissions(type_id) {
+  return axios({
+    url: `/v0.1/ci_types/${type_id}/filters/permissions`,
+    method: 'get',
+  })
+}
+
+// parent_ids, child_id
+export function postCiTypeInheritance(data) {
+  return axios({
+    url: `/v0.1/ci_types/inheritance`,
+    method: 'post',
+    data
+  })
+}
+
+// parent_id, child_id
+export function deleteCiTypeInheritance(data) {
+  return axios({
+    url: `/v0.1/ci_types/inheritance`,
+    method: 'delete',
+    data
+  })
+}
+
+export function getCITypeIcons() {
+  return axios({
+    url: '/v0.1/ci_types/icons',
+    method: 'GET',
+  })
+}

cmdb-ui/src/modules/cmdb/api/CITypeRelation.js

@@ -74,3 +74,11 @@ export function getCanEditByParentIdChildId(parent_id, child_id) {
     method: 'GET'
   })
 }
+
+export function getCITypeRelationPath(params) {
+  return axios({
+    url: `/v0.1/ci_type_relations/path`,
+    method: 'GET',
+    params
+  })
+}

cmdb-ui/src/modules/cmdb/api/ci.js

@@ -2,11 +2,12 @@ import { axios } from '@/utils/request'
 
 const urlPrefix = '/v0.1'
 
-export function searchCI(params) {
+export function searchCI(params, isShowMessage = true) {
   return axios({
     url: urlPrefix + `/ci/s`,
     method: 'GET',
-    params: params
+    params: params,
+    isShowMessage
   })
 }
 

cmdb-ui/src/modules/cmdb/api/ciTypeGroup.js

@@ -50,3 +50,13 @@ export const putCITypeGroups = (data) => {
         data: data
     })
 }
+
+// 导出模型分组
+export function exportCITypeGroups(params) {
+  return axios({
+    url: `${urlPrefix}/ci_types/template/export`,
+    method: 'GET',
+    params: params,
+    timeout: 30 * 1000,
+  })
+}

cmdb-ui/src/modules/cmdb/api/discovery.js

@@ -45,11 +45,37 @@ export function getHttpAttributes(name, params) {
     })
 }
 
-export function getSnmpAttributes(name) {
-    return axios({
-        url: `/v0.1/adr/snmp/${name}/attributes`,
-        method: 'GET',
-    })
+export function getSnmpAttributes(type, name) {
+  return axios({
+    url: `/v0.1/adr/${type}/${name}/attributes`,
+    method: 'GET',
+  })
+}
+
+export function getHttpAttrMapping(name, resource) {
+  return axios({
+    url: `/v0.1/adr/http/${name}/mapping`,
+    method: 'GET',
+    params: {
+      resource
+    }
+  })
+}
+
+export function getHTTPAccounts(params) {
+  return axios({
+      url: `/v0.1/adr/accounts`,
+      method: 'GET',
+      params
+  })
+}
+
+export function postHTTPAccounts(data) {
+  return axios({
+      url: `/v0.1/adr/accounts`,
+      method: 'POST',
+      data
+  })
 }
 
 export function getCITypeDiscovery(type_id) {
@@ -118,3 +144,65 @@ export function deleteAdc(adc_id) {
         method: 'DELETE',
     })
 }
+
+export function getAdcCounter(params) {
+  return axios({
+    url: `v0.1/adc/counter`,
+    method: 'GET',
+    params
+  })
+}
+
+export function getAdcExecHistories(params) {
+  return axios({
+    url: `v0.1/adc/exec/histories`,
+    method: 'GET',
+    params
+  })
+}
+
+export function getAdtSyncHistories(adt_id) {
+  return axios({
+    url: `/v0.1/adt/${adt_id}/sync/histories`,
+    method: 'GET',
+    params: {
+      page_size: 9999
+    }
+  })
+}
+
+export function postAdtTest(adt_id) {
+  return axios({
+    url: `/v0.1/adt/${adt_id}/test`,
+    method: 'POST',
+  })
+}
+
+export function getAdtTestResult(exec_id) {
+  return axios({
+    url: `/v0.1/adt/test/${exec_id}/result`,
+    method: 'GET'
+  })
+}
+
+export function getCITypeAttributes(type_id) {
+  return axios({
+    url: `/v0.1/adt/ci_types/${type_id}/attributes`,
+    method: 'GET',
+  })
+}
+
+export function getCITypeRelations(type_id) {
+  return axios({
+    url: `/v0.1/adt/ci_types/${type_id}/relations`,
+    method: 'GET',
+  })
+}
+
+export function postCITypeRelations(type_id, data) {
+  return axios({
+    url: `/v0.1/adt/ci_types/${type_id}/relations`,
+    method: 'POST',
+    data
+  })
+}

cmdb-ui/src/modules/cmdb/api/history.js

@@ -1,56 +1,92 @@
-import { axios } from '@/utils/request'
-
-export function getCIHistory(ciId) {
-  return axios({
-    url: `/v0.1/history/ci/${ciId}`,
-    method: 'GET'
-  })
-}
-
-export function getCIHistoryTable(params) {
-  return axios({
-    url: `/v0.1/history/records/attribute`,
-    method: 'GET',
-    params: params
-  })
-}
-
-export function getRelationTable(params) {
-  return axios({
-    url: `/v0.1/history/records/relation`,
-    method: 'GET',
-    params: params
-  })
-}
-
-export function getCITypesTable(params) {
-  return axios({
-    url: `/v0.1/history/ci_types`,
-    method: 'GET',
-    params: params
-  })
-}
-
-export function getUsers(params) {
-  return axios({
-    url: `/v1/acl/users/employee`,
-    method: 'GET',
-    params: params
-  })
-}
-
-export function getCiTriggers(params) {
-  return axios({
-    url: `/v0.1/history/ci_triggers`,
-    method: 'GET',
-    params: params
-  })
-}
-
-export function getCiTriggersByCiId(ci_id, params) {
-  return axios({
-    url: `/v0.1/history/ci_triggers/${ci_id}`,
-    method: 'GET',
-    params
-  })
-}
+import { axios } from '@/utils/request'
+
+export function getCIHistory(ciId) {
+  return axios({
+    url: `/v0.1/history/ci/${ciId}`,
+    method: 'GET'
+  })
+}
+
+export function getCIHistoryTable(params) {
+  return axios({
+    url: `/v0.1/history/records/attribute`,
+    method: 'GET',
+    params: params,
+    timeout: 30 * 1000
+  })
+}
+
+export function getRelationTable(params) {
+  return axios({
+    url: `/v0.1/history/records/relation`,
+    method: 'GET',
+    params: params,
+    timeout: 30 * 1000
+  })
+}
+
+export function getCITypesTable(params) {
+  return axios({
+    url: `/v0.1/history/ci_types`,
+    method: 'GET',
+    params: params,
+    timeout: 30 * 1000
+  })
+}
+
+export function getUsers(params) {
+  return axios({
+    url: `/v1/acl/users/employee`,
+    method: 'GET',
+    params: params
+  })
+}
+
+export function getCiTriggers(params) {
+  return axios({
+    url: `/v0.1/history/ci_triggers`,
+    method: 'GET',
+    params: params
+  })
+}
+
+export function getCiTriggersByCiId(ci_id, params) {
+  return axios({
+    url: `/v0.1/history/ci_triggers/${ci_id}`,
+    method: 'GET',
+    params
+  })
+}
+
+export function getCiRelatedTickets(params) {
+  return axios({
+    url: `/itsm/v1/process_ticket/get_tickets_by`,
+    method: 'POST',
+    data: params,
+    isShowMessage: false
+  })
+}
+
+export function judgeItsmInstalled() {
+  return axios({
+    url: `/itsm/v1/process_ticket/itsm_existed`,
+    method: 'GET',
+    isShowMessage: false
+  })
+}
+
+export function getCIsBaseline(params) {
+  return axios({
+    url: `/v0.1/ci/baseline`,
+    method: 'GET',
+    params
+  })
+}
+
+export function CIBaselineRollback(ciId, params) {
+  return axios({
+    url: `/v0.1/ci/${ciId}/baseline/rollback`,
+    method: 'POST',
+    data: params
+  })
+}