From ff701cc77097934f680155dc75fbf43f2d9f7d00 Mon Sep 17 00:00:00 2001
From: pycook <pycook@126.com>
Date: Mon, 18 Nov 2019 20:02:25 +0800
Subject: [PATCH 1/6] search by elasticsearch [doing]
---
Pipfile | 1 +
api/app.py | 2 +
api/commands/click_cmdb.py | 21 ++-
api/extensions.py | 2 +
api/lib/cmdb/ci.py | 8 +-
api/lib/cmdb/search/__init__.py | 3 +
api/lib/cmdb/search/db/__init__.py | 1 +
api/lib/cmdb/{ => search/db}/query_sql.py | 0
api/lib/cmdb/{ => search/db}/search.py | 6 +-
api/lib/cmdb/search/es/__init__.py | 1 +
api/lib/cmdb/search/es/search.py | 152 ++++++++++++++++++++++
api/lib/utils.py | 45 +++++++
api/settings.py.example | 4 +
api/tasks/cmdb.py | 17 ++-
api/views/acl/resources.py | 1 +
api/views/cmdb/ci.py | 13 +-
docs/requirements.txt | 1 +
17 files changed, 256 insertions(+), 22 deletions(-)
create mode 100644 api/lib/cmdb/search/__init__.py
create mode 100644 api/lib/cmdb/search/db/__init__.py
rename api/lib/cmdb/{ => search/db}/query_sql.py (100%)
rename api/lib/cmdb/{ => search/db}/search.py (98%)
create mode 100644 api/lib/cmdb/search/es/__init__.py
create mode 100644 api/lib/cmdb/search/es/search.py
diff --git a/Pipfile b/Pipfile
index 182432c..c0a1723 100644
--- a/Pipfile
+++ b/Pipfile
@@ -40,6 +40,7 @@ bs4 = ">=0.0.1"
toposort = ">=1.5"
requests = ">=2.22.0"
PyJWT = ">=1.7.1"
+elasticsearch = "==7.0.4"
[dev-packages]
# Testing
diff --git a/api/app.py b/api/app.py
index cfd0e93..ab3ac6c 100644
--- a/api/app.py
+++ b/api/app.py
@@ -21,6 +21,7 @@ from api.extensions import (
migrate,
celery,
rd,
+ es
)
from api.flask_cas import CAS
from api.models.acl import User
@@ -98,6 +99,7 @@ def register_extensions(app):
login_manager.init_app(app)
migrate.init_app(app, db)
rd.init_app(app)
+ es.init_app(app)
celery.conf.update(app.config)
diff --git a/api/commands/click_cmdb.py b/api/commands/click_cmdb.py
index 6b2634d..3eb5be9 100644
--- a/api/commands/click_cmdb.py
+++ b/api/commands/click_cmdb.py
@@ -4,10 +4,12 @@
import json
import click
+from flask import current_app
from flask.cli import with_appcontext
import api.lib.cmdb.ci
from api.extensions import db
+from api.extensions import es
from api.extensions import rd
from api.models.cmdb import CI
@@ -19,13 +21,22 @@ def init_cache():
cis = CI.get_by(to_dict=False)
for ci in cis:
- res = rd.get([ci.id])
- if res and list(filter(lambda x: x, res)):
- continue
+ if current_app.config.get("USE_ES"):
+ res = es.get_index_id(ci.id)
+ if res:
+ continue
+ else:
+ res = rd.get([ci.id])
+ if res and list(filter(lambda x: x, res)):
+ continue
m = api.lib.cmdb.ci.CIManager()
ci_dict = m.get_ci_by_id_from_db(ci.id, need_children=False, use_master=False)
- rd.delete(ci.id)
- rd.add({ci.id: json.dumps(ci_dict)})
+
+ if current_app.config.get("USE_ES"):
+ es.create(ci_dict)
+ else:
+ rd.delete(ci.id)
+ rd.add({ci.id: json.dumps(ci_dict)})
db.session.remove()
diff --git a/api/extensions.py b/api/extensions.py
index bec8ea6..45cf662 100644
--- a/api/extensions.py
+++ b/api/extensions.py
@@ -9,6 +9,7 @@ from flask_login import LoginManager
from flask_migrate import Migrate
from flask_sqlalchemy import SQLAlchemy
+from api.lib.utils import ESHandler
from api.lib.utils import RedisHandler
bcrypt = Bcrypt()
@@ -19,3 +20,4 @@ cache = Cache()
celery = Celery()
cors = CORS(supports_credentials=True)
rd = RedisHandler(prefix="CMDB_CI") # TODO
+es = ESHandler()
diff --git a/api/lib/cmdb/ci.py b/api/lib/cmdb/ci.py
index ffaabdb..71cc93e 100644
--- a/api/lib/cmdb/ci.py
+++ b/api/lib/cmdb/ci.py
@@ -23,8 +23,8 @@ from api.lib.cmdb.const import TableMap
from api.lib.cmdb.const import type_map
from api.lib.cmdb.history import AttributeHistoryManger
from api.lib.cmdb.history import CIRelationHistoryManager
-from api.lib.cmdb.query_sql import QUERY_CIS_BY_IDS
-from api.lib.cmdb.query_sql import QUERY_CIS_BY_VALUE_TABLE
+from api.lib.cmdb.search.db.query_sql import QUERY_CIS_BY_IDS
+from api.lib.cmdb.search.db.query_sql import QUERY_CIS_BY_VALUE_TABLE
from api.lib.cmdb.value import AttributeValueManager
from api.lib.decorator import kwargs_required
from api.lib.utils import handle_arg_list
@@ -112,8 +112,8 @@ class CIManager(object):
use_master=use_master)
res.update(_res)
- res['_type'] = ci_type.id
- res['_id'] = ci_id
+ res['type_id'] = ci_type.id
+ res['ci_id'] = ci_id
return res
diff --git a/api/lib/cmdb/search/__init__.py b/api/lib/cmdb/search/__init__.py
new file mode 100644
index 0000000..3c49dd7
--- /dev/null
+++ b/api/lib/cmdb/search/__init__.py
@@ -0,0 +1,3 @@
+# -*- coding:utf-8 -*-
+
+__all__ = ['db', 'es']
diff --git a/api/lib/cmdb/search/db/__init__.py b/api/lib/cmdb/search/db/__init__.py
new file mode 100644
index 0000000..380474e
--- /dev/null
+++ b/api/lib/cmdb/search/db/__init__.py
@@ -0,0 +1 @@
+# -*- coding:utf-8 -*-
diff --git a/api/lib/cmdb/query_sql.py b/api/lib/cmdb/search/db/query_sql.py
similarity index 100%
rename from api/lib/cmdb/query_sql.py
rename to api/lib/cmdb/search/db/query_sql.py
diff --git a/api/lib/cmdb/search.py b/api/lib/cmdb/search/db/search.py
similarity index 98%
rename from api/lib/cmdb/search.py
rename to api/lib/cmdb/search/db/search.py
index 3f3a12c..eeff10f 100644
--- a/api/lib/cmdb/search.py
+++ b/api/lib/cmdb/search/db/search.py
@@ -13,9 +13,9 @@ from api.lib.cmdb.cache import CITypeCache
from api.lib.cmdb.ci import CIManager
from api.lib.cmdb.const import RetKey
from api.lib.cmdb.const import TableMap
-from api.lib.cmdb.query_sql import FACET_QUERY
-from api.lib.cmdb.query_sql import QUERY_CI_BY_ATTR_NAME
-from api.lib.cmdb.query_sql import QUERY_CI_BY_TYPE
+from api.lib.cmdb.search.db.query_sql import FACET_QUERY
+from api.lib.cmdb.search.db.query_sql import QUERY_CI_BY_ATTR_NAME
+from api.lib.cmdb.search.db.query_sql import QUERY_CI_BY_TYPE
from api.lib.utils import handle_arg_list
from api.models.cmdb import Attribute
from api.models.cmdb import CI
diff --git a/api/lib/cmdb/search/es/__init__.py b/api/lib/cmdb/search/es/__init__.py
new file mode 100644
index 0000000..380474e
--- /dev/null
+++ b/api/lib/cmdb/search/es/__init__.py
@@ -0,0 +1 @@
+# -*- coding:utf-8 -*-
diff --git a/api/lib/cmdb/search/es/search.py b/api/lib/cmdb/search/es/search.py
new file mode 100644
index 0000000..e7904d3
--- /dev/null
+++ b/api/lib/cmdb/search/es/search.py
@@ -0,0 +1,152 @@
+# -*- coding:utf-8 -*-
+
+
+from __future__ import unicode_literals
+
+from flask import current_app
+
+from api.extensions import es
+from api.lib.cmdb.cache import AttributeCache
+from api.lib.cmdb.const import RetKey
+from api.lib.utils import handle_arg_list
+from api.models.cmdb import Attribute
+
+
+class SearchError(Exception):
+ def __init__(self, v):
+ self.v = v
+
+ def __str__(self):
+ return self.v
+
+
+class Search(object):
+ def __init__(self, query=None, fl=None, facet_field=None, page=1, ret_key=RetKey.NAME, count=1, sort=None):
+ self.orig_query = query
+ self.fl = fl
+ self.facet_field = facet_field
+ self.page = page
+ self.ret_key = ret_key
+ self.count = count or current_app.config.get("DEFAULT_PAGE_COUNT")
+ self.sort = sort
+
+ self.query = dict(query=dict(bool=dict(should=[], must=[], must_not=[])))
+
+ @staticmethod
+ def _operator_proc(key):
+ operator = "&"
+ if key.startswith("+"):
+ key = key[1:].strip()
+ elif key.startswith("-"):
+ operator = "|"
+ key = key[1:].strip()
+ elif key.startswith("~"):
+ operator = "~"
+ key = key[1:].strip()
+
+ return operator, key
+
+ def _operator2query(self, operator):
+ if operator == "&":
+ return self.query['query']['bool']['must']
+ elif operator == "|":
+ return self.query['query']['bool']['should']
+ else:
+ return self.query['query']['bool']['must_not']
+
+ def _attr_name_proc(self, key):
+ operator, key = self._operator_proc(key)
+
+ if key in ('ci_type', 'type', '_type'):
+ return 'ci_type', Attribute.TEXT, operator
+
+ if key in ('id', 'ci_id', '_id'):
+ return 'ci_id', Attribute.TEXT, operator
+
+ attr = AttributeCache.get(key)
+ if attr:
+ return attr.name, attr.value_type, operator
+ else:
+ raise SearchError("{0} is not existed".format(key))
+
+ def _in_query_handle(self, attr, v, operator):
+ self._operator2query(operator).append({})
+
+ def _range_query_handle(self, attr, v, operator):
+ self._operator2query(operator).append({
+ "range": {
+ attr: {
+ "gte": 10,
+ "lte": 10,
+ }
+ }
+ })
+
+ def _comparison_query_handle(self, attr, v, operator):
+ pass
+
+ def _match_query_handle(self, attr, v, operator):
+ pass
+
+ def __query_build_by_field(self, queries):
+
+ for q in queries:
+ if ":" in q:
+ k = q.split(":")[0].strip()
+ v = ":".join(q.split(":")[1:]).strip()
+ field_name, field_type, operator = self._attr_name_proc(k)
+ if field_name:
+ # in query
+ if v.startswith("(") and v.endswith(")"):
+ self._in_query_handle(field_name, v, operator)
+ # range query
+ elif v.startswith("[") and v.endswith("]") and "_TO_" in v:
+ self._range_query_handle(field_name, v, operator)
+ # comparison query
+ elif v.startswith(">=") or v.startswith("<=") or v.startswith(">") or v.startswith("<"):
+ self._comparison_query_handle(field_name, v, operator)
+ else:
+ self._match_query_handle(field_name, v, operator)
+ else:
+ raise SearchError("argument q format invalid: {0}".format(q))
+ elif q:
+ raise SearchError("argument q format invalid: {0}".format(q))
+
+ def _query_build_raw(self):
+
+ queries = handle_arg_list(self.orig_query)
+ current_app.logger.debug(queries)
+
+ self.__query_build_by_field(queries)
+
+ self.__paginate()
+
+ filter_path = self._fl_build()
+
+ return es.read(self.query, filter_path=filter_path)
+
+ def _facet_build(self):
+ return {}
+
+ def __paginate(self):
+ self.query.update({"from": (self.page - 1) * self.count,
+ "size": self.count})
+
+ def _fl_build(self):
+ return ['hits.hits._source.{0}'.format(i) for i in self.fl]
+
+ def search(self):
+ try:
+ numfound, cis = self._query_build_raw()
+ except Exception as e:
+ current_app.logger.error(str(e))
+ raise SearchError("unknown search error")
+
+ if self.facet_field and numfound:
+ facet = self._facet_build()
+ else:
+ facet = dict()
+
+ total = len(cis)
+
+ return cis, {}, total, self.page, numfound, facet
diff --git a/api/lib/utils.py b/api/lib/utils.py
index b350a7d..a26dfec 100644
--- a/api/lib/utils.py
+++ b/api/lib/utils.py
@@ -2,6 +2,7 @@
import redis
import six
+from elasticsearch import Elasticsearch
from flask import current_app
@@ -72,3 +73,47 @@ class RedisHandler(object):
current_app.logger.warn("[{0}] is not in redis".format(key_id))
except Exception as e:
current_app.logger.error("delete redis key error, {0}".format(str(e)))
+
+
+class ESHandler(object):
+ def __init__(self, flask_app=None):
+ self.flask_app = flask_app
+ self.es = None
+ self.index = "cmdb"
+
+ def init_app(self, app):
+ self.flask_app = app
+ config = self.flask_app.config
+ self.es = Elasticsearch(config.get("ES_HOST"))
+ if not self.es.indices.exists(index=self.index):
+ self.es.indices.create(index=self.index)
+
+ def get_index_id(self, ci_id):
+ query = {
+ 'query': {
+ 'match': {'ci_id': ci_id}
+ },
+ }
+ res = self.es.search(index=self.index, body=query)
+ if res['hits']['hits']:
+ return res['hits']['hits'][-1].get('_id')
+
+ def create(self, body):
+ return self.es.index(index=self.index, body=body).get("_id")
+
+ def update(self, ci_id, body):
+ _id = self.get_index_id(ci_id)
+
+ return self.es.index(index=self.index, id=_id, body=body).get("_id")
+
+ def delete(self, ci_id):
+ _id = self.get_index_id(ci_id)
+
+ self.es.delete(index=self.index, id=_id)
+
+ def read(self, query, filter_path=None):
+ filter_path = filter_path or []
+ if filter_path:
+ filter_path.append('hits.total')
+ res = self.es.search(index=self.index, body=query, filter_path=filter_path)
+ return res['hits']['total']['value'], [i['_source'] for i in res['hits']['hits']]
diff --git a/api/settings.py.example b/api/settings.py.example
index c9315fe..fec4f5b 100644
--- a/api/settings.py.example
+++ b/api/settings.py.example
@@ -75,3 +75,7 @@ DEFAULT_PAGE_COUNT = 50
# # permission
WHITE_LIST = ["127.0.0.1"]
USE_ACL = False
+
+# # elastic search
+ES_HOST = '127.0.0.1'
+USE_ES = False
diff --git a/api/tasks/cmdb.py b/api/tasks/cmdb.py
index d48943a..78fee55 100644
--- a/api/tasks/cmdb.py
+++ b/api/tasks/cmdb.py
@@ -10,6 +10,7 @@ import api.lib.cmdb.ci
from api.extensions import celery
from api.extensions import db
from api.extensions import rd
+from api.extensions import es
from api.lib.cmdb.const import CMDB_QUEUE
@@ -20,14 +21,22 @@ def ci_cache(ci_id):
m = api.lib.cmdb.ci.CIManager()
ci = m.get_ci_by_id_from_db(ci_id, need_children=False, use_master=False)
- rd.delete(ci_id)
- rd.add({ci_id: json.dumps(ci)})
+ if current_app.config.get("USE_ES"):
+ es.update(ci_id, ci)
+ else:
+ rd.delete(ci_id)
+ rd.add({ci_id: json.dumps(ci)})
- current_app.logger.info("%d caching.........." % ci_id)
+ current_app.logger.info("%d flush.........." % ci_id)
@celery.task(name="cmdb.ci_delete", queue=CMDB_QUEUE)
def ci_delete(ci_id):
current_app.logger.info(ci_id)
- rd.delete(ci_id)
+
+ if current_app.config.get("USE_ES"):
+ es.delete(ci_id)
+ else:
+ rd.delete(ci_id)
+
current_app.logger.info("%d delete.........." % ci_id)
diff --git a/api/views/acl/resources.py b/api/views/acl/resources.py
index 3b7bae8..94241cf 100644
--- a/api/views/acl/resources.py
+++ b/api/views/acl/resources.py
@@ -60,6 +60,7 @@ class ResourceView(APIView):
class ResourceGroupView(APIView):
url_prefix = ("/resource_groups", "/resource_groups/<int:group_id>")
+ @args_required('app_id')
@validate_app
def get(self):
page = get_page(request.values.get("page", 1))
diff --git a/api/views/cmdb/ci.py b/api/views/cmdb/ci.py
index 210f86c..c2e839b 100644
--- a/api/views/cmdb/ci.py
+++ b/api/views/cmdb/ci.py
@@ -12,8 +12,9 @@ from api.lib.cmdb.ci import CIManager
from api.lib.cmdb.const import ExistPolicy
from api.lib.cmdb.const import ResourceType, PermEnum
from api.lib.cmdb.const import RetKey
-from api.lib.cmdb.search import Search
-from api.lib.cmdb.search import SearchError
+from api.lib.cmdb.search.db.search import Search as SearchFromDB
+from api.lib.cmdb.search.es.search import Search as SearchFromES
+from api.lib.cmdb.search.db.search import SearchError
from api.lib.perm.acl.acl import has_perm_from_args
from api.lib.perm.auth import auth_abandoned
from api.lib.utils import get_page
@@ -144,14 +145,14 @@ class CISearchView(APIView):
sort = request.values.get("sort")
start = time.time()
- s = Search(query, fl, facet, page, ret_key, count, sort)
+ if current_app.config.get("USE_ES"):
+ s = SearchFromES(query, fl, facet, page, ret_key, count, sort)
+ else:
+ s = SearchFromDB(query, fl, facet, page, ret_key, count, sort)
try:
response, counter, total, page, numfound, facet = s.search()
except SearchError as e:
return abort(400, str(e))
- except Exception as e:
- current_app.logger.error(str(e))
- return abort(500, "search unknown error")
current_app.logger.debug("search time is :{0}".format(time.time() - start))
return self.jsonify(numfound=numfound,
total=total,
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 9ed3509..c9f59d0 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -35,3 +35,4 @@ bs4>=0.0.1
toposort>=1.5
requests>=2.22.0
PyJWT>=1.7.1
+elasticsearch ==7.0.4
From 0e7c52df71c554098fd6244b90fea47e72c00d9a Mon Sep 17 00:00:00 2001
From: pycook <pycook@126.com>
Date: Mon, 18 Nov 2019 22:05:59 +0800
Subject: [PATCH 2/6] es search update
---
api/lib/cmdb/search/es/search.py | 91 +++++++++++++++++++++++++++-----
api/lib/utils.py | 6 ++-
2 files changed, 84 insertions(+), 13 deletions(-)
diff --git a/api/lib/cmdb/search/es/search.py b/api/lib/cmdb/search/es/search.py
index e7904d3..7752802 100644
--- a/api/lib/cmdb/search/es/search.py
+++ b/api/lib/cmdb/search/es/search.py
@@ -30,7 +30,7 @@ class Search(object):
self.count = count or current_app.config.get("DEFAULT_PAGE_COUNT")
self.sort = sort
- self.query = dict(query=dict(bool=dict(should=[], must=[], must_not=[])))
+ self.query = dict(filter=dict(bool=dict(should=[], must=[], must_not=[])))
@staticmethod
def _operator_proc(key):
@@ -69,24 +69,65 @@ class Search(object):
else:
raise SearchError("{0} is not existed".format(key))
- def _in_query_handle(self, attr, v, operator):
- self._operator2query(operator).append({})
+ def _in_query_handle(self, attr, v):
+ terms = v[1:-1].split(";")
+ operator = "|"
+ for term in terms:
+ self._operator2query(operator).append({
+ "term": {
+ attr: term
+ }
+ })
+
+ @staticmethod
+ def _digit(s):
+ if s.isdigit():
+ return int(float(s))
+ return s
def _range_query_handle(self, attr, v, operator):
+ left, right = v.split("_TO_")
+ left, right = left.strip()[1:], right.strip()[:-1]
self._operator2query(operator).append({
"range": {
attr: {
- "gte": 10,
- "lte": 10,
+ "to": self._digit(right),
+ "from": self._digit(left),
}
}
})
def _comparison_query_handle(self, attr, v, operator):
- pass
+ if v.startswith(">="):
+ _query = dict(gte=self._digit(v[2:]))
+ elif v.startswith("<="):
+ _query = dict(lte=self._digit(v[2:]))
+ elif v.startswith(">"):
+ _query = dict(gt=self._digit(v[1:]))
+ elif v.startswith("<"):
+ _query = dict(lt=self._digit(v[1:]))
+ else:
+ return
+
+ self._operator2query(operator).append({
+ "range": {
+ attr: _query
+ }
+ })
def _match_query_handle(self, attr, v, operator):
- pass
+ if "*" in v:
+ self._operator2query(operator).append({
+ "wildcard": {
+ attr: v
+ }
+ })
+ else:
+ self._operator2query(operator).append({
+ "term": {
+ attr: v
+ }
+ })
def __query_build_by_field(self, queries):
@@ -98,7 +139,7 @@ class Search(object):
if field_name:
# in query
if v.startswith("(") and v.endswith(")"):
- self._in_query_handle(field_name, v, operator)
+ self._in_query_handle(field_name, v)
# range query
elif v.startswith("[") and v.endswith("]") and "_TO_" in v:
self._range_query_handle(field_name, v, operator)
@@ -119,16 +160,42 @@ class Search(object):
self.__query_build_by_field(queries)
- self.__paginate()
+ self._paginate_build()
filter_path = self._fl_build()
- return es.read(self.query, filter_path=filter_path)
+ sort = self._sort_build()
+
+ return es.read(self.query, filter_path=filter_path, sort=sort)
def _facet_build(self):
- return {}
+ return {
+ "aggs": {
+ self.facet_field: {
+ "cardinality": {
+ "field": self.facet_field
+ }
+ }
+ }
+ }
- def __paginate(self):
+ def _sort_build(self):
+ fields = list(filter(lambda x: x != "", self.sort or ""))
+ sorts = []
+ for field in fields:
+ sort_type = "asc"
+ if field.startswith("+"):
+ field = field[1:]
+ elif field.startswith("-"):
+ field = field[1:]
+ sort_type = "desc"
+ else:
+ continue
+ sorts.append({field: {"order": sort_type}})
+
+ return sorts
+
+ def _paginate_build(self):
self.query.update({"from": (self.page - 1) * self.count,
"size": self.count})
diff --git a/api/lib/utils.py b/api/lib/utils.py
index a26dfec..1f70db4 100644
--- a/api/lib/utils.py
+++ b/api/lib/utils.py
@@ -111,9 +111,13 @@ class ESHandler(object):
self.es.delete(index=self.index, id=_id)
- def read(self, query, filter_path=None):
+ def read(self, query, filter_path=None, sort=None):
filter_path = filter_path or []
if filter_path:
filter_path.append('hits.total')
+
+ if sort:
+ query.update(dict(sort=sort))
+
res = self.es.search(index=self.index, body=query, filter_path=filter_path)
return res['hits']['total']['value'], [i['_source'] for i in res['hits']['hits']]
From 47ded842319c9177821a8ab48064892679c66c58 Mon Sep 17 00:00:00 2001
From: pycook <pycook@126.com>
Date: Tue, 19 Nov 2019 18:16:31 +0800
Subject: [PATCH 3/6] elastic search [done]
---
Dockerfile | 8 ++-
api/commands/click_cmdb.py | 24 +++++++-
api/lib/cmdb/attribute.py | 21 ++++++-
api/lib/cmdb/ci.py | 4 +-
api/lib/cmdb/const.py | 8 +++
api/lib/cmdb/search/es/search.py | 82 +++++++++++++++++---------
api/lib/utils.py | 24 ++++++--
docker-compose.yml | 31 ++++++++++
ui/src/views/cmdb/ci/index.vue | 4 +-
ui/src/views/cmdb/tree_views/index.vue | 2 +-
10 files changed, 166 insertions(+), 42 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 2414e8f..842cacb 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -35,4 +35,10 @@ RUN pip install --no-cache-dir -r docs/requirements.txt \
&& sed -i "s#{user}:{password}@127.0.0.1:3306/{db}#cmdb:123456@mysql:3306/cmdb#g" api/settings.py \
&& sed -i "s/127.0.0.1/redis/g" api/settings.py
-CMD ["bash", "-c", "flask run"]
\ No newline at end of file
+CMD ["bash", "-c", "flask run"]
+
+
+# ================================= Search ================================
+FROM docker.elastic.co/elasticsearch/elasticsearch:7.4.2 AS cmdb-search
+
+RUN yes | ./bin/elasticsearch-plugin install https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v7.4.2/elasticsearch-analysis-ik-7.4.2.zip
diff --git a/api/commands/click_cmdb.py b/api/commands/click_cmdb.py
index 3eb5be9..0bfab82 100644
--- a/api/commands/click_cmdb.py
+++ b/api/commands/click_cmdb.py
@@ -9,7 +9,6 @@ from flask.cli import with_appcontext
import api.lib.cmdb.ci
from api.extensions import db
-from api.extensions import es
from api.extensions import rd
from api.models.cmdb import CI
@@ -19,6 +18,29 @@ from api.models.cmdb import CI
def init_cache():
db.session.remove()
+ if current_app.config.get("USE_ES"):
+ from api.extensions import es
+ from api.models.cmdb import Attribute
+ from api.lib.cmdb.const import type_map
+ attributes = Attribute.get_by(to_dict=False)
+ for attr in attributes:
+ other = dict()
+ other['index'] = True if attr.is_index else False
+ if attr.value_type == Attribute.TEXT:
+ other['analyzer'] = 'ik_max_word'
+ other['search_analyzer'] = 'ik_smart'
+ if attr.is_index:
+ other["fields"] = {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ try:
+ es.update_mapping(attr.name, type_map['es_type'][attr.value_type], other)
+ except Exception as e:
+ print(e)
+
cis = CI.get_by(to_dict=False)
for ci in cis:
if current_app.config.get("USE_ES"):
diff --git a/api/lib/cmdb/attribute.py b/api/lib/cmdb/attribute.py
index 82bce55..cfef2b4 100644
--- a/api/lib/cmdb/attribute.py
+++ b/api/lib/cmdb/attribute.py
@@ -96,9 +96,8 @@ class AttributeManager(object):
name = kwargs.pop("name")
alias = kwargs.pop("alias", "")
alias = name if not alias else alias
- Attribute.get_by(name=name, first=True) and abort(400, "attribute name <{0}> is already existed".format(name))
- Attribute.get_by(alias=alias, first=True) and abort(400,
- "attribute alias <{0}> is already existed".format(name))
+ Attribute.get_by(name=name, first=True) and abort(400, "attribute name <{0}> is duplicated".format(name))
+ Attribute.get_by(alias=alias, first=True) and abort(400, "attribute alias <{0}> is duplicated".format(name))
attr = Attribute.create(flush=True,
name=name,
@@ -118,6 +117,22 @@ class AttributeManager(object):
AttributeCache.clean(attr)
+ if current_app.config.get("USE_ES"):
+ from api.extensions import es
+ other = dict()
+ other['index'] = True if attr.is_index else False
+ if attr.value_type == Attribute.TEXT:
+ other['analyzer'] = 'ik_max_word'
+ other['search_analyzer'] = 'ik_smart'
+ if attr.is_index:
+ other["fields"] = {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ es.update_mapping(name, type_map['es_type'][attr.value_type], other)
+
return attr.id
def update(self, _id, **kwargs):
diff --git a/api/lib/cmdb/ci.py b/api/lib/cmdb/ci.py
index 71cc93e..1c1ec37 100644
--- a/api/lib/cmdb/ci.py
+++ b/api/lib/cmdb/ci.py
@@ -346,8 +346,8 @@ class CIManager(object):
if ci_id not in ci_set:
ci_dict = dict()
ci_type = CITypeCache.get(type_id)
- ci_dict["_id"] = ci_id
- ci_dict["_type"] = type_id
+ ci_dict["ci_id"] = ci_id
+ ci_dict["ci_type"] = type_id
ci_dict["ci_type"] = ci_type.name
ci_dict["ci_type_alias"] = ci_type.alias
ci_set.add(ci_id)
diff --git a/api/lib/cmdb/const.py b/api/lib/cmdb/const.py
index d7e0b39..511c599 100644
--- a/api/lib/cmdb/const.py
+++ b/api/lib/cmdb/const.py
@@ -93,6 +93,14 @@ type_map = {
'index_{0}'.format(Attribute.DATE): 'c_value_index_datetime',
'index_{0}'.format(Attribute.TIME): 'c_value_index_texts',
'index_{0}'.format(Attribute.FLOAT): 'c_value_index_floats',
+ },
+ 'es_type': {
+ Attribute.INT: 'long',
+ Attribute.TEXT: 'text',
+ Attribute.DATETIME: 'text',
+ Attribute.DATE: 'text',
+ Attribute.TIME: 'text',
+ Attribute.FLOAT: 'float'
}
}
diff --git a/api/lib/cmdb/search/es/search.py b/api/lib/cmdb/search/es/search.py
index 7752802..1650fd0 100644
--- a/api/lib/cmdb/search/es/search.py
+++ b/api/lib/cmdb/search/es/search.py
@@ -28,9 +28,9 @@ class Search(object):
self.page = page
self.ret_key = ret_key
self.count = count or current_app.config.get("DEFAULT_PAGE_COUNT")
- self.sort = sort
+ self.sort = sort or "ci_id"
- self.query = dict(filter=dict(bool=dict(should=[], must=[], must_not=[])))
+ self.query = dict(query=dict(bool=dict(should=[], must=[], must_not=[])))
@staticmethod
def _operator_proc(key):
@@ -91,21 +91,22 @@ class Search(object):
self._operator2query(operator).append({
"range": {
attr: {
- "to": self._digit(right),
- "from": self._digit(left),
+ "lte": self._digit(right),
+ "gte": self._digit(left),
+ "boost": 2.0
}
}
})
def _comparison_query_handle(self, attr, v, operator):
if v.startswith(">="):
- _query = dict(gte=self._digit(v[2:]))
+ _query = dict(gte=self._digit(v[2:]), boost=2.0)
elif v.startswith("<="):
- _query = dict(lte=self._digit(v[2:]))
+ _query = dict(lte=self._digit(v[2:]), boost=2.0)
elif v.startswith(">"):
- _query = dict(gt=self._digit(v[1:]))
+ _query = dict(gt=self._digit(v[1:]), boost=2.0)
elif v.startswith("<"):
- _query = dict(lt=self._digit(v[1:]))
+ _query = dict(lt=self._digit(v[1:]), boost=2.0)
else:
return
@@ -123,6 +124,8 @@ class Search(object):
}
})
else:
+ if attr == "ci_type" and v.isdigit():
+ attr = "type_id"
self._operator2query(operator).append({
"term": {
attr: v
@@ -164,23 +167,32 @@ class Search(object):
filter_path = self._fl_build()
- sort = self._sort_build()
+ self._sort_build()
- return es.read(self.query, filter_path=filter_path, sort=sort)
+ self._facet_build()
+
+ return es.read(self.query, filter_path=filter_path)
def _facet_build(self):
- return {
- "aggs": {
- self.facet_field: {
- "cardinality": {
- "field": self.facet_field
+ aggregations = dict(aggs={})
+ for field in self.facet_field:
+ attr = AttributeCache.get(field)
+ if not attr:
+ raise SearchError("Facet by <{0}> does not exist".format(field))
+ aggregations['aggs'].update({
+ field: {
+ "terms": {
+ "field": "{0}.keyword".format(field)
+ if attr.value_type not in (Attribute.INT, Attribute.FLOAT) else field
}
}
- }
- }
+ })
+
+ if aggregations['aggs']:
+ self.query.update(aggregations)
def _sort_build(self):
- fields = list(filter(lambda x: x != "", self.sort or ""))
+ fields = list(filter(lambda x: x != "", (self.sort or "").split(",")))
sorts = []
for field in fields:
sort_type = "asc"
@@ -190,10 +202,20 @@ class Search(object):
field = field[1:]
sort_type = "desc"
else:
+ field = field
+ if field == "ci_id":
+ sorts.append({field: {"order": sort_type}})
continue
- sorts.append({field: {"order": sort_type}})
- return sorts
+ attr = AttributeCache.get(field)
+ if not attr:
+ raise SearchError("Sort by <{0}> does not exist".format(field))
+
+ sort_by = "{0}.keyword".format(field) \
+ if attr.value_type not in (Attribute.INT, Attribute.FLOAT) else field
+ sorts.append({sort_by: {"order": sort_type}})
+
+ self.query.update(dict(sort=sorts))
def _paginate_build(self):
self.query.update({"from": (self.page - 1) * self.count,
@@ -204,16 +226,22 @@ class Search(object):
def search(self):
try:
- numfound, cis = self._query_build_raw()
+ numfound, cis, facet = self._query_build_raw()
except Exception as e:
current_app.logger.error(str(e))
raise SearchError("unknown search error")
- if self.facet_field and numfound:
- facet = self._facet_build()
- else:
- facet = dict()
-
total = len(cis)
- return cis, {}, total, self.page, numfound, facet
+ counter = dict()
+ for ci in cis:
+ ci_type = ci.get("ci_type")
+ if ci_type not in counter.keys():
+ counter[ci_type] = 0
+ counter[ci_type] += 1
+
+ facet_ = dict()
+ for k in facet:
+ facet_[k] = [[i['key'], i['doc_count'], k] for i in facet[k]["buckets"]]
+
+ return cis, counter, total, self.page, numfound, facet_
diff --git a/api/lib/utils.py b/api/lib/utils.py
index 1f70db4..b3bb7cf 100644
--- a/api/lib/utils.py
+++ b/api/lib/utils.py
@@ -88,6 +88,18 @@ class ESHandler(object):
if not self.es.indices.exists(index=self.index):
self.es.indices.create(index=self.index)
+ def update_mapping(self, field, value_type, other):
+ body = {
+ "properties": {
+ field: {"type": value_type},
+ }}
+ body['properties'][field].update(other)
+
+ self.es.indices.put_mapping(
+ index=self.index,
+ body=body
+ )
+
def get_index_id(self, ci_id):
query = {
'query': {
@@ -111,13 +123,15 @@ class ESHandler(object):
self.es.delete(index=self.index, id=_id)
- def read(self, query, filter_path=None, sort=None):
+ def read(self, query, filter_path=None):
filter_path = filter_path or []
if filter_path:
filter_path.append('hits.total')
- if sort:
- query.update(dict(sort=sort))
-
res = self.es.search(index=self.index, body=query, filter_path=filter_path)
- return res['hits']['total']['value'], [i['_source'] for i in res['hits']['hits']]
+ if res['hits'].get('hits'):
+ return res['hits']['total']['value'], \
+ [i['_source'] for i in res['hits']['hits']], \
+ res.get("aggregations", {})
+ else:
+ return 0, [], {}
diff --git a/docker-compose.yml b/docker-compose.yml
index b421b3e..8351480 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -26,6 +26,33 @@ services:
aliases:
- redis
+ cmdb-search:
+ image: registry.cn-qingdao.aliyuncs.com/pycook/cmdb-search:1.2
+# build:
+# context: .
+# target: cmdb-search
+ container_name: cmdb-search
+ environment:
+ - discovery.type=single-node
+ - cluster.name=docker-cluster
+ - bootstrap.memory_lock=true
+ - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+ ulimits:
+ memlock:
+ soft: -1
+ hard: -1
+ nofile:
+ soft: 65536
+ hard: 65536
+ volumes:
+ - esdata:/usr/share/elasticsearch/data
+ ports:
+ - 9200:9200
+ networks:
+ new:
+ aliases:
+ - cmdb-search
+
cmdb-api:
image: registry.cn-qingdao.aliyuncs.com/pycook/cmdb-api:1.0
container_name: cmdb-api
@@ -35,12 +62,15 @@ services:
- /bin/sh
- -c
- |
+ sed -i "s#USE_ES = False#USE_ES = True#g" api/settings.py
+ sed -i "s#ES_HOST = '127.0.0.1'#ES_HOST = cmdb-search#g" api/settings.py
gunicorn --workers=3 autoapp:app -b 0.0.0.0:5000 -D
flask init-cache
celery worker -A celery_worker.celery -E -Q cmdb_async --concurrency=1
depends_on:
- cmdb-db
- cmdb-cache
+ - cmdb-search
networks:
new:
aliases:
@@ -70,6 +100,7 @@ services:
volumes:
db-data:
+ esdata:
networks:
new:
\ No newline at end of file
diff --git a/ui/src/views/cmdb/ci/index.vue b/ui/src/views/cmdb/ci/index.vue
index 678f033..63d2007 100644
--- a/ui/src/views/cmdb/ci/index.vue
+++ b/ui/src/views/cmdb/ci/index.vue
@@ -40,7 +40,7 @@
bordered
ref="table"
size="middle"
- rowKey="_id"
+ rowKey="ci_id"
:columns="columns"
:data="loadInstances"
:alert="options.alert"
@@ -158,7 +158,7 @@ export default {
result.totalCount = res.numfound
result.totalPage = Math.ceil(res.numfound / params.pageSize)
result.data = Object.assign([], res.result)
- result.data.forEach((item, index) => (item.key = item._id))
+ result.data.forEach((item, index) => (item.key = item.ci_id))
setTimeout(() => {
this.setColumnWidth()
}, 200)
diff --git a/ui/src/views/cmdb/tree_views/index.vue b/ui/src/views/cmdb/tree_views/index.vue
index e2f9d00..805d2bc 100644
--- a/ui/src/views/cmdb/tree_views/index.vue
+++ b/ui/src/views/cmdb/tree_views/index.vue
@@ -105,7 +105,7 @@ export default {
result.totalCount = res.numfound
result.totalPage = Math.ceil(res.numfound / (params.pageSize || 25))
result.data = Object.assign([], res.result)
- result.data.forEach((item, index) => (item.key = item._id))
+ result.data.forEach((item, index) => (item.key = item.ci_id))
setTimeout(() => {
this.setColumnWidth()
}, 200)
From a1f63b00ddf4c89f43bb57697af091fcc3072e45 Mon Sep 17 00:00:00 2001
From: pycook <pycook@126.com>
Date: Tue, 19 Nov 2019 18:32:35 +0800
Subject: [PATCH 4/6] fix search
---
api/app.py | 3 ++-
ui/src/views/cmdb/attributes/index.vue | 1 -
ui/src/views/cmdb/ci/index.vue | 2 +-
ui/src/views/cmdb/ci/modules/CiDetail.vue | 4 ++--
ui/src/views/cmdb/tree_views/index.vue | 4 ++--
5 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/api/app.py b/api/app.py
index ab3ac6c..068a978 100644
--- a/api/app.py
+++ b/api/app.py
@@ -99,7 +99,8 @@ def register_extensions(app):
login_manager.init_app(app)
migrate.init_app(app, db)
rd.init_app(app)
- es.init_app(app)
+ if app.config.get("USE_ES"):
+ es.init_app(app)
celery.conf.update(app.config)
diff --git a/ui/src/views/cmdb/attributes/index.vue b/ui/src/views/cmdb/attributes/index.vue
index ce00592..a3fa4b7 100644
--- a/ui/src/views/cmdb/attributes/index.vue
+++ b/ui/src/views/cmdb/attributes/index.vue
@@ -75,7 +75,6 @@
<a-popconfirm
title="确认删除?"
@confirm="handleDelete(record)"
- @cancel="cancel"
okText="是"
cancelText="否"
>
diff --git a/ui/src/views/cmdb/ci/index.vue b/ui/src/views/cmdb/ci/index.vue
index 63d2007..39f6daf 100644
--- a/ui/src/views/cmdb/ci/index.vue
+++ b/ui/src/views/cmdb/ci/index.vue
@@ -324,7 +324,7 @@ export default {
return searchCI(`q=_id:${ciId}`).then(res => {
const ciMap = {}
Object.keys(res.result[0]).forEach(k => {
- if (!['ci_type', '_id', 'ci_type_alias', '_type'].includes(k)) {
+ if (!['ci_type', 'ci_id', 'ci_type_alias', 'type_id'].includes(k)) {
ciMap[k] = res.result[0][k]
}
})
diff --git a/ui/src/views/cmdb/ci/modules/CiDetail.vue b/ui/src/views/cmdb/ci/modules/CiDetail.vue
index 811b477..317dd8d 100644
--- a/ui/src/views/cmdb/ci/modules/CiDetail.vue
+++ b/ui/src/views/cmdb/ci/modules/CiDetail.vue
@@ -33,7 +33,7 @@
<div v-for="parent in parentCITypes" :key="'ctr_' + parent.ctr_id">
<a-card type="inner" :title="parent.alias || parent.name" v-if="firstCIs[parent.name]">
<a-table
- rowKey="_id"
+ rowKey="ci_id"
size="middle"
:columns="firstCIColumns[child.id]"
:dataSource="firstCIs[child.name]"
@@ -45,7 +45,7 @@
<div v-for="child in childCITypes" :key="'ctr_' + child.ctr_id">
<a-card type="inner" :title="child.alias || child.name" v-if="secondCIs[child.name]">
<a-table
- rowKey="_id"
+ rowKey="ci_id"
size="middle"
:columns="secondCIColumns[child.id]"
:dataSource="secondCIs[child.name]"
diff --git a/ui/src/views/cmdb/tree_views/index.vue b/ui/src/views/cmdb/tree_views/index.vue
index 805d2bc..69c92c6 100644
--- a/ui/src/views/cmdb/tree_views/index.vue
+++ b/ui/src/views/cmdb/tree_views/index.vue
@@ -20,7 +20,7 @@
bordered
ref="table"
size="middle"
- rowKey="_id"
+ rowKey="ci_id"
:columns="columns"
:data="loadInstances"
:scroll="{ x: scrollX, y: scrollY }"
@@ -203,7 +203,7 @@ export default {
this.current = [this.typeId]
this.loadColumns()
this.levels = res.find(item => item.id === this.typeId).levels
- this.$refs.table.refresh(true)
+ this.$refs.table && this.$refs.table.refresh(true)
}
})
},
From e5baa5012d4754092791eb1bce905e41f6dcbee1 Mon Sep 17 00:00:00 2001
From: pycook <pycook@126.com>
Date: Tue, 19 Nov 2019 21:41:46 +0800
Subject: [PATCH 5/6] acl: resource type api
---
api/lib/perm/acl/user.py | 6 +++---
api/views/acl/resources.py | 43 ++++++++++++++++++++++++++++++++++++++
2 files changed, 46 insertions(+), 3 deletions(-)
diff --git a/api/lib/perm/acl/user.py b/api/lib/perm/acl/user.py
index 178502c..fd152db 100644
--- a/api/lib/perm/acl/user.py
+++ b/api/lib/perm/acl/user.py
@@ -43,10 +43,10 @@ class UserCRUD(object):
return User.create(**kwargs)
@staticmethod
- def update(rid, **kwargs):
- user = User.get_by_id(rid) or abort(404, "User <{0}> does not exist".format(rid))
+ def update(uid, **kwargs):
+ user = User.get_by_id(uid) or abort(404, "User <{0}> does not exist".format(uid))
- UserCache.clean(rid)
+ UserCache.clean(uid)
return user.update(**kwargs)
diff --git a/api/views/acl/resources.py b/api/views/acl/resources.py
index 94241cf..72876a5 100644
--- a/api/views/acl/resources.py
+++ b/api/views/acl/resources.py
@@ -6,12 +6,55 @@ from api.lib.decorator import args_required
from api.lib.perm.acl import validate_app
from api.lib.perm.acl.resource import ResourceCRUD
from api.lib.perm.acl.resource import ResourceGroupCRUD
+from api.lib.perm.acl.resource import ResourceTypeCRUD
from api.lib.utils import get_page
from api.lib.utils import get_page_size
from api.lib.utils import handle_arg_list
from api.resource import APIView
+class ResourceTypeView(APIView):
+ url_prefix = ("/resource_types", "/resource_types/<int:type_id>")
+
+ @args_required('app_id')
+ @validate_app
+ def get(self):
+ page = get_page(request.values.get("page", 1))
+ page_size = get_page_size(request.values.get("page_size"))
+ q = request.values.get('q')
+ app_id = request.values.get('app_id')
+
+ numfound, res = ResourceTypeCRUD.search(q, app_id, page, page_size)
+
+ return self.jsonify(numfound=numfound,
+ page=page,
+ page_size=page_size,
+ groups=[i.to_dict() for i in res])
+
+ @args_required('name')
+ @args_required('app_id')
+ @args_required('perms')
+ @validate_app
+ def post(self):
+ name = request.values.get('name')
+ app_id = request.values.get('app_id')
+ perms = request.values.get('perms')
+
+ rt = ResourceTypeCRUD.add(name, app_id, perms)
+
+ return self.jsonify(rt.to_dict())
+
+ def put(self, type_id):
+ rt = ResourceTypeCRUD.update(type_id, **request.values)
+
+ return self.jsonify(rt.to_dict())
+
+ def delete(self, type_id):
+ ResourceTypeCRUD.delete(type_id)
+
+ return self.jsonify(type_id=type_id)
+
+
class ResourceView(APIView):
url_prefix = ("/resources", "/resources/<int:resource_id>")
From d3a8ef5966a33d604a6e5914be6668c3d4ec0e32 Mon Sep 17 00:00:00 2001
From: pycook <pycook@126.com>
Date: Tue, 19 Nov 2019 21:46:53 +0800
Subject: [PATCH 6/6] fix get user by uid
---
api/app.py | 2 +-
api/lib/perm/acl/user.py | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/api/app.py b/api/app.py
index 068a978..95d6059 100644
--- a/api/app.py
+++ b/api/app.py
@@ -34,7 +34,7 @@ API_PACKAGE = "api"
@login_manager.user_loader
def load_user(user_id):
"""Load user by ID."""
- return User.get_by_id(int(user_id))
+ return User.get_by(uid=int(user_id), first=True, to_dict=False)
class ReverseProxy(object):
diff --git a/api/lib/perm/acl/user.py b/api/lib/perm/acl/user.py
index fd152db..78396ff 100644
--- a/api/lib/perm/acl/user.py
+++ b/api/lib/perm/acl/user.py
@@ -44,7 +44,7 @@ class UserCRUD(object):
@staticmethod
def update(uid, **kwargs):
- user = User.get_by_id(uid) or abort(404, "User <{0}> does not exist".format(uid))
+ user = User.get_by(uid=uid, to_dict=False, first=True) or abort(404, "User <{0}> does not exist".format(uid))
UserCache.clean(uid)
@@ -59,7 +59,7 @@ class UserCRUD(object):
@classmethod
def delete(cls, uid):
- user = User.get_by_id(uid) or abort(404, "User <{0}> does not exist".format(uid))
+ user = User.get_by(uid=uid, to_dict=False, first=True) or abort(404, "User <{0}> does not exist".format(uid))
UserCache.clean(user)