7x31/cmdb
1
0
Fork 0
mirror of https://github.com/veops/cmdb.git synced 2025-08-09 01:02:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

前后端全面升级

Browse Source
This commit is contained in:
pycook
2023-07-10 17:42:15 +08:00
parent c444fed436
commit 109d4f1a2e
629 changed files with 97789 additions and 23995 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
cmdb-api/api/lib/cmdb/search/ci/__init__.py
View File

@@ -1,3 +1,25 @@
# -*- coding:utf-8 -*-
__all__ = ['db', 'es']
__all__ = ['db', 'es', 'search']
from flask import current_app
from api.lib.cmdb.const import RetKey
from api.lib.cmdb.search.ci.db.search import Search as SearchFromDB
from api.lib.cmdb.search.ci.es.search import Search as SearchFromES
def search(query=None,
fl=None,
facet=None,
page=1,
ret_key=RetKey.NAME,
count=1,
sort=None,
excludes=None):
if current_app.config.get("USE_ES"):
s = SearchFromES(query, fl, facet, page, ret_key, count, sort)
else:
s = SearchFromDB(query, fl, facet, page, ret_key, count, sort, excludes=excludes)
return s

43
cmdb-api/api/lib/cmdb/search/ci/db/query_sql.py
View File

@@ -2,7 +2,6 @@
from __future__ import unicode_literals
QUERY_CIS_BY_VALUE_TABLE = """
SELECT attr.name AS attr_name,
attr.alias AS attr_alias,
@@ -58,9 +57,51 @@ QUERY_CI_BY_ATTR_NAME = """
AND {0}.value {2}
"""
QUERY_CI_BY_ID = """
SELECT c_cis.id as ci_id
FROM c_cis
WHERE c_cis.id={}
"""
QUERY_CI_BY_TYPE = """
SELECT c_cis.id AS ci_id
FROM c_cis
WHERE c_cis.type_id in ({0})
"""
QUERY_UNION_CI_ATTRIBUTE_IS_NULL = """
SELECT *
FROM (
SELECT c_cis.id AS ci_id
FROM c_cis
WHERE c_cis.type_id IN ({0})
) {3}
LEFT JOIN (
SELECT {1}.ci_id
FROM {1}
WHERE {1}.attr_id = {2}
AND {1}.value LIKE "%"
) {4} USING (ci_id)
WHERE {4}.ci_id IS NULL
"""
QUERY_CI_BY_NO_ATTR = """
SELECT *
FROM
(SELECT c_value_index_texts.ci_id
FROM c_value_index_texts
WHERE c_value_index_texts.value LIKE "{0}"
UNION
SELECT c_value_index_integers.ci_id
FROM c_value_index_integers
WHERE c_value_index_integers.value LIKE "{0}"
UNION
SELECT c_value_index_floats.ci_id
FROM c_value_index_floats
WHERE c_value_index_floats.value LIKE "{0}"
UNION
SELECT c_value_index_datetime.ci_id
FROM c_value_index_datetime
WHERE c_value_index_datetime.value LIKE "{0}") AS {1}
GROUP BY {1}.ci_id
"""

311
cmdb-api/api/lib/cmdb/search/ci/db/search.py
View File

@@ -3,23 +3,33 @@
from __future__ import unicode_literals
import copy
import time
from flask import current_app
from flask import g
from jinja2 import Template
from api.extensions import db
from api.lib.cmdb.cache import AttributeCache
from api.lib.cmdb.cache import CITypeCache
from api.lib.cmdb.ci import CIManager
from api.lib.cmdb.const import PermEnum
from api.lib.cmdb.const import ResourceTypeEnum
from api.lib.cmdb.const import RetKey
from api.lib.cmdb.const import ValueTypeEnum
from api.lib.cmdb.perms import CIFilterPermsCRUD
from api.lib.cmdb.resp_format import ErrFormat
from api.lib.cmdb.search import SearchError
from api.lib.cmdb.search.ci.db.query_sql import FACET_QUERY
from api.lib.cmdb.search.ci.db.query_sql import QUERY_CI_BY_ATTR_NAME
from api.lib.cmdb.search.ci.db.query_sql import QUERY_CI_BY_ID
from api.lib.cmdb.search.ci.db.query_sql import QUERY_CI_BY_NO_ATTR
from api.lib.cmdb.search.ci.db.query_sql import QUERY_CI_BY_TYPE
from api.lib.cmdb.search.ci.db.query_sql import QUERY_UNION_CI_ATTRIBUTE_IS_NULL
from api.lib.cmdb.utils import TableMap
from api.lib.perm.acl.acl import ACLManager
from api.lib.perm.acl.acl import is_app_admin
from api.lib.utils import handle_arg_list
from api.models.cmdb import CI
class Search(object):
@@ -30,9 +40,11 @@ class Search(object):
ret_key=RetKey.NAME,
count=1,
sort=None,
ci_ids=None):
ci_ids=None,
excludes=None):
self.orig_query = query
self.fl = fl
self.fl = fl or []
self.excludes = excludes or []
self.facet_field = facet_field
self.page = page
self.ret_key = ret_key
@@ -43,11 +55,17 @@ class Search(object):
self.type_id_list = []
self.only_type_query = False
self.valid_type_names = []
self.type2filter_perms = dict()
@staticmethod
def _operator_proc(key):
operator = "&"
if key.startswith("+"):
key = key[1:].strip()
elif key.startswith("-~"):
operator = "|~"
key = key[2:].strip()
elif key.startswith("-"):
operator = "|"
key = key[1:].strip()
@@ -70,14 +88,41 @@ class Search(object):
if attr:
return attr.name, attr.value_type, operator, attr
else:
raise SearchError("{0} is not existed".format(key))
raise SearchError(ErrFormat.attribute_not_found.format(key))
def _type_query_handler(self, v):
def _type_query_handler(self, v, queries):
new_v = v[1:-1].split(";") if v.startswith("(") and v.endswith(")") else [v]
for _v in new_v:
ci_type = CITypeCache.get(_v)
if len(new_v) == 1 and not self.sort and ci_type.default_order_attr:
self.sort = ci_type.default_order_attr
if ci_type is not None:
self.type_id_list.append(str(ci_type.id))
if self.valid_type_names == "ALL" or ci_type.name in self.valid_type_names:
self.type_id_list.append(str(ci_type.id))
if ci_type.id in self.type2filter_perms:
ci_filter = self.type2filter_perms[ci_type.id].get('ci_filter')
if ci_filter:
sub = []
ci_filter = Template(ci_filter).render(user=g.user)
for i in ci_filter.split(','):
if i.startswith("~") and not sub:
queries.append(i)
else:
sub.append(i)
if sub:
queries.append(dict(operator="&", queries=sub))
if self.type2filter_perms[ci_type.id].get('attr_filter'):
if not self.fl:
self.fl = set(self.type2filter_perms[ci_type.id]['attr_filter'])
else:
self.fl = set(self.fl) & set(self.type2filter_perms[ci_type.id]['attr_filter'])
else:
raise SearchError(ErrFormat.no_permission.format(ci_type.alias, PermEnum.READ))
else:
raise SearchError(ErrFormat.ci_type_not_found2.format(_v))
if self.type_id_list:
type_ids = ",".join(self.type_id_list)
@@ -89,24 +134,32 @@ class Search(object):
return ""
@staticmethod
def _in_query_handler(attr, v):
def _id_query_handler(v):
return QUERY_CI_BY_ID.format(v)
@staticmethod
def _in_query_handler(attr, v, is_not):
new_v = v[1:-1].split(";")
table_name = TableMap(attr_name=attr.name).table_name
in_query = " OR {0}.value ".format(table_name).join(['LIKE "{0}"'.format(_v.replace("*", "%")) for _v in new_v])
table_name = TableMap(attr=attr).table_name
in_query = " OR {0}.value ".format(table_name).join(['{0} "{1}"'.format(
"NOT LIKE" if is_not else "LIKE",
_v.replace("*", "%")) for _v in new_v])
_query_sql = QUERY_CI_BY_ATTR_NAME.format(table_name, attr.id, in_query)
return _query_sql
@staticmethod
def _range_query_handler(attr, v):
def _range_query_handler(attr, v, is_not):
start, end = [x.strip() for x in v[1:-1].split("_TO_")]
table_name = TableMap(attr_name=attr.name).table_name
range_query = "BETWEEN '{0}' AND '{1}'".format(start.replace("*", "%"), end.replace("*", "%"))
table_name = TableMap(attr=attr).table_name
range_query = "{0} '{1}' AND '{2}'".format(
"NOT BETWEEN" if is_not else "BETWEEN",
start.replace("*", "%"), end.replace("*", "%"))
_query_sql = QUERY_CI_BY_ATTR_NAME.format(table_name, attr.id, range_query)
return _query_sql
@staticmethod
def _comparison_query_handler(attr, v):
table_name = TableMap(attr_name=attr.name).table_name
table_name = TableMap(attr=attr).table_name
if v.startswith(">=") or v.startswith("<="):
comparison_query = "{0} '{1}'".format(v[:2], v[2:].replace("*", "%"))
else:
@@ -154,18 +207,44 @@ class Search(object):
"INNER JOIN c_cis on c_cis.id=B.ci_id "
"ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format((self.page - 1) * self.count, sort_type, self.count))
def __sort_by_type(self, sort_type, query_sql):
ret_sql = "SELECT SQL_CALC_FOUND_ROWS DISTINCT B.ci_id FROM ({0}) AS B {1}"
if self.type_id_list:
self.query_sql = "SELECT B.ci_id FROM ({0}) AS B {1}".format(
query_sql,
"INNER JOIN c_cis on c_cis.id=B.ci_id WHERE c_cis.type_id IN ({0}) ".format(
",".join(self.type_id_list)))
return ret_sql.format(
query_sql,
"INNER JOIN c_cis on c_cis.id=B.ci_id WHERE c_cis.type_id IN ({3}) "
"ORDER BY c_cis.type_id {1} LIMIT {0:d}, {2};".format(
(self.page - 1) * self.count, sort_type, self.count, ",".join(self.type_id_list)))
else:
self.query_sql = "SELECT B.ci_id FROM ({0}) AS B {1}".format(
query_sql,
"INNER JOIN c_cis on c_cis.id=B.ci_id ")
return ret_sql.format(
query_sql,
"INNER JOIN c_cis on c_cis.id=B.ci_id "
"ORDER BY c_cis.type_id {1} LIMIT {0:d}, {2};".format(
(self.page - 1) * self.count, sort_type, self.count))
def __sort_by_field(self, field, sort_type, query_sql):
attr = AttributeCache.get(field)
attr_id = attr.id
table_name = TableMap(attr_name=attr.name).table_name
table_name = TableMap(attr=attr).table_name
_v_query_sql = """SELECT {0}.ci_id, {1}.value
FROM ({2}) AS {0} INNER JOIN {1} ON {1}.ci_id = {0}.ci_id
WHERE {1}.attr_id = {3}""".format("ALIAS", table_name, query_sql, attr_id)
new_table = _v_query_sql
if self.only_type_query or not self.type_id_list:
return "SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id, C.value " \
return "SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id " \
"FROM ({0}) AS C " \
"ORDER BY C.value {2} " \
"LIMIT {1:d}, {3};".format(new_table, (self.page - 1) * self.count, sort_type, self.count)
@@ -176,7 +255,7 @@ class Search(object):
INNER JOIN c_cis on c_cis.id=C.ci_id
WHERE c_cis.type_id IN ({1})""".format(new_table, ",".join(self.type_id_list))
return """SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id, C.value
return """SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id
FROM ({0}) AS C
INNER JOIN c_cis on c_cis.id=C.ci_id
WHERE c_cis.type_id IN ({4})
@@ -192,6 +271,8 @@ class Search(object):
if field in ("_id", "ci_id") or not field:
return self.__sort_by_id(sort_type, query_sql)
elif field in ("_type", "ci_type"):
return self.__sort_by_type(sort_type, query_sql)
else:
return self.__sort_by_field(field, sort_type, query_sql)
@@ -201,7 +282,7 @@ class Search(object):
query_sql = """SELECT * FROM ({0}) as {1}
INNER JOIN ({2}) as {3} USING(ci_id)""".format(query_sql, alias, _query_sql, alias + "A")
elif operator == "|":
elif operator == "|" or operator == "|~":
query_sql = "SELECT * FROM ({0}) as {1} UNION ALL ({2})".format(query_sql, alias, _query_sql)
elif operator == "~":
@@ -225,54 +306,142 @@ class Search(object):
return numfound, res
def __get_types_has_read(self):
"""
:return: _type:(type1;type2)
"""
acl = ACLManager('cmdb')
res = acl.get_resources(ResourceTypeEnum.CI)
self.valid_type_names = {i['name'] for i in res if PermEnum.READ in i['permissions']}
res2 = acl.get_resources(ResourceTypeEnum.CI_FILTER)
if res2:
self.type2filter_perms = CIFilterPermsCRUD().get_by_ids(list(map(int, [i['name'] for i in res2])))
return "_type:({})".format(";".join(self.valid_type_names))
def __confirm_type_first(self, queries):
has_type = False
result = []
sub = {}
id_query = None
for q in queries:
if q.startswith("_type"):
queries.remove(q)
queries.insert(0, q)
has_type = True
result.insert(0, q)
if len(queries) == 1 or queries[1].startswith("-") or queries[1].startswith("~"):
self.only_type_query = True
return queries
elif q.startswith("_id") and len(q.split(':')) == 2:
id_query = int(q.split(":")[1]) if q.split(":")[1].isdigit() else None
result.append(q)
elif q.startswith("(") or q[1:].startswith("(") or q[2:].startswith("("):
if not q.startswith("("):
raise SearchError(ErrFormat.ci_search_Parentheses_invalid)
def __query_build_by_field(self, queries):
query_sql, alias, operator = "", "A", "&"
is_first, only_type_query_special = True, True
operator, q = self._operator_proc(q)
if q.endswith(")"):
result.append(dict(operator=operator, queries=[q[1:-1]]))
sub = dict(operator=operator, queries=[q[1:]])
elif q.endswith(")") and sub:
sub['queries'].append(q[:-1])
result.append(copy.deepcopy(sub))
sub = {}
elif sub:
sub['queries'].append(q)
else:
result.append(q)
_is_app_admin = is_app_admin('cmdb') or g.user.username == "worker"
if result and not has_type and not _is_app_admin:
type_q = self.__get_types_has_read()
if id_query:
ci = CIManager.get_by_id(id_query)
if not ci:
raise SearchError(ErrFormat.ci_not_found.format(id_query))
result.insert(0, "_type:{}".format(ci.type_id))
else:
result.insert(0, type_q)
elif _is_app_admin:
self.valid_type_names = "ALL"
else:
self.__get_types_has_read()
current_app.logger.warning(result)
return result
def __query_by_attr(self, q, queries, alias):
k = q.split(":")[0].strip()
v = "\:".join(q.split(":")[1:]).strip()
v = v.replace("'", "\\'")
v = v.replace('"', '\\"')
field, field_type, operator, attr = self._attr_name_proc(k)
if field == "_type":
_query_sql = self._type_query_handler(v, queries)
elif field == "_id":
_query_sql = self._id_query_handler(v)
elif field:
if attr is None:
raise SearchError(ErrFormat.attribute_not_found.format(field))
is_not = True if operator == "|~" else False
# in query
if v.startswith("(") and v.endswith(")"):
_query_sql = self._in_query_handler(attr, v, is_not)
# range query
elif v.startswith("[") and v.endswith("]") and "_TO_" in v:
_query_sql = self._range_query_handler(attr, v, is_not)
# comparison query
elif v.startswith(">=") or v.startswith("<=") or v.startswith(">") or v.startswith("<"):
_query_sql = self._comparison_query_handler(attr, v)
else:
table_name = TableMap(attr=attr).table_name
if is_not and v == "*" and self.type_id_list: # special handle
_query_sql = QUERY_UNION_CI_ATTRIBUTE_IS_NULL.format(
",".join(self.type_id_list),
table_name,
attr.id,
alias,
alias + 'A'
)
alias += "AA"
else:
_query_sql = QUERY_CI_BY_ATTR_NAME.format(
table_name,
attr.id,
'{0} "{1}"'.format("NOT LIKE" if is_not else "LIKE", v.replace("*", "%")))
else:
raise SearchError(ErrFormat.argument_invalid.format("q"))
return alias, _query_sql, operator
def __query_build_by_field(self, queries, is_first=True, only_type_query_special=True, alias='A', operator='&'):
query_sql = ""
for q in queries:
_query_sql = ""
if ":" in q:
k = q.split(":")[0].strip()
v = "\:".join(q.split(":")[1:]).strip()
current_app.logger.debug(v)
field, field_type, operator, attr = self._attr_name_proc(k)
if field == "_type":
_query_sql = self._type_query_handler(v)
current_app.logger.debug(_query_sql)
elif field == "_id": # exclude all others
ci = CI.get_by_id(v)
if ci is not None:
return 1, [str(v)]
elif field:
if attr is None:
raise SearchError("{0} is not found".format(field))
if isinstance(q, dict):
alias, _query_sql, operator = self.__query_build_by_field(q['queries'], True, True, alias)
current_app.logger.info(_query_sql)
current_app.logger.info((operator, is_first, alias))
operator = q['operator']
# in query
if v.startswith("(") and v.endswith(")"):
_query_sql = self._in_query_handler(attr, v)
# range query
elif v.startswith("[") and v.endswith("]") and "_TO_" in v:
_query_sql = self._range_query_handler(attr, v)
# comparison query
elif v.startswith(">=") or v.startswith("<=") or v.startswith(">") or v.startswith("<"):
_query_sql = self._comparison_query_handler(attr, v)
else:
table_name = TableMap(attr_name=attr.name).table_name
_query_sql = QUERY_CI_BY_ATTR_NAME.format(
table_name, attr.id, 'LIKE "{0}"'.format(v.replace("*", "%")))
else:
raise SearchError("argument q format invalid: {0}".format(q))
elif ":" in q and not q.startswith("*"):
alias, _query_sql, operator = self.__query_by_attr(q, queries, alias)
elif q == "*":
continue
elif q:
raise SearchError("argument q format invalid: {0}".format(q))
q = q.replace("'", "\\'")
q = q.replace('"', '\\"')
q = q.replace("*", "%").replace('\\n', '%')
_query_sql = QUERY_CI_BY_NO_ATTR.format(q, alias)
if is_first and _query_sql and not self.only_type_query:
query_sql = "SELECT * FROM ({0}) AS {1}".format(_query_sql, alias)
@@ -285,7 +454,8 @@ class Search(object):
elif _query_sql:
query_sql = self._wrap_sql(operator, alias, _query_sql, query_sql)
alias += "AA"
return None, query_sql
return alias, query_sql, operator
def _filter_ids(self, query_sql):
if self.ci_ids:
@@ -294,21 +464,36 @@ class Search(object):
return query_sql
@staticmethod
def _extra_handle_query_expr(args): # \, or ,
result = []
if args:
result.append(args[0])
for arg in args[1:]:
if result[-1].endswith('\\'):
result[-1] = ",".join([result[-1].rstrip('\\'), arg])
# elif ":" not in arg:
# result[-1] = ",".join([result[-1], arg])
else:
result.append(arg)
return result
def _query_build_raw(self):
queries = handle_arg_list(self.orig_query)
queries = self._extra_handle_query_expr(queries)
queries = self.__confirm_type_first(queries)
current_app.logger.debug(queries)
ret, query_sql = self.__query_build_by_field(queries)
if ret is not None:
return ret, query_sql
_, query_sql, _ = self.__query_build_by_field(queries)
s = time.time()
if query_sql:
query_sql = self._filter_ids(query_sql)
self.query_sql = query_sql
current_app.logger.debug(query_sql)
# current_app.logger.debug(query_sql)
numfound, res = self._execute_sql(query_sql)
current_app.logger.debug("query ci ids is: {0}".format(time.time() - s))
return numfound, [_res[0] for _res in res]
@@ -320,9 +505,9 @@ class Search(object):
for f in self.facet_field:
k, field_type, _, attr = self._attr_name_proc(f)
if k:
table_name = TableMap(attr_name=k).table_name
table_name = TableMap(attr=attr).table_name
query_sql = FACET_QUERY.format(table_name, self.query_sql, attr.id)
current_app.logger.debug(query_sql)
# current_app.logger.debug(query_sql)
result = db.session.execute(query_sql).fetchall()
facet[k] = result
@@ -356,7 +541,7 @@ class Search(object):
response, counter = [], {}
if ci_ids:
response = CIManager.get_cis_by_ids(ci_ids, ret_key=self.ret_key, fields=_fl)
response = CIManager.get_cis_by_ids(ci_ids, ret_key=self.ret_key, fields=_fl, excludes=self.excludes)
for res in response:
ci_type = res.get("ci_type")
if ci_type not in counter.keys():

147
cmdb-api/api/lib/cmdb/search/ci/es/search.py
View File

@@ -10,6 +10,7 @@ from api.extensions import es
from api.lib.cmdb.cache import AttributeCache
from api.lib.cmdb.const import RetKey
from api.lib.cmdb.const import ValueTypeEnum
from api.lib.cmdb.resp_format import ErrFormat
from api.lib.cmdb.search import SearchError
from api.lib.utils import handle_arg_list
@@ -22,9 +23,11 @@ class Search(object):
ret_key=RetKey.NAME,
count=1,
sort=None,
ci_ids=None):
ci_ids=None,
excludes=None):
self.orig_query = query
self.fl = fl
self.fl = fl or []
self.excludes = excludes or []
self.facet_field = facet_field
self.page = page
self.ret_key = ret_key
@@ -39,6 +42,9 @@ class Search(object):
operator = "&"
if key.startswith("+"):
key = key[1:].strip()
elif key.startswith("-~"):
operator = "|~"
key = key[2:].strip()
elif key.startswith("-"):
operator = "|"
key = key[1:].strip()
@@ -53,6 +59,8 @@ class Search(object):
return self.query['query']['bool']['must']
elif operator == "|":
return self.query['query']['bool']['should']
elif operator == "|~":
return self.query['query']['bool']['should']
else:
return self.query['query']['bool']['must_not']
@@ -69,9 +77,9 @@ class Search(object):
if attr:
return attr.name, attr.value_type, operator
else:
raise SearchError("{0} is not existed".format(key))
raise SearchError(ErrFormat.attribute_not_found.format(key))
def _in_query_handle(self, attr, v):
def _in_query_handle(self, attr, v, is_not):
terms = v[1:-1].split(";")
operator = "|"
if attr in ('_type', 'ci_type', 'type_id') and terms and terms[0].isdigit():
@@ -79,11 +87,27 @@ class Search(object):
terms = map(int, terms)
current_app.logger.warning(terms)
for term in terms:
self._operator2query(operator).append({
"term": {
attr: term
}
})
if is_not:
self._operator2query(operator).append({
"bool": {
"must_not": [
{
"term": {
attr: term
}
}
]
}
})
else:
self._operator2query(operator).append({
"term": {
attr: term
}
})
def _filter_ids(self):
if self.ci_ids:
@@ -95,18 +119,36 @@ class Search(object):
return int(float(s))
return s
def _range_query_handle(self, attr, v, operator):
def _range_query_handle(self, attr, v, operator, is_not):
left, right = v.split("_TO_")
left, right = left.strip()[1:], right.strip()[:-1]
self._operator2query(operator).append({
"range": {
attr: {
"lte": self._digit(right),
"gte": self._digit(left),
"boost": 2.0
if is_not:
self._operator2query(operator).append({
"bool": {
"must_not": [
{
"range": {
attr: {
"lte": self._digit(right),
"gte": self._digit(left),
"boost": 2.0
}
}
}
]
}
}
})
})
else:
self._operator2query(operator).append({
"range": {
attr: {
"lte": self._digit(right),
"gte": self._digit(left),
"boost": 2.0
}
}
})
def _comparison_query_handle(self, attr, v, operator):
if v.startswith(">="):
@@ -126,21 +168,50 @@ class Search(object):
}
})
def _match_query_handle(self, attr, v, operator):
def _match_query_handle(self, attr, v, operator, is_not):
if "*" in v:
self._operator2query(operator).append({
"wildcard": {
attr: v.lower() if isinstance(v, six.string_types) else v
}
})
if is_not:
self._operator2query(operator).append({
"bool": {
"must_not": [
{
"wildcard": {
attr: v.lower() if isinstance(v, six.string_types) else v
}
}
]
}
})
else:
self._operator2query(operator).append({
"wildcard": {
attr: v.lower() if isinstance(v, six.string_types) else v
}
})
else:
if attr == "ci_type" and v.isdigit():
attr = "type_id"
self._operator2query(operator).append({
"term": {
attr: v.lower() if isinstance(v, six.string_types) else v
}
})
if is_not:
self._operator2query(operator).append({
"bool": {
"must_not": [
{
"term": {
attr: v.lower() if isinstance(v, six.string_types) else v
}
}
]
}
})
else:
self._operator2query(operator).append({
"term": {
attr: v.lower() if isinstance(v, six.string_types) else v
}
})
def __query_build_by_field(self, queries):
@@ -150,21 +221,23 @@ class Search(object):
v = ":".join(q.split(":")[1:]).strip()
field_name, field_type, operator = self._attr_name_proc(k)
if field_name:
is_not = True if operator == "|~" else False
# in query
if v.startswith("(") and v.endswith(")"):
self._in_query_handle(field_name, v)
self._in_query_handle(field_name, v, is_not)
# range query
elif v.startswith("[") and v.endswith("]") and "_TO_" in v:
self._range_query_handle(field_name, v, operator)
self._range_query_handle(field_name, v, operator, is_not)
# comparison query
elif v.startswith(">=") or v.startswith("<=") or v.startswith(">") or v.startswith("<"):
self._comparison_query_handle(field_name, v, operator)
else:
self._match_query_handle(field_name, v, operator)
self._match_query_handle(field_name, v, operator, is_not)
else:
raise SearchError("argument q format invalid: {0}".format(q))
raise SearchError(ErrFormat.argument_invalid.format("q"))
elif q:
raise SearchError("argument q format invalid: {0}".format(q))
raise SearchError(ErrFormat.argument_invalid.format("q"))
def _query_build_raw(self):
@@ -191,7 +264,7 @@ class Search(object):
for field in self.facet_field:
attr = AttributeCache.get(field)
if not attr:
raise SearchError("Facet by <{0}> does not exist".format(field))
raise SearchError(ErrFormat.attribute_not_found(field))
aggregations['aggs'].update({
field: {
"terms": {
@@ -222,7 +295,7 @@ class Search(object):
attr = AttributeCache.get(field)
if not attr:
raise SearchError("Sort by <{0}> does not exist".format(field))
raise SearchError(ErrFormat.attribute_not_found.format(field))
sort_by = "{0}.keyword".format(field) \
if attr.value_type not in (ValueTypeEnum.INT, ValueTypeEnum.FLOAT) else field
@@ -242,7 +315,7 @@ class Search(object):
numfound, cis, facet = self._query_build_raw()
except Exception as e:
current_app.logger.error(str(e))
raise SearchError("unknown search error")
raise SearchError(ErrFormat.unknown_search_error)
total = len(cis)

70
cmdb-api/api/lib/cmdb/search/ci_relation/search.py
View File

@@ -2,13 +2,16 @@
import json
from collections import Counter
from flask import abort
from flask import current_app
from api.extensions import rd
from api.lib.cmdb.ci import CIRelationManager
from api.lib.cmdb.ci_type import CITypeRelationManager
from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
from api.lib.cmdb.resp_format import ErrFormat
from api.lib.cmdb.search.ci.db.search import Search as SearchFromDB
from api.lib.cmdb.search.ci.es.search import Search as SearchFromES
from api.models.cmdb import CI
@@ -22,7 +25,8 @@ class Search(object):
facet_field=None,
page=1,
count=None,
sort=None):
sort=None,
reverse=False):
self.orig_query = query
self.fl = fl
self.facet_field = facet_field
@@ -32,20 +36,35 @@ class Search(object):
self.root_id = root_id
self.level = level
self.reverse = reverse
def _get_ids(self):
merge_ids = []
ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
for level in range(1, sorted(self.level)[-1] + 1):
_tmp = list(map(lambda x: list(json.loads(x).keys()),
filter(lambda x: x is not None, rd.get(ids, REDIS_PREFIX_CI_RELATION) or [])))
ids = [j for i in _tmp for j in i]
if level in self.level:
merge_ids.extend(ids)
return merge_ids
def _get_reverse_ids(self):
merge_ids = []
ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
for level in range(1, sorted(self.level)[-1] + 1):
ids = CIRelationManager.get_ancestor_ids(ids, 1)
if level in self.level:
merge_ids.extend(ids)
return merge_ids
def search(self):
ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
cis = [CI.get_by_id(_id) or abort(404, "CI <{0}> does not exist".format(_id)) for _id in ids]
cis = [CI.get_by_id(_id) or abort(404, ErrFormat.ci_not_found.format("id={}".format(_id))) for _id in ids]
merge_ids = []
for level in self.level:
ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
for _ in range(0, level):
_tmp = list(map(lambda x: list(json.loads(x).keys()),
filter(lambda x: x is not None, rd.get(ids, REDIS_PREFIX_CI_RELATION) or [])))
ids = [j for i in _tmp for j in i]
merge_ids.extend(ids)
merge_ids = self._get_ids() if not self.reverse else self._get_reverse_ids()
if not self.orig_query or ("_type:" not in self.orig_query
and "type_id:" not in self.orig_query
@@ -53,7 +72,10 @@ class Search(object):
type_ids = []
for level in self.level:
for ci in cis:
type_ids.extend(CITypeRelationManager.get_child_type_ids(ci.type_id, level))
if not self.reverse:
type_ids.extend(CITypeRelationManager.get_child_type_ids(ci.type_id, level))
else:
type_ids.extend(CITypeRelationManager.get_parent_type_ids(ci.type_id, level))
type_ids = list(set(type_ids))
if self.orig_query:
self.orig_query = "_type:({0}),{1}".format(";".join(list(map(str, type_ids))), self.orig_query)
@@ -86,24 +108,30 @@ class Search(object):
ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id
for l in range(0, int(self.level)):
if not l:
_tmp = list(map(lambda x: list(json.loads(x).keys()),
_tmp = list(map(lambda x: list(json.loads(x).items()),
[i or '{}' for i in rd.get(ids, REDIS_PREFIX_CI_RELATION) or []]))
else:
for idx, i in enumerate(_tmp):
if i:
for idx, item in enumerate(_tmp):
if item:
if type_ids and l == self.level - 1:
__tmp = list(
map(lambda x: list({_id: 1 for _id, type_id in json.loads(x).items()
if type_id in type_ids}.keys()),
map(lambda x: [(_id, type_id) for _id, type_id in json.loads(x).items()
if type_id in type_ids],
filter(lambda x: x is not None,
rd.get(i, REDIS_PREFIX_CI_RELATION) or [])))
rd.get([i[0] for i in item], REDIS_PREFIX_CI_RELATION) or [])))
else:
__tmp = list(map(lambda x: list(json.loads(x).keys()),
__tmp = list(map(lambda x: list(json.loads(x).items()),
filter(lambda x: x is not None,
rd.get(i, REDIS_PREFIX_CI_RELATION) or [])))
rd.get([i[0] for i in item], REDIS_PREFIX_CI_RELATION) or [])))
_tmp[idx] = [j for i in __tmp for j in i]
else:
_tmp[idx] = []
return {_id: len(_tmp[idx]) for idx, _id in enumerate(ids)}
result = {str(_id): len(_tmp[idx]) for idx, _id in enumerate(ids)}
result.update(
detail={str(_id): dict(Counter([i[1] for i in _tmp[idx]]).items()) for idx, _id in enumerate(ids)})
return result