mirror of
https://github.com/veops/cmdb.git
synced 2025-08-08 15:07:55 +08:00
v1 vsersion
This commit is contained in:
pycook
pycook
348
lib/search.py
Normal file
348
lib/search.py
Normal file
@@ -0,0 +1,348 @@
|
||||
# -*- coding:utf-8 -*-
|
||||
|
||||
|
||||
import time
|
||||
|
||||
from flask import current_app
|
||||
|
||||
from lib.const import TableMap
|
||||
from models.attribute import CIAttributeCache
|
||||
from models.ci_type import CITypeCache
|
||||
from extensions import db
|
||||
from models import CI
|
||||
from lib.ci import get_cis_by_ids
|
||||
from lib.query_sql import FACET_QUERY
|
||||
from lib.query_sql import QUERY_CI_BY_TYPE
|
||||
from lib.query_sql import QUERY_CI_BY_ATTR_NAME
|
||||
|
||||
|
||||
class SearchError(Exception):
|
||||
def __init__(self, v):
|
||||
self.v = v
|
||||
|
||||
def __str__(self):
|
||||
return self.v
|
||||
|
||||
|
||||
class Search(object):
|
||||
def __init__(self, query=None, fl=None, facet_field=None,
|
||||
page=1, ret_key="name", count=1, sort=None):
|
||||
self.orig_query = query
|
||||
self.fl = fl
|
||||
self.facet_field = facet_field
|
||||
self.page = page
|
||||
self.ret_key = ret_key
|
||||
try:
|
||||
self.count = int(count)
|
||||
except ValueError:
|
||||
self.count = current_app.config.get("DEFAULT_PAGE_COUNT")
|
||||
self.sort = sort
|
||||
self.query_sql = ""
|
||||
self.type_id_list = []
|
||||
|
||||
def tor_proc(self, key):
|
||||
tor = list()
|
||||
if key.startswith("+"):
|
||||
tor.append('&')
|
||||
key = key[1:].strip()
|
||||
elif key.startswith("-"):
|
||||
tor.append('|')
|
||||
key = key[1:].strip()
|
||||
elif key.startswith("~"):
|
||||
tor.append('~')
|
||||
key = key[1:].strip()
|
||||
if not tor:
|
||||
tor = ['&', '']
|
||||
if len(tor) < 2:
|
||||
tor.append('')
|
||||
return tor, key
|
||||
|
||||
def attr_name_proc(self, key):
|
||||
tor, key = self.tor_proc(key)
|
||||
if key in ('ci_type', 'type', '_type'):
|
||||
return '_type', 'text', tor, None
|
||||
if key in ('id', 'ci_id', '_id'):
|
||||
return '_id', 'text', tor, None
|
||||
attr = CIAttributeCache.get(key)
|
||||
if attr is not None:
|
||||
# if not attr.is_index:
|
||||
# raise SearchError("{0} is not indexed".format(attr.attr_name))
|
||||
field_name = attr.attr_name
|
||||
return field_name, attr.value_type, tor, attr
|
||||
else:
|
||||
raise SearchError("{0} is not existed".format(key))
|
||||
|
||||
def type_query_handler(self, v, only_type_query):
|
||||
new_v = [v]
|
||||
if v.startswith("(") and v.endswith(")"):
|
||||
new_v = v[1:-1].split(";")
|
||||
for _v in new_v:
|
||||
ci_type = CITypeCache.get(_v)
|
||||
if ci_type is not None:
|
||||
self.type_id_list.append(str(ci_type.type_id))
|
||||
if self.type_id_list:
|
||||
type_ids = ",".join(self.type_id_list)
|
||||
_query_sql = QUERY_CI_BY_TYPE.format(type_ids)
|
||||
if only_type_query:
|
||||
return _query_sql
|
||||
else:
|
||||
return ""
|
||||
return ""
|
||||
|
||||
def in_query_handler(self, attr, v):
|
||||
new_v = v[1:-1].split(";")
|
||||
table_name = TableMap(attr_name=attr.attr_name).table_name
|
||||
_query_sql = QUERY_CI_BY_ATTR_NAME.format(
|
||||
table_name, attr.attr_id,
|
||||
" OR {0}.value ".format(table_name).join(['LIKE "{0}"'.format(
|
||||
_v.replace("*", "%")) for _v in new_v]))
|
||||
return _query_sql
|
||||
|
||||
def range_query_handler(self, attr, v):
|
||||
start, end = [x.strip() for x in v[1:-1].split("_TO_")]
|
||||
table_name = TableMap(attr_name=attr.attr_name).table_name
|
||||
_query_sql = QUERY_CI_BY_ATTR_NAME.format(
|
||||
table_name, attr.attr_id, "BETWEEN '{0}' AND '{1}'".format(
|
||||
start.replace("*", "%"), end.replace("*", "%")))
|
||||
return _query_sql
|
||||
|
||||
def comparison_query_handler(self, attr, v):
|
||||
table_name = TableMap(attr_name=attr.attr_name).table_name
|
||||
if (v.startswith("<") and not v.startswith("<=")) or \
|
||||
(v.startswith(">") and not v.startswith(">=")):
|
||||
_query_sql = QUERY_CI_BY_ATTR_NAME.format(
|
||||
table_name, attr.attr_id, "{0} '{1}'".format(
|
||||
v[0], v[1:].replace("*", "%")))
|
||||
elif v.startswith(">=") or v.startswith("<="):
|
||||
_query_sql = QUERY_CI_BY_ATTR_NAME.format(
|
||||
table_name, attr.attr_id, "{0} '{1}'".format(
|
||||
v[:2], v[2:].replace("*", "%")))
|
||||
return _query_sql
|
||||
|
||||
def sort_query_handler(self, field, query_sql, only_type_query):
|
||||
if field is None:
|
||||
field = ""
|
||||
if field.startswith("+"):
|
||||
field = field[1:]
|
||||
sort_type = "ASC"
|
||||
elif field.startswith("-"):
|
||||
field = field[1:]
|
||||
sort_type = "DESC"
|
||||
else:
|
||||
sort_type = "ASC"
|
||||
|
||||
if field in ("_id", "ci_id") or not field:
|
||||
if only_type_query:
|
||||
return """SELECT SQL_CALC_FOUND_ROWS DISTINCT B.ci_id
|
||||
FROM ({0}) AS B {1}""".format(
|
||||
query_sql,
|
||||
"ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format(
|
||||
(self.page - 1) * self.count, sort_type, self.count))
|
||||
elif self.type_id_list:
|
||||
return """SELECT SQL_CALC_FOUND_ROWS DISTINCT B.ci_id
|
||||
FROM ({0}) AS B {1}""".format(
|
||||
query_sql,
|
||||
"INNER JOIN cis on cis.ci_id=B.ci_id "
|
||||
"WHERE cis.type_id in ({3}) "
|
||||
"ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format(
|
||||
(self.page - 1) * self.count, sort_type, self.count,
|
||||
",".join(self.type_id_list)))
|
||||
else:
|
||||
return """SELECT SQL_CALC_FOUND_ROWS DISTINCT B.ci_id
|
||||
FROM ({0}) AS B {1}""".format(
|
||||
query_sql,
|
||||
"INNER JOIN cis on cis.ci_id=B.ci_id "
|
||||
"ORDER BY B.ci_id {1} LIMIT {0:d}, {2};".format(
|
||||
(self.page - 1) * self.count, sort_type, self.count))
|
||||
else:
|
||||
attr = CIAttributeCache.get(field)
|
||||
attr_id = attr.attr_id
|
||||
|
||||
table_name = TableMap(attr_name=attr.attr_name).table_name
|
||||
_v_query_sql = """SELECT {0}.ci_id, {1}.value FROM
|
||||
({2}) AS {0} INNER JOIN {1} ON {1}.ci_id = {0}.ci_id
|
||||
WHERE {1}.attr_id = {3}""".format("ALIAS", table_name,
|
||||
query_sql, attr_id)
|
||||
new_table = _v_query_sql
|
||||
if only_type_query:
|
||||
return "SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id " \
|
||||
"FROM ({0}) AS C " \
|
||||
"ORDER BY C.value {2} " \
|
||||
"LIMIT {1:d}, {3};".format(new_table,
|
||||
(self.page - 1) * self.count,
|
||||
sort_type, self.count)
|
||||
elif self.type_id_list:
|
||||
return """SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id
|
||||
FROM ({0}) AS C
|
||||
INNER JOIN cis on cis.ci_id=C.ci_id
|
||||
WHERE cis.type_id in ({4})
|
||||
ORDER BY C.value {2}
|
||||
LIMIT {1:d}, {3};""".format(new_table,
|
||||
(self.page - 1) * self.count,
|
||||
sort_type, self.count,
|
||||
",".join(self.type_id_list))
|
||||
else:
|
||||
return """SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id
|
||||
FROM ({0}) AS C
|
||||
ORDER BY C.value {2}
|
||||
LIMIT {1:d}, {3};""".format(new_table,
|
||||
(self.page - 1) * self.count,
|
||||
sort_type, self.count)
|
||||
|
||||
def _wrap_sql(self, tor, alias, _query_sql, query_sql):
|
||||
if tor[0] == "&":
|
||||
query_sql = """SELECT * FROM ({0}) as {1}
|
||||
INNER JOIN ({2}) as {3} USING(ci_id)""".format(
|
||||
query_sql, alias, _query_sql, alias + "A")
|
||||
elif tor[0] == "|":
|
||||
query_sql = "SELECT * FROM ({0}) as {1} UNION ALL ({2})".format(
|
||||
query_sql, alias, _query_sql)
|
||||
elif tor[0] == "~":
|
||||
query_sql = "SELECT * FROM ({0}) as {1} LEFT JOIN ({2}) as {3} " \
|
||||
"USING(ci_id) WHERE {3}.ci_id is NULL".format(
|
||||
query_sql, alias, _query_sql, alias + "A")
|
||||
return query_sql
|
||||
|
||||
def _execute_sql(self, query_sql, only_type_query):
|
||||
v_query_sql = self.sort_query_handler(self.sort, query_sql,
|
||||
only_type_query)
|
||||
start = time.time()
|
||||
execute = db.session.execute
|
||||
current_app.logger.debug(v_query_sql)
|
||||
res = execute(v_query_sql).fetchall()
|
||||
end_time = time.time()
|
||||
current_app.logger.debug("query ci ids time is: {0}".format(
|
||||
end_time - start))
|
||||
numfound = execute("SELECT FOUND_ROWS();").fetchall()[0][0]
|
||||
current_app.logger.debug("statistics ci ids time is: {0}".format(
|
||||
time.time() - end_time)
|
||||
)
|
||||
return numfound, res
|
||||
|
||||
def query_build_raw(self):
|
||||
query_sql, alias, tor = "", "A", ["&"]
|
||||
is_first = True
|
||||
only_type_query = False
|
||||
queries = self.orig_query.split(",")
|
||||
queries = filter(lambda x: x != "", queries)
|
||||
for q in queries:
|
||||
if q.startswith("_type"):
|
||||
queries.remove(q)
|
||||
queries.insert(0, q)
|
||||
if len(queries) == 1 or queries[1].startswith("-") or \
|
||||
queries[1].startswith("~"):
|
||||
only_type_query = True
|
||||
break
|
||||
current_app.logger.debug(queries)
|
||||
special = True
|
||||
for q in queries:
|
||||
_query_sql = ""
|
||||
if ":" in q:
|
||||
k = q.split(":")[0].strip()
|
||||
v = ":".join(q.split(":")[1:]).strip()
|
||||
current_app.logger.info(v)
|
||||
field, field_type, tor, attr = self.attr_name_proc(k)
|
||||
if field == "_type":
|
||||
_query_sql = self.type_query_handler(v, only_type_query)
|
||||
current_app.logger.debug(_query_sql)
|
||||
elif field == "_id": # exclude all others
|
||||
_ci_ids = [str(v)]
|
||||
ci = db.session.query(CI.ci_id).filter(
|
||||
CI.ci_id == int(v)).first()
|
||||
if ci is not None:
|
||||
return 1, _ci_ids
|
||||
elif field:
|
||||
if attr is None:
|
||||
raise SearchError("{0} is not found".format(field))
|
||||
# in query
|
||||
if v.startswith("(") and v.endswith(")"):
|
||||
_query_sql = self.in_query_handler(attr, v)
|
||||
# range query
|
||||
elif v.startswith("[") and v.endswith("]") and "_TO_" in v:
|
||||
_query_sql = self.range_query_handler(attr, v)
|
||||
# comparison query
|
||||
elif v.startswith(">=") or v.startswith("<=") or \
|
||||
v.startswith(">") or v.startswith("<"):
|
||||
_query_sql = self.comparison_query_handler(attr, v)
|
||||
else:
|
||||
table_name = \
|
||||
TableMap(attr_name=attr.attr_name).table_name
|
||||
_query_sql = QUERY_CI_BY_ATTR_NAME.format(
|
||||
table_name, attr.attr_id,
|
||||
'LIKE "{0}"'.format(v.replace("*", "%")))
|
||||
else:
|
||||
return 0, []
|
||||
elif q:
|
||||
return 0, []
|
||||
|
||||
if is_first and _query_sql and not only_type_query:
|
||||
query_sql = "SELECT * FROM ({0}) AS {1}".format(_query_sql,
|
||||
alias)
|
||||
is_first = False
|
||||
alias += "A"
|
||||
elif only_type_query and special:
|
||||
is_first = False
|
||||
special = False
|
||||
query_sql = _query_sql
|
||||
elif _query_sql:
|
||||
query_sql = self._wrap_sql(tor, alias, _query_sql, query_sql)
|
||||
alias += "AA"
|
||||
|
||||
_start = time.time()
|
||||
if query_sql:
|
||||
self.query_sql = query_sql
|
||||
current_app.logger.debug(query_sql)
|
||||
numfound, res = self._execute_sql(query_sql, only_type_query)
|
||||
current_app.logger.info("query ci ids is: {0}".format(
|
||||
time.time() - _start))
|
||||
return numfound, [_res[0] for _res in res]
|
||||
return 0, []
|
||||
|
||||
def facet_build(self):
|
||||
facet = {}
|
||||
for f in self.facet_field:
|
||||
k, field_type, _, attr = self.attr_name_proc(f)
|
||||
if k:
|
||||
table_name = TableMap(attr_name=k).table_name
|
||||
query_sql = FACET_QUERY.format(
|
||||
table_name, self.query_sql, attr.attr_id)
|
||||
result = db.session.execute(query_sql).fetchall()
|
||||
facet[k] = result
|
||||
facet_result = dict()
|
||||
for k, v in facet.items():
|
||||
if not k.startswith('_'):
|
||||
a = getattr(CIAttributeCache.get(k), "attr_%s" % self.ret_key)
|
||||
facet_result[a] = list()
|
||||
for f in v:
|
||||
if f[1] != 0:
|
||||
facet_result[a].append((f[0], f[1], a))
|
||||
return facet_result
|
||||
|
||||
def fl_build(self):
|
||||
_fl = list()
|
||||
for f in self.fl:
|
||||
k, _, _, _ = self.attr_name_proc(f)
|
||||
if k:
|
||||
_fl.append(k)
|
||||
return _fl
|
||||
|
||||
def search(self):
|
||||
numfound, ci_ids = self.query_build_raw()
|
||||
ci_ids = map(str, ci_ids)
|
||||
_fl = self.fl_build()
|
||||
|
||||
if self.facet_field and numfound:
|
||||
facet = self.facet_build()
|
||||
else:
|
||||
facet = dict()
|
||||
|
||||
response, counter = [], {}
|
||||
if ci_ids:
|
||||
response = get_cis_by_ids(ci_ids, ret_key=self.ret_key, fields=_fl)
|
||||
for res in response:
|
||||
ci_type = res.get("ci_type")
|
||||
if ci_type not in counter.keys():
|
||||
counter[ci_type] = 0
|
||||
counter[ci_type] += 1
|
||||
total = len(response)
|
||||
return response, counter, total, self.page, numfound, facet
|
||||
Reference in New Issue
Block a user