feat(api): add builtin attributes (#631)

cmdb-api/api/commands/click_cmdb.py

@@ -1,14 +1,13 @@
 # -*- coding:utf-8 -*-
 
 
+import click
 import copy
 import datetime
 import json
+import requests
 import time
 import uuid
-
-import click
-import requests
 from flask import current_app
 from flask.cli import with_appcontext
 from flask_login import login_user
@@ -37,11 +36,14 @@ from api.lib.secrets.secrets import InnerKVManger
 from api.models.acl import App
 from api.models.acl import ResourceType
 from api.models.cmdb import Attribute
+from api.models.cmdb import AttributeHistory
 from api.models.cmdb import CI
 from api.models.cmdb import CIRelation
 from api.models.cmdb import CIType
 from api.models.cmdb import CITypeTrigger
+from api.models.cmdb import OperationRecord
 from api.models.cmdb import PreferenceRelationView
+from api.tasks.cmdb import batch_ci_cache
 
 
 @click.command()
@@ -557,5 +559,20 @@ def cmdb_patch(version):
                         existed.update(option=option, commit=False)
 
             db.session.commit()
+
+        if version >= "2.4.14":  # update ci columns: updated_at and updated_by
+            ci_ids = []
+            for i in CI.get_by(only_query=True).filter(CI.updated_at.is_(None)):
+                hist = AttributeHistory.get_by(ci_id=i.id, only_query=True).order_by(AttributeHistory.id.desc()).first()
+                if hist is not None:
+                    record = OperationRecord.get_by_id(hist.record_id)
+                    if record is not None:
+                        u = UserCache.get(record.uid)
+                        i.update(updated_at=record.created_at, updated_by=u and u.nickname, flush=True)
+                        ci_ids.append(i.id)
+
+            db.session.commit()
+
+            batch_ci_cache.apply_async(args=(ci_ids,))
     except Exception as e:
         print("cmdb patch failed: {}".format(e))

cmdb-api/api/lib/cmdb/ci.py

@@ -45,6 +45,7 @@ from api.lib.notify import notify_send
 from api.lib.perm.acl.acl import ACLManager
 from api.lib.perm.acl.acl import is_app_admin
 from api.lib.perm.acl.acl import validate_permission
+from api.lib.perm.acl.cache import UserCache
 from api.lib.secrets.inner import InnerCrypt
 from api.lib.secrets.vault import VaultClient
 from api.lib.utils import handle_arg_list
@@ -206,6 +207,8 @@ class CIManager(object):
         res['_type'] = ci_type.id
         res['ci_type_alias'] = ci_type.alias
         res['_id'] = ci_id
+        res['_updated_at'] = str(ci.updated_at)
+        res['_updated_by'] = ci.updated_by
 
         return res
 
@@ -581,6 +584,9 @@ class CIManager(object):
             else:
                 ci_relation_add(ref_ci_dict, ci.id)
 
+        u = UserCache.get(current_user.uid)
+        ci.update(updated_at=now, updated_by=u and u.nickname)
+
     @staticmethod
     def update_unique_value(ci_id, unique_name, unique_value):
         ci = CI.get_by_id(ci_id) or abort(404, ErrFormat.ci_not_found.format("id={}".format(ci_id)))

cmdb-api/api/lib/cmdb/const.py

@@ -1,6 +1,8 @@
 # -*- coding:utf-8 -*- 
 
 
+from flask_babel import lazy_gettext as _l
+
 from api.lib.utils import BaseEnum
 
 
@@ -110,17 +112,23 @@ class ExecuteStatusEnum(BaseEnum):
     FAILED = '1'
     RUNNING = '2'
 
+
 class RelationSourceEnum(BaseEnum):
     ATTRIBUTE_VALUES = "0"
     AUTO_DISCOVERY = "1"
 
 
+BUILTIN_ATTRIBUTES = {
+    "_updated_at": _l("Update Time"),
+    "_updated_by": _l("Updated By"),
+}
+
 CMDB_QUEUE = "one_cmdb_async"
 REDIS_PREFIX_CI = "ONE_CMDB"
 REDIS_PREFIX_CI_RELATION = "CMDB_CI_RELATION"
 REDIS_PREFIX_CI_RELATION2 = "CMDB_CI_RELATION2"
 
-BUILTIN_KEYWORDS = {'id', '_id', 'ci_id', 'type', '_type', 'ci_type', 'ticket_id'}
+BUILTIN_KEYWORDS = {'id', '_id', 'ci_id', 'type', '_type', 'ci_type', 'ticket_id', *BUILTIN_ATTRIBUTES.keys()}
 
 L_TYPE = None
 L_CI = None

cmdb-api/api/lib/cmdb/preference.py

@@ -16,6 +16,7 @@ from api.lib.cmdb.cache import CITypeAttributesCache
 from api.lib.cmdb.cache import CITypeCache
 from api.lib.cmdb.cache import CMDBCounterCache
 from api.lib.cmdb.ci_type import CITypeAttributeManager
+from api.lib.cmdb.const import BUILTIN_ATTRIBUTES
 from api.lib.cmdb.const import ConstraintEnum
 from api.lib.cmdb.const import PermEnum
 from api.lib.cmdb.const import ResourceTypeEnum
@@ -24,7 +25,6 @@ from api.lib.cmdb.perms import CIFilterPermsCRUD
 from api.lib.cmdb.resp_format import ErrFormat
 from api.lib.exception import AbortException
 from api.lib.perm.acl.acl import ACLManager
-from api.models.cmdb import CITypeAttribute
 from api.models.cmdb import CITypeGroup
 from api.models.cmdb import CITypeGroupItem
 from api.models.cmdb import CITypeRelation
@@ -136,17 +136,24 @@ class PreferenceManager(object):
             _type = CITypeCache.get(type_id)
             type_id = _type and _type.id
 
-        attrs = db.session.query(PreferenceShowAttributes, CITypeAttribute.order).join(
+        # attrs = db.session.query(PreferenceShowAttributes, CITypeAttribute.order).join(
-            CITypeAttribute, CITypeAttribute.attr_id == PreferenceShowAttributes.attr_id).filter(
+        #     CITypeAttribute, CITypeAttribute.attr_id == PreferenceShowAttributes.attr_id).filter(
-            PreferenceShowAttributes.uid == current_user.uid).filter(
+        #     PreferenceShowAttributes.uid == current_user.uid).filter(
-            PreferenceShowAttributes.type_id == type_id).filter(
+        #     PreferenceShowAttributes.type_id == type_id).filter(
-            PreferenceShowAttributes.deleted.is_(False)).filter(CITypeAttribute.deleted.is_(False)).group_by(
+        #     PreferenceShowAttributes.deleted.is_(False)).filter(CITypeAttribute.deleted.is_(False)).group_by(
-            CITypeAttribute.attr_id).all()
+        #     CITypeAttribute.attr_id).all()
+
+        attrs = PreferenceShowAttributes.get_by(uid=current_user.uid, type_id=type_id, to_dict=False)
 
         result = []
-        for i in sorted(attrs, key=lambda x: x.PreferenceShowAttributes.order):
+        for i in sorted(attrs, key=lambda x: x.order):
-            item = i.PreferenceShowAttributes.attr.to_dict()
+            if i.attr_id:
-            item.update(dict(is_fixed=i.PreferenceShowAttributes.is_fixed))
+                item = i.attr.to_dict()
+            elif i.builtin_attr:
+                item = dict(name=i.builtin_attr, alias=BUILTIN_ATTRIBUTES[i.builtin_attr])
+            else:
+                item = dict(name="", alias="")
+            item.update(dict(is_fixed=i.is_fixed))
             result.append(item)
 
         is_subscribed = True
@@ -155,10 +162,14 @@ class PreferenceManager(object):
                                                                       choice_web_hook_parse=False,
                                                                       choice_other_parse=False)
             result = [i for i in result if i['default_show']]
+
+            for i in BUILTIN_ATTRIBUTES:
+                result.append(dict(name=i, alias=BUILTIN_ATTRIBUTES[i]))
+
             is_subscribed = False
 
         for i in result:
-            if i["is_choice"]:
+            if i.get("is_choice"):
                 i.update(dict(choice_value=AttributeManager.get_choice_values(
                     i["id"], i["value_type"], i.get("choice_web_hook"), i.get("choice_other"))))
 
@@ -172,24 +183,34 @@ class PreferenceManager(object):
                 _attr, is_fixed = x
             else:
                 _attr, is_fixed = x, False
-            attr = AttributeCache.get(_attr) or abort(404, ErrFormat.attribute_not_found.format("id={}".format(_attr)))
+
+            if _attr in BUILTIN_ATTRIBUTES:
+                attr = None
+                builtin_attr = _attr
+            else:
+                attr = AttributeCache.get(_attr) or abort(
+                    404, ErrFormat.attribute_not_found.format("id={}".format(_attr)))
+                builtin_attr = None
             existed = PreferenceShowAttributes.get_by(type_id=type_id,
                                                       uid=current_user.uid,
-                                                      attr_id=attr.id,
+                                                      attr_id=attr and attr.id,
+                                                      builtin_attr=builtin_attr,
                                                       first=True,
                                                       to_dict=False)
             if existed is None:
                 PreferenceShowAttributes.create(type_id=type_id,
                                                 uid=current_user.uid,
-                                                attr_id=attr.id,
+                                                attr_id=attr and attr.id,
+                                                builtin_attr=builtin_attr,
                                                 order=order,
                                                 is_fixed=is_fixed)
             else:
                 existed.update(order=order, is_fixed=is_fixed)
 
-        attr_dict = {int(i[0]) if isinstance(i, list) else int(i): j for i, j in attr_order}
+        attr_dict = {(int(i[0]) if i[0].isdigit() else i[0]) if isinstance(i, list) else
+                     (int(i) if i.isdigit() else i): j for i, j in attr_order}
         for i in existed_all:
-            if i.attr_id not in attr_dict:
+            if (i.attr_id and i.attr_id not in attr_dict) or (i.builtin_attr and i.builtin_attr not in attr_dict):
                 i.soft_delete()
 
         if not existed_all and attr_order:

cmdb-api/api/lib/cmdb/search/ci/db/search.py

@@ -15,6 +15,7 @@ from api.extensions import db
 from api.lib.cmdb.cache import AttributeCache
 from api.lib.cmdb.cache import CITypeCache
 from api.lib.cmdb.ci import CIManager
+from api.lib.cmdb.const import BUILTIN_ATTRIBUTES
 from api.lib.cmdb.const import PermEnum
 from api.lib.cmdb.const import ResourceTypeEnum
 from api.lib.cmdb.const import RetKey
@@ -304,14 +305,21 @@ class Search(object):
                     (self.page - 1) * self.count, sort_type, self.count))
 
     def __sort_by_field(self, field, sort_type, query_sql):
-        attr = AttributeCache.get(field)
+        if field not in BUILTIN_ATTRIBUTES:
-        attr_id = attr.id
 
-        table_name = TableMap(attr=attr).table_name
+            attr = AttributeCache.get(field)
-        _v_query_sql = """SELECT {0}.ci_id, {1}.value
+            attr_id = attr.id
-                          FROM ({2}) AS {0} INNER JOIN {1} ON {1}.ci_id = {0}.ci_id
+
-                          WHERE {1}.attr_id = {3}""".format("ALIAS", table_name, query_sql, attr_id)
+            table_name = TableMap(attr=attr).table_name
-        new_table = _v_query_sql
+            _v_query_sql = """SELECT ALIAS.ci_id, {0}.value
+                              FROM ({1}) AS ALIAS INNER JOIN {0} ON {0}.ci_id = ALIAS.ci_id
+                              WHERE {0}.attr_id = {2}""".format(table_name, query_sql, attr_id)
+            new_table = _v_query_sql
+        else:
+            _v_query_sql = """SELECT c_cis.id AS ci_id, c_cis.{0} AS value
+                                          FROM c_cis  INNER JOIN ({1}) AS ALIAS ON ALIAS.ci_id = c_cis.id""".format(
+                field[1:], query_sql)
+            new_table = _v_query_sql
 
         if self.only_type_query or not self.type_id_list or self.multi_type_has_ci_filter:
             return ("SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id FROM ({0}) AS C ORDER BY C.value {2} "

cmdb-api/api/models/cmdb.py

@@ -253,6 +253,7 @@ class CI(Model):
     status = db.Column(db.Enum(*CIStatusEnum.all(), name="status"))
     heartbeat = db.Column(db.DateTime, default=lambda: datetime.datetime.now())
     is_auto_discovery = db.Column('a', db.Boolean, default=False)
+    updated_by = db.Column(db.String(64))
 
     ci_type = db.relationship("CIType", backref="c_cis.type_id")
 
@@ -534,6 +535,7 @@ class CustomDashboard(Model):
 
     type_id = db.Column(db.Integer, db.ForeignKey('c_ci_types.id'))
     attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'))
+    builtin_attr = db.Column(db.String(256), nullable=True)
     level = db.Column(db.Integer)
 
     options = db.Column(db.JSON)