From c4caa464d36a261b64e307ccf01723adb0e6dae8 Mon Sep 17 00:00:00 2001 From: pycook Date: Thu, 20 Jul 2023 18:36:32 +0800 Subject: [PATCH 1/4] fix docker-compose --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index f8b9a66..049be9a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,7 +48,7 @@ services: gunicorn --workers=3 autoapp:app -b 0.0.0.0:5000 -D flask cmdb-init-cache flask cmdb-init-acl - flask cmdb-counter + nohup flask cmdb-counter > counter.log 2>&1 & celery worker -A celery_worker.celery -E -Q one_cmdb_async --concurrency=2 -D celery worker -A celery_worker.celery -E -Q acl_async --concurrency=2 From 7443f96813fec8cf49555b47e9ac1a14e8674533 Mon Sep 17 00:00:00 2001 From: pycook Date: Fri, 21 Jul 2023 15:58:41 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E7=A6=81=E6=AD=A2=E5=88=A0=E9=99=A4?= =?UTF-8?q?=E5=94=AF=E4=B8=80=E6=A0=87=E8=AF=86=E7=9A=84=E5=B1=9E=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- cmdb-api/api/lib/cmdb/attribute.py | 4 ++++ cmdb-api/api/lib/cmdb/resp_format.py | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 009287d..affb374 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,6 @@ docker-compose up -d --- -_**欢迎关注我们的公众号，点击联系我们，加入微信、qq运维群，获得更多产品、行业相关资讯**_ +_**欢迎关注我们的公众号，点击联系我们，加入微信、qq运维群(336164978)，获得更多产品、行业相关资讯**_ ![公众号](docs/images/qrcode_for_gzh.jpg) diff --git a/cmdb-api/api/lib/cmdb/attribute.py b/cmdb-api/api/lib/cmdb/attribute.py index 97b2aaa..bc17d97 100644 --- a/cmdb-api/api/lib/cmdb/attribute.py +++ b/cmdb-api/api/lib/cmdb/attribute.py @@ -18,6 +18,7 @@ from api.lib.decorator import kwargs_required from api.lib.perm.acl.acl import is_app_admin from api.lib.perm.acl.acl import validate_permission from api.models.cmdb import Attribute +from api.models.cmdb import CIType from api.models.cmdb import CITypeAttribute from api.models.cmdb import CITypeAttributeGroupItem from api.models.cmdb import PreferenceShowAttributes @@ -315,6 +316,9 @@ class AttributeManager(object): attr = Attribute.get_by_id(_id) or abort(404, ErrFormat.attribute_not_found.format("id={}".format(_id))) name = attr.name + if CIType.get_by(unique_id=attr.id, first=True, to_dict=False) is not None: + return abort(400, ErrFormat.attribute_is_unique_id) + if attr.uid and attr.uid != g.user.uid: return abort(403, ErrFormat.cannot_delete_attribute) diff --git a/cmdb-api/api/lib/cmdb/resp_format.py b/cmdb-api/api/lib/cmdb/resp_format.py index ed21e86..7c53784 100644 --- a/cmdb-api/api/lib/cmdb/resp_format.py +++ b/cmdb-api/api/lib/cmdb/resp_format.py @@ -10,6 +10,7 @@ class ErrFormat(CommonErrFormat): argument_file_not_found = "文件似乎并未上传" attribute_not_found = "属性 {} 不存在!" + attribute_is_unique_id = "该属性是模型的唯一标识，不能被删除!" attribute_value_type_cannot_change = "属性的值类型不允许修改!" attribute_list_value_cannot_change = "多值不被允许修改!" attribute_index_cannot_change = "修改索引 非管理员不被允许!" From c196c7598517c46620df01cfcc42e1c4a05fe5d9 Mon Sep 17 00:00:00 2001 From: pycook Date: Mon, 24 Jul 2023 21:55:00 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E5=BA=9F=E5=BC=833=E4=B8=AA=E8=A1=A8:=20c?= =?UTF-8?q?=5Fvalue=5Fdatetime=20=20c=5Fvalue=5Ffloats=20=20c=5Fvalue=5Fin?= =?UTF-8?q?tegers,=20time=E7=B1=BB=E5=9E=8B=E5=B1=9E=E6=80=A7=E5=80=BC?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=86=99=E5=85=A5=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmdb-api/api/lib/cmdb/utils.py | 25 +++++++-------- cmdb-api/api/models/cmdb.py | 58 +++++++++++++++++----------------- 2 files changed, 41 insertions(+), 42 deletions(-) diff --git a/cmdb-api/api/lib/cmdb/utils.py b/cmdb-api/api/lib/cmdb/utils.py index 1ea82af..725a0f2 100644 --- a/cmdb-api/api/lib/cmdb/utils.py +++ b/cmdb-api/api/lib/cmdb/utils.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import datetime import json +import re import six from markupsafe import escape @@ -31,7 +32,8 @@ class ValueTypeMap(object): ValueTypeEnum.INT: string2int, ValueTypeEnum.FLOAT: float, ValueTypeEnum.TEXT: lambda x: escape(x).encode('utf-8').decode('utf-8'), - ValueTypeEnum.TIME: lambda x: escape(x).encode('utf-8').decode('utf-8'), + ValueTypeEnum.TIME: lambda x: re.compile(r'\d\d:\d\d:\d\d').findall( + escape(x).encode('utf-8').decode('utf-8'))[0], ValueTypeEnum.DATETIME: str2datetime, ValueTypeEnum.DATE: str2datetime, ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x, @@ -61,15 +63,11 @@ class ValueTypeMap(object): ValueTypeEnum.INT: model.IntegerChoice, ValueTypeEnum.FLOAT: model.FloatChoice, ValueTypeEnum.TEXT: model.TextChoice, + ValueTypeEnum.TIME: model.TextChoice, } table = { - ValueTypeEnum.INT: model.CIValueInteger, ValueTypeEnum.TEXT: model.CIValueText, - ValueTypeEnum.DATETIME: model.CIValueDateTime, - ValueTypeEnum.DATE: model.CIValueDateTime, - ValueTypeEnum.TIME: model.CIValueText, - ValueTypeEnum.FLOAT: model.CIValueFloat, ValueTypeEnum.JSON: model.CIValueJson, 'index_{0}'.format(ValueTypeEnum.INT): model.CIIndexValueInteger, 'index_{0}'.format(ValueTypeEnum.TEXT): model.CIIndexValueText, @@ -81,12 +79,7 @@ class ValueTypeMap(object): } table_name = { - ValueTypeEnum.INT: 'c_value_integers', ValueTypeEnum.TEXT: 'c_value_texts', - ValueTypeEnum.DATETIME: 'c_value_datetime', - ValueTypeEnum.DATE: 'c_value_datetime', - ValueTypeEnum.TIME: 'c_value_texts', - ValueTypeEnum.FLOAT: 'c_value_floats', ValueTypeEnum.JSON: 'c_value_json', 'index_{0}'.format(ValueTypeEnum.INT): 'c_value_index_integers', 'index_{0}'.format(ValueTypeEnum.TEXT): 'c_value_index_texts', @@ -117,8 +110,11 @@ class TableMap(object): @property def table(self): attr = AttributeCache.get(self.attr_name) if not self.attr else self.attr - if self.is_index is None: + if attr.value_type != ValueTypeEnum.TEXT and attr.value_type != ValueTypeEnum.JSON: + self.is_index = True + elif self.is_index is None: self.is_index = attr.is_index + i = "index_{0}".format(attr.value_type) if self.is_index else attr.value_type return ValueTypeMap.table.get(i) @@ -126,8 +122,11 @@ class TableMap(object): @property def table_name(self): attr = AttributeCache.get(self.attr_name) if not self.attr else self.attr - if self.is_index is None: + if attr.value_type != ValueTypeEnum.TEXT and attr.value_type != ValueTypeEnum.JSON: + self.is_index = True + elif self.is_index is None: self.is_index = attr.is_index + i = "index_{0}".format(attr.value_type) if self.is_index else attr.value_type return ValueTypeMap.table_name.get(i) diff --git a/cmdb-api/api/models/cmdb.py b/cmdb-api/api/models/cmdb.py index e1693cb..e3433d6 100644 --- a/cmdb-api/api/models/cmdb.py +++ b/cmdb-api/api/models/cmdb.py @@ -249,26 +249,26 @@ class CIIndexValueDateTime(Model): __table_args__ = (db.Index("datetime_attr_value_index", "attr_id", "value"),) -class CIValueInteger(Model): - __tablename__ = "c_value_integers" - - ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False) - attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False) - value = db.Column(db.Integer, nullable=False) - - ci = db.relationship("CI", backref="c_value_integers.ci_id") - attr = db.relationship("Attribute", backref="c_value_integers.attr_id") - - -class CIValueFloat(Model): - __tablename__ = "c_value_floats" - - ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False) - attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False) - value = db.Column(DOUBLE, nullable=False) - - ci = db.relationship("CI", backref="c_value_floats.ci_id") - attr = db.relationship("Attribute", backref="c_value_floats.attr_id") +# class CIValueInteger(Model): +# __tablename__ = "c_value_integers" +# +# ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False) +# attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False) +# value = db.Column(db.Integer, nullable=False) +# +# ci = db.relationship("CI", backref="c_value_integers.ci_id") +# attr = db.relationship("Attribute", backref="c_value_integers.attr_id") +# +# +# class CIValueFloat(Model): +# __tablename__ = "c_value_floats" +# +# ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False) +# attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False) +# value = db.Column(DOUBLE, nullable=False) +# +# ci = db.relationship("CI", backref="c_value_floats.ci_id") +# attr = db.relationship("Attribute", backref="c_value_floats.attr_id") class CIValueText(Model): @@ -282,15 +282,15 @@ class CIValueText(Model): attr = db.relationship("Attribute", backref="c_value_texts.attr_id") -class CIValueDateTime(Model): - __tablename__ = "c_value_datetime" - - ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False) - attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False) - value = db.Column(db.DateTime, nullable=False) - - ci = db.relationship("CI", backref="c_value_datetime.ci_id") - attr = db.relationship("Attribute", backref="c_value_datetime.attr_id") +# class CIValueDateTime(Model): +# __tablename__ = "c_value_datetime" +# +# ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False) +# attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False) +# value = db.Column(db.DateTime, nullable=False) +# +# ci = db.relationship("CI", backref="c_value_datetime.ci_id") +# attr = db.relationship("Attribute", backref="c_value_datetime.attr_id") class CIValueJson(Model): From 5d299bd71a9b6ac2c2d4bf5bf2ef8e867e92b1eb Mon Sep 17 00:00:00 2001 From: pycook Date: Tue, 25 Jul 2023 10:31:30 +0800 Subject: [PATCH 4/4] add command cmdb-index-table-upgrade --- cmdb-api/api/commands/click_acl.py | 3 + cmdb-api/api/commands/click_cmdb.py | 37 +++++++++++ cmdb-api/api/commands/init_common_setting.py | 6 +- cmdb-api/api/lib/cmdb/utils.py | 5 +- cmdb-api/api/models/cmdb.py | 67 +++++++++++--------- 5 files changed, 84 insertions(+), 34 deletions(-) diff --git a/cmdb-api/api/commands/click_acl.py b/cmdb-api/api/commands/click_acl.py index d35c6a3..8034c46 100644 --- a/cmdb-api/api/commands/click_acl.py +++ b/cmdb-api/api/commands/click_acl.py @@ -5,6 +5,9 @@ from flask.cli import with_appcontext @click.command() @with_appcontext def init_acl(): + """ + acl init + """ from api.models.acl import Role from api.models.acl import App from api.tasks.acl import role_rebuild diff --git a/cmdb-api/api/commands/click_cmdb.py b/cmdb-api/api/commands/click_cmdb.py index 735464d..37e83f6 100644 --- a/cmdb-api/api/commands/click_cmdb.py +++ b/cmdb-api/api/commands/click_cmdb.py @@ -29,6 +29,7 @@ from api.lib.perm.acl.role import RoleCRUD from api.lib.perm.acl.user import UserCRUD from api.models.acl import App from api.models.acl import ResourceType +from api.models.cmdb import Attribute from api.models.cmdb import CI from api.models.cmdb import CIRelation from api.models.cmdb import CIType @@ -200,6 +201,9 @@ def del_user(user): @click.command() @with_appcontext def cmdb_counter(): + """ + Dashboard calculations + """ from api.lib.cmdb.cache import CMDBCounterCache while True: @@ -217,6 +221,9 @@ def cmdb_counter(): @click.command() @with_appcontext def cmdb_trigger(): + """ + Trigger execution + """ current_day = datetime.datetime.today().strftime("%Y-%m-%d") trigger2cis = dict() trigger2completed = dict() @@ -259,3 +266,33 @@ def cmdb_trigger(): i += 1 time.sleep(10) + + +@click.command() +@with_appcontext +def cmdb_index_table_upgrade(): + """ + Migrate data from tables c_value_integers, c_value_floats, and c_value_datetime + """ + for attr in Attribute.get_by(to_dict=False): + if attr.value_type not in {ValueTypeEnum.TEXT, ValueTypeEnum.JSON}: + attr.update(is_index=True) + + from api.models.cmdb import CIValueInteger, CIIndexValueInteger + from api.models.cmdb import CIValueFloat, CIIndexValueFloat + from api.models.cmdb import CIValueDateTime, CIIndexValueDateTime + + for i in CIValueInteger.get_by(to_dict=False): + CIIndexValueInteger.create(ci_id=i.ci_id, attr_id=i.attr_id, value=i.value, commit=False) + i.delete(commit=False) + db.session.commit() + + for i in CIValueFloat.get_by(to_dict=False): + CIIndexValueFloat.create(ci_id=i.ci_id, attr_id=i.attr_id, value=i.value, commit=False) + i.delete(commit=False) + db.session.commit() + + for i in CIValueDateTime.get_by(to_dict=False): + CIIndexValueDateTime.create(ci_id=i.ci_id, attr_id=i.attr_id, value=i.value, commit=False) + i.delete(commit=False) + db.session.commit() diff --git a/cmdb-api/api/commands/init_common_setting.py b/cmdb-api/api/commands/init_common_setting.py index ddc67bc..88483c1 100644 --- a/cmdb-api/api/commands/init_common_setting.py +++ b/cmdb-api/api/commands/init_common_setting.py @@ -19,7 +19,7 @@ class InitEmployee(object): def import_user_from_acl(self): """ - 从ACL导入用户 + Import users from ACL """ acl = ACLManager('acl') @@ -149,7 +149,7 @@ class InitDepartment(object): @with_appcontext def init_import_user_from_acl(): """ - 从ACL导入用户 + Import users from ACL """ InitEmployee().import_user_from_acl() @@ -158,7 +158,7 @@ def init_import_user_from_acl(): @with_appcontext def init_department(): """ - 初始化 部门 + Department initialization """ InitDepartment().init() InitDepartment().create_acl_role_with_department() diff --git a/cmdb-api/api/lib/cmdb/utils.py b/cmdb-api/api/lib/cmdb/utils.py index 725a0f2..c12b6b7 100644 --- a/cmdb-api/api/lib/cmdb/utils.py +++ b/cmdb-api/api/lib/cmdb/utils.py @@ -13,6 +13,8 @@ import api.models.cmdb as model from api.lib.cmdb.cache import AttributeCache from api.lib.cmdb.const import ValueTypeEnum +TIME_RE = re.compile(r"^(20|21|22|23|[0-1]\d):[0-5]\d:[0-5]\d$") + def string2int(x): return int(float(x)) @@ -32,8 +34,7 @@ class ValueTypeMap(object): ValueTypeEnum.INT: string2int, ValueTypeEnum.FLOAT: float, ValueTypeEnum.TEXT: lambda x: escape(x).encode('utf-8').decode('utf-8'), - ValueTypeEnum.TIME: lambda x: re.compile(r'\d\d:\d\d:\d\d').findall( - escape(x).encode('utf-8').decode('utf-8'))[0], + ValueTypeEnum.TIME: lambda x: TIME_RE.findall(escape(x).encode('utf-8').decode('utf-8'))[0], ValueTypeEnum.DATETIME: str2datetime, ValueTypeEnum.DATE: str2datetime, ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x, diff --git a/cmdb-api/api/models/cmdb.py b/cmdb-api/api/models/cmdb.py index e3433d6..052957f 100644 --- a/cmdb-api/api/models/cmdb.py +++ b/cmdb-api/api/models/cmdb.py @@ -249,26 +249,32 @@ class CIIndexValueDateTime(Model): __table_args__ = (db.Index("datetime_attr_value_index", "attr_id", "value"),) -# class CIValueInteger(Model): -# __tablename__ = "c_value_integers" -# -# ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False) -# attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False) -# value = db.Column(db.Integer, nullable=False) -# -# ci = db.relationship("CI", backref="c_value_integers.ci_id") -# attr = db.relationship("Attribute", backref="c_value_integers.attr_id") -# -# -# class CIValueFloat(Model): -# __tablename__ = "c_value_floats" -# -# ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False) -# attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False) -# value = db.Column(DOUBLE, nullable=False) -# -# ci = db.relationship("CI", backref="c_value_floats.ci_id") -# attr = db.relationship("Attribute", backref="c_value_floats.attr_id") +class CIValueInteger(Model): + """ + Deprecated in a future version + """ + __tablename__ = "c_value_integers" + + ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False) + attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False) + value = db.Column(db.Integer, nullable=False) + + ci = db.relationship("CI", backref="c_value_integers.ci_id") + attr = db.relationship("Attribute", backref="c_value_integers.attr_id") + + +class CIValueFloat(Model): + """ + Deprecated in a future version + """ + __tablename__ = "c_value_floats" + + ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False) + attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False) + value = db.Column(DOUBLE, nullable=False) + + ci = db.relationship("CI", backref="c_value_floats.ci_id") + attr = db.relationship("Attribute", backref="c_value_floats.attr_id") class CIValueText(Model): @@ -282,15 +288,18 @@ class CIValueText(Model): attr = db.relationship("Attribute", backref="c_value_texts.attr_id") -# class CIValueDateTime(Model): -# __tablename__ = "c_value_datetime" -# -# ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False) -# attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False) -# value = db.Column(db.DateTime, nullable=False) -# -# ci = db.relationship("CI", backref="c_value_datetime.ci_id") -# attr = db.relationship("Attribute", backref="c_value_datetime.attr_id") +class CIValueDateTime(Model): + """ + Deprecated in a future version + """ + __tablename__ = "c_value_datetime" + + ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False) + attr_id = db.Column(db.Integer, db.ForeignKey('c_attributes.id'), nullable=False) + value = db.Column(db.DateTime, nullable=False) + + ci = db.relationship("CI", backref="c_value_datetime.ci_id") + attr = db.relationship("Attribute", backref="c_value_datetime.attr_id") class CIValueJson(Model):