cmdb/cmdb-api/api/commands/click_cmdb.py

# -*- coding:utf-8 -*-


import copy
import datetime
import json
import time

import click
import requests
from flask import current_app
from flask.cli import with_appcontext
from flask_login import login_user

import api.lib.cmdb.ci
from api.extensions import db
from api.extensions import rd
from api.lib.cmdb.cache import AttributeCache
from api.lib.cmdb.const import PermEnum
from api.lib.cmdb.const import REDIS_PREFIX_CI
from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
from api.lib.cmdb.const import ResourceTypeEnum
from api.lib.cmdb.const import RoleEnum
from api.lib.cmdb.const import ValueTypeEnum
from api.lib.exception import AbortException
from api.lib.perm.acl.acl import ACLManager
from api.lib.perm.acl.acl import UserCache
from api.lib.perm.acl.cache import AppCache
from api.lib.perm.acl.resource import ResourceCRUD
from api.lib.perm.acl.resource import ResourceTypeCRUD
from api.lib.perm.acl.role import RoleCRUD
from api.lib.perm.acl.user import UserCRUD
from api.lib.secrets.inner import KeyManage
from api.lib.secrets.inner import global_key_threshold
from api.lib.secrets.secrets import InnerKVManger
from api.models.acl import App
from api.models.acl import ResourceType
from api.models.cmdb import Attribute
from api.models.cmdb import CI
from api.models.cmdb import CIRelation
from api.models.cmdb import CIType
from api.models.cmdb import CITypeTrigger
from api.models.cmdb import PreferenceRelationView


@click.command()
@with_appcontext
def cmdb_init_cache():
    db.session.remove()

    ci_relations = CIRelation.get_by(to_dict=False)
    relations = dict()
    for cr in ci_relations:
        relations.setdefault(cr.first_ci_id, {}).update({cr.second_ci_id: cr.second_ci.type_id})
    for i in relations:
        relations[i] = json.dumps(relations[i])
    if relations:
        rd.create_or_update(relations, REDIS_PREFIX_CI_RELATION)

    es = None
    if current_app.config.get("USE_ES"):
        from api.extensions import es
        from api.models.cmdb import Attribute
        from api.lib.cmdb.utils import ValueTypeMap
        attributes = Attribute.get_by(to_dict=False)
        for attr in attributes:
            other = dict()
            other['index'] = True if attr.is_index else False
            if attr.value_type == ValueTypeEnum.TEXT:
                other['analyzer'] = 'ik_max_word'
                other['search_analyzer'] = 'ik_smart'
                if attr.is_index:
                    other["fields"] = {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
            try:
                es.update_mapping(attr.name, ValueTypeMap.es_type[attr.value_type], other)
            except Exception as e:
                print(e)

    cis = CI.get_by(to_dict=False)
    for ci in cis:
        if current_app.config.get("USE_ES"):
            res = es.get_index_id(ci.id)
            if res:
                continue
        else:
            res = rd.get([ci.id], REDIS_PREFIX_CI)
            if res and list(filter(lambda x: x, res)):
                continue

        m = api.lib.cmdb.ci.CIManager()
        ci_dict = m.get_ci_by_id_from_db(ci.id, need_children=False, use_master=False)

        if current_app.config.get("USE_ES"):
            es.create(ci_dict)
        else:
            rd.create_or_update({ci.id: json.dumps(ci_dict)}, REDIS_PREFIX_CI)

    db.session.remove()


@click.command()
@with_appcontext
def cmdb_init_acl():
    _app = AppCache.get('cmdb') or App.create(name='cmdb')
    app_id = _app.id

    # 1. add resource type
    for resource_type in ResourceTypeEnum.all():
        try:
            ResourceTypeCRUD.add(app_id, resource_type, '', PermEnum.all())
        except AbortException:
            pass

    # 2. add role
    try:
        RoleCRUD.add_role(RoleEnum.CONFIG, app_id, True)
    except AbortException:
        pass
    try:
        RoleCRUD.add_role(RoleEnum.CMDB_READ_ALL, app_id, False)
    except AbortException:
        pass

    # 3. add resource and grant
    ci_types = CIType.get_by(to_dict=False)
    type_id = ResourceType.get_by(name=ResourceTypeEnum.CI, first=True, to_dict=False).id
    for ci_type in ci_types:
        try:
            ResourceCRUD.add(ci_type.name, type_id, app_id)
        except AbortException:
            pass

        ACLManager().grant_resource_to_role(ci_type.name,
                                            RoleEnum.CMDB_READ_ALL,
                                            ResourceTypeEnum.CI,
                                            [PermEnum.READ])

    relation_views = PreferenceRelationView.get_by(to_dict=False)
    type_id = ResourceType.get_by(name=ResourceTypeEnum.RELATION_VIEW, first=True, to_dict=False).id
    for view in relation_views:
        try:
            ResourceCRUD.add(view.name, type_id, app_id)
        except AbortException:
            pass

        ACLManager().grant_resource_to_role(view.name,
                                            RoleEnum.CMDB_READ_ALL,
                                            ResourceTypeEnum.RELATION_VIEW,
                                            [PermEnum.READ])


@click.command()
@click.option(
    '-u',
    '--user',
    help='username'
)
@click.option(
    '-p',
    '--password',
    help='password'
)
@click.option(
    '-m',
    '--mail',
    help='mail'
)
@with_appcontext
def add_user(user, password, mail):
    """
    create a user

    is_admin: default is False

    Example:  flask add-user -u <username> -p <password> -m <mail>
    """
    assert user is not None
    assert password is not None
    assert mail is not None
    UserCRUD.add(username=user, password=password, email=mail)


@click.command()
@click.option(
    '-u',
    '--user',
    help='username'
)
@with_appcontext
def del_user(user):
    """
    delete a user

    Example:  flask del-user -u <username>
    """
    assert user is not None
    from api.models.acl import User

    u = User.get_by(username=user, first=True, to_dict=False)
    u and UserCRUD.delete(u.uid)


@click.command()
@with_appcontext
def cmdb_counter():
    """
    Dashboard calculations
    """
    from api.lib.cmdb.cache import CMDBCounterCache

    current_app.test_request_context().push()
    login_user(UserCache.get('worker'))
    while True:
        try:
            db.session.remove()

            CMDBCounterCache.reset()
        except:
            import traceback
            print(traceback.format_exc())

        time.sleep(60)


@click.command()
@with_appcontext
def cmdb_trigger():
    """
    Trigger execution for date attribute
    """
    from api.lib.cmdb.ci import CITriggerManager

    current_day = datetime.datetime.today().strftime("%Y-%m-%d")
    trigger2cis = dict()
    trigger2completed = dict()

    i = 0
    while True:
        try:
            db.session.remove()

            if datetime.datetime.today().strftime("%Y-%m-%d") != current_day:
                trigger2cis = dict()
                trigger2completed = dict()
                current_day = datetime.datetime.today().strftime("%Y-%m-%d")

            if i == 3 or i == 0:
                i = 0
                triggers = CITypeTrigger.get_by(to_dict=False, __func_isnot__key_attr_id=None)
                for trigger in triggers:
                    try:
                        ready_cis = CITriggerManager.waiting_cis(trigger)
                    except Exception as e:
                        print(e)
                        continue

                    if trigger.id not in trigger2cis:
                        trigger2cis[trigger.id] = (trigger, ready_cis)
                    else:
                        cur = trigger2cis[trigger.id]
                        cur_ci_ids = {i.ci_id for i in cur[1]}
                        trigger2cis[trigger.id] = (
                            trigger, cur[1] + [i for i in ready_cis if i.ci_id not in cur_ci_ids
                                               and i.ci_id not in trigger2completed.get(trigger.id, {})])

            for tid in trigger2cis:
                trigger, cis = trigger2cis[tid]
                for ci in copy.deepcopy(cis):
                    if CITriggerManager.trigger_notify(trigger, ci):
                        trigger2completed.setdefault(trigger.id, set()).add(ci.ci_id)

                        for _ci in cis:
                            if _ci.ci_id == ci.ci_id:
                                cis.remove(_ci)

            i += 1
            time.sleep(10)
        except Exception as e:
            import traceback
            print(traceback.format_exc())
            current_app.logger.error("cmdb trigger exception: {}".format(e))
            time.sleep(60)


@click.command()
@with_appcontext
def cmdb_index_table_upgrade():
    """
    Migrate data from tables c_value_integers, c_value_floats, and c_value_datetime
    """
    for attr in Attribute.get_by(to_dict=False):
        if attr.value_type not in {ValueTypeEnum.TEXT, ValueTypeEnum.JSON} and not attr.is_index:
            attr.update(is_index=True)
            AttributeCache.clean(attr)

    from api.models.cmdb import CIValueInteger, CIIndexValueInteger
    from api.models.cmdb import CIValueFloat, CIIndexValueFloat
    from api.models.cmdb import CIValueDateTime, CIIndexValueDateTime

    for i in CIValueInteger.get_by(to_dict=False):
        CIIndexValueInteger.create(ci_id=i.ci_id, attr_id=i.attr_id, value=i.value, commit=False)
        i.delete(commit=False)
    db.session.commit()

    for i in CIValueFloat.get_by(to_dict=False):
        CIIndexValueFloat.create(ci_id=i.ci_id, attr_id=i.attr_id, value=i.value, commit=False)
        i.delete(commit=False)
    db.session.commit()

    for i in CIValueDateTime.get_by(to_dict=False):
        CIIndexValueDateTime.create(ci_id=i.ci_id, attr_id=i.attr_id, value=i.value, commit=False)
        i.delete(commit=False)
    db.session.commit()


@click.command()
@click.option(
    '-a',
    '--address',
    help='inner cmdb api, http://127.0.0.1:8000',
)
@with_appcontext
def cmdb_inner_secrets_init(address):
    """
    init inner secrets for password feature
    """
    KeyManage(backend=InnerKVManger).init()

    if address and address.startswith("http") and current_app.config.get("INNER_TRIGGER_TOKEN", "") != "":
        resp = requests.post("{}/api/v0.1/secrets/auto_seal".format(address.strip("/")),
                             headers={"Inner-Token": current_app.config.get("INNER_TRIGGER_TOKEN", "")})
        if resp.status_code == 200:
            KeyManage.print_response(resp.json())
        else:
            KeyManage.print_response({"message": resp.text, "status": "failed"})


@click.command()
@click.option(
    '-a',
    '--address',
    help='inner cmdb api, http://127.0.0.1:8000',
    required=True,
)
@with_appcontext
def cmdb_inner_secrets_unseal(address):
    """
    unseal the secrets feature
    """
    address = "{}/api/v0.1/secrets/unseal".format(address.strip("/"))
    if not address.startswith("http"):
        KeyManage.print_response({"message": "invalid address, should start with http", "status": "failed"})
        return
    for i in range(global_key_threshold):
        token = click.prompt(f'Enter unseal token {i + 1}', hide_input=True, confirmation_prompt=False)
        assert token is not None
        resp = requests.post(address, headers={"Unseal-Token": token})
        if resp.status_code == 200:
            KeyManage.print_response(resp.json())
        else:
            KeyManage.print_response({"message": resp.text, "status": "failed"})
            return


@click.command()
@click.option(
    '-a',
    '--address',
    help='inner cmdb api, http://127.0.0.1:8000',
    required=True,
)
@click.option(
    '-k',
    '--token',
    help='root token',
    prompt=True,
    hide_input=True,
)
@with_appcontext
def cmdb_inner_secrets_seal(address, token):
    """
    seal the secrets feature
    """
    assert address is not None
    assert token is not None
    if address.startswith("http"):
        address = "{}/api/v0.1/secrets/seal".format(address.strip("/"))
        resp = requests.post(address, headers={
            "Inner-Token": token,
        })
        if resp.status_code == 200:
            KeyManage.print_response(resp.json())
        else:
            KeyManage.print_response({"message": resp.text, "status": "failed"})


@click.command()
@with_appcontext
def cmdb_password_data_migrate():
    """
    Migrate CI password data, version >= v2.3.6
    """
    from api.models.cmdb import CIIndexValueText
    from api.models.cmdb import CIValueText
    from api.lib.secrets.inner import InnerCrypt
    from api.lib.secrets.vault import VaultClient

    attrs = Attribute.get_by(to_dict=False)
    for attr in attrs:
        if attr.is_password:

            value_table = CIIndexValueText if attr.is_index else CIValueText

            for i in value_table.get_by(attr_id=attr.id, to_dict=False):
                if current_app.config.get("SECRETS_ENGINE", 'inner') == 'inner':
                    _, status = InnerCrypt().decrypt(i.value)
                    if status:
                        continue

                    encrypt_value, status = InnerCrypt().encrypt(i.value)
                    if status:
                        CIValueText.create(ci_id=i.ci_id, attr_id=attr.id, value=encrypt_value)
                    else:
                        continue
                elif current_app.config.get("SECRETS_ENGINE") == 'vault':
                    if i.value == '******':
                        continue

                    vault = VaultClient(current_app.config.get('VAULT_URL'), current_app.config.get('VAULT_TOKEN'))
                    try:
                        vault.update("/{}/{}".format(i.ci_id, i.attr_id), dict(v=i.value))
                    except Exception as e:
                        print('save password to vault failed: {}'.format(e))
                        continue
                else:
                    continue

                i.delete()