cmdb/cmdb-api/api/commands/click_cmdb.py

# -*- coding:utf-8 -*-


import copy
import datetime
import json
import time
import uuid

import click
import requests
from flask import current_app
from flask.cli import with_appcontext
from flask_login import login_user

import api.lib.cmdb.ci
from api.extensions import db
from api.extensions import rd
from api.lib.cmdb.cache import AttributeCache
from api.lib.cmdb.const import PermEnum
from api.lib.cmdb.const import REDIS_PREFIX_CI
from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION
from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION2
from api.lib.cmdb.const import ResourceTypeEnum
from api.lib.cmdb.const import RoleEnum
from api.lib.cmdb.const import ValueTypeEnum
from api.lib.exception import AbortException
from api.lib.perm.acl.acl import ACLManager
from api.lib.perm.acl.acl import UserCache
from api.lib.perm.acl.cache import AppCache
from api.lib.perm.acl.resource import ResourceCRUD
from api.lib.perm.acl.resource import ResourceTypeCRUD
from api.lib.perm.acl.role import RoleCRUD
from api.lib.secrets.inner import KeyManage
from api.lib.secrets.inner import global_key_threshold
from api.lib.secrets.secrets import InnerKVManger
from api.models.acl import App
from api.models.acl import ResourceType
from api.models.cmdb import Attribute
from api.models.cmdb import CI
from api.models.cmdb import CIRelation
from api.models.cmdb import CIType
from api.models.cmdb import CITypeTrigger
from api.models.cmdb import PreferenceRelationView


@click.command()
@with_appcontext
def cmdb_init_cache():
    db.session.remove()

    ci_relations = CIRelation.get_by(to_dict=False)
    relations = dict()
    relations2 = dict()
    for cr in ci_relations:
        relations.setdefault(cr.first_ci_id, {}).update({cr.second_ci_id: cr.second_ci.type_id})
        if cr.ancestor_ids:
            relations2.setdefault('{},{}'.format(cr.ancestor_ids, cr.first_ci_id), {}).update(
                {cr.second_ci_id: cr.second_ci.type_id})
    for i in relations:
        relations[i] = json.dumps(relations[i])
    for i in relations2:
        relations2[i] = json.dumps(relations2[i])
    if relations:
        rd.create_or_update(relations, REDIS_PREFIX_CI_RELATION)
    if relations2:
        rd.create_or_update(relations2, REDIS_PREFIX_CI_RELATION2)

    es = None
    if current_app.config.get("USE_ES"):
        from api.extensions import es
        from api.models.cmdb import Attribute
        from api.lib.cmdb.utils import ValueTypeMap
        attributes = Attribute.get_by(to_dict=False)
        for attr in attributes:
            other = dict()
            other['index'] = True if attr.is_index else False
            if attr.value_type == ValueTypeEnum.TEXT:
                other['analyzer'] = 'ik_max_word'
                other['search_analyzer'] = 'ik_smart'
                if attr.is_index:
                    other["fields"] = {
                        "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                        }
                    }
            try:
                es.update_mapping(attr.name, ValueTypeMap.es_type[attr.value_type], other)
            except Exception as e:
                print(e)

    cis = CI.get_by(to_dict=False)
    for ci in cis:
        if current_app.config.get("USE_ES"):
            res = es.get_index_id(ci.id)
            if res:
                continue
        else:
            res = rd.get([ci.id], REDIS_PREFIX_CI)
            if res and list(filter(lambda x: x, res)):
                continue

        m = api.lib.cmdb.ci.CIManager()
        ci_dict = m.get_ci_by_id_from_db(ci.id, need_children=False, use_master=False)

        if current_app.config.get("USE_ES"):
            es.create(ci_dict)
        else:
            rd.create_or_update({ci.id: json.dumps(ci_dict)}, REDIS_PREFIX_CI)

    db.session.remove()


@click.command()
@with_appcontext
def cmdb_init_acl():
    _app = AppCache.get('cmdb') or App.create(name='cmdb')
    app_id = _app.id

    current_app.test_request_context().push()

    # 1. add resource type
    for resource_type in ResourceTypeEnum.all():
        try:
            perms = PermEnum.all()
            if resource_type in (ResourceTypeEnum.CI_FILTER, ResourceTypeEnum.PAGE):
                perms = [PermEnum.READ]
            elif resource_type == ResourceTypeEnum.CI_TYPE_RELATION:
                perms = [PermEnum.ADD, PermEnum.DELETE, PermEnum.GRANT]
            elif resource_type in (ResourceTypeEnum.RELATION_VIEW, ResourceTypeEnum.TOPOLOGY_VIEW):
                perms = [PermEnum.READ, PermEnum.UPDATE, PermEnum.DELETE, PermEnum.GRANT]

            ResourceTypeCRUD.add(app_id, resource_type, '', perms)
        except AbortException:
            pass

    # 2. add role
    try:
        RoleCRUD.add_role(RoleEnum.CONFIG, app_id, True)
    except AbortException:
        pass
    try:
        RoleCRUD.add_role(RoleEnum.CMDB_READ_ALL, app_id, False)
    except AbortException:
        pass

    # 3. add resource and grant
    ci_types = CIType.get_by(to_dict=False)
    resource_type_id = ResourceType.get_by(name=ResourceTypeEnum.CI, first=True, to_dict=False).id
    for ci_type in ci_types:
        try:
            ResourceCRUD.add(ci_type.name, resource_type_id, app_id)
        except AbortException:
            pass

        ACLManager().grant_resource_to_role(ci_type.name,
                                            RoleEnum.CMDB_READ_ALL,
                                            ResourceTypeEnum.CI,
                                            [PermEnum.READ])

    relation_views = PreferenceRelationView.get_by(to_dict=False)
    resource_type_id = ResourceType.get_by(name=ResourceTypeEnum.RELATION_VIEW, first=True, to_dict=False).id
    for view in relation_views:
        try:
            ResourceCRUD.add(view.name, resource_type_id, app_id)
        except AbortException:
            pass

        ACLManager().grant_resource_to_role(view.name,
                                            RoleEnum.CMDB_READ_ALL,
                                            ResourceTypeEnum.RELATION_VIEW,
                                            [PermEnum.READ])


@click.command()
@with_appcontext
def cmdb_counter():
    """
    Dashboard calculations
    """
    from api.lib.cmdb.cache import CMDBCounterCache

    current_app.test_request_context().push()
    if not UserCache.get('worker'):
        from api.lib.perm.acl.user import UserCRUD

        UserCRUD.add(username='worker', password=uuid.uuid4().hex, email='worker@xxx.com')

    login_user(UserCache.get('worker'))

    i = 0
    today = datetime.date.today()
    while True:
        try:
            db.session.remove()

            CMDBCounterCache.reset()

            if i % 5 == 0:
                CMDBCounterCache.flush_adc_counter()
                i = 0

            if datetime.date.today() != today:
                CMDBCounterCache.clear_ad_exec_history()
                today = datetime.date.today()

            CMDBCounterCache.flush_sub_counter()

            i += 1
        except:
            import traceback
            print(traceback.format_exc())

        time.sleep(60)


@click.command()
@with_appcontext
def cmdb_trigger():
    """
    Trigger execution for date attribute
    """
    from api.lib.cmdb.ci import CITriggerManager

    current_day = datetime.datetime.today().strftime("%Y-%m-%d")
    trigger2cis = dict()
    trigger2completed = dict()

    i = 0
    while True:
        try:
            db.session.remove()

            if datetime.datetime.today().strftime("%Y-%m-%d") != current_day:
                trigger2cis = dict()
                trigger2completed = dict()
                current_day = datetime.datetime.today().strftime("%Y-%m-%d")

            if i == 3 or i == 0:
                i = 0
                triggers = CITypeTrigger.get_by(to_dict=False, __func_isnot__key_attr_id=None)
                for trigger in triggers:
                    try:
                        ready_cis = CITriggerManager.waiting_cis(trigger)
                    except Exception as e:
                        print(e)
                        continue

                    if trigger.id not in trigger2cis:
                        trigger2cis[trigger.id] = (trigger, ready_cis)
                    else:
                        cur = trigger2cis[trigger.id]
                        cur_ci_ids = {i.ci_id for i in cur[1]}
                        trigger2cis[trigger.id] = (
                            trigger, cur[1] + [i for i in ready_cis if i.ci_id not in cur_ci_ids
                                               and i.ci_id not in trigger2completed.get(trigger.id, {})])

            for tid in trigger2cis:
                trigger, cis = trigger2cis[tid]
                for ci in copy.deepcopy(cis):
                    if CITriggerManager.trigger_notify(trigger, ci):
                        trigger2completed.setdefault(trigger.id, set()).add(ci.ci_id)

                        for _ci in cis:
                            if _ci.ci_id == ci.ci_id:
                                cis.remove(_ci)

            i += 1
            time.sleep(10)
        except Exception as e:
            import traceback
            print(traceback.format_exc())
            current_app.logger.error("cmdb trigger exception: {}".format(e))
            time.sleep(60)


@click.command()
@with_appcontext
def cmdb_index_table_upgrade():
    """
    Migrate data from tables c_value_integers, c_value_floats, and c_value_datetime
    """
    for attr in Attribute.get_by(to_dict=False):
        if attr.value_type not in {ValueTypeEnum.TEXT, ValueTypeEnum.JSON} and not attr.is_index:
            attr.update(is_index=True)
            AttributeCache.clean(attr)

    from api.models.cmdb import CIValueInteger, CIIndexValueInteger
    from api.models.cmdb import CIValueFloat, CIIndexValueFloat
    from api.models.cmdb import CIValueDateTime, CIIndexValueDateTime

    for i in CIValueInteger.get_by(to_dict=False):
        CIIndexValueInteger.create(ci_id=i.ci_id, attr_id=i.attr_id, value=i.value, commit=False)
        i.delete(commit=False)
    db.session.commit()

    for i in CIValueFloat.get_by(to_dict=False):
        CIIndexValueFloat.create(ci_id=i.ci_id, attr_id=i.attr_id, value=i.value, commit=False)
        i.delete(commit=False)
    db.session.commit()

    for i in CIValueDateTime.get_by(to_dict=False):
        CIIndexValueDateTime.create(ci_id=i.ci_id, attr_id=i.attr_id, value=i.value, commit=False)
        i.delete(commit=False)
    db.session.commit()


def valid_address(address):
    if not address:
        return False

    if not address.startswith(("http://127.0.0.1", "https://127.0.0.1")):
        response = {
            "message": "Address should start with http://127.0.0.1 or https://127.0.0.1",
            "status": "failed"
        }
        KeyManage.print_response(response)
        return False
    return True


@click.command()
@click.option(
    '-a',
    '--address',
    help='inner cmdb api, http://127.0.0.1:8000',
)
@with_appcontext
def cmdb_inner_secrets_init(address):
    """
    init inner secrets for password feature
    """
    res, ok = KeyManage(backend=InnerKVManger()).init()
    if not ok:
        if res.get("status") == "failed":
            KeyManage.print_response(res)
            return

    token = res.get("details", {}).get("root_token", "")
    if valid_address(address):
        token = current_app.config.get("INNER_TRIGGER_TOKEN", "") if not token else token
        if not token:
            token = click.prompt(f'Enter root token', hide_input=True, confirmation_prompt=False)
        assert token is not None
        resp = requests.post("{}/api/v0.1/secrets/auto_seal".format(address.strip("/")),
                             headers={"Inner-Token": token})
        if resp.status_code == 200:
            KeyManage.print_response(resp.json())
        else:
            KeyManage.print_response({"message": resp.text or resp.status_code, "status": "failed"})
    else:
        KeyManage.print_response(res)


@click.command()
@click.option(
    '-a',
    '--address',
    help='inner cmdb api, http://127.0.0.1:8000',
    required=True,
)
@with_appcontext
def cmdb_inner_secrets_unseal(address):
    """
    unseal the secrets feature
    """
    # if not valid_address(address):
    #     return
    address = "{}/api/v0.1/secrets/unseal".format(address.strip("/"))
    for i in range(global_key_threshold):
        token = click.prompt(f'Enter unseal token {i + 1}', hide_input=True, confirmation_prompt=False)
        assert token is not None
        resp = requests.post(address, headers={"Unseal-Token": token}, timeout=5)
        if resp.status_code == 200:
            KeyManage.print_response(resp.json())
            if resp.json().get("status") in ["success", "skip"]:
                return
        else:
            KeyManage.print_response({"message": resp.status_code, "status": "failed"})
            return


@click.command()
@click.option(
    '-a',
    '--address',
    help='inner cmdb api, http://127.0.0.1:8000',
    required=True,
)
@click.option(
    '-k',
    '--token',
    help='root token',
    prompt=True,
    hide_input=True,
)
@with_appcontext
def cmdb_inner_secrets_seal(address, token):
    """
    seal the secrets feature
    """
    assert address is not None
    assert token is not None
    if not valid_address(address):
        return
    address = "{}/api/v0.1/secrets/seal".format(address.strip("/"))
    resp = requests.post(address, headers={
        "Inner-Token": token,
    })
    if resp.status_code == 200:
        KeyManage.print_response(resp.json())
    else:
        KeyManage.print_response({"message": resp.status_code, "status": "failed"})


@click.command()
@with_appcontext
def cmdb_password_data_migrate():
    """
    Migrate CI password data, version >= v2.3.6
    """
    from api.models.cmdb import CIIndexValueText
    from api.models.cmdb import CIValueText
    from api.lib.secrets.inner import InnerCrypt
    from api.lib.secrets.vault import VaultClient

    attrs = Attribute.get_by(to_dict=False)
    for attr in attrs:
        if attr.is_password:

            value_table = CIIndexValueText if attr.is_index else CIValueText

            failed = False
            for i in value_table.get_by(attr_id=attr.id, to_dict=False):
                if current_app.config.get("SECRETS_ENGINE", 'inner') == 'inner':
                    _, status = InnerCrypt().decrypt(i.value)
                    if status:
                        continue

                    encrypt_value, status = InnerCrypt().encrypt(i.value)
                    if status:
                        CIValueText.create(ci_id=i.ci_id, attr_id=attr.id, value=encrypt_value)
                    else:
                        failed = True
                        continue
                elif current_app.config.get("SECRETS_ENGINE") == 'vault':
                    if i.value == '******':
                        continue

                    vault = VaultClient(current_app.config.get('VAULT_URL'), current_app.config.get('VAULT_TOKEN'))
                    try:
                        vault.update("/{}/{}".format(i.ci_id, i.attr_id), dict(v=i.value))
                    except Exception as e:
                        print('save password to vault failed: {}'.format(e))
                        failed = True
                        continue
                else:
                    continue

                i.delete()

                if not failed and attr.is_index:
                    attr.update(is_index=False)


@click.command()
@with_appcontext
def cmdb_agent_init():
    """
    Initialize the agent's permissions and obtain the key and secret
    """

    from api.models.acl import User

    user = User.get_by(username="cmdb_agent", first=True, to_dict=False)
    if user is None:
        click.echo(
            click.style('user cmdb_agent does not exist, please use flask add-user to create it first', fg='red'))
        return

    # grant
    _app = AppCache.get('cmdb') or App.create(name='cmdb')
    app_id = _app.id

    ci_types = CIType.get_by(to_dict=False)
    resource_type_id = ResourceType.get_by(name=ResourceTypeEnum.CI, first=True, to_dict=False).id
    for ci_type in ci_types:
        try:
            ResourceCRUD.add(ci_type.name, resource_type_id, app_id)
        except AbortException:
            pass

        ACLManager().grant_resource_to_role(ci_type.name,
                                            "cmdb_agent",
                                            ResourceTypeEnum.CI,
                                            [PermEnum.READ, PermEnum.UPDATE, PermEnum.ADD, PermEnum.DELETE])

    click.echo("Key   : {}".format(click.style(user.key, bg='red')))
    click.echo("Secret: {}".format(click.style(user.secret, bg='red')))


@click.command()
@click.option(
    '-v',
    '--version',
    help='input cmdb version, e.g. 2.4.6',
    required=True,
)
@with_appcontext
def cmdb_patch(version):
    """
    CMDB upgrade patch
    """

    version = version[1:] if version.lower().startswith("v") else version

    if version >= '2.4.6':

        from api.models.cmdb import CITypeRelation
        for cr in CITypeRelation.get_by(to_dict=False):
            if hasattr(cr, 'parent_attr_id') and cr.parent_attr_id and not cr.parent_attr_ids:
                parent_attr_ids, child_attr_ids = [cr.parent_attr_id], [cr.child_attr_id]
                cr.update(parent_attr_ids=parent_attr_ids, child_attr_ids=child_attr_ids, commit=False)
        db.session.commit()

        from api.models.cmdb import AutoDiscoveryCIType, AutoDiscoveryCITypeRelation
        from api.lib.cmdb.cache import CITypeCache, AttributeCache
        for adt in AutoDiscoveryCIType.get_by(to_dict=False):
            if adt.relation:
                if not AutoDiscoveryCITypeRelation.get_by(ad_type_id=adt.type_id):
                    peer_type = CITypeCache.get(list(adt.relation.values())[0]['type_name'])
                    peer_type_id = peer_type and peer_type.id
                    peer_attr = AttributeCache.get(list(adt.relation.values())[0]['attr_name'])
                    peer_attr_id = peer_attr and peer_attr.id
                    if peer_type_id and peer_attr_id:
                        AutoDiscoveryCITypeRelation.create(ad_type_id=adt.type_id,
                                                           ad_key=list(adt.relation.keys())[0],
                                                           peer_type_id=peer_type_id,
                                                           peer_attr_id=peer_attr_id,
                                                           commit=False)
            if hasattr(adt, 'interval') and adt.interval and not adt.cron:
                adt.cron = "*/{} * * * *".format(adt.interval // 60 or 1)

        db.session.commit()

    if version >= "2.4.7":
        from api.lib.cmdb.auto_discovery.const import DEFAULT_INNER
        from api.models.cmdb import AutoDiscoveryRule
        for i in DEFAULT_INNER:
            existed = AutoDiscoveryRule.get_by(name=i['name'], first=True, to_dict=False)
            if existed is not None:
                if "en" in i['option'] and 'en' not in (existed.option or {}):
                    option = copy.deepcopy(existed.option)
                    option['en'] = i['option']['en']
                    existed.update(option=option, commit=False)

        db.session.commit()