dashboard/infrastructure/utils.py

# Copyright © 2022 Aravinth Manivannan <realaravinth@batsense.net>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
import os
import shutil
import yaml
import requests
from pathlib import Path
from threading import Thread, Event
from time import sleep

from django.utils.crypto import get_random_string
from django.template.loader import render_to_string
from django.core.mail import send_mail
from django.conf import settings
from git import Repo, Commit
from git.exc import InvalidGitRepositoryError

from dash.models import Instance

from infrastructure.models import InstanceCreated, JobType, Job


class Worker(Thread):
    def __init__(self, job: Job):
        self.job = job
        super().__init__()

    def run(self):
        gitea_uri = Infra.get_gitea_uri(instance=self.job.instance)
        woodpecker = Infra.get_woodpecker_uri(instance=self.job.instance)
        while True:
            try:
                print(f"[ping] Trying to reach {gitea_uri}")
                resp = requests.get(gitea_uri)
                if resp.status_code == 200:
                    break
            except Exception:
                return False
            sleep(10)

        job = self.job
        self.job = None
        email = job.instance.owned_by.email

        ctx = {
            "gitea_uri": gitea_uri,
            "woodpecker_uri": woodpecker,
            "username": job.instance.owned_by.username,
        }
        body = render_to_string(
            "infrastructure/emails/instance-created.txt",
            context=ctx,
        )

        sender = settings.DEFAULT_FROM_EMAIL

        send_mail(
            subject="[Hostea] Your Hostea instance is now online!",
            message=body,
            from_email=f"No reply Hostea<{sender}>",  # TODO read from settings.py
            recipient_list=[email],
        )
        job.delete()


def create_vm_if_not_exists(instance: Instance) -> (str, Commit):
    """
    Create VM utility. Gitea password is returned
    """
    infra = Infra()
    if not InstanceCreated.objects.filter(instance=instance).exists():
        (gitea_password, commit) = infra.add_vm(instance=instance)
        InstanceCreated.objects.create(instance=instance, created=True)
        job = Job.objects.create(instance=instance, job_type=str(JobType.PING))
        Worker(job=job).start()
        return (gitea_password, commit)
    else:
        if str.strip(infra.get_flavor(instance=instance)) != str.strip(
            infra.translate_size(instance=instance)
        ):
            # Worker.init_global()
            return infra.add_vm(instance=instance)
    return None


def delete_vm(instance: Instance):
    infra = Infra()
    infra.remove_vm(instance=instance)
    if InstanceCreated.objects.filter(instance=instance).exists():
        InstanceCreated.objects.get(instance=instance).delete()
    instance.delete()


class Infra:
    """
    Utility function to manage infrastructure repository
    """

    def __init__(self):
        conf = settings.HOSTEA["INFRA"]["HOSTEA_REPO"]
        self.repo_path = Path(conf["PATH"])
        if not self.repo_path.exists():
            os.makedirs(self.repo_path)

        ssh_cmd = f"/usr/bin/ssh -oStrictHostKeyChecking=no -i {conf['SSH_KEY']}"
        self.env = {"GIT_SSH_COMMAND": ssh_cmd}
        try:
            self.repo = Repo(path=self.repo_path)
        except InvalidGitRepositoryError:
            self.repo = Repo.clone_from(conf["REMOTE"], self.repo_path, env=self.env)

    def _host_vars_dir(self, subdomain: str) -> Path:
        """
        utility method: get host_vars directory for a subdomain
        """
        return self.repo_path.joinpath(f"inventory/host_vars/{subdomain}-host/")

    def _provision_path(self, subdomain: str) -> Path:
        """
        utility method: get provision file pay for a subdomain
        """
        return self._host_vars_dir(subdomain=subdomain).joinpath("provision.yml")

    @classmethod
    def get_gitea_uri(cls, instance: Instance) -> str:
        """
        Get an instance's Gitea URI
        """
        base = settings.HOSTEA["INFRA"]["HOSTEA_DOMAIN"]
        return f"https://{instance.name}.{base}"

    @classmethod
    def _gen_woodpecker_hostname(cls, instance: Instance) -> str:
        """
        Get Woodpecker hostname of an instance
        """
        return f"{instance.name}-ci"

    @classmethod
    def get_woodpecker_uri(cls, instance: Instance) -> str:
        """
        Get an instance's Gitea URI
        """
        base = settings.HOSTEA["INFRA"]["HOSTEA_DOMAIN"]
        return f"https://{cls._gen_woodpecker_hostname(instance=instance)}.{base}"

    def get_flavor(self, instance: Instance):
        """
        Get VM flavour/size/configuration from the fleet repository
        """
        subdomain = instance.name
        provision = self._provision_path(subdomain)
        with open(provision, "r", encoding="utf-8") as f:
            config = yaml.safe_load(f)
            if "openstack_flavor" in config:
                return config["openstack_flavor"].split("{{ ")[1].split(" }}")[0]
        return None

    def _gitea_path(self, subdomain: str) -> Path:
        """
        utility method: get gitea file for a subdomain
        """

        return self._host_vars_dir(subdomain=subdomain).joinpath("gitea.yml")

    def _backup_path(self, subdomain: str) -> Path:
        """
        utility method: get backup file for a subdomain
        """

        return self.repo_path.joinpath(f"inventory/{subdomain}-backup.yml")

    def _service_path(self, subdomain: str) -> Path:
        """
        utility method: get service file for a subdomain
        """

        return self.repo_path.joinpath(f"inventory/{subdomain}-service.yml")

    def _hostscript_path(self, subdomain: str) -> Path:
        """
        utility method: hostscript file for a subdomain
        """
        return self.repo_path.joinpath(f"hosts-scripts/{subdomain}-host.sh")

    def write_hostscript(self, subdomain: str, content: str):
        """
        Write contents to hostscript.
        Hostscript will contain the history of all actions that have been
        ordered on a particular VM. So content needs to be appended to it,
        rather than overwritten.
        """
        hostscript = self._hostscript_path(subdomain)
        with open(hostscript, "a", encoding="utf-8") as f:
            f.write(content)
            f.write("\n")

    def _add_files(self, subdomain: str):
        """
        Add all relevant files of a VM
        """
        self.repo.git.add(str(self._host_vars_dir(subdomain=subdomain)))
        self.repo.git.add(str(self._backup_path(subdomain=subdomain)))
        self.repo.git.add(str(self._service_path(subdomain=subdomain)))
        self.repo.git.add(str(self._hostscript_path(subdomain=subdomain)))

    def _commit(self, action: str, subdomain: str) -> Commit:
        """
        Commit changes to a VM configuration
        """

        self._add_files(subdomain=subdomain)
        return self.repo.git.commit(
            message=f"{action} VM {subdomain}",
            author="Dashboard Bot <bot@dashboard.hostea.org>",
        )

    def _pull(self):
        self.repo.git.fetch(env=self.env)

    # TODO: switch to using Git cmd

    @staticmethod
    def translate_size(instance: Instance) -> str:
        """
        Translate openstack(I think OVH-specific) sizes to enough.community
        normalized sizes
        """
        if instance.configuration_id.name == "s1-2":
            return "openstack_flavor_small"
        if instance.configuration_id.name == "s1-4":
            return "openstack_flavor_medium"
        if instance.configuration_id.name == "s1-8":
            return "openstack_flavor_large"
        return instance.configuration_id.name

    def add_vm(self, instance: Instance) -> (str, Commit):
        """
        Add new VM to infrastructure repository

        The gitea user password is returned
        """
        self._pull()

        subdomain = instance.name
        host_vars_dir = self._host_vars_dir(subdomain)

        if not host_vars_dir.exists():
            os.makedirs(host_vars_dir)

        hostscript_path = self.repo_path.joinpath("hosts-scripts/")
        if not hostscript_path.exists():
            os.makedirs(hostscript_path)

        woodpecker_agent_secret = get_random_string(64)
        gitea_password = get_random_string(20)

        ctx = {
            "woodpecker_agent_secret": woodpecker_agent_secret,
            "woodpecker_hostname": self._gen_woodpecker_hostname(instance=instance),
            "woodpecker_admins": f"{instance.owned_by.username}",
            "gitea_email": instance.owned_by.email,
            "gitea_password": gitea_password,
            "subdomain": subdomain,
        }

        gitea = self._gitea_path(subdomain)
        with open(gitea, "w+", encoding="utf-8") as f:
            f.write(
                render_to_string(
                    "infrastructure/yml/gitea.yml",
                    context=ctx,
                )
            )

        size = self.translate_size(instance=instance)
        provision = self._provision_path(subdomain)

        with open(provision, "w+", encoding="utf-8") as f:
            f.write(
                render_to_string(
                    "infrastructure/yml/provision.yml", context={"vm_size": size}
                )
            )

        backup = self._backup_path(subdomain)
        with open(backup, "w+", encoding="utf-8") as f:
            f.write(
                render_to_string(
                    "infrastructure/yml/backups.yml", context={"subdomain": subdomain}
                )
            )

        service = self._service_path(subdomain)
        with open(service, "w+", encoding="utf-8") as f:
            f.write(
                render_to_string(
                    "infrastructure/yml/service.yml", context={"subdomain": subdomain}
                )
            )

        hostscript = self._hostscript_path(subdomain)
        with open(hostscript, "w+", encoding="utf-8") as f:
            f.write("\n")

        self.write_hostscript(
            subdomain=subdomain,
            content=render_to_string(
                "infrastructure/sh/hostscripts/create.sh",
                context={"subdomain": subdomain},
            ),
        )

        commit = self._commit(action="add", subdomain=subdomain)
        self.repo.git.push(env=self.env)
        return (gitea_password, commit)

    def remove_vm(self, instance: Instance):
        """
        Remove a VM from infrastructure repository
        """
        self._pull()
        subdomain = instance.name

        try:

            host_vars_dir = self._host_vars_dir(subdomain)
            shutil.rmtree(host_vars_dir)

            backup = self._backup_path(subdomain)
            os.remove(backup)

            service = self._service_path(subdomain)
            os.remove(service)
        except FileNotFoundError:
            pass

        hostscript = self._hostscript_path(subdomain)
        with open(hostscript, "w+", encoding="utf-8") as f:
            f.write("\n")

        self.write_hostscript(
            subdomain=subdomain,
            content=render_to_string(
                "infrastructure/sh/hostscripts/rm.sh",
                context={"subdomain": subdomain},
            ),
        )
        self._commit(action="rm", subdomain=subdomain)
        self.repo.git.push(env=self.env)