dashboard/infrastructure/utils.py

# Copyright © 2022 Aravinth Manivannan <realaravinth@batsense.net>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
import os
import shutil
import yaml
import requests
from pathlib import Path
from threading import Thread, Event
from time import sleep

from django.utils.crypto import get_random_string
from django.template.loader import render_to_string
from django.core.mail import send_mail
from django.conf import settings
from git import Repo, Commit
from git.exc import InvalidGitRepositoryError

from dash.models import Instance

from infrastructure.models import InstanceCreated, JobType, Job


class Worker(Thread):
    def __init__(self, job: Job):
        self.job = job
        super().__init__()

    #########        self.daemon = True

    def run(self):
        gitea_uri = Infra.get_gitea_uri(instance=self.job.instance)
        woodpecker = Infra.get_woodpecker_uri(instance=self.job.instance)
        while True:
            try:
                print(f"[ping] Trying to reach {gitea_uri}")
                resp = requests.get(gitea_uri)
                print(resp.status_code)
                if resp.status_code == 200:
                    break
            except Exception:
                return False
            sleep(10)

        print("sending email")
        job = self.job
        self.job = None
        email = job.instance.owned_by.email
        send_mail(
            subject="[Hostea] Your Hostea instance is now online!",
            message=f"""
Hello,

The deployment job has run to completion and your Hostea instance is now online!
Credentials to admin account was sent in an earlier email, please contact
support if didn't receive it.

Gitea: {gitea_uri}
Woodpecker CI: {woodpecker}
""",
            from_email="No reply Hostea<no-reply@exampl.org>",  # TODO read from settings.py
            recipient_list=[email],
        )
        job.delete()
        print("job deleted")


def create_vm_if_not_exists(instance: Instance) -> (str, Commit):
    """
    Create VM utility. Gitea password is returned
    """
    infra = Infra()
    if not InstanceCreated.objects.filter(instance=instance).exists():
        (gitea_password, commit) = infra.add_vm(instance=instance)
        InstanceCreated.objects.create(instance=instance, created=True)
        job = Job.objects.create(instance=instance, job_type=str(JobType.PING))
        Worker(job=job).start()
        return (gitea_password, commit)
    else:
        if str.strip(infra.get_flavor(instance=instance)) != str.strip(
            infra.translate_size(instance=instance)
        ):
            # Worker.init_global()
            return infra.add_vm(instance=instance)
    return None


def delete_vm(instance: Instance, owner: str):
    infra = Infra()
    infra.remove_vm(instance=instance)
    if InstanceCreated.objects.filter(instance=instance).exists():
        InstanceCreated.objects.get(instance=instance).delete()
    instance.delete()


class Infra:
    """
    Utility function to manage infrastructure repository
    """

    def __init__(self):
        conf = settings.HOSTEA["INFRA"]["HOSTEA_REPO"]
        self.repo_path = Path(conf["PATH"])
        if not self.repo_path.exists():
            os.makedirs(self.repo_path)

        ssh_cmd = f"/usr/bin/ssh -oStrictHostKeyChecking=no -i {conf['SSH_KEY']}"
        self.env = {"GIT_SSH_COMMAND": ssh_cmd}
        try:
            self.repo = Repo(path=self.repo_path)
        except InvalidGitRepositoryError:
            self.repo = Repo.clone_from(conf["REMOTE"], self.repo_path, env=self.env)

    def _host_vars_dir(self, subdomain: str) -> Path:
        """
        utility method: get host_vars directory for a subdomain
        """
        return self.repo_path.joinpath(f"inventory/host_vars/{subdomain}-host/")

    def _provision_path(self, subdomain: str) -> Path:
        """
        utility method: get provision file pay for a subdomain
        """
        return self._host_vars_dir(subdomain=subdomain).joinpath("provision.yml")

    @classmethod
    def get_gitea_uri(cls, instance: Instance) -> str:
        """
        Get an instance's Gitea URI
        """
        base = settings.HOSTEA["INFRA"]["HOSTEA_DOMAIN"]
        return f"https://{instance.name}.{base}"

    @classmethod
    def _gen_woodpecker_hostname(cls, instance: Instance) -> str:
        """
        Get Woodpecker hostname of an instance
        """
        return f"{instance.name}-ci"

    @classmethod
    def get_woodpecker_uri(cls, instance: Instance) -> str:
        """
        Get an instance's Gitea URI
        """
        base = settings.HOSTEA["INFRA"]["HOSTEA_DOMAIN"]
        return f"https://{cls._gen_woodpecker_hostname(instance=instance)}.{base}"

    def get_flavor(self, instance: Instance):
        """
        Get VM flavour/size/configuration from the fleet repository
        """
        subdomain = instance.name
        provision = self._provision_path(subdomain)
        with open(provision, "r", encoding="utf-8") as f:
            config = yaml.safe_load(f)
            if "openstack_flavor" in config:
                return config["openstack_flavor"].split("{{ ")[1].split(" }}")[0]
        return None

    def _gitea_path(self, subdomain: str) -> Path:
        """
        utility method: get gitea file for a subdomain
        """

        return self._host_vars_dir(subdomain=subdomain).joinpath("gitea.yml")

    def _backup_path(self, subdomain: str) -> Path:
        """
        utility method: get backup file for a subdomain
        """

        return self.repo_path.joinpath(f"inventory/{subdomain}-backup.yml")

    def _service_path(self, subdomain: str) -> Path:
        """
        utility method: get service file for a subdomain
        """

        return self.repo_path.joinpath(f"inventory/{subdomain}-service.yml")

    def _hostscript_path(self, subdomain: str) -> Path:
        """
        utility method: hostscript file for a subdomain
        """
        return self.repo_path.joinpath(f"hosts-scripts/{subdomain}-host.sh")

    def write_hostscript(self, subdomain: str, content: str):
        """
        Write contents to hostscript.
        Hostscript will contain the history of all actions that have been
        ordered on a particular VM. So content needs to be appended to it,
        rather than overwritten.
        """
        hostscript = self._hostscript_path(subdomain)
        with open(hostscript, "a", encoding="utf-8") as f:
            f.write(content)
            f.write("\n")

    def _add_files(self, subdomain: str):
        """
        Add all relevant files of a VM
        """
        self.repo.git.add(str(self._host_vars_dir(subdomain=subdomain)))
        self.repo.git.add(str(self._backup_path(subdomain=subdomain)))
        self.repo.git.add(str(self._service_path(subdomain=subdomain)))
        self.repo.git.add(str(self._hostscript_path(subdomain=subdomain)))

    def _commit(self, action: str, subdomain: str) -> Commit:
        """
        Commit changes to a VM configuration
        """

        self._add_files(subdomain=subdomain)
        return self.repo.git.commit(
            message=f"{action} VM {subdomain}",
            author="Dashboard Bot <bot@dashboard.hostea.org>",
        )

    def _pull(self):
        self.repo.git.pull(env=self.env, rebase="true")

    @staticmethod
    def translate_size(instance: Instance) -> str:
        """
        Translate openstack(I think OVH-specific) sizes to enough.community
        normalized sizes
        """
        if instance.configuration_id.name == "s1-2":
            return "openstack_flavor_small"
        if instance.configuration_id.name == "s1-4":
            return "openstack_flavor_medium"
        if instance.configuration_id.name == "s1-8":
            return "openstack_flavor_large"
        return instance.configuration_id.name

    def add_vm(self, instance: Instance) -> (str, Commit):
        """
        Add new VM to infrastructure repository

        The gitea user password is returned
        """
        self._pull()

        subdomain = instance.name
        host_vars_dir = self._host_vars_dir(subdomain)

        if not host_vars_dir.exists():
            os.makedirs(host_vars_dir)

        hostscript_path = self.repo_path.joinpath("hosts-scripts/")
        if not hostscript_path.exists():
            os.makedirs(hostscript_path)

        woodpecker_agent_secret = get_random_string(64)
        gitea_password = get_random_string(20)

        ctx = {
            "woodpecker_agent_secret": woodpecker_agent_secret,
            "woodpecker_hostname": self._gen_woodpecker_hostname(instance=instance),
            "woodpecker_admins": f"{instance.owned_by.username}",
            "gitea_email": instance.owned_by.email,
            "gitea_password": gitea_password,
            "subdomain": subdomain,
        }

        gitea = self._gitea_path(subdomain)
        with open(gitea, "w+", encoding="utf-8") as f:
            f.write(
                render_to_string(
                    "infrastructure/yml/gitea.yml",
                    context=ctx,
                )
            )

        size = self.translate_size(instance=instance)
        provision = self._provision_path(subdomain)

        with open(provision, "w+", encoding="utf-8") as f:
            f.write(
                render_to_string(
                    "infrastructure/yml/provision.yml", context={"vm_size": size}
                )
            )

        backup = self._backup_path(subdomain)
        with open(backup, "w+", encoding="utf-8") as f:
            f.write(
                render_to_string(
                    "infrastructure/yml/backups.yml", context={"subdomain": subdomain}
                )
            )

        service = self._service_path(subdomain)
        with open(service, "w+", encoding="utf-8") as f:
            f.write(
                render_to_string(
                    "infrastructure/yml/service.yml", context={"subdomain": subdomain}
                )
            )

        hostscript = self._hostscript_path(subdomain)
        with open(hostscript, "w+", encoding="utf-8") as f:
            f.write("\n")

        self.write_hostscript(
            subdomain=subdomain,
            content=render_to_string(
                "infrastructure/sh/hostscripts/create.sh",
                context={"subdomain": subdomain},
            ),
        )

        commit = self._commit(action="add", subdomain=subdomain)
        self.repo.git.push(env=self.env)
        return (gitea_password, commit)

    def remove_vm(self, instance: Instance):
        """
        Remove a VM from infrastructure repository
        """
        self._pull()
        subdomain = instance.name

        try:

            host_vars_dir = self._host_vars_dir(subdomain)
            shutil.rmtree(host_vars_dir)

            backup = self._backup_path(subdomain)
            os.remove(backup)

            service = self._service_path(subdomain)
            os.remove(service)
        except FileNotFoundError:
            pass

        hostscript = self._hostscript_path(subdomain)
        with open(hostscript, "w+", encoding="utf-8") as f:
            f.write("\n")

        self.write_hostscript(
            subdomain=subdomain,
            content=render_to_string(
                "infrastructure/sh/hostscripts/rm.sh",
                context={"subdomain": subdomain},
            ),
        )
        self._commit(action="rm", subdomain=subdomain)
        self.repo.git.push(env=self.env)