dashboard/infrastructure/utils.py

# Copyright © 2022 Aravinth Manivannan <realaravinth@batsense.net>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
import logging
import os
import sh
import shutil
import yaml
import requests
from pathlib import Path
from threading import Thread
from time import sleep

from django.utils.crypto import get_random_string
from django.template.loader import render_to_string
from django.contrib.auth import get_user_model
from django.core.mail import send_mail
from django.conf import settings
from payments import get_payment_model

from dash.models import Instance

from infrastructure.models import InstanceCreated, JobType, Job

logging.basicConfig()
logger = logging.getLogger(__name__)


class Worker(Thread):
    def __init__(self, job: Job):
        self.job = job
        super().__init__()

    def run(self):
        forgejo_uri = Infra.get_forgejo_uri(instance=self.job.instance)
        woodpecker = Infra.get_woodpecker_uri(instance=self.job.instance)
        while True:
            try:
                print(f"[ping] Trying to reach {forgejo_uri}")
                resp = requests.get(forgejo_uri)
                if resp.status_code == 200:
                    break
            except Exception:
                return False
            sleep(10)

        job = self.job
        self.job = None
        email = job.instance.owned_by.email

        ctx = {
            "forgejo_uri": forgejo_uri,
            "woodpecker_uri": woodpecker,
            "username": job.instance.owned_by.username,
        }
        body = render_to_string(
            "infrastructure/emails/instance-created.txt",
            context=ctx,
        )

        sender = settings.DEFAULT_FROM_EMAIL

        send_mail(
            subject="[Gna!] Your Gna! instance is now online!",
            message=body,
            from_email=f"No reply Gna!<{sender}>",  # TODO read from settings.py
            recipient_list=[email],
        )
        job.delete()


def create_vm_if_not_exists(instance: Instance) -> (str, str):
    """
    Create VM utility. Forgejo password is returned
    """

    def notify_staff(instance: Instance):
        infra = Infra()
        User = get_user_model()
        forgejo_uri = Infra.get_forgejo_uri(instance=instance)
        woodpecker = Infra.get_woodpecker_uri(instance=instance)

        for staff in User.objects.filter(is_staff=True):
            ctx = {
                "forgejo_uri": forgejo_uri,
                "woodpecker_uri": woodpecker,
                "username": staff.username,
            }
            body = render_to_string(
                "infrastructure/emails/staff-new-instance-alert.txt",
                context=ctx,
            )

            sender = settings.DEFAULT_FROM_EMAIL

            send_mail(
                subject="[Gna!] New instance alert",
                message=body,
                from_email=f"No reply Gna!<{sender}>",  # TODO read from settings.py
                recipient_list=[staff.email],
            )

    infra = Infra()
    if not InstanceCreated.objects.filter(instance=instance).exists():
        (forgejo_password, commit) = infra.add_vm(instance=instance)
        InstanceCreated.objects.create(instance=instance, created=True)
        notify_staff(instance=instance)
        job = Job.objects.create(instance=instance, job_type=str(JobType.PING))
        Worker(job=job).start()
        return (forgejo_password, commit)
    else:
        if str.strip(infra.get_flavor(instance=instance)) != str.strip(
            infra.translate_size(instance=instance)
        ):
            # Worker.init_global()
            notify_staff(instance=instance)
            return infra.add_vm(instance=instance)
    return None


def delete_vm(instance: Instance):
    infra = Infra()
    Payment = get_payment_model()
    for payment in Payment.objects.filter(
        paid_by=instance.owned_by, instance_name=instance.name
    ):
        payment.vm_deleted = True
        payment.save()

    infra.remove_vm(instance=instance)
    if InstanceCreated.objects.filter(instance=instance).exists():
        InstanceCreated.objects.get(instance=instance).delete()
    instance.delete()


class Infra:
    """
    Utility function to manage infrastructure repository
    """

    def __init__(self):
        conf = settings.HOSTEA["INFRA"]["HOSTEA_REPO"]
        self.repo_path = Path(conf["PATH"])
        self._clone()

    def _clone(self):
        conf = settings.HOSTEA["INFRA"]["HOSTEA_REPO"]
        ssh_cmd = f"/usr/bin/ssh -oStrictHostKeyChecking=no -i {conf['SSH_KEY']}"
        self.git = sh.git.bake(_env={"GIT_SSH_COMMAND": ssh_cmd})
        conf = settings.HOSTEA["INFRA"]["HOSTEA_REPO"]
        if os.path.exists(self.repo_path):
            shutil.rmtree(self.repo_path)
        self.git.clone(conf["REMOTE"], self.repo_path)
        self.git = self.git.bake("-C", self.repo_path)

    def _host_vars_dir(self, subdomain: str) -> Path:
        """
        utility method: get host_vars directory for a subdomain
        """
        return self.repo_path.joinpath(f"inventory/host_vars/{subdomain}-host/")

    def _provision_path(self, subdomain: str) -> Path:
        """
        utility method: get provision file pay for a subdomain
        """
        return self._host_vars_dir(subdomain=subdomain).joinpath("provision.yml")

    @classmethod
    def get_forgejo_uri(cls, instance: Instance) -> str:
        """
        Get an instance's Forgejo URI
        """
        base = settings.HOSTEA["INFRA"]["HOSTEA_DOMAIN"]
        return f"https://{instance.name}.{base}"

    @classmethod
    def _gen_woodpecker_hostname(cls, instance: Instance) -> str:
        """
        Get Woodpecker hostname of an instance
        """
        return f"{instance.name}-ci"

    @classmethod
    def get_woodpecker_uri(cls, instance: Instance) -> str:
        """
        Get an instance's Forgejo URI
        """
        base = settings.HOSTEA["INFRA"]["HOSTEA_DOMAIN"]
        return f"https://{cls._gen_woodpecker_hostname(instance=instance)}.{base}"

    def get_flavor(self, instance: Instance):
        """
        Get VM flavour/size/configuration from the fleet repository
        """
        subdomain = instance.name
        provision = self._provision_path(subdomain)
        with open(provision, "r", encoding="utf-8") as f:
            config = yaml.safe_load(f)
            if "openstack_flavor" in config:
                return config["openstack_flavor"].split("{{ ")[1].split(" }}")[0]
        return None

    def _forgejo_path(self, subdomain: str) -> Path:
        """
        utility method: get forgejo file for a subdomain
        """

        return self._host_vars_dir(subdomain=subdomain).joinpath("forgejo.yml")

    def _backup_path(self, subdomain: str) -> Path:
        """
        utility method: get backup file for a subdomain
        """

        return self.repo_path.joinpath(f"inventory/{subdomain}-backup.yml")

    def _service_path(self, subdomain: str) -> Path:
        """
        utility method: get service file for a subdomain
        """

        return self.repo_path.joinpath(f"inventory/{subdomain}-service.yml")

    def _hostscript_path(self, subdomain: str) -> Path:
        """
        utility method: hostscript file for a subdomain
        """
        return self.repo_path.joinpath(f"hosts-scripts/{subdomain}-host.sh")

    def write_hostscript(self, subdomain: str, content: str):
        """
        Write contents to hostscript.
        Hostscript will contain the history of all actions that have been
        ordered on a particular VM. So content needs to be appended to it,
        rather than overwritten.
        """
        hostscript = self._hostscript_path(subdomain)
        with open(hostscript, "a", encoding="utf-8") as f:
            f.write(content)
            f.write("\n")

    def _push(self, message):
        self.git.add(".")
        self.git.config("user.email", settings.HOSTEA["INSTANCE_MAINTAINER_CONTACT"])
        self.git.config("user.name", "Hostea dashboard")
        try:
            self.git.commit("-m", f"dashboard: {message}")
        except sh.ErrorReturnCode_1:
            logger.debug("no change")
        else:
            self.git.push("origin", "master")
        return self._sha()

    def _sha(self):
        sha = self.git("rev-parse", "origin/master")
        return str(sha).strip()

    @staticmethod
    def translate_size(instance: Instance) -> str:
        """
        Translate openstack(I think OVH-specific) sizes to enough.community
        normalized sizes
        """
        if instance.configuration_id.name == "s1-2":
            return "openstack_flavor_small"
        if instance.configuration_id.name == "s1-4":
            return "openstack_flavor_medium"
        if instance.configuration_id.name == "s1-8":
            return "openstack_flavor_large"
        return instance.configuration_id.name

    def add_vm(self, instance: Instance) -> (str, str):
        """
        Add new VM to infrastructure repository

        The forgejo user password is returned
        """

        subdomain = instance.name
        host_vars_dir = self._host_vars_dir(subdomain)

        if not host_vars_dir.exists():
            os.makedirs(host_vars_dir)

        hostscript_path = self.repo_path.joinpath("hosts-scripts/")
        if not hostscript_path.exists():
            os.makedirs(hostscript_path)

        woodpecker_agent_secret = get_random_string(64)
        forgejo_password = get_random_string(20)

        ctx = {
            "woodpecker_agent_secret": woodpecker_agent_secret,
            "woodpecker_hostname": self._gen_woodpecker_hostname(instance=instance),
            "woodpecker_admins": f"{instance.owned_by.username}",
            "forgejo_email": instance.owned_by.email,
            "forgejo_password": forgejo_password,
            "subdomain": subdomain,
        }

        forgejo = self._forgejo_path(subdomain)
        with open(forgejo, "w+", encoding="utf-8") as f:
            f.write(
                render_to_string(
                    "infrastructure/yml/forgejo.yml",
                    context=ctx,
                )
            )

        size = self.translate_size(instance=instance)
        provision = self._provision_path(subdomain)

        with open(provision, "w+", encoding="utf-8") as f:
            f.write(
                render_to_string(
                    "infrastructure/yml/provision.yml", context={"vm_size": size}
                )
            )

        backup = self._backup_path(subdomain)
        with open(backup, "w+", encoding="utf-8") as f:
            f.write(
                render_to_string(
                    "infrastructure/yml/backups.yml", context={"subdomain": subdomain}
                )
            )

        service = self._service_path(subdomain)
        with open(service, "w+", encoding="utf-8") as f:
            f.write(
                render_to_string(
                    "infrastructure/yml/service.yml", context={"subdomain": subdomain}
                )
            )

        hostscript = self._hostscript_path(subdomain)
        with open(hostscript, "w+", encoding="utf-8") as f:
            f.write("\n")

        self.write_hostscript(
            subdomain=subdomain,
            content=render_to_string(
                "infrastructure/sh/hostscripts/create.sh",
                context={"subdomain": subdomain},
            ),
        )

        commit = self._push(f"add vm {subdomain}")
        return (forgejo_password, commit)

    def remove_vm(self, instance: Instance):
        """
        Remove a VM from infrastructure repository
        """
        subdomain = instance.name

        host_vars_dir = self._host_vars_dir(subdomain)
        if os.path.exists(host_vars_dir):
            shutil.rmtree(host_vars_dir)

        backup = self._backup_path(subdomain)
        if os.path.exists(backup):
            os.remove(backup)

        service = self._service_path(subdomain)
        if os.path.exists(service):
            os.remove(service)

        hostscript = self._hostscript_path(subdomain)
        with open(hostscript, "w+", encoding="utf-8") as f:
            f.write("\n")

        self.write_hostscript(
            subdomain=subdomain,
            content=render_to_string(
                "infrastructure/sh/hostscripts/rm.sh",
                context={"subdomain": subdomain},
            ),
        )
        return self._push(f"rm vm {subdomain}")