From f3324579c9d8f7413f52e65f09eb10afa79f1955 Mon Sep 17 00:00:00 2001 From: realaravinth Date: Fri, 24 Jun 2022 20:35:32 +0530 Subject: [PATCH] feat: utilities to add and remove VM on the Hostea repo --- infrastructure/tests.py | 76 +++++++++++++++++ infrastructure/utils.py | 181 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 257 insertions(+) create mode 100644 infrastructure/tests.py create mode 100644 infrastructure/utils.py diff --git a/infrastructure/tests.py b/infrastructure/tests.py new file mode 100644 index 0000000..71e3310 --- /dev/null +++ b/infrastructure/tests.py @@ -0,0 +1,76 @@ +# Copyright © 2022 Aravinth Manivannan +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from pathlib import Path + +from django.test import TestCase, Client, override_settings +from django.conf import settings + +from dash.models import Instance +from .utils import Infra + +from accounts.tests import register_util, login_util +from dash.tests import create_configurations, create_instance_util + + +class InfraUtilTest(TestCase): + """ + Tests billing system + """ + + def setUp(self): + self.username = "infrautil_user" + register_util(t=self, username=self.username) + create_configurations(t=self) + + @override_settings( + HOSTEA={ + "INFRA": { + "HOSTEA_REPO": { + "PATH": "/tmp/hostea/dashboard/test_path_util/repo/", + "REMOTE": "git@gitea.hostea.org:Hostea/payments.git", + "SSH_KEY": "/src/atm/.ssh/id", + } + } + } + ) + def test_path_utils(self): + infra = Infra() + subdomain = "foo" + base = infra.repo_path + + self.assertEqual( + base.joinpath(f"inventory/host_vars/{subdomain}-host/"), + infra._host_vars_dir(subdomain=subdomain), + ) + + self.assertEqual( + base.joinpath(f"inventory/host_vars/{subdomain}-host/gitea.yml"), + infra._gitea_path(subdomain=subdomain), + ) + + self.assertEqual( + base.joinpath(f"inventory/host_vars/{subdomain}-host/provision.yml"), + infra._provision_path(subdomain=subdomain), + ) + + self.assertEqual( + base.joinpath(f"inventory/{subdomain}-backup.yml"), + infra._backup_path(subdomain=subdomain), + ) + + self.assertEqual( + base.joinpath(f"inventory/hosts-scripts/{subdomain}-host.sh"), + infra._hostscript_path(subdomain=subdomain), + ) diff --git a/infrastructure/utils.py b/infrastructure/utils.py new file mode 100644 index 0000000..11612d6 --- /dev/null +++ b/infrastructure/utils.py @@ -0,0 +1,181 @@ +# Copyright © 2022 Aravinth Manivannan +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +import os +import shutil +from pathlib import Path + +from django.template.loader import render_to_string +from django.conf import settings +from git import Repo +from git.exc import InvalidGitRepositoryError + +from dash.models import Instance + + +class Infra: + """ + Utility function to manage infrastructure repository + """ + + def __init__(self): + conf = settings.HOSTEA["INFRA"]["HOSTEA_REPO"] + self.repo_path = Path(conf["PATH"]) + if not self.repo_path.exists(): + os.makedirs(self.repo_path) + + self.ssh_cmd = f"ssh -i {conf['SSH_KEY']}" + try: + self.repo = Repo(path=self.repo_path) + except InvalidGitRepositoryError: + self.repo = Repo.clone_from( + conf["REMOTE"], self.repo_path, env={"GIT_SSH_COMMAND": self.ssh_cmd} + ) + + def _host_vars_dir(self, subdomain: str) -> Path: + """ + utility method: get host_vars directory for a subdomain + """ + return self.repo_path.joinpath(f"inventory/host_vars/{subdomain}-host/") + + def _provision_path(self, subdomain: str) -> Path: + """ + utility method: get provision file pay for a subdomain + """ + + return self._host_vars_dir(subdomain=subdomain).joinpath("provision.yml") + + def _gitea_path(self, subdomain: str) -> Path: + """ + utility method: get gitea file for a subdomain + """ + + return self._host_vars_dir(subdomain=subdomain).joinpath("gitea.yml") + + def _backup_path(self, subdomain: str) -> Path: + """ + utility method: get backup file for a subdomain + """ + + return self.repo_path.joinpath(f"inventory/{subdomain}-backup.yml") + + def _hostscript_path(self, subdomain: str) -> Path: + """ + utility method: hostscript file for a subdomain + """ + return self.repo_path.joinpath(f"inventory/hosts-scripts/{subdomain}-host.sh") + + def write_hostscript(self, subdomain: str, content: str): + """ + Write contents to hostscript. + Hostscript will contain the history of all actions that have been + ordered on a particular VM. So content needs to be appended to it, + rather than overwritten. + """ + hostscript = self._hostscript_path(subdomain) + with open(hostscript, "a", encoding="utf-8") as f: + f.write(content) + f.write("\n") + + def _add_files(self, subdomain: str): + """ + Add all relevant files of a VM + """ + self.repo.index.add(self._host_vars_dir(subdomain=subdomain)) + self.repo.index.add(self._backup_path(subdomain=subdomain)) + self.repo.index.add(self._hostscript_path(subdomain=subdomain)) + + def _commit(self, action: str, subdomain: str): + """ + Commit changes to a VM configuration + """ + + self._add_files(subdomain=subdomain) + self.repo.git.commit( + f"{action} VM {subdomain}", author="bot@dashboard.hostea.org" + ) + + def add_vm(self, instance: Instance): + """ + Add new VM to infrastructure repository + """ + subdomain = instance.name + host_vars_dir = self._host_vars_dir(subdomain) + + if not host_vars_dir.exists(): + os.makedirs(host_vars_dir) + + hostscript_path = self.repo_path.joinpath("inventory/hosts-scripts/") + if not hostscript_path.exists(): + os.makedirs(hostscript_path) + + gitea = self._gitea_path(subdomain) + gitea_template = "./templates/infrastructure/yml/gitea.yml" + shutil.copy(gitea_template, gitea) + + # provision_template = "./templates/infrastructure/yml/provision.yml" + provision = self._provision_path(subdomain) + # TODO: instance config names are different the flavours expected: + # ``` + # openstack_flavor: ‘{{ openstack_flavor_medium }}’ * openstack_flavor: ‘{{ openstack_flavor_large }}’ + # ``` + # check with @dachary about this + with open(provision, "w", encoding="utf-8") as f: + f.write( + render_to_string( + "infrastructure/yml/provision.yml", + context={"vm_size": instance.instance_id.name}, + ) + ) + + # backup = self.repo_path.joinpath(f"inventory/{instance.name}-backup.yml") + backup = self._backup_path(subdomain) + # backup_template = "./templates/infrastructure/yml/provision.yml" + with open(backup, "w", encoding="utf-8") as f: + f.write( + render_to_string( + "infrastructure/yml/backups.yml", context={"subdomain": subdomain} + ) + ) + + # hostscript = self.repo_path.join("inventory/hosts-scripts/{instance.name}-host.sh") + + self.write_hostscript( + subdomain=subdomain, + content=render_to_string( + "infrastructure/sh/create.sh", context={"subdomain": subdomain} + ), + ) + + self._commit(action="add", subdomain=subdomain) + + def remove_vm(self, instance: Instance): + """ + Remove a VM from infrastructure repository + """ + subdomain = instance.name + + host_vars_dir = self._host_vars_dir(subdomain) + shutil.rmtree(host_vars_dir) + + backup = self._backup_path(subdomain) + os.remove(backup) + + self.write_hostscript( + subdomain=subdomain, + content=render_to_string( + "infrastructure/sh/rm.sh", context={"subdomain": subdomain} + ), + ) + self._commit(action="rm", subdomain=subdomain)