From e63719764a9c8ba370724ee3d5f66caa2f92261d Mon Sep 17 00:00:00 2001 From: realaravinth Date: Tue, 28 Jun 2022 23:54:14 +0530 Subject: [PATCH] fix: idempotency: change configuration in fleet repository too, when vm create is re-run for the same VM with different configuration fixes: https://gitea.hostea.org/Hostea/dashboard/issues/8 --- infrastructure/management/commands/vm.py | 4 ++ infrastructure/tests.py | 39 ++++++++++++++----- infrastructure/utils.py | 49 ++++++++++++++++-------- requirements.txt | 1 + 4 files changed, 68 insertions(+), 25 deletions(-) diff --git a/infrastructure/management/commands/vm.py b/infrastructure/management/commands/vm.py index f92ac94..2af6f91 100644 --- a/infrastructure/management/commands/vm.py +++ b/infrastructure/management/commands/vm.py @@ -104,6 +104,10 @@ class Command(BaseCommand): name=size ) instance.save() + gitea_password = create_vm_if_not_exists(instance) + print("Instance created") + print(f"Gitea admin password: {gitea_password}") + else: self.stderr.write(self.style.ERROR(f"error: {str(e)}")) except Exception as e: diff --git a/infrastructure/tests.py b/infrastructure/tests.py index d3320d2..1b00899 100644 --- a/infrastructure/tests.py +++ b/infrastructure/tests.py @@ -118,19 +118,31 @@ class InfraUtilTest(TestCase): repo.git.pull() self.assertEqual(repo.head.commit.hexsha == after_rm, True) - vm_name = "cmd_vm" + + @override_settings(HOSTEA=custom_config(test_name="test_cmd")) + def test_cmd(self): + subdomain = "cmd_vm" + infra = Infra() + c = Client() + conf = settings.HOSTEA["INFRA"]["HOSTEA_REPO"] + login_util(self, c, "accounts.home") + + base = infra.repo_path stdout = StringIO() stderr = StringIO() - self.assertEqual(Instance.objects.filter(name=vm_name).exists(), False) + self.assertEqual(Instance.objects.filter(name=subdomain).exists(), False) # username exists call_command( - "vm", "create", vm_name, f"--owner={self.username}", "--flavor=medium" + "vm", "create", subdomain, f"--owner={self.username}", "--flavor=medium" ) out = stdout.getvalue() - instance = Instance.objects.get(name=vm_name) + instance = Instance.objects.get(name=subdomain) + + self.assertEqual(infra.get_flavor(instance=instance), "openstack_flavor_medium") + self.assertEqual(instance.owned_by, self.user) self.assertEqual( instance.configuration_id, InstanceConfiguration.objects.get(name="s1-4") @@ -143,28 +155,37 @@ class InfraUtilTest(TestCase): # run create vm command again with same configuration to crudely check idempotency call_command( - "vm", "create", vm_name, f"--owner={self.username}", "--flavor=medium" + "vm", "create", subdomain, f"--owner={self.username}", "--flavor=medium" ) # run create vm command again with different configuration but same name # to crudely check idempotency old_size = instance.configuration_id call_command( - "vm", "create", vm_name, f"--owner={self.username}", "--flavor=large" + "vm", "create", subdomain, f"--owner={self.username}", "--flavor=large" ) instance.refresh_from_db() + # verify new size is updated in DB self.assertEqual( str.strip(instance.configuration_id.name) == str.strip(translate_sizes("large")), True, ) - call_command("vm", "delete", vm_name) + # verify new size is updated in repository + self.assertEqual( + str.strip(infra.translate_size(instance=instance)) == + str.strip(infra.get_flavor(instance=instance)), + True, + ) + + + call_command("vm", "delete", subdomain) out = stdout.getvalue() - self.assertEqual(Instance.objects.filter(name=vm_name).exists(), False) + self.assertEqual(Instance.objects.filter(name=subdomain).exists(), False) host_vars_dir = infra._host_vars_dir(subdomain) self.assertEqual(host_vars_dir.exists(), False) # run delete VM command to crudely check idempotency - call_command("vm", "delete", vm_name) + call_command("vm", "delete", subdomain) diff --git a/infrastructure/utils.py b/infrastructure/utils.py index 79db0b3..fcbae95 100644 --- a/infrastructure/utils.py +++ b/infrastructure/utils.py @@ -14,6 +14,7 @@ # along with this program. If not, see . import os import shutil +import yaml from pathlib import Path from django.utils.crypto import get_random_string @@ -37,6 +38,12 @@ def create_vm_if_not_exists(instance: Instance) -> str: instance = InstanceCreated.objects.create(instance=instance, created=True) instance.save() return gitea_password + else: + if str.strip(infra.get_flavor(instance=instance)) != str.strip(infra.translate_size( + instance=instance + )): + gitea_password = infra.add_vm(instance=instance) + return gitea_password def delete_vm(instance: Instance, owner: str): @@ -75,9 +82,16 @@ class Infra: """ utility method: get provision file pay for a subdomain """ - return self._host_vars_dir(subdomain=subdomain).joinpath("provision.yml") + def get_flavor(self, instance: Instance): + subdomain = instance.name + provision = self._provision_path(subdomain) + with open(provision, "r", encoding="utf-8") as f: + config = yaml.safe_load(f) + if "openstack_flavor" in config: + return config["openstack_flavor"].split("{{ ")[1].split(" }}")[0] + def _gitea_path(self, subdomain: str) -> Path: """ utility method: get gitea file for a subdomain @@ -140,6 +154,16 @@ class Infra: def _pull(self): self.repo.git.pull(env=self.env, rebase="true") + def translate_size(self, instance: Instance) -> str: + if instance.configuration_id.name == "s1-2": + return "openstack_flavor_small" + elif instance.configuration_id.name == "s1-4": + return "openstack_flavor_medium" + elif instance.configuration_id.name == "s1-8": + return "openstack_flavor_large" + else: + return instance.configuration_id.name + def add_vm(self, instance: Instance) -> str: """ Add new VM to infrastructure repository @@ -171,7 +195,7 @@ class Infra: } gitea = self._gitea_path(subdomain) - with open(gitea, "w", encoding="utf-8") as f: + with open(gitea, "w+", encoding="utf-8") as f: f.write( render_to_string( "infrastructure/yml/gitea.yml", @@ -180,6 +204,7 @@ class Infra: ) # provision_template = "./templates/infrastructure/yml/provision.yml" + size = self.translate_size(instance=instance) provision = self._provision_path(subdomain) # TODO: instance config names are different the flavours expected: # ``` @@ -187,27 +212,19 @@ class Infra: # ``` # check with @dachary about this - size = None - if instance.configuration_id.name == "s1-2": - size = "openstack_flavor_small" - elif instance.configuration_id.name == "s1-4": - size = "openstack_flavor_medium" - elif instance.configuration_id.name == "s1-8": - size = "openstack_flavor_large" - else: - size = instance.configuration_id.name - with open(provision, "w", encoding="utf-8") as f: + with open(provision, "w+", encoding="utf-8") as f: f.write( render_to_string( "infrastructure/yml/provision.yml", context={"vm_size": size} ) ) + assert provision.exists() # backup = self.repo_path.joinpath(f"inventory/{instance.name}-backup.yml") backup = self._backup_path(subdomain) # backup_template = "./templates/infrastructure/yml/provision.yml" - with open(backup, "w", encoding="utf-8") as f: + with open(backup, "w+", encoding="utf-8") as f: f.write( render_to_string( "infrastructure/yml/backups.yml", context={"subdomain": subdomain} @@ -215,7 +232,7 @@ class Infra: ) service = self._service_path(subdomain) - with open(service, "w", encoding="utf-8") as f: + with open(service, "w+", encoding="utf-8") as f: f.write( render_to_string( "infrastructure/yml/service.yml", context={"subdomain": subdomain} @@ -225,7 +242,7 @@ class Infra: # hostscript = self.repo_path.join("inventory/hosts-scripts/{instance.name}-host.sh") hostscript = self._hostscript_path(subdomain) - with open(hostscript, "w", encoding="utf-8") as f: + with open(hostscript, "w+", encoding="utf-8") as f: f.write("\n") self.write_hostscript( @@ -261,7 +278,7 @@ class Infra: pass hostscript = self._hostscript_path(subdomain) - with open(hostscript, "w", encoding="utf-8") as f: + with open(hostscript, "w+", encoding="utf-8") as f: f.write("\n") self.write_hostscript( diff --git a/requirements.txt b/requirements.txt index b771c4a..896b2a5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -35,6 +35,7 @@ pycparser==2.21 pylint==2.12.2 pynvim==0.4.3 pytz==2022.1 +PyYAML==6.0 requests==2.27.1 smmap==5.0.0 sqlparse==0.4.2