From f00746a36d20a3f66a305c4b90b5d8c2d3746386 Mon Sep 17 00:00:00 2001 From: realaravinth Date: Thu, 30 Jun 2022 01:10:55 +0530 Subject: [PATCH 01/10] feat: notify user on instance creation --- infrastructure/migrations/0006_job.py | 36 +++++++++++++++++ infrastructure/models.py | 21 +++++++++- infrastructure/tests.py | 31 +++++++++++++-- infrastructure/utils.py | 56 +++++++++++++++++++++++++-- requirements.txt | 1 + 5 files changed, 137 insertions(+), 8 deletions(-) create mode 100644 infrastructure/migrations/0006_job.py diff --git a/infrastructure/migrations/0006_job.py b/infrastructure/migrations/0006_job.py new file mode 100644 index 0000000..4561f62 --- /dev/null +++ b/infrastructure/migrations/0006_job.py @@ -0,0 +1,36 @@ +# Generated by Django 4.0.3 on 2022-06-29 18:30 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("dash", "0006_auto_20220619_0800"), + ("infrastructure", "0005_remove_instancecreated_gitea_password"), + ] + + operations = [ + migrations.CreateModel( + name="Job", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("job_type", models.CharField(max_length=10, verbose_name="Job Type")), + ( + "instance", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="dash.instance" + ), + ), + ], + ), + ] diff --git a/infrastructure/models.py b/infrastructure/models.py index 1e0673d..c71b26c 100644 --- a/infrastructure/models.py +++ b/infrastructure/models.py @@ -12,11 +12,30 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from django.db import models +from enum import Enum, unique +from django.db import models from dash.models import Instance +@unique +class JobType(Enum): + PING = "ping" + + def __str__(self): + return self.name + + class InstanceCreated(models.Model): instance = models.ForeignKey(Instance, on_delete=models.PROTECT) created = models.BooleanField(default=False) + + +class Job(models.Model): + instance = models.ForeignKey(Instance, on_delete=models.CASCADE) + + job_type = models.CharField( + "Job Type", + max_length=10, + null=False, + ) diff --git a/infrastructure/tests.py b/infrastructure/tests.py index cb52910..cb59015 100644 --- a/infrastructure/tests.py +++ b/infrastructure/tests.py @@ -15,11 +15,11 @@ import shutil import time import os -import requests from io import StringIO -from urllib.parse import urlparse, urlunparse +from urllib.parse import urlunparse from pathlib import Path +import requests from django.test import TestCase, Client, override_settings from django.conf import settings from django.core.management import call_command @@ -31,8 +31,8 @@ from accounts.tests import register_util, login_util from dash.tests import create_configurations, create_instance_util from infrastructure.management.commands.vm import translate_sizes -from .utils import Infra -from .models import InstanceCreated +from .utils import Infra, Worker +from .models import InstanceCreated, Job, JobType def custom_config(test_name: str): @@ -187,3 +187,26 @@ class InfraUtilTest(TestCase): # run delete VM command to crudely check idempotency call_command("vm", "delete", subdomain) + + def test_worker(self): + subdomain = "gitea" # yes, gitea.hostea.org exists. will use it till I + # figure out how to use requests_mock within django + c = Client() + login_util(self, c, "accounts.home") + create_instance_util( + t=self, c=c, instance_name=subdomain, config=self.instance_config[0] + ) + + instance = Instance.objects.get(name=subdomain) + job = Job.objects.create(instance=instance, job_type=JobType.PING) + gitea_uri = Infra.get_gitea_uri(instance=instance) + print(f"mocking {gitea_uri}") + + w = Worker(job=job) + w.start() + time.sleep(5) + self.assertEqual(w.is_alive(), False) + w.join() + self.assertEqual( + Job.objects.filter(instance=instance, job_type=JobType.PING).exists(), True + ) diff --git a/infrastructure/utils.py b/infrastructure/utils.py index 68ef92d..b66f741 100644 --- a/infrastructure/utils.py +++ b/infrastructure/utils.py @@ -15,17 +15,65 @@ import os import shutil import yaml +import requests from pathlib import Path +from threading import Thread, Event +from time import sleep from django.utils.crypto import get_random_string from django.template.loader import render_to_string +from django.core.mail import send_mail from django.conf import settings from git import Repo, Commit from git.exc import InvalidGitRepositoryError from dash.models import Instance -from .models import InstanceCreated +from infrastructure.models import InstanceCreated, JobType, Job + + +class Worker(Thread): + def __init__(self, job: Job): + self.job = job + super().__init__() + + ######### self.daemon = True + + def run(self): + gitea_uri = Infra.get_gitea_uri(instance=self.job.instance) + woodpecker = Infra.get_woodpecker_hostname(instance=self.job.instance) + while True: + try: + print(f"[ping] Trying to reach {gitea_uri}") + resp = requests.get(gitea_uri) + print(resp.status_code) + if resp.status_code == 200: + break + except Exception: + return False + sleep(10) + + print("sending email") + job = self.job + self.job = None + email = job.instance.owned_by.email + send_mail( + subject="[Hostea] Your Hostea instance is now online!", + message=f""" +Hello, + +The deployment job has run to completion and your Hostea instance is now online! +Credentials to admin account was sent in an earlier email, please contact +support if didn't receive it. + +Gitea: {gitea_uri} +Woodpecker CI: {woodpecker} +""", + from_email="No reply Hostea", # TODO read from settings.py + recipient_list=[email], + ) + job.delete() + print("job deleted") def create_vm_if_not_exists(instance: Instance) -> (str, Commit): @@ -35,13 +83,15 @@ def create_vm_if_not_exists(instance: Instance) -> (str, Commit): infra = Infra() if not InstanceCreated.objects.filter(instance=instance).exists(): (gitea_password, commit) = infra.add_vm(instance=instance) - instance = InstanceCreated.objects.create(instance=instance, created=True) - instance.save() + InstanceCreated.objects.create(instance=instance, created=True) + job = Job.objects.create(instance=instance, job_type=str(JobType.PING)) + Worker(job=job).start() return (gitea_password, commit) else: if str.strip(infra.get_flavor(instance=instance)) != str.strip( infra.translate_size(instance=instance) ): + # Worker.init_global() return infra.add_vm(instance=instance) return None diff --git a/requirements.txt b/requirements.txt index 896b2a5..ccdcb32 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,6 +37,7 @@ pynvim==0.4.3 pytz==2022.1 PyYAML==6.0 requests==2.27.1 +six==1.16.0 smmap==5.0.0 sqlparse==0.4.2 stripe==3.4.0 From 9d89bc071c0148b1d3551169ed375a592ad108fd Mon Sep 17 00:00:00 2001 From: realaravinth Date: Thu, 30 Jun 2022 13:05:57 +0530 Subject: [PATCH 02/10] fet: show Gitea and woodpecker URIs in view instance page --- dash/templates/dash/instances/view/index.html | 1 + dash/views.py | 6 ++++++ infrastructure/utils.py | 6 +++--- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/dash/templates/dash/instances/view/index.html b/dash/templates/dash/instances/view/index.html index a616e9b..b5bfb28 100644 --- a/dash/templates/dash/instances/view/index.html +++ b/dash/templates/dash/instances/view/index.html @@ -12,6 +12,7 @@

Created On: {{ instance.created_at }}

+

Gitea Instance|Woodpecker CI

str: + def get_woodpecker_uri(cls, instance: Instance) -> str: """ Get an instance's Gitea URI """ From faca7286b7878393f8f47d00e3683f0b8a8a323b Mon Sep 17 00:00:00 2001 From: realaravinth Date: Thu, 30 Jun 2022 14:33:56 +0530 Subject: [PATCH 03/10] feat: load Dashboard source code repository URL from settings --- dashboard/local_settings.ci.py | 1 + dashboard/local_settings.example.py | 1 + dashboard/settings.py | 1 + 3 files changed, 3 insertions(+) diff --git a/dashboard/local_settings.ci.py b/dashboard/local_settings.ci.py index d171af4..5b0dde8 100644 --- a/dashboard/local_settings.ci.py +++ b/dashboard/local_settings.ci.py @@ -54,6 +54,7 @@ PAYMENT_VARIANTS = { ### Dashbaord specific configuration options HOSTEA = { + "SOURCE_CODE": "https://gitea.hostea.org/Hostea/dashboard", "INSTANCE_MAINTAINER_CONTACT": "contact@hostea.example.org", "ACCOUNTS": { "MAX_VERIFICATION_TOLERANCE_PERIOD": 60 * 60 * 24, # in seconds diff --git a/dashboard/local_settings.example.py b/dashboard/local_settings.example.py index a5f9720..5afc69f 100644 --- a/dashboard/local_settings.example.py +++ b/dashboard/local_settings.example.py @@ -53,6 +53,7 @@ PAYMENT_VARIANTS = { ### Dashbaord specific configuration options HOSTEA = { + "SOURCE_CODE": "https://gitea.hostea.org/Hostea/dashboard", "INSTANCE_MAINTAINER_CONTACT": "contact@hostea.example.org", "ACCOUNTS": { "MAX_VERIFICATION_TOLERANCE_PERIOD": 60 * 60 * 24, # in seconds diff --git a/dashboard/settings.py b/dashboard/settings.py index 55eb1bb..315d708 100644 --- a/dashboard/settings.py +++ b/dashboard/settings.py @@ -168,6 +168,7 @@ PAYMENT_VARIANTS = { ### Dashbaord specific configuration options HOSTEA = { + "SOURCE_CODE": "https://gitea.hostea.org/Hostea/dashboard", "RESTRICT_NEW_INTEGRATION_INSTALLATION": True, "INSTANCE_MAINTAINER_CONTACT": "contact@hostea.example.org", "ACCOUNTS": { From 8be1e02a21a88b7ea25b8a4c7c609b2f4c993262 Mon Sep 17 00:00:00 2001 From: realaravinth Date: Thu, 30 Jun 2022 14:37:49 +0530 Subject: [PATCH 04/10] feat: load footer ctx in all templates --- .../templates/common/components/footer.html | 33 ++++++++++--------- accounts/views.py | 4 +++ billing/views.py | 2 ++ dash/utils.py | 22 +++++++++++++ dash/views.py | 8 ++++- infrastructure/views.py | 2 +- static/css/main.css | 8 ++--- support/views.py | 2 ++ 8 files changed, 59 insertions(+), 22 deletions(-) diff --git a/accounts/templates/common/components/footer.html b/accounts/templates/common/components/footer.html index 2004b55..e82e052 100644 --- a/accounts/templates/common/components/footer.html +++ b/accounts/templates/common/components/footer.html @@ -2,20 +2,6 @@
- Docs - | - No AMP - -
-
Home + | +   About + +
+
+ No AMP +
|
Contact Instance Maintainer|
- v{{ footer.version }}-{{ footer.git_hash }} + {{ footer.source_code.text }}
diff --git a/accounts/views.py b/accounts/views.py index db39aaf..338d619 100644 --- a/accounts/views.py +++ b/accounts/views.py @@ -23,6 +23,7 @@ from django.http import HttpResponse from django.views.decorators.csrf import csrf_protect from django.urls import reverse +from dash.utils import footer_ctx from .models import AccountConfirmChallenge from .utils import send_verification_email, ConfirmAccess @@ -35,6 +36,7 @@ def login_view(request): def default_login_ctx(): return { "title": "Login", + "footer": footer_ctx(), } if request.method == "GET": @@ -102,6 +104,7 @@ def register_view(request): "title": "Register", "username": username, "email": username, + "footer": footer_ctx(), } if request.method == "GET": @@ -213,6 +216,7 @@ def sudo(request): def default_login_ctx(): return { "title": "Confirm Access", + "footer": footer_ctx(), } if request.method == "GET": diff --git a/billing/views.py b/billing/views.py index d3cf5ed..d4ab2d3 100644 --- a/billing/views.py +++ b/billing/views.py @@ -24,6 +24,7 @@ from payments import get_payment_model, RedirectNeeded, PaymentStatus from dash.models import Instance from django.db.models import Q from infrastructure.utils import create_vm_if_not_exists +from dash.utils import footer_ctx def default_ctx(title: str, username: str): @@ -33,6 +34,7 @@ def default_ctx(title: str, username: str): return { "title": title, "username": username, + "footer": footer_ctx(), } diff --git a/dash/utils.py b/dash/utils.py index 2f3e3d6..99d45d0 100644 --- a/dash/utils.py +++ b/dash/utils.py @@ -14,7 +14,9 @@ # along with this program. If not, see . from enum import Enum, unique +from git import Repo from django.contrib.auth.models import User +from django.conf import settings from .models import Instance, InstanceConfiguration @@ -55,3 +57,23 @@ def create_instance(vm_name: str, configuration_name: str, user: User) -> Instan instance = Instance(name=vm_name, configuration_id=configuration, owned_by=user) instance.save() return instance + + +source_code = None + + +def footer_ctx(): + global source_code + if source_code is None: + source_code = {"text": "Source Code", "link": settings.HOSTEA["SOURCE_CODE"]} + try: + r = Repo(".") + commit = r.head.commit.hexsha + source_code["text"] = f"v-{commit.hexsha[0:8]}" + except: + pass + + return { + "source_code": source_code, + "admin_email": settings.HOSTEA["INSTANCE_MAINTAINER_CONTACT"], + } diff --git a/dash/views.py b/dash/views.py index 8ab7130..89825eb 100644 --- a/dash/views.py +++ b/dash/views.py @@ -25,7 +25,12 @@ from accounts.decorators import confirm_access from infrastructure.utils import Infra from .models import Instance, InstanceConfiguration -from .utils import create_instance as create_instance_util, VmErrors, VmException +from .utils import ( + create_instance as create_instance_util, + VmErrors, + VmException, + footer_ctx, +) def default_ctx(title: str, username: str): @@ -36,6 +41,7 @@ def default_ctx(title: str, username: str): "title": title, "username": username, "open_instances": "open", + "footer": footer_ctx(), } diff --git a/infrastructure/views.py b/infrastructure/views.py index 8d5c7ad..37a3292 100644 --- a/infrastructure/views.py +++ b/infrastructure/views.py @@ -62,7 +62,7 @@ your new Gitea instance. Great powers come with great responsibilities, so use the admin credentials wisely. When in doubt, consult the Gitea docs or contact support!\n - -username : root + - username : root - password: {gitea_password} """, from_email="No reply Hostea", # TODO read from settings.py diff --git a/static/css/main.css b/static/css/main.css index 4b633f9..d32d1d2 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -34,11 +34,9 @@ h2 { body { width: 100%; min-height: 100vh; - /* display: flex; flex-direction: column; justify-content: space-between; - */ } a:hover { @@ -244,6 +242,8 @@ footer { display: block; font-size: 0.7rem; margin-bottom: 5px; + margin-left: 260px; + width: 100%; } .footer__container { @@ -391,6 +391,8 @@ footer { font-size: 0.7rem; padding: 0; margin: 0; + margin-left: 260px; + width: calc(100vw - 260px); } .footer__container { @@ -607,8 +609,6 @@ fieldset { background-color: #e11d21; } - - /* .form__label { margin: 5px 0; diff --git a/support/views.py b/support/views.py index 0011745..2ad6b69 100644 --- a/support/views.py +++ b/support/views.py @@ -15,6 +15,7 @@ from django.shortcuts import render, redirect from django.contrib.auth.decorators import login_required +from dash.utils import footer_ctx from .utils import IssueTracker @@ -28,6 +29,7 @@ def default_ctx(title: str, username: str): "username": username, "open_support": "open", "support": {"list": it.get_issue_tracker(), "new": it.open_issue()}, + "footer": footer_ctx(), } From 412230bd99b7e9dde630cd25ae45e3a0f014b057 Mon Sep 17 00:00:00 2001 From: realaravinth Date: Fri, 1 Jul 2022 19:52:10 +0530 Subject: [PATCH 05/10] feat: create repo and add deploy key util --- integration/ci.sh | 7 ++++++- integration/lib.sh | 38 +++++++++++++++++++++++--------------- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/integration/ci.sh b/integration/ci.sh index 5d4b0f4..a25fa34 100755 --- a/integration/ci.sh +++ b/integration/ci.sh @@ -11,6 +11,7 @@ init() { else docker_compose_down || true docker_compose_up + sed -i /localhost.*/d ~/.ssh/known_hosts setup_env sleep 5 # wait_for_env @@ -29,4 +30,8 @@ teardown() { fi } -$1 +new_fleet_repo() { + new_fleet_repo_init $2 +} + +$1 $@ diff --git a/integration/lib.sh b/integration/lib.sh index e9d8048..079b9c2 100755 --- a/integration/lib.sh +++ b/integration/lib.sh @@ -120,31 +120,20 @@ support_repo_init() { $GITEA_HOSTEA_SUPPORT_REPO } -# register user "Hostea" on Gitea and create support repository -fleet_repo_init() { - python -m integration \ - gitea register \ - $GITEA_HOSTEA_USERNAME $GITEA_HOSTEA_PASSWORD \ - $GITEA_HOSTEA_EMAIL \ - $GITEA_URL || true - python -m integration \ - gitea login \ - $GITEA_HOSTEA_USERNAME $GITEA_HOSTEA_PASSWORD \ - $GITEA_HOSTEA_EMAIL \ - $GITEA_URL +new_fleet_repo_init() { python -m integration \ gitea create_repo \ $GITEA_HOSTEA_USERNAME $GITEA_HOSTEA_PASSWORD \ $GITEA_HOSTEA_EMAIL \ $GITEA_URL \ - $GITEA_HOSTEA_FLEET_REPO + $1 python -m integration \ gitea add_deploy_key \ $GITEA_HOSTEA_USERNAME $GITEA_HOSTEA_PASSWORD \ $GITEA_HOSTEA_EMAIL \ $GITEA_URL \ - $GITEA_HOSTEA_FLEET_REPO \ + $1 \ $GITEA_HOSTEA_FLEET_DEPLOY_KEY tmp_dir=$(mktemp -d) @@ -159,7 +148,8 @@ fleet_repo_init() { git init git add README git commit -m "init" - git remote add origin $GITEA_HOSTEA_FLEET_REPO_REMOTE + REMOTE="$GITEA_SSH_URL/$GITEA_HOSTEA_USERNAME/$1.git" + git remote add origin $REMOTE GIT_SSH_COMMAND="/usr/bin/ssh -oStrictHostKeyChecking=no -i $GITEA_HOSTEA_FLEET_DEPLOY_KEY_PRIVATE" \ git push --set-upstream origin master popd @@ -167,6 +157,24 @@ fleet_repo_init() { } + +# register user "Hostea" on Gitea and create support repository +fleet_repo_init() { + python -m integration \ + gitea register \ + $GITEA_HOSTEA_USERNAME $GITEA_HOSTEA_PASSWORD \ + $GITEA_HOSTEA_EMAIL \ + $GITEA_URL || true + python -m integration \ + gitea login \ + $GITEA_HOSTEA_USERNAME $GITEA_HOSTEA_PASSWORD \ + $GITEA_HOSTEA_EMAIL \ + $GITEA_URL + + new_fleet_repo_init $GITEA_HOSTEA_FLEET_REPO + +} + # Create user on Hostea to simulate a Hostea customer hostea_customer_simulation() { python -m integration \ From bbcd373fe4b2b309ae2ad8fe19face98383e99fd Mon Sep 17 00:00:00 2001 From: realaravinth Date: Fri, 1 Jul 2022 19:53:00 +0530 Subject: [PATCH 06/10] feat: redirect to VM deletion post sudo authentication --- dash/views.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dash/views.py b/dash/views.py index 89825eb..383a072 100644 --- a/dash/views.py +++ b/dash/views.py @@ -142,5 +142,4 @@ def delete_instance(request, name): ctx["instance"] = instance return render(request, "dash/instances/delete/index.html", context=ctx) - instance.delete() - return redirect(reverse("dash.home")) + return redirect(reverse("infra.rm", args=(instance.name,))) From 8fc20d16be026d862890558640a3ff7cc573cb37 Mon Sep 17 00:00:00 2001 From: realaravinth Date: Fri, 1 Jul 2022 19:54:15 +0530 Subject: [PATCH 07/10] feat: ues separate fleet repo for each unit test --- billing/tests.py | 45 +++++-------------------------- dash/tests.py | 59 ++++++++++++++++++++++++++++++++++++++--- infrastructure/tests.py | 29 ++++++++------------ 3 files changed, 73 insertions(+), 60 deletions(-) diff --git a/billing/tests.py b/billing/tests.py index 16a6c37..ac6cca2 100644 --- a/billing/tests.py +++ b/billing/tests.py @@ -26,7 +26,7 @@ from django.conf import settings from payments import get_payment_model, RedirectNeeded, PaymentStatus from accounts.tests import register_util, login_util -from dash.tests import create_configurations, create_instance_util +from dash.tests import create_configurations, create_instance_util, infra_custom_config from dash.models import Instance from .models import Payment @@ -43,55 +43,21 @@ class BillingTest(TestCase): register_util(t=self, username=self.username) create_configurations(t=self) + @override_settings(HOSTEA=infra_custom_config(test_name="test_payments")) def test_payments(self): c = Client() login_util(self, c, "accounts.home") - instance_name = "test_create_instance_renders" + instance_name = "test_payments" create_instance_util( t=self, c=c, instance_name=instance_name, config=self.instance_config[0] ) instance = Instance.objects.get(name=instance_name) - payment_uri = reverse("billing.invoice.generate", args=(instance_name,)) - - # generate invoice - resp = c.get(payment_uri) - self.assertEqual(resp.status_code, 302) - invoice_uri = resp.headers["Location"] - self.assertEqual("invoice/payment/" in invoice_uri, True) - - # try to generate duplicate invoice, but should get redirected to previous invoice - resp = c.get(payment_uri) - self.assertEqual(resp.status_code, 302) - self.assertEqual(invoice_uri == resp.headers["Location"], True) - - # check if invoice details page is displaying the invoice - # if payment is yet to be made: - # template will show payment button - # else: - # template will show payment date - resp = c.get(invoice_uri) - self.assertEqual(str.encode(instance_name) in resp.content, True) - self.assertEqual( - str.encode(str(self.instance_config[0].rent)) in resp.content, True - ) - self.assertEqual(str.encode("Paid on") in resp.content, False) - - # check if the unpaid invoice is displayed in the pending invoice view - resp = c.get(reverse("billing.invoice.pending")) - self.assertEqual(str.encode(invoice_uri) in resp.content, True) - - self.assertEqual(payment_fullfilled(instance=instance), False) - - # simulate payment. There's probably a better way to do this - payment = get_payment_model().objects.get(paid_by=self.user) - payment.status = PaymentStatus.CONFIRMED - payment.save() - self.assertEqual(payment_fullfilled(instance=instance), True) - # + payment = get_payment_model().objects.get(paid_by=self.user) + invoice_uri = reverse("billing.invoice.details", args=(payment.public_ref,)) # check if paid invoice is listed in paid invoice list view resp = c.get(reverse("billing.invoice.paid")) @@ -111,6 +77,7 @@ class BillingTest(TestCase): # try to generate an invoice for the second time on the same VM # shouldn't be possible since payment is already made for the duration + payment_uri = reverse("billing.invoice.generate", args=(instance.name,)) resp = c.get(payment_uri) self.assertEqual(resp.status_code, 400) diff --git a/dash/tests.py b/dash/tests.py index 8d02936..899adea 100644 --- a/dash/tests.py +++ b/dash/tests.py @@ -12,12 +12,20 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import subprocess +import shutil +import os +from time import sleep +from pathlib import Path + + from django.contrib.auth import get_user_model from django.utils.http import urlencode from django.urls import reverse from django.test import TestCase, Client, override_settings from django.conf import settings from django.db.utils import IntegrityError +from payments import get_payment_model, RedirectNeeded, PaymentStatus from accounts.tests import login_util, register_util @@ -44,6 +52,28 @@ def create_configurations(t: TestCase): ) +def infra_custom_config(test_name: str): + def create_fleet_repo(test_name: str): + subprocess.run( + ["./integration/ci.sh", "new_fleet_repo", test_name], + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + ) + sleep(10) + + create_fleet_repo(test_name=test_name) + + c = settings.HOSTEA + path = Path(f"/tmp/hostea/dashboard/{test_name}/repo") + if path.exists(): + shutil.rmtree(path) + c["INFRA"]["HOSTEA_REPO"]["PATH"] = str(path) + remote_base = os.environ.get("HOSTEA_INFRA_HOSTEA_REPO_REMOTE") + c["INFRA"]["HOSTEA_REPO"]["REMOTE"] = f"{remote_base}{test_name}.git" + print(c["INFRA"]["HOSTEA_REPO"]["REMOTE"]) + return c + + def create_instance_util( t: TestCase, c: Client, instance_name: str, config: InstanceConfiguration ): @@ -55,6 +85,22 @@ def create_instance_util( resp.headers["location"], reverse("billing.invoice.generate", args=(instance_name,)), ) + # generate invoice + payment_uri = reverse("billing.invoice.generate", args=(instance_name,)) + resp = c.get(payment_uri) + t.assertEqual(resp.status_code, 302) + invoice_uri = resp.headers["Location"] + t.assertEqual("invoice/payment/" in invoice_uri, True) + + # simulate payment. There's probably a better way to do this + payment = get_payment_model().objects.get( + paid_by=t.user, instance_name=instance_name + ) + payment.status = PaymentStatus.CONFIRMED + payment.save() + + resp = c.get(reverse("infra.create", args=(instance_name,))) + t.assertEqual(resp.status_code, 200) class DashHome(TestCase): @@ -151,6 +197,9 @@ class CreateInstance(TestCase): register_util(t=self, username="createinstance_user") create_configurations(t=self) + @override_settings( + HOSTEA=infra_custom_config(test_name="test_create_instance_renders") + ) def test_create_instance_renders(self): c = Client() login_util(self, c, "accounts.home") @@ -236,12 +285,16 @@ class CreateInstance(TestCase): resp = c.post(delete_uri) self.assertEqual(resp.status_code, 302) - self.assertEqual(resp.headers["location"], reverse("dash.home")) + self.assertEqual( + resp.headers["location"], reverse("infra.rm", args=(instance.name,)) + ) + resp = c.get(resp.headers["location"]) + self.assertEqual(resp.status_code, 302) + self.assertEqual(resp.headers["location"], reverse("dash.instances.list")) self.assertEqual( Instance.objects.filter( - name=instance.name, + name=instance_name, owned_by=self.user, - configuration_id=self.instance_config[0], ).exists(), False, ) diff --git a/infrastructure/tests.py b/infrastructure/tests.py index cb59015..71d576d 100644 --- a/infrastructure/tests.py +++ b/infrastructure/tests.py @@ -14,7 +14,6 @@ # along with this program. If not, see . import shutil import time -import os from io import StringIO from urllib.parse import urlunparse from pathlib import Path @@ -28,22 +27,13 @@ from git import Repo from dash.models import Instance, InstanceConfiguration from accounts.tests import register_util, login_util -from dash.tests import create_configurations, create_instance_util +from dash.tests import create_configurations, create_instance_util, infra_custom_config from infrastructure.management.commands.vm import translate_sizes from .utils import Infra, Worker from .models import InstanceCreated, Job, JobType -def custom_config(test_name: str): - c = settings.HOSTEA - path = Path(f"/tmp/hostea/dashboard/{test_name}/repo") - if path.exists(): - shutil.rmtree(path) - c["INFRA"]["HOSTEA_REPO"]["PATH"] = str(path) - return c - - class InfraUtilTest(TestCase): """ Tests billing system @@ -54,7 +44,7 @@ class InfraUtilTest(TestCase): register_util(t=self, username=self.username) create_configurations(t=self) - @override_settings(HOSTEA=custom_config(test_name="test_path_util")) + @override_settings(HOSTEA=infra_custom_config(test_name="test_path_util")) def test_path_utils(self): infra = Infra() subdomain = "foo" @@ -85,7 +75,7 @@ class InfraUtilTest(TestCase): infra._hostscript_path(subdomain=subdomain), ) - @override_settings(HOSTEA=custom_config(test_name="test_add_vm")) + @override_settings(HOSTEA=infra_custom_config(test_name="test_add_vm")) def test_add_vm(self): infra = Infra() c = Client() @@ -105,9 +95,12 @@ class InfraUtilTest(TestCase): after_add = infra.repo.head.commit.hexsha self.assertEqual(before_add is not after_add, True) - c = custom_config(test_name="test_add_vm--get-head") - path = c["INFRA"]["HOSTEA_REPO"]["PATH"] + # c = infra_custom_config(test_name="test_add_vm--get-head") + path = Path("/tmp/hostea/dashboard/check-test_add_vm") + if path.exists(): + shutil.rmtree(path) repo = Repo.clone_from(conf["REMOTE"], path, env=infra.env) + repo.git.pull(env=infra.env) self.assertEqual(repo.head.commit.hexsha == after_add, True) before_rm = infra.repo.head.commit.hexsha @@ -115,10 +108,10 @@ class InfraUtilTest(TestCase): after_rm = infra.repo.head.commit.hexsha self.assertEqual(before_add is not after_add, True) - repo.git.pull() + repo.git.pull(env=infra.env) self.assertEqual(repo.head.commit.hexsha == after_rm, True) - @override_settings(HOSTEA=custom_config(test_name="test_cmd")) + @override_settings(HOSTEA=infra_custom_config(test_name="test_cmd")) def test_cmd(self): subdomain = "cmd_vm" infra = Infra() @@ -204,7 +197,7 @@ class InfraUtilTest(TestCase): w = Worker(job=job) w.start() - time.sleep(5) + time.sleep(15) self.assertEqual(w.is_alive(), False) w.join() self.assertEqual( From f86dd2ff3713136871834aaec1e66d9a063f6aca Mon Sep 17 00:00:00 2001 From: realaravinth Date: Fri, 1 Jul 2022 19:54:20 +0530 Subject: [PATCH 08/10] feat: delete_vm takes only one parameter --- infrastructure/management/commands/vm.py | 2 +- infrastructure/utils.py | 12 +++++------- infrastructure/views.py | 8 +++----- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/infrastructure/management/commands/vm.py b/infrastructure/management/commands/vm.py index ed44ab2..2495a20 100644 --- a/infrastructure/management/commands/vm.py +++ b/infrastructure/management/commands/vm.py @@ -119,7 +119,7 @@ class Command(BaseCommand): vm_name = options[self.vm_name_key] if Instance.objects.filter(name=vm_name).exists(): instance = Instance.objects.get(name=vm_name) - delete_vm(instance=instance, owner=instance.owned_by.username) + delete_vm(instance=instance) def handle(self, *args, **options): for i in [self.action_key, self.vm_name_key]: diff --git a/infrastructure/utils.py b/infrastructure/utils.py index 9261b0b..ce6251e 100644 --- a/infrastructure/utils.py +++ b/infrastructure/utils.py @@ -20,6 +20,7 @@ from pathlib import Path from threading import Thread, Event from time import sleep +import sh from django.utils.crypto import get_random_string from django.template.loader import render_to_string from django.core.mail import send_mail @@ -37,8 +38,6 @@ class Worker(Thread): self.job = job super().__init__() - ######### self.daemon = True - def run(self): gitea_uri = Infra.get_gitea_uri(instance=self.job.instance) woodpecker = Infra.get_woodpecker_uri(instance=self.job.instance) @@ -46,14 +45,12 @@ class Worker(Thread): try: print(f"[ping] Trying to reach {gitea_uri}") resp = requests.get(gitea_uri) - print(resp.status_code) if resp.status_code == 200: break except Exception: return False sleep(10) - print("sending email") job = self.job self.job = None email = job.instance.owned_by.email @@ -73,7 +70,6 @@ Woodpecker CI: {woodpecker} recipient_list=[email], ) job.delete() - print("job deleted") def create_vm_if_not_exists(instance: Instance) -> (str, Commit): @@ -96,7 +92,7 @@ def create_vm_if_not_exists(instance: Instance) -> (str, Commit): return None -def delete_vm(instance: Instance, owner: str): +def delete_vm(instance: Instance): infra = Infra() infra.remove_vm(instance=instance) if InstanceCreated.objects.filter(instance=instance).exists(): @@ -229,7 +225,9 @@ class Infra: ) def _pull(self): - self.repo.git.pull(env=self.env, rebase="true") + self.repo.git.fetch(env=self.env) + + # TODO: switch to using Git cmd @staticmethod def translate_size(instance: Instance) -> str: diff --git a/infrastructure/views.py b/infrastructure/views.py index 37a3292..7607c6c 100644 --- a/infrastructure/views.py +++ b/infrastructure/views.py @@ -26,7 +26,7 @@ from accounts.decorators import confirm_access from dash.models import Instance from billing.utils import payment_fullfilled -from .utils import create_vm_if_not_exists, Infra +from .utils import create_vm_if_not_exists, Infra, delete_vm def default_ctx(title: str, username: str): @@ -84,7 +84,5 @@ def delete_instance(request, instance_name: str): Dashboard homepage view """ instance = get_object_or_404(Instance, name=instance_name, owned_by=request.user) - infra = Infra() - infra.remove_vm(instance=instance) - # TODO: push isn't implemented yet - return HttpResponse() + delete_vm(instance=instance) + return redirect(reverse("dash.instances.list")) From e5ebdc29ce9647e4762a2c000dd2471fab352761 Mon Sep 17 00:00:00 2001 From: realaravinth Date: Fri, 1 Jul 2022 19:54:56 +0530 Subject: [PATCH 09/10] feat: CI: replace fleet repo remote URI with remote URI template --- .woodpecker.yml | 2 +- infrastructure/utils.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index b485198..c05b5d8 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -4,7 +4,7 @@ pipeline: environment: - DATABSE_URL=postgres://postgres:password@database:5432/postgres - EMAIL_URL=smtp://admin:password@smtp:10025 - - HOSTEA_INFRA_HOSTEA_REPO_REMOTE=ssh://git@gitea:22/hostea/fleet.git + - HOSTEA_INFRA_HOSTEA_REPO_REMOTE=ssh://git@gitea:22/hostea/ - HOSTEA_META_GITEA_INSTANCE=http://gitea:3000 commands: - export HOSTEA_INFRA_HOSTEA_REPO_SSH_KEY="$(realpath ./tests/fleet-deploy-key)" diff --git a/infrastructure/utils.py b/infrastructure/utils.py index ce6251e..45393a5 100644 --- a/infrastructure/utils.py +++ b/infrastructure/utils.py @@ -20,7 +20,6 @@ from pathlib import Path from threading import Thread, Event from time import sleep -import sh from django.utils.crypto import get_random_string from django.template.loader import render_to_string from django.core.mail import send_mail From 4a47543a0fdf02e6cdf33db03c9f809e52be117d Mon Sep 17 00:00:00 2001 From: realaravinth Date: Fri, 1 Jul 2022 20:59:10 +0530 Subject: [PATCH 10/10] hostfix: use default source code link when settings.py doesn't provide one --- dash/utils.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dash/utils.py b/dash/utils.py index 99d45d0..f54a96a 100644 --- a/dash/utils.py +++ b/dash/utils.py @@ -65,7 +65,14 @@ source_code = None def footer_ctx(): global source_code if source_code is None: - source_code = {"text": "Source Code", "link": settings.HOSTEA["SOURCE_CODE"]} + if "SOURCE_CODE" in settings.HOSTEA: + source_code = { + "text": "Source Code", + "link": settings.HOSTEA["SOURCE_CODE"], + } + else: + link = "https://gitea.hostea.org/Hostea/dashboard" + source_code = {"text": "Source Code", "link": link} try: r = Repo(".") commit = r.head.commit.hexsha