From ce0498b01372aac5ca87e1dc7aa30b3936bf2d3b Mon Sep 17 00:00:00 2001 From: realaravinth Date: Fri, 8 Jul 2022 18:43:04 +0530 Subject: [PATCH 1/4] fix: add instance link in primary nav bar points to right page --- dash/templates/dash/common/components/primary-nav.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dash/templates/dash/common/components/primary-nav.html b/dash/templates/dash/common/components/primary-nav.html index f502c0d..6eb930e 100644 --- a/dash/templates/dash/common/components/primary-nav.html +++ b/dash/templates/dash/common/components/primary-nav.html @@ -18,7 +18,7 @@
- Add Instance
-- 2.40.1 From 2c8a5909cb76f66671b95e0cb6b9fbd2e09ac3e9 Mon Sep 17 00:00:00 2001 From: realaravinth Date: Fri, 8 Jul 2022 18:51:04 +0530 Subject: [PATCH 2/4] fix: generate absolute URI when attaching links in invoice and payment notification fixes: https://gitea.hostea.org/Hostea/dashboard/issues/37 --- billing/templates/billing/emails/payment-receipt.txt | 2 +- billing/utils.py | 9 ++++++++- billing/views.py | 3 +++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/billing/templates/billing/emails/payment-receipt.txt b/billing/templates/billing/emails/payment-receipt.txt index 5236f9e..f2477c0 100644 --- a/billing/templates/billing/emails/payment-receipt.txt +++ b/billing/templates/billing/emails/payment-receipt.txt @@ -14,7 +14,7 @@ Hostea Receipt - {{payment.date.month}}/{{payment.date.day}}/{{payment.date.year To view the receipt online, please see the following link: -{% url 'billing.invoice.details' payment_public_id=payment.public_ref %} +{{link}} We appreciate your business! diff --git a/billing/utils.py b/billing/utils.py index 57dd0fd..8598c45 100644 --- a/billing/utils.py +++ b/billing/utils.py @@ -13,6 +13,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . from enum import Enum, unique +from urllib.parse import urlparse, urlunparse from datetime import datetime, timedelta, timezone from payments import get_payment_model, RedirectNeeded, PaymentStatus @@ -33,6 +34,10 @@ def __get_delta(): delta = now - timedelta(seconds=(60 * 60 * 24 * 30)) # one month return delta +def get_invoice_link(payment: Payment): + invoice_link = reverse("billing.invoice.details", args=(payment.public_ref,)) + parsed = urlparse(settings.PAYMENT_HOST) + return urlunparse((parsed.scheme, parsed.netloc, invoice_link, "", "", "")) def payment_fullfilled(instance: Instance) -> bool: delta = __get_delta() @@ -87,7 +92,8 @@ def generate_invoice(instance: Instance) -> Payment: instance=instance, ) - invoice_link = reverse("billing.invoice.details", args=(payment.public_ref,)) + invoice_link = get_invoice_link(payment=payment) + ctx = { "username": instance.owned_by.username, "link": invoice_link, @@ -102,6 +108,7 @@ def generate_invoice(instance: Instance) -> Payment: email = instance.owned_by.email sender = settings.DEFAULT_FROM_EMAIL + send_mail( subject="[Hostea] An invoice is generated for your Hostea VM", message=body, diff --git a/billing/views.py b/billing/views.py index 00ac7f3..a282eb6 100644 --- a/billing/views.py +++ b/billing/views.py @@ -33,6 +33,7 @@ from .utils import ( generate_invoice as generate_invoice_util, GenerateInvoiceErrorCode, GenerateInvoiceException, + get_invoice_link ) @@ -111,9 +112,11 @@ def payment_success(request, payment_public_id): payment = get_object_or_404( get_payment_model(), public_ref=payment_public_id, paid_by=request.user ) + ctx = { "username": request.user.username, "payment": payment, + "link": get_invoice_link(payment=payment), } body = render_to_string( -- 2.40.1 From 6e84746a2cb29b0e7bd901fe7db6b841534614c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Dachary?= Date: Fri, 8 Jul 2022 14:26:59 +0200 Subject: [PATCH 3/4] git config before push --- Makefile | 8 +---- dash/utils.py | 1 - infrastructure/tests.py | 46 ++++-------------------- infrastructure/utils.py | 77 +++++++++++++++++++---------------------- integration/lib.sh | 6 ++-- manage.py | 4 +-- requirements.txt | 3 +- 7 files changed, 49 insertions(+), 96 deletions(-) diff --git a/Makefile b/Makefile index ad56d80..c7757df 100644 --- a/Makefile +++ b/Makefile @@ -38,13 +38,7 @@ integration-test: ## run integration tests . ./venv/bin/activate && integration/tests.sh lint: ## Run linter - @./venv/bin/black ./dashboard/ - @./venv/bin/black ./accounts/ - @./venv/bin/black ./dash/ - @./venv/bin/black ./support/ - @./venv/bin/black ./billing/ - @./venv/bin/black ./infrastructure/ - @./venv/bin/black ./integration/ + @./venv/bin/black dashboard accounts dash support billing infrastructure integration migrate: ## Run migrations $(call run_migrations) diff --git a/dash/utils.py b/dash/utils.py index f54a96a..ac3d7b3 100644 --- a/dash/utils.py +++ b/dash/utils.py @@ -14,7 +14,6 @@ # along with this program. If not, see . from enum import Enum, unique -from git import Repo from django.contrib.auth.models import User from django.conf import settings diff --git a/infrastructure/tests.py b/infrastructure/tests.py index 71d576d..7ac8e08 100644 --- a/infrastructure/tests.py +++ b/infrastructure/tests.py @@ -12,19 +12,11 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import shutil import time -from io import StringIO -from urllib.parse import urlunparse -from pathlib import Path -import requests from django.test import TestCase, Client, override_settings -from django.conf import settings from django.core.management import call_command -from git import Repo - from dash.models import Instance, InstanceConfiguration from accounts.tests import register_util, login_util from dash.tests import create_configurations, create_instance_util, infra_custom_config @@ -77,59 +69,37 @@ class InfraUtilTest(TestCase): @override_settings(HOSTEA=infra_custom_config(test_name="test_add_vm")) def test_add_vm(self): - infra = Infra() c = Client() - conf = settings.HOSTEA["INFRA"]["HOSTEA_REPO"] login_util(self, c, "accounts.home") subdomain = "add_vm" - base = infra.repo_path - create_instance_util( t=self, c=c, instance_name=subdomain, config=self.instance_config[0] ) - before_add = infra.repo.head.commit.hexsha instance = Instance.objects.get(name=subdomain) - woodpecker_agent_secret = infra.add_vm(instance=instance) - after_add = infra.repo.head.commit.hexsha - self.assertEqual(before_add is not after_add, True) - # c = infra_custom_config(test_name="test_add_vm--get-head") - path = Path("/tmp/hostea/dashboard/check-test_add_vm") - if path.exists(): - shutil.rmtree(path) - repo = Repo.clone_from(conf["REMOTE"], path, env=infra.env) - repo.git.pull(env=infra.env) - self.assertEqual(repo.head.commit.hexsha == after_add, True) + infra = Infra() + before_add = infra._sha() + (password, after_add) = infra.add_vm(instance=instance) + self.assertNotEqual(before_add, after_add) - before_rm = infra.repo.head.commit.hexsha - infra.remove_vm(instance=instance) - after_rm = infra.repo.head.commit.hexsha - self.assertEqual(before_add is not after_add, True) - - repo.git.pull(env=infra.env) - self.assertEqual(repo.head.commit.hexsha == after_rm, True) + before_rm = after_add + after_rm = infra.remove_vm(instance=instance) + self.assertNotEqual(before_rm, after_rm) @override_settings(HOSTEA=infra_custom_config(test_name="test_cmd")) def test_cmd(self): subdomain = "cmd_vm" infra = Infra() c = Client() - conf = settings.HOSTEA["INFRA"]["HOSTEA_REPO"] login_util(self, c, "accounts.home") - base = infra.repo_path - - stdout = StringIO() - stderr = StringIO() - self.assertEqual(Instance.objects.filter(name=subdomain).exists(), False) # username exists call_command( "vm", "create", subdomain, f"--owner={self.username}", "--flavor=medium" ) - out = stdout.getvalue() instance = Instance.objects.get(name=subdomain) @@ -152,7 +122,6 @@ class InfraUtilTest(TestCase): # run create vm command again with different configuration but same name # to crudely check idempotency - old_size = instance.configuration_id call_command( "vm", "create", subdomain, f"--owner={self.username}", "--flavor=large" ) @@ -172,7 +141,6 @@ class InfraUtilTest(TestCase): ) call_command("vm", "delete", subdomain) - out = stdout.getvalue() self.assertEqual(Instance.objects.filter(name=subdomain).exists(), False) host_vars_dir = infra._host_vars_dir(subdomain) diff --git a/infrastructure/utils.py b/infrastructure/utils.py index 671b0d0..3cb098b 100644 --- a/infrastructure/utils.py +++ b/infrastructure/utils.py @@ -12,25 +12,28 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import logging import os +import sh import shutil import yaml import requests from pathlib import Path -from threading import Thread, Event +from threading import Thread from time import sleep from django.utils.crypto import get_random_string from django.template.loader import render_to_string from django.core.mail import send_mail from django.conf import settings -from git import Repo, Commit -from git.exc import InvalidGitRepositoryError from dash.models import Instance from infrastructure.models import InstanceCreated, JobType, Job +logging.basicConfig() +logger = logging.getLogger(__name__) + class Worker(Thread): def __init__(self, job: Job): @@ -75,7 +78,7 @@ class Worker(Thread): job.delete() -def create_vm_if_not_exists(instance: Instance) -> (str, Commit): +def create_vm_if_not_exists(instance: Instance) -> (str, str): """ Create VM utility. Gitea password is returned """ @@ -111,18 +114,17 @@ class Infra: def __init__(self): conf = settings.HOSTEA["INFRA"]["HOSTEA_REPO"] self.repo_path = Path(conf["PATH"]) - if not self.repo_path.exists(): - os.makedirs(self.repo_path) - - ssh_cmd = f"/usr/bin/ssh -oStrictHostKeyChecking=no -i {conf['SSH_KEY']}" - self.env = {"GIT_SSH_COMMAND": ssh_cmd} self._clone() def _clone(self): + conf = settings.HOSTEA["INFRA"]["HOSTEA_REPO"] + ssh_cmd = f"/usr/bin/ssh -oStrictHostKeyChecking=no -i {conf['SSH_KEY']}" + self.git = sh.git.bake(_env={"GIT_SSH_COMMAND": ssh_cmd}) conf = settings.HOSTEA["INFRA"]["HOSTEA_REPO"] if os.path.exists(self.repo_path): shutil.rmtree(self.repo_path) - self.repo = Repo.clone_from(conf["REMOTE"], self.repo_path, env=self.env) + self.git.clone(conf["REMOTE"], self.repo_path) + self.git = self.git.bake("-C", self.repo_path) def _host_vars_dir(self, subdomain: str) -> Path: """ @@ -210,25 +212,21 @@ class Infra: f.write(content) f.write("\n") - def _add_files(self, subdomain: str): - """ - Add all relevant files of a VM - """ - self.repo.git.add(str(self._host_vars_dir(subdomain=subdomain))) - self.repo.git.add(str(self._backup_path(subdomain=subdomain))) - self.repo.git.add(str(self._service_path(subdomain=subdomain))) - self.repo.git.add(str(self._hostscript_path(subdomain=subdomain))) + def _push(self, message): + self.git.add(".") + self.git.config("user.email", settings.HOSTEA["INSTANCE_MAINTAINER_CONTACT"]) + self.git.config("user.name", "Hostea dashboard") + try: + self.git.commit("-m", f"dashboard: {message}") + except sh.ErrorReturnCode_1: + logger.debug("no change") + else: + self.git.push("origin", "master") + return self._sha() - def _commit(self, action: str, subdomain: str) -> Commit: - """ - Commit changes to a VM configuration - """ - - self._add_files(subdomain=subdomain) - return self.repo.git.commit( - message=f"{action} VM {subdomain}", - author="Dashboard Bot ", - ) + def _sha(self): + sha = self.git("rev-parse", "origin/master") + return str(sha).strip() @staticmethod def translate_size(instance: Instance) -> str: @@ -244,13 +242,12 @@ class Infra: return "openstack_flavor_large" return instance.configuration_id.name - def add_vm(self, instance: Instance) -> (str, Commit): + def add_vm(self, instance: Instance) -> (str, str): """ Add new VM to infrastructure repository The gitea user password is returned """ - self._clone() subdomain = instance.name host_vars_dir = self._host_vars_dir(subdomain) @@ -321,29 +318,26 @@ class Infra: ), ) - commit = self._commit(action="add", subdomain=subdomain) - self.repo.git.push(env=self.env) + commit = self._push(f"add vm {subdomain}") return (gitea_password, commit) def remove_vm(self, instance: Instance): """ Remove a VM from infrastructure repository """ - self._clone() subdomain = instance.name - try: - - host_vars_dir = self._host_vars_dir(subdomain) + host_vars_dir = self._host_vars_dir(subdomain) + if os.path.exists(host_vars_dir): shutil.rmtree(host_vars_dir) - backup = self._backup_path(subdomain) + backup = self._backup_path(subdomain) + if os.path.exists(backup): os.remove(backup) - service = self._service_path(subdomain) + service = self._service_path(subdomain) + if os.path.exists(service): os.remove(service) - except FileNotFoundError: - pass hostscript = self._hostscript_path(subdomain) with open(hostscript, "w+", encoding="utf-8") as f: @@ -356,5 +350,4 @@ class Infra: context={"subdomain": subdomain}, ), ) - self._commit(action="rm", subdomain=subdomain) - self.repo.git.push(env=self.env) + return self._push(f"rm vm {subdomain}") diff --git a/integration/lib.sh b/integration/lib.sh index 079b9c2..7e79870 100755 --- a/integration/lib.sh +++ b/integration/lib.sh @@ -139,13 +139,13 @@ new_fleet_repo_init() { tmp_dir=$(mktemp -d) pushd $tmp_dir echo "init" >> README + git init if is_ci then - git config --global user.email "${CI_COMMIT_AUTHOR_EMAIL}" - git config --global user.name "${CI_COMMIT_AUTHOR}" + git config user.email "${CI_COMMIT_AUTHOR_EMAIL}" + git config user.name "${CI_COMMIT_AUTHOR}" chmod 600 $GITEA_HOSTEA_FLEET_DEPLOY_KEY_PRIVATE fi - git init git add README git commit -m "init" REMOTE="$GITEA_SSH_URL/$GITEA_HOSTEA_USERNAME/$1.git" diff --git a/manage.py b/manage.py index 73d4f5b..f2be5c4 100755 --- a/manage.py +++ b/manage.py @@ -6,7 +6,7 @@ import sys def main(): """Run administrative tasks.""" - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dashboard.settings') + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dashboard.settings") try: from django.core.management import execute_from_command_line except ImportError as exc: @@ -18,5 +18,5 @@ def main(): execute_from_command_line(sys.argv) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/requirements.txt b/requirements.txt index 1aa7e5f..bb351ab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,8 +15,6 @@ django-oauth-toolkit==2.0.0 django-payments==1.0.0 django-phonenumber-field==6.3.0 djangorestframework==3.13.1 -gitdb==4.0.9 -GitPython==3.1.27 greenlet==1.1.2 idna==3.3 install==1.3.5 @@ -40,6 +38,7 @@ pytz==2022.1 PyYAML==6.0 requests==2.27.1 six==1.16.0 +sh==1.14.2 smmap==5.0.0 sqlparse==0.4.2 stripe==3.4.0 -- 2.40.1 From bebf18946a5f0a668575b2a7b6868d2140e8c86d Mon Sep 17 00:00:00 2001 From: realaravinth Date: Fri, 8 Jul 2022 19:31:06 +0530 Subject: [PATCH 4/4] fix: re-enable billing app tests Commented out for debugging, forgot to enable(!!) --- billing/tests.py | 202 +++++++++++++++++++++++------------------------ billing/utils.py | 3 +- billing/views.py | 2 +- 3 files changed, 104 insertions(+), 103 deletions(-) diff --git a/billing/tests.py b/billing/tests.py index 36f23e8..54cf554 100644 --- a/billing/tests.py +++ b/billing/tests.py @@ -35,107 +35,107 @@ from .models import Payment from .utils import payment_fullfilled -# class BillingTest(TestCase): -# """ -# Tests billing system -# """ -# -# def setUp(self): -# self.username = "billing_user" -# register_util(t=self, username=self.username) -# create_configurations(t=self) -# -# @override_settings(HOSTEA=infra_custom_config(test_name="test_payments")) -# def test_payments(self): -# c = Client() -# login_util(self, c, "accounts.home") -# instance_name = "test_payments" -# create_instance_util( -# t=self, c=c, instance_name=instance_name, config=self.instance_config[0] -# ) -# -# instance = Instance.objects.get(name=instance_name) -# -# self.assertEqual(payment_fullfilled(instance=instance), True) -# -# payment = get_payment_model().objects.get(paid_by=self.user) -# invoice_uri = reverse("billing.invoice.details", args=(payment.public_ref,)) -# -# # check if paid invoice is listed in paid invoice list view -# resp = c.get(reverse("billing.invoice.paid")) -# self.assertEqual(str.encode(invoice_uri) in resp.content, True) -# -# # check if the paid invoice is displayed in the pending invoice view, should not be displayed -# resp = c.get(reverse("billing.invoice.pending")) -# self.assertEqual(str.encode(invoice_uri) in resp.content, False) -# -# # check if the invoice details view is rendering paid invoice version -# resp = c.get(invoice_uri) -# self.assertEqual(str.encode(instance_name) in resp.content, True) -# self.assertEqual( -# str.encode(str(self.instance_config[0].rent)) in resp.content, True -# ) -# self.assertEqual(str.encode("Paid on") in resp.content, True) -# -# # try to generate an invoice for the second time on the same VM -# # shouldn't be possible since payment is already made for the duration -# payment_uri = reverse("billing.invoice.generate", args=(instance.name,)) -# resp = c.get(payment_uri) -# self.assertEqual(resp.status_code, 400) -# -# ## payment success page; no real functionality but user is redirected here -# # by stripe if payment is successful -# resp = c.get(reverse("billing.invoice.success", args=(payment.public_ref,))) -# self.assertEqual( -# resp.headers["Location"], -# reverse("infra.create", args=(payment.instance_name,)), -# ) -# -# # create_instance_util creates an instance and pays for it. An email is -# # sent when the invoice is generated and one after payment is made -# # -# # So we are first checking for the last email that was sent(receipt) -# # and then the Gitea instance credentials notification followed by the -# # invoice generation email. -# receipt_mail = mail.outbox.pop() -# self.assertEqual( -# all( -# [ -# receipt_mail.to[0] == self.email, -# "This is a receipt for your latest Hostea payment" -# in receipt_mail.body, -# ] -# ), -# True, -# ) -# -# instance_notificaiton = mail.outbox.pop() -# self.assertEqual( -# all( -# [ -# instance_notificaiton.to[0] == self.email, -# "Congratulations on your new Gitea instance!" -# in instance_notificaiton.body, -# ] -# ), -# True, -# ) -# -# invoice_generated_mail = mail.outbox.pop() -# self.assertEqual( -# all( -# [ -# invoice_generated_mail.to[0] == self.email, -# "An invoice is generated" in invoice_generated_mail.body, -# ] -# ), -# True, -# ) -# -# ## payment failure page; no real functionality but user is redirected here -# # by stripe if payment is successful -# resp = c.get(reverse("billing.invoice.fail", args=(payment.public_ref,))) -# self.assertEqual(b"failed" in resp.content, True) +class BillingTest(TestCase): + """ + Tests billing system + """ + + def setUp(self): + self.username = "billing_user" + register_util(t=self, username=self.username) + create_configurations(t=self) + + @override_settings(HOSTEA=infra_custom_config(test_name="test_payments")) + def test_payments(self): + c = Client() + login_util(self, c, "accounts.home") + instance_name = "test_payments" + create_instance_util( + t=self, c=c, instance_name=instance_name, config=self.instance_config[0] + ) + + instance = Instance.objects.get(name=instance_name) + + self.assertEqual(payment_fullfilled(instance=instance), True) + + payment = get_payment_model().objects.get(paid_by=self.user) + invoice_uri = reverse("billing.invoice.details", args=(payment.public_ref,)) + + # check if paid invoice is listed in paid invoice list view + resp = c.get(reverse("billing.invoice.paid")) + self.assertEqual(str.encode(invoice_uri) in resp.content, True) + + # check if the paid invoice is displayed in the pending invoice view, should not be displayed + resp = c.get(reverse("billing.invoice.pending")) + self.assertEqual(str.encode(invoice_uri) in resp.content, False) + + # check if the invoice details view is rendering paid invoice version + resp = c.get(invoice_uri) + self.assertEqual(str.encode(instance_name) in resp.content, True) + self.assertEqual( + str.encode(str(self.instance_config[0].rent)) in resp.content, True + ) + self.assertEqual(str.encode("Paid on") in resp.content, True) + + # try to generate an invoice for the second time on the same VM + # shouldn't be possible since payment is already made for the duration + payment_uri = reverse("billing.invoice.generate", args=(instance.name,)) + resp = c.get(payment_uri) + self.assertEqual(resp.status_code, 400) + + ## payment success page; no real functionality but user is redirected here + # by stripe if payment is successful + resp = c.get(reverse("billing.invoice.success", args=(payment.public_ref,))) + self.assertEqual( + resp.headers["Location"], + reverse("infra.create", args=(payment.instance_name,)), + ) + + # create_instance_util creates an instance and pays for it. An email is + # sent when the invoice is generated and one after payment is made + # + # So we are first checking for the last email that was sent(receipt) + # and then the Gitea instance credentials notification followed by the + # invoice generation email. + receipt_mail = mail.outbox.pop() + self.assertEqual( + all( + [ + receipt_mail.to[0] == self.email, + "This is a receipt for your latest Hostea payment" + in receipt_mail.body, + ] + ), + True, + ) + + instance_notificaiton = mail.outbox.pop() + self.assertEqual( + all( + [ + instance_notificaiton.to[0] == self.email, + "Congratulations on your new Gitea instance!" + in instance_notificaiton.body, + ] + ), + True, + ) + + invoice_generated_mail = mail.outbox.pop() + self.assertEqual( + all( + [ + invoice_generated_mail.to[0] == self.email, + "An invoice is generated" in invoice_generated_mail.body, + ] + ), + True, + ) + + ## payment failure page; no real functionality but user is redirected here + # by stripe if payment is successful + resp = c.get(reverse("billing.invoice.fail", args=(payment.public_ref,))) + self.assertEqual(b"failed" in resp.content, True) class GenerateInvoiceCommand(TestCase): diff --git a/billing/utils.py b/billing/utils.py index 8598c45..afbb4d4 100644 --- a/billing/utils.py +++ b/billing/utils.py @@ -34,11 +34,13 @@ def __get_delta(): delta = now - timedelta(seconds=(60 * 60 * 24 * 30)) # one month return delta + def get_invoice_link(payment: Payment): invoice_link = reverse("billing.invoice.details", args=(payment.public_ref,)) parsed = urlparse(settings.PAYMENT_HOST) return urlunparse((parsed.scheme, parsed.netloc, invoice_link, "", "", "")) + def payment_fullfilled(instance: Instance) -> bool: delta = __get_delta() @@ -108,7 +110,6 @@ def generate_invoice(instance: Instance) -> Payment: email = instance.owned_by.email sender = settings.DEFAULT_FROM_EMAIL - send_mail( subject="[Hostea] An invoice is generated for your Hostea VM", message=body, diff --git a/billing/views.py b/billing/views.py index a282eb6..403e4dd 100644 --- a/billing/views.py +++ b/billing/views.py @@ -33,7 +33,7 @@ from .utils import ( generate_invoice as generate_invoice_util, GenerateInvoiceErrorCode, GenerateInvoiceException, - get_invoice_link + get_invoice_link, ) -- 2.40.1