Backend integration #1
11
Makefile
|
@ -33,11 +33,12 @@ help: ## Prints help for targets with comments
|
||||||
@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
|
@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
|
||||||
|
|
||||||
lint: ## Run linter
|
lint: ## Run linter
|
||||||
@./venv/bin/black ./dashboard/*
|
@./venv/bin/black ./dashboard/
|
||||||
@./venv/bin/black ./accounts/*
|
@./venv/bin/black ./accounts/
|
||||||
@./venv/bin/black ./dash/*
|
@./venv/bin/black ./dash/
|
||||||
@./venv/bin/black ./support/*
|
@./venv/bin/black ./support/
|
||||||
@./venv/bin/black ./billing/*
|
@./venv/bin/black ./billing/
|
||||||
|
@./venv/bin/black ./infrastructure/
|
||||||
|
|
||||||
migrate: ## Run migrations
|
migrate: ## Run migrations
|
||||||
$(call run_migrations)
|
$(call run_migrations)
|
||||||
|
|
|
@ -27,8 +27,10 @@ from payments import get_payment_model, RedirectNeeded, PaymentStatus
|
||||||
|
|
||||||
from accounts.tests import register_util, login_util
|
from accounts.tests import register_util, login_util
|
||||||
from dash.tests import create_configurations, create_instance_util
|
from dash.tests import create_configurations, create_instance_util
|
||||||
|
from dash.models import Instance
|
||||||
|
|
||||||
from .models import Payment
|
from .models import Payment
|
||||||
|
from .utils import payment_fullfilled
|
||||||
|
|
||||||
|
|
||||||
class BillingTest(TestCase):
|
class BillingTest(TestCase):
|
||||||
|
@ -49,6 +51,8 @@ class BillingTest(TestCase):
|
||||||
t=self, c=c, instance_name=instance_name, config=self.instance_config[0]
|
t=self, c=c, instance_name=instance_name, config=self.instance_config[0]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
instance = Instance.objects.get(name=instance_name)
|
||||||
|
|
||||||
payment_uri = reverse("billing.invoice.generate", args=(instance_name,))
|
payment_uri = reverse("billing.invoice.generate", args=(instance_name,))
|
||||||
|
|
||||||
# generate invoice
|
# generate invoice
|
||||||
|
@ -78,11 +82,17 @@ class BillingTest(TestCase):
|
||||||
resp = c.get(reverse("billing.invoice.pending"))
|
resp = c.get(reverse("billing.invoice.pending"))
|
||||||
self.assertEqual(str.encode(invoice_uri) in resp.content, True)
|
self.assertEqual(str.encode(invoice_uri) in resp.content, True)
|
||||||
|
|
||||||
|
self.assertEqual(payment_fullfilled(instance=instance), False)
|
||||||
|
|
||||||
# simulate payment. There's probably a better way to do this
|
# simulate payment. There's probably a better way to do this
|
||||||
payment = get_payment_model().objects.get(paid_by=self.user)
|
payment = get_payment_model().objects.get(paid_by=self.user)
|
||||||
payment.status = PaymentStatus.CONFIRMED
|
payment.status = PaymentStatus.CONFIRMED
|
||||||
payment.save()
|
payment.save()
|
||||||
|
|
||||||
|
self.assertEqual(payment_fullfilled(instance=instance), True)
|
||||||
|
|
||||||
|
#
|
||||||
|
|
||||||
# check if paid invoice is listed in paid invoice list view
|
# check if paid invoice is listed in paid invoice list view
|
||||||
resp = c.get(reverse("billing.invoice.paid"))
|
resp = c.get(reverse("billing.invoice.paid"))
|
||||||
self.assertEqual(str.encode(invoice_uri) in resp.content, True)
|
self.assertEqual(str.encode(invoice_uri) in resp.content, True)
|
||||||
|
|
|
@ -0,0 +1,33 @@
|
||||||
|
# Copyright © 2022 Aravinth Manivannan <realaravinth@batsense.net>
|
||||||
|
#
|
||||||
|
# This program is free software: you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU Affero General Public License as
|
||||||
|
# published by the Free Software Foundation, either version 3 of the
|
||||||
|
# License, or (at your option) any later version.
|
||||||
|
#
|
||||||
|
# This program is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU Affero General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from payments import get_payment_model, RedirectNeeded, PaymentStatus
|
||||||
|
from django.contrib.auth.models import User
|
||||||
|
from django.shortcuts import get_object_or_404
|
||||||
|
|
||||||
|
from dash.models import Instance
|
||||||
|
|
||||||
|
|
||||||
|
def payment_fullfilled(instance: Instance) -> bool:
|
||||||
|
Payment = get_payment_model()
|
||||||
|
now = datetime.now(tz=timezone.utc)
|
||||||
|
delta = now - timedelta(seconds=(60 * 60 * 24 * 30)) # one month
|
||||||
|
|
||||||
|
payment = None
|
||||||
|
for p in Payment.objects.filter(date__gt=(delta), instance_name=instance.name):
|
||||||
|
if p.status == PaymentStatus.CONFIRMED:
|
||||||
|
return True
|
||||||
|
|
||||||
|
return False
|
|
@ -68,7 +68,7 @@ def generate_invoice(request, instance_name: str):
|
||||||
delta = now - timedelta(seconds=(60 * 60 * 24 * 30)) # one month
|
delta = now - timedelta(seconds=(60 * 60 * 24 * 30)) # one month
|
||||||
|
|
||||||
payment = None
|
payment = None
|
||||||
for p in Payment.objects.filter(date__gt=(delta)):
|
for p in Payment.objects.filter(date__gt=(delta), instance_name=instance_name):
|
||||||
if p.status == PaymentStatus.CONFIRMED:
|
if p.status == PaymentStatus.CONFIRMED:
|
||||||
return HttpResponse("BAD REQUEST: Already paid", status=400)
|
return HttpResponse("BAD REQUEST: Already paid", status=400)
|
||||||
elif any([p.status == PaymentStatus.INPUT, p.status == PaymentStatus.WAITING]):
|
elif any([p.status == PaymentStatus.INPUT, p.status == PaymentStatus.WAITING]):
|
||||||
|
|
|
@ -26,18 +26,18 @@ from .models import InstanceConfiguration, Instance
|
||||||
|
|
||||||
def create_configurations(t: TestCase):
|
def create_configurations(t: TestCase):
|
||||||
t.instance_config = [
|
t.instance_config = [
|
||||||
InstanceConfiguration(name="Personal", rent=5.0, ram=0.5, cpu=1, storage=25),
|
InstanceConfiguration.objects.get(
|
||||||
InstanceConfiguration(name="Enthusiast", rent=10.0, ram=2, cpu=2, storage=50),
|
name="s1-2", rent=10, ram=2, cpu=1, storage=10
|
||||||
InstanceConfiguration(
|
|
||||||
name="Small Business", rent=20.0, ram=8, cpu=4, storage=64
|
|
||||||
),
|
),
|
||||||
InstanceConfiguration(
|
InstanceConfiguration.objects.get(
|
||||||
name="Enterprise", rent=100.0, ram=64, cpu=24, storage=1024
|
name="s1-4", rent=20, ram=4, cpu=1, storage=20
|
||||||
|
),
|
||||||
|
InstanceConfiguration.objects.get(
|
||||||
|
name="s1-8", rent=40, ram=8, cpu=2, storage=40
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
for instance in t.instance_config:
|
for instance in t.instance_config:
|
||||||
instance.save()
|
|
||||||
print(f"[*][init] Instance {instance.name} is saved")
|
print(f"[*][init] Instance {instance.name} is saved")
|
||||||
t.assertEqual(
|
t.assertEqual(
|
||||||
InstanceConfiguration.objects.filter(name=instance.name).exists(), True
|
InstanceConfiguration.objects.filter(name=instance.name).exists(), True
|
||||||
|
|
|
@ -47,6 +47,7 @@ INSTALLED_APPS = [
|
||||||
"oauth2_provider",
|
"oauth2_provider",
|
||||||
"payments",
|
"payments",
|
||||||
"billing",
|
"billing",
|
||||||
|
"infrastructure",
|
||||||
]
|
]
|
||||||
|
|
||||||
MIDDLEWARE = [
|
MIDDLEWARE = [
|
||||||
|
@ -177,8 +178,25 @@ HOSTEA = {
|
||||||
# ref: https://gitea.hostea.org/Hostea/july-mvp/issues/17
|
# ref: https://gitea.hostea.org/Hostea/july-mvp/issues/17
|
||||||
"SUPPORT_REPOSITORY": "support",
|
"SUPPORT_REPOSITORY": "support",
|
||||||
},
|
},
|
||||||
|
"INFRA": {
|
||||||
|
"HOSTEA_REPO": {
|
||||||
|
# where to clone the repository
|
||||||
|
"PATH": "/srv/hostea/dashboard/infrastructure",
|
||||||
|
# Git remote URI of the repository
|
||||||
|
"REMOTE": "git@localhost:Hostea/enough.git",
|
||||||
|
# SSH key that can push to the Git repository remote mentioned above
|
||||||
|
"SSH_KEY": "/srv/hostea/deploy",
|
||||||
|
}
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
EMAIL_CONFIG = env.email("EMAIL_URL", default="smtp://admin:password@localhost:10025")
|
EMAIL_CONFIG = env.email("EMAIL_URL", default="smtp://admin:password@localhost:10025")
|
||||||
|
|
||||||
vars().update(EMAIL_CONFIG)
|
vars().update(EMAIL_CONFIG)
|
||||||
|
|
||||||
|
try:
|
||||||
|
import dashboard.local_settings
|
||||||
|
|
||||||
|
print("Found local_settings")
|
||||||
|
except ModuleNotFoundError:
|
||||||
|
pass
|
||||||
|
|
|
@ -23,5 +23,6 @@ urlpatterns = [
|
||||||
path("admin/", admin.site.urls),
|
path("admin/", admin.site.urls),
|
||||||
path("dash/", include("dash.urls")),
|
path("dash/", include("dash.urls")),
|
||||||
path("support/", include("support.urls")),
|
path("support/", include("support.urls")),
|
||||||
|
path("infra/", include("infrastructure.urls")),
|
||||||
path("", include("accounts.urls")),
|
path("", include("accounts.urls")),
|
||||||
]
|
]
|
||||||
|
|
|
@ -72,3 +72,24 @@ PAYMENT_VARIANTS = {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Infrastructure(via [Enough](https://enough.community))
|
||||||
|
|
||||||
|
```python
|
||||||
|
HOSTEA = {
|
||||||
|
# <------snip-------->
|
||||||
|
"INFRA": {
|
||||||
|
"HOSTEA_REPO": {
|
||||||
|
# where to clone the repository
|
||||||
|
"PATH": "/srv/hostea/dashboard/infrastructure",
|
||||||
|
# Git remote URI of the repository
|
||||||
|
"REMOTE": "git@localhost:Hostea/enough.git",
|
||||||
|
# SSH key that can push to the Git repository remote mentioned above
|
||||||
|
"SSH_KEY": "/srv/hostea/deploy",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
```
|
||||||
|
|
||||||
|
### References:
|
||||||
|
|
||||||
|
https://enough-community.readthedocs.io/en/latest/services/hostea.html
|
||||||
|
|
|
@ -0,0 +1,3 @@
|
||||||
|
from django.contrib import admin
|
||||||
|
|
||||||
|
# Register your models here.
|
|
@ -0,0 +1,6 @@
|
||||||
|
from django.apps import AppConfig
|
||||||
|
|
||||||
|
|
||||||
|
class InfrastructureConfig(AppConfig):
|
||||||
|
default_auto_field = "django.db.models.BigAutoField"
|
||||||
|
name = "infrastructure"
|
|
@ -0,0 +1,37 @@
|
||||||
|
# Generated by Django 4.0.3 on 2022-06-25 10:48
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
import django.db.models.deletion
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
initial = True
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
("dash", "0006_auto_20220619_0800"),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.CreateModel(
|
||||||
|
name="InstanceCreated",
|
||||||
|
fields=[
|
||||||
|
(
|
||||||
|
"id",
|
||||||
|
models.BigAutoField(
|
||||||
|
auto_created=True,
|
||||||
|
primary_key=True,
|
||||||
|
serialize=False,
|
||||||
|
verbose_name="ID",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
("creted", models.BooleanField(default=False)),
|
||||||
|
(
|
||||||
|
"instance",
|
||||||
|
models.ForeignKey(
|
||||||
|
on_delete=django.db.models.deletion.PROTECT, to="dash.instance"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
]
|
|
@ -0,0 +1,20 @@
|
||||||
|
# Generated by Django 4.0.3 on 2022-06-25 12:26
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
("infrastructure", "0001_initial"),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="instancecreated",
|
||||||
|
name="gitea_password",
|
||||||
|
field=models.CharField(
|
||||||
|
default=None, max_length=32, verbose_name="Name of this configuration"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]
|
|
@ -0,0 +1,27 @@
|
||||||
|
# Copyright © 2022 Aravinth Manivannan <realaravinth@batsense.net>
|
||||||
|
#
|
||||||
|
# This program is free software: you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU Affero General Public License as
|
||||||
|
# published by the Free Software Foundation, either version 3 of the
|
||||||
|
# License, or (at your option) any later version.
|
||||||
|
#
|
||||||
|
# This program is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU Affero General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
from django.db import models
|
||||||
|
|
||||||
|
from dash.models import Instance
|
||||||
|
|
||||||
|
|
||||||
|
class InstanceCreated(models.Model):
|
||||||
|
instance = models.ForeignKey(Instance, on_delete=models.PROTECT)
|
||||||
|
gitea_password = models.CharField(
|
||||||
|
"Name of this configuration",
|
||||||
|
default=None,
|
||||||
|
max_length=32,
|
||||||
|
)
|
||||||
|
creted = models.BooleanField(default=False)
|
|
@ -0,0 +1,2 @@
|
||||||
|
enough --domain $domain host create {{subdomain}}-host
|
||||||
|
|||||||
|
enough --domain $domain service create --host {{subdomain}}-host gitea
|
|
@ -0,0 +1 @@
|
||||||
|
enough --domain $domain host delete hostea001-host
|
|
@ -0,0 +1,3 @@
|
||||||
|
pets:
|
||||||
|
hosts:
|
||||||
|
{{ subdomain }}-host:
|
|
@ -0,0 +1,83 @@
|
||||||
|
---
|
||||||
|
#
|
||||||
|
#######################################
|
||||||
|
#
|
||||||
|
# Public hostname of the Gitea instance
|
||||||
|
#
|
||||||
|
#
|
||||||
|
gitea_host: "{{ subdomain }}.{{ '{' }}{{ '{' }} domain {{ '}' }}{{ '}' }}"
|
||||||
dachary
commented
Outdated
Review
Note: Enough will replace this {{ domain }} with the actual value, the dashboard does not need to do it. But the gitea. must be replaced by subdomain which is presumably the name asssigned to the instance by the dashboard. Note: Enough will replace this {{ domain }} with the actual value, the dashboard does not need to do it. But the **gitea.** must be replaced by **subdomain** which is presumably the name asssigned to the instance by the dashboard.
|
|||||||
|
#
|
||||||
|
#######################################
|
||||||
|
#
|
||||||
|
# Mailer from
|
||||||
|
#
|
||||||
|
#
|
||||||
|
gitea_mailer_from: "noreply@{{ '{' }}{{ '{' }} domain {{ '}' }}{{ '}' }}"
|
||||||
dachary
commented
Outdated
Review
This must be replaced with noreply@{{ domain }}. It is set to enough.community to facilitate tests because the SMTP relay is very picky about non-existent domain names. This must be replaced with **noreply@{{ domain }}**. It is set to **enough.community** to facilitate tests because the SMTP relay is very picky about non-existent domain names.
|
|||||||
|
#
|
||||||
|
#######################################
|
||||||
|
#
|
||||||
|
# SSH port of the Gitea instance
|
||||||
|
#
|
||||||
|
#
|
||||||
|
gitea_ssh_port: "22"
|
||||||
|
#
|
||||||
|
#######################################
|
||||||
|
#
|
||||||
|
# Gitea version
|
||||||
|
#
|
||||||
|
#
|
||||||
|
gitea_version: "1.16.8"
|
||||||
|
#
|
||||||
|
#######################################
|
||||||
|
#
|
||||||
|
# Admin user name
|
||||||
|
#
|
||||||
|
gitea_user: root
|
||||||
|
#
|
||||||
|
#######################################
|
||||||
|
#
|
||||||
|
# Admin user password
|
||||||
|
#
|
||||||
|
gitea_password: "{{ gitea_password }}"
|
||||||
dachary
commented
Outdated
Review
This must be set by the dashboard to a password that is then communicated to the user so they can login. This must be set by the dashboard to a password that is then communicated to the user so they can login.
|
|||||||
|
#
|
||||||
|
#######################################
|
||||||
|
#
|
||||||
|
# Admin user email
|
||||||
|
#
|
||||||
|
gitea_email: "{{ gitea_email }}"
|
||||||
dachary
commented
Outdated
Review
This must be set to the email of the user so they can receive notifications. This must be set to the email of the user so they can receive notifications.
|
|||||||
|
#
|
||||||
|
#######################################
|
||||||
|
#
|
||||||
|
# Unique hostname of the woodpecker server relative to {{ '{' }}{{ '{' }} domain {{ '}' }}{{ '}' }}
|
||||||
|
#
|
||||||
|
woodpecker_hostname: "{{ woodpecker_hostname }}"
|
||||||
dachary marked this conversation as resolved
Outdated
dachary
commented
Outdated
Review
This should probably be derived from the subdomain, like subdomainci? This should probably be derived from the **subdomain**, like **subdomain**ci?
realaravinth
commented
Outdated
Review
can we do sub-subdomain? If the user's Gitea is available at mygitea.hostea.org, then the CI at can we do sub-subdomain? If the user's Gitea is available at mygitea.hostea.org, then the CI at `woodpecker.mygitea.hostea.org`?
dachary
commented
Outdated
Review
I'd rather not try my luck with this 😓 But it would be prettier. I'd rather not try my luck with this 😓 But it would be prettier.
realaravinth
commented
Outdated
Review
Acknowledged, post-MVP goal then :) Acknowledged, post-MVP goal then :)
|
|||||||
|
#
|
||||||
|
#######################################
|
||||||
|
#
|
||||||
|
# Public hostname of the Woodpecker instance
|
||||||
|
#
|
||||||
|
woodpecker_host: "{{ '{' }}{{ '{' }} woodpecker_hostname {{ '}' }}{{ '}' }}.{{ '{' }}{{ '{' }} domain {{ '}' }}{{ '}' }}"
|
||||||
|
#
|
||||||
|
#######################################
|
||||||
|
#
|
||||||
|
# Gitea users with admin rights on woodpecker
|
||||||
|
#
|
||||||
|
woodpecker_admins: "{{ '{' }}{{ '{' }} gitea_user {{ '}' }}{{ '}' }}"
|
||||||
|
#
|
||||||
|
#######################################
|
||||||
|
#
|
||||||
|
# Woodpecker shared agent secret `openssl rand -hex 32`
|
||||||
|
#
|
||||||
|
woodpecker_agent_secret: {{ woodpecker_agent_secret }}
|
||||||
dachary
commented
Outdated
Review
This must be set by the dashboard but the user does not need to know about it. This must be set by the dashboard but the user does not need to know about it.
|
|||||||
|
#
|
||||||
|
#######################################
|
||||||
|
#
|
||||||
|
# Woodpecker version
|
||||||
|
#
|
||||||
|
woodpecker_version: "v0.15.2"
|
||||||
|
#
|
||||||
|
#######################################
|
||||||
|
#
|
||||||
|
# Woodpecker max procs
|
||||||
|
#
|
||||||
|
woodpecker_max_procs: 1
|
|
@ -0,0 +1 @@
|
||||||
|
openstack_flavor: {{ vm_size }}
|
|
@ -0,0 +1,104 @@
|
||||||
|
# Copyright © 2022 Aravinth Manivannan <realaravinth@batsense.net>
|
||||||
|
#
|
||||||
|
# This program is free software: you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU Affero General Public License as
|
||||||
|
# published by the Free Software Foundation, either version 3 of the
|
||||||
|
# License, or (at your option) any later version.
|
||||||
|
#
|
||||||
|
# This program is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU Affero General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from django.test import TestCase, Client, override_settings
|
||||||
|
from django.conf import settings
|
||||||
|
|
||||||
|
from dash.models import Instance
|
||||||
|
from .utils import Infra
|
||||||
|
|
||||||
|
from accounts.tests import register_util, login_util
|
||||||
|
from dash.tests import create_configurations, create_instance_util
|
||||||
|
|
||||||
|
|
||||||
|
class InfraUtilTest(TestCase):
|
||||||
|
"""
|
||||||
|
Tests billing system
|
||||||
|
"""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.username = "infrautil_user"
|
||||||
|
register_util(t=self, username=self.username)
|
||||||
|
create_configurations(t=self)
|
||||||
|
|
||||||
|
@override_settings(
|
||||||
|
HOSTEA={
|
||||||
|
"INFRA": {
|
||||||
|
"HOSTEA_REPO": {
|
||||||
|
"PATH": "/tmp/hostea/dashboard/test_path_util/repo/",
|
||||||
|
"REMOTE": "git@git.batsense.net:realaravinth/dummy-hostea-dash-test",
|
||||||
|
"SSH_KEY": "/src/atm/.ssh/aravinth",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
def test_path_utils(self):
|
||||||
|
infra = Infra()
|
||||||
|
subdomain = "foo"
|
||||||
|
base = infra.repo_path
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
base.joinpath(f"inventory/host_vars/{subdomain}-host/"),
|
||||||
|
infra._host_vars_dir(subdomain=subdomain),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
base.joinpath(f"inventory/host_vars/{subdomain}-host/gitea.yml"),
|
||||||
|
infra._gitea_path(subdomain=subdomain),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
base.joinpath(f"inventory/host_vars/{subdomain}-host/provision.yml"),
|
||||||
|
infra._provision_path(subdomain=subdomain),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
base.joinpath(f"inventory/{subdomain}-backup.yml"),
|
||||||
|
infra._backup_path(subdomain=subdomain),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
base.joinpath(f"inventory/hosts-scripts/{subdomain}-host.sh"),
|
||||||
|
infra._hostscript_path(subdomain=subdomain),
|
||||||
|
)
|
||||||
|
|
||||||
|
@override_settings(
|
||||||
|
HOSTEA={
|
||||||
|
"INFRA": {
|
||||||
|
"HOSTEA_REPO": {
|
||||||
|
"PATH": "/tmp/hostea/dashboard/test_add_vm/repo/",
|
||||||
|
"REMOTE": "git@git.batsense.net:realaravinth/dummy-hostea-dash-test",
|
||||||
|
"SSH_KEY": "/src/atm/.ssh/aravinth",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
def test_add_vm(self):
|
||||||
|
infra = Infra()
|
||||||
|
c = Client()
|
||||||
|
login_util(self, c, "accounts.home")
|
||||||
|
subdomain = "add_vm"
|
||||||
|
|
||||||
|
base = infra.repo_path
|
||||||
|
|
||||||
|
create_instance_util(
|
||||||
|
t=self, c=c, instance_name=subdomain, config=self.instance_config[0]
|
||||||
|
)
|
||||||
|
|
||||||
|
instance = Instance.objects.get(name=subdomain)
|
||||||
|
woodpecker_agent_secret = infra.add_vm(instance=instance)
|
||||||
|
|
||||||
|
# infra.remove_vm(instance=instance)
|
|
@ -0,0 +1,23 @@
|
||||||
|
# Copyright © 2022 Aravinth Manivannan <realaravinth@batsense.net>
|
||||||
|
#
|
||||||
|
# This program is free software: you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU Affero General Public License as
|
||||||
|
# published by the Free Software Foundation, either version 3 of the
|
||||||
|
# License, or (at your option) any later version.
|
||||||
|
#
|
||||||
|
# This program is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU Affero General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
from django.contrib import admin
|
||||||
|
from django.urls import path, include
|
||||||
|
|
||||||
|
from .views import create_instance, delete_instance
|
||||||
|
|
||||||
|
urlpatterns = [
|
||||||
|
path("create/<str:instance_name>/", create_instance, name="infra.create"),
|
||||||
|
path("rm/<str:instance_name>/", delete_instance, name="infra.rm"),
|
||||||
|
]
|
|
@ -0,0 +1,220 @@
|
||||||
|
# Copyright © 2022 Aravinth Manivannan <realaravinth@batsense.net>
|
||||||
|
#
|
||||||
|
# This program is free software: you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU Affero General Public License as
|
||||||
|
# published by the Free Software Foundation, either version 3 of the
|
||||||
|
# License, or (at your option) any later version.
|
||||||
|
#
|
||||||
|
# This program is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU Affero General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from django.utils.crypto import get_random_string
|
||||||
|
from django.template.loader import render_to_string
|
||||||
|
from django.conf import settings
|
||||||
|
from git import Repo
|
||||||
|
from git.exc import InvalidGitRepositoryError
|
||||||
|
|
||||||
|
from dash.models import Instance
|
||||||
|
|
||||||
|
|
||||||
|
class Infra:
|
||||||
|
"""
|
||||||
|
Utility function to manage infrastructure repository
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
conf = settings.HOSTEA["INFRA"]["HOSTEA_REPO"]
|
||||||
|
self.repo_path = Path(conf["PATH"])
|
||||||
|
if not self.repo_path.exists():
|
||||||
|
os.makedirs(self.repo_path)
|
||||||
|
|
||||||
|
self.ssh_cmd = f"ssh -i {conf['SSH_KEY']}"
|
||||||
|
try:
|
||||||
|
self.repo = Repo(path=self.repo_path)
|
||||||
|
except InvalidGitRepositoryError:
|
||||||
|
self.repo = Repo.clone_from(
|
||||||
|
conf["REMOTE"], self.repo_path, env={"GIT_SSH_COMMAND": self.ssh_cmd}
|
||||||
|
)
|
||||||
|
|
||||||
|
def _host_vars_dir(self, subdomain: str) -> Path:
|
||||||
|
"""
|
||||||
|
utility method: get host_vars directory for a subdomain
|
||||||
|
"""
|
||||||
|
return self.repo_path.joinpath(f"inventory/host_vars/{subdomain}-host/")
|
||||||
|
|
||||||
|
def _provision_path(self, subdomain: str) -> Path:
|
||||||
|
"""
|
||||||
|
utility method: get provision file pay for a subdomain
|
||||||
|
"""
|
||||||
|
|
||||||
|
return self._host_vars_dir(subdomain=subdomain).joinpath("provision.yml")
|
||||||
|
|
||||||
|
def _gitea_path(self, subdomain: str) -> Path:
|
||||||
|
"""
|
||||||
|
utility method: get gitea file for a subdomain
|
||||||
|
"""
|
||||||
|
|
||||||
|
return self._host_vars_dir(subdomain=subdomain).joinpath("gitea.yml")
|
||||||
|
|
||||||
|
def _backup_path(self, subdomain: str) -> Path:
|
||||||
|
"""
|
||||||
|
utility method: get backup file for a subdomain
|
||||||
|
"""
|
||||||
|
|
||||||
|
return self.repo_path.joinpath(f"inventory/{subdomain}-backup.yml")
|
||||||
|
|
||||||
|
def _hostscript_path(self, subdomain: str) -> Path:
|
||||||
|
"""
|
||||||
|
utility method: hostscript file for a subdomain
|
||||||
|
"""
|
||||||
|
return self.repo_path.joinpath(f"inventory/hosts-scripts/{subdomain}-host.sh")
|
||||||
|
|
||||||
|
def write_hostscript(self, subdomain: str, content: str):
|
||||||
|
"""
|
||||||
|
Write contents to hostscript.
|
||||||
|
Hostscript will contain the history of all actions that have been
|
||||||
|
ordered on a particular VM. So content needs to be appended to it,
|
||||||
|
rather than overwritten.
|
||||||
|
"""
|
||||||
|
hostscript = self._hostscript_path(subdomain)
|
||||||
|
with open(hostscript, "a", encoding="utf-8") as f:
|
||||||
|
f.write(content)
|
||||||
|
f.write("\n")
|
||||||
|
|
||||||
|
def _add_files(self, subdomain: str):
|
||||||
|
"""
|
||||||
|
Add all relevant files of a VM
|
||||||
|
"""
|
||||||
|
self.repo.git.add(str(self._host_vars_dir(subdomain=subdomain)))
|
||||||
|
self.repo.git.add(str(self._backup_path(subdomain=subdomain)))
|
||||||
|
self.repo.git.add(str(self._hostscript_path(subdomain=subdomain)))
|
||||||
|
|
||||||
|
def _commit(self, action: str, subdomain: str):
|
||||||
|
"""
|
||||||
|
Commit changes to a VM configuration
|
||||||
|
"""
|
||||||
|
|
||||||
|
self._add_files(subdomain=subdomain)
|
||||||
|
self.repo.git.commit(
|
||||||
|
message=f"{action} VM {subdomain}",
|
||||||
|
author="Dashboard Bot <bot@dashboard.hostea.org>",
|
||||||
|
)
|
||||||
|
|
||||||
|
def add_vm(self, instance: Instance) -> str:
|
||||||
|
"""
|
||||||
|
Add new VM to infrastructure repository
|
||||||
|
|
||||||
|
The gitea user password is returned
|
||||||
|
"""
|
||||||
|
self.repo.git.pull()
|
||||||
|
|
||||||
|
subdomain = instance.name
|
||||||
|
host_vars_dir = self._host_vars_dir(subdomain)
|
||||||
|
|
||||||
|
if not host_vars_dir.exists():
|
||||||
|
os.makedirs(host_vars_dir)
|
||||||
|
|
||||||
|
hostscript_path = self.repo_path.joinpath("inventory/hosts-scripts/")
|
||||||
|
if not hostscript_path.exists():
|
||||||
|
os.makedirs(hostscript_path)
|
||||||
|
|
||||||
|
woodpecker_agent_secret = get_random_string(64)
|
||||||
|
gitea_password = get_random_string(20)
|
||||||
|
|
||||||
|
ctx = {
|
||||||
|
"woodpecker_agent_secret": woodpecker_agent_secret,
|
||||||
|
"woodpecker_hostname": f"{subdomain}-ci",
|
||||||
|
"woodpecker_admins": f"{instance.owned_by.username}",
|
||||||
|
"gitea_email": instance.owned_by.email,
|
||||||
|
"gitea_password": gitea_password,
|
||||||
|
"subdomain": subdomain,
|
||||||
|
}
|
||||||
|
|
||||||
|
gitea = self._gitea_path(subdomain)
|
||||||
|
with open(gitea, "w", encoding="utf-8") as f:
|
||||||
|
f.write(
|
||||||
|
render_to_string(
|
||||||
|
"infrastructure/yml/gitea.yml",
|
||||||
|
context=ctx,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# provision_template = "./templates/infrastructure/yml/provision.yml"
|
||||||
|
provision = self._provision_path(subdomain)
|
||||||
|
# TODO: instance config names are different the flavours expected:
|
||||||
|
# ```
|
||||||
|
# openstack_flavor: ‘{{ openstack_flavor_medium }}’ * openstack_flavor: ‘{{ openstack_flavor_large }}’
|
||||||
|
# ```
|
||||||
|
# check with @dachary about this
|
||||||
|
|
||||||
|
size = None
|
||||||
|
if instance.configuration_id.name == "s1-2":
|
||||||
|
size = "openstack_flavor_small"
|
||||||
|
elif instance.configuration_id.name == "s1-4":
|
||||||
|
size = "openstack_flavor_medium"
|
||||||
|
elif instance.configuration_id.name == "s1-8":
|
||||||
|
size = "openstack_flavor_large"
|
||||||
|
else:
|
||||||
|
size = instance.configuration_id.name
|
||||||
|
|
||||||
|
with open(provision, "w", encoding="utf-8") as f:
|
||||||
|
f.write(
|
||||||
|
render_to_string(
|
||||||
|
"infrastructure/yml/provision.yml", context={"vm_size": size}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# backup = self.repo_path.joinpath(f"inventory/{instance.name}-backup.yml")
|
||||||
|
backup = self._backup_path(subdomain)
|
||||||
|
# backup_template = "./templates/infrastructure/yml/provision.yml"
|
||||||
|
with open(backup, "w", encoding="utf-8") as f:
|
||||||
|
f.write(
|
||||||
|
render_to_string(
|
||||||
|
"infrastructure/yml/backups.yml", context={"subdomain": subdomain}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# hostscript = self.repo_path.join("inventory/hosts-scripts/{instance.name}-host.sh")
|
||||||
|
|
||||||
|
self.write_hostscript(
|
||||||
|
subdomain=subdomain,
|
||||||
|
content=render_to_string(
|
||||||
|
"infrastructure/sh/hostscripts/create.sh",
|
||||||
|
context={"subdomain": subdomain},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
self._commit(action="add", subdomain=subdomain)
|
||||||
|
self.repo.git.push()
|
||||||
|
return gitea_password
|
||||||
|
|
||||||
|
def remove_vm(self, instance: Instance):
|
||||||
|
"""
|
||||||
|
Remove a VM from infrastructure repository
|
||||||
|
"""
|
||||||
|
self.repo.git.pull()
|
||||||
|
subdomain = instance.name
|
||||||
|
|
||||||
|
host_vars_dir = self._host_vars_dir(subdomain)
|
||||||
|
shutil.rmtree(host_vars_dir)
|
||||||
|
|
||||||
|
backup = self._backup_path(subdomain)
|
||||||
|
os.remove(backup)
|
||||||
|
|
||||||
|
self.write_hostscript(
|
||||||
|
subdomain=subdomain,
|
||||||
|
content=render_to_string(
|
||||||
|
"infrastructure/sh/hostscripts/rm.sh",
|
||||||
|
context={"subdomain": subdomain},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self._commit(action="rm", subdomain=subdomain)
|
||||||
|
self.repo.git.push()
|
|
@ -0,0 +1,73 @@
|
||||||
|
# Copyright © 2022 Aravinth Manivannan <realaravinth@batsense.net>
|
||||||
|
#
|
||||||
|
# This program is free software: you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU Affero General Public License as
|
||||||
|
# published by the Free Software Foundation, either version 3 of the
|
||||||
|
# License, or (at your option) any later version.
|
||||||
|
#
|
||||||
|
# This program is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU Affero General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
from django.shortcuts import render, redirect, get_object_or_404
|
||||||
|
from django.utils.http import urlencode
|
||||||
|
from django.contrib.auth import authenticate, login, logout
|
||||||
|
from django.contrib.auth import get_user_model
|
||||||
|
from django.contrib.auth.decorators import login_required
|
||||||
|
from django.http import HttpResponse
|
||||||
|
from django.views.decorators.csrf import csrf_protect
|
||||||
|
from django.urls import reverse
|
||||||
|
|
||||||
|
from accounts.decorators import confirm_access
|
||||||
|
from dash.models import Instance
|
||||||
|
from billing.utils import payment_fullfilled
|
||||||
|
|
||||||
|
from .utils import Infra
|
||||||
|
from .models import InstanceCreated
|
||||||
|
|
||||||
|
|
||||||
|
def default_ctx(title: str, username: str):
|
||||||
|
"""
|
||||||
|
Default context for all dashboard pages
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"title": title,
|
||||||
|
"username": username,
|
||||||
|
"open_instances": "open",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@login_required
|
||||||
|
def create_instance(request, instance_name: str):
|
||||||
|
"""
|
||||||
|
Dashboard homepage view
|
||||||
|
"""
|
||||||
|
instance = get_object_or_404(Instance, name=instance_name, owned_by=request.user)
|
||||||
|
if not payment_fullfilled(instance=instance):
|
||||||
|
return redirect(reverse("billing.invoice.generate", args=(instance_name,)))
|
||||||
|
|
||||||
|
infra = Infra()
|
||||||
|
if not InstanceCreated.objects.filter(instance=instance).exists():
|
||||||
|
instance = InstanceCreated.objects.create(instance=instance, created=True)
|
||||||
|
instance.save()
|
||||||
|
gitea_password = infra.add_vm(instance=instance)
|
||||||
|
instance.gitea_password = gitea_password
|
||||||
|
instance.save()
|
||||||
|
|
||||||
|
return HttpResponse()
|
||||||
|
|
||||||
|
|
||||||
|
@login_required
|
||||||
|
@confirm_access
|
||||||
|
def delete_instance(request, instance_name: str):
|
||||||
|
"""
|
||||||
|
Dashboard homepage view
|
||||||
|
"""
|
||||||
|
instance = get_object_or_404(Instance, name=instance_name, owned_by=request.user)
|
||||||
|
infra = Infra()
|
||||||
|
infra.remove_vm(instance=instance)
|
||||||
|
# TODO: push isn't implemented yet
|
||||||
|
return HttpResponse()
|
|
@ -14,6 +14,8 @@ django-oauth-toolkit==2.0.0
|
||||||
django-payments==1.0.0
|
django-payments==1.0.0
|
||||||
django-phonenumber-field==6.3.0
|
django-phonenumber-field==6.3.0
|
||||||
djangorestframework==3.13.1
|
djangorestframework==3.13.1
|
||||||
|
gitdb==4.0.9
|
||||||
|
GitPython==3.1.27
|
||||||
dachary
commented
Review
I tend to prefer using http://amoffat.github.io/sh/ like so👍
because (i) I know the git CLI and learning the python modules API has no appeal (ii) they struggle to catch up with what git does. My 2cts ;-) I tend to prefer using http://amoffat.github.io/sh/ like so👍
```
self.git = sh.git.bake(_cwd=self.git_directory.name)
self.git.init()
p = self.gitea.projects.get(self.user, self.project)
url = p.http_url_to_repo_with_auth
self.git.config('http.sslVerify', 'false')
self.git.remote.add.origin(url)
self.git.fetch()
```
because (i) I know the git CLI and learning the python modules API has no appeal (ii) they struggle to catch up with what git does.
My 2cts ;-)
realaravinth
commented
Review
Cool library, thanks for the suggestion. Works perfectly for our use case, I'll switch in a bit. Cool library, thanks for the suggestion. Works perfectly for our use case, I'll switch in a bit.
|
|||||||
greenlet==1.1.2
|
greenlet==1.1.2
|
||||||
idna==3.3
|
idna==3.3
|
||||||
isort==5.10.1
|
isort==5.10.1
|
||||||
|
@ -34,6 +36,7 @@ pylint==2.12.2
|
||||||
pynvim==0.4.3
|
pynvim==0.4.3
|
||||||
pytz==2022.1
|
pytz==2022.1
|
||||||
requests==2.27.1
|
requests==2.27.1
|
||||||
|
smmap==5.0.0
|
||||||
sqlparse==0.4.2
|
sqlparse==0.4.2
|
||||||
stripe==3.4.0
|
stripe==3.4.0
|
||||||
tblib==1.7.0
|
tblib==1.7.0
|
||||||
|
|
d.hostea.org is correct but it should be set to d.$HOSTEA_DOMAIN (when HOSTEA_DOMAIN should be made available at installation time in the settings file created by the playbook).