Hostea
/
dashboard
2
2
Fork 1
Code Issues 11 Pull Requests Projects Releases Wiki Activity

git config before push #43

Merged
realaravinth merged 1 commits from dachary/dashboard:wip-config-3 into master 2022-07-08 14:18:00 +00:00
Conversation 1 Commits 1 Files Changed 7
7 changed files with 49 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Makefile
View File

@ -38,13 +38,7 @@ integration-test: ## run integration tests
. ./venv/bin/activate && integration/tests.sh
lint: ## Run linter
@./venv/bin/black ./dashboard/
@./venv/bin/black ./accounts/
@./venv/bin/black ./dash/
@./venv/bin/black ./support/
@./venv/bin/black ./billing/
@./venv/bin/black ./infrastructure/
@./venv/bin/black ./integration/
@./venv/bin/black dashboard accounts dash support billing infrastructure integration
migrate: ## Run migrations
$(call run_migrations)

1
dash/utils.py
View File

@ -14,7 +14,6 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from enum import Enum, unique
from git import Repo
from django.contrib.auth.models import User
from django.conf import settings

46
infrastructure/tests.py
View File

@ -12,19 +12,11 @@
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import shutil
import time
from io import StringIO
from urllib.parse import urlunparse
from pathlib import Path
import requests
from django.test import TestCase, Client, override_settings
from django.conf import settings
from django.core.management import call_command
from git import Repo
from dash.models import Instance, InstanceConfiguration
from accounts.tests import register_util, login_util
from dash.tests import create_configurations, create_instance_util, infra_custom_config
@ -77,59 +69,37 @@ class InfraUtilTest(TestCase):
@override_settings(HOSTEA=infra_custom_config(test_name="test_add_vm"))
def test_add_vm(self):
infra = Infra()
c = Client()
conf = settings.HOSTEA["INFRA"]["HOSTEA_REPO"]
login_util(self, c, "accounts.home")
subdomain = "add_vm"
base = infra.repo_path
create_instance_util(
t=self, c=c, instance_name=subdomain, config=self.instance_config[0]
)
before_add = infra.repo.head.commit.hexsha
instance = Instance.objects.get(name=subdomain)
woodpecker_agent_secret = infra.add_vm(instance=instance)
after_add = infra.repo.head.commit.hexsha
self.assertEqual(before_add is not after_add, True)
# c = infra_custom_config(test_name="test_add_vm--get-head")
path = Path("/tmp/hostea/dashboard/check-test_add_vm")
if path.exists():
shutil.rmtree(path)
repo = Repo.clone_from(conf["REMOTE"], path, env=infra.env)
repo.git.pull(env=infra.env)
self.assertEqual(repo.head.commit.hexsha == after_add, True)
infra = Infra()
before_add = infra._sha()
(password, after_add) = infra.add_vm(instance=instance)
self.assertNotEqual(before_add, after_add)
before_rm = infra.repo.head.commit.hexsha
infra.remove_vm(instance=instance)
after_rm = infra.repo.head.commit.hexsha
self.assertEqual(before_add is not after_add, True)
repo.git.pull(env=infra.env)
self.assertEqual(repo.head.commit.hexsha == after_rm, True)
before_rm = after_add
after_rm = infra.remove_vm(instance=instance)
self.assertNotEqual(before_rm, after_rm)
@override_settings(HOSTEA=infra_custom_config(test_name="test_cmd"))
def test_cmd(self):
subdomain = "cmd_vm"
infra = Infra()
c = Client()
conf = settings.HOSTEA["INFRA"]["HOSTEA_REPO"]
login_util(self, c, "accounts.home")
base = infra.repo_path
stdout = StringIO()
stderr = StringIO()
self.assertEqual(Instance.objects.filter(name=subdomain).exists(), False)
# username exists
call_command(
"vm", "create", subdomain, f"--owner={self.username}", "--flavor=medium"
)
out = stdout.getvalue()
instance = Instance.objects.get(name=subdomain)
@ -152,7 +122,6 @@ class InfraUtilTest(TestCase):
# run create vm command again with different configuration but same name
# to crudely check idempotency
old_size = instance.configuration_id
call_command(
"vm", "create", subdomain, f"--owner={self.username}", "--flavor=large"
)
@ -172,7 +141,6 @@ class InfraUtilTest(TestCase):
)
call_command("vm", "delete", subdomain)
out = stdout.getvalue()
self.assertEqual(Instance.objects.filter(name=subdomain).exists(), False)
host_vars_dir = infra._host_vars_dir(subdomain)

77
infrastructure/utils.py
View File

@ -12,25 +12,28 @@
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import logging
import os
import sh
import shutil
import yaml
import requests
from pathlib import Path
from threading import Thread, Event
from threading import Thread
from time import sleep
from django.utils.crypto import get_random_string
from django.template.loader import render_to_string
from django.core.mail import send_mail
from django.conf import settings
from git import Repo, Commit
from git.exc import InvalidGitRepositoryError
from dash.models import Instance
from infrastructure.models import InstanceCreated, JobType, Job
logging.basicConfig()
logger = logging.getLogger(__name__)
class Worker(Thread):
def __init__(self, job: Job):
@ -75,7 +78,7 @@ class Worker(Thread):
job.delete()
def create_vm_if_not_exists(instance: Instance) -> (str, Commit):
def create_vm_if_not_exists(instance: Instance) -> (str, str):
"""
Create VM utility. Gitea password is returned
"""
@ -111,18 +114,17 @@ class Infra:
def __init__(self):
conf = settings.HOSTEA["INFRA"]["HOSTEA_REPO"]
self.repo_path = Path(conf["PATH"])
if not self.repo_path.exists():
os.makedirs(self.repo_path)
ssh_cmd = f"/usr/bin/ssh -oStrictHostKeyChecking=no -i {conf['SSH_KEY']}"
self.env = {"GIT_SSH_COMMAND": ssh_cmd}
self._clone()
def _clone(self):
conf = settings.HOSTEA["INFRA"]["HOSTEA_REPO"]
ssh_cmd = f"/usr/bin/ssh -oStrictHostKeyChecking=no -i {conf['SSH_KEY']}"
self.git = sh.git.bake(_env={"GIT_SSH_COMMAND": ssh_cmd})
conf = settings.HOSTEA["INFRA"]["HOSTEA_REPO"]
if os.path.exists(self.repo_path):
shutil.rmtree(self.repo_path)
self.repo = Repo.clone_from(conf["REMOTE"], self.repo_path, env=self.env)
self.git.clone(conf["REMOTE"], self.repo_path)
self.git = self.git.bake("-C", self.repo_path)
def _host_vars_dir(self, subdomain: str) -> Path:
"""
@ -210,25 +212,21 @@ class Infra:
f.write(content)
f.write("\n")
def _add_files(self, subdomain: str):
"""
Add all relevant files of a VM
"""
self.repo.git.add(str(self._host_vars_dir(subdomain=subdomain)))
self.repo.git.add(str(self._backup_path(subdomain=subdomain)))
self.repo.git.add(str(self._service_path(subdomain=subdomain)))
self.repo.git.add(str(self._hostscript_path(subdomain=subdomain)))
def _push(self, message):
self.git.add(".")
self.git.config("user.email", settings.HOSTEA["INSTANCE_MAINTAINER_CONTACT"])
self.git.config("user.name", "Hostea dashboard")
try:
self.git.commit("-m", f"dashboard: {message}")
except sh.ErrorReturnCode_1:
logger.debug("no change")
else:
self.git.push("origin", "master")
return self._sha()
def _commit(self, action: str, subdomain: str) -> Commit:
"""
Commit changes to a VM configuration
"""
self._add_files(subdomain=subdomain)
return self.repo.git.commit(
message=f"{action} VM {subdomain}",
author="Dashboard Bot <bot@dashboard.hostea.org>",
)
def _sha(self):
sha = self.git("rev-parse", "origin/master")
return str(sha).strip()
@staticmethod
def translate_size(instance: Instance) -> str:
@ -244,13 +242,12 @@ class Infra:
return "openstack_flavor_large"
return instance.configuration_id.name
def add_vm(self, instance: Instance) -> (str, Commit):
def add_vm(self, instance: Instance) -> (str, str):
"""
Add new VM to infrastructure repository
The gitea user password is returned
"""
self._clone()
subdomain = instance.name
host_vars_dir = self._host_vars_dir(subdomain)
@ -321,29 +318,26 @@ class Infra:
),
)
commit = self._commit(action="add", subdomain=subdomain)
self.repo.git.push(env=self.env)
commit = self._push(f"add vm {subdomain}")
return (gitea_password, commit)
def remove_vm(self, instance: Instance):
"""
Remove a VM from infrastructure repository
"""
self._clone()
subdomain = instance.name
try:
host_vars_dir = self._host_vars_dir(subdomain)
host_vars_dir = self._host_vars_dir(subdomain)
if os.path.exists(host_vars_dir):
shutil.rmtree(host_vars_dir)
backup = self._backup_path(subdomain)
backup = self._backup_path(subdomain)
if os.path.exists(backup):
os.remove(backup)
service = self._service_path(subdomain)
service = self._service_path(subdomain)
if os.path.exists(service):
os.remove(service)
except FileNotFoundError:
pass
hostscript = self._hostscript_path(subdomain)
with open(hostscript, "w+", encoding="utf-8") as f:
@ -356,5 +350,4 @@ class Infra:
context={"subdomain": subdomain},
),
)
self._commit(action="rm", subdomain=subdomain)
self.repo.git.push(env=self.env)
return self._push(f"rm vm {subdomain}")

6
integration/lib.sh
View File

@ -139,13 +139,13 @@ new_fleet_repo_init() {
tmp_dir=$(mktemp -d)
pushd $tmp_dir
echo "init" >> README
git init
if is_ci
then
git config --global user.email "${CI_COMMIT_AUTHOR_EMAIL}"
git config --global user.name "${CI_COMMIT_AUTHOR}"
git config user.email "${CI_COMMIT_AUTHOR_EMAIL}"
git config user.name "${CI_COMMIT_AUTHOR}"
chmod 600 $GITEA_HOSTEA_FLEET_DEPLOY_KEY_PRIVATE
fi
git init
git add README
git commit -m "init"
REMOTE="$GITEA_SSH_URL/$GITEA_HOSTEA_USERNAME/$1.git"

4
manage.py
View File

@ -6,7 +6,7 @@ import sys
def main():
"""Run administrative tasks."""
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dashboard.settings')
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dashboard.settings")
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
@ -18,5 +18,5 @@ def main():
execute_from_command_line(sys.argv)
if __name__ == '__main__':
if __name__ == "__main__":
main()

3
requirements.txt
View File

@ -15,8 +15,6 @@ django-oauth-toolkit==2.0.0
django-payments==1.0.0
django-phonenumber-field==6.3.0
djangorestframework==3.13.1
gitdb==4.0.9
GitPython==3.1.27
greenlet==1.1.2
idna==3.3
install==1.3.5
@ -40,6 +38,7 @@ pytz==2022.1
PyYAML==6.0
requests==2.27.1
six==1.16.0
sh==1.14.2
smmap==5.0.0
sqlparse==0.4.2
stripe==3.4.0