git config before push #43
8
Makefile
8
Makefile
|
@ -38,13 +38,7 @@ integration-test: ## run integration tests
|
||||||
. ./venv/bin/activate && integration/tests.sh
|
. ./venv/bin/activate && integration/tests.sh
|
||||||
|
|
||||||
lint: ## Run linter
|
lint: ## Run linter
|
||||||
@./venv/bin/black ./dashboard/
|
@./venv/bin/black dashboard accounts dash support billing infrastructure integration
|
||||||
@./venv/bin/black ./accounts/
|
|
||||||
@./venv/bin/black ./dash/
|
|
||||||
@./venv/bin/black ./support/
|
|
||||||
@./venv/bin/black ./billing/
|
|
||||||
@./venv/bin/black ./infrastructure/
|
|
||||||
@./venv/bin/black ./integration/
|
|
||||||
|
|
||||||
migrate: ## Run migrations
|
migrate: ## Run migrations
|
||||||
$(call run_migrations)
|
$(call run_migrations)
|
||||||
|
|
|
@ -14,7 +14,6 @@
|
||||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
from enum import Enum, unique
|
from enum import Enum, unique
|
||||||
|
|
||||||
from git import Repo
|
|
||||||
from django.contrib.auth.models import User
|
from django.contrib.auth.models import User
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
|
|
||||||
|
|
|
@ -12,19 +12,11 @@
|
||||||
#
|
#
|
||||||
# You should have received a copy of the GNU Affero General Public License
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
import shutil
|
|
||||||
import time
|
import time
|
||||||
from io import StringIO
|
|
||||||
from urllib.parse import urlunparse
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import requests
|
|
||||||
from django.test import TestCase, Client, override_settings
|
from django.test import TestCase, Client, override_settings
|
||||||
from django.conf import settings
|
|
||||||
from django.core.management import call_command
|
from django.core.management import call_command
|
||||||
|
|
||||||
from git import Repo
|
|
||||||
|
|
||||||
from dash.models import Instance, InstanceConfiguration
|
from dash.models import Instance, InstanceConfiguration
|
||||||
from accounts.tests import register_util, login_util
|
from accounts.tests import register_util, login_util
|
||||||
from dash.tests import create_configurations, create_instance_util, infra_custom_config
|
from dash.tests import create_configurations, create_instance_util, infra_custom_config
|
||||||
|
@ -77,59 +69,37 @@ class InfraUtilTest(TestCase):
|
||||||
|
|
||||||
@override_settings(HOSTEA=infra_custom_config(test_name="test_add_vm"))
|
@override_settings(HOSTEA=infra_custom_config(test_name="test_add_vm"))
|
||||||
def test_add_vm(self):
|
def test_add_vm(self):
|
||||||
infra = Infra()
|
|
||||||
c = Client()
|
c = Client()
|
||||||
conf = settings.HOSTEA["INFRA"]["HOSTEA_REPO"]
|
|
||||||
login_util(self, c, "accounts.home")
|
login_util(self, c, "accounts.home")
|
||||||
subdomain = "add_vm"
|
subdomain = "add_vm"
|
||||||
|
|
||||||
base = infra.repo_path
|
|
||||||
|
|
||||||
create_instance_util(
|
create_instance_util(
|
||||||
t=self, c=c, instance_name=subdomain, config=self.instance_config[0]
|
t=self, c=c, instance_name=subdomain, config=self.instance_config[0]
|
||||||
)
|
)
|
||||||
|
|
||||||
before_add = infra.repo.head.commit.hexsha
|
|
||||||
instance = Instance.objects.get(name=subdomain)
|
instance = Instance.objects.get(name=subdomain)
|
||||||
woodpecker_agent_secret = infra.add_vm(instance=instance)
|
|
||||||
after_add = infra.repo.head.commit.hexsha
|
|
||||||
self.assertEqual(before_add is not after_add, True)
|
|
||||||
|
|
||||||
# c = infra_custom_config(test_name="test_add_vm--get-head")
|
infra = Infra()
|
||||||
path = Path("/tmp/hostea/dashboard/check-test_add_vm")
|
before_add = infra._sha()
|
||||||
if path.exists():
|
(password, after_add) = infra.add_vm(instance=instance)
|
||||||
shutil.rmtree(path)
|
self.assertNotEqual(before_add, after_add)
|
||||||
repo = Repo.clone_from(conf["REMOTE"], path, env=infra.env)
|
|
||||||
repo.git.pull(env=infra.env)
|
|
||||||
self.assertEqual(repo.head.commit.hexsha == after_add, True)
|
|
||||||
|
|
||||||
before_rm = infra.repo.head.commit.hexsha
|
before_rm = after_add
|
||||||
infra.remove_vm(instance=instance)
|
after_rm = infra.remove_vm(instance=instance)
|
||||||
after_rm = infra.repo.head.commit.hexsha
|
self.assertNotEqual(before_rm, after_rm)
|
||||||
self.assertEqual(before_add is not after_add, True)
|
|
||||||
|
|
||||||
repo.git.pull(env=infra.env)
|
|
||||||
self.assertEqual(repo.head.commit.hexsha == after_rm, True)
|
|
||||||
|
|
||||||
@override_settings(HOSTEA=infra_custom_config(test_name="test_cmd"))
|
@override_settings(HOSTEA=infra_custom_config(test_name="test_cmd"))
|
||||||
def test_cmd(self):
|
def test_cmd(self):
|
||||||
subdomain = "cmd_vm"
|
subdomain = "cmd_vm"
|
||||||
infra = Infra()
|
infra = Infra()
|
||||||
c = Client()
|
c = Client()
|
||||||
conf = settings.HOSTEA["INFRA"]["HOSTEA_REPO"]
|
|
||||||
login_util(self, c, "accounts.home")
|
login_util(self, c, "accounts.home")
|
||||||
|
|
||||||
base = infra.repo_path
|
|
||||||
|
|
||||||
stdout = StringIO()
|
|
||||||
stderr = StringIO()
|
|
||||||
|
|
||||||
self.assertEqual(Instance.objects.filter(name=subdomain).exists(), False)
|
self.assertEqual(Instance.objects.filter(name=subdomain).exists(), False)
|
||||||
# username exists
|
# username exists
|
||||||
call_command(
|
call_command(
|
||||||
"vm", "create", subdomain, f"--owner={self.username}", "--flavor=medium"
|
"vm", "create", subdomain, f"--owner={self.username}", "--flavor=medium"
|
||||||
)
|
)
|
||||||
out = stdout.getvalue()
|
|
||||||
|
|
||||||
instance = Instance.objects.get(name=subdomain)
|
instance = Instance.objects.get(name=subdomain)
|
||||||
|
|
||||||
|
@ -152,7 +122,6 @@ class InfraUtilTest(TestCase):
|
||||||
|
|
||||||
# run create vm command again with different configuration but same name
|
# run create vm command again with different configuration but same name
|
||||||
# to crudely check idempotency
|
# to crudely check idempotency
|
||||||
old_size = instance.configuration_id
|
|
||||||
call_command(
|
call_command(
|
||||||
"vm", "create", subdomain, f"--owner={self.username}", "--flavor=large"
|
"vm", "create", subdomain, f"--owner={self.username}", "--flavor=large"
|
||||||
)
|
)
|
||||||
|
@ -172,7 +141,6 @@ class InfraUtilTest(TestCase):
|
||||||
)
|
)
|
||||||
|
|
||||||
call_command("vm", "delete", subdomain)
|
call_command("vm", "delete", subdomain)
|
||||||
out = stdout.getvalue()
|
|
||||||
|
|
||||||
self.assertEqual(Instance.objects.filter(name=subdomain).exists(), False)
|
self.assertEqual(Instance.objects.filter(name=subdomain).exists(), False)
|
||||||
host_vars_dir = infra._host_vars_dir(subdomain)
|
host_vars_dir = infra._host_vars_dir(subdomain)
|
||||||
|
|
|
@ -12,25 +12,28 @@
|
||||||
#
|
#
|
||||||
# You should have received a copy of the GNU Affero General Public License
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import sh
|
||||||
import shutil
|
import shutil
|
||||||
import yaml
|
import yaml
|
||||||
import requests
|
import requests
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from threading import Thread, Event
|
from threading import Thread
|
||||||
from time import sleep
|
from time import sleep
|
||||||
|
|
||||||
from django.utils.crypto import get_random_string
|
from django.utils.crypto import get_random_string
|
||||||
from django.template.loader import render_to_string
|
from django.template.loader import render_to_string
|
||||||
from django.core.mail import send_mail
|
from django.core.mail import send_mail
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from git import Repo, Commit
|
|
||||||
from git.exc import InvalidGitRepositoryError
|
|
||||||
|
|
||||||
from dash.models import Instance
|
from dash.models import Instance
|
||||||
|
|
||||||
from infrastructure.models import InstanceCreated, JobType, Job
|
from infrastructure.models import InstanceCreated, JobType, Job
|
||||||
|
|
||||||
|
logging.basicConfig()
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class Worker(Thread):
|
class Worker(Thread):
|
||||||
def __init__(self, job: Job):
|
def __init__(self, job: Job):
|
||||||
|
@ -75,7 +78,7 @@ class Worker(Thread):
|
||||||
job.delete()
|
job.delete()
|
||||||
|
|
||||||
|
|
||||||
def create_vm_if_not_exists(instance: Instance) -> (str, Commit):
|
def create_vm_if_not_exists(instance: Instance) -> (str, str):
|
||||||
"""
|
"""
|
||||||
Create VM utility. Gitea password is returned
|
Create VM utility. Gitea password is returned
|
||||||
"""
|
"""
|
||||||
|
@ -111,18 +114,17 @@ class Infra:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
conf = settings.HOSTEA["INFRA"]["HOSTEA_REPO"]
|
conf = settings.HOSTEA["INFRA"]["HOSTEA_REPO"]
|
||||||
self.repo_path = Path(conf["PATH"])
|
self.repo_path = Path(conf["PATH"])
|
||||||
if not self.repo_path.exists():
|
|
||||||
os.makedirs(self.repo_path)
|
|
||||||
|
|
||||||
ssh_cmd = f"/usr/bin/ssh -oStrictHostKeyChecking=no -i {conf['SSH_KEY']}"
|
|
||||||
self.env = {"GIT_SSH_COMMAND": ssh_cmd}
|
|
||||||
self._clone()
|
self._clone()
|
||||||
|
|
||||||
def _clone(self):
|
def _clone(self):
|
||||||
|
conf = settings.HOSTEA["INFRA"]["HOSTEA_REPO"]
|
||||||
|
ssh_cmd = f"/usr/bin/ssh -oStrictHostKeyChecking=no -i {conf['SSH_KEY']}"
|
||||||
|
self.git = sh.git.bake(_env={"GIT_SSH_COMMAND": ssh_cmd})
|
||||||
conf = settings.HOSTEA["INFRA"]["HOSTEA_REPO"]
|
conf = settings.HOSTEA["INFRA"]["HOSTEA_REPO"]
|
||||||
if os.path.exists(self.repo_path):
|
if os.path.exists(self.repo_path):
|
||||||
shutil.rmtree(self.repo_path)
|
shutil.rmtree(self.repo_path)
|
||||||
self.repo = Repo.clone_from(conf["REMOTE"], self.repo_path, env=self.env)
|
self.git.clone(conf["REMOTE"], self.repo_path)
|
||||||
|
self.git = self.git.bake("-C", self.repo_path)
|
||||||
|
|
||||||
def _host_vars_dir(self, subdomain: str) -> Path:
|
def _host_vars_dir(self, subdomain: str) -> Path:
|
||||||
"""
|
"""
|
||||||
|
@ -210,25 +212,21 @@ class Infra:
|
||||||
f.write(content)
|
f.write(content)
|
||||||
f.write("\n")
|
f.write("\n")
|
||||||
|
|
||||||
def _add_files(self, subdomain: str):
|
def _push(self, message):
|
||||||
"""
|
self.git.add(".")
|
||||||
Add all relevant files of a VM
|
self.git.config("user.email", settings.HOSTEA["INSTANCE_MAINTAINER_CONTACT"])
|
||||||
"""
|
self.git.config("user.name", "Hostea dashboard")
|
||||||
self.repo.git.add(str(self._host_vars_dir(subdomain=subdomain)))
|
try:
|
||||||
self.repo.git.add(str(self._backup_path(subdomain=subdomain)))
|
self.git.commit("-m", f"dashboard: {message}")
|
||||||
self.repo.git.add(str(self._service_path(subdomain=subdomain)))
|
except sh.ErrorReturnCode_1:
|
||||||
self.repo.git.add(str(self._hostscript_path(subdomain=subdomain)))
|
logger.debug("no change")
|
||||||
|
else:
|
||||||
|
self.git.push("origin", "master")
|
||||||
|
return self._sha()
|
||||||
|
|
||||||
def _commit(self, action: str, subdomain: str) -> Commit:
|
def _sha(self):
|
||||||
"""
|
sha = self.git("rev-parse", "origin/master")
|
||||||
Commit changes to a VM configuration
|
return str(sha).strip()
|
||||||
"""
|
|
||||||
|
|
||||||
self._add_files(subdomain=subdomain)
|
|
||||||
return self.repo.git.commit(
|
|
||||||
message=f"{action} VM {subdomain}",
|
|
||||||
author="Dashboard Bot <bot@dashboard.hostea.org>",
|
|
||||||
)
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def translate_size(instance: Instance) -> str:
|
def translate_size(instance: Instance) -> str:
|
||||||
|
@ -244,13 +242,12 @@ class Infra:
|
||||||
return "openstack_flavor_large"
|
return "openstack_flavor_large"
|
||||||
return instance.configuration_id.name
|
return instance.configuration_id.name
|
||||||
|
|
||||||
def add_vm(self, instance: Instance) -> (str, Commit):
|
def add_vm(self, instance: Instance) -> (str, str):
|
||||||
"""
|
"""
|
||||||
Add new VM to infrastructure repository
|
Add new VM to infrastructure repository
|
||||||
|
|
||||||
The gitea user password is returned
|
The gitea user password is returned
|
||||||
"""
|
"""
|
||||||
self._clone()
|
|
||||||
|
|
||||||
subdomain = instance.name
|
subdomain = instance.name
|
||||||
host_vars_dir = self._host_vars_dir(subdomain)
|
host_vars_dir = self._host_vars_dir(subdomain)
|
||||||
|
@ -321,29 +318,26 @@ class Infra:
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
commit = self._commit(action="add", subdomain=subdomain)
|
commit = self._push(f"add vm {subdomain}")
|
||||||
self.repo.git.push(env=self.env)
|
|
||||||
return (gitea_password, commit)
|
return (gitea_password, commit)
|
||||||
|
|
||||||
def remove_vm(self, instance: Instance):
|
def remove_vm(self, instance: Instance):
|
||||||
"""
|
"""
|
||||||
Remove a VM from infrastructure repository
|
Remove a VM from infrastructure repository
|
||||||
"""
|
"""
|
||||||
self._clone()
|
|
||||||
subdomain = instance.name
|
subdomain = instance.name
|
||||||
|
|
||||||
try:
|
host_vars_dir = self._host_vars_dir(subdomain)
|
||||||
|
if os.path.exists(host_vars_dir):
|
||||||
host_vars_dir = self._host_vars_dir(subdomain)
|
|
||||||
shutil.rmtree(host_vars_dir)
|
shutil.rmtree(host_vars_dir)
|
||||||
|
|
||||||
backup = self._backup_path(subdomain)
|
backup = self._backup_path(subdomain)
|
||||||
|
if os.path.exists(backup):
|
||||||
os.remove(backup)
|
os.remove(backup)
|
||||||
|
|
||||||
service = self._service_path(subdomain)
|
service = self._service_path(subdomain)
|
||||||
|
if os.path.exists(service):
|
||||||
os.remove(service)
|
os.remove(service)
|
||||||
except FileNotFoundError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
hostscript = self._hostscript_path(subdomain)
|
hostscript = self._hostscript_path(subdomain)
|
||||||
with open(hostscript, "w+", encoding="utf-8") as f:
|
with open(hostscript, "w+", encoding="utf-8") as f:
|
||||||
|
@ -356,5 +350,4 @@ class Infra:
|
||||||
context={"subdomain": subdomain},
|
context={"subdomain": subdomain},
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
self._commit(action="rm", subdomain=subdomain)
|
return self._push(f"rm vm {subdomain}")
|
||||||
self.repo.git.push(env=self.env)
|
|
||||||
|
|
|
@ -139,13 +139,13 @@ new_fleet_repo_init() {
|
||||||
tmp_dir=$(mktemp -d)
|
tmp_dir=$(mktemp -d)
|
||||||
pushd $tmp_dir
|
pushd $tmp_dir
|
||||||
echo "init" >> README
|
echo "init" >> README
|
||||||
|
git init
|
||||||
if is_ci
|
if is_ci
|
||||||
then
|
then
|
||||||
git config --global user.email "${CI_COMMIT_AUTHOR_EMAIL}"
|
git config user.email "${CI_COMMIT_AUTHOR_EMAIL}"
|
||||||
git config --global user.name "${CI_COMMIT_AUTHOR}"
|
git config user.name "${CI_COMMIT_AUTHOR}"
|
||||||
chmod 600 $GITEA_HOSTEA_FLEET_DEPLOY_KEY_PRIVATE
|
chmod 600 $GITEA_HOSTEA_FLEET_DEPLOY_KEY_PRIVATE
|
||||||
fi
|
fi
|
||||||
git init
|
|
||||||
git add README
|
git add README
|
||||||
git commit -m "init"
|
git commit -m "init"
|
||||||
REMOTE="$GITEA_SSH_URL/$GITEA_HOSTEA_USERNAME/$1.git"
|
REMOTE="$GITEA_SSH_URL/$GITEA_HOSTEA_USERNAME/$1.git"
|
||||||
|
|
|
@ -6,7 +6,7 @@ import sys
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
"""Run administrative tasks."""
|
"""Run administrative tasks."""
|
||||||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dashboard.settings')
|
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dashboard.settings")
|
||||||
try:
|
try:
|
||||||
from django.core.management import execute_from_command_line
|
from django.core.management import execute_from_command_line
|
||||||
except ImportError as exc:
|
except ImportError as exc:
|
||||||
|
@ -18,5 +18,5 @@ def main():
|
||||||
execute_from_command_line(sys.argv)
|
execute_from_command_line(sys.argv)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|
|
@ -15,8 +15,6 @@ django-oauth-toolkit==2.0.0
|
||||||
django-payments==1.0.0
|
django-payments==1.0.0
|
||||||
django-phonenumber-field==6.3.0
|
django-phonenumber-field==6.3.0
|
||||||
djangorestframework==3.13.1
|
djangorestframework==3.13.1
|
||||||
gitdb==4.0.9
|
|
||||||
GitPython==3.1.27
|
|
||||||
greenlet==1.1.2
|
greenlet==1.1.2
|
||||||
idna==3.3
|
idna==3.3
|
||||||
install==1.3.5
|
install==1.3.5
|
||||||
|
@ -40,6 +38,7 @@ pytz==2022.1
|
||||||
PyYAML==6.0
|
PyYAML==6.0
|
||||||
requests==2.27.1
|
requests==2.27.1
|
||||||
six==1.16.0
|
six==1.16.0
|
||||||
|
sh==1.14.2
|
||||||
smmap==5.0.0
|
smmap==5.0.0
|
||||||
sqlparse==0.4.2
|
sqlparse==0.4.2
|
||||||
stripe==3.4.0
|
stripe==3.4.0
|
||||||
|
|
Loading…
Reference in New Issue