Initial revision

master
madmaurice 2 years ago
parent ed69701311
commit 8414256088
  1. 3
      .gitignore
  2. 0
      giteamigrate/__init__.py
  3. 16
      giteamigrate/asgi.py
  4. 115
      giteamigrate/settings.dist.py
  5. 22
      giteamigrate/urls.py
  6. 16
      giteamigrate/wsgi.py
  7. 21
      manage.py
  8. 0
      migrator/__init__.py
  9. 3
      migrator/admin.py
  10. 33
      migrator/api.py
  11. 5
      migrator/apps.py
  12. 11
      migrator/helper.py
  13. 34
      migrator/migrations/0001_initial.py
  14. 19
      migrator/migrations/0002_migration_username.py
  15. 25
      migrator/migrations/0003_auto_20200803_1947.py
  16. 18
      migrator/migrations/0004_auto_20200803_1948.py
  17. 0
      migrator/migrations/__init__.py
  18. 35
      migrator/models.py
  19. 7
      migrator/settings.py
  20. 135
      migrator/tasks.py
  21. 15
      migrator/templates/base.html
  22. 122
      migrator/templates/migration_create.html
  23. 59
      migrator/templates/migration_progress.html
  24. 3
      migrator/tests.py
  25. 12
      migrator/urls.py
  26. 117
      migrator/views.py
  27. 7
      static/bootstrap.min.css
  28. 2
      static/jquery-3.5.1.min.js

3
.gitignore vendored

@ -0,0 +1,3 @@
__pycache__
/db.sqlite3
/giteamigrate/settings.py

@ -0,0 +1,16 @@
"""
ASGI config for giteamigrate project.
It exposes the ASGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/
"""
import os
from django.core.asgi import get_asgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'giteamigrate.settings')
application = get_asgi_application()

@ -0,0 +1,115 @@
import os
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
GITLAB_API = 'https://git.zom.bi/api/v4'
GITEA_API = 'https://gitea.zom.bi/api/v1'
GITLAB_REPO_URL = 'https://%s:%s@git.zom.bi/%s.git'
GITEA_REPO_URL = 'https://%s:%s@gitea.zom.bi/%s.git'
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '' # Fill with key
DEBUG = False
ALLOWED_HOSTS = [
'*'
]
# Application definition
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'migrator',
'workers',
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
ROOT_URLCONF = 'giteamigrate.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
WSGI_APPLICATION = 'giteamigrate.wsgi.application'
# Database
# https://docs.djangoproject.com/en/3.0/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
}
}
# Password validation
# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
# Internationalization
# https://docs.djangoproject.com/en/3.0/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/3.0/howto/static-files/
STATIC_URL = '/static/'
STATICFILES_DIRS = [
os.path.join(BASE_DIR, "static")
]

@ -0,0 +1,22 @@
"""giteamigrate URL Configuration
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/3.0/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: path('', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
Including another URLconf
1. Import the include() function: from django.urls import include, path
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path, include
urlpatterns = [
path('', include('migrator.urls')),
path('admin/', admin.site.urls),
]

@ -0,0 +1,16 @@
"""
WSGI config for giteamigrate project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/
"""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'giteamigrate.settings')
application = get_wsgi_application()

@ -0,0 +1,21 @@
#!/usr/bin/env python
"""Django's command-line utility for administrative tasks."""
import os
import sys
def main():
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'giteamigrate.settings')
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?"
) from exc
execute_from_command_line(sys.argv)
if __name__ == '__main__':
main()

@ -0,0 +1,3 @@
from django.contrib import admin
# Register your models here.

@ -0,0 +1,33 @@
import requests
from . import settings
class API:
def __init__(self, prefix, params={}):
self.prefix=prefix
self.default_params=params
def get(self, url, *args, **kwargs):
params = kwargs.get('params', {})
for k,v in self.default_params.items():
if k not in params:
params[k] = v
kwargs['params'] = params
return requests.get(self.prefix + url, *args, **kwargs)
def post(self, url, *args, **kwargs):
params = kwargs.get('params', {})
for k,v in self.default_params.items():
if k not in params:
params[k] = v
kwargs['params'] = params
return requests.post(self.prefix + url, *args, **kwargs)
class GiteaAPI(API):
def __init__(self, token):
super().__init__(settings.GITEA_API, {'token': token})
class GitlabAPI(API):
def __init__(self, token):
super().__init__(settings.GITLAB_API, {'private_token': token})

@ -0,0 +1,5 @@
from django.apps import AppConfig
class MigratorConfig(AppConfig):
name = 'migrator'

@ -0,0 +1,11 @@
import re
def get_dict_from_query(query, name):
d = {}
matcher = re.compile("^%s\[(.+)\]$" % name)
for key, value in query.items():
match = matcher.match(key)
if match is not None:
d[match.group(1)] = value
return d

@ -0,0 +1,34 @@
# Generated by Django 3.0.8 on 2020-08-01 15:33
from django.db import migrations, models
import django.db.models.deletion
import migrator.models
import uuid
class Migration(migrations.Migration):
initial = True
dependencies = [
]
operations = [
migrations.CreateModel(
name='Migration',
fields=[
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('gitlab_token', models.CharField(max_length=200)),
('gitea_token', models.CharField(max_length=200)),
],
),
migrations.CreateModel(
name='Repository',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('path_with_namespace', models.CharField(max_length=200)),
('result', models.PositiveIntegerField(choices=[(migrator.models.Status['PENDING'], 1), (migrator.models.Status['SUCCESS'], 2), (migrator.models.Status['EXISTS'], 3), (migrator.models.Status['ERROR'], 4)], default=migrator.models.Status['PENDING'])),
('migration', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='repositories', to='migrator.Migration')),
],
),
]

@ -0,0 +1,19 @@
# Generated by Django 3.0.8 on 2020-08-01 15:54
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('migrator', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='migration',
name='username',
field=models.CharField(default='anonymous', max_length=200),
preserve_default=False,
),
]

@ -0,0 +1,25 @@
# Generated by Django 3.0.8 on 2020-08-03 19:47
from django.db import migrations, models
import migrator.models
class Migration(migrations.Migration):
dependencies = [
('migrator', '0002_migration_username'),
]
operations = [
migrations.AddField(
model_name='repository',
name='error_message',
field=models.TextField(default=''),
preserve_default=False,
),
migrations.AlterField(
model_name='repository',
name='result',
field=models.PositiveIntegerField(choices=[(migrator.models.Status['PENDING'], 1), (migrator.models.Status['IN_PROGRESS'], 5), (migrator.models.Status['SUCCESS'], 2), (migrator.models.Status['EXISTS'], 3), (migrator.models.Status['ERROR'], 4)], default=migrator.models.Status['PENDING']),
),
]

@ -0,0 +1,18 @@
# Generated by Django 3.0.8 on 2020-08-03 19:48
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('migrator', '0003_auto_20200803_1947'),
]
operations = [
migrations.AlterField(
model_name='repository',
name='error_message',
field=models.TextField(default=''),
),
]

@ -0,0 +1,35 @@
from django.db import models
from enum import IntEnum
import uuid
class Migration(models.Model):
id = models.UUIDField(
primary_key = True,
default = uuid.uuid4,
editable = False
)
username = models.CharField(max_length=200)
gitlab_token = models.CharField(max_length=200)
gitea_token = models.CharField(max_length=200)
class Status(IntEnum):
PENDING = 1
IN_PROGRESS = 5
SUCCESS = 2
EXISTS = 3
ERROR = 4
class Repository(models.Model):
migration = models.ForeignKey(
Migration,
on_delete = models.CASCADE,
related_name = "repositories",
)
path_with_namespace = models.CharField(max_length=200)
result = models.PositiveIntegerField(
choices=[(tag, tag.value) for tag in Status],
default = Status.PENDING
)
error_message = models.TextField(default="")

@ -0,0 +1,7 @@
from django.conf import settings
GITLAB_API = getattr(settings, 'GITLAB_API', 'https://localhost/api/v4')
GITEA_API = getattr(settings, 'GITEA_API', 'https://localhost/api/v1')
GITLAB_REPO_URL = getattr(settings, 'GITLAB_REPO_URL', 'https://%s:%s@localhost/%s.git')
GITEA_REPO_URL = getattr(settings, 'GITEA_REPO_URL', 'https://%s:%s@localhost/%s.git')

@ -0,0 +1,135 @@
from workers import task
from .models import *
from time import sleep
from random import randint
from tempfile import TemporaryDirectory
from . import settings
from .api import *
from os import path
import re
import subprocess
import traceback
class MigrationError(Exception):
pass
def handle_migration(repository):
gitea = GiteaAPI(repository.migration.gitea_token)
gitlab = GitlabAPI(repository.migration.gitlab_token)
match = re.search('^([^/]+)/',repository.path_with_namespace)
if match is None:
raise MigrationError("Could not get organization")
else:
organization = match.group(1)
match = re.search('([^/]+)$', repository.path_with_namespace)
if match is None:
raise MigrationError("Could not get repo name")
else:
reponame = match.group(1)
# Create organization if necessary
if organization == repository.migration.username:
organization = None
if organization is None:
response = gitea.get("/orgs/%s" % organization)
if response.status_code != 200:
response = gitea.post("/orgs", json={
"full_name" : organization,
"username" : organization,
})
if response.status_code != 201:
raise MigrationError("Could not create organization: %s" % (response.text) )
owner = organization or repository.migration.username
# Create repo if necessary
if organization is None:
response = gitea.post('/user/repos', data={
'auto_init': False,
'default_branch': 'master',
'description': "Automatically created",
'name': reponame,
})
else:
response = gitea.post('/org/%s/repos' % organization, data={
'name': reponame,
'private': True,
})
if response.status_code == 409:
pass # exists
elif response.status_code != 201:
raise MigrationError("Could not create repo: %s" % (response.text))
gitlab_remote = settings.GITLAB_REPO_URL % (
repository.migration.username,
repository.migration.gitlab_token,
repository.path_with_namespace
)
gitea_remote = settings.GITEA_REPO_URL % (
repository.migration.username,
repository.migration.gitea_token,
repository.path_with_namespace
)
# Migrate
with TemporaryDirectory() as tempdir:
gitdir = path.join(tempdir,'git')
try:
subprocess.check_call([
'git','clone','--mirror',
gitlab_remote,
gitdir
])
except subprocess.CalledProcessError:
raise MigrationError("Could not clone")
try:
subprocess.check_call([
'git', '-C', gitdir,
'remote','add',
'gitea', gitea_remote
])
except subprocess.CalledProcessError:
raise MigrationError("Could not set remote")
try:
subprocess.check_call([
'git', '-C', gitdir,
'push', '--all', 'gitea'
])
except subprocess.CalledProcessError:
raise MigrationError("Could not push")
return Status.SUCCESS
@task()
def migrate_repository(repository_id):
try:
repository = Repository.objects.select_related('migration').get(pk=repository_id)
except Repository.DoesNotExist:
return
print(">>> Migrating repository %s..." % (repository.path_with_namespace), end="\n\n")
repository.result = Status.IN_PROGRESS
repository.save()
# Might take a while
try:
result = handle_migration(repository)
repository.result = result
except Exception as e:
result = Status.ERROR
repository.result = result
repository.error_message = traceback.format_exc()
finally:
repository.save()
print(end="\n\n")

@ -0,0 +1,15 @@
{% load static %}
<html>
<head>
<title>Gitlab to Gitea migrator</title>
<link rel="stylesheet" href="{% static 'bootstrap.min.css' %}">
</head>
<body>
<div class="container mt-4">
{% block content %}
{% endblock %}
</div>
<script src="{% static 'jquery-3.5.1.min.js' %}"></script>
{% block javascript %}{% endblock %}
</body>
</html>

@ -0,0 +1,122 @@
{% extends 'base.html' %}
{% block content %}
<form method="post">
{% csrf_token %}
<div class="form-group row">
<label for="gitea_token" class="col-form-label col-md-auto">Gitea Access Token:</label>
<div class="col">
<input name="gitea_token"
id="gitea_token"
class="form-control">
<small class="text-muted">
Can be created
<a href="https://gitea.zom.bi/user/settings/applications">here</a>
</small>
</div>
<div class="col-md-auto">
<button type="button" class="btn btn-primary" id="buttonCheckGitea">
Check Gitea Access
</button>
</div>
</div>
<div class="form-group row">
<label for="gitlab_token" class="col-form-label col-md-auto">Gitlab Access Token:</label>
<div class="col">
<input name="gitlab_token"
id="gitlab_token"
class="form-control">
<small class="text-muted">
Can be created
<a href="https://git.zom.bi/profile/personal_access_tokens">here</a> (with api scope)
</small>
</div>
</div>
<div class="form-group" id="projectfetch">
<button type="button" class="btn btn-primary" id="buttonFetch">
Fetch projects from GitLab
</button>
</div>
<div class="form-group d-none" id="projectselect">
<h4>Select projects to migrate</h4>
<div class="d-flex flex-row">
<div class="btn-group btn-group-sm mr-auto">
<button type="button" class="btn btn-outline-secondary"
id="buttonSelectAll">
Select all
</button>
<button type="button" class="btn btn-outline-secondary"
id="buttonSelectNone">
Select none
</button>
</div>
<div>
<button type="submit" class="btn btn-primary">
Start migration
</button>
</div>
</div>
<ul id="projectslist" class="list-group mt-4">
</ul>
<input type="hidden" id="usernamefield" name="username" value="">
</div>
</div>
</form>
{% endblock %}
{% block javascript %}
<script type="text/javascript">
$(document).ready(function() {
$('#buttonFetch').click(function() {
$('#projectselect').addClass('d-none');
$('#gitlab_token').prop('readonly', true);
$.getJSON("{% url 'fetch_gitlab_projects' %}?gitlab_token="+$('#gitlab_token').val(), function(data) {
if(data.error) {
alert(data.error);
$('#gitlab_token').prop('readonly', false);
return;
}
var items = [];
data.projects.forEach(function(project) {
items.push(
"<li class='list-group-item'>" +
"<input name='projects[" + project + "]' type=checkbox>"+
"<span class='pl-2'>" + project + "</span>"+
"</li>"
);
});
$('#usernamefield').val(data.username);
$('#projectslist').html(items.join(""));
$('#projectselect').removeClass('d-none');
$('#projectfetch').addClass('d-none');
$('#buttonFetch').off('click');
}).fail(function() {
$('#gitlab_token').prop('readonly', false);
});
});
$('#buttonCheckGitea').click(function() {
$.getJSON("{% url 'check_gitea_access' %}?gitea_token="+$('#gitea_token').val(), function(result) {
if(result.success) {
alert("Gitea access works");
} else {
alert("Gitea access failed");
}
});
});
$('#buttonSelectAll').click(function() {
$('#projectslist input[type=checkbox]').prop('checked',true);
});
$('#buttonSelectNone').click(function() {
$('#projectslist input[type=checkbox]').prop('checked',false);
});
});
</script>
{% endblock %}

@ -0,0 +1,59 @@
{% extends 'base.html' %}
{% block content %}
<h1>Migration progress for {{ migration.username }}</h1>
<div class="mt-4">
<ul class="list-group">
{% for repository in migration.repositories.all %}
<li id="revision-{{ repository.id }}" class="list-group-item">
<div class="d-flex flex-row">
<div class="d-inline-block mr-auto">
{{ repository.path_with_namespace }}
</div>
<div class="d-inline-block">
<span class="status status-{{ repository.result }}">
</span>
</div>
</div>
<pre style="height:140px" class="error_msg form-control is-invalid mt-2 mb-0{% if repository.result != 4 %} d-none{% endif %}">{{ repository.error_message }}</pre>
</li>
{% endfor %}
</ul>
</div>
{% endblock %}
{% block javascript %}
<style>
.status-1::after { content: "pending"; color: #998800; }
.status-2::after { content: "success"; color: #339933; font-weight: bold; }
.status-3::after { content: "exists"; color: #666666; }
.status-4::after { content: "error"; color: #992222; }
.status-5::after { content: "in progress"; color: #998800; }
</style>
<script type="text/javascript">
$(document).ready(function() {
var PENDING = 1;
var intervalId;
function updateStatus() {
$.getJSON("{% url 'migration_status' migration.id %}", function(status) {
var alldone = true;
status.forEach(function (item) {
if(item.status == PENDING) { alldone = false; }
$('#revision-'+item.id+' span.status').attr('class','status status-'+item.status);
if(typeof(item.message) !== "undefined") {
$('#revision-'+item.id+' .error_msg').text(item.message).removeClass('d-none');
}
});
if(alldone) {
clearInterval(intervalId);
}
});
}
intervalId = setInterval(updateStatus, 10000);
});
</script>
{% endblock %}

@ -0,0 +1,3 @@
from django.test import TestCase
# Create your tests here.

@ -0,0 +1,12 @@
from django.urls import path, include
from . import views
urlpatterns = [
path('api/', include([
path('gitlab/projects', views.fetch_gitlab_projects, name="fetch_gitlab_projects"),
path('gitea/access', views.check_gitea_access, name="check_gitea_access"),
path('migration/status/<str:migration_key>', views.migration_status, name="migration_status"),
])),
path('', views.migration_create, name="migration_create"),
path('progress/<str:migration_key>', views.migration_progess, name="migration_progress"),
]

@ -0,0 +1,117 @@
from django.shortcuts import render, redirect
from django.http import JsonResponse, HttpResponse
from . import settings
from .helper import get_dict_from_query
from .models import *
from . import tasks
from .api import *
def migration_create(request):
if request.method == "POST":
projects = get_dict_from_query(request.POST,"projects")
migration = Migration(
gitlab_token = request.POST["gitlab_token"],
gitea_token = request.POST["gitea_token"],
username = request.POST["username"],
)
migration.save()
for project in projects:
repo = Repository(
migration = migration,
path_with_namespace = project
)
repo.save()
tasks.migrate_repository(repo.id)
return redirect("migration_progress", migration.id)
return render(request, "migration_create.html")
def migration_progess(request, migration_key):
try:
migration = Migration.objects.prefetch_related('repositories').get(pk=migration_key)
except Migration.DoesNotExist:
return redirect("migration_create")
return render(request, "migration_progress.html", {
'migration': migration,
})
def migration_status(request, migration_key):
repositories = Repository.objects.filter(
migration_id = migration_key,
).all()
data = []
for repo in repositories:
info = { 'id': repo.id, 'status': repo.result }
if repo.result == Status.ERROR:
info['message'] = repo.error_message
data.append(info)
return JsonResponse(data,safe=False)
def fetch_gitlab_projects(request):
gitlab_token = request.GET.get("gitlab_token", None)
if gitlab_token is None:
return JsonResponse({ 'error' : 'Missing token' })
gitlabapi = GitlabAPI(gitlab_token)
response = gitlabapi.get('/user')
if response.status_code != 200:
return JsonResponse({ 'error' : 'API error' })
data = response.json()
try:
username = data['username']
except KeyError:
return JsonResponse({ 'error' : 'API error' })
projects = []
page = 1
while True:
response = gitlabapi.get('/projects', params={
'owned': 1,
'page': page,
'per_page': 100,
})
if response.status_code != 200:
return JsonResponse({ 'error' : 'API error' })
data = response.json()
if len(data) == 0:
break
for gitlab_project in data:
try:
projects.append(gitlab_project['path_with_namespace'])
except KeyError:
return JsonResponse({ 'error' : 'API error' })
page+=1
return JsonResponse({
'username' : username,
'projects' : sorted(projects)
})
def check_gitea_access(request):
gitea_token = request.GET.get("gitea_token", None)
if gitea_token is None:
return JsonResponse({ 'success': False })
response = GiteaAPI(gitea_token).get('/user')
if response.status_code == 200:
return JsonResponse({ 'success': True })
else:
return JsonResponse({ 'success': False })

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long
Loading…
Cancel
Save