initial commit
This commit is contained in:
commit
4a98f2316b
|
|
@ -0,0 +1,57 @@
|
|||
from mirror import SSHAgent
|
||||
from subprocess import check_call
|
||||
import os
|
||||
import json
|
||||
import gitlab
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
|
||||
class DlGitlabProjects(object):
|
||||
def __init__(self, agent, gl):
|
||||
self.agent = agent
|
||||
self.gl = gl
|
||||
|
||||
def run(self):
|
||||
projects = self.get_gitlab_projects()
|
||||
futures = []
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
for name, project in projects.items():
|
||||
# import ipdb ; ipdb.set_trace()
|
||||
if not os.path.exists(os.path.join("tmp", project.name)):
|
||||
futures.append(pool.submit(check_call, ["git", "clone", project.ssh_url_to_repo, project.name], cwd="./tmp/"))
|
||||
else:
|
||||
with open(os.path.join("tmp", project.name + ".txt"), "w") as f:
|
||||
f.write(project.description)
|
||||
for item in futures:
|
||||
e = item.exception()
|
||||
if e:
|
||||
raise e
|
||||
|
||||
def get_gitlab_projects(self):
|
||||
page = 0
|
||||
all_projects = {}
|
||||
while True:
|
||||
projects = self.gl.projects.owned(per_page=10, page=page)
|
||||
page += 1
|
||||
if not projects:
|
||||
break
|
||||
for item in projects:
|
||||
if item.visibility_level == gitlab.VISIBILITY_PUBLIC: # or "githubmirror" not in item.tag_list:
|
||||
continue
|
||||
all_projects[item.name] = item
|
||||
return all_projects
|
||||
|
||||
|
||||
def main():
|
||||
with open("creds.json") as f:
|
||||
creds = json.load(f)
|
||||
|
||||
gitlab_api = gitlab.Gitlab(creds["gitlab"]["url"], creds["gitlab"]["token"], api_version=3)
|
||||
|
||||
with SSHAgent() as agent:
|
||||
check_call(["ssh-add", creds["identity"]], env={"SSH_AUTH_SOCK": agent})
|
||||
DlGitlabProjects(agent, gitlab_api).run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
from mirror import SSHAgent
|
||||
from subprocess import check_call
|
||||
from collections import defaultdict
|
||||
import json
|
||||
import gitlab
|
||||
|
||||
|
||||
class DlGitlabProjects(object):
|
||||
def __init__(self, agent, gl):
|
||||
self.agent = agent
|
||||
self.gl = gl
|
||||
|
||||
def run(self):
|
||||
projects = self.get_gitlab_projects()
|
||||
hooks = defaultdict(list)
|
||||
for name, project in projects.items():
|
||||
# import ipdb ; ipdb.set_trace()
|
||||
for hook in self.gl.project_hooks.list(project_id=project.id):
|
||||
hooks[name].append(hook.url)
|
||||
|
||||
with open("./hooks.json", "w") as f:
|
||||
json.dump(dict(hooks), f, sort_keys=True, indent=4)
|
||||
|
||||
def get_gitlab_projects(self):
|
||||
page = 0
|
||||
all_projects = {}
|
||||
while True:
|
||||
projects = self.gl.projects.list(per_page=10, page=page, owned=True)
|
||||
page += 1
|
||||
if not projects:
|
||||
break
|
||||
for item in projects:
|
||||
# if item.visibility == 'public': # or "githubmirror" not in item.tag_list:
|
||||
# continue
|
||||
all_projects[item.name] = item
|
||||
return all_projects
|
||||
|
||||
|
||||
def main():
|
||||
with open("creds.json") as f:
|
||||
creds = json.load(f)
|
||||
|
||||
gitlab_api = gitlab.Gitlab(creds["gitlab"]["url"], creds["gitlab"]["token"], api_version=4)
|
||||
|
||||
with SSHAgent() as agent:
|
||||
check_call(["ssh-add", creds["identity"]], env={"SSH_AUTH_SOCK": agent})
|
||||
DlGitlabProjects(agent, gitlab_api).run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -0,0 +1,335 @@
|
|||
import json
|
||||
import requests
|
||||
|
||||
|
||||
class ApiException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class Gitea(object):
|
||||
def __init__(self, url, token):
|
||||
self.url = url
|
||||
self.session = requests.session()
|
||||
self.session.headers = {"Authorization": "token " + token}
|
||||
|
||||
def get_url(self, endpoint):
|
||||
url = self.url + endpoint
|
||||
print(url)
|
||||
return url
|
||||
|
||||
def parse_result(self, result):
|
||||
print(result.status_code)
|
||||
if (result.text and len(result.text) > 3):
|
||||
return json.loads(result.text)
|
||||
return {}
|
||||
|
||||
def requests_get(self, endpoint):
|
||||
return self.parse_result(self.session.get(self.get_url(endpoint)))
|
||||
|
||||
def requests_post(self, endpoint, data):
|
||||
return self.parse_result(self.session.post(self.get_url(endpoint), json=data))
|
||||
|
||||
def get_version(self):
|
||||
path = '/version'
|
||||
return self.requests_get(path)
|
||||
|
||||
def create_repo(self, **kwargs):
|
||||
path = '/user/repos'
|
||||
return self.requests_post(path, kwargs)
|
||||
|
||||
def create_hook(self, owner, repo, url):
|
||||
path = '/repos/{owner}/{repo}/hooks'.format(owner=owner, repo=repo)
|
||||
return self.requests_post(path,
|
||||
{"active": True,
|
||||
"config": {
|
||||
"url": url,
|
||||
"content_type": "json",
|
||||
"secret": ""
|
||||
},
|
||||
"events": ["push"],
|
||||
"type": "gitea"})
|
||||
|
||||
# def get_repos_search(self, q, uid, limit):
|
||||
# path = '/repos/search'
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def get_users_gpg_keys(self, username):
|
||||
# path = '/users/' + username + '/gpg_keys'
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def get_orgs_repos(self, orgname):
|
||||
# path = '/orgs/' + orgname + '/repos'
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def put_user_starred(self, username, reponame):
|
||||
# path = '/user/starred/' + username + '/' + reponame
|
||||
# return self.requests.put(path)
|
||||
|
||||
# def delete_user_starred(self, username, reponame):
|
||||
# path = '/user/starred/' + username + '/' + reponame
|
||||
# return self.requests.delete(path)
|
||||
|
||||
# def get_user_starred(self, username, reponame):
|
||||
# path = '/user/starred/' + username + '/' + reponame
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def get_users_search(self, ):
|
||||
# path = '/users/search'
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def delete_repos(self, username, reponame):
|
||||
# path = '/repos/' + username + '/' + reponame
|
||||
# return self.requests.delete(path)
|
||||
|
||||
# def get_repos(self, username, reponame):
|
||||
# path = '/repos/' + username + '/' + reponame
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def get_users_repos(self, username):
|
||||
# path = '/users/' + username + '/repos'
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def post_repos__mirror_sync(self, username, reponame):
|
||||
# path = '/repos/' + username + '/' + reponame + '/mirror-sync'
|
||||
# return self.requests_post(path, data={})
|
||||
|
||||
# def post_admin_users_orgs(self, username, full_name, description, website, location):
|
||||
# path = '/admin/users/' + username + '/orgs'
|
||||
# return self.requests_post(path, data={'username': username, 'full_name': full_name, 'description': description,
|
||||
# 'website': website, 'location': location})
|
||||
|
||||
# def post_user_gpg_keys(self, armored_public_key):
|
||||
# path = '/user/gpg_keys'
|
||||
# return self.requests_post(path, data={'armored_public_key': armored_public_key})
|
||||
|
||||
# def get_user_gpg_keys_all(self):
|
||||
# path = '/user/gpg_keys'
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def get_user_subscriptions(self, ):
|
||||
# path = '/user/subscriptions'
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def post_markdown(self, Text, Mode, Context, Wiki):
|
||||
# path = '/markdown'
|
||||
# return self.requests_post(path, data={'Text': Text, 'Mode': Mode, 'Context': Context, 'Wiki': Wiki})
|
||||
|
||||
# def patch_repos__hooks(self, config, events, active, username, reponame, id):
|
||||
# path = '/repos/' + username + '/' + reponame + '/hooks/' + id
|
||||
# return self.requests.patch(path)
|
||||
|
||||
# def delete_repos_hooks(self, username, reponame, id):
|
||||
# path = '/repos/' + username + '/' + reponame + '/hooks/' + id
|
||||
# return self.requests.delete(path)
|
||||
|
||||
# def get_users_keys(self, username):
|
||||
# path = '/users/' + username + '/keys'
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def put_user_following(self, username):
|
||||
# path = '/user/following/' + username
|
||||
# return self.requests.put(path)
|
||||
|
||||
# def delete_user_following(self, username):
|
||||
# path = '/user/following/' + username
|
||||
# return self.requests.delete(path)
|
||||
|
||||
# def get_user_following(self, username):
|
||||
# path = '/user/following/' + username
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def post_markdown_raw(self, ):
|
||||
# path = '/markdown/raw'
|
||||
# return self.requests_post(path, data={})
|
||||
|
||||
# def get_users_subscriptions(self, username):
|
||||
# path = '/users/' + username + '/subscriptions'
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def delete_user_keys(self, id):
|
||||
# path = '/user/keys/' + id
|
||||
# return self.requests.delete(path)
|
||||
|
||||
# def get_user_keys(self, id):
|
||||
# path = '/user/keys/' + id
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def post_orgs_hooks(self, type, config, events, active, orgname):
|
||||
# path = '/orgs/' + orgname + '/hooks/'
|
||||
# return self.requests_post(path, data={'type': type, 'config': config, 'events': events, 'active': active})
|
||||
|
||||
# def get_user(self, ):
|
||||
# path = '/user'
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def get_orgs_members_all(self, orgname):
|
||||
# path = '/orgs/' + orgname + '/members'
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def patch_admin_users(self, source_id, login_name, full_name, email, password, website, location, active, admin,
|
||||
# allow_git_hook, allow_import_local, max_repo_creation, username):
|
||||
# path = '/admin/users/' + username
|
||||
# return self.requests.patch(path)
|
||||
|
||||
# def delete_admin_users(self, username):
|
||||
# path = '/admin/users/' + username
|
||||
# return self.requests.delete(path)
|
||||
|
||||
# def post_admin_users_keys(self, title, key, username):
|
||||
# path = '/admin/users/' + username + '/keys'
|
||||
# return self.requests_post(path, data={'title': title, 'key': key})
|
||||
|
||||
# def post_user_keys(self, title, key):
|
||||
# path = '/user/keys'
|
||||
# return self.requests_post(path, data={'title': title, 'key': key})
|
||||
|
||||
# def get_user_keys_all(self):
|
||||
# path = '/user/keys'
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def get_users_tokens(self, username):
|
||||
# path = '/users/' + username + '/tokens'
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def get_orgs_hooks_all(self, orgname):
|
||||
# path = '/orgs/' + orgname + '/hooks'
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def get_users(self, username):
|
||||
# path = '/users/' + username
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def get_orgs_public_members_all(self, orgname):
|
||||
# path = '/orgs/' + orgname + '/public_members'
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def post_repos__forks(self, organization, repo, owner):
|
||||
# path = '/repos/' + owner + '/' + repo + '/forks'
|
||||
# return self.requests_post(path, data={'organization': organization})
|
||||
|
||||
# def get_repos_forks(self, repo, owner):
|
||||
# path = '/repos/' + owner + '/' + repo + '/forks'
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def post_admin_users(self, source_id, login_name, username, full_name, email, password, send_notify):
|
||||
# path = '/admin/users'
|
||||
# return self.requests_post(path, data={'source_id': source_id, 'login_name': login_name, 'username': username,
|
||||
# 'full_name': full_name, 'email': email, 'password': password,
|
||||
# 'send_notify': send_notify})
|
||||
|
||||
# def get_user_starred_all(self):
|
||||
# path = '/user/starred'
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def get_users_followers(self, username):
|
||||
# path = '/users/' + username + '/followers'
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def get_repositories(self, id):
|
||||
# path = '/repositories/' + id
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def put_repos__subscription(self, username, reponame):
|
||||
# path = '/repos/' + username + '/' + reponame + '/subscription'
|
||||
# return self.requests.put(path)
|
||||
|
||||
# def delete_repos_subscription(self, username, reponame):
|
||||
# path = '/repos/' + username + '/' + reponame + '/subscription'
|
||||
# return self.requests.delete(path)
|
||||
|
||||
# def get_repos_subscription(self, username, reponame):
|
||||
# path = '/repos/' + username + '/' + reponame + '/subscription'
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def get_users_following(self, username):
|
||||
# path = '/users/' + username + '/following'
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def get_users_starred(self, username):
|
||||
# path = '/users/' + username + '/starred'
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def put_orgs_public_members(self, username, orgname):
|
||||
# path = '/orgs/' + orgname + '/public_members/' + username
|
||||
# return self.requests.put(path)
|
||||
|
||||
# def delete_orgs_public_members(self, username, orgname):
|
||||
# path = '/orgs/' + orgname + '/public_members/' + username
|
||||
# return self.requests.delete(path)
|
||||
|
||||
# def get_orgs_public_members(self, username, orgname):
|
||||
# path = '/orgs/' + orgname + '/public_members/' + username
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def get_user_following_all(self, ):
|
||||
# path = '/user/following'
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def get_user_followers(self, ):
|
||||
# path = '/user/followers'
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def post_admin_users_repos(self, name, description, private, auto_init, gitignores, license, readme, username):
|
||||
# path = '/admin/users/' + username + '/repos'
|
||||
# return self.requests_post(path, data={'name': name, 'description': description, 'private': private,
|
||||
# 'auto_init': auto_init, 'gitignores': gitignores, 'license': license,
|
||||
# 'readme': readme})
|
||||
|
||||
# def patch_orgs_hooks(self, config, events, active, orgname, id):
|
||||
# path = '/orgs/' + orgname + '/hooks/' + id
|
||||
# return self.requests.patch(path)
|
||||
|
||||
# def delete_orgs_hooks(self, orgname, id):
|
||||
# path = '/orgs/' + orgname + '/hooks/' + id
|
||||
# return self.requests.delete(path)
|
||||
|
||||
# def get_orgs_hooks(self, orgname, id):
|
||||
# path = '/orgs/' + orgname + '/hooks/' + id
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def post_org_repos(self, name, description, private, auto_init, gitignores, license, readme, org):
|
||||
# path = '/org/' + org + '/repos'
|
||||
# return self.requests_post(path, data={'name': name, 'description': description, 'private': private,
|
||||
# 'auto_init': auto_init, 'gitignores': gitignores, 'license': license,
|
||||
# 'readme': readme})
|
||||
|
||||
# def delete_orgs_members(self, orgname, username):
|
||||
# path = '/orgs/' + orgname + '/members/' + username
|
||||
# return self.requests.delete(path)
|
||||
|
||||
# def get_orgs_members(self, orgname, username):
|
||||
# path = '/orgs/' + orgname + '/members/' + username
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def post_repos__hooks(self, type, config, events, active, reponame, username):
|
||||
# path = '/repos/' + username + '/' + reponame + '/hooks'
|
||||
# return self.requests_post(path, data={'type': type, 'config': config, 'events': events, 'active': active})
|
||||
|
||||
# def get_repos_hooks(self, reponame, username):
|
||||
# path = '/repos/' + username + '/' + reponame + '/hooks'
|
||||
# return self.requests_get(path)
|
||||
|
||||
# def post_repos_migrate(self, clone_addr, auth_username, auth_password, uid, repo_name, mirror, private,
|
||||
# description):
|
||||
# path = '/repos/migrate'
|
||||
# return self.requests_post(path, data={'clone_addr': clone_addr, 'auth_username': auth_username,
|
||||
# 'auth_password': auth_password, 'uid': uid, 'repo_name': repo_name,
|
||||
# 'mirror': mirror, 'private': private, 'description': description})
|
||||
|
||||
# def post_user_repos(self, name, description, private, auto_init, gitignores, license, readme):
|
||||
# path = '/user/repos'
|
||||
# return self.requests_post(path, data={'name': name, 'description': description, 'private': private,
|
||||
# 'auto_init': auto_init, 'gitignores': gitignores, 'license': license,
|
||||
# 'readme': readme})
|
||||
|
||||
def get_user_repos(self, ):
|
||||
path = '/user/repos'
|
||||
return self.requests_get(path)
|
||||
|
||||
# def delete_user_gpg_keys(self, id):
|
||||
# path = '/user/gpg_keys/' + id
|
||||
# return self.requests.delete(path)
|
||||
|
||||
# def get_user_gpg_keys(self, id):
|
||||
# path = '/user/gpg_keys/' + id
|
||||
# return self.requests_get(path)
|
||||
|
|
@ -0,0 +1,112 @@
|
|||
#!/usr/bin/env python3
|
||||
import json
|
||||
import gitea
|
||||
import github
|
||||
import os
|
||||
from shutil import rmtree
|
||||
import re
|
||||
from subprocess import check_call, check_output
|
||||
from time import time
|
||||
import signal
|
||||
|
||||
|
||||
class SSHAgent(object):
|
||||
sockre = re.compile(r'SSH_AUTH_SOCK=(.*?);')
|
||||
pidre = re.compile(r'SSH_AGENT_PID=(\d*);', )
|
||||
|
||||
def __init__(self, cluster=None):
|
||||
self.addargs = []
|
||||
if cluster:
|
||||
self.addargs.extend(['-a', os.path.join(cluster, 'agent.sock.{}'.format(int(time())))])
|
||||
|
||||
def __enter__(self):
|
||||
output = check_output(['ssh-agent'] + self.addargs).decode("UTF-8")
|
||||
self.sock = self.sockre.search(output).group(1)
|
||||
self.pid = int(self.pidre.search(output).group(1))
|
||||
return self.sock
|
||||
|
||||
def __exit__(self, errtype, value, traceback):
|
||||
os.kill(self.pid, signal.SIGHUP)
|
||||
|
||||
|
||||
class GiteaToGithubMirror(object):
|
||||
def __init__(self, creds, agent):
|
||||
self.creds = creds
|
||||
self.agent = agent
|
||||
|
||||
self.ge = gitea.Gitea(self.creds["gitea"]["url"], self.creds["gitea"]["token"])
|
||||
self.gh = github.Github(self.creds["github"]["username"], self.creds["github"]["password"])
|
||||
self.ghu = self.gh.get_user()
|
||||
|
||||
if not os.path.exists("./tmp"):
|
||||
os.mkdir("./tmp")
|
||||
|
||||
def run(self):
|
||||
self.github_projects = self.get_github_projects()
|
||||
self.gitea_projects = self.get_gitea_projects()
|
||||
|
||||
print("{} projects to check".format(len(self.gitea_projects)))
|
||||
visited = []
|
||||
for project_name, project in self.gitea_projects.items():
|
||||
if project_name in visited:
|
||||
continue
|
||||
visited.append(project_name)
|
||||
print("({}/{}) Mirroring {}".format(len(visited), len(self.gitea_projects), project_name))
|
||||
self.mirror_to_gh(project_name)
|
||||
|
||||
def get_gitea_projects(self):
|
||||
projects = {}
|
||||
for project in filter(lambda x: x["private"] == False and x["empty"] == False, self.ge.get_user_repos()):
|
||||
projects[project['name']] = project
|
||||
return projects
|
||||
|
||||
def get_github_projects(self):
|
||||
all_projects = {}
|
||||
for repo in self.ghu.get_repos():
|
||||
all_projects[repo.name] = repo
|
||||
return all_projects
|
||||
|
||||
def mirror_to_gh(self, project_name):
|
||||
assert project_name
|
||||
source = self.gitea_projects[project_name]
|
||||
dest = None
|
||||
|
||||
if project_name in self.github_projects:
|
||||
dest = self.github_projects[project_name]
|
||||
else:
|
||||
print("{} not found in github, creating".format(project_name))
|
||||
dest = self.ghu.create_repo(name=source['name'],
|
||||
description=source['description'],
|
||||
homepage=source['html_url']) # Gitea project url
|
||||
|
||||
repo_dir = os.path.join("tmp", source['name'])
|
||||
|
||||
if not os.path.exists(repo_dir):
|
||||
try:
|
||||
check_call(["git", "clone", source['ssh_url'], source['name']], cwd="./tmp/")
|
||||
except:
|
||||
rmtree(repo_dir)
|
||||
raise
|
||||
try:
|
||||
check_call(["git", "remote", "add", "github", dest.ssh_url], cwd=repo_dir)
|
||||
except: # Remote already exists
|
||||
pass
|
||||
else:
|
||||
check_call(["git", "fetch"], cwd=repo_dir)
|
||||
check_call(["git", "pull", "origin", "master"], cwd=repo_dir)
|
||||
|
||||
check_call(["git", "push", "-f", "-u", "github", "master"], cwd=repo_dir, env={"SSH_AUTH_SOCK": self.agent})
|
||||
print("Completed {}\n".format(project_name))
|
||||
|
||||
|
||||
def main():
|
||||
with open("creds.json") as f:
|
||||
creds = json.load(f)
|
||||
|
||||
with SSHAgent() as agent:
|
||||
check_call(["ssh-add", creds["identity"]], env={"SSH_AUTH_SOCK": agent})
|
||||
GiteaToGithubMirror(creds, agent).run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -0,0 +1,120 @@
|
|||
#!/usr/bin/env python3
|
||||
import json
|
||||
import gitlab
|
||||
import github
|
||||
import os
|
||||
from shutil import rmtree
|
||||
import re
|
||||
from subprocess import check_call, check_output
|
||||
from time import time
|
||||
import signal
|
||||
|
||||
|
||||
class SSHAgent(object):
|
||||
sockre = re.compile(r'SSH_AUTH_SOCK=(.*?);')
|
||||
pidre = re.compile(r'SSH_AGENT_PID=(\d*);', )
|
||||
|
||||
def __init__(self, cluster=None):
|
||||
self.addargs = []
|
||||
if cluster:
|
||||
self.addargs.extend(['-a', os.path.join(cluster, 'agent.sock.{}'.format(int(time())))])
|
||||
|
||||
def __enter__(self):
|
||||
output = check_output(['ssh-agent'] + self.addargs).decode("UTF-8")
|
||||
self.sock = self.sockre.search(output).group(1)
|
||||
self.pid = int(self.pidre.search(output).group(1))
|
||||
return self.sock
|
||||
|
||||
def __exit__(self, errtype, value, traceback):
|
||||
os.kill(self.pid, signal.SIGHUP)
|
||||
|
||||
|
||||
class GitlabToGithubMirror(object):
|
||||
def __init__(self, creds, agent):
|
||||
self.creds = creds
|
||||
self.agent = agent
|
||||
|
||||
self.gl = gitlab.Gitlab(self.creds["gitlab"]["url"], self.creds["gitlab"]["token"], api_version=3)
|
||||
self.gh = github.Github(self.creds["github"]["username"], self.creds["github"]["password"])
|
||||
self.ghu = self.gh.get_user()
|
||||
|
||||
if not os.path.exists("./tmp"):
|
||||
os.mkdir("./tmp")
|
||||
|
||||
def get_gitlab_projects(self):
|
||||
page = 0
|
||||
all_projects = {}
|
||||
while True:
|
||||
projects = self.gl.projects.owned(per_page=10, page=page)
|
||||
page += 1
|
||||
if not projects:
|
||||
break
|
||||
for item in projects:
|
||||
if item.visibility_level != gitlab.VISIBILITY_PUBLIC or "githubmirror" not in item.tag_list:
|
||||
continue
|
||||
all_projects[item.name] = item
|
||||
return all_projects
|
||||
|
||||
def get_github_projects(self):
|
||||
all_projects = {}
|
||||
for repo in self.ghu.get_repos():
|
||||
all_projects[repo.name] = repo
|
||||
return all_projects
|
||||
|
||||
def run(self):
|
||||
self.github_projects = self.get_github_projects()
|
||||
self.gitlab_projects = self.get_gitlab_projects()
|
||||
|
||||
print("{} projects to check".format(len(self.gitlab_projects)))
|
||||
visited = []
|
||||
for project_name, project in self.gitlab_projects.items():
|
||||
if project_name in visited:
|
||||
continue
|
||||
visited.append(project_name)
|
||||
print("({}/{}) Mirroring {}".format(len(visited), len(self.gitlab_projects), project_name))
|
||||
self.mirror_to_gh(project_name)
|
||||
|
||||
def mirror_to_gh(self, project_name):
|
||||
assert project_name
|
||||
source = self.gitlab_projects[project_name]
|
||||
dest = None
|
||||
|
||||
if project_name in self.github_projects:
|
||||
dest = self.github_projects[project_name]
|
||||
else:
|
||||
print("{} not found in github, creating".format(project_name))
|
||||
dest = self.ghu.create_repo(name=source.name,
|
||||
description=source.description,
|
||||
homepage=source.web_url) # Gitlab url
|
||||
|
||||
repo_dir = os.path.join("tmp", source.name)
|
||||
|
||||
if not os.path.exists(repo_dir):
|
||||
try:
|
||||
check_call(["git", "clone", source.ssh_url_to_repo, source.name], cwd="./tmp/")
|
||||
except:
|
||||
rmtree(repo_dir)
|
||||
raise
|
||||
try:
|
||||
check_call(["git", "remote", "add", "github", dest.ssh_url], cwd=repo_dir)
|
||||
except: # Remote already exists
|
||||
pass
|
||||
else:
|
||||
check_call(["git", "fetch"], cwd=repo_dir)
|
||||
check_call(["git", "pull", "origin", "master"], cwd=repo_dir)
|
||||
|
||||
check_call(["git", "push", "-f", "-u", "github", "master"], cwd=repo_dir, env={"SSH_AUTH_SOCK": self.agent})
|
||||
print("Completed {}\n".format(project_name))
|
||||
|
||||
|
||||
def main():
|
||||
with open("creds.json") as f:
|
||||
creds = json.load(f)
|
||||
|
||||
with SSHAgent() as agent:
|
||||
check_call(["ssh-add", creds["identity"]], env={"SSH_AUTH_SOCK": agent})
|
||||
GitlabToGithubMirror(creds, agent).run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
import gitea
|
||||
from pprint import pprint
|
||||
import os
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from subprocess import check_call
|
||||
import json
|
||||
|
||||
|
||||
def main():
|
||||
with open("creds.json") as f:
|
||||
creds = json.load(f)
|
||||
client = gitea.Gitea(creds["gitea"]["url"], creds["gitea"]["token"])
|
||||
|
||||
projects = [i.name for i in os.scandir("tmp") if i.is_dir()]
|
||||
|
||||
# import ipdb ; ipdb.set_trace()
|
||||
|
||||
futures = []
|
||||
with ThreadPoolExecutor(max_workers=1) as pool:
|
||||
for project in projects:
|
||||
projdir = os.path.join("tmp", project)
|
||||
with open(projdir + ".txt") as f:
|
||||
description = f.read()
|
||||
|
||||
new_repo = client.create_repo(**{"auto_init": False,
|
||||
"description": description,
|
||||
# "gitignores": "string",
|
||||
# "license": "string",
|
||||
"name": project,
|
||||
"private": True,
|
||||
# "readme": "string"
|
||||
})
|
||||
|
||||
if "ssh_url" in new_repo:
|
||||
check_call(["git", "remote", "add", "gitea", new_repo["ssh_url"]], cwd=projdir)
|
||||
|
||||
futures.append(pool.submit(check_call,
|
||||
["git", "push", "gitea"],
|
||||
cwd=projdir))
|
||||
errors = []
|
||||
for item in futures:
|
||||
e = item.exception()
|
||||
if e:
|
||||
errors.append(e)
|
||||
print("Errors: ", len(errors))
|
||||
pprint(errors)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
certifi==2018.4.16
|
||||
chardet==3.0.4
|
||||
idna==2.7
|
||||
PyGithub==1.40
|
||||
PyJWT==1.6.4
|
||||
python-gitlab==1.5.1
|
||||
requests==2.19.1
|
||||
six==1.11.0
|
||||
urllib3==1.23
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
import gitea
|
||||
import json
|
||||
|
||||
|
||||
def main():
|
||||
with open("creds.json") as f:
|
||||
creds = json.load(f)
|
||||
client = gitea.Gitea(creds["gitea"]["url"], creds["gitea"]["token"])
|
||||
with open("hooks.json") as f:
|
||||
hooks = json.load(f)
|
||||
|
||||
for repo, urls in hooks.items():
|
||||
print(repo)
|
||||
for url in urls:
|
||||
print(client.create_hook("dave", repo, url))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
Reference in New Issue