<commit_before>#!/usr/bin/env python3
#
# Author: <NAME> <kyle [at] kylemana [dot] com >
#
# Example invocation:
# ./manifest-archive.py -u https://android.googlesource.com/platform/manifest -b android-1.6_r1

import os
import sys
import shutil
import logging
import tarfile
import tempfile
import subprocess
import argparse
from time import gmtime, strftime

#logging.basicConfig(level=logging.INFO)
logging.basicConfig(level=logging.DEBUG)

parser = argparse.ArgumentParser()
parser.add_argument("-b", "--manifest-branch", help="manifest branch or revision")
parser.add_argument("-u", "--manifest-url", help="manifest repository location")
parser.add_argument("-j", "--jobs", help="projects to fetch simultaneously")
parser.add_argument("-o", "--output", help="tarball output")

args = parser.parse_args()

name = args.manifest_branch
origdir = os.getcwd()

# Set TMPDIR to change the base
temp = tempfile.TemporaryDirectory()
logging.debug(temp)
base = os.path.join(temp.name, name)
os.mkdir(base)
os.chdir(base)

if args.output:
    output = os.path.join(origdir, args.output)
else:
    output = os.path.join(origdir, "{}_{}.tar.xz".format(name, strftime("%Y%m%d")))

# Build repo command
cmd = ['repo', 'init']

if args.manifest_branch:
    cmd.extend(['--manifest-branch', args.manifest_branch])

if args.manifest_url:
    cmd.extend(['--manifest-url', args.manifest_url])

# Run repo init
logging.info("Running: {}".format(str(cmd)))
ret = subprocess.call(cmd)

# Run repo sync
cmd_sync = ['repo', 'sync']

if args.jobs:
    cmd_sync.extend(['--jobs', args.jobs])

logging.info("Running: {}".format(str(cmd_sync)))
ret = subprocess.call(cmd_sync)<commit_msg>archive: Complain if less then 10GB at TMPDIR<commit_after>#!/usr/bin/env python3
#
# Author: <NAME> <kyle [at] kylemana [dot] com >
#
# Example invocation:
# ./manifest-archive.py -u https://android.googlesource.com/platform/manifest -b android-1.6_r1

import os
import sys
import shutil
import logging
import tarfile
import tempfile
import subprocess
import argparse
from time import gmtime, strftime

#logging.basicConfig(level=logging.INFO)
logging.basicConfig(level=logging.DEBUG)

parser = argparse.ArgumentParser()
parser.add_argument("-b", "--manifest-branch", help="manifest branch or revision")
parser.add_argument("-u", "--manifest-url", help="manifest repository location")
parser.add_argument("-j", "--jobs", help="projects to fetch simultaneously")
parser.add_argument("-o", "--output", help="tarball output")

args = parser.parse_args()

name = args.manifest_branch
origdir = os.getcwd()

# Set TMPDIR to change the base
temp = tempfile.TemporaryDirectory()

# Check that temp directory isn't on a small tmpfs
temp_free_gb = shutil.disk_usage(temp.name)[2] / (2**30)
logging.debug('Free diskspace @ {}: {:.4} GB'.format(temp.name, temp_free_gb))

if temp_free_gb < 10:
    print('Insufficent free diskspace @ {}: {:.4} GB'.format(
        temp.name, temp_free_gb), file=sys.stderr)
    print('Set the env TMPDIR variable to move temp directory')
    sys.exit(1)

base = os.path.join(temp.name, name)
os.mkdir(base)
os.chdir(base)

if args.output:
    output = os.path.join(origdir, args.output)
else:
    output = os.path.join(origdir, "{}_{}.tar.xz".format(name, strftime("%Y%m%d")))

# Build repo command
cmd = ['repo', 'init']

if args.manifest_branch:
    cmd.extend(['--manifest-branch', args.manifest_branch])

if args.manifest_url:
    cmd.extend(['--manifest-url', args.manifest_url])

# Run repo init
logging.info("Running: {}".format(str(cmd)))
ret = subprocess.call(cmd)

# Run repo sync
cmd_sync = ['repo', 'sync']

if args.jobs:
    cmd_sync.extend(['--jobs', args.jobs])

logging.info("Running: {}".format(str(cmd_sync)))
ret = subprocess.call(cmd_sync)
{'fraction_non_alphanumeric': 0.163063784360822, 'fraction_numerical': 0.005070723245262877, 'mean_word_length': 5.094308943089431, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '467771', 'n_tokens_mistral': 1307, 'n_tokens_neox': 1257, 'n_words': 297}