<commit_before>#!/usr/bin/env python3
import os

from xdg import BaseDirectory
from subprocess import Popen


def mirror(args):
    # The URIs for the document types
    uris = {'charter': 'ietf.org::everything-ftp/ietf/',
            'conflict': 'rsync.ietf.org::everything-ftp/conflict-reviews/',
            'draft': 'rsync.ietf.org::internet-drafts',
            'iana': 'rsync.ietf.org::everything-ftp/iana/',
            'iesg': 'rsync.ietf.org::iesg-minutes/',
            'rfc': 'ftp.rfc-editor.org::everything-ftp/in-notes/',
            'status': 'rsync.ietf.org::everything-ftp/status-changes/'}
    # Strings to be passed to rsync as arguments to `--exclude`
    excludes = {'charter': [],
                'conflict': [],
                'draft': ['*.xml', '*.pdf'],
                'iana': [],
                'iesg': [],
                'rfc': ['tar*', 'search*', 'PDF-RFC*', 'tst/', 'pdfrfc/',
                        'internet-drafts/', 'ien/'],
                'status': []}

    # Set the top-level mirror directory
    top_dir = os.path.abspath(
        os.path.expandvars(
            os.path.expanduser(
                args.dir[0])))
    # Attempt to create directory
    try:
        os.makedirs(top_dir)
    except FileExistsError:
        pass  # Directory already exists

    # No document type passed as an argument
    if args.type is None:
        # Generate a dict of command arrays from the above dicts
        commands = {}
        for doc_type, uri in uris.items():
            command = ['rsync', '-az', '--delete-during']
            for exclude in excludes[doc_type]:
                command.append("--exclude='{}'".format(exclude))
            command.append(uri)
            command.append(top_dir + '/' + doc_type)
            print(command)
            commands[doc_type] = command
        processes = []
        for doc_type, command in commands.items():
            # Create subdirectory for the document type
            try:
                os.makedirs(top_dir + '/' + doc_type)
            except FileExistsError:  # Directory already exists
                pass
            # Start the rsync process and add its PID to processes
            process = Popen(command)
            processes.append(process)
        # Wait for each rsync process to complete
        exitcodes = [p.wait() for p in processes]
    # One or multiple document types passed as arguments
    else:
        # Generate a dict of command arrays
        commands = {}
        for doc_type in args.type:
            command = ['rsync', '-az', '--delete-during']
            for exclude in excludes[doc_type]:
                command.append("--exclude='{}'".format(exclude))
            command.append(uris[doc_type])
            command.append(top_dir + '/' + doc_type)
            print(command)
            commands[doc_type] = command
        processes = []
        for doc_type, command in commands.items():
            # Create subdirectory for the document type
            try:
                os.makedirs(top_dir + '/' + doc_type)
            except FileExistsError:  # Directory already exists
                pass
            # Start the rsync process and add its PID to processes
            process = Popen(command)
            processes.append(process)
        # Wait for each rsync process to complete
        exitcodes = [p.wait() for p in processes]
<commit_msg>mirror: Make URI and exclude dicts global<commit_after>#!/usr/bin/env python3
import os

from xdg import BaseDirectory
from subprocess import Popen

URI_DICT = {'charter': 'ietf.org::everything-ftp/ietf/',
            'conflict': 'rsync.ietf.org::everything-ftp/conflict-reviews/',
            'draft': 'rsync.ietf.org::internet-drafts',
            'iana': 'rsync.ietf.org::everything-ftp/iana/',
            'iesg': 'rsync.ietf.org::iesg-minutes/',
            'rfc': 'ftp.rfc-editor.org::everything-ftp/in-notes/',
            'status': 'rsync.ietf.org::everything-ftp/status-changes/'}

EXCLUDE_DICT = {'charter': [],
                'conflict': [],
                'draft': ['*.xml', '*.pdf'],
                'iana': [],
                'iesg': [],
                'rfc': ['tar*', 'search*', 'PDF-RFC*', 'tst/', 'pdfrfc/',
                        'internet-drafts/', 'ien/'],
                'status': []}


def mirror(args):
    # Set the top-level mirror directory
    top_dir = os.path.abspath(
        os.path.expandvars(
            os.path.expanduser(
                args.dir[0])))
    # Attempt to create directory
    try:
        os.makedirs(top_dir)
    except FileExistsError:
        pass  # Directory already exists

    # No document type passed as an argument
    if args.type is None:
        # Generate a dict of command arrays from the above dicts
        commands = {}
        for doc_type, uri in URI_DICT.items():
            command = ['rsync', '-az', '--delete-during']
            for exclude in EXCLUDE_DICT[doc_type]:
                command.append("--exclude='{}'".format(exclude))
            command.append(uri)
            command.append(top_dir + '/' + doc_type)
            print(command)
            commands[doc_type] = command
        processes = []
        for doc_type, command in commands.items():
            # Create subdirectory for the document type
            try:
                os.makedirs(top_dir + '/' + doc_type)
            except FileExistsError:  # Directory already exists
                pass
            # Start the rsync process and add its PID to processes
            process = Popen(command)
            processes.append(process)
        # Wait for each rsync process to complete
        exitcodes = [p.wait() for p in processes]
    # One or multiple document types passed as arguments
    else:
        # Generate a dict of command arrays
        commands = {}
        for doc_type in args.type:
            command = ['rsync', '-az', '--delete-during']
            for exclude in EXCLUDE_DICT[doc_type]:
                command.append("--exclude='{}'".format(exclude))
            command.append(URI_DICT[doc_type])
            command.append(top_dir + '/' + doc_type)
            print(command)
            commands[doc_type] = command
        processes = []
        for doc_type, command in commands.items():
            # Create subdirectory for the document type
            try:
                os.makedirs(top_dir + '/' + doc_type)
            except FileExistsError:  # Directory already exists
                pass
            # Start the rsync process and add its PID to processes
            process = Popen(command)
            processes.append(process)
        # Wait for each rsync process to complete
        exitcodes = [p.wait() for p in processes]

{'fraction_non_alphanumeric': 0.11966701352757544, 'fraction_numerical': 0.0005946187007581388, 'mean_word_length': 1.836424957841484, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7416917', 'n_tokens_mistral': 1947, 'n_tokens_neox': 1870, 'n_words': 500}