<commit_before>#!/usr/bin/env python2

import json
import os
import sys

def main(transfer_path):
    basename = os.path.basename(transfer_path)
    try:
        _, dc_id, _ = basename.split('---')
    except ValueError:
        return 1
    metadata = [
        {
            'parts': 'objects',
            'dc.identifier': dc_id,
        }
    ]
    metadata_path = os.path.join(transfer_path, 'metadata')
    if not os.path.exists(metadata_path):
        os.makedirs(metadata_path)
    metadata_path = os.path.join(metadata_path, 'metadata.json')
    with open(metadata_path, 'w') as f:
        json.dump(metadata, f)
    return 0

if __name__ == '__main__':
    transfer_path = sys.argv[1]
    sys.exit(main(transfer_path))
<commit_msg>Automate Transfers: Change default metadata added<commit_after>#!/usr/bin/env python2

import json
import os
import sys

def main(transfer_path):
    basename = os.path.basename(transfer_path)
    try:
        dc_id, _, _ = basename.split('---')
    except ValueError:
        return 1
    metadata = [
        {
            'parts': 'objects',
            'dc.identifier': dc_id,
        }
    ]
    metadata_path = os.path.join(transfer_path, 'metadata')
    if not os.path.exists(metadata_path):
        os.makedirs(metadata_path)
    metadata_path = os.path.join(metadata_path, 'metadata.json')
    with open(metadata_path, 'w') as f:
        json.dump(metadata, f)
    return 0

if __name__ == '__main__':
    transfer_path = sys.argv[1]
    sys.exit(main(transfer_path))

{'fraction_non_alphanumeric': 0.12450592885375494, 'fraction_numerical': 0.005270092226613966, 'mean_word_length': 2.6339712918660285, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5093404', 'n_tokens_mistral': 507, 'n_tokens_neox': 502, 'n_words': 103}