<commit_before>#!/usr/bin/python3
#
# Copyright 2020 The ANGLE Project Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# download_restricted_traces.py:
#   download restricted traces and update their timestamp for build

import fnmatch
import json
import os
import subprocess
import sys


def reject_duplicate_keys(pairs):
    found_keys = {}
    for key, value in pairs:
        if key in found_keys:
            raise ValueError("duplicate key: %r" % (key,))
        else:
            found_keys[key] = value
    return found_keys


def read_json(json_file):
    with open(json_file) as map_file:
        return json.loads(map_file.read(), object_pairs_hook=reject_duplicate_keys)


def run_command(command):
    env = os.environ.copy()
    env['PYTHONUNBUFFERED'] = '1'
    with subprocess.Popen(
            command, stdout=subprocess.PIPE, universal_newlines=True, env=env) as proc:
        while proc.poll() is None:
            out = proc.stdout.read(1)
            sys.stdout.write(out)
            sys.stdout.flush()
        if proc.returncode:
            raise subprocess.CalledProcessError(proc.returncode, cmd)


def main():
    if (len(sys.argv) != 2):
        print('Missing restricted traces directory')
        return 1

    trace_dir = sys.argv[1]

    # issue download command
    download_script = 'download_from_google_storage'
    if os.name == 'nt':
        download_script += '.bat'
    cmd = [
        download_script,
        '--directory',
        '--recursive',
        '--extract',<commit_msg>Traces: Use 4 download threads.<commit_after>#!/usr/bin/python3
#
# Copyright 2020 The ANGLE Project Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# download_restricted_traces.py:
#   download restricted traces and update their timestamp for build

import fnmatch
import json
import os
import subprocess
import sys


def reject_duplicate_keys(pairs):
    found_keys = {}
    for key, value in pairs:
        if key in found_keys:
            raise ValueError("duplicate key: %r" % (key,))
        else:
            found_keys[key] = value
    return found_keys


def read_json(json_file):
    with open(json_file) as map_file:
        return json.loads(map_file.read(), object_pairs_hook=reject_duplicate_keys)


def run_command(command):
    env = os.environ.copy()
    env['PYTHONUNBUFFERED'] = '1'
    with subprocess.Popen(
            command, stdout=subprocess.PIPE, universal_newlines=True, env=env) as proc:
        while proc.poll() is None:
            out = proc.stdout.read(1)
            sys.stdout.write(out)
            sys.stdout.flush()
        if proc.returncode:
            raise subprocess.CalledProcessError(proc.returncode, cmd)


def main():
    if (len(sys.argv) != 2):
        print('Missing restricted traces directory')
        return 1

    trace_dir = sys.argv[1]

    # issue download command
    download_script = 'download_from_google_storage'
    if os.name == 'nt':
        download_script += '.bat'
    cmd = [
        download_script,
        '--directory',
        '--recursive',
        '--extract',
        '--num_threads=4',
{'fraction_non_alphanumeric': 0.10058264336093223, 'fraction_numerical': 0.006746396810794235, 'mean_word_length': 2.677564825253664, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6343158', 'n_tokens_mistral': 1004, 'n_tokens_neox': 976, 'n_words': 274}