<commit_before>#!/usr/bin/env python
from __future__ import absolute_import, division, print_function, unicode_literals
import os
import sys

from .config import get_config, configure_job
from .uri import download, cleanup

def write_to_stderr(file_status, file_size, file_name):
    sys.stderr.write("{},{},{}\n".format(file_status, file_size, file_name))
    sys.stderr.flush()

def run(config):
    configure_job(config)
    try:
        for uri in iter(sys.stdin.readline, ""):
            uri = uri.rstrip() # remove trailing linebreak
            temp_filename = None
            try:
                temp_filename = download(config, uri)
                file_size = os.path.getsize(temp_filename)
                config.MAP_FUNC(temp_filename)
                write_to_stderr("+", file_size, uri)
            except (KeyboardInterrupt, SystemExit):
                sys.stderr.write("map worker {} aborted\n".format(os.getpid()))
                sys.exit(1)
            except Exception as e:
                sys.stderr.write("{}\n".format(e))<commit_msg>Allow for passing the original file uri to mapper without requiring old map funcs to change<commit_after>#!/usr/bin/env python
from __future__ import absolute_import, division, print_function, unicode_literals
from inspect import getargspec
import os
import sys

from .config import get_config, configure_job
from .uri import download, cleanup

def write_to_stderr(file_status, file_size, file_name):
    sys.stderr.write("{},{},{}\n".format(file_status, file_size, file_name))
    sys.stderr.flush()

def run(config):
    configure_job(config)
    try:
        for uri in iter(sys.stdin.readline, ""):
            uri = uri.rstrip() # remove trailing linebreak
            temp_filename = None
            try:
                temp_filename = download(config, uri)
                file_size = os.path.getsize(temp_filename)
                # allow passing uri to mapper, without breaking existing code
                if len(getargspec(config.MAP_FUNC).args) == 2:
                    config.MAP_FUNC(temp_filename, uri)
                else:
                    config.MAP_FUNC(temp_filename)
                write_to_stderr("+", file_size, uri)
            except (KeyboardInterrupt, SystemExit):
                sys.stderr.write("map worker {} aborted\n".format(os.getpid()))
                sys.exit(1)
            except Exception as e:
                sys.stderr.write("{}\n".format(e))
{'fraction_non_alphanumeric': 0.10350448247758762, 'fraction_numerical': 0.0012224938875305623, 'mean_word_length': 2.4480337078651684, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4168763', 'n_tokens_mistral': 706, 'n_tokens_neox': 682, 'n_words': 169}