<commit_before>package net.barik.spreadsheet;

import java.io.IOException;
import java.io.InputStream;
import java.net.URLEncoder;

import org.apache.hadoop.conf.Configuration;

import net.barik.spreadsheet.analysis.AnalysisOutput;
import net.barik.spreadsheet.analysis.SpreadsheetAnalyzer;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class AnalysisMapper extends Mapper<LongWritable, Text, Text, Text> {

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
    	Configuration conf = context.getConfiguration();
    	
    	String corpusName = conf.get("corpus.name", "[unspecified]");

        String importBucket = conf.get("import.bucket", "barik-cc");
        String importKeyPrefix = conf.get("import.keyprefix", "analysis/binaries/");
 
        String exportBucket = conf.get("export.bucket", "barik-cc");
        String exportKeyPrefix = conf.get("export.keyprefix", "analysis/output/");
    	
    	
    	String fileName = URLEncoder.encode(value.toString().trim(), "UTF-8");
    	String path = importKeyPrefix + fileName;
    	if (fileName.isEmpty()) {
    		context.write(new Text(path), new Text("[Empty Path Name]"));
    		return;
    	}

        InputStream is = S3Load.loadSpreadsheet(importBucket, path);
        
        AnalysisOutput ao = SpreadsheetAnalyzer.doAnalysisAndGetObject(is, corpusName, fileName);

        JacksonS3Export.exportItem(ao, exportBucket, exportKeyPrefix, fileName);
        is.close();

        context.write(new Text(path), new Text(ao.errorNotification));

    }
}<commit_msg>URLEncoding doesn't fix the problem<commit_after>package net.barik.spreadsheet;

import java.io.IOException;
import java.io.InputStream;
import java.net.URLEncoder;

import org.apache.hadoop.conf.Configuration;

import net.barik.spreadsheet.analysis.AnalysisOutput;
import net.barik.spreadsheet.analysis.SpreadsheetAnalyzer;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class AnalysisMapper extends Mapper<LongWritable, Text, Text, Text> {

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
    	Configuration conf = context.getConfiguration();
    	
    	String corpusName = conf.get("corpus.name", "[unspecified]");

        String importBucket = conf.get("import.bucket", "barik-cc");
        String importKeyPrefix = conf.get("import.keyprefix", "analysis/binaries/");
 
        String exportBucket = conf.get("export.bucket", "barik-cc");
        String exportKeyPrefix = conf.get("export.keyprefix", "analysis/output/");
    	
    	
    	String fileName = value.toString().trim();
    	String path = importKeyPrefix + fileName;
    	if (fileName.isEmpty()) {
    		context.write(new Text(path), new Text("[Empty Path Name]"));
    		return;
    	}

    	try {
	        InputStream is = S3Load.loadSpreadsheet(importBucket, path);
	        
	        AnalysisOutput ao = SpreadsheetAnalyzer.doAnalysisAndGetObject(is, corpusName, fileName);
	
	        JacksonS3Export.exportItem(ao, exportBucket, exportKeyPrefix, fileName);
	        is.close();
	
	        context.write(new Text(path), new Text(ao.errorNotification));
    	}
    	catch (Exception e) {
    		throw new IOException(String.format("Problem with %s;%s;%s;%s;",fileName, path, importKeyPrefix, exportKeyPrefix)
    				, e);
    	}

    }
}
{'fraction_non_alphanumeric': 0.11928934010152284, 'fraction_numerical': 0.0014100394811054709, 'mean_word_length': 3.7610738255033556, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5567011', 'n_tokens_mistral': 1074, 'n_tokens_neox': 1038, 'n_words': 212}