<commit_before>package io.github.mattthomson.depijp.tap;

import cascading.scheme.Scheme;
import cascading.scheme.local.TextDelimited;
import cascading.tuple.Fields;
import cascading.tuple.Tuple;
import cascading.tuple.TupleEntry;
import cascading.tuple.Tuples;
import com.google.common.collect.ImmutableList;

import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;
import java.util.Properties;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

public class TsvPijpTap extends FilePijpTap<List<String>> {
    private final List<Fields> fields;

    public TsvPijpTap(String path, int numFields) {
        super(path);

        fields = IntStream.range(0, numFields)
                .mapToObj(i -> new Fields(String.format("tsv-field-%s", i)))
                .collect(Collectors.toList());
    }

    @Override
    protected Scheme<Properties, InputStream, OutputStream, ?, ?> getScheme() {
        return new TextDelimited(getOutputFields(), "\t");
    }

    @Override
    public Fields getOutputFields() {
        return fields.stream()
                .collect(Collectors.reducing(new Fields(), Fields::append));
    }

    @Override
    public List<String> fromTupleEntry(TupleEntry tupleEntry) {
        return fields.stream()
                .map(tupleEntry::getString)
                .collect(Collectors.toList());
    }

    @Override
    public Tuple toTuple(List<String> value) {
        return Tuples.create(ImmutableList.copyOf(value));
    }
}
<commit_msg>Include file name in field name.<commit_after>package io.github.mattthomson.depijp.tap;

import cascading.scheme.Scheme;
import cascading.scheme.local.TextDelimited;
import cascading.tuple.Fields;
import cascading.tuple.Tuple;
import cascading.tuple.TupleEntry;
import cascading.tuple.Tuples;
import com.google.common.collect.ImmutableList;

import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;
import java.util.Properties;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

public class TsvPijpTap extends FilePijpTap<List<String>> {
    private final List<Fields> fields;

    public TsvPijpTap(String path, int numFields) {
        super(path);

        fields = IntStream.range(0, numFields)
                .mapToObj(i -> new Fields(String.format("%s-field-%s", path, i)))
                .collect(Collectors.toList());
    }

    @Override
    protected Scheme<Properties, InputStream, OutputStream, ?, ?> getScheme() {
        return new TextDelimited(getOutputFields(), "\t");
    }

    @Override
    public Fields getOutputFields() {
        return fields.stream()
                .collect(Collectors.reducing(new Fields(), Fields::append));
    }

    @Override
    public List<String> fromTupleEntry(TupleEntry tupleEntry) {
        return fields.stream()
                .map(tupleEntry::getString)
                .collect(Collectors.toList());
    }

    @Override
    public Tuple toTuple(List<String> value) {
        return Tuples.create(ImmutableList.copyOf(value));
    }
}

{'fraction_non_alphanumeric': 0.11535947712418301, 'fraction_numerical': 0.00065359477124183, 'mean_word_length': 3.2991573033707864, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5830644', 'n_tokens_mistral': 947, 'n_tokens_neox': 898, 'n_words': 178}