<commit_before>package org.rakam.collection;

import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import com.fasterxml.jackson.core.JsonTokenId;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JsonDeserializer;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.dataformat.csv.CsvParser;
import com.google.common.collect.ImmutableList;
import io.netty.handler.codec.http.HttpResponseStatus;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.rakam.analysis.ConfigManager;
import org.rakam.analysis.metadata.Metastore;
import org.rakam.collection.FieldDependencyBuilder.FieldDependency;
import org.rakam.util.DateTimeUtils;
import org.rakam.util.RakamException;

import javax.inject.Inject;

import java.io.IOException;
import java.time.LocalTime;
import java.time.temporal.ChronoField;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

import static com.fasterxml.jackson.core.JsonToken.VALUE_STRING;
import static java.lang.String.format;
import static java.util.stream.IntStream.range;
import static org.rakam.analysis.InternalConfig.USER_TYPE;
import static org.rakam.collection.FieldType.STRING;
import static org.rakam.collection.JsonEventDeserializer.getValueOfMagicField;
import static org.rakam.util.AvroUtil.convertAvroSchema;

public class CsvEventDeserializer
        extends JsonDeserializer<EventList>
{

    private final Metastore metastore;
    private final Set<SchemaField> constantFields;
    private final ConfigManager configManager;

    @Inject
    public CsvEventDeserializer(Metastore metastore, ConfigManager configManager,
            FieldDependency fieldDependency)
    {
        this.metastore = metastore;

        this.configManager = configManager;
        this.constantFields = fieldDependency.constantFields;
    }

    @Override
    public EventList deserialize(JsonParser jp, DeserializationContext ctxt)
            throws IOException
    {
        String project = (String) ctxt.getAttribute("project");
        String collection = (String) ctxt.getAttribute("collection");
        String apiKey = (String) ctxt.getAttribute("apiKey");

        boolean useheader = (boolean) ctxt.getAttribute("useHeader");

        Map.Entry<List<SchemaField>, int[]> header;
        if (useheader) {
            header = readHeader((CsvParser) jp, project, collection);
        }
        else {
            List<SchemaField> vall = metastore.getCollection(project, collection);
            header = new AbstractMap.SimpleImmutableEntry<>(vall, IntStream.range(0, vall.size()).toArray());
        }

        List<SchemaField> fields = header.getKey();
        int[] indexes = header.getValue();
        List<FieldType> types = Arrays.stream(indexes)
                .mapToObj(i -> header.getKey().get(i).getType()).collect(Collectors.toList());<commit_msg>make useHeader in CSV parser optional<commit_after>package org.rakam.collection;

import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import com.fasterxml.jackson.core.JsonTokenId;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JsonDeserializer;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.dataformat.csv.CsvParser;
import com.google.common.collect.ImmutableList;
import io.netty.handler.codec.http.HttpResponseStatus;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.rakam.analysis.ConfigManager;
import org.rakam.analysis.metadata.Metastore;
import org.rakam.collection.FieldDependencyBuilder.FieldDependency;
import org.rakam.util.DateTimeUtils;
import org.rakam.util.RakamException;

import javax.inject.Inject;

import java.io.IOException;
import java.time.LocalTime;
import java.time.temporal.ChronoField;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

import static com.fasterxml.jackson.core.JsonToken.VALUE_STRING;
import static java.lang.String.format;
import static java.util.stream.IntStream.range;
import static org.rakam.analysis.InternalConfig.USER_TYPE;
import static org.rakam.collection.FieldType.STRING;
import static org.rakam.collection.JsonEventDeserializer.getValueOfMagicField;
import static org.rakam.util.AvroUtil.convertAvroSchema;

public class CsvEventDeserializer
        extends JsonDeserializer<EventList>
{

    private final Metastore metastore;
    private final Set<SchemaField> constantFields;
    private final ConfigManager configManager;

    @Inject
    public CsvEventDeserializer(Metastore metastore, ConfigManager configManager,
            FieldDependency fieldDependency)
    {
        this.metastore = metastore;

        this.configManager = configManager;
        this.constantFields = fieldDependency.constantFields;
    }

    @Override
    public EventList deserialize(JsonParser jp, DeserializationContext ctxt)
            throws IOException
    {
        String project = (String) ctxt.getAttribute("project");
        String collection = (String) ctxt.getAttribute("collection");
        String apiKey = (String) ctxt.getAttribute("apiKey");

        boolean useheader = Boolean.TRUE == ctxt.getAttribute("useHeader");

        Map.Entry<List<SchemaField>, int[]> header;
        if (useheader) {
            header = readHeader((CsvParser) jp, project, collection);
        }
        else {
            List<SchemaField> vall = metastore.getCollection(project, collection);
            header = new AbstractMap.SimpleImmutableEntry<>(vall, IntStream.range(0, vall.size()).toArray());
        }

        List<SchemaField> fields = header.getKey();
        int[] indexes = header.getValue();
        List<FieldType> types = Arrays.stream(indexes)
                .mapToObj(i -> header.getKey().get(i).getType()).collect(Collectors.toList());
{'fraction_non_alphanumeric': 0.10069117185045555, 'fraction_numerical': 0.00031416902293433867, 'mean_word_length': 4.323578595317726, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4627280', 'n_tokens_mistral': 1877, 'n_tokens_neox': 1785, 'n_words': 344}