<commit_before>package org.cratedb.import_;

import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableMap;
import org.cratedb.action.import_.ImportContext;
import org.cratedb.action.import_.NodeImportRequest;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.ElasticSearchParseException;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.bulk.BulkProcessor;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexRequest.OpType;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.Requests;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.inject.Injector;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.ImmutableSettings.Builder;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParser.Token;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.VersionType;
import org.elasticsearch.index.mapper.internal.*;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.indices.IndexMissingException;

import java.io.*;
import java.util.*;
import java.util.concurrent.ExecutionException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import java.util.zip.GZIPInputStream;

public class Importer {

    private Client client;
    private final Injector injector;
    private final ClusterService clusterService;

    private final ByteSizeValue bulkByteSize = new ByteSizeValue(5, ByteSizeUnit.MB);
    private final TimeValue flushInterval = TimeValue.timeValueSeconds(5);
    private final int concurrentRequests = 4;

    private Map<Tuple<String, String>, List<String>> indexTypePrimaryKeysCache = new HashMap<>();

    @Inject
    public Importer(Injector injector, ClusterService clusterService) {
        this.injector = injector;
        this.clusterService = clusterService;
    }

    public Result execute(ImportContext context, NodeImportRequest request) {
        if (this.client == null) {
            // Inject here to avoid injection loop in constructor
            this.client = injector.getInstance(Client.class);
        }
        String index = request.index();
        String type = request.type();
        int bulkSize = request.bulkSize();
        Result result = new Result();
        Date start = new Date();

        File path = new File(context.path());
        Pattern file_pattern = context.file_pattern();
        File[] files;

        if (!path.isDirectory() && !path.isFile() && context.file_pattern() == null) {
            // try to parse last path segment as file pattern
            Tuple<String, Pattern> tuple = parsePathPattern(context.path());
            path = new File(tuple.v1());
            file_pattern = tuple.v2();
        }

        if (path.isDirectory()) {
            if (context.file_pattern() == null) {
                files = path.listFiles();
            } else {
                final Pattern file_pattern_inner = file_pattern;<commit_msg>Fix: using regex pattern on ``COPY`` did not work properly<commit_after>package org.cratedb.import_;

import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableMap;
import org.cratedb.action.import_.ImportContext;
import org.cratedb.action.import_.NodeImportRequest;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.ElasticSearchParseException;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.bulk.BulkProcessor;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexRequest.OpType;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.Requests;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.inject.Injector;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.ImmutableSettings.Builder;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParser.Token;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.VersionType;
import org.elasticsearch.index.mapper.internal.*;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.indices.IndexMissingException;

import java.io.*;
import java.util.*;
import java.util.concurrent.ExecutionException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import java.util.zip.GZIPInputStream;

public class Importer {

    private Client client;
    private final Injector injector;
    private final ClusterService clusterService;

    private final ByteSizeValue bulkByteSize = new ByteSizeValue(5, ByteSizeUnit.MB);
    private final TimeValue flushInterval = TimeValue.timeValueSeconds(5);
    private final int concurrentRequests = 4;

    private Map<Tuple<String, String>, List<String>> indexTypePrimaryKeysCache = new HashMap<>();

    @Inject
    public Importer(Injector injector, ClusterService clusterService) {
        this.injector = injector;
        this.clusterService = clusterService;
    }

    public Result execute(ImportContext context, NodeImportRequest request) {
        if (this.client == null) {
            // Inject here to avoid injection loop in constructor
            this.client = injector.getInstance(Client.class);
        }
        String index = request.index();
        String type = request.type();
        int bulkSize = request.bulkSize();
        Result result = new Result();
        Date start = new Date();

        File path = new File(context.path());
        Pattern file_pattern = context.file_pattern();
        File[] files;

        if (!path.isDirectory() && !path.isFile() && context.file_pattern() == null) {
            // try to parse last path segment as file pattern
            Tuple<String, Pattern> tuple = parsePathPattern(context.path());
            path = new File(tuple.v1());
            file_pattern = tuple.v2();
        }

        if (path.isDirectory()) {
            if (file_pattern == null) {
                files = path.listFiles();
            } else {
                final Pattern file_pattern_inner = file_pattern;
{'fraction_non_alphanumeric': 0.0973374613003096, 'fraction_numerical': 0.001238390092879257, 'mean_word_length': 4.3096646942800785, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7054712', 'n_tokens_mistral': 2089, 'n_tokens_neox': 2134, 'n_words': 444}