<commit_before>package org.codelibs.elasticsearch.runner;

import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;

import java.io.IOException;
import java.io.InputStream;
import java.net.ConnectException;
import java.net.Socket;
import java.nio.file.FileSystems;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.List;

import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.admin.indices.flush.FlushResponse;
import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse;
import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.AdminClient;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.Requests;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.logging.log4j.LogConfigurator;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.internal.InternalNode;
import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.kohsuke.args4j.CmdLineException;
import org.kohsuke.args4j.CmdLineParser;
import org.kohsuke.args4j.Option;

/**
 * ElasticsearchClusterRunner manages multiple Elasticsearch instances.
 *
 * @author shinsuke
 *
 */
public class ElasticsearchClusterRunner {
    private static final ESLogger logger = Loggers
            .getLogger(ElasticsearchClusterRunner.class);

    protected static final String LOGGING_YAML = "logging.yml";

    protected static final String ELASTICSEARCH_YAML = "elasticsearch.yml";

    protected static final String WORK_DIR = "work";

    protected static final String DATA_DIR = "data";

    protected static final String LOGS_DIR = "logs";

    protected static final String PLUGINS_DIR = "plugins";

    protected static final String CONFIG_DIR = "config";

    protected List<Node> nodeList = new ArrayList<>();

    protected int maxHttpPort = 9299;

    protected int maxTransportPort = 9399;

    @Option(name = "-basePath", usage = "Base path for Elasticsearch.")
    protected String basePath;

    @Option(name = "-numOfNode", usage = "The number of Elasticsearch node.")
    protected int numOfNode = 3;

    @Option(name = "-baseTransportPort", usage = "Base transport port.")
    protected int baseTransportPort = 9300;

    @Option(name = "-baseHttpPort", usage = "Base http port.")
    protected int baseHttpPort = 9200;

    @Option(name = "-clusterName", usage = "Cluster name.")
    protected String clusterName = "elasticsearch-cluster-runner";

    @Option(name = "-indexStoreType", usage = "Index store type.")
    protected String indexStoreType = "default";

    @Option(name = "-useStdOut", usage = "Print logs to stdout.")
    protected boolean useStdOut = true;

    @Option(name = "-throwOnFailure", usage = "Throw an exception on a failure.")
    protected boolean throwOnFailure = true;

    protected Builder builder;

    public static void main(final String[] args) {
        final ElasticsearchClusterRunner runner = new ElasticsearchClusterRunner();
        Runtime.getRuntime().addShutdownHook(new Thread() {
            @Override
            public void run() {
                runner.close();
            }
        });

        runner.build(args);

        while (true) {
            if (runner.isClosed()) {
                break;
            }
            try {
                Thread.sleep(5000);
            } catch (final InterruptedException e) {
                // no-op
            }
        }
    }

    public ElasticsearchClusterRunner() {
    }

    public boolean isClosed() {
        for (final Node node : nodeList) {
            if (!node.isClosed()) {
                return false;
            }
        }
        return true;
    }

    public void close() {
        for (final Node node : nodeList) {
            node.close();
        }
        print("Closed all nodes.");
    }

    public void clean() {
        final Path bPath = FileSystems.getDefault().getPath(basePath);
        try {
            Files.walkFileTree(bPath, new FileVisitor<Path>() {
                @Override
                public FileVisitResult preVisitDirectory(final Path dir,
                        final BasicFileAttributes attrs) throws IOException {
                    return FileVisitResult.CONTINUE;
                }

                @Override
                public FileVisitResult visitFile(final Path file,
                        final BasicFileAttributes attrs) throws IOException {
                    Files.delete(file);
                    return checkIfExist(file);
                }

                @Override
                public FileVisitResult visitFileFailed(final Path file,
                        final IOException exc) throws IOException {
                    throw exc;
                }

                @Override
                public FileVisitResult postVisitDirectory(final Path dir,
                        final IOException exc) throws IOException {
                    if (exc == null) {
                        Files.delete(dir);
                        if (!Files.exists(dir)) {
                            return FileVisitResult.CONTINUE;
                        } else {
                            throw new IOException();
                        }
                    } else {
                        throw exc;
                    }
                }

                private FileVisitResult checkIfExist(final Path path)
                        throws IOException {
                    if (Files.exists(path)) {
                        throw new IOException("Failed to delete " + path);
                    }
                    return FileVisitResult.CONTINUE;
                }
            });
            print("Deleted " + basePath);
        } catch (final IOException e) {
            print("Failed to delete " + basePath);
        }
    }

    public ElasticsearchClusterRunner onBuild(Builder builder) {
        this.builder = builder;
        return this;
    }

    public void build(final String... args) {
        if (args != null) {
            final CmdLineParser parser = new CmdLineParser(this);
            parser.setUsageWidth(80);

            try {
                parser.parseArgument(args);
            } catch (final CmdLineException e) {
                throw new ClusterRunnerException("Failed to parse args: "
                        + Strings.arrayToDelimitedString(args, " "));
            }
        }

        if (basePath == null) {
            try {
                basePath = Files.createTempDirectory("es-cluster")
                        .toAbsolutePath().toString();
            } catch (final IOException e) {
                throw new ClusterRunnerException("Could not create $ES_HOME.",
                        e);
            }
        }

        final Path esBasePath = Paths.get(basePath);
        createDir(esBasePath);

        final Path confPath = Paths.get(basePath, CONFIG_DIR);
        createDir(confPath);

        final Path esConfPath = confPath.resolve(ELASTICSEARCH_YAML);
        if (!Files.exists(esConfPath)) {
            try (InputStream is = Thread.currentThread()
                    .getContextClassLoader()
                    .getResourceAsStream(CONFIG_DIR + "/" + ELASTICSEARCH_YAML)) {
                Files.copy(is, esConfPath, StandardCopyOption.REPLACE_EXISTING);
            } catch (final IOException e) {
                throw new ClusterRunnerException("Could not create: "
                        + esConfPath, e);
            }
        }

        final Path logConfPath = confPath.resolve(LOGGING_YAML);
        if (!Files.exists(logConfPath)) {
            try (InputStream is = Thread.currentThread()
                    .getContextClassLoader()
                    .getResourceAsStream(CONFIG_DIR + "/" + LOGGING_YAML)) {
                Files.copy(is, logConfPath, StandardCopyOption.REPLACE_EXISTING);
            } catch (final IOException e) {
                throw new ClusterRunnerException("Could not create: "
                        + logConfPath, e);
            }
        }

        print("----------------------------------------");
        print("Cluster Name: " + clusterName);
        print("Base Path:    " + basePath);
        print("Num Of Node:  " + numOfNode);
        print("----------------------------------------");

        for (int i = 0; i < numOfNode; i++) {
            nodeList.add(buildNode(i + 1));
        }
    }

    protected Node buildNode(final int number) {
        final Path confPath = Paths.get(basePath, CONFIG_DIR);
        final Path pluginsPath = Paths.get(basePath, PLUGINS_DIR);
        final Path logsPath = Paths.get(basePath, LOGS_DIR, "node_" + number);
        final Path dataPath = Paths.get(basePath, DATA_DIR, "node_" + number);
        final Path workPath = Paths.get(basePath, WORK_DIR, "node_" + number);

        createDir(logsPath);
        createDir(dataPath);
        createDir(workPath);

        final ImmutableSettings.Builder settingsBuilder = settingsBuilder();

        if (builder != null) {
            builder.build(number, settingsBuilder);
        }

        putIfAbsent(settingsBuilder, "path.conf", confPath.toAbsolutePath()
                .toString());
        putIfAbsent(settingsBuilder, "path.data", dataPath.toAbsolutePath()
                .toString());
        putIfAbsent(settingsBuilder, "path.work", workPath.toAbsolutePath()
                .toString());
        putIfAbsent(settingsBuilder, "path.logs", logsPath.toAbsolutePath()
                .toString());
        putIfAbsent(settingsBuilder, "path.plugins", pluginsPath
                .toAbsolutePath().toString());

        final String nodeName = "Node " + number;
        final int transportPort = getTransportPort(number);
        final int httpPort = getHttpPort(number);
        putIfAbsent(settingsBuilder, "cluster.name", clusterName);
        putIfAbsent(settingsBuilder, "node.name", nodeName);
        putIfAbsent(settingsBuilder, "node.master", String.valueOf(true));
        putIfAbsent(settingsBuilder, "node.data", String.valueOf(true));
        putIfAbsent(settingsBuilder, "http.enabled", String.valueOf(true));
        putIfAbsent(settingsBuilder, "transport.tcp.port",
                String.valueOf(transportPort));
        putIfAbsent(settingsBuilder, "http.port", String.valueOf(httpPort));
        putIfAbsent(settingsBuilder, "index.store.type", indexStoreType);

        print("Node Name:      " + nodeName);
        print("HTTP Port:      " + httpPort);
        print("Transport Port: " + transportPort);
        print("Data Directory: " + dataPath);
        print("Log Directory:  " + logsPath);
        print("----------------------------------------");

        final Settings settings = settingsBuilder.build();
        LogConfigurator.configure(settings);
        final Node node = new InternalNode(settings, true);
        node.start();
        return node;
    }

    private int getHttpPort(final int number) {
        int httpPort = baseHttpPort + number;
        if (maxHttpPort < 0) {
            return httpPort;
        }
        while (httpPort <= maxHttpPort) {
            try (Socket socket = new Socket("localhost", httpPort)) {
                httpPort++;
            } catch (ConnectException e) {
                return httpPort;
            } catch (IOException e) {
                print(e.getMessage());
                httpPort++;
            }
        }
        throw new ClusterRunnerException("The http port " + httpPort
                + " is unavailable.");
    }

    protected int getTransportPort(final int number) {
        int transportPort = baseTransportPort + number;
        if (maxTransportPort < 0) {
            return transportPort;
        }
        while (transportPort <= maxTransportPort) {
            try (Socket socket = new Socket("localhost", transportPort)) {
                transportPort++;
            } catch (ConnectException e) {
                return transportPort;
            } catch (IOException e) {
                print(e.getMessage());
                transportPort++;
            }
        }
        throw new ClusterRunnerException("The transport port " + transportPort
                + " is unavailable.");
    }

    protected void putIfAbsent(final ImmutableSettings.Builder settingsBuilder,
            final String key, final String value) {
        if (settingsBuilder.get(key) == null) {
            settingsBuilder.put(key, value);
        }
    }

    public void setMaxHttpPort(int maxHttpPort) {<commit_msg>null check<commit_after>package org.codelibs.elasticsearch.runner;

import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;

import java.io.IOException;
import java.io.InputStream;
import java.net.ConnectException;
import java.net.Socket;
import java.nio.file.FileSystems;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.List;

import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.admin.indices.flush.FlushResponse;
import org.elasticsearch.action.admin.indices.optimize.OptimizeResponse;
import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.AdminClient;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.Requests;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.logging.log4j.LogConfigurator;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.internal.InternalNode;
import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.kohsuke.args4j.CmdLineException;
import org.kohsuke.args4j.CmdLineParser;
import org.kohsuke.args4j.Option;

/**
 * ElasticsearchClusterRunner manages multiple Elasticsearch instances.
 *
 * @author shinsuke
 *
 */
public class ElasticsearchClusterRunner {
    private static final ESLogger logger = Loggers
            .getLogger(ElasticsearchClusterRunner.class);

    protected static final String LOGGING_YAML = "logging.yml";

    protected static final String ELASTICSEARCH_YAML = "elasticsearch.yml";

    protected static final String WORK_DIR = "work";

    protected static final String DATA_DIR = "data";

    protected static final String LOGS_DIR = "logs";

    protected static final String PLUGINS_DIR = "plugins";

    protected static final String CONFIG_DIR = "config";

    protected List<Node> nodeList = new ArrayList<>();

    protected int maxHttpPort = 9299;

    protected int maxTransportPort = 9399;

    @Option(name = "-basePath", usage = "Base path for Elasticsearch.")
    protected String basePath;

    @Option(name = "-numOfNode", usage = "The number of Elasticsearch node.")
    protected int numOfNode = 3;

    @Option(name = "-baseTransportPort", usage = "Base transport port.")
    protected int baseTransportPort = 9300;

    @Option(name = "-baseHttpPort", usage = "Base http port.")
    protected int baseHttpPort = 9200;

    @Option(name = "-clusterName", usage = "Cluster name.")
    protected String clusterName = "elasticsearch-cluster-runner";

    @Option(name = "-indexStoreType", usage = "Index store type.")
    protected String indexStoreType = "default";

    @Option(name = "-useStdOut", usage = "Print logs to stdout.")
    protected boolean useStdOut = true;

    @Option(name = "-throwOnFailure", usage = "Throw an exception on a failure.")
    protected boolean throwOnFailure = true;

    protected Builder builder;

    public static void main(final String[] args) {
        final ElasticsearchClusterRunner runner = new ElasticsearchClusterRunner();
        Runtime.getRuntime().addShutdownHook(new Thread() {
            @Override
            public void run() {
                runner.close();
            }
        });

        runner.build(args);

        while (true) {
            if (runner.isClosed()) {
                break;
            }
            try {
                Thread.sleep(5000);
            } catch (final InterruptedException e) {
                // no-op
            }
        }
    }

    public ElasticsearchClusterRunner() {
    }

    public boolean isClosed() {
        for (final Node node : nodeList) {
            if (!node.isClosed()) {
                return false;
            }
        }
        return true;
    }

    public void close() {
        for (final Node node : nodeList) {
            node.close();
        }
        print("Closed all nodes.");
    }

    public void clean() {
        final Path bPath = FileSystems.getDefault().getPath(basePath);
        try {
            Files.walkFileTree(bPath, new FileVisitor<Path>() {
                @Override
                public FileVisitResult preVisitDirectory(final Path dir,
                        final BasicFileAttributes attrs) throws IOException {
                    return FileVisitResult.CONTINUE;
                }

                @Override
                public FileVisitResult visitFile(final Path file,
                        final BasicFileAttributes attrs) throws IOException {
                    Files.delete(file);
                    return checkIfExist(file);
                }

                @Override
                public FileVisitResult visitFileFailed(final Path file,
                        final IOException exc) throws IOException {
                    throw exc;
                }

                @Override
                public FileVisitResult postVisitDirectory(final Path dir,
                        final IOException exc) throws IOException {
                    if (exc == null) {
                        Files.delete(dir);
                        if (!Files.exists(dir)) {
                            return FileVisitResult.CONTINUE;
                        } else {
                            throw new IOException();
                        }
                    } else {
                        throw exc;
                    }
                }

                private FileVisitResult checkIfExist(final Path path)
                        throws IOException {
                    if (Files.exists(path)) {
                        throw new IOException("Failed to delete " + path);
                    }
                    return FileVisitResult.CONTINUE;
                }
            });
            print("Deleted " + basePath);
        } catch (final IOException e) {
            print("Failed to delete " + basePath);
        }
    }

    public ElasticsearchClusterRunner onBuild(Builder builder) {
        this.builder = builder;
        return this;
    }

    public void build(final String... args) {
        if (args != null) {
            final CmdLineParser parser = new CmdLineParser(this);
            parser.setUsageWidth(80);

            try {
                parser.parseArgument(args);
            } catch (final CmdLineException e) {
                throw new ClusterRunnerException("Failed to parse args: "
                        + Strings.arrayToDelimitedString(args, " "));
            }
        }

        if (basePath == null) {
            try {
                basePath = Files.createTempDirectory("es-cluster")
                        .toAbsolutePath().toString();
            } catch (final IOException e) {
                throw new ClusterRunnerException("Could not create $ES_HOME.",
                        e);
            }
        }

        final Path esBasePath = Paths.get(basePath);
        createDir(esBasePath);

        final Path confPath = Paths.get(basePath, CONFIG_DIR);
        createDir(confPath);

        final Path esConfPath = confPath.resolve(ELASTICSEARCH_YAML);
        if (!Files.exists(esConfPath)) {
            try (InputStream is = Thread.currentThread()
                    .getContextClassLoader()
                    .getResourceAsStream(CONFIG_DIR + "/" + ELASTICSEARCH_YAML)) {
                Files.copy(is, esConfPath, StandardCopyOption.REPLACE_EXISTING);
            } catch (final IOException e) {
                throw new ClusterRunnerException("Could not create: "
                        + esConfPath, e);
            }
        }

        final Path logConfPath = confPath.resolve(LOGGING_YAML);
        if (!Files.exists(logConfPath)) {
            try (InputStream is = Thread.currentThread()
                    .getContextClassLoader()
                    .getResourceAsStream(CONFIG_DIR + "/" + LOGGING_YAML)) {
                Files.copy(is, logConfPath, StandardCopyOption.REPLACE_EXISTING);
            } catch (final IOException e) {
                throw new ClusterRunnerException("Could not create: "
                        + logConfPath, e);
            }
        }

        print("----------------------------------------");
        print("Cluster Name: " + clusterName);
        print("Base Path:    " + basePath);
        print("Num Of Node:  " + numOfNode);
        print("----------------------------------------");

        for (int i = 0; i < numOfNode; i++) {
            nodeList.add(buildNode(i + 1));
        }
    }

    protected Node buildNode(final int number) {
        final Path confPath = Paths.get(basePath, CONFIG_DIR);
        final Path pluginsPath = Paths.get(basePath, PLUGINS_DIR);
        final Path logsPath = Paths.get(basePath, LOGS_DIR, "node_" + number);
        final Path dataPath = Paths.get(basePath, DATA_DIR, "node_" + number);
        final Path workPath = Paths.get(basePath, WORK_DIR, "node_" + number);

        createDir(logsPath);
        createDir(dataPath);
        createDir(workPath);

        final ImmutableSettings.Builder settingsBuilder = settingsBuilder();

        if (builder != null) {
            builder.build(number, settingsBuilder);
        }

        putIfAbsent(settingsBuilder, "path.conf", confPath.toAbsolutePath()
                .toString());
        putIfAbsent(settingsBuilder, "path.data", dataPath.toAbsolutePath()
                .toString());
        putIfAbsent(settingsBuilder, "path.work", workPath.toAbsolutePath()
                .toString());
        putIfAbsent(settingsBuilder, "path.logs", logsPath.toAbsolutePath()
                .toString());
        putIfAbsent(settingsBuilder, "path.plugins", pluginsPath
                .toAbsolutePath().toString());

        final String nodeName = "Node " + number;
        final int transportPort = getTransportPort(number);
        final int httpPort = getHttpPort(number);
        putIfAbsent(settingsBuilder, "cluster.name", clusterName);
        putIfAbsent(settingsBuilder, "node.name", nodeName);
        putIfAbsent(settingsBuilder, "node.master", String.valueOf(true));
        putIfAbsent(settingsBuilder, "node.data", String.valueOf(true));
        putIfAbsent(settingsBuilder, "http.enabled", String.valueOf(true));
        putIfAbsent(settingsBuilder, "transport.tcp.port",
                String.valueOf(transportPort));
        putIfAbsent(settingsBuilder, "http.port", String.valueOf(httpPort));
        putIfAbsent(settingsBuilder, "index.store.type", indexStoreType);

        print("Node Name:      " + nodeName);
        print("HTTP Port:      " + httpPort);
        print("Transport Port: " + transportPort);
        print("Data Directory: " + dataPath);
        print("Log Directory:  " + logsPath);
        print("----------------------------------------");

        final Settings settings = settingsBuilder.build();
        LogConfigurator.configure(settings);
        final Node node = new InternalNode(settings, true);
        node.start();
        return node;
    }

    private int getHttpPort(final int number) {
        int httpPort = baseHttpPort + number;
        if (maxHttpPort < 0) {
            return httpPort;
        }
        while (httpPort <= maxHttpPort) {
            try (Socket socket = new Socket("localhost", httpPort)) {
                httpPort++;
            } catch (ConnectException e) {
                return httpPort;
            } catch (IOException e) {
                print(e.getMessage());
                httpPort++;
            }
        }
        throw new ClusterRunnerException("The http port " + httpPort
                + " is unavailable.");
    }

    protected int getTransportPort(final int number) {
        int transportPort = baseTransportPort + number;
        if (maxTransportPort < 0) {
            return transportPort;
        }
        while (transportPort <= maxTransportPort) {
            try (Socket socket = new Socket("localhost", transportPort)) {
                transportPort++;
            } catch (ConnectException e) {
                return transportPort;
            } catch (IOException e) {
                print(e.getMessage());
                transportPort++;
            }
        }
        throw new ClusterRunnerException("The transport port " + transportPort
                + " is unavailable.");
    }

    protected void putIfAbsent(final ImmutableSettings.Builder settingsBuilder,
            final String key, final String value) {
        if (settingsBuilder.get(key) == null && value != null) {
            settingsBuilder.put(key, value);
        }
    }

    public void setMaxHttpPort(int maxHttpPort) {
{'fraction_non_alphanumeric': 0.10458504282732312, 'fraction_numerical': 0.002231339523501044, 'mean_word_length': 2.1369383608037933, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2772922', 'n_tokens_mistral': 7136, 'n_tokens_neox': 6877, 'n_words': 1607}