<commit_before>package com.github.tkurz.sesame.vocab;

import org.apache.commons.cli.*;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.ResponseHandler;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.openrdf.model.util.GraphUtilException;
import org.openrdf.rio.RDFFormat;
import org.openrdf.rio.RDFParseException;
import org.openrdf.rio.RDFParserRegistry;
import org.openrdf.rio.Rio;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Iterator;
import java.util.Properties;
import java.util.Set;

/**
 * ...
 * <p/>
 * @author  <NAME> (<EMAIL>)
 * @author  <NAME> (<EMAIL>)
 */
public class Main {

    public static void main(String [] args) {
        try {
            CommandLineParser parser = new PosixParser();
            CommandLine cli = parser.parse(getCliOpts(), args);

            if (cli.hasOption('h')) {
                printHelp();
                return;
            }

            // two args must be left over: <input-inputFile> <output-inputFile>
            String[] cliArgs = cli.getArgs();
            final String input, output;
            switch (cliArgs.length) {
                case 0:
                    throw new ParseException("Missing input-file");
                case 1:
                    input = cliArgs[0];
                    output = null;
                    break;
                case 2:
                    input = cliArgs[0];
                    output = cliArgs[1];
                    break;
                default:
                    throw new ParseException("too many arguments");
            }

            RDFFormat format = Rio.getParserFormatForMIMEType(cli.getOptionValue('f', null));

            Path tempFile = null;
            final VocabBuilder builder;
            if (input.startsWith("http://")) {
                tempFile = Files.createTempFile("vocab-builder", "."+(format!=null?format.getDefaultFileExtension():"cache"));
                URL url = new URL(input);

                try {
                    fetchVocab(url, tempFile);
                } catch (URISyntaxException e) {
                    throw new ParseException("Invalid input URL: " +e.getMessage());
                }

                builder = new VocabBuilder(tempFile.toString(), format);
            } else
                builder = new VocabBuilder(input, format);

            if (cli.hasOption('p')) {
                builder.setPackageName(cli.getOptionValue('p'));
            }
            if (cli.hasOption('n')) {
                builder.setName(cli.getOptionValue('n'));
            }
            if (cli.hasOption('u')) {
                builder.setPrefix(cli.getOptionValue('u'));
            }
            if (cli.hasOption('l')) {
                builder.setPreferredLanguage(cli.getOptionValue('l'));
            }
            if (cli.hasOption('s')) {
                try {
                    builder.setIndent(StringUtils.repeat(' ', Integer.parseInt(cli.getOptionValue('s', "4"))));
                } catch (NumberFormatException e) {
                    throw new ParseException("indent must be numeric");
                }
            } else {
                builder.setIndent("\t");
            }

            if (output != null) {
                System.out.printf("Starting generation%n");
                Path outFile = Paths.get(output);
                builder.generate(outFile);
                System.out.printf("Generation finished, result available in '%s'%n", output);
            } else {
                builder.generate(System.out);
            }

            if (tempFile != null) {
                Files.deleteIfExists(tempFile);
            }
        } catch (ParseException e) {
            printHelp(e.getMessage());
        } catch (RDFParseException e) {
            System.err.println("Could not parse input file: " + e.getMessage());
        } catch (FileNotFoundException e) {
            System.err.println("Could not read input-file: " + e.getMessage());
        } catch (IOException e) {
            System.err.println("Error during file-access: " + e.getMessage());
        } catch (GraphUtilException e) {
            e.printStackTrace();
        } catch (GenerationException e) {
            System.err.printf(e.getMessage());
        }
    }

    private static void printHelp() {
        printHelp(null);
    }

    private static void printHelp(String error) {
        HelpFormatter hf = new HelpFormatter();
        PrintWriter w = new PrintWriter(System.out);
        if (error != null) {
            hf.printWrapped(w, 80, error);<commit_msg>Improved error message on unknown formats<commit_after>package com.github.tkurz.sesame.vocab;

import org.apache.commons.cli.*;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.ResponseHandler;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.openrdf.model.util.GraphUtilException;
import org.openrdf.rio.RDFFormat;
import org.openrdf.rio.RDFParseException;
import org.openrdf.rio.RDFParserRegistry;
import org.openrdf.rio.Rio;
import org.openrdf.rio.UnsupportedRDFormatException;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Iterator;
import java.util.Properties;
import java.util.Set;

/**
 * ...
 * <p/>
 * @author  <NAME> (<EMAIL>)
 * @author  <NAME> (<EMAIL>)
 */
public class Main {

    public static void main(String [] args) {
        try {
            CommandLineParser parser = new PosixParser();
            CommandLine cli = parser.parse(getCliOpts(), args);

            if (cli.hasOption('h')) {
                printHelp();
                return;
            }

            // two args must be left over: <input-inputFile> <output-inputFile>
            String[] cliArgs = cli.getArgs();
            final String input, output;
            switch (cliArgs.length) {
                case 0:
                    throw new ParseException("Missing input-file");
                case 1:
                    input = cliArgs[0];
                    output = null;
                    break;
                case 2:
                    input = cliArgs[0];
                    output = cliArgs[1];
                    break;
                default:
                    throw new ParseException("too many arguments");
            }

            RDFFormat format = Rio.getParserFormatForMIMEType(cli.getOptionValue('f', null));

            Path tempFile = null;
            final VocabBuilder builder;
            if (input.startsWith("http://")) {
                tempFile = Files.createTempFile("vocab-builder", "."+(format!=null?format.getDefaultFileExtension():"cache"));
                URL url = new URL(input);

                try {
                    fetchVocab(url, tempFile);
                } catch (URISyntaxException e) {
                    throw new ParseException("Invalid input URL: " +e.getMessage());
                }

                builder = new VocabBuilder(tempFile.toString(), format);
            } else
                builder = new VocabBuilder(input, format);

            if (cli.hasOption('p')) {
                builder.setPackageName(cli.getOptionValue('p'));
            }
            if (cli.hasOption('n')) {
                builder.setName(cli.getOptionValue('n'));
            }
            if (cli.hasOption('u')) {
                builder.setPrefix(cli.getOptionValue('u'));
            }
            if (cli.hasOption('l')) {
                builder.setPreferredLanguage(cli.getOptionValue('l'));
            }
            if (cli.hasOption('s')) {
                try {
                    builder.setIndent(StringUtils.repeat(' ', Integer.parseInt(cli.getOptionValue('s', "4"))));
                } catch (NumberFormatException e) {
                    throw new ParseException("indent must be numeric");
                }
            } else {
                builder.setIndent("\t");
            }

            if (output != null) {
                System.out.printf("Starting generation%n");
                Path outFile = Paths.get(output);
                builder.generate(outFile);
                System.out.printf("Generation finished, result available in '%s'%n", output);
            } else {
                builder.generate(System.out);
            }

            if (tempFile != null) {
                Files.deleteIfExists(tempFile);
            }
        } catch (UnsupportedRDFormatException e) {
            System.err.printf("%s%nTry setting the format explicitly%n", e.getMessage());
        } catch (ParseException e) {
            printHelp(e.getMessage());
        } catch (RDFParseException e) {
            System.err.println("Could not parse input file: " + e.getMessage());
        } catch (FileNotFoundException e) {
            System.err.println("Could not read input-file: " + e.getMessage());
        } catch (IOException e) {
            System.err.println("Error during file-access: " + e.getMessage());
        } catch (GraphUtilException e) {
            e.printStackTrace();
        } catch (GenerationException e) {
            System.err.println(e.getMessage());
        }
    }

    private static void printHelp() {
        printHelp(null);
    }

    private static void printHelp(String error) {
        HelpFormatter hf = new HelpFormatter();
        PrintWriter w = new PrintWriter(System.out);
        if (error != null) {
            hf.printWrapped(w, 80, error);
{'fraction_non_alphanumeric': 0.1191586561340059, 'fraction_numerical': 0.0019034929094889123, 'mean_word_length': 1.9035645205857972, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7252783', 'n_tokens_mistral': 2890, 'n_tokens_neox': 2791, 'n_words': 575}