<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.tika.cli;

import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.PrintStream;
import java.net.URL;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;

import javax.xml.transform.OutputKeys;
import javax.xml.transform.TransformerConfigurationException;
import javax.xml.transform.sax.SAXTransformerFactory;
import javax.xml.transform.sax.TransformerHandler;
import javax.xml.transform.stream.StreamResult;

import org.apache.log4j.BasicConfigurator;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.SimpleLayout;
import org.apache.log4j.WriterAppender;
import org.apache.tika.gui.TikaGUI;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.Parser;
import org.apache.tika.sax.BodyContentHandler;
import org.xml.sax.ContentHandler;
import org.xml.sax.helpers.DefaultHandler;

/**
 * Simple command line interface for Apache Tika.
 */
public class TikaCLI {

    public static void main(String[] args) throws Exception {
        BasicConfigurator.configure(
                new WriterAppender(new SimpleLayout(), System.err));
        Logger.getRootLogger().setLevel(Level.INFO);

        TikaCLI cli = new TikaCLI();
        for (int i = 0; i < args.length; i++) {
            cli.process(args[i]);
        }
        if (cli.pipeMode) {
            cli.process("-");
        }
    }

    private Map<String, Object> context;

    private Parser parser;

    private Metadata metadata;

    private ContentHandler handler;

    private boolean pipeMode = true;

    public TikaCLI() throws TransformerConfigurationException {
        context = new HashMap<String, Object>();
        parser = new AutoDetectParser();
        context.put(Parser.class.getName(), parser);
        handler = getXmlContentHandler();
    }

    public void process(String arg) throws Exception {
        if (arg.equals("-?") || arg.equals("--help")) {
            pipeMode = false;
            usage();
        } else if (arg.equals("-v") || arg.equals("--verbose")) {
            Logger.getRootLogger().setLevel(Level.DEBUG);
        } else if (arg.equals("-g") || arg.equals("--gui")) {
            pipeMode = false;
            TikaGUI.main(new String[0]);
        } else if (arg.equals("-x") || arg.equals("--xml")) {
            handler = getXmlContentHandler();
        } else if (arg.equals("-h") || arg.equals("--html")) {
            handler = getHtmlContentHandler();
        } else if (arg.equals("-t") || arg.equals("--text")) {
            handler = getTextContentHandler();
        } else if (arg.equals("-m") || arg.equals("--metadata")) {
            handler = getMetadataContentHandler();
        } else {
            pipeMode = false;
            metadata = new Metadata();
            if (arg.equals("-")) {
                parser.parse(System.in, handler, metadata, context);
            } else {
                InputStream input;
                File file = new File(arg);
                if (file.isFile()) {
                    metadata.set(Metadata.RESOURCE_NAME_KEY, file.getName());
                    input = new FileInputStream(file);
                } else {
                    URL url = new URL(arg);
                    String path = url.getPath();
                    int slash = path.lastIndexOf('/');
                    String name = path.substring(slash + 1);
                    if (name.length() > 0) {
                        metadata.set(Metadata.RESOURCE_NAME_KEY, name);
                    }
                    input = url.openStream();
                }
                try {
                    parser.parse(input, handler, metadata, context);
                } finally {
                    input.close();
                }
            }
        }
    }

    private void usage() {
        PrintStream out = System.out;
        out.println("usage: tika [option] [file]");
        out.println();
        out.println("Options:");
        out.println("    -? or --help       Print this usage message");
        out.println("    -v or --verbose    Print debug level messages");
        out.println("    -g or --gui        Start the Apache Tika GUI");
        out.println("    -x or --xml        Output XHTML content (default)");
        out.println("    -h or --html       Output HTML content");
        out.println("    -t or --text       Output plain text content");
        out.println("    -m or --metadata   Output only metadata");
        out.println();
        out.println("Description:");
        out.println("    Apache Tika will parse the file(s) specified on the");
        out.println("    command line and output the extracted text content");
        out.println("    or metadata to standard output.");
        out.println();
        out.println("    Instead of a file name you can also specify the URL");
        out.println("    of a document to be parsed.");
        out.println();
        out.println("    If no file name or URL is specified (or the special");
        out.println("    name \"-\" is used), then the standard input stream");
        out.println("    is parsed.");
        out.println();
        out.println("    Use the \"--gui\" (or \"-g\") option to start");
        out.println("    the Apache Tika GUI. You can drag and drop files");
        out.println("    from a normal file explorer to the GUI window to");
        out.println("    extract text content and metadata from the files.");
    }

    private ContentHandler getXmlContentHandler()
            throws TransformerConfigurationException {
        SAXTransformerFactory factory = (SAXTransformerFactory)
            SAXTransformerFactory.newInstance();
        TransformerHandler handler = factory.newTransformerHandler();
        handler.getTransformer().setOutputProperty(OutputKeys.METHOD, "xml");
        handler.getTransformer().setOutputProperty(OutputKeys.INDENT, "yes");
        handler.setResult(new StreamResult(System.out));
        return handler;
    }

    private ContentHandler getHtmlContentHandler()
            throws TransformerConfigurationException {
        SAXTransformerFactory factory = (SAXTransformerFactory)
        SAXTransformerFactory.newInstance();
        TransformerHandler handler = factory.newTransformerHandler();
        handler.getTransformer().setOutputProperty(OutputKeys.METHOD, "html");
        handler.getTransformer().setOutputProperty(OutputKeys.INDENT, "yes");
        handler.setResult(new StreamResult(System.out));
        return handler;
    }

    private ContentHandler getTextContentHandler() {
        return new BodyContentHandler(System.out);
    }

    private ContentHandler getMetadataContentHandler() {
        return new DefaultHandler() {
            public void endDocument() {
                String[] names = metadata.names();
                Arrays.sort(names);
                for (String name : names) {
                    System.out.println(name + ": " + metadata.get(name));
                }
            }
        };
    }

}<commit_msg>TIKA-277: Tika stand alone CLI --possibility to specify output encoding (--text)<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.tika.cli;

import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.net.URL;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;

import javax.xml.transform.OutputKeys;
import javax.xml.transform.TransformerConfigurationException;
import javax.xml.transform.sax.SAXTransformerFactory;
import javax.xml.transform.sax.TransformerHandler;
import javax.xml.transform.stream.StreamResult;

import org.apache.log4j.BasicConfigurator;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.SimpleLayout;
import org.apache.log4j.WriterAppender;
import org.apache.tika.gui.TikaGUI;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.Parser;
import org.apache.tika.sax.BodyContentHandler;
import org.xml.sax.ContentHandler;
import org.xml.sax.helpers.DefaultHandler;

/**
 * Simple command line interface for Apache Tika.
 */
public class TikaCLI {

    public static void main(String[] args) throws Exception {
        BasicConfigurator.configure(
                new WriterAppender(new SimpleLayout(), System.err));
        Logger.getRootLogger().setLevel(Level.INFO);

        TikaCLI cli = new TikaCLI();
        for (int i = 0; i < args.length; i++) {
            cli.process(args[i]);
        }
        if (cli.pipeMode) {
            cli.process("-");
        }
    }

    private Map<String, Object> context;

    private Parser parser;

    private Metadata metadata;

    private ContentHandler handler;

    /**
     * Output character encoding, or <code>null</code> for platform default
     */
    private String encoding = null;

    private boolean pipeMode = true;

    public TikaCLI() throws TransformerConfigurationException {
        context = new HashMap<String, Object>();
        parser = new AutoDetectParser();
        context.put(Parser.class.getName(), parser);
        handler = getXmlContentHandler();
    }

    public void process(String arg) throws Exception {
        if (arg.equals("-?") || arg.equals("--help")) {
            pipeMode = false;
            usage();
        } else if (arg.equals("-v") || arg.equals("--verbose")) {
            Logger.getRootLogger().setLevel(Level.DEBUG);
        } else if (arg.equals("-g") || arg.equals("--gui")) {
            pipeMode = false;
            TikaGUI.main(new String[0]);
        } else if (arg.startsWith("-e")) {
            encoding = arg.substring("-e".length());
        } else if (arg.startsWith("--encoding=")) {
            encoding = arg.substring("--encoding=".length());
        } else if (arg.equals("-x") || arg.equals("--xml")) {
            handler = getXmlContentHandler();
        } else if (arg.equals("-h") || arg.equals("--html")) {
            handler = getHtmlContentHandler();
        } else if (arg.equals("-t") || arg.equals("--text")) {
            handler = getTextContentHandler();
        } else if (arg.equals("-m") || arg.equals("--metadata")) {
            handler = getMetadataContentHandler();
        } else {
            pipeMode = false;
            metadata = new Metadata();
            if (arg.equals("-")) {
                parser.parse(System.in, handler, metadata, context);
            } else {
                InputStream input;
                File file = new File(arg);
                if (file.isFile()) {
                    metadata.set(Metadata.RESOURCE_NAME_KEY, file.getName());
                    input = new FileInputStream(file);
                } else {
                    URL url = new URL(arg);
                    String path = url.getPath();
                    int slash = path.lastIndexOf('/');
                    String name = path.substring(slash + 1);
                    if (name.length() > 0) {
                        metadata.set(Metadata.RESOURCE_NAME_KEY, name);
                    }
                    input = url.openStream();
                }
                try {
                    parser.parse(input, handler, metadata, context);
                } finally {
                    input.close();
                }
            }
        }
    }

    private void usage() {
        PrintStream out = System.out;
        out.println("usage: tika [option] [file]");
        out.println();
        out.println("Options:");
        out.println("    -?  or --help        Print this usage message");
        out.println("    -v  or --verbose     Print debug level messages");
        out.println("    -g  or --gui         Start the Apache Tika GUI");
        out.println("    -eX or --encoding=X  Use output encoding X");
        out.println("    -x  or --xml         Output XHTML content (default)");
        out.println("    -h  or --html        Output HTML content");
        out.println("    -t  or --text        Output plain text content");
        out.println("    -m  or --metadata    Output only metadata");
        out.println();
        out.println("Description:");
        out.println("    Apache Tika will parse the file(s) specified on the");
        out.println("    command line and output the extracted text content");
        out.println("    or metadata to standard output.");
        out.println();
        out.println("    Instead of a file name you can also specify the URL");
        out.println("    of a document to be parsed.");
        out.println();
        out.println("    If no file name or URL is specified (or the special");
        out.println("    name \"-\" is used), then the standard input stream");
        out.println("    is parsed.");
        out.println();
        out.println("    Use the \"--gui\" (or \"-g\") option to start");
        out.println("    the Apache Tika GUI. You can drag and drop files");
        out.println("    from a normal file explorer to the GUI window to");
        out.println("    extract text content and metadata from the files.");
    }

    private ContentHandler getXmlContentHandler()
            throws TransformerConfigurationException {
        SAXTransformerFactory factory = (SAXTransformerFactory)
            SAXTransformerFactory.newInstance();
        TransformerHandler handler = factory.newTransformerHandler();
        handler.getTransformer().setOutputProperty(OutputKeys.METHOD, "xml");
        handler.getTransformer().setOutputProperty(OutputKeys.INDENT, "yes");
        if (encoding != null) {
            handler.getTransformer().setOutputProperty(
                    OutputKeys.ENCODING, encoding);
        }
        handler.setResult(new StreamResult(System.out));
        return handler;
    }

    private ContentHandler getHtmlContentHandler()
            throws TransformerConfigurationException {
        SAXTransformerFactory factory = (SAXTransformerFactory)
        SAXTransformerFactory.newInstance();
        TransformerHandler handler = factory.newTransformerHandler();
        handler.getTransformer().setOutputProperty(OutputKeys.METHOD, "html");
        handler.getTransformer().setOutputProperty(OutputKeys.INDENT, "yes");
        if (encoding != null) {
            handler.getTransformer().setOutputProperty(
                    OutputKeys.ENCODING, encoding);
        }
        handler.setResult(new StreamResult(System.out));
        return handler;
    }

    private ContentHandler getTextContentHandler()
            throws UnsupportedEncodingException {
        return new BodyContentHandler(getSystemOutWriter());
    }

    private ContentHandler getMetadataContentHandler()
            throws UnsupportedEncodingException {
        final PrintWriter writer = new PrintWriter(getSystemOutWriter());
        return new DefaultHandler() {
            public void endDocument() {
                String[] names = metadata.names();
                Arrays.sort(names);
                for (String name : names) {
                    writer.println(name + ": " + metadata.get(name));
                }
            }
        };
    }

    /**
     * Returns a {@link System#out} writer with the configured output encoding.
     *
     * @see <a href="https://issues.apache.org/jira/browse/TIKA-277">TIKA-277</a>
     * @return writer
     * @throws UnsupportedEncodingException
     *         if the configured encoding is not supported
     */
    private Writer getSystemOutWriter() throws UnsupportedEncodingException {
        if (encoding != null) {
            return new OutputStreamWriter(System.out, encoding);
        } else {
            return new OutputStreamWriter(System.out);
        }
    }

}
{'fraction_non_alphanumeric': 0.11355603693991563, 'fraction_numerical': 0.0019952114924181963, 'mean_word_length': 2.264421287681429, 'pattern_counts': {'":': 2, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 2, 'xml': 20}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1213902', 'n_tokens_mistral': 4575, 'n_tokens_neox': 4475, 'n_words': 1275}