<commit_before>/*
 * Copyright (c) 2013, <NAME>. Licensed under an MIT-style license.
 * You may obtain a copy of the License at
 *
 *		 http://mattcg.mit-license.org
 */

package cg.m.nodetika;

import java.util.Map;
import java.util.HashMap;
import java.util.Set;

import java.io.IOException;
import java.io.FileNotFoundException;
import java.io.File;
import java.io.InputStream;
import java.io.FileInputStream;
import java.io.ByteArrayOutputStream;
import java.io.OutputStreamWriter;

import java.net.URL;
import java.net.MalformedURLException;

import java.lang.Exception;

import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.TikaMetadataKeys;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.html.HtmlParser;
import org.apache.tika.mime.MediaType;
import org.apache.tika.detect.Detector;
import org.apache.tika.detect.AutoDetectReader;
import org.apache.tika.language.LanguageIdentifier;
import org.apache.tika.sax.BodyContentHandler;
import org.apache.tika.config.TikaConfig;
import org.apache.tika.exception.TikaException;
import org.apache.tika.exception.EncryptedDocumentException;

import org.xml.sax.helpers.DefaultHandler;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;

import com.google.gson.Gson;

public class NodeTika {

	private static TikaInputStream createInputStream(String uri) throws FileNotFoundException, MalformedURLException, IOException {
		InputStream inputStream;

		if (uri.startsWith("http://") || uri.startsWith("https://") || uri.startsWith("ftp://")) {
			inputStream = new URL(uri).openStream();
		} else {
			inputStream = new FileInputStream(uri);
		}

		return TikaInputStream.get(inputStream);
	}

	private static AutoDetectParser createParser() {
		final AutoDetectParser parser = new AutoDetectParser();

		Map<MediaType, Parser> parsers = parser.getParsers();
		parsers.put(MediaType.APPLICATION_XML, new HtmlParser());
		parser.setParsers(parsers);

		parser.setFallback(new Parser() {
			public Set<MediaType> getSupportedTypes(ParseContext parseContext) {
				return parser.getSupportedTypes(parseContext);
			}

			public void parse(InputStream inputStream, ContentHandler contentHandler, Metadata metadata, ParseContext parseContext) throws TikaException {
				throw new TikaException("Unsupported Media Type");
			}
		});

		return parser;
	}

	private static Metadata createMetadata(AutoDetectParser parser, String contentType, String uri) {
		Metadata metadata = new Metadata();
		metadata.set(TikaMetadataKeys.RESOURCE_NAME_KEY, new File(uri).getName());

		if (contentType != null && "xml".equals(MediaType.parse(contentType).getSubtype())) {
			contentType = null;
		}

		if (contentType != null && contentType.equals(MediaType.OCTET_STREAM)) {
			contentType = null;
		}

		if (contentType != null) {
			metadata.add(HttpHeaders.CONTENT_TYPE, contentType);

			final Detector detector = parser.getDetector();

			parser.setDetector(new Detector() {
				public MediaType detect(InputStream inputStream, Metadata metadata) throws IOException {
					String ct = metadata.get(HttpHeaders.CONTENT_TYPE);

					if (ct != null) {
						return MediaType.parse(ct);
					} else {
						return detector.detect(inputStream, metadata);
					}
				}
			});
		}

		return metadata;
	}

	public static String extractText(String uri, String contentType) throws Exception {
		return extractText(uri, contentType, "UTF8");
	}

	public static String extractText(String uri, String contentType, String outputEncoding) throws Exception {<commit_msg>Dash in UTF-8<commit_after>/*
 * Copyright (c) 2013, <NAME>. Licensed under an MIT-style license.
 * You may obtain a copy of the License at
 *
 *		 http://mattcg.mit-license.org
 */

package cg.m.nodetika;

import java.util.Map;
import java.util.HashMap;
import java.util.Set;

import java.io.IOException;
import java.io.FileNotFoundException;
import java.io.File;
import java.io.InputStream;
import java.io.FileInputStream;
import java.io.ByteArrayOutputStream;
import java.io.OutputStreamWriter;

import java.net.URL;
import java.net.MalformedURLException;

import java.lang.Exception;

import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.TikaMetadataKeys;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.html.HtmlParser;
import org.apache.tika.mime.MediaType;
import org.apache.tika.detect.Detector;
import org.apache.tika.detect.AutoDetectReader;
import org.apache.tika.language.LanguageIdentifier;
import org.apache.tika.sax.BodyContentHandler;
import org.apache.tika.config.TikaConfig;
import org.apache.tika.exception.TikaException;
import org.apache.tika.exception.EncryptedDocumentException;

import org.xml.sax.helpers.DefaultHandler;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;

import com.google.gson.Gson;

public class NodeTika {

	private static TikaInputStream createInputStream(String uri) throws FileNotFoundException, MalformedURLException, IOException {
		InputStream inputStream;

		if (uri.startsWith("http://") || uri.startsWith("https://") || uri.startsWith("ftp://")) {
			inputStream = new URL(uri).openStream();
		} else {
			inputStream = new FileInputStream(uri);
		}

		return TikaInputStream.get(inputStream);
	}

	private static AutoDetectParser createParser() {
		final AutoDetectParser parser = new AutoDetectParser();

		Map<MediaType, Parser> parsers = parser.getParsers();
		parsers.put(MediaType.APPLICATION_XML, new HtmlParser());
		parser.setParsers(parsers);

		parser.setFallback(new Parser() {
			public Set<MediaType> getSupportedTypes(ParseContext parseContext) {
				return parser.getSupportedTypes(parseContext);
			}

			public void parse(InputStream inputStream, ContentHandler contentHandler, Metadata metadata, ParseContext parseContext) throws TikaException {
				throw new TikaException("Unsupported Media Type");
			}
		});

		return parser;
	}

	private static Metadata createMetadata(AutoDetectParser parser, String contentType, String uri) {
		Metadata metadata = new Metadata();
		metadata.set(TikaMetadataKeys.RESOURCE_NAME_KEY, new File(uri).getName());

		if (contentType != null && "xml".equals(MediaType.parse(contentType).getSubtype())) {
			contentType = null;
		}

		if (contentType != null && contentType.equals(MediaType.OCTET_STREAM)) {
			contentType = null;
		}

		if (contentType != null) {
			metadata.add(HttpHeaders.CONTENT_TYPE, contentType);

			final Detector detector = parser.getDetector();

			parser.setDetector(new Detector() {
				public MediaType detect(InputStream inputStream, Metadata metadata) throws IOException {
					String ct = metadata.get(HttpHeaders.CONTENT_TYPE);

					if (ct != null) {
						return MediaType.parse(ct);
					} else {
						return detector.detect(inputStream, metadata);
					}
				}
			});
		}

		return metadata;
	}

	public static String extractText(String uri, String contentType) throws Exception {
		return extractText(uri, contentType, "UTF-8");
	}

	public static String extractText(String uri, String contentType, String outputEncoding) throws Exception {
{'fraction_non_alphanumeric': 0.1097839216212589, 'fraction_numerical': 0.0014763119044423566, 'mean_word_length': 4.9951729686242965, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6974918', 'n_tokens_mistral': 2438, 'n_tokens_neox': 2210, 'n_words': 490}