<commit_before>/*
 * The MIT License
 *
 * Copyright 2017 georgi.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */
package netrank;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import mark.core.DetectionAgentInterface;
import mark.core.DetectionAgentProfile;
import mark.core.Evidence;
import mark.core.RawData;
import mark.core.ServerInterface;
import org.apache.tika.language.detect.LanguageDetector;
import org.apache.tika.langdetect.OptimaizeLangDetector;
import org.apache.tika.language.detect.LanguageResult;

/**
 * LanguageDetection class for language detection in email.
 * Note: can be intensive on large collections of email's.
 * @author georgi
 */
public class LanguageDetection implements DetectionAgentInterface<Link> {

    private static final double RATIO_THRESHOLD = 0.3;

    /**
     * Method for parsing the email data and language detection.
     * Extracts the text from it and running a language detection to
     * determine the language of the text
     * @param text
     * @return List<String> a list of Strings with the different possible
     * language in the body of the email
     * @throws IOException
     */
    private List<String> detectLanguage(final String text) throws IOException {
        List<String> lang = null;
        String[] data = text.split(" , ");
        LanguageDetector detector =
                        new OptimaizeLangDetector().loadModels();
        for (String data1 : data) {
            if (data1.contains("Text=")) {
                String text_body = data1.substring(5);

                List<LanguageResult> result = detector.detectAll(text_body);
                lang = new ArrayList<>(result.size());
                for (int i = 0; i < result.size(); i++) {
                    lang.add(result.get(i).getLanguage());
                }
            }
        }
        return lang;
    }

    /**<commit_msg>Updated and corrected error to LanguageDetection<commit_after>/*
 * The MIT License
 *
 * Copyright 2017 georgi.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */
package netrank;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import mark.core.DetectionAgentInterface;
import mark.core.DetectionAgentProfile;
import mark.core.Evidence;
import mark.core.RawData;
import mark.core.ServerInterface;
import org.apache.tika.language.detect.LanguageDetector;
import org.apache.tika.langdetect.OptimaizeLangDetector;
import org.apache.tika.language.detect.LanguageResult;

/**
 * LanguageDetection class for language detection in email.
 * Note: can be intensive on large collections of email's.
 * @author georgi
 */
public class LanguageDetection implements DetectionAgentInterface<Link> {

    private static final double RATIO_THRESHOLD = 0.3;

    /**
     * Method for parsing the email data and language detection.
     * Extracts the text from it and running a language detection to
     * determine the language of the text
     * @param text
     * @return List<String> a list of Strings with the different possible
     * language in the body of the email
     * @throws IOException
     */
    private List<String> detectLanguage(final String text) throws IOException {
        List<String> lang = null;
        MIMEParser parser = new MIMEParser(text);
        String parsed_data = parser.toString();
        String[] data = parsed_data.split(" , ");
        LanguageDetector detector =
                        new OptimaizeLangDetector().loadModels();
        for (String data1 : data) {
            if (data1.contains("Text=")) {
                String text_body = data1.substring(5);

                List<LanguageResult> result = detector.detectAll(text_body);
                lang = new ArrayList<>(result.size());
                for (int i = 0; i < result.size(); i++) {
                    lang.add(result.get(i).getLanguage());
                }
            }
        }
        return lang;
    }

    /**
{'fraction_non_alphanumeric': 0.07818725099601594, 'fraction_numerical': 0.0036520584329349268, 'mean_word_length': 3.1380494505494507, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '131005', 'n_tokens_mistral': 1565, 'n_tokens_neox': 1488, 'n_words': 634}