<commit_before>/**
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 *               http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 *
 *
 * **************************************************************************
 * NOTICE This software was produced for the U. S. Government under Contract No.
 * W15P7T-12-C-F600, and is subject to the Rights in Noncommercial Computer
 * Software and Noncommercial Computer Software Documentation Clause
 * 252.227-7014 (JUN 1995)
 *
 * (c) 2012 The MITRE Corporation. All Rights Reserved.
 * **************************************************************************
 *
 * Continue contributions:
 *    Copyright 2013-2015 The MITRE Corporation.
 */
///** ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|
//
// _____                                ____                     __                       __
///\  __`\                             /\  _`\                  /\ \__                   /\ \__
//\ \ \/\ \   _____      __     ___    \ \,\L\_\      __   __  _\ \ ,_\     __       ___ \ \ ,_\
//\ \ \ \ \ /\ '__`\  /'__`\ /' _ `\   \/_\__ \    /'__`\/\ \/'\\ \ \/   /'__`\   /' _ `\\ \ \/
//\ \ \_\ \\ \ \L\ \/\  __/ /\ \/\ \    /\ \L\ \ /\  __/\/>  </ \ \ \_ /\ \L\.\_ /\ \/\ \\ \ \_
// \ \_____\\ \ ,__/\ \____\\ \_\ \_\   \ `\____\\ \____\/\_/\_\ \ \__\\ \__/.\_\\ \_\ \_\\ \__\
//  \/_____/ \ \ \/  \/____/ \/_/\/_/    \/_____/ \/____/\//\/_/  \/__/ \/__/\/_/ \/_/\/_/ \/__/
//          \ \_\
//           \/_/
//
// OpenSextant PoLi - Patterns extractor
//*  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|
//*/
package org.opensextant.extractors.poli;

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;

import org.opensextant.data.TextInput;
import org.opensextant.extraction.TextMatch;
import org.opensextant.extractors.flexpat.AbstractFlexPat;
import org.opensextant.extractors.flexpat.RegexPattern;
import org.opensextant.extractors.flexpat.RegexPatternManager;
import org.opensextant.extractors.flexpat.TextMatchResult;
import org.opensextant.util.TextUtils;

/**
 *
 * @author <NAME>, MITRE, ubaldino at mitre dot org
 */
public class PatternsOfLife extends AbstractFlexPat {

    public static final String DEFAULT_POLI_CFG = "/poli_patterns.cfg";

    public PatternsOfLife(boolean debugmode) {
        super(debugmode);
        patterns_file = DEFAULT_POLI_CFG;
    }

    /**
     * Default constructor, debugging off.
     */
    public PatternsOfLife() {
        this(false);
    }

    /**
     * Extractor interface: extractors are responsible for cleaning up after
     * themselves.
     */
    @Override
    public void cleanup() {
    }

    /**
     * Extractor interface: getName
     *
     * @return
     */
    @Override
    public String getName() {
        return "PoLi";
    }

    @Override
    protected RegexPatternManager createPatternManager(InputStream s, String n) throws IOException {
        return new PoliPatternManager(s, n);
    }

    /**
     * Support the standard Extractor interface. This provides access to the
     * most common extraction; For PoLi extraction, you would process ALL
     * patterns in your configuration file, or if you enable only certain
     * patterns -- those enabled at the time of this call would be executed.
     * extract_patterns( family = null ) implies ALL patterns.
     */
    @Override
    public List<TextMatch> extract(TextInput input) {
        TextMatchResult results = extract_patterns(input.buffer, input.id, null);
        return results.matches;
    }

    @Override
    public List<TextMatch> extract(String input_buf) {
        TextMatchResult results = extract_patterns(input_buf, NO_DOC_ID, null);
        return results.matches;
    }

    /**<commit_msg>streamline API call<commit_after>/**
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 *               http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 *
 *
 * **************************************************************************
 * NOTICE This software was produced for the U. S. Government under Contract No.
 * W15P7T-12-C-F600, and is subject to the Rights in Noncommercial Computer
 * Software and Noncommercial Computer Software Documentation Clause
 * 252.227-7014 (JUN 1995)
 *
 * (c) 2012 The MITRE Corporation. All Rights Reserved.
 * **************************************************************************
 *
 * Continue contributions:
 *    Copyright 2013-2015 The MITRE Corporation.
 */
///** ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|
//
// _____                                ____                     __                       __
///\  __`\                             /\  _`\                  /\ \__                   /\ \__
//\ \ \/\ \   _____      __     ___    \ \,\L\_\      __   __  _\ \ ,_\     __       ___ \ \ ,_\
//\ \ \ \ \ /\ '__`\  /'__`\ /' _ `\   \/_\__ \    /'__`\/\ \/'\\ \ \/   /'__`\   /' _ `\\ \ \/
//\ \ \_\ \\ \ \L\ \/\  __/ /\ \/\ \    /\ \L\ \ /\  __/\/>  </ \ \ \_ /\ \L\.\_ /\ \/\ \\ \ \_
// \ \_____\\ \ ,__/\ \____\\ \_\ \_\   \ `\____\\ \____\/\_/\_\ \ \__\\ \__/.\_\\ \_\ \_\\ \__\
//  \/_____/ \ \ \/  \/____/ \/_/\/_/    \/_____/ \/____/\//\/_/  \/__/ \/__/\/_/ \/_/\/_/ \/__/
//          \ \_\
//           \/_/
//
// OpenSextant PoLi - Patterns extractor
//*  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|
//*/
package org.opensextant.extractors.poli;

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;

import org.opensextant.data.TextInput;
import org.opensextant.extraction.TextMatch;
import org.opensextant.extractors.flexpat.AbstractFlexPat;
import org.opensextant.extractors.flexpat.RegexPattern;
import org.opensextant.extractors.flexpat.RegexPatternManager;
import org.opensextant.extractors.flexpat.TextMatchResult;
import org.opensextant.util.TextUtils;

/**
 *
 * @author <NAME>, MITRE, ubaldino at mitre dot org
 */
public class PatternsOfLife extends AbstractFlexPat {

    public static final String DEFAULT_POLI_CFG = "/poli_patterns.cfg";

    public PatternsOfLife(boolean debugmode) {
        super(debugmode);
        patterns_file = DEFAULT_POLI_CFG;
    }

    /**
     * Default constructor, debugging off.
     */
    public PatternsOfLife() {
        this(false);
    }

    /**
     * Extractor interface: extractors are responsible for cleaning up after
     * themselves.
     */
    @Override
    public void cleanup() {
    }

    /**
     * Extractor interface: getName
     *
     * @return
     */
    @Override
    public String getName() {
        return "PoLi";
    }

    @Override
    protected RegexPatternManager createPatternManager(InputStream s, String n) throws IOException {
        return new PoliPatternManager(s, n);
    }

    /**
     * Support the standard Extractor interface. This provides access to the
     * most common extraction; For PoLi extraction, you would process ALL
     * patterns in your configuration file, or if you enable only certain
     * patterns -- those enabled at the time of this call would be executed.
     * extract_patterns( family = null ) implies ALL patterns.
     */
    @Override
    public List<TextMatch> extract(TextInput input) {
        TextMatchResult results = extract_patterns(input.buffer, input.id, null);
        return results.matches;
    }

    public List<TextMatch> extract(TextInput input, String family) {
        TextMatchResult results = extract_patterns(input.buffer, input.id, family);        
        return results.matches;
    }

    @Override
    public List<TextMatch> extract(String input_buf) {
        TextMatchResult results = extract_patterns(input_buf, NO_DOC_ID, null);
        return results.matches;
    }

    /**
{'fraction_non_alphanumeric': 0.19096671949286848, 'fraction_numerical': 0.008603124292506226, 'mean_word_length': 2.213895962168061, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6745757', 'n_tokens_mistral': 2672, 'n_tokens_neox': 2476, 'n_words': 625}