<commit_before><?php

/** @file Lemmatizer.php
 * Perl-based lemmatizer similar to what RFTagger uses.
 *
 * @author <NAME>
 * @date March 2014
 */

require_once( "AutomaticAnnotator.php" );

class Lemmatizer extends AutomaticAnnotator {
    private $tmpfiles = array();
    private $dictionary = array();
    private $lowercase_all = false;
    private $use_pos = true;
    private $use_norm = false;

    // lemmatizer return value if lemma could not be found:
    private $unknown_lemma = "<unknown>";

    public function __construct($prfx, $opts) {
        parent::__construct($prfx, $opts);
        if(!array_key_exists("par", $this->options)) {
            $this->options["par"] = $this->prefix . "Lemmatizer.par";
        }
        if(array_key_exists("use_pos", $this->options)) {
            $this->use_pos = ($this->options["use_pos"] == 1) ? true : false;
        }
        if(array_key_exists("use_norm", $this->options)) {
            $this->use_norm = (bool) $this->options["use_norm"];
        }
        if(array_key_exists("lowercase_all", $this->options)) {
            $this->lowercase_all = (bool) $this->options["lowercase_all"];
        }
    }

    public function __destruct() {
        foreach($this->tmpfiles as $tmpfile) {
            unlink($tmpfile);
        }
    }

    /** Writes an input file for the lemmatization script.
     *
     * @param array $tokens Tokens to write
     *
     * @return Name of the newly created file
     */
    private function writeLemmatizerInput($tokens) {
        $filename = tempnam(sys_get_temp_dir(), "cora_lem");
        $this->tmpfiles[] = $filename;
        $handle = fopen($filename, "w");
        foreach($tokens as $tok) {
            fwrite($handle, $tok['ascii']);
            if($this->use_pos) {
                fwrite($handle, "\t".$tok['tags']['pos']);
            }
            fwrite($handle, "\n");
        }
        fclose($handle);
        return $filename;
    }

    private function lowercaseAscii(&$tokens) {
        foreach($tokens as &$tok) {
            if(array_key_exists('ascii', $tok)) {
                $tok['ascii'] = mb_strtolower($tok['ascii'], 'UTF-8');
            }
        }
        unset($tok);
    }

    /** Converts lemmatizer output to an array for saving it back to CorA.
     *
     * @param array $mod The original input token
     * @param string $line Output line from lemmatizer
     *
     * @return An array element suitable for DBInterface::saveLines()
     */
    private function makeAnnotationArray($mod, $line) {
        if($line == $this->unknown_lemma) {
            return array();
        }
        return array("id" => $mod['id'],
                     "ascii" => $mod['ascii'],
                     "anno_lemma" => $line);<commit_msg>Lemmatizer memory fix attempt.<commit_after><?php

/** @file Lemmatizer.php
 * Perl-based lemmatizer similar to what RFTagger uses.
 *
 * @author <NAME>
 * @date March 2014
 */

require_once( "AutomaticAnnotator.php" );

class Lemmatizer extends AutomaticAnnotator {
    private $tmpfiles = array();
    private $dictionary = array();
    private $lowercase_all = false;
    private $use_pos = true;
    private $use_norm = false;

    // lemmatizer return value if lemma could not be found:
    private $unknown_lemma = "<unknown>";

    public function __construct($prfx, $opts) {
        parent::__construct($prfx, $opts);
        if(!array_key_exists("par", $this->options)) {
            $this->options["par"] = $this->prefix . "Lemmatizer.par";
        }
        if(array_key_exists("use_pos", $this->options)) {
            $this->use_pos = ($this->options["use_pos"] == 1) ? true : false;
        }
        if(array_key_exists("use_norm", $this->options)) {
            $this->use_norm = (bool) $this->options["use_norm"];
        }
        if(array_key_exists("lowercase_all", $this->options)) {
            $this->lowercase_all = (bool) $this->options["lowercase_all"];
        }
    }

    public function __destruct() {
        foreach($this->tmpfiles as $tmpfile) {
            unlink($tmpfile);
        }
    }

    /** Writes an input file for the lemmatization script.
     *
     * @param array $tokens Tokens to write
     *
     * @return Name of the newly created file
     */
    private function writeLemmatizerInput($tokens) {
        $filename = tempnam(sys_get_temp_dir(), "cora_lem");
        $this->tmpfiles[] = $filename;
        $handle = fopen($filename, "w");
        foreach($tokens as $tok) {
            fwrite($handle, $tok['ascii']);
            if($this->use_pos) {
                fwrite($handle, "\t".$tok['tags']['pos']);
            }
            fwrite($handle, "\n");
        }
        fclose($handle);
        return $filename;
    }

    private function lowercaseAscii(&$tokens) {
        foreach($tokens as &$tok) {
            if(isset($tok['ascii'])) {
                $tok['ascii'] = mb_strtolower($tok['ascii'], 'UTF-8');
            }
        }
        unset($tok);
    }

    /** Converts lemmatizer output to an array for saving it back to CorA.
     *
     * @param array $mod The original input token
     * @param string $line Output line from lemmatizer
     *
     * @return An array element suitable for DBInterface::saveLines()
     */
    private function makeAnnotationArray($mod, $line) {
        if($line == $this->unknown_lemma) {
            return array();
        }
        return array("id" => $mod['id'],
                     "ascii" => $mod['ascii'],
                     "anno_lemma" => $line);
{'fraction_non_alphanumeric': 0.14942112879884226, 'fraction_numerical': 0.002170767004341534, 'mean_word_length': 2.256183745583039, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '790841', 'n_tokens_mistral': 1694, 'n_tokens_neox': 1636, 'n_words': 377}