<commit_before><?php
/**
 * Tatoeba Project, free collaborative creation of multilingual corpuses project
 * Copyright (C) 2016  <NAME> <<EMAIL>>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * PHP version 5
 *
 * @category PHP
 * @package  Tatoeba
 * @author   <NAME> <<EMAIL>>
 * @license  Affero General Public License
 * @link     http://tatoeba.org
 */

/**
 * Model class for vocabulary.
 *
 * @package  Models
 * @author   <NAME> <<EMAIL>>
 * @license  Affero General Public License
 * @link     http://tatoeba.org
 */

class Vocabulary extends AppModel
{
    public $useTable = 'vocabulary';
    public $belongsTo = array('UsersVocabulary', 'Sentence');
    public $actsAs = array('Hashable');

    /**
     * Adds an item into the vocabulary list of current user.
     *
     * @param $lang string Language of the vocabulary item.
     * @param $text string Text of the vocabulary item.
     *
     * @return $data array
     */
    public function addItem($lang, $text)
    {
        $text = trim($text);
        if (empty($text) || empty($lang)) {
            return null;
        }

        $hash = $this->makeHash($lang, $text);

        $data = array(
            'hash' => $hash,
            'lang' => $lang,
            'text' => $text
        );

        $duplicate = $this->_hasDuplicate($hash, $lang, $text);

        if ($duplicate) {
            $numSentences = $this->_updateNumSentences($duplicate['Vocabulary']);

            $this->id = $duplicate['Vocabulary']['id'];

            $data['numSentences'] = $numSentences;
        } else {
            $numSentences = $this->_getNumberOfSentences($lang, $text);

            $data['numSentences'] = $numSentences;

            $this->save($data);
        }

        $data['id'] = $this->id;

        $this->UsersVocabulary->add($this->id, CurrentUser::get('id'));

        return $data;
    }

    /**
     * Return true if land and text have true duplicate.
     *
     * @param  string  $hash Vocabulary hash value.
     * @param  string  $lang Vocabulary language.
     * @param  string  $text Vocabulary text.
     *
     * @return boolean|array
     */
    private function _hasDuplicate($hash, $lang, $text)
    {
        $vocabularyItems = $this->findAllByBinary($hash, 'hash');

        foreach ($vocabularyItems as $vocabularyItem) {
            if ($this->confirmDuplicate($text, $lang, $vocabularyItem['Vocabulary'])) {
                return $vocabularyItem;
            }
        }

        return false;
    }

    /**
     * Returns the number of sentences for $text in language $lang.
     *
     * This uses the search engine (Sphinx) to count the number of search result
     * for an exact search on $text in language $lang.
     *
     * @param $lang string Language of the vocabulary item
     * @param $text string Text of the vocabulary item.
     *
     * @return int
     */
    private function _getNumberOfSentences($lang, $text)
    {
        $this->Behaviors->attach('Sphinx');
        $index = array($lang . '_main_index', $lang . '_delta_index');
        $sphinx = array(
            'index' => $index,
            'matchMode' => SPH_MATCH_EXTENDED2
        );
        $query = '="'.$text.'"';
        return $this->Sentence->find('count', array(
            'sphinx' => $sphinx,
            'search' => $query
        ));<commit_msg>Use Sphinx phrase search instead of our own quotes<commit_after><?php
/**
 * Tatoeba Project, free collaborative creation of multilingual corpuses project
 * Copyright (C) 2016  <NAME> <<EMAIL>>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * PHP version 5
 *
 * @category PHP
 * @package  Tatoeba
 * @author   <NAME> <<EMAIL>>
 * @license  Affero General Public License
 * @link     http://tatoeba.org
 */

/**
 * Model class for vocabulary.
 *
 * @package  Models
 * @author   <NAME> <<EMAIL>>
 * @license  Affero General Public License
 * @link     http://tatoeba.org
 */

class Vocabulary extends AppModel
{
    public $useTable = 'vocabulary';
    public $belongsTo = array('UsersVocabulary', 'Sentence');
    public $actsAs = array('Hashable');

    /**
     * Adds an item into the vocabulary list of current user.
     *
     * @param $lang string Language of the vocabulary item.
     * @param $text string Text of the vocabulary item.
     *
     * @return $data array
     */
    public function addItem($lang, $text)
    {
        $text = trim($text);
        if (empty($text) || empty($lang)) {
            return null;
        }

        $hash = $this->makeHash($lang, $text);

        $data = array(
            'hash' => $hash,
            'lang' => $lang,
            'text' => $text
        );

        $duplicate = $this->_hasDuplicate($hash, $lang, $text);

        if ($duplicate) {
            $numSentences = $this->_updateNumSentences($duplicate['Vocabulary']);

            $this->id = $duplicate['Vocabulary']['id'];

            $data['numSentences'] = $numSentences;
        } else {
            $numSentences = $this->_getNumberOfSentences($lang, $text);

            $data['numSentences'] = $numSentences;

            $this->save($data);
        }

        $data['id'] = $this->id;

        $this->UsersVocabulary->add($this->id, CurrentUser::get('id'));

        return $data;
    }

    /**
     * Return true if land and text have true duplicate.
     *
     * @param  string  $hash Vocabulary hash value.
     * @param  string  $lang Vocabulary language.
     * @param  string  $text Vocabulary text.
     *
     * @return boolean|array
     */
    private function _hasDuplicate($hash, $lang, $text)
    {
        $vocabularyItems = $this->findAllByBinary($hash, 'hash');

        foreach ($vocabularyItems as $vocabularyItem) {
            if ($this->confirmDuplicate($text, $lang, $vocabularyItem['Vocabulary'])) {
                return $vocabularyItem;
            }
        }

        return false;
    }

    /**
     * Returns the number of sentences for $text in language $lang.
     *
     * This uses the search engine (Sphinx) to count the number of search result
     * for an exact search on $text in language $lang.
     *
     * @param $lang string Language of the vocabulary item
     * @param $text string Text of the vocabulary item.
     *
     * @return int
     */
    private function _getNumberOfSentences($lang, $text)
    {
        $this->Behaviors->attach('Sphinx');
        $index = array($lang . '_main_index', $lang . '_delta_index');
        $sphinx = array(
            'index' => $index,
            'matchMode' => SPH_MATCH_PHRASE,
        );
        return $this->Sentence->find('count', array(
            'sphinx' => $sphinx,
            'search' => $text,
        ));
{'fraction_non_alphanumeric': 0.12490547012856063, 'fraction_numerical': 0.0016385177716158306, 'mean_word_length': 2.36228813559322, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1039286', 'n_tokens_mistral': 2347, 'n_tokens_neox': 2269, 'n_words': 730}