<commit_before><?php
/**
 * This is the implementation of a dictionary plugin. 
 * This plugin provides a stopword list.
 *  
 */

class StopWordPlugin extends MGDictionaryPlugin  {
  public $hasAdmin = TRUE;
  public $accessRole = "editor";
  public $enableOnInstall = true;
  
  /**
   * Checks whether one or more of the given tags are in the stop word list. It will return the tags
   * as an array in this form
   * 
   * array(
   *  'tag1', true/false // true if found
   *  'tag2', true/false // true if found
   *   ...
   * )
   * 
   * @param object $game The game object
   * @param object $game_model The game model
   * @param array $tags the tags to be looked up as a single dimension array array('tag1', 'tag2', ...)
   * @param array $user_ids optional user_ids that might influence the lookup
   * @param array $image_ids optional image_ids that might influence the lookup
   * @return array the checked tags
   */
  function lookup(&$game, &$game_model, $tags, $user_id=null, $image_ids=null) {
    $tags_return = array();
    $tags_lookup = array();
    
    if (is_array($tags) && count($tags) > 0) {
      foreach ($tags as $tag) {
        $tags_return[strtolower($tag)] = false;
        $tags_lookup[] = $tag;
      }
      
      $stopped_words = Yii::app()->db->createCommand()
                        ->select('s.word')
                        ->from('{{stop_word}} s')
                        ->where(array('in', 's.word', array_values($tags_lookup))) 
                        ->queryAll();
      
      if ($stopped_words) {
        foreach ($stopped_words as $stop_word) {
          if (array_key_exists(strtolower($stop_word["word"]), $tags_return)) {
            $tags_return[strtolower($stop_word["word"])] = true;
          }
        }  
      }
    }
    
    return $tags_return;
  }
  
  function add($tag, $info) {
    $model = new StopWord;
    $model->word = $tag;
    $model->created = date('Y-m-d H:i:s'); 
    $model->modified = date('Y-m-d H:i:s'); 
    $model->source = $info;
    
    if (!$model->save()) {
      print_r($model->getErrors());
    } 
  }
  
  function setWeights(&$game, &$game_model, $tags) {
    $stop_words = StopWord::getStopWordList();
    
    if (count($stop_words)) {
      $arr_stopwords = array();
      $arr_used_stopwords = array();
      
      foreach ($stop_words as $stop_word) {
        $arr_stopwords[strtolower($stop_word["word"])] = $stop_word;
      }
      
      foreach ($tags as $image_id => $image_tags) {<commit_msg>Fix for Bug #906698 'STOPWORD PLUGIN: Banning tag results in Error 500'<commit_after><?php
/**
 * This is the implementation of a dictionary plugin. 
 * This plugin provides a stopword list.
 *  
 */

class StopWordPlugin extends MGDictionaryPlugin  {
  public $hasAdmin = TRUE;
  public $accessRole = "editor";
  public $enableOnInstall = true;
  
  /**
   * Checks whether one or more of the given tags are in the stop word list. It will return the tags
   * as an array in this form
   * 
   * array(
   *  'tag1', true/false // true if found
   *  'tag2', true/false // true if found
   *   ...
   * )
   * 
   * @param object $game The game object
   * @param object $game_model The game model
   * @param array $tags the tags to be looked up as a single dimension array array('tag1', 'tag2', ...)
   * @param array $user_ids optional user_ids that might influence the lookup
   * @param array $image_ids optional image_ids that might influence the lookup
   * @return array the checked tags
   */
  function lookup(&$game, &$game_model, $tags, $user_id=null, $image_ids=null) {
    $tags_return = array();
    $tags_lookup = array();
    
    if (is_array($tags) && count($tags) > 0) {
      foreach ($tags as $tag) {
        $tags_return[strtolower($tag)] = false;
        $tags_lookup[] = $tag;
      }
      
      $stopped_words = Yii::app()->db->createCommand()
                        ->select('s.word')
                        ->from('{{stop_word}} s')
                        ->where(array('in', 's.word', array_values($tags_lookup))) 
                        ->queryAll();
      
      if ($stopped_words) {
        foreach ($stopped_words as $stop_word) {
          if (array_key_exists(strtolower($stop_word["word"]), $tags_return)) {
            $tags_return[strtolower($stop_word["word"])] = true;
          }
        }  
      }
    }
    
    return $tags_return;
  }
  
  function add($tag, $info) {
    $model = new StopWord;
    $model->word = $tag;
    $model->counter = 0; 
    $model->modified = date('Y-m-d H:i:s'); 
    $model->source = $info;
    
    if (!$model->save()) {
      print_r($model->getErrors());
    } 
  }
  
  function setWeights(&$game, &$game_model, $tags) {
    $stop_words = StopWord::getStopWordList();
    
    if (count($stop_words)) {
      $arr_stopwords = array();
      $arr_used_stopwords = array();
      
      foreach ($stop_words as $stop_word) {
        $arr_stopwords[strtolower($stop_word["word"])] = $stop_word;
      }
      
      foreach ($tags as $image_id => $image_tags) {
{'fraction_non_alphanumeric': 0.14297029702970296, 'fraction_numerical': 0.0039603960396039604, 'mean_word_length': 2.374081496325985, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '421925', 'n_tokens_mistral': 1638, 'n_tokens_neox': 1603, 'n_words': 418}