<commit_before>class WordData < ActiveRecord::Base
  include SecureSerialize
  include Async
  include GlobalId
  include Processable

  secure_serialize :data
  replicated_model  
  before_save :generate_defaults
  
  def generate_defaults
    self.data ||= {}
    self.reviews = (self.data['reviewer_ids'] || []).length
    if !self.priority && self.locale
      self.priority = -1
      self.schedule(:assert_priority)
    end
  end

  def reviewed_by?(user)
    return (self.data['reviewer_ids'] || []).include?(user && user.global_id)
  end
  
  def process_params(params, non_user_params)
    updater = non_user_params[:updater]
    if updater && updater.allows?(non_user_params[:updater], 'admin_support_actions')
      if !params['skip']
        self.data['reviewer_ids'] = ((self.data['reviewer_ids'] || []) + [updater.global_id]).uniq
        self.data['reviews'] ||= {}
        self.data['reviews'][updater.global_id] = {
          'updated' => Time.now.iso8601,
          'primary_part_of_speech' => params['primary_part_of_speech'],
          'inflection_overrides' => params['inflect_overrides']
        }
        if params['primary_part_of_speech']
          self.data['types'] = ([params['primary_part_of_speech']] + (self.data['types'] || [])).uniq
        end
        if params['inflection_overrides']
          hash = self.data['inflection_overrides'] || {}
          params['inflection_overrides'].each do |key, str|
            if str == ""
              hash.delete(key)
            else
              hash[key] = str
            end
          end
          self.data['inflection_overrides'] = hash
        end
      end
    end
  end

  def assert_priority
    cores = WordData.core_lists.select{|l| l['locale'] == self.locale }
    fringes = WordData.fringe_lists.select{|l| l['locale'] == self.locale }
    score = nil
    return unless self.locale == 'en'
    @@english_buttons ||= nil
    if !@english_buttons
      bs = Board.find_by_path('example/core-112').board_downstream_button_set rescue nil
      if bs
        bs.assert_extra_data
        # priority = 8 if it's in core-112
        bs
        @@english_buttons = bs.buttons
    end
    if @@english_buttons
      score = 8 if @@english_buttons.buttons.any?{|b| b['label'] == self.word }
    end
    if cores.length > 0
      core_count = cores.select{|l| l['words'].include?(self.word) }.length
      fringe_count = fringes.select{|l| l['categories'].any?{|c| c['words'].include?(self.word) } }.length
      # something in all the core lists is top priority
      score = 10 if core_count == cores.length
      # something in some of the core lists is high priority
      score = 9 if core_count > 0
      # something in the fringe lists is moderate priority
      score == 7 if fringe_count > 0
    end
    if !score
      # download word frequency list
      @@english_with_counts ||= Typhoeus.get("https://coughdrop.s3.amazonaws.com/language/english_with_counts.txt")
      lines = @@english_with_counts.body.split(/\n/)
      hash = {}
      # top 5,000 - 5 points
      # top 10,000 - 4 points
      # top 25,000 - 3 points
      # top 50,000 - 2 points
      # any        - 1 point
      list = lines.map{|s| s.split(/\t/)[0] }
      idx = list.index(self.word)
      # score it from 0-5 based on word frequency
      score ||= 5 if idx && idx < 5000
      score ||= 4 if idx && idx < 10000
      score ||= 3 if idx && idx < 25000
      score ||= 2 if idx && idx < 50000
      score ||= 1 if idx
    end
    if score
      self.priority = score
      self.save
    end
  end
  
  def self.find_word(text, locale='en') 
    word = find_word_record(text, locale)
    word && word.data
  end
  
  def self.find_word_record(text, locale='en')
    return nil if text && text.match(/^[\+\:]/)
    locale ||= 'en'
    word = self.find_by(:word => text.downcase, :locale => locale)
    word ||= self.find_by(:word => text.downcase.gsub(/[^A-Za-z0-9'\s]/, ''), :locale => locale)
    if !word && locale.match(/-/)
      locale = locale.split(/-/)[0]
      word ||= self.find_by(:word => text.downcase, :locale => locale)
      word ||= self.find_by(:word => text.downcase.gsub(/[^A-Za-z0-9'\s]/, ''), :locale => locale)<commit_msg>batch priority assertion for word_data<commit_after>class WordData < ActiveRecord::Base
  include SecureSerialize
  include Async
  include GlobalId
  include Processable

  secure_serialize :data
  replicated_model  
  before_save :generate_defaults
  
  def generate_defaults
    self.data ||= {}
    self.reviews = (self.data['reviewer_ids'] || []).length
    if !self.priority && self.locale
      self.priority = -1
      self.schedule(:assert_priority)
    end
  end

  def reviewed_by?(user)
    return (self.data['reviewer_ids'] || []).include?(user && user.global_id)
  end
  
  def process_params(params, non_user_params)
    updater = non_user_params[:updater]
    if updater && updater.allows?(non_user_params[:updater], 'admin_support_actions')
      if !params['skip']
        self.data['reviewer_ids'] = ((self.data['reviewer_ids'] || []) + [updater.global_id]).uniq
        self.data['reviews'] ||= {}
        self.data['reviews'][updater.global_id] = {
          'updated' => Time.now.iso8601,
          'primary_part_of_speech' => params['primary_part_of_speech'],
          'inflection_overrides' => params['inflect_overrides']
        }
        if params['primary_part_of_speech']
          self.data['types'] = ([params['primary_part_of_speech']] + (self.data['types'] || [])).uniq
        end
        if params['inflection_overrides']
          hash = self.data['inflection_overrides'] || {}
          params['inflection_overrides'].each do |key, str|
            if str == ""
              hash.delete(key)
            else
              hash[key] = str
            end
          end
          self.data['inflection_overrides'] = hash
        end
      end
    end
  end

  def self.assert_priority(relation)
    bs = Board.find_by_path('example/core-112').board_downstream_button_set rescue nil
    buttons = nil
    if bs
      bs.assert_extra_data
      buttons = bs.buttons
    end
    counts = Typhoeus.get("https://coughdrop.s3.amazonaws.com/language/english_with_counts.txt")
    relation.find_in_batches(batch_size: 20) do |batch|
      batch.each do |wd|
        wd.assert_priority({'buttons' => buttons, 'counts' => counts})
      end
    end
  end

  def assert_priority(opts=nil)
    cores = WordData.core_lists.select{|l| l['locale'] == self.locale }
    fringes = WordData.fringe_lists.select{|l| l['locale'] == self.locale }
    score = nil
    return unless self.locale == 'en'
    buttons = nil
    if !opts
      bs = Board.find_by_path('example/core-112').board_downstream_button_set rescue nil
      if bs
        bs.assert_extra_data
        # priority = 8 if it's in core-112
        bs
        buttons = bs.buttons
      end
    end
    if buttons
      score = 8 if buttons.buttons.any?{|b| b['label'] == self.word }
    end
    if cores.length > 0
      core_count = cores.select{|l| l['words'].include?(self.word) }.length
      fringe_count = fringes.select{|l| l['categories'].any?{|c| c['words'].include?(self.word) } }.length
      # something in all the core lists is top priority
      score = 10 if core_count == cores.length
      # something in some of the core lists is high priority
      score = 9 if core_count > 0
      # something in the fringe lists is moderate priority
      score == 7 if fringe_count > 0
    end
    if !score
      # download word frequency list
      counts = opts && opts['counts']
      counts ||= Typhoeus.get("https://coughdrop.s3.amazonaws.com/language/english_with_counts.txt") if !opts
      if counts
        lines = counts.body.split(/\n/)
        hash = {}
        # top 5,000 - 5 points
        # top 10,000 - 4 points
        # top 25,000 - 3 points
        # top 50,000 - 2 points
        # any        - 1 point
        list = lines.map{|s| s.split(/\t/)[0] }
        idx = list.index(self.word)
        # score it from 0-5 based on word frequency
        score ||= 5 if idx && idx < 5000
        score ||= 4 if idx && idx < 10000
        score ||= 3 if idx && idx < 25000
        score ||= 2 if idx && idx < 50000
        score ||= 1 if idx
      end
    end
    if score
      self.priority = score
      self.save
    end
  end
  
  def self.find_word(text, locale='en') 
    word = find_word_record(text, locale)
    word && word.data
  end
  
  def self.find_word_record(text, locale='en')
    return nil if text && text.match(/^[\+\:]/)
    locale ||= 'en'
    word = self.find_by(:word => text.downcase, :locale => locale)
    word ||= self.find_by(:word => text.downcase.gsub(/[^A-Za-z0-9'\s]/, ''), :locale => locale)
    if !word && locale.match(/-/)
      locale = locale.split(/-/)[0]
      word ||= self.find_by(:word => text.downcase, :locale => locale)
      word ||= self.find_by(:word => text.downcase.gsub(/[^A-Za-z0-9'\s]/, ''), :locale => locale)
{'fraction_non_alphanumeric': 0.13919821826280623, 'fraction_numerical': 0.017817371937639197, 'mean_word_length': 2.4864130434782608, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 33, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2722905', 'n_tokens_mistral': 3067, 'n_tokens_neox': 2898, 'n_words': 666}