<commit_before>require 'bundler/setup'
require 'dotenv'
Dotenv.load

require 'active_record'
require 'activerecord-import'
require 'logger'
require 'yaml'
require 'pry'
require 'searchkick'

EZFS_ENV = ENV.fetch('RACK_ENV', 'development')

ActiveRecord::Base.logger = Logger.new('debug.log')
configuration = YAML::load(IO.read('db/config.yml'))
ActiveRecord::Base.establish_connection(configuration[EZFS_ENV])

class Filing < ActiveRecord::Base
  searchkick
  validates_uniqueness_of :fcc_id, scope: :docket_number
  
  def self.docket_search(docket_number, query)    
    results = self.search(where: {docket_number: docket_number, citation: query})
    if results.length == 0
      results = self.search(query, where: {docket_number: docket_number})
    end

    return results
  end

  def self.all_search(query)
    re1 = '(\\d+)'	# Integer Number 1
    re2 = '(-)'	# Any Single Character 1
    re3 = '(\\d+)'	# Integer Number 2
    re = (re1+re2+re3)
    m = Regexp.new(re,Regexp::IGNORECASE)
    
    query.gsub!("nprm", "Notice of Proposed Rulemaking")
    query.gsub!("NPRM", "Notice of Proposed Rulemaking")
    
    query.gsub!("fnprm", "Further Notice of Proposed Rulemaking")
    query.gsub!("FNPRM", "Further Notice of Proposed Rulemaking")
    
    query.gsub!("r&o", "Report and Order")
    query.gsub!("R&O", "Report and Order")
    
    results = []
    docket_number = nil
    
    if m.match(query)
      docket_number = m.match(query)[0]   
      stripped_query = query.gsub(docket_number, "").strip
      
      if stripped_query.empty?
        results = self.search(where: {docket_number: docket_number})
      else
        results = self.docket_search(docket_number, stripped_query)<commit_msg>strip out *<commit_after>require 'bundler/setup'
require 'dotenv'
Dotenv.load

require 'active_record'
require 'activerecord-import'
require 'logger'
require 'yaml'
require 'pry'
require 'searchkick'

EZFS_ENV = ENV.fetch('RACK_ENV', 'development')

ActiveRecord::Base.logger = Logger.new('debug.log')
configuration = YAML::load(IO.read('db/config.yml'))
ActiveRecord::Base.establish_connection(configuration[EZFS_ENV])

class Filing < ActiveRecord::Base
  searchkick
  validates_uniqueness_of :fcc_id, scope: :docket_number
  
  def self.docket_search(docket_number, query)    
    results = self.search(where: {docket_number: docket_number, citation: query})
    if results.length == 0
      results = self.search(query, where: {docket_number: docket_number})
    end

    return results
  end

  def self.all_search(query)
    re1 = '(\\d+)'	# Integer Number 1
    re2 = '(-)'	# Any Single Character 1
    re3 = '(\\d+)'	# Integer Number 2
    re = (re1+re2+re3)
    m = Regexp.new(re,Regexp::IGNORECASE)
    
    query.gsub!("*", "")
    
    query.gsub!("nprm", "Notice of Proposed Rulemaking")
    query.gsub!("NPRM", "Notice of Proposed Rulemaking")
    
    query.gsub!("fnprm", "Further Notice of Proposed Rulemaking")
    query.gsub!("FNPRM", "Further Notice of Proposed Rulemaking")
    
    query.gsub!("r&o", "Report and Order")
    query.gsub!("R&O", "Report and Order")
    
    results = []
    docket_number = nil
    
    if m.match(query)
      docket_number = m.match(query)[0]   
      stripped_query = query.gsub(docket_number, "").strip
      
      if stripped_query.empty?
        results = self.search(where: {docket_number: docket_number})
      else
        results = self.docket_search(docket_number, stripped_query)
{'fraction_non_alphanumeric': 0.13316002310803005, 'fraction_numerical': 0.006354708261120739, 'mean_word_length': 3.73734610123119, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '848845', 'n_tokens_mistral': 1225, 'n_tokens_neox': 1190, 'n_words': 252}