<commit_before>class RestoreTbPhinPdfs < ActiveRecord::Migration
  def self.up
    drop_table :tb_phin_pdfs
    create_table :tb_phin_pdfs do |t|
    t.string :phin_var, :null => true
    t.string :pdf_var, :null => true
    t.string :page, :null => true
    t.integer :var_order, :null => true
    
      t.timestamps
    end
  
  #Locate db/phin_pdf_var.txt
  fileRoot = File.expand_path(File.dirname(__FILE__))
  modRoot = fileRoot.split(File::SEPARATOR).map {|x| x=="" ? File::SEPARATOR : x + File::SEPARATOR}
  modRoot.delete_at(-1)
  newPath = modRoot.join
  newPath = newPath + 'phin_pdf_var.txt'
  
  open(newPath) do |data_file|
    data_file.read.each_line do |line|
      phin_var, pdf_var, page, var_order = line.chomp.split("|")
      TbPhinPdf.create(:phin_var => phin_var, :pdf_var => pdf_var, :page => page, :var_order => var_order)
    end
  end
  end

  def self.down
  end
end<commit_msg>make migration work on production<commit_after>class RestoreTbPhinPdfs < ActiveRecord::Migration
  def self.up
    drop_table :tb_phin_pdfs
    create_table :tb_phin_pdfs do |t|
    t.string :phin_var, :null => true
    t.string :pdf_var, :null => true
    t.string :page, :null => true
    t.integer :var_order, :null => true
    
      t.timestamps
    end
  
    #Locate db/phin_pdf_var.txt
    fileRoot = File.expand_path(File.dirname(__FILE__))
    modRoot = fileRoot.split(File::SEPARATOR).map {|x| x=="" ? File::SEPARATOR : x + File::SEPARATOR}
    modRoot.delete_at(-1)
    newPath = modRoot.join
    newPath = newPath + 'phin_pdf_var.txt'
    
    open(newPath) do |data_file|
      id = 1
      data_file.read.each_line do |line|
        phin_var, pdf_var, page, var_order = line.chomp.split("|")
        if(var_order.nil?) 
           execute("INSERT INTO tb_phin_pdfs(id, phin_var, pdf_var, page) VALUES (" +
              id.to_s + ",'" + phin_var + "','" + pdf_var + "','" + page + "')")
        else
           execute("INSERT INTO tb_phin_pdfs(id, phin_var, pdf_var, page, var_order) VALUES (" +
              id.to_s + ",'" + phin_var + "','" + pdf_var + "','" + page + "','" + var_order + "')")
        end
        id += 1
      end
    end
  end

  def self.down
  end
end
{'fraction_non_alphanumeric': 0.13876251137397635, 'fraction_numerical': 0.0018198362147406734, 'mean_word_length': 2.885159010600707, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3868716', 'n_tokens_mistral': 852, 'n_tokens_neox': 842, 'n_words': 170}