<commit_before>require 'lib/migrate'

class RemoveSegments < ActiveRecord::Migration
  def self.up
    have_segs = select_value("SELECT count(*) FROM current_segments").to_i != 0

    if have_segs
      prefix = File.join Dir.tmpdir, "008_remove_segments.#{$$}."

      cmd = "db/migrate/008_remove_segments_helper"
      src = "#{cmd}.cc"
      if not File.exists? cmd or File.mtime(cmd) < File.mtime(src) then 
	system 'c++ -O3 -Wall `mysql_config --cflags --libs` ' +
	  "#{src} -o #{cmd}" or fail
      end

      conn_opts = ActiveRecord::Base.connection.
	instance_eval { @connection_options }
      args = conn_opts.map { |arg| arg.to_s } + [prefix]
      fail "#{cmd} failed" unless system cmd, *args

      tempfiles = ['ways', 'way_nodes', 'way_tags',
	'relations', 'relation_members', 'relation_tags'].
	map { |base| prefix + base }
      ways, way_nodes, way_tags,
	relations, relation_members, relation_tags = tempfiles
    end

    drop_table :segments
    drop_table :way_segments
    create_table :way_nodes, myisam_table do |t|
      t.column :id,          :bigint, :limit => 64, :null => false
      t.column :node_id,     :bigint, :limit => 64, :null => false
      t.column :version,     :bigint, :limit => 20, :null => false
      t.column :sequence_id, :bigint, :limit => 11, :null => false
    end
    add_primary_key :way_nodes, [:id, :version, :sequence_id]

    drop_table :current_segments
    drop_table :current_way_segments
    create_table :current_way_nodes, innodb_table do |t|
      t.column :id,          :bigint, :limit => 64, :null => false
      t.column :node_id,     :bigint, :limit => 64, :null => false
      t.column :sequence_id, :bigint, :limit => 11, :null => false
    end
    add_primary_key :current_way_nodes, [:id, :sequence_id]

    execute "TRUNCATE way_tags"
    execute "TRUNCATE ways"
    execute "TRUNCATE current_way_tags"
    execute "TRUNCATE current_ways"

    # now get the data back
    csvopts = "FIELDS TERMINATED BY ',' ENCLOSED BY '\"' ESCAPED BY '\"' LINES TERMINATED BY '\\n'"

    tempfiles.each { |fn| File.chmod 0644, fn } if have_segs

    if have_segs
      execute "LOAD DATA INFILE '#{ways}' INTO TABLE ways #{csvopts} (id, user_id, timestamp) SET visible = 1, version = 1"
      execute "LOAD DATA INFILE '#{way_nodes}' INTO TABLE way_nodes #{csvopts} (id, node_id, sequence_id) SET version = 1"
      execute "LOAD DATA INFILE '#{way_tags}' INTO TABLE way_tags #{csvopts} (id, k, v) SET version = 1"

      execute "INSERT INTO current_ways SELECT id, user_id, timestamp, visible FROM ways"
      execute "INSERT INTO current_way_nodes SELECT id, node_id, sequence_id FROM way_nodes"
      execute "INSERT INTO current_way_tags SELECT id, k, v FROM way_tags"
    end

    # and then readd the index
    add_index :current_way_nodes, [:node_id], :name => "current_way_nodes_node_idx"
<commit_msg>rails_port_0.5: Add the index before loading the data as suggested by <NAME>.<commit_after>require 'lib/migrate'

class RemoveSegments < ActiveRecord::Migration
  def self.up
    have_segs = select_value("SELECT count(*) FROM current_segments").to_i != 0

    if have_segs
      prefix = File.join Dir.tmpdir, "008_remove_segments.#{$$}."

      cmd = "db/migrate/008_remove_segments_helper"
      src = "#{cmd}.cc"
      if not File.exists? cmd or File.mtime(cmd) < File.mtime(src) then 
	system 'c++ -O3 -Wall `mysql_config --cflags --libs` ' +
	  "#{src} -o #{cmd}" or fail
      end

      conn_opts = ActiveRecord::Base.connection.
	instance_eval { @connection_options }
      args = conn_opts.map { |arg| arg.to_s } + [prefix]
      fail "#{cmd} failed" unless system cmd, *args

      tempfiles = ['ways', 'way_nodes', 'way_tags',
	'relations', 'relation_members', 'relation_tags'].
	map { |base| prefix + base }
      ways, way_nodes, way_tags,
	relations, relation_members, relation_tags = tempfiles
    end

    drop_table :segments
    drop_table :way_segments
    create_table :way_nodes, myisam_table do |t|
      t.column :id,          :bigint, :limit => 64, :null => false
      t.column :node_id,     :bigint, :limit => 64, :null => false
      t.column :version,     :bigint, :limit => 20, :null => false
      t.column :sequence_id, :bigint, :limit => 11, :null => false
    end
    add_primary_key :way_nodes, [:id, :version, :sequence_id]

    drop_table :current_segments
    drop_table :current_way_segments
    create_table :current_way_nodes, innodb_table do |t|
      t.column :id,          :bigint, :limit => 64, :null => false
      t.column :node_id,     :bigint, :limit => 64, :null => false
      t.column :sequence_id, :bigint, :limit => 11, :null => false
    end
    add_primary_key :current_way_nodes, [:id, :sequence_id]
    add_index :current_way_nodes, [:node_id], :name => "current_way_nodes_node_idx"

    execute "TRUNCATE way_tags"
    execute "TRUNCATE ways"
    execute "TRUNCATE current_way_tags"
    execute "TRUNCATE current_ways"

    # now get the data back
    csvopts = "FIELDS TERMINATED BY ',' ENCLOSED BY '\"' ESCAPED BY '\"' LINES TERMINATED BY '\\n'"

    tempfiles.each { |fn| File.chmod 0644, fn } if have_segs

    if have_segs
      execute "LOAD DATA INFILE '#{ways}' INTO TABLE ways #{csvopts} (id, user_id, timestamp) SET visible = 1, version = 1"
      execute "LOAD DATA INFILE '#{way_nodes}' INTO TABLE way_nodes #{csvopts} (id, node_id, sequence_id) SET version = 1"
      execute "LOAD DATA INFILE '#{way_tags}' INTO TABLE way_tags #{csvopts} (id, k, v) SET version = 1"

      execute "INSERT INTO current_ways SELECT id, user_id, timestamp, visible FROM ways"
      execute "INSERT INTO current_way_nodes SELECT id, node_id, sequence_id FROM way_nodes"
      execute "INSERT INTO current_way_tags SELECT id, k, v FROM way_tags"
    end

{'fraction_non_alphanumeric': 0.122533748701973, 'fraction_numerical': 0.010730356524749048, 'mean_word_length': 3.381349507202426, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7568051', 'n_tokens_mistral': 2056, 'n_tokens_neox': 1941, 'n_words': 574}