<commit_before>namespace :cassandra do

  # TODO (davebenvenuti 10/4/12) these tasks should use the hosts field from configuration, but we need to figure out a way to deal with the embedded ruby first

  desc "Reset (Drop, Create, Remigrate) the Cassandra schema"
  task :reset do
    begin
      Rake::Task['cassandra:drop'].invoke
    rescue Thrift::Exception => e
      puts "ignoring thrift exception #{e} (keyspace probably doesn't exist)"
    end

    Rake::Task['cassandra:create'].invoke
    Rake::Task['cassandra:migrate'].invoke
  end

  desc "Drop the keyspace from Cassandra as defined in config/cassandra.yml"
  task :drop do
    client = Cassandra.new "system", ["#{`hostname`.strip}:9160"]

    begin
      puts "Dropping keyspace #{keyspace_name}..."

      with_thrift_timeout_retry do
        client.drop_keyspace keyspace_name
      end

    rescue Thrift::Exception => e
      if ENV['IGNORE_THRIFT_EXCEPTIONS']
        puts "ignoring thrift exception #{e}"
      else
        raise e
      end
    ensure
      client.disconnect!
    end
  end

  desc "Create the keyspace in Cassandra as defined in config/cassandra.yml"
  task :create do
    client = Cassandra.new "system", ["#{`hostname`.strip}:9160"]

    begin
      puts "Creating keyspace #{keyspace_name}..."

      keyspace_definition = CassandraThrift::KsDef.new({
          :name => keyspace_name,
          :strategy_class => 'org.apache.cassandra.locator.SimpleStrategy',
          :strategy_options => { 'replication_factor' => '1' },
          :cf_defs => []
      })

      with_thrift_timeout_retry do
        client.add_keyspace keyspace_definition
      end

    rescue Thrift::Exception => e
      if ENV['IGNORE_THRIFT_EXCEPTIONS']
        puts "ignoring thrift exception #{e}"
      else
        raise e
      end
    ensure
      client.disconnect!
    end

    true
  end

  desc "Create column families as defined in config/cassandra.yml"
  task :migrate do
    client = Cassandra.new keyspace_name, ["#{`hostname`.strip}:9160"]


    # the migrate task needs a little more resilience with respect to timeouts and thrift errors.  we should try 3 times with a random sleep in between

    begin
      column_families.each do |cf|
        cf_def = CassandraThrift::CfDef.new({
                                       :name => cf['name'],
                                       :column_type => cf['column_type'],
                                       :comparator_type => cf['compare_with'],
                                       :subcomparator_type => cf['compare_subcolumns_with'],
                                       :keyspace => keyspace_name
        })

        with_thrift_timeout_retry do

          client.keyspace = keyspace_name # reloads the schema so the column_families are up to date

          if client.column_families.has_key?(cf['name'])
            puts "Skipping column family #{cf['name']}, already exsits"<commit_msg>Adding Cassandra connection timeout to rake tasks.<commit_after>namespace :cassandra do

  # TODO (davebenvenuti 10/4/12) these tasks should use the hosts field from configuration, but we need to figure out a way to deal with the embedded ruby first

  desc "Reset (Drop, Create, Remigrate) the Cassandra schema"
  task :reset do
    begin
      Rake::Task['cassandra:drop'].invoke
    rescue Thrift::Exception => e
      puts "ignoring thrift exception #{e} (keyspace probably doesn't exist)"
    end

    Rake::Task['cassandra:create'].invoke
    Rake::Task['cassandra:migrate'].invoke
  end

  desc "Drop the keyspace from Cassandra as defined in config/cassandra.yml"
  task :drop do
    client = Cassandra.new "system", ["#{`hostname`.strip}:9160"], { :connect_timeout => 1 }

    begin
      puts "Dropping keyspace #{keyspace_name}..."

      with_thrift_timeout_retry do
        client.drop_keyspace keyspace_name
      end

    rescue Thrift::Exception => e
      if ENV['IGNORE_THRIFT_EXCEPTIONS']
        puts "ignoring thrift exception #{e}"
      else
        raise e
      end
    ensure
      client.disconnect!
    end
  end

  desc "Create the keyspace in Cassandra as defined in config/cassandra.yml"
  task :create do
    client = Cassandra.new "system", ["#{`hostname`.strip}:9160"]. { :connect_timeout => 1 }

    begin
      puts "Creating keyspace #{keyspace_name}..."

      keyspace_definition = CassandraThrift::KsDef.new({
          :name => keyspace_name,
          :strategy_class => 'org.apache.cassandra.locator.SimpleStrategy',
          :strategy_options => { 'replication_factor' => '1' },
          :cf_defs => []
      })

      with_thrift_timeout_retry do
        client.add_keyspace keyspace_definition
      end

    rescue Thrift::Exception => e
      if ENV['IGNORE_THRIFT_EXCEPTIONS']
        puts "ignoring thrift exception #{e}"
      else
        raise e
      end
    ensure
      client.disconnect!
    end

    true
  end

  desc "Create column families as defined in config/cassandra.yml"
  task :migrate do
    client = Cassandra.new keyspace_name, ["#{`hostname`.strip}:9160"], { :connect_timeout => 1 }


    # the migrate task needs a little more resilience with respect to timeouts and thrift errors.  we should try 3 times with a random sleep in between

    begin
      column_families.each do |cf|
        cf_def = CassandraThrift::CfDef.new({
                                       :name => cf['name'],
                                       :column_type => cf['column_type'],
                                       :comparator_type => cf['compare_with'],
                                       :subcomparator_type => cf['compare_subcolumns_with'],
                                       :keyspace => keyspace_name
        })

        with_thrift_timeout_retry do

          client.keyspace = keyspace_name # reloads the schema so the column_families are up to date

          if client.column_families.has_key?(cf['name'])
            puts "Skipping column family #{cf['name']}, already exsits"
{'fraction_non_alphanumeric': 0.09365609348914858, 'fraction_numerical': 0.006844741235392321, 'mean_word_length': 2.3657303370786518, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2733910', 'n_tokens_mistral': 1773, 'n_tokens_neox': 1708, 'n_words': 491}