<commit_before>#!/usr/bin/env ruby

require 'yaml'

class Hbase_Backup

  def initialize(dt)
    @dt = dt
  end
 
  def backup(table, start, versions, type)

    `hbase org.apache.hadoop.hbase.mapreduce.Export -D mapred.output.compression.codec=org.apache.hadoop.io.compress.SnappyCodec -D mapred.output.compression.type=BLOCK -D mapred.output.compress=true #{table} /tmp/backups.#{@dt}/#{table}.#{type}/ #{versions} #{start}`
  end 

end

def s3(bucket, dt)
  `hadoop fs -mkdir s3n://#{bucket}/hbase_backups/#{dt}/`
  `hadoop fs -cp /tmp/backups.#{dt}/* s3n://#{bucket}/hbase_backups/#{dt}/`
  `hadoop fs -rmr /tmp/backups.#{dt}`
end

config = YAML::load(File.open('<%= @node[:backups][:hbase][:conf] %>', 'r').read())
tables = config['tables']
versions = config['backups']['versions']

dt = Date.today().strftime("%Y%m%d")
today = Date.today().strftime("%A")
start = nil
type = nil

tables.each do |table|
  if config.has_key?(table)
    full = config[table]['full']
    diff = config[table]['differential'] 
    incr = config[table]['incrimental'] 
  else
    full = config['default']['full']
    diff = config['default']['differential']
    incr = config['default']['incrimental']
  end
 
  last_full = full.last
  last_diff = diff.last
  hour_offset = config['backups']['offset']

  if full.include? today
    start = 0 
    type = 'full'
  elsif diff.include? today
    start = `date -d "last-#{last_full}" +%s`.chomp # This is hacky, but it works...  
    type = 'differential'
  elsif incr.include? today
    start = `date -d "last-#{last_diff}" +%s`.chomp
    type = 'incrimental'
  end
    
  backup = Hbase_Backup.new(dt)  
  backup.backup(table, start, versions, type)

end
  
s3(config['backups']['s3'], dt)<commit_msg>HBase backups now using distcp<commit_after>#!/usr/bin/env ruby

require 'yaml'

class Hbase_Backup

  def initialize(dt)
    @dt = dt
  end
 
  def backup(table, start, versions, type)

    `hbase org.apache.hadoop.hbase.mapreduce.Export -D mapred.output.compression.codec=org.apache.hadoop.io.compress.SnappyCodec -D mapred.output.compression.type=BLOCK -D mapred.output.compress=true #{table} /tmp/backups/#{type}/#{@dt}/#{table}/ #{versions} #{start}`
  end

end

def s3(bucket, type, dt)
  `hadoop distcp /tmp/backups/#{type}/#{dt}/ s3n://#{bucket}/hbase_backups/#{type}/#{dt}/`
  `hadoop fs -rmr /tmp/backups`
end

config = YAML::load(File.open('<%= @node[:backups][:hbase][:conf] %>', 'r').read())
tables = config['tables']
versions = config['backups']['versions']

dt = Date.today().strftime("%Y%m%d")
today = Date.today().strftime("%A")
start = nil
type = nil

tables.each do |table|
  if config.has_key?(table)
    full = config[table]['full']
    diff = config[table]['differential'] 
    incr = config[table]['incrimental'] 
  else
    full = config['default']['full']
    diff = config['default']['differential']
    incr = config['default']['incrimental']
  end
 
  last_full = full.last
  last_diff = diff.last
  hour_offset = config['backups']['offset']

  if full.include? today
    start = 0 
    type = 'full'
  elsif diff.include? today
    start = `date -d "last-#{last_full}" +%s`.chomp # This is hacky, but it works...  
    type = 'differential'
  elsif incr.include? today
    start = `date -d "last-#{last_diff}" +%s`.chomp
    type = 'incrimental'
  end
    
  backup = Hbase_Backup.new(dt)  
  backup.backup(table, start, versions, type)

end
  
s3(config['backups']['s3'], type, dt)
{'fraction_non_alphanumeric': 0.18559395875689805, 'fraction_numerical': 0.003194888178913738, 'mean_word_length': 3.8990042674253202, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1458888', 'n_tokens_mistral': 1273, 'n_tokens_neox': 1250, 'n_words': 262}