<commit_before>require 'rubygems'
require 'google_drive'
require 'open-uri'
require 'trollop'
require 'yaml'

OPTS = Trollop::options do
    banner <<-EOS
Download image files listed in fireslurp's Google Spreadsheet and make a
timelapse movie.

Usage:

  bundle exec ruby moviemaker.rb -c fireslurp.yaml morganfire01

where [options] are:
EOS
  opt :debug, "Print debug statements", :type => :boolean, :short => "-d"
  opt :config, "Path to YAML configuration file", :type => :string, :short => "-c"
  opt :folder, "Folder to write images and movie", :type => :string, :short => "-f"
  opt :width, "Width of the movie", :type => :integer, :short => "-w", :default => 500
  opt :skip_download, "Skip the downloads and assume files are already in the work path", :type => :boolean, :default => false
  opt :skip_alignment, "Skip the alignment and assume files are already in the work path", :type => :boolean, :default => false
  opt :google_email, "Google email address, used to add data to a Google spreadsheet", :type => :string
  opt :google_password, "Google account password", :type => :string
  opt :google_spreadsheet_id, "
    Write data to a Google Spreadsheet instead of CSV. The value should be the
    Google Spreadhseet ID. If it's blank and the flag is used, the script will
    look for an internal variable.
  ".strip, :type => :string, :short => '-g'
  opt :timelapsing, "timelapsing_detailed command (could be full path to binary)", :type => :string, :default => "timelapsing_detailed"
end

config = if OPTS[:config] && File.exists?(OPTS[:config])
  YAML.load_file(OPTS[:config])
else
  {}
end

## CONFIG ###############################################
GOOGLE_EMAIL          = OPTS[:google_email]           || config['google_email']
GOOGLE_PASSWORD       = OPTS[:google_password]        || config['google_password']
GOOGLE_SPREADSHEET_ID = OPTS[:google_spreadsheet_id]  || config['google_spreadsheet_id']
###############################################################

HEADERS = %w(provider tag datetime username usable_tag image_url url image_url_s image_url_m license title)

@tag = ARGV[0].to_s.gsub(/[#\s]/, '')
@work_path = if OPTS[:folder]
  File.expand_path(OPTS[:folder])
else
  File.join(Dir::tmpdir, "#{@tag}-#{Time.now.to_i}")
end
FileUtils.mkdir_p @work_path, :mode => 0755

def system_call(cmd)
  puts "Running #{cmd}" if OPTS[:debug]
  system cmd
end

def download_images
  puts "DOWNLOADING IMAGES"  if OPTS[:debug]
  session = GoogleDrive.login(GOOGLE_EMAIL, GOOGLE_PASSWORD)
  ws = session.spreadsheet_by_key(GOOGLE_SPREADSHEET_ID).worksheets[0]
  rows = ws.rows.select {|r| @tag == r[HEADERS.index('usable_tag')]}
  rows.each_with_index do |row,i|
    provider, row_tag, datetime, username, usable_tag, image_url, url, image_url_s, image_url_m, license, title = row
    next unless @tag == usable_tag
    base = File.basename(image_url)
    ext = File.extname(image_url).split(':')[0]
    fname = "#{@tag}-#{datetime}-#{provider}-#{username}-#{base.split('.')[0]}#{ext}".gsub(/[^A-z0-9\-\_\.]+/, '_')
    path = File.join(@work_path, fname)
    puts "#{i+1} of #{rows.size}, writing\n\t#{image_url} to \n\t#{path}"
    begin
      f = open(image_url)
      open(path, 'wb') do |file|
        file << f.read
      end
      f.close
    rescue OpenURI::HTTPError => e
      if e.message =~ /404 Not Found/
        puts "\tImage gone"
      end
    end
  end
end

def resize_images(options = {})
  puts "RESIZING RAW IMAGES"  if OPTS[:debug]
  prefix = "#{options[:prefix] || "#{@tag}-"}*"
  Dir.glob(File.join(@work_path, prefix)).each do |path|
    basename = File.basename(path)
    system_call "convert #{path} -resize #{OPTS[:width]} #{@work_path}/resized-#{basename}"
  end
end

def align_images
  puts "ALIGNING" if OPTS[:debug]
  system_call "cd #{@work_path} && #{OPTS[:timelapsing]} resized-*"
end

def create_movie
  puts "CREATING MOVIE" if OPTS[:debug]
  system_call "convert -delay 30 #{@work_path}/resized-fixed_* #{@work_path}/#{@tag}.gif"
end

download_images unless OPTS[:skip_download]
resize_images
align_images unless OPTS[:skip_alignment]
resize_images(:prefix => "fixed_")
create_movie<commit_msg>Minor changes to moviemaker.<commit_after>require 'rubygems'
require 'google_drive'
require 'open-uri'
require 'trollop'
require 'yaml'

OPTS = Trollop::options do
    banner <<-EOS
Download image files listed in fireslurp's Google Spreadsheet and make a
timelapse movie.

Usage:

  bundle exec ruby moviemaker.rb -c fireslurp.yaml morganfire01

where [options] are:
EOS
  opt :debug, "Print debug statements", :type => :boolean, :short => "-d"
  opt :config, "Path to YAML configuration file", :type => :string, :short => "-c"
  opt :folder, "Folder to write images and movie", :type => :string, :short => "-f"
  opt :width, "Width of the movie", :type => :integer, :short => "-w", :default => 500
  opt :skip_download, "Skip the downloads and assume files are already in the work path", :type => :boolean, :default => false
  opt :skip_alignment, "Skip the alignment and assume files are already in the work path", :type => :boolean, :default => false
  opt :google_email, "Google email address, used to add data to a Google spreadsheet", :type => :string
  opt :google_password, "Google account password", :type => :string
  opt :google_spreadsheet_id, "
    Write data to a Google Spreadsheet instead of CSV. The value should be the
    Google Spreadhseet ID. If it's blank and the flag is used, the script will
    look for an internal variable.
  ".strip, :type => :string, :short => '-g'
  opt :timelapsing, "timelapsing_detailed command (could be full path to binary)", :type => :string, :default => "timelapsing_detailed"
  opt :limit, "Limit the stitch to the first n files. Just for testing.", :type => :integer
end

config = if OPTS[:config] && File.exists?(OPTS[:config])
  YAML.load_file(OPTS[:config])
else
  {}
end

## CONFIG ###############################################
GOOGLE_EMAIL          = OPTS[:google_email]           || config['google_email']
GOOGLE_PASSWORD       = OPTS[:google_password]        || config['google_password']
GOOGLE_SPREADSHEET_ID = OPTS[:google_spreadsheet_id]  || config['google_spreadsheet_id']
###############################################################

HEADERS = %w(provider tag datetime username usable_tag image_url url image_url_s image_url_m license title)

@tag = ARGV[0].to_s.gsub(/[#\s]/, '')
@work_path = if OPTS[:folder]
  File.expand_path(OPTS[:folder])
else
  File.join(Dir::tmpdir, "#{@tag}-#{Time.now.to_i}")
end
FileUtils.mkdir_p @work_path, :mode => 0755

def system_call(cmd)
  puts "Running #{cmd}" if OPTS[:debug]
  system cmd
end

def download_images
  puts "DOWNLOADING IMAGES"  if OPTS[:debug]
  session = GoogleDrive.login(GOOGLE_EMAIL, GOOGLE_PASSWORD)
  ws = session.spreadsheet_by_key(GOOGLE_SPREADSHEET_ID).worksheets[0]
  rows = ws.rows.select {|r| @tag == r[HEADERS.index('usable_tag')]}
  rows.each_with_index do |row,i|
    next if OPTS.limit && OPTS.limit < i+1
    provider, row_tag, datetime, username, usable_tag, image_url, url, image_url_s, image_url_m, license, title = row
    next unless @tag == usable_tag
    if image_url.empty?
      puts "\timage_url blank, skipping..."
      next
    end
    base = File.basename(image_url)
    ext = File.extname(image_url).split(':')[0]
    fname = "#{@tag}-#{datetime}-#{provider}-#{username}-#{base.split('.')[0]}#{ext}".gsub(/[^A-z0-9\-\_\.]+/, '_')
    path = File.join(@work_path, fname)
    puts "#{i+1} of #{rows.size}, writing\n\t#{image_url} to \n\t#{path}"
    begin
      f = open(image_url)
      open(path, 'wb') do |file|
        file << f.read
      end
      f.close
    rescue OpenURI::HTTPError => e
      if e.message =~ /404 Not Found/
        puts "\tImage gone"
      end
    end
  end
end

def resize_images(options = {})
  puts "RESIZING RAW IMAGES"  if OPTS[:debug]
  prefix = "#{options[:prefix] || "#{@tag}-"}*"
  Dir.glob(File.join(@work_path, prefix)).each do |path|
    basename = File.basename(path)
    system_call "convert #{path} -resize #{OPTS[:width]} #{@work_path}/resized-#{basename}"
  end
end

def align_images
  puts "ALIGNING" if OPTS[:debug]
  system_call "cd #{@work_path} && #{OPTS[:timelapsing]} resized-*"
end

def create_movie
  puts "CREATING MOVIE" if OPTS[:debug]
  prefix = if OPTS[:skip_alignment]
    "resized-"
  else
    "resized-fixed_"
  end
  system_call "convert -delay 30 #{@work_path}/#{prefix}* #{@work_path}/#{@tag}.gif"
end

download_images unless OPTS[:skip_download]
resize_images
align_images unless OPTS[:skip_alignment]
resize_images(:prefix => "fixed_")
create_movie
{'fraction_non_alphanumeric': 0.16869565217391305, 'fraction_numerical': 0.0049855072463768114, 'mean_word_length': 4.364427860696518, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '292450', 'n_tokens_mistral': 2994, 'n_tokens_neox': 2922, 'n_words': 815}