<commit_before>#!/usr/bin/ruby

require 'optparse'
require 'rubygems'
require 'votigoto'
require 'duration'
require 'tempfile'

class TivoFetcher

  def initialize(tivo, show, options)
    @mak = tivo.mak
    @show = show
    @options = options
  end

  def make_filename(extension)
    if @options[:show_dir] && @show.episode_number then
      Dir.mkdir(@show.title) unless Dir.exists?(@show.title)
      season_num = @show.episode_number / 100;
      season_episode = @show.episode_number % 100;
      season_dir = @show.title + '/' \
                   + 'Season ' + sprintf("%02d", season_num)
      Dir.mkdir(season_dir) unless Dir.exists?(season_dir)
      base_name = season_dir + '/' \
                  + @show.title + ' - ' \
                  + sprintf("s%02de%02d", season_num, season_episode) \
                  + (@show.episode_title ? ' - ' + @show.episode_title : '')
    else
      base_name = @show.to_s
    end
    filename = base_name + '.' + extension
    if @options[:skip] then
      if File.exists?(filename) && File.size(filename) > 100 then
        puts "skipping #{filename}"
        return nil
      end
    else 
      counter = 0
      while File.exists?(filename) && File.size(filename) > 100 do
        counter += 1
        filename = (base_name + '.' + counter.to_s + '.' + extension)
      end
    end
    puts "fetch #{filename}"
    return filename
  end

  def fetch_cmd
    url = @show.content_url
    cookie_file = Tempfile.new('cookies').path
    return %Q{curl #{@options[:verbose] ? '-v' : '-q'} --cookie-jar #{cookie_file} --digest --user "tivo:#{@mak}" "#{url}"}
  end

  def smart_fetch
    if @options[:encode] then
      fetch_and_encode
    elsif @options[:decode] then
      fetch_and_decode
    else
      fetch
    end
  end

  def run_cmd(extension, &get_cmd)
    filename = make_filename(extension)
    return unless filename
    temp_filename = filename + ".tmp"
    cmd = get_cmd.call(temp_filename)
    puts cmd if @options[:verbose]
    system(cmd) or raise "#{cmd} failed: $?"
    File.rename(temp_filename, filename)
    sleep(2)
  end
  
  def fetch
    run_cmd("tivo") { |temp_filename|
      fetch_cmd() \
        + %Q{ -o "#{temp_filename}"}
    }
  end

  def fetch_and_decode
    run_cmd("mpg") { |temp_filename|
      fetch_cmd() \
        + %Q{ | tivodecode --mak #{@mak} -o "#{temp_filename}" -}
    }
  end

  def fetch_and_encode
    run_cmd("mp4") { |temp_filename|
      fetch_cmd() \
        + %Q{ | tivodecode --mak #{@mak} -} \
        + %Q{ | ffmpeg -loglevel #{@options[:verbose] ? 'info' : 'error' } -i - -c:v libx264 -preset fast -c:a copy -f mp4 "#{temp_filename}"}
      #%Q{ | mencoder -quiet -profile low -o "#{temp_filename}" -}
    }
  end

end

def nice_file_size(size)
    units = %w{B KB MB GB TB} 
    e = (Math.log(size)/Math.log(1024)).floor 
    s = "%.3f" % (size.to_f / 1024.0**e)
    s.sub(/\.?0*$/, " " + units[e])
end

def nice_duration(msec)
    Duration.new(:seconds => (msec / 1000)).strftime("%h:%M:%S")
end

def print_details(show)
    puts "title: #{show.title}"
    puts "episode_title: #{show.episode_title}"
    puts "source_station: #{show.source_station}"
    puts "program_id: #{show.program_id}"
    puts "series_id: #{show.series_id}"
    puts "source_size: #{nice_file_size(show.source_size)}"
    puts "duration: #{nice_duration(show.duration)}"
    puts "source_channel: #{show.source_channel}"
    puts "capture_date: #{show.capture_date.strftime("%Y-%m-%d %H:%M:%S")}"
    puts "in_progress: #{show.in_progress}"
    puts "episode_number: #{show.episode_number}"
    puts
end

host = nil
mak = nil

options = {}
OptionParser.new do |opts|
  opts.banner = "Usage: tivo_fetch.rb [options]"
  
  opts.on("--host HOST", "Host") { |val|
    host = val
  }
  opts.on("--mak MAK", "Media Access Key") { |val|
    mak = val
  }
  opts.on("--detail", "Details") { |bool|
    options[:detail] = bool
  }
  opts.on("--decode", "Decode") { |bool|
    options[:decode] = bool
  }
  opts.on("--mp4", "Encode MP4") { |bool|
    options[:encode] = bool
  }
  opts.on("--fetch", "Fetch") { |bool|
    options[:fetch] = bool
  }
  opts.on("--skip", "Skip existing files") { |bool|
    options[:skip] = bool
  }
  opts.on("--show", "Show directories") { |bool|
    options[:show_dir] = bool
  }
  opts.on("--verbose", "Verbose") { |bool|
    options[:verbose] = bool
  }
end.parse!

tivo = Votigoto::Base.new(host, mak)

if ! ARGV.empty? then
  ARGV.each do |id|
    if options[:detail] then
      print_details(tivo.show(id))
    else
      TivoFetcher.new(tivo, tivo.show(id), options).smart_fetch()
    end
  end
else
  tivo.shows.each do |show|
    if options[:detail] then
      print_details(show)
    elsif options[:decode] || options[:fetch] || options[:encode] then
      if show.in_progress then
        puts "in progress #{show.to_s}"
        next
      elsif show.copy_protected then
        puts "copy protected #{show.to_s}"
        next
      end
      TivoFetcher.new(tivo, show, options).smart_fetch()
    else
      puts "[#{show.program_id}] #{show}"
    end
  end
end
<commit_msg>Display program_id, ffmpeg quality, verbose fixes<commit_after>#!/usr/bin/ruby

require 'optparse'
require 'rubygems'
require 'votigoto'
require 'duration'
require 'tempfile'

class TivoFetcher

  def initialize(tivo, show, options)
    @mak = tivo.mak
    @show = show
    @options = options
  end

  def make_filename(extension)
    if @options[:show_dir] && @show.episode_number then
      Dir.mkdir(@show.title) unless Dir.exists?(@show.title)
      season_num = @show.episode_number / 100;
      season_episode = @show.episode_number % 100;
      season_dir = @show.title + '/' \
                   + 'Season ' + sprintf("%02d", season_num)
      Dir.mkdir(season_dir) unless Dir.exists?(season_dir)
      base_name = season_dir + '/' \
                  + @show.title + ' - ' \
                  + sprintf("s%02de%02d", season_num, season_episode) \
                  + (@show.episode_title ? ' - ' + @show.episode_title : '')
    else
      base_name = @show.to_s
    end
    filename = base_name + '.' + extension
    if @options[:skip] then
      if File.exists?(filename) && File.size(filename) > 100 then
        puts "skipping #{@show.program_id} #{filename}"
        return nil
      end
    else 
      counter = 0
      while File.exists?(filename) && File.size(filename) > 100 do
        counter += 1
        filename = (base_name + '.' + counter.to_s + '.' + extension)
      end
    end
    puts "fetch #{@show.program_id} #{filename}"
    return filename
  end

  def fetch_cmd
    url = @show.content_url
    cookie_file = Tempfile.new('cookies').path
    return %Q{curl #{@options[:verbose] ? '-v' : '-s'} --cookie-jar #{cookie_file} --digest --user "tivo:#{@mak}" "#{url}"}
  end

  def smart_fetch
    if @options[:encode] then
      fetch_and_encode
    elsif @options[:decode] then
      fetch_and_decode
    else
      fetch
    end
  end

  def run_cmd(extension, &get_cmd)
    filename = make_filename(extension)
    return unless filename
    temp_filename = filename + ".tmp"
    cmd = get_cmd.call(temp_filename)
    puts cmd if @options[:verbose]
    system(cmd) or raise "#{cmd} failed: #{$?}"
    File.rename(temp_filename, filename)
    sleep(5)
  end
  
  def fetch
    run_cmd("tivo") { |temp_filename|
      fetch_cmd() \
        + %Q{ -o "#{temp_filename}"}
    }
  end

  def fetch_and_decode
    run_cmd("mpg") { |temp_filename|
      fetch_cmd() \
        + %Q{ | tivodecode --mak #{@mak} -o "#{temp_filename}" - #{@options[:verbose] ? '' : ' 2>/dev/null'}}
    }
  end

  def fetch_and_encode
    run_cmd("m4v") { |temp_filename|
      fetch_cmd() \
        + %Q{ | tivodecode --mak #{@mak} - 2>/dev/null} \
        + %Q{ | ffmpeg -loglevel #{@options[:verbose] ? 'info' : 'fatal' } -i - -c:v libx264 -crf 23 -c:a copy -f mp4 "#{temp_filename}"}
      #%Q{ | mencoder -quiet -profile low -o "#{temp_filename}" -}
    }
  end

end

def nice_file_size(size)
    units = %w{B KB MB GB TB} 
    e = (Math.log(size)/Math.log(1024)).floor 
    s = "%.3f" % (size.to_f / 1024.0**e)
    s.sub(/\.?0*$/, " " + units[e])
end

def nice_duration(msec)
    Duration.new(:seconds => (msec / 1000)).strftime("%h:%M:%S")
end

def print_details(show)
    puts "title: #{show.title}"
    puts "episode_title: #{show.episode_title}"
    puts "source_station: #{show.source_station}"
    puts "program_id: #{show.program_id}"
    puts "series_id: #{show.series_id}"
    puts "source_size: #{nice_file_size(show.source_size)}"
    puts "duration: #{nice_duration(show.duration)}"
    puts "source_channel: #{show.source_channel}"
    puts "capture_date: #{show.capture_date.strftime("%Y-%m-%d %H:%M:%S")}"
    puts "in_progress: #{show.in_progress}"
    puts "episode_number: #{show.episode_number}"
    puts
end

host = nil
mak = nil

options = {}
OptionParser.new do |opts|
  opts.banner = "Usage: tivo_fetch.rb [options]"
  
  opts.on("--host HOST", "Host") { |val|
    host = val
  }
  opts.on("--mak MAK", "Media Access Key") { |val|
    mak = val
  }
  opts.on("--detail", "Details") { |bool|
    options[:detail] = bool
  }
  opts.on("--decode", "Decode") { |bool|
    options[:decode] = bool
  }
  opts.on("--mp4", "Encode MP4") { |bool|
    options[:encode] = bool
  }
  opts.on("--encode", "Encode MP4") { |bool|
    options[:encode] = bool
  }
  opts.on("--fetch", "Fetch") { |bool|
    options[:fetch] = bool
  }
  opts.on("--skip", "Skip existing files") { |bool|
    options[:skip] = bool
  }
  opts.on("--show", "Show directories") { |bool|
    options[:show_dir] = bool
  }
  opts.on("--verbose", "Verbose") { |bool|
    options[:verbose] = bool
  }
end.parse!

tivo = Votigoto::Base.new(host, mak)

if ! ARGV.empty? then
  ARGV.each do |id|
    if options[:detail] then
      print_details(tivo.show(id))
    else
      TivoFetcher.new(tivo, tivo.show(id), options).smart_fetch()
    end
  end
else
  tivo.shows.each do |show|
    if options[:detail] then
      print_details(show)
    elsif options[:decode] || options[:fetch] || options[:encode] then
      if show.in_progress then
        puts "in progress #{show.program_id} #{show.to_s}"
        next
      elsif show.copy_protected then
        puts "copy protected #{show.program_id} #{show.to_s}"
        next
      end
      TivoFetcher.new(tivo, show, options).smart_fetch()
    else
      puts "#{show.program_id} #{show}"
    end
  end
end

{'fraction_non_alphanumeric': 0.16342561548041, 'fraction_numerical': 0.008717310087173101, 'mean_word_length': 2.7245808062789867, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1299915', 'n_tokens_mistral': 3866, 'n_tokens_neox': 3747, 'n_words': 812}