<commit_before># encoding: utf-8

class Unstuck < Exception
end

# Grabs all the fragments that make up the media file and joins
# them within a local file.
#
class Grabber
  attr_reader :urls

  SANE_FRAGMENT_MAX = 10_000
  SANE_FRAGMENT_MIN = 1

  def initialize(manifest, bootstrap, io = nil, do_download = true)
    # As we've hardcoded 1 into the @url below
    fail Unstuck, 'Only one segment can be handled' if bootstrap.segments != 1
    fail Unstuck, 'Not enough fragments' if bootstrap.fragments < SANE_FRAGMENT_MIN
    fail Unstuck, 'Too many fragments' if bootstrap.fragments > SANE_FRAGMENT_MAX

    @uri = "#{manifest.media_filename}.flv"
    @url = "#{manifest.base_ref}/#{manifest.media_filename}Seg1-Frag"
    @total_fragments = bootstrap.fragments
    @urls = []
    @downloaded_fragments = []
    @fragments_downloaded = 0

    # TODO: Track how much has already been downloaded and append from that point
    fail Unstuck, "Aborting as the download target file '#{@uri}' already exists" if File.exist? @uri

    @out = io
    @out = File.new(@uri, 'ab') if @out.nil?

    # TODO: Test first fragment for audio and video write header accordingly
    @out.write(FLV.header)

    build
    download_all if do_download
  end

  private

  def build
    # TODO, correctly set the fragment start
    fragment = 1
    while fragment <= @total_fragments
      @urls << "#{@url}#{fragment}"
      fragment += 1
    end
  end

  def download_all
    @urls.each { |url| download url }
  end<commit_msg>method extraction for easy reading<commit_after># encoding: utf-8

class Unstuck < Exception
end

# Grabs all the fragments that make up the media file and joins
# them within a local file.
#
class Grabber
  attr_reader :urls

  SANE_FRAGMENT_MAX = 10_000
  SANE_FRAGMENT_MIN = 1

  def initialize(manifest, bootstrap, io = nil, do_download = true)
    # As we've hardcoded 1 into the @url below
    fail Unstuck, 'Only one segment can be handled' if bootstrap.segments != 1
    fail Unstuck, 'Not enough fragments' if bootstrap.fragments < SANE_FRAGMENT_MIN
    fail Unstuck, 'Too many fragments' if bootstrap.fragments > SANE_FRAGMENT_MAX

    @uri = "#{manifest.media_filename}.flv"
    @url = "#{manifest.base_ref}/#{manifest.media_filename}Seg1-Frag"
    @total_fragments = bootstrap.fragments
    @urls = []
    @downloaded_fragments = []
    @fragments_downloaded = 0
    @out = io

    create_download_file
    build
    download_all if do_download
  end

  private

  def create_download_file
    # TODO: Track how much has already been downloaded and append from that point
    fail Unstuck, "Aborting as the download target file '#{@uri}' already exists" if File.exist? @uri

    @out = File.new(@uri, 'ab') if @out.nil?

    # TODO: Test first fragment for audio and video write header accordingly
    @out.write(FLV.header)
  end

  def build
    # TODO, correctly set the fragment start
    fragment = 1
    while fragment <= @total_fragments
      @urls << "#{@url}#{fragment}"
      fragment += 1
    end
  end

  def download_all
    @urls.each { |url| download url }
  end
{'fraction_non_alphanumeric': 0.08835859400193485, 'fraction_numerical': 0.00838439213157046, 'mean_word_length': 3.3751763046544427, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6571620', 'n_tokens_mistral': 1034, 'n_tokens_neox': 968, 'n_words': 331}