<commit_before># encoding: UTF-8

require 'net/http'
require 'erb'
require 'iconv'
require 'open-uri'
require 'date'
require 'nokogiri'

class Hash
  def to_binding
    res = Object.new
    res.instance_eval("def binding_for(#{keys.join(',')}) binding end")
    res.binding_for(*values)
  end
end

class Leecher
  FicSourceURL = "http://www.fanfiction.net" 
  ChapterTemplate = 'chapter.html.erb'
  OpfTemplate = 'book.opf.erb'
  TocTemplate = 'toc.html.erb'
  
  # constructor
  def initialize(fic_id, chapter_range, legal_mode=false)
    @fic_id = fic_id
    @chapter_range = chapter_range
    @chapters = []
    @opf = nil
    @legal_mode = legal_mode
  end
  
  def run
    # get story info
    puts "Fetching http://www.fanfiction.net/s/#{@fic_id}"
    story_meta = leech_story_info
    
    # generate chapters
    @titles = []
    for i in @chapter_range do
      # a pause is needed to comply with FF.net TOS
      # plese read README file to more information about the TOS
      if @legal_mode
        puts "Making a 30 seconds pause to comply with fanfiction.net TOS..."
        puts "Read README file for more information about this issue"
        sleep 30
      end
      
      raw = leech_chapter(i)
      raw[:index] = i
      
      puts "Parsing chapter #{i}"
      create_html_file("chapter#{i}.html", ChapterTemplate, raw)
              
      @chapters.push("chapter#{i}.html")
      @titles.push(raw[:title])    
    end
    
    # generate TOC
    create_html_file("toc.html", TocTemplate, {
      :chapters => @chapters,
      :titles => @titles,
      :story_title => story_meta[:title],
      :story_author => story_meta[:author]
    })
    
    # generate OPF file
    create_html_file("book.opf", OpfTemplate, {
      :chapters => @chapters,
      :title => story_meta[:title],
      :lang => "en",
      :author => story_meta[:author],
      :toc => "toc.html"
    })
        
    puts "Generating ebook..."
    puts `kindlegen book.opf`
    
    cleanup
  end
  
  protected
  
  # creates a HTML file using the template and data provided
  def create_html_file(filename, template, data)
    File.open(filename, "w+:utf-8") do |file|
      file << ERB.new(File.read(template)).result(data.to_binding)
    end
  end
  
  # gets the book info from the front chapter in the story
  def leech_story_info
    doc = Nokogiri::HTML(open("#{FicSourceURL}/s/#{@fic_id}"), "UTF-8")    
    # get author name
    author = doc.xpath("//table[@id='gui_table1i']/tbody/tr[@class='alt2']/td/a[1]").first.inner_text
    # get story title
    title = doc.xpath("//table[@id='gui_table1i']/tbody[1]/tr[1]/td[1]/b[1]").first.inner_text
    
    {:title => title, :author => author}
  end
  
  # downloads & process a single chapter
  def leech_chapter(index)
    # puts "Leeching chapter \##{index}..."
    doc = Nokogiri::HTML(open("#{FicSourceURL}/s/#{@fic_id}/#{index}"), "UTF-8")
    
    # get chapter content inside #storytext
    content = doc.xpath('//div[@id="storytext"]').first.inner_html<commit_msg>FIX crash when trying to get title and author<commit_after># encoding: UTF-8

require 'net/http'
require 'erb'
require 'iconv'
require 'open-uri'
require 'date'
require 'nokogiri'

class Hash
  def to_binding
    res = Object.new
    res.instance_eval("def binding_for(#{keys.join(',')}) binding end")
    res.binding_for(*values)
  end
end

class Leecher
  FicSourceURL = "http://www.fanfiction.net" 
  ChapterTemplate = 'chapter.html.erb'
  OpfTemplate = 'book.opf.erb'
  TocTemplate = 'toc.html.erb'
  
  # constructor
  def initialize(fic_id, chapter_range, legal_mode=false)
    @fic_id = fic_id
    @chapter_range = chapter_range
    @chapters = []
    @opf = nil
    @legal_mode = legal_mode
  end
  
  def run
    # get story info
    puts "Fetching http://www.fanfiction.net/s/#{@fic_id}"
    story_meta = leech_story_info
    
    # generate chapters
    @titles = []
    for i in @chapter_range do
      # a pause is needed to comply with FF.net TOS
      # plese read README file to more information about the TOS
      if @legal_mode
        puts "Making a 30 seconds pause to comply with fanfiction.net TOS..."
        puts "Read README file for more information about this issue"
        sleep 30
      end
      
      raw = leech_chapter(i)
      raw[:index] = i
      
      puts "Parsing chapter #{i}"
      create_html_file("chapter#{i}.html", ChapterTemplate, raw)
              
      @chapters.push("chapter#{i}.html")
      @titles.push(raw[:title])    
    end
    
    # generate TOC
    create_html_file("toc.html", TocTemplate, {
      :chapters => @chapters,
      :titles => @titles,
      :story_title => story_meta[:title],
      :story_author => story_meta[:author]
    })
    
    # generate OPF file
    create_html_file("book.opf", OpfTemplate, {
      :chapters => @chapters,
      :title => story_meta[:title],
      :lang => "en",
      :author => story_meta[:author],
      :toc => "toc.html"
    })
        
    puts "Generating ebook..."
    puts `kindlegen book.opf`
    
    cleanup
  end
  
  protected
  
  # creates a HTML file using the template and data provided
  def create_html_file(filename, template, data)
    File.open(filename, "w+:utf-8") do |file|
      file << ERB.new(File.read(template)).result(data.to_binding)
    end
  end
  
  # gets the book info from the front chapter in the story
  def leech_story_info
    doc = Nokogiri::HTML(open("#{FicSourceURL}/s/#{@fic_id}"), "UTF-8")    
    # get author name
    author = doc.xpath("//table[@id='gui_table1i']/tbody/tr[1]/td[1]/a[1]").first.inner_text
    # get story title
    title = doc.xpath("//table[@id='gui_table1i']/tbody/tr[1]/td[1]/b[1]").first.inner_text
    {:title => title, :author => author}
  end
  
  # downloads & process a single chapter
  def leech_chapter(index)
    # puts "Leeching chapter \##{index}..."
    doc = Nokogiri::HTML(open("#{FicSourceURL}/s/#{@fic_id}/#{index}"), "UTF-8")
    
    # get chapter content inside #storytext
    content = doc.xpath('//div[@id="storytext"]').first.inner_html
{'fraction_non_alphanumeric': 0.13604094436189532, 'fraction_numerical': 0.005283143470364867, 'mean_word_length': 2.7957393483709274, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6734175', 'n_tokens_mistral': 2107, 'n_tokens_neox': 2088, 'n_words': 487}