<commit_before>module Fanfic

  class Story

    attr_accessor :id, :identifier,
                  :title, :author,
                  :published_at, :publisher,
                  :rating, :language, :genre, :chapters,
                  :domain

    def initialize(uri)
      #https://www.fictionpress.com/s/3172150/1/r9kElsa-Is-Suffering
      @id = uri[/(http|https):\/\/(www|m)?.(fanfiction.net|fictionpress.com)\/s\/(\d+).*/, 4]
      @domain = uri[/(http|https):\/\/(www|m)?.(fanfiction.net|fictionpress.com)\/s\/(\d+).*/, 3]
      raise Exception.new "Cannot parse uri: #{uri}" unless id
      @chapters_loaded = false
    end

    def uri
      "https://www.#{domain}/s/#{id}"
    end

    def chapter_uri(chapter_id)
      "https://www.#{domain}/s/#{id}/#{chapter_id}"
    end

    def publisher
      'fanfiction.net'
    end

    def one_shot?
      @chapters.count == 0
    end

    def load_details
      puts "Loading details from: #{uri}"

      @identifier = uri

      doc = Nokogiri::HTML(open(uri))

      @title = doc.xpath('//*[@id="profile_top"]/b')[0].text
      @author = doc.xpath('//*[@id="profile_top"]/a[1]')[0].text

      @rating = doc.xpath('//*[@id="profile_top"]/span[last()]/a[1]')[0].text
      details = doc.xpath('//*[@id="profile_top"]/span[last()]/text()[2]').text<commit_msg>Correct domain is used as publisher now.<commit_after>module Fanfic

  class Story

    attr_accessor :id, :identifier,
                  :title, :author,
                  :published_at, :publisher,
                  :rating, :language, :genre, :chapters,
                  :domain

    def initialize(uri)
      #https://www.fictionpress.com/s/3172150/1/r9kElsa-Is-Suffering
      @id = uri[/(http|https):\/\/(www|m)?.(fanfiction.net|fictionpress.com)\/s\/(\d+).*/, 4]
      @domain = uri[/(http|https):\/\/(www|m)?.(fanfiction.net|fictionpress.com)\/s\/(\d+).*/, 3]
      raise Exception.new "Cannot parse uri: #{uri}" unless id
      @chapters_loaded = false
    end

    def uri
      "https://www.#{domain}/s/#{id}"
    end

    def chapter_uri(chapter_id)
      "https://www.#{domain}/s/#{id}/#{chapter_id}"
    end

    def publisher
      domain
    end

    def one_shot?
      @chapters.count == 0
    end

    def load_details
      puts "Loading details from: #{uri}"

      @identifier = uri

      doc = Nokogiri::HTML(open(uri))

      @title = doc.xpath('//*[@id="profile_top"]/b')[0].text
      @author = doc.xpath('//*[@id="profile_top"]/a[1]')[0].text

      @rating = doc.xpath('//*[@id="profile_top"]/span[last()]/a[1]')[0].text
      details = doc.xpath('//*[@id="profile_top"]/span[last()]/text()[2]').text
{'fraction_non_alphanumeric': 0.20279984865682937, 'fraction_numerical': 0.01362088535754824, 'mean_word_length': 2.385403329065301, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 10, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4359184', 'n_tokens_mistral': 992, 'n_tokens_neox': 935, 'n_words': 151}