<commit_before># require 'mangad/site_parser_base'
require 'mangad/site_parser_base'
module Mangad
  class MangaHostParser < SiteParserBase

    # url used to search in the site
    SEARCH_URL = "http://mangahost.net/find/"

    # regex to extract url of found mangas
    SEARCH_LINK_REGEX = /entry-title">\s*<a\s*href="(.*)?"\s*title="(.*)"/

    # Regex to extract chapters' url from manga page.
    # Manga host has two diferent manga pages.
    # One to medium/short mangas and one to big mangas
    # /capitulo['"]\stitle=['"]Ler Online\s+-\s+(.*?)['"]\s+href=['"](.*?)['"]/, # for short/medium mangas
    CHAPTER_LINK_REGEX = [
      /capitulo.*?Ler\s+Online\s+-\s+(.*?)['"]\s+href=['"](.*?)['"]/, # for short/medium mangas
      /<a\s+href=['"](.*?)['"]\s+title=['"]Ler\s+Online\s+-\s+(.*?)\s+\[\]/ # for big mangas
    ]

    # regex to extract images' url from chapter page
    IMG_LINK_REGEX = [/img_\d+['"]\s+src=['"](.*?)['"]/,
                      /url['"]:['"](.*?)['"]\}/]

    def download

      # white space is not allowed in the search url. 
      guess_manga_name = @options.manga.gsub(/\s/, '+') # Replace ' ' by  '+'

      search("#{SEARCH_URL}#{guess_manga_name}", SEARCH_LINK_REGEX)

      # puts "executing chapters".green ## DEBUG
      # Due the organazation of the chapters page the chapters are
      # extracted in reverse order
      @chapters = parse(@manga_url, CHAPTER_LINK_REGEX[0]) do |resul, page|

        # use long mangas regex if short mangas regex
        # returns empty array
        if resul.empty?
          # Extract chapters name and url and
          # swap chapter[0](name) with chapter[1](url)
          # to conform with result from CHAPTER_LINK_REGEX[0]
          page.scan(CHAPTER_LINK_REGEX[1]) {|chapter| resul << chapter.rotate}
        end
        resul
      end

      # correct utf-8 errors
      correct_chapters_name

      # prompt user to select chapters to download
      select_chapters

      # create manga directory
      create_dir(@manga_name)

      # download selected chapters
      @chapters.reverse_each do |chapter|
        imgs_url = parse(chapter[1], IMG_LINK_REGEX[0]) do |resul, page|
          # use second pattern if the first returns a empty
          # array
          if resul.empty?
            page.scan(IMG_LINK_REGEX[1]) do |img|
              resul << img.sub!(%{//}, "")
            end
          end

          resul.each do |img|
            # some images urls are incorrect and need to be corrected. For exemple:
            # img.mangahost.net/br/images/img.png.webp => img.mangahost.net/br/mangas_files/img.png
            img.sub!(/images/, "mangas_files")
            img.sub!(/\.webp/, "")
          end
          resul
        end

        # create chapter directory relative to manga directory
        chapter_name = "#{chapter[0]}"<commit_msg>added block to use second img link regex<commit_after># require 'mangad/site_parser_base'
require 'mangad/site_parser_base'
module Mangad
  class MangaHostParser < SiteParserBase

    # url used to search in the site
    SEARCH_URL = "http://mangahost.net/find/"

    # regex to extract url of found mangas
    SEARCH_LINK_REGEX = /entry-title">\s*<a\s*href="(.*)?"\s*title="(.*)"/

    # Regex to extract chapters' url from manga page.
    # Manga host has two diferent manga pages.
    # One to medium/short mangas and one to big mangas
    # /capitulo['"]\stitle=['"]Ler Online\s+-\s+(.*?)['"]\s+href=['"](.*?)['"]/, # for short/medium mangas
    CHAPTER_LINK_REGEX = [
      /capitulo.*?Ler\s+Online\s+-\s+(.*?)['"]\s+href=['"](.*?)['"]/, # for short/medium mangas
      /<a\s+href=['"](.*?)['"]\s+title=['"]Ler\s+Online\s+-\s+(.*?)\s+\[\]/ # for big mangas
    ]

    # regex to extract images' url from chapter page
    IMG_LINK_REGEX = [/img_\d+['"]\s+src=['"](.*?)['"]/,
                      /url['"]:['"](.*?)['"]\}/]

    def download

      # white space is not allowed in the search url. 
      guess_manga_name = @options.manga.gsub(/\s/, '+') # Replace ' ' by  '+'

      search("#{SEARCH_URL}#{guess_manga_name}", SEARCH_LINK_REGEX)

      # puts "executing chapters".green ## DEBUG
      # Due the organazation of the chapters page the chapters are
      # extracted in reverse order
      @chapters = parse(@manga_url, CHAPTER_LINK_REGEX[0]) do |resul, page|

        # use long mangas regex if short mangas regex
        # returns empty array
        if resul.empty?
          # Extract chapters name and url and
          # swap chapter[0](name) with chapter[1](url)
          # to conform with result from CHAPTER_LINK_REGEX[0]
          page.scan(CHAPTER_LINK_REGEX[1]) {|chapter| resul << chapter.rotate}
        end
        resul
      end

      # correct utf-8 errors
      correct_chapters_name

      # prompt user to select chapters to download
      select_chapters

      # create manga directory
      create_dir(@manga_name)

      # download selected chapters
      @chapters.reverse_each do |chapter|
        imgs_url = parse(chapter[1], IMG_LINK_REGEX[0]) do |resul, page|
          # use second pattern if the first returns a empty
          # array
          # binding.pry # DEBUG
          if resul.empty?
            page.scan(IMG_LINK_REGEX[1]) do |img|
              resul << img[0].gsub!(%r{\\}, "")
            end
          end

          resul.each do |img|
            # some images urls are incorrect and need to be corrected. For exemple:
            # img.mangahost.net/br/images/img.png.webp => img.mangahost.net/br/mangas_files/img.png
            img.sub!(/images/, "mangas_files")
            img.sub!(/\.webp/, "")
          end
          resul
        end

        # create chapter directory relative to manga directory
        chapter_name = "#{chapter[0]}"
{'fraction_non_alphanumeric': 0.14729354614850798, 'fraction_numerical': 0.0036433032616238724, 'mean_word_length': 2.3381586566299943, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6515113', 'n_tokens_mistral': 1945, 'n_tokens_neox': 1865, 'n_words': 474}