<commit_before>"""Extractor for hpfanficarchive.com."""

from fanfic_scraper.base_fanfic import BaseFanfic, BaseChapter
from urllib.parse import urlparse, urljoin, parse_qs
import requests
from bs4 import BeautifulSoup, Comment
from collections import defaultdict
import re
import os
import shutil
import textwrap
from datetime import datetime
from random import shuffle, uniform
from copy import deepcopy
from time import sleep

def chapter_nav(tag):
    test = (tag.name == 'select')
    test = (test and 'chap_select' in tag['id'])

    return test

def send_request(url,verify):

    #Needed to bypass paranoid filtering for some hosts...
    proxies = {
               'http': 'socks5:127.0.0.1:9050',
               'https': 'socks5:127.0.0.1:9050'
              }

    r = requests.get(url, proxies=proxies, verify=verify)

    return r

class HPFanficArchive(BaseFanfic):

    def get_fanfic_title(self,r):
        soup = BeautifulSoup(r.text, 'html5lib')

        for div in soup.find_all('div', {'id':'pagetitle'}):
            title = div.find_all('a', href=re.compile(r'^viewstory.php\?sid='))[0]
            title = title.get_text()
            break

        return title

    def get_story_url(self, storyid):
        return 'https://www.hpfanficarchive.com/stories/viewstory.php?sid='+storyid

    def extract_chapters(self):
        """Extract chapters function (backbone)."""
        fanfic_name = self.name

        url = self.url
        urlscheme = urlparse(url)

        #set story_id from url
        self.fanfic_id = parse_qs(urlscheme.query, keep_blank_values=True)['sid'][0]

        # Get chapters
        r = send_request(url, self.verify_https)
        soup = BeautifulSoup(r.text, 'html5lib')

        self.title = self.get_fanfic_title(r)


        chapters = defaultdict(HPFanficArchiveChapter)

        try:

            chapter_list = soup.find_all('a', href=re.compile(r'^viewstory.php\?sid='))

            for link in chapter_list:
                chapter = link.get('href')
                if 'chapter' in chapter:
                    chapter_link = urljoin(
                        urlscheme.scheme + "://" + urlscheme.netloc,
                        'stories/' + str(chapter))
                    chapter_num = parse_qs(urlparse(chapter_link).query)['chapter'][0]
                    chapter_num = int(chapter_num)
                    chapters[chapter_num] = HPFanficArchiveChapter(
                        self, chapter_num, chapter_link)

            return chapters

        except:
            return chapters

    def get_update_date(self):
        r = send_request(self.url, self.verify_https)
        soup = BeautifulSoup(r.text, 'lxml')

        for c in soup.find_all(text=lambda text:isinstance(text, Comment)):
            if c in [' UPDATED START ']:
                update_date = c.next_element.strip()
                update_date = datetime.strptime(update_date, '%B %d, %Y')
                break

        return update_date

class HPFanficArchiveChapter(BaseChapter):
    """Base chapter class."""

    def get_fanfic_title(self,r):
        soup = BeautifulSoup(r.text, 'html5lib')

        regex = re.compile(r'^viewstory.php\?sid=')

        for div in soup.find_all('div', {'id':'pagetitle'}):
            title = div.find_all('a', href=regex)[0]
            title = title.get_text()
            break

        return title

    def get_fanfic_author(self,r):
        soup = BeautifulSoup(r.text, 'html5lib')

        regex = re.compile(r'^viewuser.php\?uid=')

        for div in soup.find_all('div', {'id':'pagetitle'}):
            author = div.find_all('a', href=regex)[0]
            author = author.get_text()
            break

        return author

    def get_fanfic_category(self,r):
        soup = BeautifulSoup(r.text, 'html5lib')
        category = ''

        regex = re.compile(r'^browse.php\?type=categories')

        desc = soup.find_all('div', {'class':'content'})[2]
        cat = desc.find_all('a', href=regex)
        cat2 = []
        for a in cat:
            cat2.append(a.get_text())
        s = ', '
        category = s.join(cat2)

        return category

    def get_fanfic_genre(self,r):
        soup = BeautifulSoup(r.text, 'html5lib')
        category = ''

        regex = re.compile(r'type_id=1')

        desc = soup.find_all('div', {'class':'content'})[2]
        cat = desc.find_all('a', href=regex)
        cat2 = []
        for a in cat:
            cat2.append(a.get_text())
        s = ', '
        category = s.join(cat2)

        return category

    def get_fanfic_description(self,r):
        soup = BeautifulSoup(r.text, 'html5lib')

        desc = soup.find_all('div', {'class':'content'})[2]
        desc = desc.find_all('p')[0]
        desc = desc.get_text()

        return desc

    def get_update_date(self,r):
        soup = BeautifulSoup(r.text, 'lxml')

        for c in soup.find_all(text=lambda text:isinstance(text, Comment)):
            if c in [' UPDATED START ']:
                update_date = c.next_element.strip()
                break

        return update_date

    def get_publish_date(self,r):
        soup = BeautifulSoup(r.text, 'lxml')

        for c in soup.find_all(text=lambda text:isinstance(text, Comment)):
            if c in [' PUBLISHED START ']:
                publish_date = c.next_element.strip()
                break
<commit_msg>Correct extraction for summary of HPFFA stories.<commit_after>"""Extractor for hpfanficarchive.com."""

from fanfic_scraper.base_fanfic import BaseFanfic, BaseChapter
from urllib.parse import urlparse, urljoin, parse_qs
import requests
from bs4 import BeautifulSoup, Comment
from collections import defaultdict
import re
import os
import shutil
import textwrap
from datetime import datetime
from random import shuffle, uniform
from copy import deepcopy
from time import sleep

def chapter_nav(tag):
    test = (tag.name == 'select')
    test = (test and 'chap_select' in tag['id'])

    return test

def send_request(url,verify):

    #Needed to bypass paranoid filtering for some hosts...
    proxies = {
               'http': 'socks5:127.0.0.1:9050',
               'https': 'socks5:127.0.0.1:9050'
              }

    r = requests.get(url, proxies=proxies, verify=verify)

    return r

class HPFanficArchive(BaseFanfic):

    def get_fanfic_title(self,r):
        soup = BeautifulSoup(r.text, 'html5lib')

        for div in soup.find_all('div', {'id':'pagetitle'}):
            title = div.find_all('a', href=re.compile(r'^viewstory.php\?sid='))[0]
            title = title.get_text()
            break

        return title

    def get_story_url(self, storyid):
        return 'https://www.hpfanficarchive.com/stories/viewstory.php?sid='+storyid

    def extract_chapters(self):
        """Extract chapters function (backbone)."""
        fanfic_name = self.name

        url = self.url
        urlscheme = urlparse(url)

        #set story_id from url
        self.fanfic_id = parse_qs(urlscheme.query, keep_blank_values=True)['sid'][0]

        # Get chapters
        r = send_request(url, self.verify_https)
        soup = BeautifulSoup(r.text, 'html5lib')

        self.title = self.get_fanfic_title(r)


        chapters = defaultdict(HPFanficArchiveChapter)

        try:

            chapter_list = soup.find_all('a', href=re.compile(r'^viewstory.php\?sid='))

            for link in chapter_list:
                chapter = link.get('href')
                if 'chapter' in chapter:
                    chapter_link = urljoin(
                        urlscheme.scheme + "://" + urlscheme.netloc,
                        'stories/' + str(chapter))
                    chapter_num = parse_qs(urlparse(chapter_link).query)['chapter'][0]
                    chapter_num = int(chapter_num)
                    chapters[chapter_num] = HPFanficArchiveChapter(
                        self, chapter_num, chapter_link)

            return chapters

        except:
            return chapters

    def get_update_date(self):
        r = send_request(self.url, self.verify_https)
        soup = BeautifulSoup(r.text, 'lxml')

        for c in soup.find_all(text=lambda text:isinstance(text, Comment)):
            if c in [' UPDATED START ']:
                update_date = c.next_element.strip()
                update_date = datetime.strptime(update_date, '%B %d, %Y')
                break

        return update_date

class HPFanficArchiveChapter(BaseChapter):
    """Base chapter class."""

    def get_fanfic_title(self,r):
        soup = BeautifulSoup(r.text, 'html5lib')

        regex = re.compile(r'^viewstory.php\?sid=')

        for div in soup.find_all('div', {'id':'pagetitle'}):
            title = div.find_all('a', href=regex)[0]
            title = title.get_text()
            break

        return title

    def get_fanfic_author(self,r):
        soup = BeautifulSoup(r.text, 'html5lib')

        regex = re.compile(r'^viewuser.php\?uid=')

        for div in soup.find_all('div', {'id':'pagetitle'}):
            author = div.find_all('a', href=regex)[0]
            author = author.get_text()
            break

        return author

    def get_fanfic_category(self,r):
        soup = BeautifulSoup(r.text, 'html5lib')
        category = ''

        regex = re.compile(r'^browse.php\?type=categories')

        desc = soup.find_all('div', {'class':'content'})[2]
        cat = desc.find_all('a', href=regex)
        cat2 = []
        for a in cat:
            cat2.append(a.get_text())
        s = ', '
        category = s.join(cat2)

        return category

    def get_fanfic_genre(self,r):
        soup = BeautifulSoup(r.text, 'html5lib')
        category = ''

        regex = re.compile(r'type_id=1')

        desc = soup.find_all('div', {'class':'content'})[2]
        cat = desc.find_all('a', href=regex)
        cat2 = []
        for a in cat:
            cat2.append(a.get_text())
        s = ', '
        category = s.join(cat2)

        return category

    def get_fanfic_description(self,r):
        soup = BeautifulSoup(r.text, 'html5lib')

        desc = soup.find_all('div', {'class':'content'})[2]
        para = desc.find_all('p')
        temp = []
        for p in para:
            temp.append(p.get_text())
        desc = "".join(temp)
        return desc

    def get_update_date(self,r):
        soup = BeautifulSoup(r.text, 'lxml')

        for c in soup.find_all(text=lambda text:isinstance(text, Comment)):
            if c in [' UPDATED START ']:
                update_date = c.next_element.strip()
                break

        return update_date

    def get_publish_date(self,r):
        soup = BeautifulSoup(r.text, 'lxml')

        for c in soup.find_all(text=lambda text:isinstance(text, Comment)):
            if c in [' PUBLISHED START ']:
                publish_date = c.next_element.strip()
                break

{'fraction_non_alphanumeric': 0.12196244477172312, 'fraction_numerical': 0.008376288659793814, 'mean_word_length': 2.179689786362306, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 6}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5757774', 'n_tokens_mistral': 3521, 'n_tokens_neox': 3361, 'n_words': 716}