<commit_before>import os

from picdump.webadapter import WebAdapter
from picdump.utils import URLBuilder, PageIterator, app_path
from picdump.pixiv import csv
from picdump.pixiv import cache
from picdump import app


API_HOST = "spapi.pixiv.net"


class API(app.HasAppMixin):
    def __init__(self, adapter=None):
        super().__init__()
        self.adapter = adapter or WebAdapter()
        self.ranking = Ranking(self)
        self.search = Search(self)
        self.cache = cache.VoidCache()

    def image_cache_dir(self):
        config = self.app.config
        return app_path(config.cache_dir, 'pixiv')

    def on_app_injected(self, app):
        self.cache = cache.ImageCache(self.image_cache_dir())

    def set_cache_dir(self, dirpath):
        self.cache = cache.ImageCache(dirpath)


# ---------------------------------
# Enums
# ---------------------------------
class RankingContentType:
    all = 'all'
    original = 'original'
    female = 'female'
    male = 'male'
    rookie = 'rookie'


class RankingSpan:
    daily = 'daily'
    weekly = 'weekly'
    monthly = 'monthly'


class SearchMode:
    by_tag = 's_tag'
    by_tag_full = 's_tag_full'
    by_keyword = 's_tc'  # tc = Title and Caption


# ---------------------------------
# Config Facades
# ---------------------------------

class APIFacade:
    def __init__(self, api):
        self.api = api
        self.urlbuilder = URLBuilder(
            host=API_HOST,
            path=self.path
        )

    @property
    def path(self):
        raise NotImplemented()

    def mk_iterator(self, requrl):
        return PageIterator(PageFetcher(self.api, requrl))


class Ranking(APIFacade):
    path = '/iphone/ranking.php'

    def __call__(self, span, content=RankingContentType.all):
        requrl = self.urlbuilder.update_with(
            params={
                'mode': span,
                'content': content
            }
        )
        return self.mk_iterator(requrl)


class Search(APIFacade):
    path = '/iphone/search.php'

    def __call__(self, query, mode=SearchMode.by_tag):
        requrl = self.urlbuilder.update_with(
            params={
                'word': query,
                's_mode': mode
            }
        )
        return self.mk_iterator(requrl)


# ---------------------------------
# Utilities
# ---------------------------------
class PageFetcher:
    def __init__(self, api, urlbuilder, begin_page=1):
        self.adapter = api.adapter
        self.api = api
        self.current_page = begin_page
        self.urlbuilder = urlbuilder

    def __next__(self):
        url = self.urlbuilder.update_params(p=self.current_page)
        self.current_page += 1
        csv_page = self.adapter.get_text(url)
        contents = list(csv.parse(csv_page, self.api))
        if len(contents) == 0:
            raise StopIteration()
        return contents

    def reset(self):
        self.current_page = 1
<commit_msg>Add DynamicPageFetcher for live updating page<commit_after>import os

from picdump.webadapter import WebAdapter
from picdump.utils import URLBuilder, PageIterator, app_path
from picdump.pixiv import csv
from picdump.pixiv import cache
from picdump import app


API_HOST = "spapi.pixiv.net"


class API(app.HasAppMixin):
    def __init__(self, adapter=None):
        super().__init__()
        self.adapter = adapter or WebAdapter()
        self.ranking = Ranking(self)
        self.search = Search(self)
        self.cache = cache.VoidCache()

    def image_cache_dir(self):
        config = self.app.config
        return app_path(config.cache_dir, 'pixiv')

    def on_app_injected(self, app):
        self.cache = cache.ImageCache(self.image_cache_dir())

    def set_cache_dir(self, dirpath):
        self.cache = cache.ImageCache(dirpath)


# ---------------------------------
# Enums
# ---------------------------------
class RankingContentType:
    all = 'all'
    original = 'original'
    female = 'female'
    male = 'male'
    rookie = 'rookie'


class RankingSpan:
    daily = 'daily'
    weekly = 'weekly'
    monthly = 'monthly'


class SearchMode:
    by_tag = 's_tag'
    by_tag_full = 's_tag_full'
    by_keyword = 's_tc'  # tc = Title and Caption


# ---------------------------------
# Config Facades
# ---------------------------------

class APIFacade:
    def __init__(self, api):
        self.api = api
        self.urlbuilder = URLBuilder(
            host=API_HOST,
            path=self.path
        )

    @property
    def path(self):
        raise NotImplemented()

    def mk_static_iterator(self, requrl):
        return PageIterator(StaticPageFetcher(self.api, requrl))

    mk_iterator = mk_static_iterator

    def mk_dynamic_iterator(self, url_factory):
        return PageIterator(DynamicPageFetcher(self.api, url_factory))


class Ranking(APIFacade):
    path = '/iphone/ranking.php'

    def __call__(self, span, content=RankingContentType.all):
        requrl = self.urlbuilder.update_with(
            params={
                'mode': span,
                'content': content
            }
        )
        return self.mk_iterator(requrl)


class Search(APIFacade):
    path = '/iphone/search.php'

    def __call__(self, query, mode=SearchMode.by_tag):
        requrl = self.urlbuilder.update_with(
            params={
                'word': query,
                's_mode': mode
            }
        )
        return self.mk_iterator(requrl)


# ---------------------------------
# Utilities
# ---------------------------------
class PageFetcher:
    def __init__(self, api, begin_page=1):
        self.adapter = api.adapter
        self.api = api
        self.current_page = begin_page

    def get_next_urlbuilder(self):
        raise NotImplementedError()

    def __next__(self):
        builder = self.get_next_urlbuilder()
        url = builder.update_params(p=self.current_page)
        self.current_page += 1
        csv_page = self.adapter.get_text(url)
        contents = list(csv.parse(csv_page, self.api))
        if len(contents) == 0:
            raise StopIteration()
        return contents

    def reset(self):
        self.current_page = 1


class StaticPageFetcher(PageFetcher):
    def __init__(self, api, urlbuilder, begin_page=1):
        super().__init__(api, begin_page)
        self.urlbuilder = urlbuilder

    def get_next_urlbuilder(self):
        return self.urlbuilder


class DynamicPageFetcher(PageFetcher):
    def __init__(self, api, url_factory, begin_page=1):
        super().__init__(api, begin_page)
        self.url_factory = url_factory

    def get_next_urlbuilder(self):
        return self.url_factory()

{'fraction_non_alphanumeric': 0.15310427968655818, 'fraction_numerical': 0.0015069318866787222, 'mean_word_length': 2.004526935264826, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1417317', 'n_tokens_mistral': 2022, 'n_tokens_neox': 1965, 'n_words': 422}