<commit_before>#! /usr/bin/python
# -*- coding: utf-8 -*-

'''
A simple sample reader for a source...
'''

import collections

ChapterInfo = collections.namedtuple('ChapterInfo', ('name', 'url'))
class ImageInfo(collections.namedtuple('innerImageInfo', ('chapter_info', 'url', 'img_url', 'next_page'))):
    def __new__(cls, chapter_info, url, img_url=None, next_page=None):
      super(cls, ImageInfo).__new__(chapter_info, url, img_url, next_page)


class History(object):
  def __init__(self):
    self.dt={}
    self.dt['info'] = {}
    self.dt['info']['chapters'] = {} # chapter name -> (chapter_url, )
    self.dt['info']['images'] = {} # url -> (chapter name, img url)
    self.dt['info']['url_map'] = {} # page/image url -> next page url
    self.dt['done']['chapters'] = {} # chapter name -> local path
    self.dt['done']['images'] = {} # img url -> local file

  def save(self, dest):
    json.dump(self.dt, dest)

  def load(self, dest):
    self.dt = json.load(dh)

  def add_chapter_info(self, ci):
    self.dt['info']['chapters'][ci.name] = (ci.url, )

  def add_image_info(self, ii):
    self.dt['info']['images'][ii.url] = (ii.chapter_info.name, ii.img_url)
    self.dt['info']['url_map'][ii.url] = ii.next_page


class Reader(object):
    def __init__(self, data):
      self._data = data
      pass

    def parse_serie(self, serie_page):
      '''
      Read the serie page and return a list of ImageInfo

      The page is a BeautifulSoup document

      Best practice is to yield ImageInfo
      '''
      return []<commit_msg>Add getter/setter for history information<commit_after>#! /usr/bin/python
# -*- coding: utf-8 -*-

'''
A simple sample reader for a source...
'''

import collections

ChapterInfo = collections.namedtuple('ChapterInfo', ('name', 'url'))
class ImageInfo(collections.namedtuple('innerImageInfo', ('chapter_info', 'url', 'img_url', 'next_page'))):
    def __new__(cls, chapter_info, url, img_url=None, next_page=None):
      super(cls, ImageInfo).__new__(chapter_info, url, img_url, next_page)


class History(object):
  def __init__(self, chapters_to_skip=tuple()):
    self.chapters_to_skip = chapters_to_skip
    self.dt={}
    self.dt['info'] = {}
    self.dt['info']['chapters'] = {} # chapter name -> (chapter_url, )
    self.dt['info']['images'] = {} # url -> (chapter name, img url)
    self.dt['info']['url_map'] = {} # page/image url -> next page url
    self.dt['done']['chapters'] = {} # chapter name -> local path
    self.dt['done']['images'] = {} # img url -> local file

  def save(self, dest):
    json.dump(self.dt, dest)

  def load(self, dest):
    self.dt = json.load(dh)

  def add_chapter_info(self, ci):
    self.dt['info']['chapters'][ci.name] = (ci.url, )

  def add_image_info(self, ii):
    self.dt['info']['images'][ii.url] = (ii.chapter_info.name, ii.img_url)
    self.dt['info']['url_map'][ii.url] = ii.next_page


  def skip_chapter(self, chapter_name):
    return self.dt['done']['chapters'].get(chapter_name) is not None or \
      chapter_name in self.chapters_to_skip


  def get_next_page(self, url):
    return self.dt['info']['url_map'].get(url)



class Reader(object):
    def __init__(self, data):
      self._data = data
      pass

    def parse_serie(self, serie_page):
      '''
      Read the serie page and return a list of ImageInfo

      The page is a BeautifulSoup document

      Best practice is to yield ImageInfo
      '''
      return []
{'fraction_non_alphanumeric': 0.17301541145681884, 'fraction_numerical': 0.0005815644082582146, 'mean_word_length': 3.6236559139784945, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5357290', 'n_tokens_mistral': 1175, 'n_tokens_neox': 1143, 'n_words': 270}