<commit_before>import chardet
import codecs
import os
import re


class Clippings(object):

    def __init__(self, source, dest=None):
        '''
        Prepares the import and store it into the 'clippings' dict.
        '''
        self.source = source
        self.dest = self._get_default_dest() if dest is None else dest
        self.book_author_couples = ()
        self.clippings = []
        self._fetch()

    def _get_default_dest(self):
        '''
        When no destination is specified, output to <InputFilename>.html
        '''
        source_full_path = os.path.realpath(self.source)
        dirname, filename_with_ext = os.path.split(source_full_path)
        filename = os.path.splitext(filename_with_ext)[0]
        default_destination = os.path.join(dirname, filename + '.html')
        return default_destination

    def _fetch(self):
        '''
        Imports clippings and book_author_couples from the source file
        '''
        clippings = ClippingsIterator(self.source)
        for clipping in clippings:
            self.clippings.append(clipping)

        # will be useful in the HTML to group by book/author
        self.book_author_couples = set((clipping['book'], clipping['author'])
                                       for clipping in self.clippings)

    def export_to_html(self):
        '''
        Output the clippings dict to HTML, using a Jinja2 template
        '''
        from jinja2 import Environment, PackageLoader  # available from pip
        env = Environment(loader=PackageLoader('whoarder', 'templates'),
                          autoescape=True,
                          extensions=['jinja2.ext.autoescape'])<commit_msg>Only highlights should be displayed<commit_after>import chardet
import codecs
import os
import re


class Clippings(object):

    def __init__(self, source, dest=None):
        '''
        Prepares the import and store it into the 'clippings' dict.
        '''
        self.source = source
        self.dest = self._get_default_dest() if dest is None else dest
        self.book_author_couples = ()
        self.clippings = []
        self._fetch()

    def _get_default_dest(self):
        '''
        When no destination is specified, output to <InputFilename>.html
        '''
        source_full_path = os.path.realpath(self.source)
        dirname, filename_with_ext = os.path.split(source_full_path)
        filename = os.path.splitext(filename_with_ext)[0]
        default_destination = os.path.join(dirname, filename + '.html')
        return default_destination

    def _fetch(self):
        '''
        Imports clippings and book_author_couples from the source file
        '''
        clippings = ClippingsIterator(self.source)
        for clipping in clippings:
            # Only highlights
            if clipping["type"] == "Highlight":
                self.clippings.append(clipping)

        # will be useful in the HTML to group by book/author
        self.book_author_couples = set((clipping['book'], clipping['author'])
                                       for clipping in self.clippings)

    def export_to_html(self):
        '''
        Output the clippings dict to HTML, using a Jinja2 template
        '''
        from jinja2 import Environment, PackageLoader  # available from pip
        env = Environment(loader=PackageLoader('whoarder', 'templates'),
                          autoescape=True,
                          extensions=['jinja2.ext.autoescape'])
{'fraction_non_alphanumeric': 0.0892343120322395, 'fraction_numerical': 0.002302820955670697, 'mean_word_length': 2.2235621521335807, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4082121', 'n_tokens_mistral': 954, 'n_tokens_neox': 933, 'n_words': 246}