<commit_before>class Asset:
    """
    This is an ooni-probe asset. It is a python
    iterator object, allowing it to be efficiently looped.
    To create your own custom asset your should subclass this
    and override the next_asset method and the len method for
    computing the length of the asset.
    """
    def __init__(self, file=None, *args, **argv):
        self.fh = None
        if file:
            self.name = file
            self.fh = open(file, 'r')
        self.eof = False

    def __iter__(self):
        return self

    def len(self):
        """
        Returns the length of the asset
        """
        for i, l in enumerate(self.fh):
            pass
        # rewind the file
        self.fh.seek(0)
        return i + 1

    def parse_line(self, line):
        """
        Override this method if you need line
        by line parsing of an Asset.
        """
        return line.replace('\n','')

    def next_asset(self):
        """
        Return the next asset.
        """
        # XXX this is really written with my feet.
        #     clean me up please...
        line = self.fh.readline()
        if line:
            return self.parse_line(line)
        else:
            self.fh.seek(0)
            raise StopIteration

    def next(self):
        try:
            return self.next_asset()
        except:
            raise StopIteration
<commit_msg>Add a method for line by line asset parsing<commit_after>class Asset:
    """
    This is an ooni-probe asset. It is a python
    iterator object, allowing it to be efficiently looped.
    To create your own custom asset your should subclass this
    and override the next_asset method and the len method for
    computing the length of the asset.
    """
    def __init__(self, file=None, *args, **argv):
        self.fh = None
        if file:
            self.name = file
            self.fh = open(file, 'r')
        self.eof = False

    def __iter__(self):
        return self

    def len(self):
        """
        Returns the length of the asset
        """
        for i, l in enumerate(self.fh):
            pass
        # rewind the file
        self.fh.seek(0)
        return i + 1

    def parse_line(self, line):
        """
        Override this method if you need line
        by line parsing of an Asset.
        """
        return line.replace('\n','')

    def next_asset(self):
        """
        Return the next asset.
        """
        # XXX this is really written with my feet.
        #     clean me up please...
        line = self.fh.readline()
        if line:
            parsed_line = self.parse_line(line)
            if parsed_line:
                return parsed_line
        else:
            self.fh.seek(0)
            raise StopIteration

    def next(self):
        try:
            return self.next_asset()
        except:
            raise StopIteration

{'fraction_non_alphanumeric': 0.08367911479944674, 'fraction_numerical': 0.002074688796680498, 'mean_word_length': 1.5466549295774648, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5727174', 'n_tokens_mistral': 815, 'n_tokens_neox': 781, 'n_words': 292}