<commit_before>import sys, re
from itertools import izip_longest
from datetime import datetime
from hotdoc.core.comment_block import *

#http://stackoverflow.com/questions/434287/what-is-the-most-pythonic-way-to-iterate-over-a-list-in-chunks
def grouper(iterable, n, fillvalue=None):
    args = [iter(iterable)] * n
    return izip_longest(*args, fillvalue=fillvalue)

class GtkDocRawCommentParser (object):
    def __init__(self):
        """
        Lifted from
        http://stackoverflow.com/questions/5323703/regex-how-to-match-sequence-of-key-value-pairs-at-end-of-string
        """
        self.kv_regex = re.compile(r'''
                                   [\S]+=
                                   (?:
                                   \s*
                                   (?!\S+=)\S+
                                   )+
                                   ''', re.VERBOSE)

    def parse_title (self, title):
        split = title.split (': ', 1)
        title = split[0].rstrip(':')
        annotations = []
        if len (split) > 1:
            annotations = self.parse_annotations (split[1])
        return title, annotations

    def parse_key_value_annotation (self, name, string):
        arg = {}
        kvs = self.kv_regex.findall (string)
        kvs = dict([kv.split('=', 1) for kv in kvs])
        return GtkDocAnnotation (name, kvs)

    def parse_annotation (self, string):
        split = string.split ()
        name = split[0].strip()<commit_msg>Make sure we get section comments, even if they contain spaces.<commit_after>import sys, re
from itertools import izip_longest
from datetime import datetime
from hotdoc.core.comment_block import *

#http://stackoverflow.com/questions/434287/what-is-the-most-pythonic-way-to-iterate-over-a-list-in-chunks
def grouper(iterable, n, fillvalue=None):
    args = [iter(iterable)] * n
    return izip_longest(*args, fillvalue=fillvalue)

class GtkDocRawCommentParser (object):
    def __init__(self):
        """
        Lifted from
        http://stackoverflow.com/questions/5323703/regex-how-to-match-sequence-of-key-value-pairs-at-end-of-string
        """
        self.kv_regex = re.compile(r'''
                                   [\S]+=
                                   (?:
                                   \s*
                                   (?!\S+=)\S+
                                   )+
                                   ''', re.VERBOSE)

    def parse_title (self, title):
        # Section comments never contain annotations,
        # We also normalize them here to not contain any spaces.
        if "SECTION" in title:
            return ''.join(title.split(' ')), []

        split = title.split (': ', 1)
        title = split[0].rstrip(':')
        annotations = []
        if len (split) > 1:
            annotations = self.parse_annotations (split[1])
        return title, annotations

    def parse_key_value_annotation (self, name, string):
        arg = {}
        kvs = self.kv_regex.findall (string)
        kvs = dict([kv.split('=', 1) for kv in kvs])
        return GtkDocAnnotation (name, kvs)

    def parse_annotation (self, string):
        split = string.split ()
        name = split[0].strip()
{'fraction_non_alphanumeric': 0.12274143302180686, 'fraction_numerical': 0.011838006230529595, 'mean_word_length': 1.7657192075796726, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4313204', 'n_tokens_mistral': 974, 'n_tokens_neox': 916, 'n_words': 204}