<commit_before>from builtins import str
from builtins import object
import os
import re
import datetime

import mammoth
import bs4


def strip_dict(d):
    """
    Return a new dictionary, like d, but with any string value stripped

    >>> d = {'a': ' foo   ', 'b': 3, 'c': '   bar'}
    >>> result = strip_dict(d)
    >>> type(result)
    <class 'dict'>
    >>> sorted(result.items())
    [('a', 'foo'), ('b', 3), ('c', 'bar')]
    """
    return dict((k, v.strip() if hasattr(v, 'strip') else v) for k, v in d.items())


class QuestionAnswerScraper(object):
    """ Parses answer documents from parliament.
    """

    DOCUMENT_NAME_REGEX = re.compile(r'^R(?P<house>[NC])(?:O(?P<president>D?P)?(?P<oral_number>\d+))?(?:W(?P<written_number>\d+))?-+(?P<date_string>(\d{6}|\d{4}-\d{2}-\d{2}))$')
    BAR_REGEX = re.compile(r'^_+$', re.MULTILINE)

    QUESTION_RE = re.compile(
        r"""
          (?P<intro>
            (?:(?P<number1>\d+)\.?\s+)?         # Question number
            (?P<askedby>[-a-zA-Z\s]+?)          # Name of question asker
            \s*\((?P<party>[-\w\s]+)\)?<commit_msg>Make number dot optional and name must only be non-numeric<commit_after>from builtins import str
from builtins import object
import os
import re
import datetime

import mammoth
import bs4


def strip_dict(d):
    """
    Return a new dictionary, like d, but with any string value stripped

    >>> d = {'a': ' foo   ', 'b': 3, 'c': '   bar'}
    >>> result = strip_dict(d)
    >>> type(result)
    <class 'dict'>
    >>> sorted(result.items())
    [('a', 'foo'), ('b', 3), ('c', 'bar')]
    """
    return dict((k, v.strip() if hasattr(v, 'strip') else v) for k, v in d.items())


class QuestionAnswerScraper(object):
    """ Parses answer documents from parliament.
    """

    DOCUMENT_NAME_REGEX = re.compile(r'^R(?P<house>[NC])(?:O(?P<president>D?P)?(?P<oral_number>\d+))?(?:W(?P<written_number>\d+))?-+(?P<date_string>(\d{6}|\d{4}-\d{2}-\d{2}))$')
    BAR_REGEX = re.compile(r'^_+$', re.MULTILINE)

    QUESTION_RE = re.compile(
        r"""
          (?P<intro>
            (?:(?P<number1>\d+)\.?[ \t]?)?      # Question number
            (?P<askedby>[a-zA-Z]+[-a-zA-Z ]+?)  # Name of question asker
            \s*\((?P<party>[-\w\s]+)\)?
{'fraction_non_alphanumeric': 0.22454424188528235, 'fraction_numerical': 0.007114273010226767, 'mean_word_length': 2.996447602131439, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1866964', 'n_tokens_mistral': 879, 'n_tokens_neox': 816, 'n_words': 187}