<commit_before>#!/usr/bin/python

import lxml.html
from lxml import etree
import urlparse
import dateutil.parser
import datetime
import re
import sys
import urllib
import urllib2
import os
from optparse import OptionParser
import time
import gzip
from StringIO import StringIO
import random

output_directory = "../../../parldata/cmpages/sp/official-reports-new/"

official_report_url_format = 'http://www.scottish.parliament.uk/parliamentarybusiness/28862.aspx?r={}&mode=html'

user_agent = 'Mozilla/5.0 (Ubuntu; X11; Linux i686; rv:9.0.1) Gecko/20100101 Firefox/9.0.1'

parser = OptionParser()
parser.add_option('-v', '--verbose', dest='verbose', action='store_true',
                  default=False, help='Print progress messages')<commit_msg>For running under 2.6, specify format number.<commit_after>#!/usr/bin/python

import lxml.html
from lxml import etree
import urlparse
import dateutil.parser
import datetime
import re
import sys
import urllib
import urllib2
import os
from optparse import OptionParser
import time
import gzip
from StringIO import StringIO
import random

output_directory = "../../../parldata/cmpages/sp/official-reports-new/"

official_report_url_format = 'http://www.scottish.parliament.uk/parliamentarybusiness/28862.aspx?r={0}&mode=html'

user_agent = 'Mozilla/5.0 (Ubuntu; X11; Linux i686; rv:9.0.1) Gecko/20100101 Firefox/9.0.1'

parser = OptionParser()
parser.add_option('-v', '--verbose', dest='verbose', action='store_true',
                  default=False, help='Print progress messages')
{'fraction_non_alphanumeric': 0.12131147540983607, 'fraction_numerical': 0.037377049180327866, 'mean_word_length': 4.846743295019157, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7045964', 'n_tokens_mistral': 534, 'n_tokens_neox': 490, 'n_words': 125}