<commit_before># Install custom version of scraperwiki library
-e git+http://github.com/openaustralia/scraperwiki-python.git@morph_defaults#egg=scraperwiki

# numpy, gemsim, matplotlib, pandas, scipy are commented out because
# all those libraries require numpy
# it looks like the python heroku buildpack doesn't support the installation of numpy

#numpy==1.6.1
BeautifulSoup==3.2.0
Creoleparser==0.7.4
Genshi==0.6
Jinja2==2.6
Markdown==2.2.0
Pygments==1.4
SQLAlchemy==0.6.6
Twisted==11.1.0
Unidecode==0.04.9
anyjson==0.3.3
argparse==1.2.1
beautifulsoup4==4.1.3
bitlyapi==0.1.1
blinker==1.2
cartodb==0.6
certifi==0.0.8
chardet==2.1.1
ckanclient==0.10
colormath==1.0.8
csvkit==0.3.0
dataset==0.5.2
demjson==1.6
dropbox==1.4
errorhandler==1.1.1
feedparser==5.0.1
fluidinfo.py==1.1.2
gdata==2.0.15
geopy==0.94.1
gevent==0.13.6
google-api-python-client==1.0beta8
googlemaps==1.0.2
greenlet==0.3.2
html5lib==0.90
httplib2==0.7.4
imposm.parser==1.0.3
jellyfish==0.2.0
mechanize==0.2.5
mock==0.7.2
networkx==1.6
ngram==3.3.0
nose==1.1.2
oauth2==1.5.170
oauth==1.0.1
oauthlib==0.1.2
openpyxl==1.5.7
ordereddict==1.1
pbkdf2==1.3
pdfminer==20110515
pexpect==2.4
pipe2py==0.9.2
pyOpenSSL==0.13
pycrypto==2.5
pycurl==7.19.0
pyephem==3.7.5.1
pyparsing==1.5.6
pyth==0.5.6
python-Levenshtein==0.10.2
python-dateutil==1.5
python-gflags==2.0
python-modargs==1.2
python-stdnum==0.7
pytz==2011k
rdflib==3.1.0
requests-foauth==0.1.1
requests==1.0.4
selenium==2.5.0
simplejson==2.2.1
suds==0.4
tweepy==1.7.1
tweetstream==1.1.1
w3lib==1.0
wsgiref==0.1.2
xlrd==0.7.1
xlutils==1.4.1
xlwt==0.7.2
xmltodict==0.4
yql==0.7.2
zope.interface==3.8.0
lxml==2.3.3
chromium-compact-language-detector==0.031415
#distribute==0.6.21
#gensim==0.8.3
icalendar==3.0.1b1
#matplotlib==1.0.1
#pandas==0.4.3
pyquery==1.0
#scipy==0.10.0
scrapely==0.9

# These are ones where the original version wasn't available. So we went to the next
# nearest available one
# Changed this from 0.9.3 to 0.9.8
Fom==0.9.8
# Changed from 3.09 to 3.10
PyYAML==3.10
# Changed from 0.14.0.2841 to 0.14.1
Scrapy==0.14.1
# Changed from 15.2.4 to 15.6.2
adspygoogle.adwords==15.6.2
# Changed this from 2.0b9 to 3.0.2
nltk==3.0.2
# Changed this from 1.0.25 to 1.0.2
pydot==1.0.2
# Changed this from 0.21.1 to 0.22.3
M2Crypto==0.22.3


# TODO Need to install parsley<commit_msg>Get python buildpack going again despite yql having vanished<commit_after># Install custom version of scraperwiki library
-e git+http://github.com/openaustralia/scraperwiki-python.git@morph_defaults#egg=scraperwiki

# numpy, gemsim, matplotlib, pandas, scipy are commented out because
# all those libraries require numpy
# it looks like the python heroku buildpack doesn't support the installation of numpy

#numpy==1.6.1
BeautifulSoup==3.2.0
Creoleparser==0.7.4
Genshi==0.6
Jinja2==2.6
Markdown==2.2.0
Pygments==1.4
SQLAlchemy==0.6.6
Twisted==11.1.0
Unidecode==0.04.9
anyjson==0.3.3
argparse==1.2.1
beautifulsoup4==4.1.3
bitlyapi==0.1.1
blinker==1.2
cartodb==0.6
certifi==0.0.8
chardet==2.1.1
ckanclient==0.10
colormath==1.0.8
csvkit==0.3.0
dataset==0.5.2
demjson==1.6
dropbox==1.4
errorhandler==1.1.1
feedparser==5.0.1
fluidinfo.py==1.1.2
gdata==2.0.15
geopy==0.94.1
gevent==0.13.6
google-api-python-client==1.0beta8
googlemaps==1.0.2
greenlet==0.3.2
html5lib==0.90
httplib2==0.7.4
imposm.parser==1.0.3
jellyfish==0.2.0
mechanize==0.2.5
mock==0.7.2
networkx==1.6
ngram==3.3.0
nose==1.1.2
oauth2==1.5.170
oauth==1.0.1
oauthlib==0.1.2
openpyxl==1.5.7
ordereddict==1.1
pbkdf2==1.3
pdfminer==20110515
pexpect==2.4
pipe2py==0.9.2
pyOpenSSL==0.13
pycrypto==2.5
pycurl==7.19.0
pyephem==3.7.5.1
pyparsing==1.5.6
pyth==0.5.6
python-Levenshtein==0.10.2
python-dateutil==1.5
python-gflags==2.0
python-modargs==1.2
python-stdnum==0.7
pytz==2011k
rdflib==3.1.0
requests-foauth==0.1.1
requests==1.0.4
selenium==2.5.0
simplejson==2.2.1
suds==0.4
tweepy==1.7.1
tweetstream==1.1.1
w3lib==1.0
wsgiref==0.1.2
xlrd==0.7.1
xlutils==1.4.1
xlwt==0.7.2
xmltodict==0.4
# Temporarily commenting out ygl because it's gone missing from
# https://pypi.python.org/pypi/yql/0.7.2
#yql==0.7.2
zope.interface==3.8.0
lxml==2.3.3
chromium-compact-language-detector==0.031415
#distribute==0.6.21
#gensim==0.8.3
icalendar==3.0.1b1
#matplotlib==1.0.1
#pandas==0.4.3
pyquery==1.0
#scipy==0.10.0
scrapely==0.9

# These are ones where the original version wasn't available. So we went to the next
# nearest available one
# Changed this from 0.9.3 to 0.9.8
Fom==0.9.8
# Changed from 3.09 to 3.10
PyYAML==3.10
# Changed from 0.14.0.2841 to 0.14.1
Scrapy==0.14.1
# Changed from 15.2.4 to 15.6.2
adspygoogle.adwords==15.6.2
# Changed this from 2.0b9 to 3.0.2
nltk==3.0.2
# Changed this from 1.0.25 to 1.0.2
pydot==1.0.2
# Changed this from 0.21.1 to 0.22.3
M2Crypto==0.22.3


# TODO Need to install parsley
{'fraction_non_alphanumeric': 0.18688455897758224, 'fraction_numerical': 0.1493819400796145, 'mean_word_length': 4.437357630979498, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '233271', 'n_tokens_mistral': 2511, 'n_tokens_neox': 2348, 'n_words': 405}