<commit_before>#!/usr/local/bin/python27
# -*- coding: utf-8 -*-

#
# Feed Me! - OpenRice Mobile Website Scraper
# Copyright (c) 2017 loadfix
#
from bs4 import BeautifulSoup
import urllib2
import re
import csv
import requests
import signal
import sys
import zhon.hanzi
import time
from datetime import datetime

# Set codepage
reload(sys)  
sys.setdefaultencoding('utf8')

# ---------- Start of User Configurable Variables ------------- #
apikey = ''
api = "http://api.openrice.com/english/english/mobile/sr2.htm?shopid="
searchurl = 'https://maps.googleapis.com/maps/api/geocode/json'
start = 1
end = 1000000
debug = True
# ---------- End of User Configurable Variables --------------- #

# Check arguments
if len(sys.argv) == 1:
   pass
elif len(sys.argv) == 3:
   try:
      start = int(sys.argv[1])
      end = int(sys.argv[2])
   except:
      print("Incorrect arguments")
      sys.exit(1)<commit_msg>python3 hard codes to utf8<commit_after>#!/usr/local/bin/python27
# -*- coding: utf-8 -*-

#
# Feed Me! - OpenRice Mobile Website Scraper
# Copyright (c) 2017 loadfix
#
from bs4 import BeautifulSoup
import urllib2
import re
import csv
import requests
import signal
import zhon.hanzi
import time
from datetime import datetime

# ---------- Start of User Configurable Variables ------------- #
apikey = ''
api = "http://api.openrice.com/english/english/mobile/sr2.htm?shopid="
searchurl = 'https://maps.googleapis.com/maps/api/geocode/json'
start = 1
end = 1000000
debug = True
# ---------- End of User Configurable Variables --------------- #

# Check arguments
if len(sys.argv) == 1:
   pass
elif len(sys.argv) == 3:
   try:
      start = int(sys.argv[1])
      end = int(sys.argv[2])
   except:
      print("Incorrect arguments")
      sys.exit(1)
{'fraction_non_alphanumeric': 0.17094017094017094, 'fraction_numerical': 0.02792022792022792, 'mean_word_length': 2.8849557522123894, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1906885', 'n_tokens_mistral': 610, 'n_tokens_neox': 563, 'n_words': 157}