<commit_before>import cookielib
import os
import re
import sys
import urllib
import urllib2
from bs4 import BeautifulSoup
from time import sleep

user_agent = u"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; " + \
    u"rv:1.9.2.11) Gecko/20101012 Firefox/3.6.11"

courts_with_fm_case_number = [
    'Charlotte Circuit Court',
    'Chesterfield Circuit Court',
    'Page Circuit Court',
    'Richmond City Circuit Court',
    'Wise Circuit Court'
]

def get_case_number_with_mf(case_number):
    return [
        case_number[:4] + 'M' + case_number[5:],
        case_number[:4] + 'F' + case_number[5:]
    ]

def get_case_number(court, case_count, charge_count):
    case_number = 'CR14' + format(case_count, '06') + \
        '-' + format(charge_count, '02')
    if court in courts_with_fm_case_number:
        return get_case_number_with_mf(case_number)
    return [case_number]

def getFilePath(court_name):
    court_name = court_name.replace(' ', '').replace('/', '')
    return '../va-circuit-court-search-files/' + court_name + '/'

def run():
    # Get cookie and list of courts
    cookieJar = cookielib.CookieJar()
    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookieJar))
    opener.addheaders = [('User-Agent', user_agent)]
    
    home = opener.open('http://ewsocis1.courts.state.va.us/CJISWeb/circuit.jsp')
    
    courts = []
    html = BeautifulSoup(home.read())
    for option in html.find_all('option'):
        courts.append({
            'fullName': option['value'],
            'id': option['value'][:3],
            'name': option['value'][5:]
        })
    
    # Go to court
    for index, court in enumerate(courts):
        print court['name']
        file_path = getFilePath(court['name'])
        if not os.path.exists(file_path):
            os.makedirs(file_path)
        #elif len(courts) > index + 1:
        #    next_file_path = getFilePath(courts[index+1]['name'])
        #    if os.path.exists(next_file_path):
        #        continue
        
        data = urllib.urlencode({
            'courtId': court['id'],
            'courtType': 'C',
            'caseType': 'ALL',
            'testdos': False,
            'sessionCreate': 'NEW',
            'whichsystem': court['fullName']})
        place_url = u"http://ewsocis1.courts.state.va.us/CJISWeb/MainMenu.do"
        opener.open(place_url, data)
        
        # search by case numbers
        sequential_cases_missed_count = 0
        case_count = -1
        file_list = os.listdir(file_path)
        if len(file_list) > 0:
            case_count = int(file_list[-1][5:10]) - 1
        
        search_next_case = True
        while search_next_case:
            case_count += 1<commit_msg>Correctly find last file in courts with F/M case numbers<commit_after>import cookielib
import os
import re
import sys
import urllib
import urllib2
from bs4 import BeautifulSoup
from time import sleep

user_agent = u"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; " + \
    u"rv:1.9.2.11) Gecko/20101012 Firefox/3.6.11"

courts_with_fm_case_number = [
    'Charlotte Circuit Court',
    'Chesterfield Circuit Court',
    'Page Circuit Court',
    'Richmond City Circuit Court',
    'Wise Circuit Court'
]

def get_case_number_with_mf(case_number):
    return [
        case_number[:4] + 'M' + case_number[5:],
        case_number[:4] + 'F' + case_number[5:]
    ]

def get_case_number(court, case_count, charge_count):
    case_number = 'CR14' + format(case_count, '06') + \
        '-' + format(charge_count, '02')
    if court in courts_with_fm_case_number:
        return get_case_number_with_mf(case_number)
    return [case_number]

def getFilePath(court_name):
    court_name = court_name.replace(' ', '').replace('/', '')
    return '../va-circuit-court-search-files/' + court_name + '/'

def run():
    # Get cookie and list of courts
    cookieJar = cookielib.CookieJar()
    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookieJar))
    opener.addheaders = [('User-Agent', user_agent)]
    
    home = opener.open('http://ewsocis1.courts.state.va.us/CJISWeb/circuit.jsp')
    
    courts = []
    html = BeautifulSoup(home.read())
    for option in html.find_all('option'):
        courts.append({
            'fullName': option['value'],
            'id': option['value'][:3],
            'name': option['value'][5:]
        })
    
    # Go to court
    for index, court in enumerate(courts):
        print court['name']
        file_path = getFilePath(court['name'])
        if not os.path.exists(file_path):
            os.makedirs(file_path)
        #elif len(courts) > index + 1:
        #    next_file_path = getFilePath(courts[index+1]['name'])
        #    if os.path.exists(next_file_path):
        #        continue
        
        data = urllib.urlencode({
            'courtId': court['id'],
            'courtType': 'C',
            'caseType': 'ALL',
            'testdos': False,
            'sessionCreate': 'NEW',
            'whichsystem': court['fullName']})
        place_url = u"http://ewsocis1.courts.state.va.us/CJISWeb/MainMenu.do"
        opener.open(place_url, data)
        
        # search by case numbers
        sequential_cases_missed_count = 0
        case_count = -1
        file_list = os.listdir(file_path)
        for file_name in file_list:
            cur_count = int(file_name[5:10])
            if cur_count > case_count:
                case_count = cur_count - 1
        
        search_next_case = True
        while search_next_case:
            case_count += 1
{'fraction_non_alphanumeric': 0.12965964343598055, 'fraction_numerical': 0.018008283810552854, 'mean_word_length': 2.4690818238600873, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1040633', 'n_tokens_mistral': 1921, 'n_tokens_neox': 1845, 'n_words': 376}