<commit_before>from bs4 import BeautifulSoup
from progressbar import ProgressBar

import requests
import os
import urllib2


def getChapterURLs():
    url = 'http://www.deltau.org/ChapterExcellence'
    page = urllib2.urlopen('http://www.deltau.org/ChapterExcellence').read()
    soup = BeautifulSoup(page, 'html.parser')
    chapter_links = soup.select('ul li h3 a')
    chapter_URLs = []
    for a in chapter_links:
        chapter_URLs.append(str(a['href']))
    return chapter_URLs

def walkChapterCriteria(url):
    chapter_criteria_URL = 'http://www.deltau.org' + url + 'CriteriaList'
    r = requests.post(chapter_criteria_URL, data = {'filter': 'Fall Criteria'})

    soup = BeautifulSoup(r.text, 'html.parser')
    criteria = soup.find_all('li')

    for i in criteria:
        checkSubmission(i)

def checkSubmission(criteria):
    if criteria.find('a', 'fancybox'):
        checkFolder(criteria)

def checkFolder(criteria):
    folder_title = "CEP/" + criteria.h4.a.encode_contents().replace('/','|')
    print(folder_title)
    if not os.path.exists(folder_title):
        os.makedirs(folder_title)

def main():
    chapter_URLs = getChapterURLs()

    pbar = ProgressBar()
    # for url in pbar(chapter_URLs):
    #     walkChapterCriteria(url)
    walkChapterCriteria(chapter_URLs[0])

if __name__ == '__main__':
    main()
else:
    print("This program should be run directly!")
<commit_msg>Creates Chapter folder within each Criteria folder<commit_after>from bs4 import BeautifulSoup
from progressbar import ProgressBar

import requests
import os
import urllib2


def getChapterURLs():
    url = 'http://www.deltau.org/ChapterExcellence'
    page = urllib2.urlopen('http://www.deltau.org/ChapterExcellence').read()
    soup = BeautifulSoup(page, 'html.parser')
    chapter_links = soup.select('ul li h3 a')
    chapter_URLs = []
    for a in chapter_links:
        chapter_URLs.append(str(a['href']))
    return chapter_URLs


def walkChapterCriteria(url):
    chapter_criteria_URL = 'http://www.deltau.org' + url + 'CriteriaList'
    r = requests.post(chapter_criteria_URL, data = {'filter': 'Fall Criteria'})

    soup = BeautifulSoup(r.text, 'html.parser')
    criteria_list = soup.find_all('li')
    chapter_name = urllib2.unquote(url.split('/')[2])

    for criteria in criteria_list:
        checkSubmission(criteria, chapter_name)


def checkSubmission(criteria, chapter_name):
    if criteria.find('a', 'fancybox'):
        folder_title = "CEP/" + criteria.h4.a.encode_contents().replace('/','|')
        checkFolder(folder_title)
        makeFolder(folder_title + '/' + chapter_name)


def checkFolder(folder_title):
    if not os.path.exists(folder_title):
        os.makedirs(folder_title)


def makeFolder(folder_title):
    if not os.path.exists(folder_title):
        os.makedirs(folder_title)
    else:
        print("This Chapter's folder already exists: " + folder_title)


def main():
    chapter_URLs = getChapterURLs()

    pbar = ProgressBar()
    # for url in pbar(chapter_URLs):
    #     walkChapterCriteria(url)
    walkChapterCriteria(chapter_URLs[0])


if __name__ == '__main__':
    main()
else:
    print("This program should be run directly!")

{'fraction_non_alphanumeric': 0.12539184952978055, 'fraction_numerical': 0.00438871473354232, 'mean_word_length': 3.842185128983308, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '851885', 'n_tokens_mistral': 1073, 'n_tokens_neox': 1035, 'n_words': 218}