<commit_before>import nltk, os, sys, re, random, string
from bs4 import BeautifulSoup

TRAIN = 'training'
TEST = 'test'


def BuildDicts(path):
    train = []
    test = []

    testAndTrain = GetPath(path)

    if testAndTrain:
        #parse the html files
        for file in os.listdir(path + '/' + TEST):
            filePath = path + '/' + TEST + '/' + file
            test.append(CleanHtml(filePath))

        for file in os.listdir(path + '/' + TRAIN):
            filePath = path + '/' + TRAIN + '/' + file
            train.append(CleanHtml(filePath))
    else:
        #files are in review folders
        allReviews = []
        ratio = 4
        for i in range(1,4):
            newPath = path + "/Review" + str(i) + "/"
            for folders in os.listdir(newPath):
                if(os.path.isdir(newPath + folders)):
                    for file in os.listdir(newPath + folders):
                        reviewer = folders.split('_')[0]
                        CleanHtml(newPath + folders + '/' + file, reviewer=reviewer)

        splitPoint = len(allReviews)//ratio
        random.shuffle(allReviews)
        test, train = allReviews[:splitPoint], allReviews[splitPoint:]
    return (test, train)


def GetPath(path):
    folders = os.listdir(path)
    return TEST in folders and TRAIN in folders


def CleanHtml(htmlPath, reviewer=None):
    fd = open(htmlPath).read()
    soup = BeautifulSoup(fd, 'html.parser')
    reviewDict = {}
    #order of paragraphs:
    #reviewer, name, address, city, food, service, venue,
    #rating, written review, 4 paragraphs
    stop = False
    count = 1
    paras = []
    for paragraphs in soup.findAll("p"):
        print("paragraphs: |{}|".format(paragraphs))
        paragraph = re.sub(r'<[^<]+?>', '', str(paragraphs))
        if paragraph != "":
            print("paragraph: |{}|".format(paragraph))
            #paragraph = paragraph.encode('ascii', 'ignore').decode('utf-8', 'ignore')
            splitParagraph = ""
            if not stop:
                if (':' not in paragraph and paragraph != ""):
                    match = re.match(r"^(\S*)\s([\w\s]+)$", paragraph)
                    if match:
                        splitParagraph = list(match.groups())
                else:
                    splitParagraph = paragraph.split(':')
                print("splitParagraph: |{}|".format(splitParagraph))
                key = splitParagraph[0].lower()
                if(key == "written review"):
                    stop = True
                else:
                    if(key == 'reviewer' and reviewer):
                        reviewDict[key] = reviewer
                    else:
                        if(key != ""):
                            reviewDict[key] = splitParagraph[1].strip()
            if stop:
                if(len(splitParagraph) > 1):
                    paras.append(splitParagraph[1])
                else:
                    paras.append(paragraph)
                count += 1
        paras = [p for p in paras if p != ""]
        for i in range (1, len(paras) - 1):
            reviewDict["para" + str(i)] = paras[i-1]

    #print(sorted(reviewDict.items()))
    return sorted(reviewDict.items())


def main():
    path = ""
    if(len(sys.argv) > 1):
        if("-h" in sys.argv):
            print("Usage: restaurants.py DATA_DIR")
            print("The people who worked on this project are: <NAME> and <NAME>")
            if("-h" == sys.argv[1] and len(sys.argv) > 2):
                path = sys.argv[2]
            elif("-h" == sys.argv[2] and len(sys.argv) > 2):
                path = sys.argv[1]
        else:
            path = sys.argv[1]
    else:
        print("Usage: restaurants.py DATA_DIR")
        sys.exit(1)

    test,train = BuildDicts(path)
    print("Test: {}\n\nTrain: {}".format(test, train))


    return

if __name__ == "__main__":
    main()<commit_msg>Finish creating test and train sets<commit_after>import nltk, os, sys, re, random, string
from bs4 import BeautifulSoup

TRAIN = 'training'
TEST = 'test'


def BuildDicts(path):
    train = []
    test = []

    testAndTrain = GetPath(path)

    if testAndTrain:
        #parse the html files
        for file in os.listdir(path + '/' + TEST):
            filePath = path + '/' + TEST + '/' + file
            test.append(CleanHtml(filePath))

        for file in os.listdir(path + '/' + TRAIN):
            filePath = path + '/' + TRAIN + '/' + file
            train.append(CleanHtml(filePath))
    else:
        #files are in review folders
        allReviews = []
        for i in range(1,4):
            newPath = path + "/Review" + str(i) + "/"
            for folders in os.listdir(newPath):
                if(os.path.isdir(newPath + folders)):
                    for file in os.listdir(newPath + folders):
                        reviewer = folders.split('_')[0]
                        allReviews.append(CleanHtml(newPath + folders + '/' + file, reviewer=reviewer))

        random.shuffle(allReviews)

        #put one review from each reviewer in the test set and the rest in train
        reviewers_in_test = set()
        for review in allReviews:
            if (review != {}):
                if (review["reviewer"] in reviewers_in_test):
                    train.append(review)
                else:
                    test.append(review)
                    reviewers_in_test.add(review["reviewer"])

    return (test, train)


def GetPath(path):
    folders = os.listdir(path)
    return TEST in folders and TRAIN in folders


def CleanHtml(htmlPath, reviewer=None):
    fd = open(htmlPath).read()
    soup = BeautifulSoup(fd, 'html.parser')
    reviewDict = {}
    #order of paragraphs:
    #reviewer, name, address, city, food, service, venue,
    #rating, written review, 4 paragraphs
    stop = False
    count = 1
    paras = []
    seen = []
    for paragraphs in soup.findAll(["p", "span"]):
        #print("paragraphs: |{}|".format(paragraphs))
        paragraph = re.sub(r'<[^<]+?>', '', str(paragraphs))
        paragraph = re.sub(r'\n', ' ', str(paragraph)).strip()
        if (paragraph not in seen):
            seen.append(paragraph)
            if paragraph != "\n" and paragraph != "":
                #print("paragraph: |{}|".format(paragraph))
                #paragraph = paragraph.encode('ascii', 'ignore').decode('utf-8', 'ignore')
                splitParagraph = None
                if not stop:
                    if (':' not in paragraph and paragraph != ""):
                        match = re.match(r"^(\S*)\s([\w\s]+)$", paragraph)
                        if match:
                            splitParagraph = list(match.groups())
                    else:
                        splitParagraph = paragraph.split(':')
                    #print("splitParagraph: |{}|".format(splitParagraph))
                    key = splitParagraph[0].lower().strip()
                    if(key == "written review"):
                        stop = True
                    else:
                        if(key == 'reviewer' and reviewer):
                            reviewDict[key] = reviewer
                        else:
                            if(key != ""):
                                reviewDict[key] = splitParagraph[1].strip()
                if stop:
                    if(splitParagraph and len(splitParagraph) > 1):
                        paras.append(splitParagraph[1].strip())
                    else:
                        paras.append(paragraph)
                    count += 1
            paras = [p for p in paras if p != ""]
            for i in range (1, len(paras) + 1):
                reviewDict["para" + str(i)] = paras[i-1]

    return reviewDict


def main():
    path = ""
    if(len(sys.argv) > 1):
        if("-h" in sys.argv):
            print("Usage: restaurants.py DATA_DIR")
            print("The people who worked on this project are: <NAME> and <NAME>")
            if("-h" == sys.argv[1] and len(sys.argv) > 2):
                path = sys.argv[2]
            elif("-h" == sys.argv[2] and len(sys.argv) > 2):
                path = sys.argv[1]
        else:
            path = sys.argv[1]
    else:
        print("Usage: restaurants.py DATA_DIR")
        sys.exit(1)

    test,train = BuildDicts(path)
    print("Test: {}\n\nTrain: {}".format(test, train))


    return

if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.13689482470784642, 'fraction_numerical': 0.005843071786310518, 'mean_word_length': 1.6390811831340466, 'pattern_counts': {'":': 4, '<': 11, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1623936', 'n_tokens_mistral': 2503, 'n_tokens_neox': 2367, 'n_words': 591}