<commit_before>from bs4 import BeautifulSoup
import sys
import urllib2

def URLtoTitle(url):
    """This function takes in the URL of a Craigslist listing and extracts the
        title from the post"""
    if url == '':
        return ''
    else:
        response = soup = None
        assert(url.startswith("http"))
        try:
            response = urllib2.urlopen(url)
            page_source = response.read()
            soup = BeautifulSoup(page_source)
        finally:
            if (response != None): 
                response.close()
        if (soup.title == None):
            return ''
        else:
            return soup.title.string

def readFile(filename, mode="rt"):
    """This is a function taken from the 15-112 website. It allows the input of
        a string from the text file defined by filename."""
    # rt stands for "read text"
    fin = contents = None
    try:
        fin = open(filename, mode)
        contents = fin.read()
    finally:
        if (fin != None): fin.close()
    return contents

def removeScore(raw_text):
    """This is a function which removes the score from a string containing
    a score followed by the corresponding URL for a Craigslist entry"""
    data_arr = raw_text.split()
    len_d = len(data_arr)
    if (len_d > 1):
        return data_arr[1]
    elif (len_d == 0):
        return ''
    else:
        return ''

def raw_to_tuple(raw_text):
    scores = []
    titles = []
    for line in raw_text:
        data_arr = line.split()
        if (len(data_arr) > 1):
            scores.append(data_arr[0])
            titles.append(data_arr[1])
    return (scores, titles)

def writeFile(filename, contents, mode="wt"):
    """Taken from the 15-112 website"""
    # wt stands for "write text"
    fout = None
    try:
        fout = open(filename, mode)
        fout.write(contents)
    finally:
        if (fout != None): fout.close()
    return True

def getTitles(textfile):
    """This function extracts all of the titles from a file containing
    Craigslist URL's and places them into a list of strings"""
    raw_file = readFile(textfile).split('\n')
    (scores, URL_list) = (raw_to_tuple(raw_file))
    scores = [int(score) for score in scores]
    result = filter(None, map(URLtoTitle, URL_list))
    writeFile('titles.txt',unicode(result))
    return (scores,result)

#print getTitles(sys.argv[1])<commit_msg>scrapes description function added<commit_after>from bs4 import BeautifulSoup
import sys
import urllib2

def URLtoTitle(url):
    """This function takes in the URL of a Craigslist listing and extracts the
        title from the post"""
    if url == '':
        return ''
    else:
        response = soup = None
        assert(url.startswith("http"))
        try:
            response = urllib2.urlopen(url)
            page_source = response.read()
            soup = BeautifulSoup(page_source)
        finally:
            if (response != None): 
                response.close()
        if (soup.title == None):
            return ''
        else:
            title = soup.title.string
            with open("titles.txt", "a") as myTitles:
                myTitles.write((title+"\n").encode('utf8'))
            paras = soup.find_all('p')
            for para in paras:
                print para.get_text()
                with open("descriptions.txt", "a") as desc_f:
                    replaced = (para.get_text()).replace('\n',' ')
                    desc_f.write((replaced+' ').encode('utf8'))
            return title

def readFile(filename, mode="rt"):
    """This is a function taken from the 15-112 website. It allows the input of
        a string from the text file defined by filename."""
    # rt stands for "read text"
    fin = contents = None
    try:
        fin = open(filename, mode)
        contents = fin.read()
    finally:
        if (fin != None): fin.close()
    return contents

def removeScore(raw_text):
    """This is a function which removes the score from a string containing
    a score followed by the corresponding URL for a Craigslist entry"""
    data_arr = raw_text.split()
    len_d = len(data_arr)
    if (len_d > 1):
        return data_arr[1]
    elif (len_d == 0):
        return ''
    else:
        return ''

def raw_to_tuple(raw_text):
    scores = []
    titles = []
    for line in raw_text:
        data_arr = line.split()
        if (len(data_arr) > 1):
            scores.append(data_arr[0])
            titles.append(data_arr[1])
    return (scores, titles)

def writeFile(filename, contents, mode="wt"):
    """Taken from the 15-112 website"""
    # wt stands for "write text"
    fout = None
    try:
        fout = open(filename, mode)
        fout.write(contents)
    finally:
        if (fout != None): fout.close()
    return True

def getTitles(textfile):
    """This function extracts all of the titles from a file containing
    Craigslist URL's and places them into a list of strings"""
    raw_file = readFile(textfile).split('\n')
    (scores, URL_list) = (raw_to_tuple(raw_file))
    scores = [int(score) for score in scores]
    result = filter(None, map(URLtoTitle, URL_list))
    return (scores,result)

#print getTitles(sys.argv[1])
{'fraction_non_alphanumeric': 0.11108971890643049, 'fraction_numerical': 0.008086253369272238, 'mean_word_length': 2.277602523659306, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2302490', 'n_tokens_mistral': 1582, 'n_tokens_neox': 1554, 'n_words': 486}