<commit_before>import re
import requests
import sys
import json

from multiprocessing import Pool, freeze_support
from BeautifulSoup import BeautifulSoup

links = []
urlPattern = re.compile("(http://|https://)(\S+)")

def parseLink(link):
    try:
        r = requests.get(link)
        r.encoding = 'UTF-8'
        title = ''
        desc = ''

        if r.status_code == 200:
            if r.headers['Content-Type'] is 'text/html':
                html = BeautifulSoup(r.content)
                desc = html.find('meta', attrs={'name': 'description'})
                title = html.title.string
                desc = ["" if desc is None else desc['content']]

        return {
                'url': link,
                'title': title,
                'desc': desc
            }
    except:
        print "Error %s - %s\n" % (link, sys.exc_info()[0])
        return None


if __name__ == '__main__':
    freeze_support()
    p = Pool()

    f = open("../Version-1/academia-de-ursarie-version-1.txt", "r")
    for line in f:
        url = urlPattern.search(line)
        if url is not None:
            links.append(url.group(0))
    f.close()

    jsonData = p.map(parseLink, links)
    jsonData

    f = open("../links.json", "w+")
    f.write(json.dumps(jsonData))
    f.close()
<commit_msg>Refactor parser script<commit_after>import re
import requests
import sys
import threading
import io
import json

from multiprocessing import Pool, freeze_support
from bs4 import BeautifulSoup

lockObj = threading.Lock()
links = []
urlPattern = re.compile(
        "(\[([\d\/]+\s*[\d:]+\s*\w+)\]).*((http|https)://([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:/~+#-]*[\w@?^=%&/~+#-])?)")


class ParseLink():
    def __init__(self, *data):
        (self.__date, self.__link) = data
        self.__title = ''
        self.__desc = ''

    def parse(self):
        try:
            r = requests.get(self.__link)
            r.encoding = 'UTF-8'

            if r.status_code == 200:
                if "text/html" in r.headers['Content-Type']:
                    (self.__title, self.__desc) = self.__extract_data(r.content)
                self.__write_to_file()

        except:
            print "Error %s - %s\n" % (self.__link, sys.exc_info()[0])

    @staticmethod
    def __extract_data(body):
        html = BeautifulSoup(body)
        desc = html.find('meta', attrs={'name': 'description'})
        title = html.title.string.strip(' \t\n\r')
        desc = desc['content'].strip(' \t\n\r') if desc is not None else ''

        return title, desc

    def __write_to_file(self):
        lockObj.acquire()
        with open("../links.json", "a") as jf:
            jf.write(str(self) + ",")
        lockObj.release()

    def __str__(self):
        return json.dumps({"url": self.__link, "date": self.__date, "title": self.__title, "desc": self.__desc})


def parseLink(data):
    print "Parse: %s" % data[1]
    ParseLink(*data).parse()


if __name__ == '__main__':
    freeze_support()
    p = Pool()

    f = open("../Version-1/academia-de-ursarie-version-1.txt", "r")
    for line in f:
        url = urlPattern.search(line)
        if url is not None:
            links.append((url.group(2), url.group(3)))
    f.close()

    jsonFile = open("../links.json", "w+")
    jsonFile.write("[")
    jsonFile.close()

    p.map(parseLink, links)

    jsonFile = open("../links.json", "r+")
    jsonFile.seek(-1, io.SEEK_END)
    jsonFile.write("]")
    jsonFile.close()

{'fraction_non_alphanumeric': 0.1743145743145743, 'fraction_numerical': 0.005772005772005772, 'mean_word_length': 2.150909090909091, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3573679', 'n_tokens_mistral': 1181, 'n_tokens_neox': 1142, 'n_words': 251}