<commit_before># -*- coding: utf-8 -*-
from os import mkdir
from os.path import exists
from re import search
from BeautifulSoup import BeautifulSoup
from urllib import urlopen
from json import dumps
from dateutil.parser import parse

TO_IGNORE = [
    "[date inconnue]",
    u"[Premier trimestre 2010] Européana (librairie numérique)",
    "Internet",
    u"[Date inconnue] Libertés publiques, libertés fondamentales",
    u"[date inconnue probablement fin 2009] Internet, vente de médicaments",
    u"contrefaçon de médicaments",
    "[date inconnue]",
    u"[Premier trimestre 2010] Européana (librairie numérique)",
    u"[date inconnue] Wikileaks, Internet",
    u"Paquet télécom, avant le vote",
]


def read_or_dl(url, name):
    if exists('dumps/%s' % name):
        return BeautifulSoup(open('dumps/%s' % name, 'r').read())
    html = urlopen(url).read()
    open('dumps/%s' % name, 'w').write(html)
    return BeautifulSoup(html)


def handle_reps_set(set_name, section):
    opinions = []
    size = len(open(set_name, "r").read().split('\n'))
    for a, i in enumerate(open(set_name, "r"), 1):
        i = i.replace('\n', '')
        print "[%s/%s]" % (a, size), i
        soup = read_or_dl("http://www.laquadrature.net/wiki/index.php?title=%s/fr" % i, i)
        if soup.find('div', id='bodyContent').h5 is None:
            continue
        opinion = {}
        if soup.find('div', id='bodyContent').h5.a:
            opinion['url'] = soup.find('div', id='bodyContent').h5.a["href"]
            opinion['title'] = soup.find('div', id='bodyContent').h5.a.text
        else:<commit_msg>[fix] url scheme for scraping old opinions<commit_after># -*- coding: utf-8 -*-
from os import mkdir
from os.path import exists
from re import search
from BeautifulSoup import BeautifulSoup
from urllib import urlopen
from json import dumps
from dateutil.parser import parse

TO_IGNORE = [
    "[date inconnue]",
    u"[Premier trimestre 2010] Européana (librairie numérique)",
    "Internet",
    u"[Date inconnue] Libertés publiques, libertés fondamentales",
    u"[date inconnue probablement fin 2009] Internet, vente de médicaments",
    u"contrefaçon de médicaments",
    "[date inconnue]",
    u"[Premier trimestre 2010] Européana (librairie numérique)",
    u"[date inconnue] Wikileaks, Internet",
    u"Paquet télécom, avant le vote",
]


def read_or_dl(url, name):
    if exists('dumps/%s' % name):
        return BeautifulSoup(open('dumps/%s' % name, 'r').read())
    html = urlopen(url).read()
    open('dumps/%s' % name, 'w').write(html)
    return BeautifulSoup(html)


def handle_reps_set(set_name, section):
    opinions = []
    size = len(open(set_name, "r").read().split('\n'))
    for a, i in enumerate(open(set_name, "r"), 1):
        i = i.replace('\n', '')
        print "[%s/%s]" % (a, size), i
        soup = read_or_dl("http://www.laquadrature.net/wiki/%s?test=true" % i, i)
        if soup.find('div', id='bodyContent').h5 is None:
            continue
        opinion = {}
        if soup.find('div', id='bodyContent').h5.a:
            opinion['url'] = soup.find('div', id='bodyContent').h5.a["href"]
            opinion['title'] = soup.find('div', id='bodyContent').h5.a.text
        else:
{'fraction_non_alphanumeric': 0.15930304915992533, 'fraction_numerical': 0.01120099564405725, 'mean_word_length': 3.3504736129905277, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '486917', 'n_tokens_mistral': 1140, 'n_tokens_neox': 1091, 'n_words': 275}