<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import re
from sys import argv
from os import mkdir
from os.path import sep
from urllib2 import urlopen, HTTPError
from utils import *

# Deuxième argument optionnel active l'usage du cache pour le debug
cache = len(argv) > 2

# Prend comme argument l'url de la liste des représentants et reconstruit les urls relatives en fonction du domaine
url = argv[1]
rooturl = re.sub(r'^(https?://[^/]+).*$', r'\1', url, re.I)
geturl = lambda t: re_href.sub(r'%s\1' % rooturl, t)
filename = "registre-lobbying-AN-v2"

re_clean_title = re.compile(r'<title>.*</title>')
clean_nonutf_title = lambda x: re_clean_title.sub("", x)

def download(url, attempts_left=3):
    try:
        return urlopen(url).read()
    except HTTPError as e:
        if attempts_left:
            time.sleep(5)
            return download(url, attempts_left-1)
        sys.stderr.write("ERROR downloading %s: %s" % (url, e))
        sys.exit(1)

try:
    mkdir('cache')
except:
    pass
# Télécharge archive en cache et nettoie une page
def get_html(url, name):
    cachefile = sep.join(["cache", "%s.html" % name])
    if not cache:
        h = download(url)
        with open(cachefile, 'w') as f:
            f.write(h)
    else:
        with open(cachefile, 'r') as f:
            h = f.read()
    h = clean_nonutf_title(h)
    h = h.decode('utf-8')
    h = clean_html(h)
    return h

# Extrait les informations de la fiche détaillée d'un représentant<commit_msg>fix bad import<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import re
from sys import argv
from time import sleep
from os import mkdir
from os.path import sep
from urllib2 import urlopen, HTTPError
from utils import *

# Deuxième argument optionnel active l'usage du cache pour le debug
cache = len(argv) > 2

# Prend comme argument l'url de la liste des représentants et reconstruit les urls relatives en fonction du domaine
url = argv[1]
rooturl = re.sub(r'^(https?://[^/]+).*$', r'\1', url, re.I)
geturl = lambda t: re_href.sub(r'%s\1' % rooturl, t)
filename = "registre-lobbying-AN-v2"

re_clean_title = re.compile(r'<title>.*</title>')
clean_nonutf_title = lambda x: re_clean_title.sub("", x)

def download(url, attempts_left=3):
    try:
        return urlopen(url).read()
    except HTTPError as e:
        if attempts_left:
            sleep(5)
            return download(url, attempts_left-1)
        sys.stderr.write("ERROR downloading %s: %s" % (url, e))
        sys.exit(1)

try:
    mkdir('cache')
except:
    pass
# Télécharge archive en cache et nettoie une page
def get_html(url, name):
    cachefile = sep.join(["cache", "%s.html" % name])
    if not cache:
        h = download(url)
        with open(cachefile, 'w') as f:
            f.write(h)
    else:
        with open(cachefile, 'r') as f:
            h = f.read()
    h = clean_nonutf_title(h)
    h = h.decode('utf-8')
    h = clean_html(h)
    return h

# Extrait les informations de la fiche détaillée d'un représentant
{'fraction_non_alphanumeric': 0.13069504489524442, 'fraction_numerical': 0.007981376787495843, 'mean_word_length': 3.0, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2153278', 'n_tokens_mistral': 1060, 'n_tokens_neox': 1063, 'n_words': 319}