<commit_before>#! /usr/bin/python2.7
# -*- coding: utf-8 -*-

import os
import sys
import urllib2
from operator import itemgetter

from settings import BASEURL, DIR, HTML_FIELDS
import utils

def get_metadata(assembly_id, range=(None, None)):
    with open('%s/%d.csv' % (DIR['meta'], assembly_id), 'r') as f:
        data = []
        for row in f.readlines():
            items = [item.strip('"') for item in row.decode('utf-8').split(',')]
            data.append(itemgetter(0,3,8)(items))

    meta = {}
    for d in data:
        meta[d[0]] = (d[1], d[2])

    return meta

def get_page(assembly_id, bill_id, link_id, field):
    url = '%s%s' % (BASEURL[field], link_id)
    outp = '%s/%s/%s.html' % (DIR[field], assembly_id, bill_id)

    if not os.path.isfile(outp):
        is_first = True
        while is_first or 'TEXTAREA ID="MSG" STYLE="display:none"' in doc:
            r = urllib2.urlopen(url)
            doc = r.read()
            is_first = False

        with open(outp, 'w') as f:
            f.write(doc)

def get_specifics(assembly_id, bill_id, link_id):
    if assembly_id > 16:
        baseurl = BASEURL['specifics']
    else:
        baseurl = BASEURL['specifics_old']<commit_msg>Bill page를 10번 이상 요청하지 않기<commit_after>#! /usr/bin/python2.7
# -*- coding: utf-8 -*-

import os
import sys
import urllib2
from operator import itemgetter

from settings import BASEURL, DIR, HTML_FIELDS
import utils

def get_metadata(assembly_id, range=(None, None)):
    with open('%s/%d.csv' % (DIR['meta'], assembly_id), 'r') as f:
        data = []
        for row in f.readlines():
            items = [item.strip('"') for item in row.decode('utf-8').split(',')]
            data.append(itemgetter(0,3,8)(items))

    meta = {}
    for d in data:
        meta[d[0]] = (d[1], d[2])

    return meta

def get_page(assembly_id, bill_id, link_id, field):
    url = '%s%s' % (BASEURL[field], link_id)
    outp = '%s/%s/%s.html' % (DIR[field], assembly_id, bill_id)

    if not os.path.isfile(outp):
        i = 0
        while i==0 or ('TEXTAREA ID="MSG" STYLE="display:none"' in doc and i<10):
            r = urllib2.urlopen(url)
            doc = r.read()
            i += 1

        with open(outp, 'w') as f:
            f.write(doc)

def get_specifics(assembly_id, bill_id, link_id):
    if assembly_id > 16:
        baseurl = BASEURL['specifics']
    else:
        baseurl = BASEURL['specifics_old']
{'fraction_non_alphanumeric': 0.15678141849854832, 'fraction_numerical': 0.014516798009124844, 'mean_word_length': 2.5892857142857144, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4474240', 'n_tokens_mistral': 901, 'n_tokens_neox': 887, 'n_words': 218}