<commit_before>import requests
import re
from lxml import etree
from HTMLParser import HTMLParser

from regenesis.cube import Cube

QUADER = re.compile(r"<quaderDaten>\* (.*)<\/quaderDaten>", re.S | re.M)

def fetch_index(catalog):
    for i in range(10):
        params = [
            ('method', 'DatenKatalog'),
            ('kennung', catalog.get('username')),
            ('passwort', catalog.get('password')),
            ('filter', '%s*' % i),
            ('bereich', 'Alle'),
            ('listenLaenge', ''),
            ('sprache', 'de')
            ]
        doc = requests.get(catalog.get('index_url'), params=params)
        doc = etree.fromstring(doc.content)
        for entry in doc.findall('.//datenKatalogEintraege/datenKatalogEintraege'):
            yield entry.findtext('./code')

def fetch_cube(catalog, name):
    params = [
        ('method', 'DatenExport'),
        ('kennung', catalog.get('username')),
        ('passwort', catalog.get('password')),
        ('namen', name),
        ('bereich', 'Alle'),
        ('format', 'csv'),
        ('werte', True),
        ('metadaten', True),
        ('zusatz', True),
        ('startjahr', 1900),
        ('endjahr', ''),
        ('zeitscheiben', ''),
        ('inhalte', ''),
        ('regionalmerkmal', ''),
        ('regionalschluessel', ''),
        ('sachmerkmal', ''),
        ('sachschluessel', ''),
        ('sachmerkmal2', ''),
        ('sachschluessel2', ''),
        ('sachmerkmal3', ''),
        ('sachschluessel3', ''),
        ('stand', ''),
        ('sprache', 'de')
        ]
    doc = requests.get(catalog.get('export_url'), params=params)
    #print [doc.url]
    #doc = etree.fromstring(doc.content)
    #return doc.find('.//quaderDaten').text
    m = QUADER.search(doc.content)
    if m is None:
        return
    return HTMLParser().unescape(m.group(1))
    #return Cube(name, data)
<commit_msg>attempt to fix #11 by running more granular catalog queries. <commit_after>import requests
import re
from lxml import etree
from HTMLParser import HTMLParser

from regenesis.cube import Cube

QUADER = re.compile(r"<quaderDaten>\* (.*)<\/quaderDaten>", re.S | re.M)

def fetch_index(catalog):
    for i in range(10, 100):
        params = [
            ('method', 'DatenKatalog'),
            ('kennung', catalog.get('username')),
            ('passwort', catalog.get('password')),
            ('filter', '%s*' % i),
            ('bereich', 'Alle'),
            ('listenLaenge', '500'),
            ('sprache', 'de')
            ]
        doc = requests.get(catalog.get('index_url'), params=params)
        doc = etree.fromstring(doc.content)
        for entry in doc.findall('.//datenKatalogEintraege/datenKatalogEintraege'):
            #print [entry.findtext('./code')]
            yield entry.findtext('./code')

def fetch_cube(catalog, name):
    params = [
        ('method', 'DatenExport'),
        ('kennung', catalog.get('username')),
        ('passwort', catalog.get('password')),
        ('namen', name),
        ('bereich', 'Alle'),
        ('format', 'csv'),
        ('werte', True),
        ('metadaten', True),
        ('zusatz', True),
        ('startjahr', 1900),
        ('endjahr', ''),
        ('zeitscheiben', ''),
        ('inhalte', ''),
        ('regionalmerkmal', ''),
        ('regionalschluessel', ''),
        ('sachmerkmal', ''),
        ('sachschluessel', ''),
        ('sachmerkmal2', ''),
        ('sachschluessel2', ''),
        ('sachmerkmal3', ''),
        ('sachschluessel3', ''),
        ('stand', ''),
        ('sprache', 'de')
        ]
    doc = requests.get(catalog.get('export_url'), params=params)
    #print [doc.url]
    #doc = etree.fromstring(doc.content)
    #return doc.find('.//quaderDaten').text
    m = QUADER.search(doc.content)
    if m is None:
        print "NO CUBE CONTENT", catalog, name
        return
    return HTMLParser().unescape(m.group(1))
    #return Cube(name, data)

{'fraction_non_alphanumeric': 0.19153277225197654, 'fraction_numerical': 0.0076511094108645756, 'mean_word_length': 2.3636363636363638, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2810419', 'n_tokens_mistral': 1307, 'n_tokens_neox': 1280, 'n_words': 244}