<commit_before>'''

@author: awuerl

'''

import sys
import urllib2

import builder

class Url(object):

    host = 'http://blitzortung.net'
    base = host + '/Data_%d/Protected/'

    def __init__(self, baseurl):
        self.url = baseurl
        self.username = ''
        self.password = ''

    def set_credentials(self, username, password):
        self.username = username
        self.password = password

    def read(self):
        password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
        password_mgr.add_password("<PASSWORD>", "http://blitzortung.net", self.username, self.password)
        handler = urllib2.HTTPBasicAuthHandler(password_mgr)

        opener = urllib2.build_opener(handler)

        try:
            urlconnection = opener.open(self.url, timeout=60)
        except urllib2.URLError, error:
            sys.stderr.write("%s when opening '%s'\n" % (error, self.url))
            return None

        data = urlconnection.read().strip()
        urlconnection.close()

        return data.decode('ISO-8859-1')

class StrokesBase(Url):

    def __init__(self, base_url, config):
        super(StrokesBase, self).__init__(base_url)
        self.set_credentials(config.get_username(), config.get_password())

    def get_strokes(self, time_interval=None):
        strokes = []
        for line in self.read().split('\n'):
            if line.strip():
                stroke_builder = builder.Stroke()
                stroke_builder.from_string(line)
                stroke = stroke_builder.build()
                if not time_interval or time_interval.contains(stroke.get_timestamp()):
                    strokes.append(stroke)
        return strokes

class Strokes(StrokesBase):

    def __init__(self, config, region=1):
        super(Strokes, self).__init__(Url.base %(region) + 'strikes.txt', config)

class Participants(StrokesBase):

    def __init__(self, config, region=1):
        super(Participants, self).__init__(Url.base %(region) + 'participants.txt')

class Stations(Url):

    def __init__(self, config, region=1):
        super(Stations, self).__init__(Url.base %(region) + 'stations.txt')
        self.set_credentials(config.get_username(), config.get_password())

class Raw(Url):

    def __init__(self, config, region, station_id, hour=0):
        super(Raw, self).__init__(Url.base %(region) + 'raw_data/%s/%02d.log' %(station_id, hour))
        self.set_credentials(config.get('USERNAME'), config.get('PASSWORD'))<commit_msg>fixed use of new config object<commit_after>'''

@author: awuerl

'''

import sys
import urllib2

import builder

class Url(object):

    host = 'http://blitzortung.net'
    base = host + '/Data_%d/Protected/'

    def __init__(self, baseurl):
        self.url = baseurl
        self.username = ''
        self.password = ''

    def set_credentials(self, username, password):
        self.username = username
        self.password = password

    def read(self):
        password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
        password_mgr.add_password("<PASSWORD>", "http://blitzortung.net", self.username, self.password)
        handler = urllib2.HTTPBasicAuthHandler(password_mgr)

        opener = urllib2.build_opener(handler)

        try:
            urlconnection = opener.open(self.url, timeout=60)
        except urllib2.URLError, error:
            sys.stderr.write("%s when opening '%s'\n" % (error, self.url))
            return None

        data = urlconnection.read().strip()
        urlconnection.close()

        return data.decode('ISO-8859-1')

class StrokesBase(Url):

    def __init__(self, base_url, config):
        super(StrokesBase, self).__init__(base_url)
        self.set_credentials(config.get_username(), config.get_password())

    def get_strokes(self, time_interval=None):
        strokes = []
        for line in self.read().split('\n'):
            if line.strip():
                stroke_builder = builder.Stroke()
                stroke_builder.from_string(line)
                stroke = stroke_builder.build()
                if not time_interval or time_interval.contains(stroke.get_timestamp()):
                    strokes.append(stroke)
        return strokes

class Strokes(StrokesBase):

    def __init__(self, config, region=1):
        super(Strokes, self).__init__(Url.base %(region) + 'strikes.txt', config)

class Participants(StrokesBase):

    def __init__(self, config, region=1):
        super(Participants, self).__init__(Url.base %(region) + 'participants.txt')

class Stations(Url):

    def __init__(self, config, region=1):
        super(Stations, self).__init__(Url.base %(region) + 'stations.txt')
        self.set_credentials(config.get_username(), config.get_password())

class Raw(Url):

    def __init__(self, config, region, station_id, hour=0):
        super(Raw, self).__init__(Url.base %(region) + 'raw_data/%s/%02d.log' %(station_id, hour))
        self.set_credentials(config.get_username(), config.get_password())
{'fraction_non_alphanumeric': 0.12904522613065328, 'fraction_numerical': 0.007236180904522613, 'mean_word_length': 2.89358372456964, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6392580', 'n_tokens_mistral': 1574, 'n_tokens_neox': 1519, 'n_words': 294}