<commit_before>import os
import numpy as np
import pandas as pd
import datetime as dt
import folium
import getpass
import urllib.request
from tqdm import tqdm


class Storm(object):
    """Main analysis class"""
    def __init__(self):
        pass

    def read_data(self):
        """Class for reading data"""
        pass

    def foo(self):
        """Class for foo"""
        pass


def get_data(start, end, dl_link, freq='10min'):
    """**Download data from Blitzorg**

    Using a specified time stamp for start and end, data is downloaded at a
    default frequency of 10 minute intervals. If a directory called data is not
    present, it will be added to the cwd as the target for the downloads.

    :paramter start: string
    :parameter end: string
    :parameter freq: string
    :paramater dl_link: string

    :Example:

    >>> get_data(start="2015-02-01T06:30", end="2015-02-01T10:05",
                dl_link="http://data.blitzortung.org/Data_1/Protected/Strokes/")
    """
    path = './data'
    try:
        os.stat(path)
    except:
        os.mkdir(path)
    username = input("Username to access Blitzorg with:")
    password = getpass.getpass(
        prompt='Blitzorg password for {0}:'.format(username))
    auth_handler = urllib.request.HTTPBasicAuthHandler()
    auth_handler.add_password(realm='Blitzortung',
                              uri='http://data.blitzortung.org',
                              user=username,
                              passwd=password)
    opener = urllib.request.build_opener(auth_handler)
    urllib.request.install_opener(opener)
    time_range = pd.date_range(start, end, freq=freq)
    for time_stamp in tqdm(time_range):
        tmp_link = dl_link+'/'.join(return_time_elements(time_stamp))+'.json.gz'
        tmp_name = "./data/bz-"+'-'.join(return_time_elements(time_stamp))+".json.gz"
        if os.path.isfile(tmp_name):
            print("{} exists. Aborting download attempt".format(tmp_name))
        else:
            # print('Downloading: ' + tmp_name) # print name if all okay...
            try:
                    urllib.request.urlretrieve(tmp_link, tmp_name)
            except Exception as inst:
                    print(inst)
                    print('  Encountered unknown error. Continuing.')


def return_time_elements(time_stamp):<commit_msg>freq=10 moved inside the function<commit_after>import os
import numpy as np
import pandas as pd
import datetime as dt
import folium
import getpass
import urllib.request
from tqdm import tqdm


class Storm(object):
    """Main analysis class"""
    def __init__(self):
        pass

    def read_data(self):
        """Class for reading data"""
        pass

    def foo(self):
        """Class for foo"""
        pass


def get_data(start, end, dl_link):
    """**Download data from Blitzorg**

    Using a specified time stamp for start and end, data is downloaded at a
    default frequency (10 minute intervals). If a directory called data is not
    present, it will be added to the cwd as the target for the downloads.

    :paramter start: string
    :parameter end: string
    :parameter freq: string
    :paramater dl_link: string

    :Example:

    >>> get_data(start="2015-02-01T06:30", end="2015-02-01T10:05",
                dl_link="http://data.blitzortung.org/Data_1/Protected/Strokes/")
    """
    path = './data'
    freq='10min'
    try:
        os.stat(path)
    except:
        os.mkdir(path)
    username = input("Username to access Blitzorg with:")
    password = getpass.getpass(
        prompt='Blitzorg password for {0}:'.format(username))
    auth_handler = urllib.request.HTTPBasicAuthHandler()
    auth_handler.add_password(realm='Blitzortung',
                              uri='http://data.blitzortung.org',
                              user=username,
                              passwd=password)
    opener = urllib.request.build_opener(auth_handler)
    urllib.request.install_opener(opener)
    time_range = pd.date_range(start, end, freq=freq)
    for time_stamp in tqdm(time_range):
        tmp_link = dl_link+'/'.join(return_time_elements(time_stamp))+'.json.gz'
        tmp_name = "./data/bz-"+'-'.join(return_time_elements(time_stamp))+".json.gz"
        if os.path.isfile(tmp_name):
            print("{} exists. Aborting download attempt".format(tmp_name))
        else:
            # print('Downloading: ' + tmp_name) # print name if all okay...
            try:
                    urllib.request.urlretrieve(tmp_link, tmp_name)
            except Exception as inst:
                    print(inst)
                    print('  Encountered unknown error. Continuing.')


def return_time_elements(time_stamp):
{'fraction_non_alphanumeric': 0.1088348271446863, 'fraction_numerical': 0.013230900554844216, 'mean_word_length': 2.4463235294117647, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4781735', 'n_tokens_mistral': 1473, 'n_tokens_neox': 1395, 'n_words': 362}