<commit_before>import csv
import datetime
import gzip
import itertools
import os
import tarfile

import numpy as np
import requests

from ulmo import util

NCDC_GSOD_DIR = os.path.join(util.get_ulmo_dir(), 'ncdc_gsod')
NCDC_GSOD_STATIONS_FILE = os.path.join(NCDC_GSOD_DIR, 'ish-history.csv')
NCDC_GSOD_START_DATE = datetime.date(1929, 1, 1)


def get_data(station_codes, start_date=None, end_date=None, parameters=None):
    if start_date:
        if isinstance(start_date, datetime.datetime):
            start_date = start_date.date()
    else:
        start_date = NCDC_GSOD_START_DATE
    if end_date:
        if isinstance(end_date, datetime.datetime):
            end_date = end_date.date()
    else:
        end_date = datetime.datetime.now().date()<commit_msg>change ncdc file cache directory : 'ncdc_gsod' -> 'ncdc/gsod'<commit_after>import csv
import datetime
import gzip
import itertools
import os
import tarfile

import numpy as np
import requests

from ulmo import util

NCDC_GSOD_DIR = os.path.join(util.get_ulmo_dir(), 'ncdc/gsod')
NCDC_GSOD_STATIONS_FILE = os.path.join(NCDC_GSOD_DIR, 'ish-history.csv')
NCDC_GSOD_START_DATE = datetime.date(1929, 1, 1)


def get_data(station_codes, start_date=None, end_date=None, parameters=None):
    if start_date:
        if isinstance(start_date, datetime.datetime):
            start_date = start_date.date()
    else:
        start_date = NCDC_GSOD_START_DATE
    if end_date:
        if isinstance(end_date, datetime.datetime):
            end_date = end_date.date()
    else:
        end_date = datetime.datetime.now().date()
{'fraction_non_alphanumeric': 0.09406565656565656, 'fraction_numerical': 0.007575757575757576, 'mean_word_length': 3.922360248447205, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '761717', 'n_tokens_mistral': 569, 'n_tokens_neox': 546, 'n_words': 119}