<commit_before># -*- coding: utf-8 -*-
"""Supports Dst values. Downloads data from NGDC.

Parameters
----------
platform : string
    'sw'
name : string
    'dst'
tag : string
    None supported

Note
----
Will only work until 2057.

Download method should be invoked on a yearly frequency,
dst.download(date1, date2, freq='AS')

This material is based upon work supported by the 
National Science Foundation under Grant Number 1259508. 

Any opinions, findings, and conclusions or recommendations expressed in this 
material are those of the author(s) and do not necessarily reflect the views 
of the National Science Foundation.
       
"""

import os
import pandas as pds
import numpy as np

import pysat

def load(fnames, tag=None, sat_id=None):
    """Load Kp index files

    Parameters
    ------------
    fnames : (pandas.Series)
        Series of filenames
    tag : (str or NoneType)
        tag or None (default=None)
    sat_id : (str or NoneType)
        satellite id or None (default=None)

    Returns
    ---------
    data : (pandas.DataFrame)
        Object containing satellite data
    meta : (pysat.Meta)
        Object containing metadata such as column names and units

    Note
    ----
    Called by pysat. Not intended for direct use by user.
    
    """

    data = pds.DataFrame()
    
    for filename in fnames:
        # need to remove date appended to dst filename
        fname = filename[0:-11]
        #f = open(fname)
        with open(fname) as f:
            lines = f.readlines()
            idx = 0
            yr = np.zeros(len(lines)*24, dtype=int)
            mo = np.zeros(len(lines)*24, dtype=int)
            day = np.zeros(len(lines)*24, dtype=int)
            ut = np.zeros(len(lines)*24, dtype=int)
            dst = np.zeros(len(lines)*24, dtype=int)
            for line in lines:
                if len(line) > 0:
                    temp_year = int(line[14:16] + line[3:5]) 
                    if temp_year > 57:
                        temp_year += 1900
                    else:
                        temp_year += 2000
                        
                    yr[idx:idx+24] = temp_year
                    mo[idx:idx+24] = int(line[5:7])
                    day[idx:idx+24] = int(line[8:10])
                    ut[idx:idx+24] = np.arange(24)<commit_msg>Improved loading robustness<commit_after># -*- coding: utf-8 -*-
"""Supports Dst values. Downloads data from NGDC.

Parameters
----------
platform : string
    'sw'
name : string
    'dst'
tag : string
    None supported

Note
----
Will only work until 2057.

Download method should be invoked on a yearly frequency,
dst.download(date1, date2, freq='AS')

This material is based upon work supported by the 
National Science Foundation under Grant Number 1259508. 

Any opinions, findings, and conclusions or recommendations expressed in this 
material are those of the author(s) and do not necessarily reflect the views 
of the National Science Foundation.
       
"""

import os
import pandas as pds
import numpy as np

import pysat

def load(fnames, tag=None, sat_id=None):
    """Load Kp index files

    Parameters
    ------------
    fnames : (pandas.Series)
        Series of filenames
    tag : (str or NoneType)
        tag or None (default=None)
    sat_id : (str or NoneType)
        satellite id or None (default=None)

    Returns
    ---------
    data : (pandas.DataFrame)
        Object containing satellite data
    meta : (pysat.Meta)
        Object containing metadata such as column names and units

    Note
    ----
    Called by pysat. Not intended for direct use by user.
    
    """

    data = pds.DataFrame()
    
    for filename in fnames:
        # need to remove date appended to dst filename
        fname = filename[0:-11]
        #f = open(fname)
        with open(fname) as f:
            lines = f.readlines()
            idx = 0
            # check if all lines are good
            max_lines=0
            for line in lines:
                if len(line) > 1:
                    max_lines+=1
            yr = np.zeros(max_lines*24, dtype=int)
            mo = np.zeros(max_lines*24, dtype=int)
            day = np.zeros(max_lines*24, dtype=int)
            ut = np.zeros(max_lines*24, dtype=int)
            dst = np.zeros(max_lines*24, dtype=int)
            for line in lines:
                if len(line) > 1:
                    temp_year = int(line[14:16] + line[3:5]) 
                    if temp_year > 57:
                        temp_year += 1900
                    else:
                        temp_year += 2000
                        
                    yr[idx:idx+24] = temp_year
                    mo[idx:idx+24] = int(line[5:7])
                    day[idx:idx+24] = int(line[8:10])
                    ut[idx:idx+24] = np.arange(24)
{'fraction_non_alphanumeric': 0.10122635626688838, 'fraction_numerical': 0.02556641030970692, 'mean_word_length': 1.7434435575826681, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6245671', 'n_tokens_mistral': 1499, 'n_tokens_neox': 1378, 'n_words': 460}