<commit_before>""" Interface to the noaa site and grab daily 10.7 cm solar flux measurements

"""

import datetime
import numpy as np
import os
import glob
from ftplib import FTP

from astropy.time import Time

#-------------------------------------------------------------------------------

def grab_solar_files(file_dir):
    ftp = FTP('ftp.swpc.noaa.gov')
    ftp.login()

    ftp.cwd('/pub/indices/old_indices/')

    for item in sorted(ftp.nlst()):
        if (item.endswith('_DSD.txt')):
            year = int(item[:4])
            if year >= 1997:
                print 'Retrieving: {}'.format(item)
                destination = os.path.join(file_dir, item)
                ftp.retrbinary('RETR {}'.format(item), open(destination, 'wb').write)


#-------------------------------------------------------------------------------

def compile_txt( file_dir ):
    date = []
    flux = []
    input_list = glob.glob(os.path.join(file_dir, '*DSD.txt'))
    input_list.sort()
    for item in input_list:
        try:
            data = np.genfromtxt(item, skiprows=13, dtype=None)
        except IOError:
            continue
        except StopIteration:
            continue
        for line in data:
            line_date = Time('{}-{}-{} 00:00:00'.format(line[0], line[1], line[2]),
                             scale='utc', format='iso').mjd
            line_flux = line[3]
            if line_flux > 0:
                date.append(line_date)
                flux.append(line_flux)

    return np.array(date), np.array(flux)

#-------------------------------------------------------------------------------

def get_solar_data( file_dir ):
    print 'Gettting Solar flux data'
    for txtfile in glob.glob(os.path.join(file_dir, '*_D?D.txt')):<commit_msg>Had to deal with improperly formatted solar data<commit_after>""" Interface to the noaa site and grab daily 10.7 cm solar flux measurements

"""

from astropy.io import ascii
import datetime
import numpy as np
import os
import glob
from ftplib import FTP

from astropy.time import Time

#-------------------------------------------------------------------------------

def grab_solar_files(file_dir):
    ftp = FTP('ftp.swpc.noaa.gov')
    ftp.login()

    ftp.cwd('/pub/indices/old_indices/')

    for item in sorted(ftp.nlst()):
        if (item.endswith('_DSD.txt')):
            year = int(item[:4])
            if year >= 2000:
                print 'Retrieving: {}'.format(item)
                destination = os.path.join(file_dir, item)
                ftp.retrbinary('RETR {}'.format(item), open(destination, 'wb').write)


#-------------------------------------------------------------------------------

def compile_txt( file_dir ):
    date = []
    flux = []
    input_list = glob.glob(os.path.join(file_dir, '*DSD.txt'))
    input_list.sort()
    for item in input_list:
        print 'Reading {}'.format(item)        
        data = ascii.read(item, data_start=1, comment='[#,:]')
        for line in data:
            line_date = Time('{}-{}-{} 00:00:00'.format(line['col1'], line['col2'], line['col3']),
                             scale='utc', format='iso').mjd
            line_flux = line[3]
            if line_flux > 0:
                date.append(line_date)
                flux.append(line_flux)

    return np.array(date), np.array(flux)

#-------------------------------------------------------------------------------

def get_solar_data( file_dir ):
    print 'Gettting Solar flux data'
    for txtfile in glob.glob(os.path.join(file_dir, '*_D?D.txt')):
{'fraction_non_alphanumeric': 0.2527503526093089, 'fraction_numerical': 0.01156558533145275, 'mean_word_length': 1.5220483641536273, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4748598', 'n_tokens_mistral': 1082, 'n_tokens_neox': 1003, 'n_words': 240}