<jupyter_start><jupyter_text>Surfcast.comA BSC Creation<jupyter_code># Reset Notebook
%reset
# Import python packages
from bs4 import BeautifulSoup
from urllib import request
import requests
from datetime import datetime
import matplotlib
import numpy as np
import matplotlib.pyplot as plt

# Matplotlib inline
%matplotlib inline
# Set URL path to NOAA database
url = "http://www.glerl.noaa.gov/ftp/EMF/glcfs/gridded_fields/FCAST/"
# Get HTML from database page
html = requests.get(url)
# Create BeautifulSoup object
html_obj = BeautifulSoup(html.content)
# File types of interest [wave, wind, surface current, surface temperature]
file_types = ['wav', 'wnd', 'cur', 'swt']
# Create empty dictionary
files = []
# Define NOAA file class
class NoaaFile:
    
    """
    Class: NoaaFile
        - This class decribes FCAST|NCAST database files.
    """

    # Initialize object
    def __init__(self, **kwargs):
        
        # Set object attributes
        self.filename = kwargs.get('filename')                  # File name
        self.file_extension = kwargs.get('file_extension')      # File extension
        self.filedate = kwargs.get('filedate')                  # File upload date
        self.filetime = kwargs.get('filetime')                  # File upload time
        self.filesize = kwargs.get('filesize')                  # File size (MB)
        
        # Convert data and time strings to a date|time object
        self.file_datetime = datetime.strptime(self.filedate + '-' + self.filetime, '%d-%b-%Y-%H:%M')

        # Set file type
        if self.file_extension == 'wav':           # Wave
            self.filetype = 'WAVES'
        elif self.file_extension == 'wnd':         # Wind
            self.filetype = 'WINDS'
        elif self.file_extension == 'cur':         # Surface Current
            self.filetype = 'SURFACE CURRENTS'
        elif self.file_extension == 'swt':         # Surface Temperature
            self.filetype = 'SURFACE TEMPS'
            
        # Set great lake
        if self.filename[0] == 'e':       # Lake Erie
            self.lake = 'erie'
        elif self.filename[0] == 'h':     # Lake Huron
            self.lake = 'huron'
        elif self.filename[0] == 'o':     # Lake Ontario
            self.lake = 'ontario'
        elif self.filename[0] == 's':     # Lake Superior
            self.lake = 'superior'
        elif self.filename[0] == 'm':     # Lake Michigan
            self.lake = 'michigan'
# Create list of NoaaFile objects for all current database entires
for link in html_obj.findAll('tr'):
    if len(link.contents) == 5 and link.contents[1].text.split('.')[-1] in file_types:
        files.append(NoaaFile(filename=link.contents[1].text,
                              file_extension=link.contents[1].text.split('.')[-1],
                              filedate=link.contents[2].text.split(' ')[0],
                              filetime=link.contents[2].text.split(' ')[1],
                              filesize=link.contents[3].text[0:len(link.contents[3].text)-1]))
# Define file download funtion
def download_file(url, files, lake, attribute):
    
    """
    Function: download_file
        - This function will download the most recently uploaded file to the NOAA database that 
          corresponds to the lake and attribute input by the user.
          
    Inputs:
        - url:        Database URL (NCAST or FCAST)
        - files:      List of all NOAA file objects currently in database
        - lake:       Lake of interest [ontario, michigan, erie, superior, huron] 
        - attribute:  Data measurement [winds, waves, surface currents, surface temperatures] 
        
    Outputs:
        - file:       The corresponding text file
    """
    
    # Find most up to date FORECAST
    date_time = files[0].file_datetime
    for file in files:
        if file.file_datetime > date_time and file.lake == lake and file.filetype == attribute.upper():
            date_time = file.file_datetime
            filename = file.filename

    # Join url and filename
    file_url = str(url + filename)

    # Send file request to server and download
    response = request.urlopen(file_url)

    # Parse test file by line breaks
    file = str(response.read()).split('\\n')

    return file
# Download most up to date FORECAST file
file = download_file(url, files, 'ontario', 'waves')
# Define Lake data class
class LakeDataFile(NoaaFile):
    
    # Initialize object
    def __init__(self, **kwargs):
        
        # Set object attributes
        self.textfile = []                  # Raw text file
        self.lake data
        
    # Download text file
    def download_file(self, url, files, lake, attribute):
        
        """
        Function: download_file
            - This function will download the most recently uploaded file to the NOAA database that 
              corresponds to the lake and attribute input by the user.

        Inputs:
            - url:        Database URL (NCAST or FCAST)
            - files:      List of all NOAA file objects currently in database
            - lake:       Lake of interest [ontario, michigan, erie, superior, huron] 
            - attribute:  Data measurement [winds, waves, surface currents, surface temperatures] 

        Outputs:
            - file:       The corresponding text file
        """

        # Find most up to date FORECAST
        date_time = files[0].file_datetime
        for file in files:
            if file.file_datetime > date_time and file.lake == lake and file.filetype == attribute.upper():
                date_time = file.file_datetime
                filename = file.filename

        # Join url and filename
        file_url = str(url + filename)

        # Send file request to server and download
        response = request.urlopen(file_url)

        # Parse test file by line breaks
        self.textfile = str(response.read()).split('\\n')
        
    
#         # Extract header information  
#         for row in range(len(file)):
#             if files[file_num].filetype in file[row]:
#                 # Year, day, hour
#                 if file[row][0] == 'b':  
#                     self.year = file[row][2:6]  # year
#                     self.day = file[row][7:10]  # day of the year
#                     print(self.year)
#                     print(self.day)
#                     print('')
#                 else: 
#                     self.year = file[row][0:4]  # year
#                     self.day = file[row][5:8]  # day of the year
#                     print(self.year)
#                     print(self.day)
#                     print('')
dummy = LakeData()
dummy.textfile
dummy.download_file(url, files, 'ontario', 'waves')
dummy.textfile<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09917355371900827, 'fraction_numerical': 0.004722550177095631, 'mean_word_length': 1.9363084922010398, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '637602', 'license': 'MIT', 'n_tokens_mistral': 1912, 'n_tokens_neox': 1853, 'n_words': 565, 'path': 'archive/GetData/GetData.ipynb', 'repo_name': 'Seb-Good/Surfcast', 'size': '6776'}