<commit_before>#!/usr/bin/env python3

"""
Documentation
"""
from ftplib import FTP
import zipfile
import io
import pandas as pd
from FTPData import rename_columns


def get_ftp_path(era):
    
    """
    Returns the path to the ftp server for given timeperiod.    
    
    INPUT
    -----
    era: string specifying the path to return, either 'recent' or 'historical'
    
    OUTPUT
    -----
    path: string specifying the webpath to the desired online data
    """
    
    if era == 'historical':
        return '/pub/CDC/observations_germany/climate/daily/kl/historical/'
    elif era == 'recent':
        return '/pub/CDC/observations_germany/climate/daily/kl/recent/'
    else:
        raise NameError('Input string not recognized')
        
        

def zipfilestring_to_stationID(fullzipstring, era):
    
    """
    Returns the station ID for the full string from the ftp-server.
    
    INPUT
    ------
    fullzipstring : string - a correct input should look like 
                        "<ftp_path>/<string_containing_id>.zip"
    
    OUTPUT
    ------
    station_ID : Station ID of type string as 5 digit number
    
    """    
    
    ftp_path =  get_ftp_path(era)
    
    if fullzipstring.startswith(ftp_path) and fullzipstring.endswith('.zip'):      
        zipstring = fullzipstring[len(ftp_path):]
        filestring = zipstring[:-4]      
    else:       
        raise NameError('This is not the correct full zipstring!')
        
    
    if era == 'historical':        
        stationID = int(filestring.split('_')[1])        
    elif era == 'recent':        
        stationID = filestring.split('_')[2]        
    else:        
        raise NameError('Requested era is not valid! It has to be historical or\
                                                                    recent.')
    
    return stationID

    
def get_era_from_zipstring(fullzipstring):

    """
    Extracts the era from a given zipstring.

    INPUT
    ------
    fullzipstring:   string of type '<ftp_path>/<string_containing_id>.zip'
    
    OUTPUT
    ------
    era:     string with either 'historical' or 'recent'
         
    """
    
    if fullzipstring.startswith(get_ftp_path('historical')):      
        return 'historical'
    elif  fullzipstring.startswith(get_ftp_path('recent')):        
        return 'recent'        
    else:
        raise NameError('Input string does not start with recent or historical\
                            ftp-path!')




def get_txtfile_savename(fullzipstring):
    
    """
    Returns the string needed to save data given by fullzipstring as .txt.
    
    INPUT
    ------
    fullzipstring:  string of type '<ftp_path>/<string_containing_id>.zip'

    OUTPUT
    ------
    txtsavename:  the name of the txt file that will contain the data

    """    
    
    era = get_era_from_zipstring(fullzipstring)
                            
    stationID = zipfilestring_to_stationID(fullzipstring, era)
    
    txtsavename = era+'_'+stationID
    
    return txtsavename
    
    
    
def get_all_zipfile_names(path):    
    
    """
    Return the list of zipfiles for given ftp-path.
    
    INPUT
    ------
    path : String specifying the webpath to the desired online data
    
    OUTPUT
    ------
    zipfile_names : list of strings with all the zipfile names 
                    with the station number and full ftp-path
    """
    
    ftp = FTP('ftp-cdc.dwd.de')
    ftp.login()
    zipfile_names = ftp.nlst(path)  
    
    return zipfile_names
    
    

#if __name__ in '__main__':    <commit_msg>added function to set up the directories for the download<commit_after>#!/usr/bin/env python3

"""
Documentation
"""
from ftplib import FTP
import zipfile
import io
import pandas as pd
import os


def get_ftp_path(era):
    
    """
    Returns the path to the ftp server for given timeperiod.    
    
    INPUT
    -----
    era: string specifying the path to return, either 'recent' or 'historical'
    
    OUTPUT
    -----
    path: string specifying the webpath to the desired online data
    """
    
    if era == 'historical':
        return '/pub/CDC/observations_germany/climate/daily/kl/historical/'
    elif era == 'recent':
        return '/pub/CDC/observations_germany/climate/daily/kl/recent/'
    else:
        raise NameError('Input string not recognized')
        
        

def zipfilestring_to_stationID(fullzipstring, era):
    
    """
    Returns the station ID for the full string from the ftp-server.
    
    INPUT
    ------
    fullzipstring : string - a correct input should look like 
                        "<ftp_path>/<string_containing_id>.zip"
    
    OUTPUT
    ------
    station_ID : Station ID of type string as 5 digit number
    
    """    
    
    ftp_path =  get_ftp_path(era)
    
    if fullzipstring.startswith(ftp_path) and fullzipstring.endswith('.zip'):      
        zipstring = fullzipstring[len(ftp_path):]
        filestring = zipstring[:-4]      
    else:       
        raise NameError('This is not the correct full zipstring!')
        
    
    if era == 'historical':        
        stationID = filestring.split('_')[1]       
    elif era == 'recent':        
        stationID = filestring.split('_')[2]        
    else:        
        raise NameError('Requested era is not valid! It has to be historical or\
                                                                    recent.')
    
    return stationID

    
def get_era_from_zipstring(fullzipstring):

    """
    Extracts the era from a given zipstring.

    INPUT
    ------
    fullzipstring:   string of type '<ftp_path>/<string_containing_id>.zip'
    
    OUTPUT
    ------
    era:     string with either 'historical' or 'recent'
         
    """
    
    if fullzipstring.startswith(get_ftp_path('historical')):      
        return 'historical'
    elif  fullzipstring.startswith(get_ftp_path('recent')):        
        return 'recent'        
    else:
        raise NameError('Input string does not start with recent or historical\
                            ftp-path!')




def get_txtfile_savename(fullzipstring):
    
    """
    Returns the string needed to save data given by fullzipstring as .txt.
    
    INPUT
    ------
    fullzipstring:  string of type '<ftp_path>/<string_containing_id>.zip'

    OUTPUT
    ------
    txtsavename:  the name of the txt file that will contain the data

    """    
    
    era = get_era_from_zipstring(fullzipstring)
                            
    stationID = zipfilestring_to_stationID(fullzipstring, era)
    
    txtsavename = era+'_'+stationID
    
    return txtsavename
    
    
    
def get_all_zipfile_names(path):    
    
    """
    Return the list of zipfiles for given ftp-path.
    
    INPUT
    ------
    path : String specifying the webpath to the desired online data
    
    OUTPUT
    ------
    zipfile_names : list of strings with all the zipfile names 
                    with the station number and full ftp-path
    """
    
    ftp = FTP('ftp-cdc.dwd.de')
    ftp.login()
    zipfile_names = ftp.nlst(path)  
    
    return zipfile_names
    
    
    

def set_up_directories(era='all'):
    
    """
    Delete old directories in the selected folder of the specified era(s) and 
    create new empty directories.
    
    INPUT
    ------
    era: string specifying the path to return, either 'recent', 'historical' or
            'all'
    OUTPUT
    ------
    no output.
    
    """
    string_1 ='downloaded_data'
    string_1a = 'historical'
    string_1b = 'recent'
    masterpath = os.getcwd()
    
    if era == 'all':
        #Check if directory exists and delete it if so
        if os.path.isdir('downloaded_data'):
            import shutil
            shutil.rmtree('downloaded_data')
        #Create directories
        os.mkdir(string_1)
        os.chdir(string_1)
        os.mkdir(string_1a)
        os.mkdir(string_1b)
        os.chdir(masterpath)
        
    elif era == 'historical':
        #Check if directory exists and delete it if so
        if os.path.isdir('downloaded_data/'+era):
            import shutil
            shutil.rmtree('downloaded_data/'+era)
        #Create directories
        if os.path.isdir('downloaded_data/'):
            os.chdir(string_1)
            os.mkdir(string_1a)
            os.chdir(masterpath)
        else:
            os.mkdir(string_1)
            os.chdir(string_1)
            os.mkdir(string_1a)
            os.chdir(masterpath)

    elif era == 'recent':
        #Check if directory exists and delete it if so
        if os.path.isdir('downloaded_data/'+era):
            import shutil
            shutil.rmtree('downloaded_data/'+era)
        #Create directories
        if os.path.isdir('downloaded_data/'):
            os.chdir(string_1)
            os.mkdir(string_1b)
            os.chdir(masterpath)
        else:
            os.mkdir(string_1)
            os.chdir(string_1)
            os.mkdir(string_1b)
            os.chdir(masterpath)
            
            
            

#if __name__ in '__main__':    
{'fraction_non_alphanumeric': 0.09340659340659341, 'fraction_numerical': 0.0029670329670329672, 'mean_word_length': 1.764580801944107, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1833862', 'n_tokens_mistral': 2815, 'n_tokens_neox': 2651, 'n_words': 699}