<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;Objectifs :2&nbsp;&nbsp;Via https://github.com/weatherbit/weatherbit-python2.1&nbsp;&nbsp;Téléchargement et lecture de la météo2.1.1&nbsp;&nbsp;dev création du fichier2.1.2&nbsp;&nbsp;Industrialisation2.1.3&nbsp;&nbsp;Lecture du fichier2.1.4&nbsp;&nbsp;Analyse des features2.1.5&nbsp;&nbsp;Faire un call solo historique3&nbsp;&nbsp;OLD3.0.1&nbsp;&nbsp;Via api.meteo-concept.com3.0.1.1&nbsp;&nbsp;Commentaires3.0.2&nbsp;&nbsp;Via www.historique-meteo.net/france/aquitaine/bordeaux/3.0.2.1&nbsp;&nbsp;Commentaires :<jupyter_code>from dotenv import load_dotenv
load_dotenv()

import pandas as pd
import requests
import datetime
from weatherbit.api import Api

from vcub_keeper.reader.reader import *
from vcub_keeper.config import *

import plotly.express as px
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_style('whitegrid')
%matplotlib inline

%load_ext autoreload
%autoreload 2<jupyter_output><empty_output><jupyter_text>Objectifs : - Test des différentes API et site météo afin de créer un historique et pouvoir aussi query la météo au jour J (. - Création d'un fichier météo- Export (dans `ROOT_DATA_REF/meteo.csv`)- Lecture                                                                                                                github : https://github.com/armgilles/vcub_keeper/issues/24  Via https://github.com/weatherbit/weatherbit-python  Téléchargement et lecture de la météo  dev création du fichier<jupyter_code>api = Api(API_METEO)
#api.set_granularity('daily')
api.set_granularity('hourly')

# To forecast
#forecast = api.get_forecast(city="Bordeaux", country="FR")
# Init DataFrame
meteo_full = pd.DataFrame()

min_date_history = '2018-12-01'
max_date_history = '2020-09-18'
date_list = pd.date_range(start=min_date_history, end=max_date_history)
for date in date_list:
    # Date processing
    date_minus_one_day = date - datetime.timedelta(days=1)
    date_str = date.strftime(format='%Y-%m-%d')
    date_minus_one_day_str = date_minus_one_day.strftime(format='%Y-%m-%d')
    print(date_minus_one_day_str + " " + date_str)
    
    # Call API
    try:
        history = api.get_history(city="Bordeaux", country="FR",
                                  start_date=date_minus_one_day_str,
                                  end_date=date_str)
        meteo_day = pd.DataFrame(history.get_series(['temp', 'precip',
                                                     'rh', 'pres',
                                                     'wind_spd']))
        meteo_full = pd.concat([meteo_full, meteo_day])
    except requests.HTTPError as exception:
        print(exception)
meteo_full.head()
# Accumulated precipitation (default mm)
meteo_full.rename(columns = {'precip':'precipitation'}, inplace = True)
# Average temperature
meteo_full.rename(columns = {'temp':'temperature'}, inplace = True)
# Average relative humidity (%)
meteo_full.rename(columns = {'rh':'humidity'}, inplace = True)
# Average pressure (mb)
meteo_full.rename(columns = {'pres':'pressure'}, inplace = True)
# Wind_speed (m/s)
meteo_full.rename(columns = {'wind_spd':'wind_speed'}, inplace = True)

# date
meteo_full.rename(columns = {'datetime':'date'}, inplace = True)

meteo_full = meteo_full[['date', 'temperature',
                         'pressure', 'humidity',
                         'precipitation', 'wind_speed']]
meteo_full.tail()
meteo_full.shape
# Check
min_date = meteo_full.date.min()
max_date = meteo_full.date.max()
date_ref = pd.date_range(start=min_date, end=max_date, freq='h')

# Si le référenciel n'a pas toutes les dates dans Timestamp
assert date_ref.isin(meteo_full['date']).all() == True # Doit etre à True
    
# Si il n'y a pas de différence symetrique entre les 2 séries de dates
assert len(date_ref.symmetric_difference(meteo_full['date'])) == 0 # Doit etre à 0

# Si il y a des doublons
assert meteo_full['date'].is_unique == True # Doit etre à True

assert meteo_full['date'].is_monotonic_increasing == True # Doit etre à True
# export
meteo_full.to_csv(ROOT_DATA_REF+'meteo.csv',
                 index=False)<jupyter_output><empty_output><jupyter_text>Industrialisation<jupyter_code>def create_meteo(min_date_history="2018-12-01", max_date_history='2020-09-18'):
    """
    Multiple call API afin de créer un fichier et d'exporte celui-ci
    dans ROOT_DATA_REF/meteo.csv
    
    Parameters
    ----------
    min_date_history : str
        date du début de l'historique (format 'yyyy-mm-dd')
    max_date_history : str
        date de fin de l'historique (format 'yyyy-mm-dd')
    
    Returns
    -------
    None
        
    Examples
    --------
    
    create_meteo(min_date_history="2018-12-01", max_date_history='2021-03-18')
    """
    api = Api(API_METEO)
    api.set_granularity('hourly')
    
    # Init DataFrame
    meteo_full = pd.DataFrame()

    # Calls API
    date_list = pd.date_range(start=min_date_history, end=max_date_history)
    for date in date_list:
        # Date processing
        date_minus_one_day = date - datetime.timedelta(days=1)
        date_str = date.strftime(format='%Y-%m-%d')
        date_minus_one_day_str = date_minus_one_day.strftime(format='%Y-%m-%d')
        print(date_minus_one_day_str + " " + date_str)

        # Call API
        try:
            history = api.get_history(city="Bordeaux", country="FR",
                                      start_date=date_minus_one_day_str,
                                      end_date=date_str)
            meteo_day = pd.DataFrame(history.get_series(['temp', 'precip',
                                                         'rh', 'pres',
                                                         'wind_spd']))
            meteo_full = pd.concat([meteo_full, meteo_day])
        except requests.HTTPError as exception:
            print(exception)

    # Naming DataFrame
    
    # Accumulated precipitation (default mm)
    meteo_full.rename(columns = {'precip':'precipitation'}, inplace = True)
    # Average temperature
    meteo_full.rename(columns = {'temp':'temperature'}, inplace = True)
    # Average relative humidity (%)
    meteo_full.rename(columns = {'rh':'humidity'}, inplace = True)
    # Average pressure (mb)
    meteo_full.rename(columns = {'pres':'pressure'}, inplace = True)
    # Wind_speed (m/s)
    meteo_full.rename(columns = {'wind_spd':'wind_speed'}, inplace = True)
    # date
    meteo_full.rename(columns = {'datetime':'date'}, inplace = True)

    meteo_full = meteo_full[['date', 'temperature',
                             'pressure', 'humidity',
                             'precipitation', 'wind_speed']]
    
    # Check
    min_date = meteo_full.date.min()
    max_date = meteo_full.date.max()
    date_ref = pd.date_range(start=min_date, end=max_date, freq='h')

    # Si le référenciel n'a pas toutes les dates dans Timestamp
    assert date_ref.isin(meteo_full['date']).all() == True

    # Si il n'y a pas de différence symetrique entre les 2 séries de dates
    assert len(date_ref.symmetric_difference(meteo_full['date'])) == 0 

    # Si il y a des doublons
    assert meteo_full['date'].is_unique == True

    # Si les date augmentent
    assert meteo_full['date'].is_monotonic_increasing == True
    
    # export
    max_date_full = meteo_full.date.max().strftime('%Y-%m-%d')
    meteo_full.to_csv(ROOT_DATA_REF+'meteo/meteo_'+max_date_full+'.csv',
                     index=False)<jupyter_output><empty_output><jupyter_text>Lecture du fichier<jupyter_code>def read_meteo(path_directory, file_name='meteo.csv'):
    """
    Lecture du fichier météo dans le répertoire dans ROOT_DATA_REF
    Parameters
    ----------
    path_directory : str
        chemin d'accès (ROOT_DATA_REF)
    file_name : str
        Nom du fichier

    Returns
    -------
    meteo : DataFrame
        
    Examples
    --------
    
    meteo = read_meteo(path_directory=ROOT_DATA_REF)
    """
    
    meteo = pd.read_csv(path_directory+file_name, parse_dates=['date'])
    
    return meteo
from vcub_keeper.reader.reader import read_meteo
from vcub_keeper.config import ROOT_DATA_REF
meteo = read_meteo(path_directory=ROOT_DATA_REF)
print(meteo.tail().to_markdown())<jupyter_output>|       | date                |   temperature |   pressure |   humidity |   precipitation |   wind_speed |
|------:|:--------------------|--------------:|-----------:|-----------:|----------------:|-------------:|
| 15787 | 2020-09-17 19:00:00 |          26.4 |     1006.2 |         39 |               0 |          1.5 |
| 15788 | 2020-09-17 20:00:00 |          24.2 |     1006.3 |         49 |               0 |          0.5 |
| 15789 | 2020-09-17 21:00:00 |          23.9 |     1006.5 |         46 |               0 |          2.6 |
| 15790 | 2020-09-17 22:00:00 |          24.4 |     1006.3 |         45 |               0 |          3.1 |
| 15791 | 2020-09-17 23:00:00 |          24.4 |     1005.9 |         47 |               0 |          2.6 |<jupyter_text>Analyse des features<jupyter_code>plt.figure(figsize=(12,5))
sns.ecdfplot(data=meteo, x='precipitation');<jupyter_output><empty_output><jupyter_text>Faire un call solo historique<jupyter_code># Historique
api = Api(API_METEO)
api.set_granularity('hourly')

try:
    history = api.get_history(city="Bordeaux", country="FR", 
                              start_date='2020-09-25',end_date='2020-09-26')
except requests.HTTPError as exception:
    print(exception)<jupyter_output><empty_output><jupyter_text>[Documentation hourly features](https://www.weatherbit.io/api/weather-history-hourly)<jupyter_code>history.json
history.get_series(['temp', 'precip','rh', 'pres', 'wind_spd'])
pd.DataFrame(history.get_series(['max_temp', 'min_temp', 'precip','temp','rh', 'pres', 'wind_spd']))<jupyter_output><empty_output><jupyter_text>OLD  Via api.meteo-concept.com [Descriptif variables](https://api.meteo-concept.com/documentationforecast-map-day)  Commentaires Impossible en gratuit d'obtenir l'historique de la météo pour Bordeaux.<jupyter_code>token = "f4f60f6d565884dc7c946cccf2f49e2c7a994e1777462cea5a8168c8d8612e2d"
url = "https://api.meteo-concept.com/api/forecast/daily/0?token="+token+"&insee=33063"
response = requests.get(url=url)
response.status_code
all_meteo_bdx = response.json()
all_meteo_bdx
all_meteo_bdx['forecast']<jupyter_output><empty_output><jupyter_text>Via www.historique-meteo.net/france/aquitaine/bordeaux/  Commentaires :  Pas possible de récupérer les données météo de la journée... `https://www.historique-meteo.net/site/export.php?ville_id=156&annee=2019&mois=01`<jupyter_code># Lecture du fichier activité pour connaitre l'historique
ts_activity = read_time_serie_activity()
date_unique = ts_activity.drop_duplicates(subset=['date'])

# Format date
date_unique['date_year_month'] = date_unique['date'].dt.strftime('%Y-%m')

date_year_month = date_unique['date_year_month'].unique()
for date in date_year_month:
    print(date)
    date_year = date.split('-')[0]
    date_month = date.split('-')[1]
    url = "https://www.historique-meteo.net/site/export.php?ville_id=156&annee="+date_year+\
        "&mois="+date_month
date_year = '2019'
date_month = '01'
url = "https://www.historique-meteo.net/site/export.php?ville_id=156&annee="+date_year+\
        "&mois="+date_month
pd.read_csv?
meteo_month = pd.read_csv("https://www.historique-meteo.net/site/export.php?ville_id=156&annee=2019&mois=01",
                          skiprows=3)
response = requests.get(url="https://www.historique-meteo.net/site/export.php?ville_id=156&annee=2019&mois=01")
response.text<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13623413623413624, 'fraction_numerical': 0.030993697660364326, 'mean_word_length': 3.0208261020479, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 11, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '291154', 'license': 'MIT', 'n_tokens_mistral': 4290, 'n_tokens_neox': 4024, 'n_words': 686, 'path': 'notebooks/01_Analyse/02_Ref/02_telechargement_lecture_meteo #24.ipynb', 'repo_name': 'armgilles/vcub_keeper', 'size': '11583'}