<commit_before># -*- coding: utf-8 -*-
"""
General caching functionality. This module defines:

1. the Cache class
2. generic cache functions to store daily results

Created on Thu Jan  7 09:34:04 2016

@author: roel
"""
import os
import numpy as np
import pandas as pd
from opengrid import config
cfg = config.Config()
from opengrid.library import misc
from opengrid.library import analysis

class Cache(object):
    """
    A class to handle daily aggregated data or intermediate results
    
    The file format for the data is result_sensor.csv
    """
    
    def __init__(self, variable, folder=None):
        """
        Create a cache object specifically for the specified variable
        
        Arguments
        ---------
        variable : str
            The name of the aggregated variable to be stored.
            Examples: elec_standby, water_max, ...
        folder : path
            Path where the files are stored
            If None, use the path specified in the opengrid configuration
            
        """
        self.variable = variable
        if folder is None:
            try:
                self.folder = os.path.join(os.path.abspath(cfg.get('data', 'folder')),'cache_day')
            except:
                raise ValueError("Specify a folder, either in the opengrid.cfg or when creating this cache object.")
        else:
            self.folder = os.path.abspath(folder)
            
        if not os.path.exists(self.folder):
            print("This folder does not exist: {}, it will be created".format(self.folder))
            os.mkdir(self.folder)
            
        print("Cache object created for variable: {}".format(self.variable))

   
    def _load(self, sensorkey):
        """
        Return a dataframe with cached data for this sensor.  If there is no
        cached data, return an empty dataframe

        Arguments
        ---------
        sensor : str
            sensor key, unique string identifier of the sensor.

        Returns
        -------
        df : tz-aware dataframe with cached daily results or empty dataframe.
        
        """
        # Find the file and read into a dataframe
        filename = self.variable + '_' + sensorkey + '.csv'
        path = os.path.join(self.folder, filename)
        
        if not os.path.exists(path):
            #print("Could not find {}".format(path))
            return pd.DataFrame()
        
        df = pd.read_csv(path, index_col = 0, header=0, parse_dates=True)
        df.index = df.index.tz_localize('UTC')
        return df
    
    
    def _write_single(self, df):
        """
        Write the dataframe with single sensor to csv according to the filename conventions

        Arguments
        ---------
        df : pandas DataFrame or Series
        """

        # handle both cases correctly: dataframe or series
        if isinstance(df, pd.DataFrame):
            if len(df.columns) != 1:
                raise ValueError("Wrong number of columns")
            df_temp = df.copy()
        elif isinstance(df, pd.Series):
            if df.name is None:
                raise ValueError("pandas Series needs a name with sensor id")
            df_temp = pd.DataFrame(df)

        sensor = df_temp.columns[0]
        # Find the file and read into a dataframe
        filename = self.variable + '_' + sensor + '.csv'
        path = os.path.join(self.folder, filename)
        
        df_temp.to_csv(path)
        #print("Values for {} written to {}".format(sensor, path))<commit_msg>better date parsing when reading from cache<commit_after># -*- coding: utf-8 -*-
"""
General caching functionality. This module defines:

1. the Cache class
2. generic cache functions to store daily results

Created on Thu Jan  7 09:34:04 2016

@author: roel
"""
import os
import numpy as np
import pandas as pd
import dateutil
from opengrid import config
cfg = config.Config()
from opengrid.library import misc
from opengrid.library import analysis

class Cache(object):
    """
    A class to handle daily aggregated data or intermediate results
    
    The file format for the data is result_sensor.csv
    """
    
    def __init__(self, variable, folder=None):
        """
        Create a cache object specifically for the specified variable
        
        Arguments
        ---------
        variable : str
            The name of the aggregated variable to be stored.
            Examples: elec_standby, water_max, ...
        folder : path
            Path where the files are stored
            If None, use the path specified in the opengrid configuration
            
        """
        self.variable = variable
        if folder is None:
            try:
                self.folder = os.path.join(os.path.abspath(cfg.get('data', 'folder')),'cache_day')
            except:
                raise ValueError("Specify a folder, either in the opengrid.cfg or when creating this cache object.")
        else:
            self.folder = os.path.abspath(folder)
            
        if not os.path.exists(self.folder):
            print("This folder does not exist: {}, it will be created".format(self.folder))
            os.mkdir(self.folder)
            
        print("Cache object created for variable: {}".format(self.variable))

   
    def _load(self, sensorkey):
        """
        Return a dataframe with cached data for this sensor.  If there is no
        cached data, return an empty dataframe

        Arguments
        ---------
        sensor : str
            sensor key, unique string identifier of the sensor.

        Returns
        -------
        df : tz-aware dataframe with cached daily results or empty dataframe.
        
        """
        # Find the file and read into a dataframe
        filename = self.variable + '_' + sensorkey + '.csv'
        path = os.path.join(self.folder, filename)
        
        if not os.path.exists(path):
            #print("Could not find {}".format(path))
            return pd.DataFrame()
        
        df = pd.read_csv(path, index_col = 0, header=0, parse_dates=True, date_parser=dateutil.parser.parse)
        return df
    
    
    def _write_single(self, df):
        """
        Write the dataframe with single sensor to csv according to the filename conventions

        Arguments
        ---------
        df : pandas DataFrame or Series
        """

        # handle both cases correctly: dataframe or series
        if isinstance(df, pd.DataFrame):
            if len(df.columns) != 1:
                raise ValueError("Wrong number of columns")
            df_temp = df.copy()
        elif isinstance(df, pd.Series):
            if df.name is None:
                raise ValueError("pandas Series needs a name with sensor id")
            df_temp = pd.DataFrame(df)

        sensor = df_temp.columns[0]
        # Find the file and read into a dataframe
        filename = self.variable + '_' + sensor + '.csv'
        path = os.path.join(self.folder, filename)
        
        df_temp.to_csv(path)
        #print("Values for {} written to {}".format(sensor, path))
{'fraction_non_alphanumeric': 0.09211641706696808, 'fraction_numerical': 0.005086182537439954, 'mean_word_length': 1.8659919028340082, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2855841', 'n_tokens_mistral': 1867, 'n_tokens_neox': 1802, 'n_words': 664}