<commit_before>'''
Copyright (c) 2014, <NAME>
All rights reserved.
License BSD
'''
import logging
import datetime
from iotrelay import Reading
from pywws import DataStore

__version__ = "1.0.0"

logger = logging.getLogger(__name__)

# hours to look back for readings
DEFAULT_LOOKBACK = 2
READING_TYPE = 'weather'


class Poll(object):
    def __init__(self, config):
        self.keys = [k.strip() for k in config['series keys'].split(',')]
        self.data_store = config['data store']
        poll_frequency = int(config['poll frequency'])
        self.delta = datetime.timedelta(seconds=poll_frequency)
        lookback = int(config.get('lookback', DEFAULT_LOOKBACK))
        hours = datetime.timedelta(hours=lookback)
        self.last_ts = datetime.datetime.utcnow() - hours

    def get_readings(self):
        current_timestamp = datetime.datetime.utcnow()
        if current_timestamp < self.last_ts + self.delta:
            logger.debug('pywws {0!s} until next reading'.format(
                         self.last_ts + self.delta - current_timestamp))
            return ()
        readings = DataStore.data_store(self.data_store)
        for reading in readings[self.last_ts:]:
            for key in self.keys:
                yield Reading(READING_TYPE, reading[key], reading['idx'], key)
        self.last_ts = current_timestamp
<commit_msg>use last reading timestamp as the early endpoint when fetching new readings<commit_after>'''
Copyright (c) 2014, <NAME>
All rights reserved.
License BSD
'''
import logging
import datetime
from iotrelay import Reading
from pywws import DataStore

__version__ = "1.0.0"

logger = logging.getLogger(__name__)

# hours to look back for readings
DEFAULT_LOOKBACK = 2
READING_TYPE = 'weather'


class Poll(object):
    def __init__(self, config):
        self.keys = [k.strip() for k in config['series keys'].split(',')]
        self.data_store = config['data store']
        lookback = int(config.get('lookback', DEFAULT_LOOKBACK))
        hours = datetime.timedelta(hours=lookback)
        self.last_ts = datetime.datetime.utcnow() - hours

    def get_readings(self):
        for data in DataStore.data_store(self.data_store)[self.last_ts:]:
            if data['idx'] <= self.last_ts:
                continue
            self.last_ts = data['idx']
            for key in self.keys:
                yield Reading(READING_TYPE, data[key], data['idx'], key)

{'fraction_non_alphanumeric': 0.11083437110834371, 'fraction_numerical': 0.007056870070568701, 'mean_word_length': 3.064080944350759, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5785687', 'n_tokens_mistral': 734, 'n_tokens_neox': 707, 'n_words': 189}