<commit_before>import tempoiq.session
from os import path
import getopt
import sys
import itertools
from tempoiq.protocol.device import Device
from tempoiq.protocol.sensor import Sensor
from tempoiq.protocol.encoder import CreateEncoder
import datetime
import json

DEFAULT_START = datetime.datetime(2000, 1, 1)
DEFAULT_END = datetime.datetime(2018, 1, 1)
DEFAULT_FILTERS = []
# Add device filters like:
#DEFAULT_FILTERS.append(Device.attributes['region'] == 'north')

class Exporter(object):
    def __init__(self, credentials, path='.'):
        cli = tempoiq.session.get_session(credentials['host'],
                                          credentials['key'],
                                          credentials['secret'])
        self.client = cli
        self.devices = []
        self.path = path

    def export_devices(self):
        req = self.client.query(Device)
        for filt in DEFAULT_FILTERS:
            req = req.filter(filt)
        response = req.read()

        if response.successful != tempoiq.response.SUCCESS:
            print("Error reading devices {}-{}"
                  .format(response.status, response.reason))
            return

        for device in response.data:
            self.devices.append(device)

        with open(self.file_in_path('devices.json'), 'w') as outfile:
            json.dump(self.devices, outfile, default=CreateEncoder().default)

    def export_datapoints(self, start, end):
        if not self.devices:
            print("Can't export data without devices!")
            return

        outfile = open(self.file_in_path('datapoints.tsv'), 'w')
        for device in self.devices:
            for (dev, sen, ts, val) in self._read_device(start, end, device):
                outfile.write('{}\t{}\t{}\t{}\n'
                              .format(dev, sen, ts.isoformat(), val))
        outfile.close()

    def file_in_path(self, filename):
        return path.join(self.path, filename)

    def _read_device(self, start, end, device):
        res = self.client.query(Sensor) \
                         .filter(Device.key == device.key) \
                         .read(start=start, end=end)
        if res.successful != tempoiq.response.SUCCESS:
            print("Error reading data from device {}: {}-{}"
                  .format(device.key, res.status, res.reason))
            return

        for row in res.data:
            for ((device, sensor), value) in row:
                yield (device, sensor, row.timestamp, value)


def main(argv):
    creds = {}
    try:
        opts, args = getopt.getopt(argv, "n:k:s:")
    except getopt.GetoptError:
        print('pull_data.py -n <backend> -k <key> -s <secret>')
        sys.exit(2)
    for opt, arg in opts:
        if opt == "-k":
            creds['key'] = arg
        elif opt == "-s":
            creds['secret'] = arg
        elif opt == "-n":
            creds['host'] = arg

    if not creds['host'].startswith('http'):
        creds['host'] = 'https://' + creds['host']

    exporter = Exporter(creds)
    exporter.export_devices()
    exporter.export_datapoints(start=DEFAULT_START, end=DEFAULT_END)

if __name__ == "__main__":
    main(sys.argv[1:])<commit_msg>Resume-able exports<commit_after>import tempoiq.session
from os import path
import getopt
import sys
import itertools
from tempoiq.protocol.device import Device
from tempoiq.protocol.sensor import Sensor
from tempoiq.protocol.encoder import CreateEncoder
import datetime
import json

DEFAULT_START = datetime.datetime(2000, 1, 1)
DEFAULT_END = datetime.datetime(2018, 1, 1)
DEFAULT_FILTERS = []
# Add device filters like:
#DEFAULT_FILTERS.append(Device.attributes['region'] == 'north')

class Exporter(object):
    def __init__(self, opts, path='.'):
        cli = tempoiq.session.get_session(opts['host'],
                                          opts['key'],
                                          opts['secret'])
        self.client = cli
        self.devices = []
        self.path = path
        if 'resume_device' in opts:
            self.resume_device = opts['resume_device']
        else:
            self.resume_device = None

    def export(self):
        if self.resume_device is None:
            self.export_devices()

        self.export_datapoints(start=DEFAULT_START, end=DEFAULT_END)

    def export_devices(self):
        req = self.client.query(Device)
        for filt in DEFAULT_FILTERS:
            req = req.filter(filt)
        response = req.read()

        if response.successful != tempoiq.response.SUCCESS:
            print("Error reading devices {}-{}"
                  .format(response.status, response.reason))
            return

        for device in response.data:
            self.devices.append(device)

        with open(self.file_in_path('devices.json'), 'w') as outfile:
            json.dump(self.devices, outfile, default=CreateEncoder().default)

    def export_datapoints(self, start, end):
        if not self.devices:
            with open(self.file_in_path('devices.json')) as infile:
                self.devices = json.load(infile)

        outfile = open(self.file_in_path('datapoints.tsv'), 'a')

        started = (self.resume_device is None)
        if started:
            print("Exporting data from {} devices".format(len(self.devices)))

        for device in self.devices:
            try:
                deviceKey = device.key
            except AttributeError:
                deviceKey = device["key"]

            if not started:
                if deviceKey == self.resume_device:
                    print("resuming with device {}".format(deviceKey))
                    started = True
                else:
                    continue

            for (dev, sen, ts, val) in self._read_device(start, end, deviceKey):
                outfile.write('{}\t{}\t{}\t{}\n'
                              .format(dev, sen, ts.isoformat(), val))
        outfile.close()

    def file_in_path(self, filename):
        return path.join(self.path, filename)

    def _read_device(self, start, end, deviceKey):
        res = self.client.query(Sensor) \
                         .filter(Device.key == deviceKey) \
                         .read(start=start, end=end)
        if res.successful != tempoiq.response.SUCCESS:
            print("Error reading data from device {}: {}-{}"
                  .format(deviceKey, res.status, res.reason))
            return

        for row in res.data:
            for ((device, sensor), value) in row:
                yield (device, sensor, row.timestamp, value)


def main(argv):
    creds = {}
    try:
        opts, args = getopt.getopt(argv, "n:k:s:r:")
    except getopt.GetoptError:
        print('export.py -n <backend> -k <key> -s <secret> -r <device>')
        sys.exit(2)
    for opt, arg in opts:
        if opt == "-k":
            creds['key'] = arg
        elif opt == "-s":
            creds['secret'] = arg
        elif opt == "-n":
            creds['host'] = arg
        elif opt == "-r":
            creds['resume_device'] = arg

    if not creds['host'].startswith('http'):
        creds['host'] = 'https://' + creds['host']

    exporter = Exporter(creds)
    exporter.export()

if __name__ == "__main__":
    main(sys.argv[1:])
{'fraction_non_alphanumeric': 0.12425790418334944, 'fraction_numerical': 0.0038658014634819827, 'mean_word_length': 2.015820149875104, 'pattern_counts': {'":': 9, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4590171', 'n_tokens_mistral': 2110, 'n_tokens_neox': 2042, 'n_words': 502}