<commit_before># Copyright 2016 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0

import asyncio
import logging
import numbers
import itertools
import zlib
import pickle
import time
import datetime

import numpy as np
from functools import reduce

from auspex.log import logger

def cartesian(arrays, out=None, dtype='f'):
    """http://stackoverflow.com/questions/28684492/numpy-equivalent-of-itertools-product"""

    arrays = [np.asarray(x) for x in arrays]
    # dtype = arrays[0].dtype

    n = np.prod([x.size for x in arrays])
    if out is None:
        out = np.zeros([n, len(arrays)], dtype=dtype)

    m = int(n / arrays[0].size)
    out[:,0] = np.repeat(arrays[0], m)
    if arrays[1:]:
        cartesian(arrays[1:], out=out[0:m,1:])
        for j in range(1, arrays[0].size):
            out[j*m:(j+1)*m,1:] = out[0:m,1:]
    return out

class DataAxis(object):
    """An axis in a data stream"""
    def __init__(self, name, points=[], unit=None, metadata=None, dtype=np.float32):
        super(DataAxis, self).__init__()
        if isinstance(name, list):
            self.unstructured = True
            self.name = name
        else:
            self.unstructured = False
            self.name         = str(name)

        # self.points holds the CURRENT set of points. During adaptive sweeps
        # this will hold the most recently added points of the axis.
        self.points        = np.array(points)
        self.unit          = unit
        self.refine_func   = None
        self.metadata      = None
        self.metadata_enum = None
        if metadata is not None:
            self.set_metadata(metadata)

        # By definition data axes will be done after every experiment.run() call
        self.done         = True

        # For adaptive sweeps, etc., keep a record of the original points that we had around
        self.original_points = self.points
        self.has_been_extended = False
        self.num_new_points = 0
        self.dtype         = dtype

        if self.unstructured:
            if unit is not None and len(name) != len(unit):
                raise ValueError("DataAxis unit length {} and tuples length {} must match.".format(len(unit),len(name)))
        if self.unstructured and len(name) != len(points[0]):
            raise ValueError("DataAxis points length {} and names length {} must match.".format(len(points[0]), len(name)))

    def set_metadata(self, metadata):
        # Convert the metadata to an enum
        self.metadata_enum, self.metadata = np.unique(metadata, return_inverse=True)

    def data_type(self, with_metadata=False):
        dtype = []
        if self.unstructured:
            name = "+".join(self.name)
            dtype.extend([(p.name, 'f') for p in self.parameter])
        else:
            name = self.name
            dtype.append((name, 'f'))

        if with_metadata and self.metadata is not None:
            dtype.append((name + "_metadata", 'f'))
        return dtype

    def points_with_metadata(self):
        if self.metadata is not None:
            if self.unstructured:
                return [list(self.original_points[i]) + [self.metadata[i]] for i in range(len(self.original_points))]
            return [(self.original_points[i], self.metadata[i], ) for i in range(len(self.original_points))]
        if self.unstructured:
            return [tuple(self.original_points[i]) for i in range(len(self.original_points))]
        return [(self.original_points[i],) for i in range(len(self.original_points))]

    def tuple_width(self):
        if self.unstructured:
            width = len(name)
        else:
            width = 1
        if self.metadata:
            width += 1
        return width

    def num_points(self):
        if self.has_been_extended:
            return len(self.points)
        else:
            return len(self.original_points)

    def add_points(self, points):
        if self.unstructured and len(self.parameter) != len(points[0]):
            raise ValueError("Parameter value tuples must be the same length as the number of parameters.")

        if type(points) in [list, np.ndarray]:
            points = np.array(points)
        else:
            # Somebody gave one point to the "add_points" method...
            points = np.array([points])

        self.num_new_points = len(points)
        self.points = np.append(self.points, points, axis=0)
        self.has_been_extended = True

    def reset(self):
        self.points = self.original_points
        self.has_been_extended = False
        self.num_new_points = 0

    def __repr__(self):
        return "<DataAxis(name={}, start={}, stop={}, num={}, unit={})>".format(
            self.name, min(self.points), max(self.points), len(self.points), self.unit)

class SweepAxis(DataAxis):
    """ Structure for sweep axis, separate from DataAxis.
    Can be an unstructured axis, in which case 'parameter' is actually a list of parameters. """
    def __init__(self, parameter, points = [], metadata=None, refine_func=None, callback_func=None):

        self.unstructured = hasattr(parameter, '__iter__')
        self.parameter    = parameter
        if self.unstructured:
            unit = [p.unit for p in parameter]
            super(SweepAxis, self).__init__([p.name for p in parameter], points=points, unit=unit, metadata=metadata)
            self.value = points[0]
        else:
            super(SweepAxis, self).__init__(parameter.name, points, unit=parameter.unit, metadata=metadata)
            self.value     = points[0]

        # Current value of the metadata
        if self.metadata is not None:
            self.metadata_value = self.metadata[0]
<commit_msg>Fixed repr for DataAxis<commit_after># Copyright 2016 Raytheon BBN Technologies
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0

import asyncio
import logging
import numbers
import itertools
import zlib
import pickle
import time
import datetime

import numpy as np
from functools import reduce

from auspex.log import logger

def cartesian(arrays, out=None, dtype='f'):
    """http://stackoverflow.com/questions/28684492/numpy-equivalent-of-itertools-product"""

    arrays = [np.asarray(x) for x in arrays]
    # dtype = arrays[0].dtype

    n = np.prod([x.size for x in arrays])
    if out is None:
        out = np.zeros([n, len(arrays)], dtype=dtype)

    m = int(n / arrays[0].size)
    out[:,0] = np.repeat(arrays[0], m)
    if arrays[1:]:
        cartesian(arrays[1:], out=out[0:m,1:])
        for j in range(1, arrays[0].size):
            out[j*m:(j+1)*m,1:] = out[0:m,1:]
    return out

class DataAxis(object):
    """An axis in a data stream"""
    def __init__(self, name, points=[], unit=None, metadata=None, dtype=np.float32):
        super(DataAxis, self).__init__()
        if isinstance(name, list):
            self.unstructured = True
            self.name = name
        else:
            self.unstructured = False
            self.name         = str(name)

        # self.points holds the CURRENT set of points. During adaptive sweeps
        # this will hold the most recently added points of the axis.
        self.points        = np.array(points)
        self.unit          = unit
        self.refine_func   = None
        self.metadata      = None
        self.metadata_enum = None
        if metadata is not None:
            self.set_metadata(metadata)

        # By definition data axes will be done after every experiment.run() call
        self.done         = True

        # For adaptive sweeps, etc., keep a record of the original points that we had around
        self.original_points = self.points
        self.has_been_extended = False
        self.num_new_points = 0
        self.dtype         = dtype

        if self.unstructured:
            if unit is not None and len(name) != len(unit):
                raise ValueError("DataAxis unit length {} and tuples length {} must match.".format(len(unit),len(name)))
        if self.unstructured and len(name) != len(points[0]):
            raise ValueError("DataAxis points length {} and names length {} must match.".format(len(points[0]), len(name)))

    def set_metadata(self, metadata):
        # Convert the metadata to an enum
        self.metadata_enum, self.metadata = np.unique(metadata, return_inverse=True)

    def data_type(self, with_metadata=False):
        dtype = []
        if self.unstructured:
            name = "+".join(self.name)
            dtype.extend([(p.name, 'f') for p in self.parameter])
        else:
            name = self.name
            dtype.append((name, 'f'))

        if with_metadata and self.metadata is not None:
            dtype.append((name + "_metadata", 'f'))
        return dtype

    def points_with_metadata(self):
        if self.metadata is not None:
            if self.unstructured:
                return [list(self.original_points[i]) + [self.metadata[i]] for i in range(len(self.original_points))]
            return [(self.original_points[i], self.metadata[i], ) for i in range(len(self.original_points))]
        if self.unstructured:
            return [tuple(self.original_points[i]) for i in range(len(self.original_points))]
        return [(self.original_points[i],) for i in range(len(self.original_points))]

    def tuple_width(self):
        if self.unstructured:
            width = len(name)
        else:
            width = 1
        if self.metadata:
            width += 1
        return width

    def num_points(self):
        if self.has_been_extended:
            return len(self.points)
        else:
            return len(self.original_points)

    def add_points(self, points):
        if self.unstructured and len(self.parameter) != len(points[0]):
            raise ValueError("Parameter value tuples must be the same length as the number of parameters.")

        if type(points) in [list, np.ndarray]:
            points = np.array(points)
        else:
            # Somebody gave one point to the "add_points" method...
            points = np.array([points])

        self.num_new_points = len(points)
        self.points = np.append(self.points, points, axis=0)
        self.has_been_extended = True

    def reset(self):
        self.points = self.original_points
        self.has_been_extended = False
        self.num_new_points = 0

    def __repr__(self):
        return "<DataAxis(name={}, start={}, stop={}, num={}, unit={})>".format(
            self.name, self.points[0], self.points[-1], len(self.points), self.unit)

class SweepAxis(DataAxis):
    """ Structure for sweep axis, separate from DataAxis.
    Can be an unstructured axis, in which case 'parameter' is actually a list of parameters. """
    def __init__(self, parameter, points = [], metadata=None, refine_func=None, callback_func=None):

        self.unstructured = hasattr(parameter, '__iter__')
        self.parameter    = parameter
        if self.unstructured:
            unit = [p.unit for p in parameter]
            super(SweepAxis, self).__init__([p.name for p in parameter], points=points, unit=unit, metadata=metadata)
            self.value = points[0]
        else:
            super(SweepAxis, self).__init__(parameter.name, points, unit=parameter.unit, metadata=metadata)
            self.value     = points[0]

        # Current value of the metadata
        if self.metadata is not None:
            self.metadata_value = self.metadata[0]

{'fraction_non_alphanumeric': 0.1107611101651626, 'fraction_numerical': 0.0074919121403030815, 'mean_word_length': 2.4, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7420863', 'n_tokens_mistral': 3484, 'n_tokens_neox': 3351, 'n_words': 993}