<commit_before>/*-
 *******************************************************************************
 * Copyright (c) 2011, 2014 Diamond Light Source Ltd.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *    <NAME> - initial API and implementation and/or initial documentation
 *******************************************************************************/

package org.eclipse.dawnsci.analysis.dataset.impl;

import java.io.Serializable;
import java.lang.annotation.Annotation;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.eclipse.dawnsci.analysis.api.dataset.ILazyDataset;
import org.eclipse.dawnsci.analysis.api.dataset.Slice;
import org.eclipse.dawnsci.analysis.api.dataset.SliceND;
import org.eclipse.dawnsci.analysis.api.io.ILazyLoader;
import org.eclipse.dawnsci.analysis.api.metadata.MetadataType;
import org.eclipse.dawnsci.analysis.api.metadata.Reshapeable;
import org.eclipse.dawnsci.analysis.api.metadata.Sliceable;
import org.eclipse.dawnsci.analysis.api.metadata.Transposable;
import org.eclipse.dawnsci.analysis.api.monitor.IMonitor;
import org.eclipse.dawnsci.analysis.dataset.metadata.OriginMetadataImpl;

public class LazyDataset extends LazyDatasetBase implements Serializable, Cloneable {

	protected int[]     oShape; // original shape
	protected long      size;   // number of items
	protected int       dtype;
	protected int       isize; // number of elements per item

	protected ILazyLoader loader;
	protected LazyDataset base = null; // used for transpose

	// relative to loader or base
	protected int         prepShape = 0; // prepending and post-pending 
	protected int         postShape = 0; // changes to shape
	protected int[]       begSlice = null; // slice begin
	protected int[]       delSlice = null; // slice delta
	protected int[]       map; // transposition map (same length as current shape)
	protected Map<Class<? extends MetadataType>, List<MetadataType>> oMetadata = null;

	/**
	 * Create a lazy dataset
	 * @param name
	 * @param dtype dataset type
	 * @param elements
	 * @param shape
	 * @param loader
	 */
	public LazyDataset(String name, int dtype, int elements, int[] shape, ILazyLoader loader) {
		this.name = name;
		this.shape = shape.clone();
		this.oShape = this.shape;
		this.loader = loader;
		this.dtype = dtype;
		this.isize = elements;
		try {
			size = AbstractDataset.calcLongSize(shape);
		} catch (IllegalArgumentException e) {
			size = Long.MAX_VALUE; // this indicates that the entire dataset cannot be read in! 
		}
	}

	/**
	 * Create a lazy dataset
	 * @param name
	 * @param dtype dataset type
	 * @param shape
	 * @param loader
	 */
	public LazyDataset(String name, int dtype, int[] shape, ILazyLoader loader) {
		this(name, dtype, 1, shape, loader);
	}

	/**
	 * Create a lazy dataset based on in-memory data (handy for testing)
	 * @param dataset
	 */
	public static LazyDataset createLazyDataset(final Dataset dataset) {
		return new LazyDataset(dataset.getName(), dataset.getDtype(), dataset.getElementsPerItem(), dataset.getShape(),
		new ILazyLoader() {
			final Dataset d = dataset;
			@Override
			public boolean isFileReadable() {
				return true;
			}

			@Override
			public Dataset getDataset(IMonitor mon, SliceND slice)
					throws Exception {
				return d.getSlice(mon, slice);
			}
		});
	}

	/**
	 * Can return -1 for unknown
	 */
	@Override
	public int getDtype() {
		return dtype;
	}

	/**
	 * Can return -1 for unknown
	 */
	@Override
	public int getElementsPerItem() {
		return isize;
	}

	@Override
	public int getSize() {
		return (int) size;
	}

	@Override
	public String toString() {
		StringBuilder out = new StringBuilder();

		if (name != null && name.length() > 0) {
			out.append("Lazy dataset '");
			out.append(name);
			out.append("' has shape [");
		} else {
			out.append("Lazy dataset shape is [");
		}
		int rank = shape == null ? 0 : shape.length;

		if (rank > 0 && shape[0] >= 0) {
			out.append(shape[0]);
		}
		for (int i = 1; i < rank; i++) {
			out.append(", " + shape[i]);
		}
		out.append(']');

		return out.toString();
	}

	@Override
	public boolean equals(Object obj) {
		if (!super.equals(obj))
			return false;

		LazyDataset other = (LazyDataset) obj;
		if (dtype != other.dtype) {
			return false;
		}
		if (isize != other.isize) {
			return false;
		}

		if (!Arrays.equals(shape, other.shape)) {
			return false;
		}

		if (loader != other.loader) {
			return false;
		}

		if (prepShape != other.prepShape) {
			return false;
		}

		if (postShape != other.postShape) {
			return false;
		}

		if (!Arrays.equals(begSlice, other.begSlice)) {
			return false;
		}
		if (!Arrays.equals(delSlice, other.delSlice)) {
			return false;
		}
		if (!Arrays.equals(map, other.map)) {
			return false;
		}
		return true;
	}

	@Override
	public LazyDataset clone() {
		LazyDataset ret = new LazyDataset(new String(name), dtype, isize, oShape, loader);
		ret.shape = shape;
		ret.size = size;
		ret.prepShape = prepShape;
		ret.postShape = postShape;
		ret.begSlice = begSlice;
		ret.delSlice = delSlice;
		ret.map = map;
		ret.base = base;
		ret.metadata = copyMetadata();
		ret.oMetadata = oMetadata;
		return ret;
	}

	@Override
	public void setShape(int... shape) {
		setShapeInternal(false, shape);
	}

	@Override
	public LazyDataset squeezeEnds() {
		setShapeInternal(false, AbstractDataset.squeezeShape(shape, true));
		return this;
	}

	@Override
	public Dataset getSlice(int[] start, int[] stop, int[] step) {
		try {
			return getSlice(null, start, stop, step);
		} catch (Exception e) {
			logger.error("Problem slicing lazy dataset", e);
		}
		return null;
	}

	@Override
	public Dataset getSlice(Slice... slice) {
		try {
			if (slice == null || slice.length == 0) {
				return getSlice(null, new SliceND(shape));
			}
			return getSlice(null, new SliceND(shape, slice));
		} catch (Exception e) {
			logger.error("Problem slicing lazy dataset", e);
		}
		return null;
	}

	@Override
	public Dataset getSlice(SliceND slice) {
		try {
			return getSlice(null, slice);
		} catch (Exception e) {
			logger.error("Problem slicing lazy dataset", e);
		}
		return null;
	}

	@Override
	public Dataset getSlice(IMonitor monitor, Slice... slice) throws Exception {
		if (slice == null || slice.length == 0) {
			return getSlice(monitor, new SliceND(shape));
		}
		return getSlice(monitor, new SliceND(shape, slice));
	}

	@Override
	public LazyDataset getSliceView(Slice... slice) {
		if (slice == null || slice.length == 0) {
			return getSliceView(new SliceND(shape));
		}
		return getSliceView(new SliceND(shape, slice));
	}

	/**
	 * 
	 * @param newSize - Used by remote dataset to reshape the data shape when the remote data changes.
	 * @param nShape
	 */
	protected void setShapeInternal(boolean newSize, int... nShape) {
		
		long nsize = AbstractDataset.calcLongSize(nShape);
		if (newSize) {
			size = nsize;
			shape = nShape;
		} else {
			if (nsize != size) {
				throw new IllegalArgumentException("Size of new shape is not equal to current size");
			}
		}
		if (nsize == 1) {
			shape = nShape.clone();
			return;
		}

		int ob = -1; // first non-unit dimension
		int or = shape.length;
		for (int i = 0; i < or; i++) {
			if (shape[i] != 1) {
				ob = i;
				break;
			}
		}
		assert ob >= 0;
		int oe = -1; // last non-unit dimension
		for (int i = or - 1; i >= ob; i--) {
			if (shape[i] != 1) {
				oe = i;
				break;
			}
		}
		assert oe >= 0;
		oe++;

		int nb = -1; // first non-unit dimension
		int nr = nShape.length;
		for (int i = 0; i < nr; i++) {
			if (nShape[i] != 1) {
				nb = i;
				break;
			}
		}

		int i = ob;
		int j = nb;
		if (begSlice == null) {
			for (; i < oe && j < nr; i++, j++) {
				if (shape[i] != nShape[j]) {
					throw new IllegalArgumentException("New shape not allowed - can only change shape by adding or removing ones to ends of old shape");
				}
			}
		} else {
			int[] nBegSlice = new int[nr];
			int[] nDelSlice = new int[nr];
			Arrays.fill(nDelSlice, 1);
			for (; i < oe && j < nr; i++, j++) {
				if (shape[i] != nShape[j]) {
					throw new IllegalArgumentException("New shape not allowed - can only change shape by adding or removing ones to ends of old shape");
				}
				nBegSlice[j] = begSlice[i];
				nDelSlice[j] = delSlice[i];
			}
	
			begSlice = nBegSlice;
			delSlice = nDelSlice;
		}
		prepShape += nb - ob;
		postShape += nr - oe;

		storeMetadata(metadata, Reshapeable.class);
		metadata = copyMetadata();
		reshapeMetadata(shape, nShape);
		shape = nShape;
	}

	@Override
	public LazyDataset getSliceView(int[] start, int[] stop, int[] step) {
		return getSliceView(new SliceND(shape, start, stop, step));
	}

	@Override
	public LazyDataset getSliceView(SliceND slice) {
		LazyDataset view = clone();
		if (slice.isAll())
			return view;

		int[] lstart = slice.getStart();
		int[] lstep  = slice.getStep();
		final int rank = shape.length;

		int[] nShape = slice.getShape();
		view.shape = nShape;
		view.size = AbstractDataset.calcLongSize(nShape);
		if (begSlice == null) {
			view.begSlice = lstart.clone();
			view.delSlice = lstep.clone();
		} else {
			view.begSlice = new int[rank];
			view.delSlice = new int[rank];
			for (int i = 0; i < rank; i++) {
				view.begSlice[i] = begSlice[i] + lstart[i] * delSlice[i];
				view.delSlice[i] = delSlice[i] * lstep[i];
			}
		}
		view.storeMetadata(metadata, Sliceable.class);
		
		view.sliceMetadata(true, slice);
		return view;
	}

	@Override
	public LazyDataset getTransposedView(int... axes) {
		LazyDataset view = clone();

		// everything now is seen through a map
		axes = checkPermutatedAxes(shape, axes);
		if (axes == null)
			return view;

		int r = shape.length;
		view.shape = new int[r];
		for (int i = 0; i < r; i++) {
			view.shape[i] = shape[axes[i]];
		}

		view.prepShape = 0;
		view.postShape = 0;
		view.begSlice = null;
		view.delSlice = null;
		view.map = axes;
		view.base = this;
		view.storeMetadata(metadata, Transposable.class);
		view.transposeMetadata(axes);
		return view;
	}

	@Override
	public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) throws Exception {
		return getSlice(monitor, new SliceND(shape, start, stop, step));
	}

	@Override
	public Dataset getSlice(IMonitor monitor, SliceND slice) throws Exception {

		if (loader != null && !loader.isFileReadable())
			return null; // TODO add interaction to use plot (or remote) server to load dataset

		SliceND nslice = calcTrueSlice(slice);

		Dataset a;
		if (base != null) {
			a = base.getSlice(monitor, nslice);
		} else {
			try {
				a = DatasetUtils.convertToDataset(loader.getDataset(monitor, nslice));
			} catch (Exception e) {
				// return a fake dataset to show that this has not worked, should not be used in general though.
				logger.debug("Problem getting {}: {}", String.format("slice %s %s %s", Arrays.toString(slice.getStart()), Arrays.toString(slice.getStop()),
								Arrays.toString(slice.getStep())), e);
				a = new DoubleDataset(1);
			}
			a.setName(name + AbstractDataset.BLOCK_OPEN + nslice.toString() + AbstractDataset.BLOCK_CLOSE);
			if (metadata != null && a instanceof LazyDatasetBase) {
				LazyDatasetBase ba = (LazyDatasetBase) a;
				ba.metadata = copyMetadata();
				if (oMetadata != null)
					ba.restoreMetadata(oMetadata);
				if (!nslice.isAll())
					ba.sliceMetadata(true, nslice);
			}
		}
		if (map != null) {
			a = a.getTransposedView(map);
		}
		a.setShape(slice.getShape());
		a.addMetadata(new OriginMetadataImpl(this, nslice.convertToSlice(), oShape, null, null));
		
		return a;
	}

	// reverse transform
	private int[] getOriginal(int[] values) {
		if (values == null)
			return null;
		int r = values.length;
		if (map == null || r < 2)
			return values;
		int[] ovalues = new int[r];
		for (int i = 0; i < r; i++) {
			ovalues[map[i]] = values[i];
		}
		return ovalues;
	}

	protected SliceND calcTrueSlice(SliceND slice) {
		int[] lstart = slice.getStart();
		int[] lstop  = slice.getStop();
		int[] lstep  = slice.getStep();

		int[] nstart;
		int[] nstop;
		int[] nstep;

		int r = base == null ? oShape.length : base.shape.length;
		nstart = new int[r];
		nstop = new int[r];
		nstep = new int[r];
		Arrays.fill(nstop, 1);
		Arrays.fill(nstep, 1);
		{
			int i = 0;
			int j = 0;
			if (prepShape < 0) { // ignore entries from new slice 
				i = -prepShape;<commit_msg>Allow null ND slices<commit_after>/*-
 *******************************************************************************
 * Copyright (c) 2011, 2014 Diamond Light Source Ltd.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *    <NAME> - initial API and implementation and/or initial documentation
 *******************************************************************************/

package org.eclipse.dawnsci.analysis.dataset.impl;

import java.io.Serializable;
import java.lang.annotation.Annotation;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.eclipse.dawnsci.analysis.api.dataset.ILazyDataset;
import org.eclipse.dawnsci.analysis.api.dataset.Slice;
import org.eclipse.dawnsci.analysis.api.dataset.SliceND;
import org.eclipse.dawnsci.analysis.api.io.ILazyLoader;
import org.eclipse.dawnsci.analysis.api.metadata.MetadataType;
import org.eclipse.dawnsci.analysis.api.metadata.Reshapeable;
import org.eclipse.dawnsci.analysis.api.metadata.Sliceable;
import org.eclipse.dawnsci.analysis.api.metadata.Transposable;
import org.eclipse.dawnsci.analysis.api.monitor.IMonitor;
import org.eclipse.dawnsci.analysis.dataset.metadata.OriginMetadataImpl;

public class LazyDataset extends LazyDatasetBase implements Serializable, Cloneable {

	protected int[]     oShape; // original shape
	protected long      size;   // number of items
	protected int       dtype;
	protected int       isize; // number of elements per item

	protected ILazyLoader loader;
	protected LazyDataset base = null; // used for transpose

	// relative to loader or base
	protected int         prepShape = 0; // prepending and post-pending 
	protected int         postShape = 0; // changes to shape
	protected int[]       begSlice = null; // slice begin
	protected int[]       delSlice = null; // slice delta
	protected int[]       map; // transposition map (same length as current shape)
	protected Map<Class<? extends MetadataType>, List<MetadataType>> oMetadata = null;

	/**
	 * Create a lazy dataset
	 * @param name
	 * @param dtype dataset type
	 * @param elements
	 * @param shape
	 * @param loader
	 */
	public LazyDataset(String name, int dtype, int elements, int[] shape, ILazyLoader loader) {
		this.name = name;
		this.shape = shape.clone();
		this.oShape = this.shape;
		this.loader = loader;
		this.dtype = dtype;
		this.isize = elements;
		try {
			size = AbstractDataset.calcLongSize(shape);
		} catch (IllegalArgumentException e) {
			size = Long.MAX_VALUE; // this indicates that the entire dataset cannot be read in! 
		}
	}

	/**
	 * Create a lazy dataset
	 * @param name
	 * @param dtype dataset type
	 * @param shape
	 * @param loader
	 */
	public LazyDataset(String name, int dtype, int[] shape, ILazyLoader loader) {
		this(name, dtype, 1, shape, loader);
	}

	/**
	 * Create a lazy dataset based on in-memory data (handy for testing)
	 * @param dataset
	 */
	public static LazyDataset createLazyDataset(final Dataset dataset) {
		return new LazyDataset(dataset.getName(), dataset.getDtype(), dataset.getElementsPerItem(), dataset.getShape(),
		new ILazyLoader() {
			final Dataset d = dataset;
			@Override
			public boolean isFileReadable() {
				return true;
			}

			@Override
			public Dataset getDataset(IMonitor mon, SliceND slice)
					throws Exception {
				return d.getSlice(mon, slice);
			}
		});
	}

	/**
	 * Can return -1 for unknown
	 */
	@Override
	public int getDtype() {
		return dtype;
	}

	/**
	 * Can return -1 for unknown
	 */
	@Override
	public int getElementsPerItem() {
		return isize;
	}

	@Override
	public int getSize() {
		return (int) size;
	}

	@Override
	public String toString() {
		StringBuilder out = new StringBuilder();

		if (name != null && name.length() > 0) {
			out.append("Lazy dataset '");
			out.append(name);
			out.append("' has shape [");
		} else {
			out.append("Lazy dataset shape is [");
		}
		int rank = shape == null ? 0 : shape.length;

		if (rank > 0 && shape[0] >= 0) {
			out.append(shape[0]);
		}
		for (int i = 1; i < rank; i++) {
			out.append(", " + shape[i]);
		}
		out.append(']');

		return out.toString();
	}

	@Override
	public boolean equals(Object obj) {
		if (!super.equals(obj))
			return false;

		LazyDataset other = (LazyDataset) obj;
		if (dtype != other.dtype) {
			return false;
		}
		if (isize != other.isize) {
			return false;
		}

		if (!Arrays.equals(shape, other.shape)) {
			return false;
		}

		if (loader != other.loader) {
			return false;
		}

		if (prepShape != other.prepShape) {
			return false;
		}

		if (postShape != other.postShape) {
			return false;
		}

		if (!Arrays.equals(begSlice, other.begSlice)) {
			return false;
		}
		if (!Arrays.equals(delSlice, other.delSlice)) {
			return false;
		}
		if (!Arrays.equals(map, other.map)) {
			return false;
		}
		return true;
	}

	@Override
	public LazyDataset clone() {
		LazyDataset ret = new LazyDataset(new String(name), dtype, isize, oShape, loader);
		ret.shape = shape;
		ret.size = size;
		ret.prepShape = prepShape;
		ret.postShape = postShape;
		ret.begSlice = begSlice;
		ret.delSlice = delSlice;
		ret.map = map;
		ret.base = base;
		ret.metadata = copyMetadata();
		ret.oMetadata = oMetadata;
		return ret;
	}

	@Override
	public void setShape(int... shape) {
		setShapeInternal(false, shape);
	}

	@Override
	public LazyDataset squeezeEnds() {
		setShapeInternal(false, AbstractDataset.squeezeShape(shape, true));
		return this;
	}

	@Override
	public Dataset getSlice(int[] start, int[] stop, int[] step) {
		try {
			return getSlice(null, start, stop, step);
		} catch (Exception e) {
			logger.error("Problem slicing lazy dataset", e);
		}
		return null;
	}

	@Override
	public Dataset getSlice(Slice... slice) {
		try {
			if (slice == null || slice.length == 0) {
				return getSlice(null, new SliceND(shape));
			}
			return getSlice(null, new SliceND(shape, slice));
		} catch (Exception e) {
			logger.error("Problem slicing lazy dataset", e);
		}
		return null;
	}

	@Override
	public Dataset getSlice(SliceND slice) {
		try {
			return getSlice(null, slice);
		} catch (Exception e) {
			logger.error("Problem slicing lazy dataset", e);
		}
		return null;
	}

	@Override
	public Dataset getSlice(IMonitor monitor, Slice... slice) throws Exception {
		if (slice == null || slice.length == 0) {
			return getSlice(monitor, new SliceND(shape));
		}
		return getSlice(monitor, new SliceND(shape, slice));
	}

	@Override
	public LazyDataset getSliceView(Slice... slice) {
		if (slice == null || slice.length == 0) {
			return getSliceView(new SliceND(shape));
		}
		return getSliceView(new SliceND(shape, slice));
	}

	/**
	 * 
	 * @param newSize - Used by remote dataset to reshape the data shape when the remote data changes.
	 * @param nShape
	 */
	protected void setShapeInternal(boolean newSize, int... nShape) {
		
		long nsize = AbstractDataset.calcLongSize(nShape);
		if (newSize) {
			size = nsize;
			shape = nShape;
		} else {
			if (nsize != size) {
				throw new IllegalArgumentException("Size of new shape is not equal to current size");
			}
		}
		if (nsize == 1) {
			shape = nShape.clone();
			return;
		}

		int ob = -1; // first non-unit dimension
		int or = shape.length;
		for (int i = 0; i < or; i++) {
			if (shape[i] != 1) {
				ob = i;
				break;
			}
		}
		assert ob >= 0;
		int oe = -1; // last non-unit dimension
		for (int i = or - 1; i >= ob; i--) {
			if (shape[i] != 1) {
				oe = i;
				break;
			}
		}
		assert oe >= 0;
		oe++;

		int nb = -1; // first non-unit dimension
		int nr = nShape.length;
		for (int i = 0; i < nr; i++) {
			if (nShape[i] != 1) {
				nb = i;
				break;
			}
		}

		int i = ob;
		int j = nb;
		if (begSlice == null) {
			for (; i < oe && j < nr; i++, j++) {
				if (shape[i] != nShape[j]) {
					throw new IllegalArgumentException("New shape not allowed - can only change shape by adding or removing ones to ends of old shape");
				}
			}
		} else {
			int[] nBegSlice = new int[nr];
			int[] nDelSlice = new int[nr];
			Arrays.fill(nDelSlice, 1);
			for (; i < oe && j < nr; i++, j++) {
				if (shape[i] != nShape[j]) {
					throw new IllegalArgumentException("New shape not allowed - can only change shape by adding or removing ones to ends of old shape");
				}
				nBegSlice[j] = begSlice[i];
				nDelSlice[j] = delSlice[i];
			}
	
			begSlice = nBegSlice;
			delSlice = nDelSlice;
		}
		prepShape += nb - ob;
		postShape += nr - oe;

		storeMetadata(metadata, Reshapeable.class);
		metadata = copyMetadata();
		reshapeMetadata(shape, nShape);
		shape = nShape;
	}

	@Override
	public LazyDataset getSliceView(int[] start, int[] stop, int[] step) {
		return getSliceView(new SliceND(shape, start, stop, step));
	}

	@Override
	public LazyDataset getSliceView(SliceND slice) {
		LazyDataset view = clone();
		if (slice.isAll())
			return view;

		int[] lstart = slice.getStart();
		int[] lstep  = slice.getStep();
		final int rank = shape.length;

		int[] nShape = slice.getShape();
		view.shape = nShape;
		view.size = AbstractDataset.calcLongSize(nShape);
		if (begSlice == null) {
			view.begSlice = lstart.clone();
			view.delSlice = lstep.clone();
		} else {
			view.begSlice = new int[rank];
			view.delSlice = new int[rank];
			for (int i = 0; i < rank; i++) {
				view.begSlice[i] = begSlice[i] + lstart[i] * delSlice[i];
				view.delSlice[i] = delSlice[i] * lstep[i];
			}
		}
		view.storeMetadata(metadata, Sliceable.class);
		
		view.sliceMetadata(true, slice);
		return view;
	}

	@Override
	public LazyDataset getTransposedView(int... axes) {
		LazyDataset view = clone();

		// everything now is seen through a map
		axes = checkPermutatedAxes(shape, axes);
		if (axes == null)
			return view;

		int r = shape.length;
		view.shape = new int[r];
		for (int i = 0; i < r; i++) {
			view.shape[i] = shape[axes[i]];
		}

		view.prepShape = 0;
		view.postShape = 0;
		view.begSlice = null;
		view.delSlice = null;
		view.map = axes;
		view.base = this;
		view.storeMetadata(metadata, Transposable.class);
		view.transposeMetadata(axes);
		return view;
	}

	@Override
	public Dataset getSlice(IMonitor monitor, int[] start, int[] stop, int[] step) throws Exception {
		return getSlice(monitor, new SliceND(shape, start, stop, step));
	}

	@Override
	public Dataset getSlice(IMonitor monitor, SliceND slice) throws Exception {

		if (loader != null && !loader.isFileReadable())
			return null; // TODO add interaction to use plot (or remote) server to load dataset

		SliceND nslice = calcTrueSlice(slice);

		Dataset a;
		if (base != null) {
			a = base.getSlice(monitor, nslice);
		} else {
			try {
				a = DatasetUtils.convertToDataset(loader.getDataset(monitor, nslice));
			} catch (Exception e) {
				// return a fake dataset to show that this has not worked, should not be used in general though.
				logger.debug("Problem getting {}: {}", String.format("slice %s %s %s", Arrays.toString(slice.getStart()), Arrays.toString(slice.getStop()),
								Arrays.toString(slice.getStep())), e);
				a = new DoubleDataset(1);
			}
			a.setName(name + AbstractDataset.BLOCK_OPEN + nslice.toString() + AbstractDataset.BLOCK_CLOSE);
			if (metadata != null && a instanceof LazyDatasetBase) {
				LazyDatasetBase ba = (LazyDatasetBase) a;
				ba.metadata = copyMetadata();
				if (oMetadata != null)
					ba.restoreMetadata(oMetadata);
				if (!nslice.isAll())
					ba.sliceMetadata(true, nslice);
			}
		}
		if (map != null) {
			a = a.getTransposedView(map);
		}
		if (slice != null)
			a.setShape(slice.getShape());
		a.addMetadata(new OriginMetadataImpl(this, nslice.convertToSlice(), oShape, null, null));
		
		return a;
	}

	// reverse transform
	private int[] getOriginal(int[] values) {
		if (values == null)
			return null;
		int r = values.length;
		if (map == null || r < 2)
			return values;
		int[] ovalues = new int[r];
		for (int i = 0; i < r; i++) {
			ovalues[map[i]] = values[i];
		}
		return ovalues;
	}

	protected SliceND calcTrueSlice(SliceND slice) {
		if (slice == null) {
			slice = new SliceND(shape);
		}
		int[] lstart = slice.getStart();
		int[] lstop  = slice.getStop();
		int[] lstep  = slice.getStep();

		int[] nstart;
		int[] nstop;
		int[] nstep;

		int r = base == null ? oShape.length : base.shape.length;
		nstart = new int[r];
		nstop = new int[r];
		nstep = new int[r];
		Arrays.fill(nstop, 1);
		Arrays.fill(nstep, 1);
		{
			int i = 0;
			int j = 0;
			if (prepShape < 0) { // ignore entries from new slice 
				i = -prepShape;
{'fraction_non_alphanumeric': 0.14498009381528637, 'fraction_numerical': 0.004099491505380582, 'mean_word_length': 3.316093909493025, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6992807', 'n_tokens_mistral': 9581, 'n_tokens_neox': 8415, 'n_words': 2405}