<commit_before>/*-
 * Copyright 2015 Diamond Light Source Ltd.
 *
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 */

package org.eclipse.dawnsci.analysis.dataset.slicer;

import java.util.Arrays;

import org.eclipse.january.DatasetException;
import org.eclipse.january.dataset.Dataset;
import org.eclipse.january.dataset.DatasetFactory;
import org.eclipse.january.dataset.DatasetUtils;
import org.eclipse.january.dataset.IDataset;
import org.eclipse.january.dataset.IDynamicDataset;
import org.eclipse.january.dataset.ILazyDataset;
import org.eclipse.january.dataset.LongDataset;
import org.eclipse.january.dataset.SliceND;
import org.eclipse.january.metadata.AxesMetadata;
import org.eclipse.january.metadata.DynamicMetadataUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class DynamicSliceViewIterator implements ISliceViewIterator {

	private static final Logger logger = LoggerFactory.getLogger(DynamicSliceViewIterator.class);
	
	private DynamicSliceNDIterator iterator = null;
	
	private IDynamicDataset lazy;
	private IDynamicDataset[] keys;
	private IDynamicDataset finished;
	private int[] axes;
	private SourceInformation source;
	private boolean next = false;
	
	private boolean last = false;
	
	private int maxTimeout = 60000;
	private int timeout = 1000;
	
	public DynamicSliceViewIterator(IDynamicDataset lazy, IDynamicDataset[] keys, IDynamicDataset finished, int dataSize) {
		this(lazy, keys, finished, dataSize, false);
	}
	
	public DynamicSliceViewIterator(IDynamicDataset lazy, IDynamicDataset[] keys, IDynamicDataset finished, int dataSize, boolean repeating) {
		try {
			iterator = new DynamicSliceNDIterator(lazy.getShape(), mergeKeys(keys), lazy.getRank()-dataSize);
			if (repeating) iterator.enableRepeating();
		} catch (DatasetException e) {
			logger.error("Could not get data from lazy dataset", e);
		}
		this.lazy = lazy;
		this.keys = keys;
		this.finished = finished;
		
		int lr = lazy.getRank();
		
		this.axes = new int[dataSize];
		
		for (int i = 0; i < axes.length; i++) {
			axes[i] = lr - 1 -i;
		}
		updateShape();
		next = iterator.hasNext();
		
		Object ssm = lazy.getFirstMetadata(SliceFromSeriesMetadata.class);
		if (ssm != null && ssm instanceof SliceFromSeriesMetadata && ((SliceFromSeriesMetadata)ssm).getSourceInfo() != null) source = ((SliceFromSeriesMetadata)ssm).getSourceInfo();
		else logger.warn("Lazy dataset contains no source information");

	}
	
	public void setMaxTimeout(int maxTimeout) {
		this.maxTimeout = maxTimeout;
		timeout = maxTimeout/50;
	}

	public void updateShape() {
		try {
			lazy.refreshShape();
			int[] current = lazy.getShape();
			for (IDynamicDataset k : keys) k.refreshShape();
			int[] s = DynamicMetadataUtils.refreshDynamicAxesMetadata(lazy.getMetadata(AxesMetadata.class), lazy.getShape());
			for (int i = 0; i < axes.length;i++) {
				s[axes[i]] = current[axes[i]];
			}
			lazy.resize(s);
			iterator.updateShape(lazy.getShape(), mergeKeys(keys));
		} catch (Exception e) {
			logger.error("Error refreshing axes",e);
		}
		
	}
	<commit_msg>Log in update shape<commit_after>/*-
 * Copyright 2015 Diamond Light Source Ltd.
 *
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 */

package org.eclipse.dawnsci.analysis.dataset.slicer;

import java.util.Arrays;

import org.eclipse.january.DatasetException;
import org.eclipse.january.dataset.Dataset;
import org.eclipse.january.dataset.DatasetFactory;
import org.eclipse.january.dataset.DatasetUtils;
import org.eclipse.january.dataset.IDataset;
import org.eclipse.january.dataset.IDynamicDataset;
import org.eclipse.january.dataset.ILazyDataset;
import org.eclipse.january.dataset.LongDataset;
import org.eclipse.january.dataset.SliceND;
import org.eclipse.january.metadata.AxesMetadata;
import org.eclipse.january.metadata.DynamicMetadataUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class DynamicSliceViewIterator implements ISliceViewIterator {

	private static final Logger logger = LoggerFactory.getLogger(DynamicSliceViewIterator.class);
	
	private DynamicSliceNDIterator iterator = null;
	
	private IDynamicDataset lazy;
	private IDynamicDataset[] keys;
	private IDynamicDataset finished;
	private int[] axes;
	private SourceInformation source;
	private boolean next = false;
	
	private boolean last = false;
	
	private int maxTimeout = 60000;
	private int timeout = 1000;
	
	public DynamicSliceViewIterator(IDynamicDataset lazy, IDynamicDataset[] keys, IDynamicDataset finished, int dataSize) {
		this(lazy, keys, finished, dataSize, false);
	}
	
	public DynamicSliceViewIterator(IDynamicDataset lazy, IDynamicDataset[] keys, IDynamicDataset finished, int dataSize, boolean repeating) {
		try {
			iterator = new DynamicSliceNDIterator(lazy.getShape(), mergeKeys(keys), lazy.getRank()-dataSize);
			if (repeating) iterator.enableRepeating();
		} catch (DatasetException e) {
			logger.error("Could not get data from lazy dataset", e);
		}
		this.lazy = lazy;
		this.keys = keys;
		this.finished = finished;
		
		int lr = lazy.getRank();
		
		this.axes = new int[dataSize];
		
		for (int i = 0; i < axes.length; i++) {
			axes[i] = lr - 1 -i;
		}
		updateShape();
		next = iterator.hasNext();
		
		Object ssm = lazy.getFirstMetadata(SliceFromSeriesMetadata.class);
		if (ssm != null && ssm instanceof SliceFromSeriesMetadata && ((SliceFromSeriesMetadata)ssm).getSourceInfo() != null) source = ((SliceFromSeriesMetadata)ssm).getSourceInfo();
		else logger.warn("Lazy dataset contains no source information");

	}
	
	public void setMaxTimeout(int maxTimeout) {
		this.maxTimeout = maxTimeout;
		timeout = maxTimeout/50;
	}

	public void updateShape() {
		try {
			lazy.refreshShape();
			int[] current = lazy.getShape();
			for (IDynamicDataset k : keys) k.refreshShape();
			int[] s = DynamicMetadataUtils.refreshDynamicAxesMetadata(lazy.getMetadata(AxesMetadata.class), lazy.getShape());
			logger.info("Dynamic iterator setting max shape " + Arrays.toString(s) + " for lazy shape " + Arrays.toString(current));
			for (int i = 0; i < axes.length;i++) {
				s[axes[i]] = current[axes[i]];
			}
			lazy.resize(s);
			iterator.updateShape(lazy.getShape(), mergeKeys(keys));
		} catch (Exception e) {
			logger.error("Error refreshing axes",e);
		}
		
	}
	
{'fraction_non_alphanumeric': 0.11052631578947368, 'fraction_numerical': 0.007218045112781955, 'mean_word_length': 4.844463971880492, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6547133', 'n_tokens_mistral': 2208, 'n_tokens_neox': 1992, 'n_words': 505}