<commit_before>package be.iminds.iot.dianne.tensor.impl.java;

import java.util.Arrays;
import java.util.Random;

import be.iminds.iot.dianne.tensor.Tensor;

public class JavaTensor implements Tensor<JavaTensor> {

	static Random random = new Random(System.currentTimeMillis());
	
	int[] dims;
	int offset = 0;
	int[] strides;
	float[] data;
	int[] indices = null;
	
	public JavaTensor(final int... d){
		this(d, null);
	
		data = new float[size()];
	}

	JavaTensor(final int[] dims, final float[] data){
		this.dims = dims;
		this.data = data;

		this.strides = new int[dims.length];
		int stride = 1;
		for(int i=dims.length-1;i>=0;i--){
			strides[i] = stride;
			stride*= dims[i];
		}
		
		if(data!=null){
			assert data.length == size();
		}
	}
	
	@Override
	public int dim() {
		return dims.length;
	}
	
	@Override
	public int[] dims() {
		return Arrays.copyOf(dims, dims.length);
	}

	@Override
	public int size() {
		int size = 1;
		for(int i=0;i<dims.length;i++){
			size *= dims[i];
		}
		return size;
	}

	@Override
	public int size(int d) {
		return dims[d];
	}
	
	@Override
	public float get(final int... d) {
		int[] i = d;
		if(d.length!=dims.length){
			i = new int[dims.length];
			int l = 0;
			for(int k=0;k<dims.length;k++){
				if(dims[k]==1){
					i[k] = 0;
				} else {
					i[k] = d[l];
					l++;
				}
			}
		}
		
		int index = getIndex(i);
		return data[index];
	}
	
	@Override
	public void set(final float v, final int... d) {
		assert d.length == dims.length;
		
		int index = getIndex(d);
		data[index] = v;
	}
	
	@Override
	public String toString(){
		if(dims.length == 2){
			// format as matrix
			String s = "";
			for(int i=0;i<dims[0];i++){
				s+="[";
				for(int j = 0; j < dims[1]-1; j++){
					s+= get(i,j)+", ";
				}
				s+= get(i, dims[1]-1) + "]";
				s+="\n";
			}
			return s;
		}
		return Arrays.toString(data);
	}

	@Override
	public void fill(float v) {
		if(indices==null){
			for(int i=0;i<data.length;i++)
				data[i] = v;
		} else {
			for(int i=0;i<indices.length;i++){
				data[indices[i]] = v;
			}
		}
	}

	@Override
	public void rand() {
		if(indices==null){
			for(int i=0;i<data.length;i++)
				data[i] = random.nextFloat();
		} else {
			for(int i=0;i<indices.length;i++){
				data[indices[i]] = random.nextFloat();
			}
		}
	}
	
	@Override
	public void grand() {
		if(indices==null){
			for(int i=0;i<data.length;i++)
				data[i] = (float) random.nextGaussian();
		} else {
			for(int i=0;i<indices.length;i++){
				data[indices[i]] = (float) random.nextGaussian();
			}
		}
	}

	@Override
	public boolean equals(Object other){
		if(!(other instanceof JavaTensor)){
			return false;
		} 
		JavaTensor o = (JavaTensor) other;
		
		if(o.dim() != dim()){
			return false;
		}
		
		if(o.size() != size()){
			return false;
		}
		
		for(int i=0;i<dims.length;i++){
			if(o.size(i) != dims[i]){
				return false;
			}
		}
		
		if(indices==null){
			for(int i=0;i<data.length;i++)
				if(data[i] != o.data[i])
					return false;
		} else {
			for(int i=0;i<indices.length;i++){
				if(data[indices[i]] != o.data[o.indices[i]])
					return false;
			}
		}
<commit_msg>fix indices bug in JavaTensor equals<commit_after>package be.iminds.iot.dianne.tensor.impl.java;

import java.util.Arrays;
import java.util.Random;

import be.iminds.iot.dianne.tensor.Tensor;

public class JavaTensor implements Tensor<JavaTensor> {

	static Random random = new Random(System.currentTimeMillis());
	
	int[] dims;
	int offset = 0;
	int[] strides;
	float[] data;
	int[] indices = null;
	
	public JavaTensor(final int... d){
		this(d, null);
	
		data = new float[size()];
	}

	JavaTensor(final int[] dims, final float[] data){
		this.dims = dims;
		this.data = data;

		this.strides = new int[dims.length];
		int stride = 1;
		for(int i=dims.length-1;i>=0;i--){
			strides[i] = stride;
			stride*= dims[i];
		}
		
		if(data!=null){
			assert data.length == size();
		}
	}
	
	@Override
	public int dim() {
		return dims.length;
	}
	
	@Override
	public int[] dims() {
		return Arrays.copyOf(dims, dims.length);
	}

	@Override
	public int size() {
		int size = 1;
		for(int i=0;i<dims.length;i++){
			size *= dims[i];
		}
		return size;
	}

	@Override
	public int size(int d) {
		return dims[d];
	}
	
	@Override
	public float get(final int... d) {
		int[] i = d;
		if(d.length!=dims.length){
			i = new int[dims.length];
			int l = 0;
			for(int k=0;k<dims.length;k++){
				if(dims[k]==1){
					i[k] = 0;
				} else {
					i[k] = d[l];
					l++;
				}
			}
		}
		
		int index = getIndex(i);
		return data[index];
	}
	
	@Override
	public void set(final float v, final int... d) {
		assert d.length == dims.length;
		
		int index = getIndex(d);
		data[index] = v;
	}
	
	@Override
	public String toString(){
		if(dims.length == 2){
			// format as matrix
			String s = "";
			for(int i=0;i<dims[0];i++){
				s+="[";
				for(int j = 0; j < dims[1]-1; j++){
					s+= get(i,j)+", ";
				}
				s+= get(i, dims[1]-1) + "]";
				s+="\n";
			}
			return s;
		}
		return Arrays.toString(data);
	}

	@Override
	public void fill(float v) {
		for(int i=0;i< (indices==null? data.length : indices.length);i++){
			data[(indices==null ? i : indices[i])] = v;
		}
	}

	@Override
	public void rand() {
		for(int i=0;i< (indices==null? data.length : indices.length);i++){
			data[(indices==null ? i : indices[i])] = random.nextFloat();
		}
	}
	
	@Override
	public void grand() {
		for(int i=0;i< (indices==null? data.length : indices.length);i++){
			data[(indices==null ? i : indices[i])] = (float) random.nextGaussian();
		}
	}

	@Override
	public boolean equals(Object other){
		if(!(other instanceof JavaTensor)){
			return false;
		} 
		JavaTensor o = (JavaTensor) other;
		
		if(o.dim() != dim()){
			return false;
		}
		
		if(o.size() != size()){
			return false;
		}
		
		for(int i=0;i<dims.length;i++){
			if(o.size(i) != dims[i]){
				return false;
			}
		}
		
		for(int i=0;i< (indices==null? data.length : indices.length);i++){
			if(data[(indices==null ? i : indices[i])] 
					!= o.data[(o.indices==null ? i : o.indices[i])]){
				return false;
			}
		}

{'fraction_non_alphanumeric': 0.19358552631578949, 'fraction_numerical': 0.008223684210526315, 'mean_word_length': 2.920696324951644, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '423202', 'n_tokens_mistral': 2857, 'n_tokens_neox': 2441, 'n_words': 524}