<commit_before>/*
 * Zorbage: an Algebraic data hierarchy for use in numeric processing.
 *
 * Copyright (C) 2016-2018 <NAME>
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 * 
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */
package nom.bdezonia.zorbage.type.storage;

import nom.bdezonia.zorbage.algebras.G;
import nom.bdezonia.zorbage.type.data.bool.BooleanMember;

// TODO: this class does a lot of recalculation. It is likely slow. But it can adapt to changes
// in the underlying list and the underlying mask. It is completely dynamic. Users must take
// care in a multithreaded environment.

/**
 * 
 * @author <NAME>
 *
 */
public class MaskedDataSource<T extends IndexedDataSource<T,U>,U>
	implements
		IndexedDataSource<T,U>
{
	private final IndexedDataSource<T,U> list;
	private final IndexedDataSource<?,BooleanMember> mask;
	
	/**
	 * 
	 * @param list
	 * @param mask
	 */
	public MaskedDataSource(IndexedDataSource<T,U> list, IndexedDataSource<?,BooleanMember> mask) {
		this.list = list;
		this.mask = mask;
		if (mask.size() == 0)
			throw new IllegalArgumentException("mask must not be of length 0");
	}
	
	@Override
	public T duplicate() {
		// shallow copy
		// TODO: WTH? Why does this warning keep cropping up?
		return (T) new MaskedDataSource<>(list, mask);
	}

	@Override
	public void set(long index, U value) {
		if (index < 0)
			throw new IllegalArgumentException("negative index exception");
		BooleanMember b = G.BOOL.construct();
		long count = -1;
		long maskSize = mask.size();
		for (long i = 0; i < list.size(); i++) {
			mask.get(i % maskSize, b);
			if (b.v()) {
				count++;
				if (count == index) {
					list.set(i, value);
					return;
				}
			}
		}
		throw new IllegalArgumentException("accessing masked data source off the end");
	}

	@Override
	public void get(long index, U value) {
		if (index < 0)
			throw new IllegalArgumentException("negative index exception");
		BooleanMember b = G.BOOL.construct();
		long count = -1;
		long maskSize = mask.size();
		for (long i = 0; i < list.size(); i++) {
			mask.get(i % maskSize, b);
			if (b.v()) {
				count++;
				if (count == index) {
					list.get(i, value);
					return;
				}
			}
		}
		throw new IllegalArgumentException("accessing masked data source off the end");
	}

	@Override
	public long size() {
		BooleanMember b = G.BOOL.construct();
		long maskSize = mask.size();
		long sz = 0;
		for (long i = 0; i < list.size(); i++) {
			mask.get(i % maskSize, b);
			if (b.v()) sz++;
		}
		return sz;
	}

}<commit_msg>Make MaskedDataSource faster and yet less dynamic<commit_after>/*
 * Zorbage: an Algebraic data hierarchy for use in numeric processing.
 *
 * Copyright (C) 2016-2018 <NAME>
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 * 
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */
package nom.bdezonia.zorbage.type.storage;

import nom.bdezonia.zorbage.algebras.G;
import nom.bdezonia.zorbage.type.data.bool.BooleanMember;

/**
 * 
 * @author <NAME>
 *
 */
public class MaskedDataSource<T extends IndexedDataSource<T,U>,U>
	implements
		IndexedDataSource<T,U>
{
	private final IndexedDataSource<T,U> list;
	private final IndexedDataSource<?,BooleanMember> mask;
	private final long listSize;
	private final long maskSize;
	private final long trueCount;
	
	/**
	 * 
	 * @param list
	 * @param mask
	 */
	public MaskedDataSource(IndexedDataSource<T,U> list, IndexedDataSource<?,BooleanMember> mask) {
		this.list = list;
		this.mask = mask;
		this.listSize = list.size();
		this.maskSize = mask.size();
		if (maskSize == 0)
			throw new IllegalArgumentException("mask must not be of length 0");
		BooleanMember b = G.BOOL.construct();
		long count = 0;
		for (long i = 0; i < Math.min(maskSize, listSize); i++) {
			mask.get(i, b);
			if (b.v())
				count++;
		}
		trueCount = count;
	}
	
	@Override
	public T duplicate() {
		// shallow copy
		// TODO: WTH? Why does this warning keep cropping up?
		return (T) new MaskedDataSource<>(list, mask);
	}

	@Override
	public void set(long index, U value) {
		if (index < 0)
			throw new IllegalArgumentException("negative index exception");
		long pos = findPosition(index);
		if (pos >= 0)
			list.set(pos, value);
		else
			throw new IllegalArgumentException("accessing masked data source off the end");
	}

	@Override
	public void get(long index, U value) {
		if (index < 0)
			throw new IllegalArgumentException("negative index exception");
		long pos = findPosition(index);
		if (pos >= 0)
			list.get(pos, value);
		else
			throw new IllegalArgumentException("accessing masked data source off the end");
	}

	@Override
	public long size() {
		BooleanMember b = G.BOOL.construct();
		long numFullReps = listSize / maskSize;
		long sz = numFullReps * trueCount;
		long partialMaskSize = listSize % maskSize;
		for (long i = 0; i < partialMaskSize; i++) {
			mask.get(i, b);
			if (b.v())
				sz++;
		}
		return sz;
	}

	private long findPosition(long index) {
		BooleanMember b = G.BOOL.construct();
		long numFullReps = index / trueCount;
		long pos = numFullReps * maskSize;
		long found = 0;
		long mustFind = index % trueCount;
		long i = 0;
		while (found < mustFind) {
			mask.get(i, b);
			if (b.v())
				found++;
			if (found != mustFind)
				i++;
		}
		pos += i;
		return pos;
	}
}
{'fraction_non_alphanumeric': 0.10155129709294747, 'fraction_numerical': 0.00534480511015513, 'mean_word_length': 3.7096378146101903, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4941137', 'n_tokens_mistral': 2568, 'n_tokens_neox': 2222, 'n_words': 889}