<commit_before>/*
 * Copyright (c) 2017 Data and Web Science Group, University of Mannheim, Germany (http://dws.informatik.uni-mannheim.de/)
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and limitations under the License.
 */
package de.uni_mannheim.informatik.dws.winter.processing.parallel;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentLinkedQueue;

import de.uni_mannheim.informatik.dws.winter.model.Pair;
import de.uni_mannheim.informatik.dws.winter.processing.Action;
import de.uni_mannheim.informatik.dws.winter.processing.AggregateCollector;
import de.uni_mannheim.informatik.dws.winter.processing.DataAggregator;
import de.uni_mannheim.informatik.dws.winter.processing.DataIterator;
import de.uni_mannheim.informatik.dws.winter.processing.Function;
import de.uni_mannheim.informatik.dws.winter.processing.Group;
import de.uni_mannheim.informatik.dws.winter.processing.GroupCollector;
import de.uni_mannheim.informatik.dws.winter.processing.Processable;
import de.uni_mannheim.informatik.dws.winter.processing.ProcessableCollection;
import de.uni_mannheim.informatik.dws.winter.processing.ProcessableCollector;
import de.uni_mannheim.informatik.dws.winter.processing.RecordKeyValueMapper;
import de.uni_mannheim.informatik.dws.winter.processing.RecordMapper;
import de.uni_mannheim.informatik.dws.winter.utils.parallel.Consumer;
import de.uni_mannheim.informatik.dws.winter.utils.parallel.Parallel;
import de.uni_mannheim.informatik.dws.winter.utils.query.Q;

/**
 * 
 * Multi-threaded implementation of {@link Processable}
 * 
 * @author <NAME> (<EMAIL>)
 *
 */
public class ParallelProcessableCollection<RecordType> extends ProcessableCollection<RecordType> {
	
	private static final long serialVersionUID = 1L;

	public ParallelProcessableCollection() {
		super(new ConcurrentLinkedQueue<RecordType>());
	}
	
	public ParallelProcessableCollection(Collection<RecordType> elements) {
		super(new ConcurrentLinkedQueue<RecordType>(elements));
	}
	
	public ParallelProcessableCollection(Processable<RecordType> elements) {
		super(new ConcurrentLinkedQueue<RecordType>(elements.get()));
	}
	
	/* (non-Javadoc)
	 * @see de.uni_mannheim.informatik.wdi.processing.DataProcessingEngine#createResultSet(java.lang.Object)
	 */
	@Override
	public  <OutputRecordType> Processable<OutputRecordType> createProcessable(OutputRecordType dummyForTypeInference) {
		return new ParallelProcessableCollection<>();
	}
	
	/* (non-Javadoc)
	 * @see de.uni_mannheim.informatik.wdi.processing.ProcessableCollection#createProcessableFromCollection(java.util.Collection)
	 */
	@Override
	public <OutputRecordType> Processable<OutputRecordType> createProcessableFromCollection(
			Collection<OutputRecordType> data) {
		return new ParallelProcessableCollection<>(data);
	}
	
	/* (non-Javadoc)
	 * @see de.uni_mannheim.informatik.wdi.processing.DataProcessingEngine#iterateDataset(de.uni_mannheim.informatik.wdi.model.DataSet, de.uni_mannheim.informatik.wdi.processing.DatasetIterator)
	 */
	@Override
	public void foreach(final DataIterator<RecordType> iterator) {
		iterator.initialise();
		
		new Parallel<RecordType>().tryForeach(get(), new Consumer<RecordType>() {

			@Override
			public void execute(RecordType parameter) {
				iterator.next(parameter);
			}
		});
		iterator.finalise();
	}
	
	@Override
	public void foreach(Action<RecordType> action) {
		new Parallel<RecordType>().tryForeach(get(), (r)->action.execute(r));
	}
	
	public Collection<Collection<RecordType>> partitionRecords() {
		int partitionSize = (int)Math.ceil(size() / Runtime.getRuntime().availableProcessors());
		
		Collection<Collection<RecordType>> partitions = new LinkedList<>();
		
		Collection<RecordType> partition = new LinkedList<>();
		
		Iterator<RecordType> it = get().iterator();
		
		while(it.hasNext()) {
			partition.add(it.next());
			
			if(partition.size()==partitionSize) {
				partitions.add(partition);
				partition = new LinkedList<>();
			}
		}
		partitions.add(partition);
		
		return partitions;
	}
	
	/* (non-Javadoc)
	 * @see de.uni_mannheim.informatik.wdi.processing.DataProcessingEngine#transform(de.uni_mannheim.informatik.wdi.model.BasicCollection, de.uni_mannheim.informatik.wdi.processing.RecordMapper)
	 */
	@Override
	public <OutputRecordType> Processable<OutputRecordType> map(final RecordMapper<RecordType, OutputRecordType> transformation) {
		final ProcessableCollector<OutputRecordType> resultCollector = new ThreadSafeProcessableCollector<>();
		
		resultCollector.setResult(createProcessable((OutputRecordType)null));
		
		resultCollector.initialise();
		
		new Parallel<Collection<RecordType>>().tryForeach(partitionRecords(), new Consumer<Collection<RecordType>>() {

			@Override
			public void execute(Collection<RecordType> parameter) {
				for(RecordType r : parameter) {
					transformation.mapRecord(r, resultCollector);<commit_msg>changed partitioning strategy<commit_after>/*
 * Copyright (c) 2017 Data and Web Science Group, University of Mannheim, Germany (http://dws.informatik.uni-mannheim.de/)
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and limitations under the License.
 */
package de.uni_mannheim.informatik.dws.winter.processing.parallel;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentLinkedQueue;

import de.uni_mannheim.informatik.dws.winter.model.Pair;
import de.uni_mannheim.informatik.dws.winter.processing.Action;
import de.uni_mannheim.informatik.dws.winter.processing.AggregateCollector;
import de.uni_mannheim.informatik.dws.winter.processing.DataAggregator;
import de.uni_mannheim.informatik.dws.winter.processing.DataIterator;
import de.uni_mannheim.informatik.dws.winter.processing.Function;
import de.uni_mannheim.informatik.dws.winter.processing.Group;
import de.uni_mannheim.informatik.dws.winter.processing.GroupCollector;
import de.uni_mannheim.informatik.dws.winter.processing.Processable;
import de.uni_mannheim.informatik.dws.winter.processing.ProcessableCollection;
import de.uni_mannheim.informatik.dws.winter.processing.ProcessableCollector;
import de.uni_mannheim.informatik.dws.winter.processing.RecordKeyValueMapper;
import de.uni_mannheim.informatik.dws.winter.processing.RecordMapper;
import de.uni_mannheim.informatik.dws.winter.utils.parallel.Consumer;
import de.uni_mannheim.informatik.dws.winter.utils.parallel.Parallel;
import de.uni_mannheim.informatik.dws.winter.utils.query.Q;

/**
 * 
 * Multi-threaded implementation of {@link Processable}
 * 
 * @author <NAME> (<EMAIL>)
 *
 */
public class ParallelProcessableCollection<RecordType> extends ProcessableCollection<RecordType> {
	
	private static final long serialVersionUID = 1L;

	public ParallelProcessableCollection() {
		super(new ConcurrentLinkedQueue<RecordType>());
	}
	
	public ParallelProcessableCollection(Collection<RecordType> elements) {
		super(new ConcurrentLinkedQueue<RecordType>(elements));
	}
	
	public ParallelProcessableCollection(Processable<RecordType> elements) {
		super(new ConcurrentLinkedQueue<RecordType>(elements.get()));
	}
	
	/* (non-Javadoc)
	 * @see de.uni_mannheim.informatik.wdi.processing.DataProcessingEngine#createResultSet(java.lang.Object)
	 */
	@Override
	public  <OutputRecordType> Processable<OutputRecordType> createProcessable(OutputRecordType dummyForTypeInference) {
		return new ParallelProcessableCollection<>();
	}
	
	/* (non-Javadoc)
	 * @see de.uni_mannheim.informatik.wdi.processing.ProcessableCollection#createProcessableFromCollection(java.util.Collection)
	 */
	@Override
	public <OutputRecordType> Processable<OutputRecordType> createProcessableFromCollection(
			Collection<OutputRecordType> data) {
		return new ParallelProcessableCollection<>(data);
	}
	
	/* (non-Javadoc)
	 * @see de.uni_mannheim.informatik.wdi.processing.DataProcessingEngine#iterateDataset(de.uni_mannheim.informatik.wdi.model.DataSet, de.uni_mannheim.informatik.wdi.processing.DatasetIterator)
	 */
	@Override
	public void foreach(final DataIterator<RecordType> iterator) {
		iterator.initialise();
		
		new Parallel<RecordType>().tryForeach(get(), new Consumer<RecordType>() {

			@Override
			public void execute(RecordType parameter) {
				iterator.next(parameter);
			}
		});
		iterator.finalise();
	}
	
	@Override
	public void foreach(Action<RecordType> action) {
		new Parallel<RecordType>().tryForeach(get(), (r)->action.execute(r));
	}
	
	public Collection<Collection<RecordType>> partitionRecords() {
		// create more partitions than available threads so we can compensate for partitions which create less workload than others (so no thread runs idle) 
		int partitionSize = (int)Math.floor(size() / (Runtime.getRuntime().availableProcessors() * 10));
		
		Collection<Collection<RecordType>> partitions = new LinkedList<>();
		
		Collection<RecordType> partition = new LinkedList<>();
		
		Iterator<RecordType> it = get().iterator();
		
		while(it.hasNext()) {
			partition.add(it.next());
			
			if(partition.size()==partitionSize) {
				partitions.add(partition);
				partition = new LinkedList<>();
			}
		}
		if(partition.size()>0) {
			partitions.add(partition);
		}
		
		return partitions;
	}
	
	/* (non-Javadoc)
	 * @see de.uni_mannheim.informatik.wdi.processing.DataProcessingEngine#transform(de.uni_mannheim.informatik.wdi.model.BasicCollection, de.uni_mannheim.informatik.wdi.processing.RecordMapper)
	 */
	@Override
	public <OutputRecordType> Processable<OutputRecordType> map(final RecordMapper<RecordType, OutputRecordType> transformation) {
		final ProcessableCollector<OutputRecordType> resultCollector = new ThreadSafeProcessableCollector<>();
		
		resultCollector.setResult(createProcessable((OutputRecordType)null));
		
		resultCollector.initialise();
		
		new Parallel<Collection<RecordType>>().tryForeach(partitionRecords(), new Consumer<Collection<RecordType>>() {

			@Override
			public void execute(Collection<RecordType> parameter) {
				for(RecordType r : parameter) {
					transformation.mapRecord(r, resultCollector);
{'fraction_non_alphanumeric': 0.11078483715963694, 'fraction_numerical': 0.0018686599038974907, 'mean_word_length': 5.882424984690753, 'pattern_counts': {'":': 0, '<': 83, '<?xml version=': 0, '>': 86, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1470691', 'n_tokens_mistral': 3578, 'n_tokens_neox': 3317, 'n_words': 612}