<commit_before>#include "NeuronExtractor.h"
#include <util/Logger.h>

logger::LogChannel neuronextractorlog("neuronextractorlog", "[NeuronExtractor] ");

NeuronExtractor::NeuronExtractor() {

	registerInput(_segments, "segments");
	registerOutput(_neurons, "neurons");
}

void
NeuronExtractor::updateOutputs() {

	if (!_neurons)
		_neurons = new SegmentTrees();

	prepareSliceMaps();

	findConnectedSlices();

	createNeuronsFromSlices();

	LOG_ALL(neuronextractorlog) << "found " << _neurons->size() << " neurons" << std::endl;
}

void
NeuronExtractor::prepareSliceMaps() {

	_sliceSegments.clear();
	_sliceNeighbors.clear();
	_unvisitedSlices.clear();

	foreach (boost::shared_ptr<Segment> segment, _segments->getSegments()) {

		std::vector<boost::shared_ptr<Slice> > sources = segment->getSourceSlices();
		std::vector<boost::shared_ptr<Slice> > targets = segment->getTargetSlices();

		foreach (boost::shared_ptr<Slice> slice, sources) {

			_sliceSegments[slice->getId()].push_back(segment);
			_unvisitedSlices.insert(slice->getId());
		}

		foreach (boost::shared_ptr<Slice> slice, targets) {

			_sliceSegments[slice->getId()].push_back(segment);
			_unvisitedSlices.insert(slice->getId());
		}

		foreach (boost::shared_ptr<Slice> source, sources)
			foreach (boost::shared_ptr<Slice> target, sources) {

				_sliceNeighbors[source->getId()].push_back(target->getId());
				_sliceNeighbors[target->getId()].push_back(source->getId());
			}
	}
}

void
NeuronExtractor::findConnectedSlices() {

	_connectedSlices.clear();

	while (!_unvisitedSlices.empty()) {

		// create a new component
		std::set<unsigned int> connectedSlices;

		// pick a unvisited slice and add it to the stack
		unsigned int seedSlice = *(_unvisitedSlices.begin());
		_boundarySlices.push(seedSlice);

		// while there are slices on the boundary stack
		while (!_boundarySlices.empty()) {
<commit_msg>bugfix: NeuronExtractor found one neuron per slice<commit_after>#include "NeuronExtractor.h"
#include <util/Logger.h>

logger::LogChannel neuronextractorlog("neuronextractorlog", "[NeuronExtractor] ");

NeuronExtractor::NeuronExtractor() {

	registerInput(_segments, "segments");
	registerOutput(_neurons, "neurons");
}

void
NeuronExtractor::updateOutputs() {

	if (!_neurons)
		_neurons = new SegmentTrees();

	prepareSliceMaps();

	findConnectedSlices();

	createNeuronsFromSlices();

	LOG_ALL(neuronextractorlog) << "found " << _neurons->size() << " neurons" << std::endl;
}

void
NeuronExtractor::prepareSliceMaps() {

	_sliceSegments.clear();
	_sliceNeighbors.clear();
	_unvisitedSlices.clear();

	foreach (boost::shared_ptr<Segment> segment, _segments->getSegments()) {

		std::vector<boost::shared_ptr<Slice> > sources = segment->getSourceSlices();
		std::vector<boost::shared_ptr<Slice> > targets = segment->getTargetSlices();

		foreach (boost::shared_ptr<Slice> slice, sources) {

			_sliceSegments[slice->getId()].push_back(segment);
			_unvisitedSlices.insert(slice->getId());
		}

		foreach (boost::shared_ptr<Slice> slice, targets) {

			_sliceSegments[slice->getId()].push_back(segment);
			_unvisitedSlices.insert(slice->getId());
		}

		foreach (boost::shared_ptr<Slice> source, sources)
			foreach (boost::shared_ptr<Slice> target, targets) {

				_sliceNeighbors[source->getId()].push_back(target->getId());
				_sliceNeighbors[target->getId()].push_back(source->getId());
			}
	}
}

void
NeuronExtractor::findConnectedSlices() {

	_connectedSlices.clear();

	while (!_unvisitedSlices.empty()) {

		// create a new component
		std::set<unsigned int> connectedSlices;

		// pick a unvisited slice and add it to the stack
		unsigned int seedSlice = *(_unvisitedSlices.begin());
		_boundarySlices.push(seedSlice);

		// while there are slices on the boundary stack
		while (!_boundarySlices.empty()) {

{'fraction_non_alphanumeric': 0.1518021573270192, 'fraction_numerical': 0.0, 'mean_word_length': 6.106542056074766, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3300923', 'n_tokens_mistral': 1416, 'n_tokens_neox': 1289, 'n_words': 208}