<commit_before>/*
 * fibers.cpp
 *
 *  Created on: Jan 24, 2013
 *      Author: schurade
 */
#include <limits>

#include "../data/datasets/datasetfibers.h"

#include "fibers.h"

Fibers::Fibers( DatasetFibers* ds ) :
    m_dataset( ds )
{
}

Fibers::~Fibers()
{
}

DatasetFibers* Fibers::thinOut()
{
    float maxDistance = 1.0;
    bool merged = true;
    bool currentMerged = false;
    int iteration = 1;
    QVector< QVector< float > > fibs = m_dataset->getFibs();

    QVector< QVector< float > > mergedFibs;

    while ( merged )
    {
        qDebug() << "iteration:" << iteration;
        ++iteration;
        qDebug() << fibs.size() << "fibers before merging";
        merged = false;
        for ( int i = 0; i < fibs.size(); ++i )
        {
            if ( i % 1000 == 0 ) qDebug() << i << fibs.size() << mergedFibs.size();
            currentMerged = false;
            QVector< float > currentFib = fibs[ i ];
            for( int k = i + 1; k < fibs.size(); ++k )
            {
                if ( currentFib.size() == fibs[k].size() )
                {
                    float dist = getFiberDist( currentFib, fibs[k] );
                    if ( dist < maxDistance )
                    {
                        mergedFibs.push_back( mergeFibs( currentFib, fibs[k]) );
                        merged = true;
                        currentMerged = true;
                        fibs.remove( k );
                        break;
                    }
                }
            }
            if ( !currentMerged )
            {
                mergedFibs.push_back( currentFib );
            }
        }
        fibs = mergedFibs;
        mergedFibs.clear();
        qDebug() << fibs.size() << "fibers after merging";
    }
    return new DatasetFibers( fibs );
}

QVector<float> Fibers::mergeFibs( QVector< float >& lhs, QVector< float >& rhs )
{
    QVector<float> out( lhs.size() );
    for ( int i = 0; i < lhs.size(); ++i )
    {
        out[i] = ( lhs[i] + rhs[i] ) / 2;
    }
    return out;
}

float Fibers::getFiberDist( QVector< float >& lhs, QVector< float >& rhs )
{
    float distSum = 0;
    float minSum;
    float lhsx;
    float lhsy;
    float lhsz;
    for ( int i = 0; i < lhs.size(); i += 3 )
    {
        minSum = std::numeric_limits<float>::max();
        lhsx = lhs.at( i );
        lhsy = lhs.at( i + 1 );
        lhsz = lhs.at( i + 2 );
        for( int k = 0; k < rhs.size(); k += 3 )
        {
            minSum = qMin( minSum, getDist( lhsx, lhsy, lhsz, rhs.at( i ), rhs.at( i + 1 ), rhs.at( i + 2 ) ) );<commit_msg>improved fib thin out algorithm<commit_after>/*
 * fibers.cpp
 *
 *  Created on: Jan 24, 2013
 *      Author: schurade
 */
#include <limits>

#include "../data/datasets/datasetfibers.h"

#include "fibers.h"

Fibers::Fibers( DatasetFibers* ds ) :
    m_dataset( ds )
{
}

Fibers::~Fibers()
{
}

DatasetFibers* Fibers::thinOut()
{
    float maxDistance = 1.0;
    bool merged = true;
    bool currentMerged = false;
    int iteration = 1;
    QVector< QVector< float > > fibs = m_dataset->getFibs();
    QVector< QVector< float > > mergedFibs;
    QVector< QVector< float > > unmergedFibs;

    while ( merged )
    {
        qDebug() << "iteration:" << iteration;
        ++iteration;
        qDebug() << fibs.size() << "fibers before merging";

        merged = false;
        QVector<bool>processed( fibs.size(), false );

        for ( int i = 0; i < fibs.size(); ++i )
        {
            if ( i % 1000 == 0 ) qDebug() << i << fibs.size() << mergedFibs.size() << unmergedFibs.size();
            currentMerged = false;
            QVector< float > currentFib = fibs[ i ];
            for( int k = i + 1; k < fibs.size(); ++k )
            {
                if ( currentFib.size() == fibs[k].size() )
                {
                    float dist = getFiberDist( currentFib, fibs[k] );
                    if ( dist < maxDistance )
                    {
                        mergedFibs.push_back( mergeFibs( currentFib, fibs[k]) );
                        merged = true;
                        currentMerged = true;
                        processed[k] = true;
                        break;
                    }
                }
            }
            if ( !currentMerged && !processed[i] )
            {
                unmergedFibs.push_back( currentFib );
            }
        }
        fibs = mergedFibs;
        mergedFibs.clear();
        qDebug() << fibs.size() + unmergedFibs.size() << "fibers after merging";
    }
    return new DatasetFibers( fibs + unmergedFibs );
}

QVector<float> Fibers::mergeFibs( QVector< float >& lhs, QVector< float >& rhs )
{
    QVector<float> out( lhs.size() );
    for ( int i = 0; i < lhs.size(); ++i )
    {
        out[i] = ( lhs[i] + rhs[i] ) / 2;
    }
    return out;
}

float Fibers::getFiberDist( QVector< float >& lhs, QVector< float >& rhs )
{
    float distSum = 0;
    float minSum;
    float lhsx;
    float lhsy;
    float lhsz;
    for ( int i = 0; i < lhs.size(); i += 3 )
    {
        minSum = std::numeric_limits<float>::max();
        lhsx = lhs.at( i );
        lhsy = lhs.at( i + 1 );
        lhsz = lhs.at( i + 2 );
        for( int k = 0; k < rhs.size(); k += 3 )
        {
            minSum = qMin( minSum, getDist( lhsx, lhsy, lhsz, rhs.at( i ), rhs.at( i + 1 ), rhs.at( i + 2 ) ) );
{'fraction_non_alphanumeric': 0.1478912839737582, 'fraction_numerical': 0.010121836925960637, 'mean_word_length': 1.5978578383641675, 'pattern_counts': {'":': 0, '<': 82, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1638601', 'n_tokens_mistral': 1935, 'n_tokens_neox': 1833, 'n_words': 439}