<commit_before>/*******************************************************************************
 * Copyright (c) 2016 IBH SYSTEMS GmbH and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *     IBH SYSTEMS GmbH - initial API and implementation
 *******************************************************************************/
package org.eclipse.packagedrone.utils;

import java.util.Arrays;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.stream.Collectors;

public class PathInformation implements Iterator<String>
{
    private final String path;

    private final int length;

    private int current;

    public PathInformation ( final String path )
    {
        Objects.requireNonNull ( path );

        this.path = path;
        this.length = path.length ();
        this.current = 0;

        eatUp ();
    }

    @Override
    public boolean hasNext ()
    {
        return this.current < this.length;
    }

    @Override
    public String next ()
    {
        if ( !hasNext () )
        {
            throw new NoSuchElementException ();
        }

        final StringBuilder sb = new StringBuilder ();
        while ( this.current < this.length )
        {
            final char c = this.path.charAt ( this.current );
            if ( c == '/' )
            {
                eatUp ();
                break;
            }
            sb.append ( c );
            this.current++;
        }
        return sb.toString ();
    }

    private void eatUp ()
    {
        while ( this.current < this.length )
        {
            final char c = this.path.charAt ( this.current );
            if ( c != '/' )
            {
                break;
            }
            this.current++;
        }
    }

    public String getRemainder ()
    {
        return Arrays.stream ( getRawRemainder ().split ( "/+" ) ).collect ( Collectors.joining ( "/" ) );<commit_msg>Add helper methods<commit_after>/*******************************************************************************
 * Copyright (c) 2016 IBH SYSTEMS GmbH and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *     IBH SYSTEMS GmbH - initial API and implementation
 *******************************************************************************/
package org.eclipse.packagedrone.utils;

import java.util.Arrays;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.function.Supplier;
import java.util.stream.Collectors;

public class PathInformation implements Iterator<String>
{
    private final String path;

    private final int length;

    private int current;

    public PathInformation ( final String path )
    {
        Objects.requireNonNull ( path );

        this.path = path;
        this.length = path.length ();
        this.current = 0;

        eatUp ();
    }

    @Override
    public boolean hasNext ()
    {
        return this.current < this.length;
    }

    @Override
    public String next ()
    {
        if ( !hasNext () )
        {
            throw new NoSuchElementException ();
        }

        final StringBuilder sb = new StringBuilder ();
        while ( this.current < this.length )
        {
            final char c = this.path.charAt ( this.current );
            if ( c == '/' )
            {
                eatUp ();
                break;
            }
            sb.append ( c );
            this.current++;
        }
        return sb.toString ();
    }

    public <T extends Throwable> String nextOrThrow ( final Supplier<T> supplier ) throws T
    {
        try
        {
            return next ();
        }
        catch ( final NoSuchElementException e )
        {
            throw supplier.get ();
        }
    }

    public String nextOrElse ( final String defaultValue )
    {
        try
        {
            return next ();
        }
        catch ( final NoSuchElementException e )
        {
            return defaultValue;
        }
    }

    public String nextOrElseGet ( final Supplier<String> defaultValue )
    {
        try
        {
            return next ();
        }
        catch ( final NoSuchElementException e )
        {
            return defaultValue.get ();
        }
    }

    private void eatUp ()
    {
        while ( this.current < this.length )
        {
            final char c = this.path.charAt ( this.current );
            if ( c != '/' )
            {
                break;
            }
            this.current++;
        }
    }

    public String getRemainder ()
    {
        return Arrays.stream ( getRawRemainder ().split ( "/+" ) ).collect ( Collectors.joining ( "/" ) );
{'fraction_non_alphanumeric': 0.15204678362573099, 'fraction_numerical': 0.0035087719298245615, 'mean_word_length': 1.8442350332594235, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1895984', 'n_tokens_mistral': 1352, 'n_tokens_neox': 1268, 'n_words': 356}