<commit_before>package norswap.violin.stream
import java.util.stream.Stream as JStream

/// Conversion to Stream ---------------------------------------------------------------------------

/**
 * Converts an iterator into a stream.
 *
 * Be cautious: this assumes the iterator is finite and so returns a [Stream].
 * If not so, assign the result to a [Stream] to avoid errors.
 */
fun <T: Any> Iterator<T>.tune(): Stream<T>
    = Stream { if (hasNext()) next() else null }

/**
 * Converts a java stream into a violin stream.
 */
fun <T: Any> java.util.stream.Stream<T>.tune(): Stream<T> {
    val iterator = iterator()
    return Stream { if (iterator.hasNext()) iterator.next() else null }
}

/**
 * Converts a kotlin sequence into a stream.
 */
fun <T: Any> Sequence<T>.tune(): Stream<T>
    = iterator().tune()

/// Conversion to Streamable -----------------------------------------------------------------------

/**
 * Converts an iterable into a streamable.
 *
 * Be cautious: this assumes the iterable yields finite iterators and so returns a [Streamable].
 * If not so, assign the result to a [Streamable] to avoid errors.
 */
fun <T: Any> Iterable<T>.tune(): Streamable<T>
    = object: Streamable<T> {
        override fun stream() = iterator().tune()
    }

/// Stream from standard types ---------------------------------------------------------------------

/**
 * Returns a stream consisting of the items of the array.
 */
fun <T: Any> Array<out T>.stream(): Stream<T>{
    var i = 0
    return Stream { if (i < size) get(i++) else null }
}

/**
 * Returns a stream consisting of the items of the array, in reverse order.
 */
fun <T: Any> Array<out T>.reverseStream(): Stream<T> {
    var i = size
    return Stream { if (i > 0) get(--i) else null }
}

/**
 * Returns a stream consisting of the non-null items of the array.
 */
fun <T: Any> Array<out T?>.pureStream(): Stream<T> {
    var i = 0
    return Stream {
        var item: T? = null
        while (i < size && item == null) item = get(i++)
        item
    }
}

/**
 * Returns a stream consisting of the non-null items of the array, in reverse order.
 */
fun <T: Any> Array<out T?>.pureReverseStream(): Stream<T> {
    var i = size
    return Stream {
        var item: T? = null
        while (i > 0 && item == null) item = get(--i)
        item
    }
}

/**
 * Returns a stream consisting of the items of the iterable.
 */
fun <T: Any> Iterable<T>.stream(): Stream<T>
    = iterator().tune()

/**
 * Returns a stream consisting of the items of the list, in reverse order.
 */
fun <T: Any> List<T>.reverseStream(): Stream<T>
    = reversed().stream()

// -------------------------------------------------------------------------------------------------<commit_msg>add conversions from stream and streamable<commit_after>package norswap.violin.stream
import norswap.violin.utils.after
import java.util.NoSuchElementException
import java.util.stream.StreamSupport
import java.util.stream.Stream as JStream

/// Conversion to Stream ---------------------------------------------------------------------------

/**
 * Converts an iterator into a stream.
 *
 * Be cautious: this assumes the iterator is finite and so returns a [Stream].
 * If not so, assign the result to a [Stream] to avoid errors.
 */
fun <T: Any> Iterator<T>.tune(): Stream<T>
    = Stream { if (hasNext()) next() else null }

/**
 * Converts a java stream into a violin stream.
 */
fun <T: Any> java.util.stream.Stream<T>.tune(): Stream<T> {
    val iterator = iterator()
    return Stream { if (iterator.hasNext()) iterator.next() else null }
}

/**
 * Converts a kotlin sequence into a stream.
 */
fun <T: Any> Sequence<T>.tune(): Stream<T>
    = iterator().tune()

/// Conversion to Streamable -----------------------------------------------------------------------

/**
 * Converts an iterable into a streamable.
 *
 * Be cautious: this assumes the iterable yields finite iterators and so returns a [Streamable].
 * If not so, assign the result to a [Streamable] to avoid errors.
 */
fun <T: Any> Iterable<T>.tune(): Streamable<T>
    = object: Streamable<T> {
        override fun stream() = iterator().tune()
    }

/// Stream from standard types ---------------------------------------------------------------------

/**
 * Returns a stream consisting of the items of the array.
 */
fun <T: Any> Array<out T>.stream(): Stream<T>{
    var i = 0
    return Stream { if (i < size) get(i++) else null }
}

/**
 * Returns a stream consisting of the items of the array, in reverse order.
 */
fun <T: Any> Array<out T>.reverseStream(): Stream<T> {
    var i = size
    return Stream { if (i > 0) get(--i) else null }
}

/**
 * Returns a stream consisting of the non-null items of the array.
 */
fun <T: Any> Array<out T?>.pureStream(): Stream<T> {
    var i = 0
    return Stream {
        var item: T? = null
        while (i < size && item == null) item = get(i++)
        item
    }
}

/**
 * Returns a stream consisting of the non-null items of the array, in reverse order.
 */
fun <T: Any> Array<out T?>.pureReverseStream(): Stream<T> {
    var i = size
    return Stream {
        var item: T? = null
        while (i > 0 && item == null) item = get(--i)
        item
    }
}

/**
 * Returns a stream consisting of the items of the iterable.
 */
fun <T: Any> Iterable<T>.stream(): Stream<T>
    = iterator().tune()

/**
 * Returns a stream consisting of the items of the list, in reverse order.
 */
fun <T: Any> List<T>.reverseStream(): Stream<T>
    = reversed().stream()

/// Conversion from Stream -------------------------------------------------------------------------

/**
 * Returns an iterator backed by the stream.
 */
fun <T: Any> Stream<T>.iterator() = object: Iterator<T> {
    private var peek: T? = this@iterator.next()
    override fun hasNext() = peek != null
    override fun next(): T
        = (peek ?: throw NoSuchElementException()) after { peek = this@iterator.next() }
}

/**
 * Returns a java stream backed by the stream.
 */
fun <T: Any> Stream<T>.javaStream() = StreamSupport.stream(
    java.lang.Iterable<T> { this@javaStream.iterator() }.spliterator(),
    false)

/// Conversion from Streamable ---------------------------------------------------------------------

/**
 * Returns an iterable backed by the streamable.
 */
fun <T: Any> Streamable<T>.iterable() = object: Iterable<T> {
    override fun iterator() = this@iterable.stream().toIterator()
}

/**
 * Returns a sequence backed by the streamable.
 */
fun <T: Any> Streamable<T>.sequence() = object: Sequence<T> {
    override fun iterator() = this@sequence.stream().toIterator()
}

/// ------------------------------------------------------------------------------------------------
{'fraction_non_alphanumeric': 0.2420770718643059, 'fraction_numerical': 0.0011902990626394881, 'mean_word_length': 1.9743362831858406, 'pattern_counts': {'":': 0, '<': 81, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 5, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2214607', 'n_tokens_mistral': 1907, 'n_tokens_neox': 1877, 'n_words': 701}