<commit_before>@file:Suppress("BlockingMethodInNonBlockingContext", "KDocUnresolvedReference")

package com.fireflysource.common.io

import com.fireflysource.common.coroutine.CoroutineDispatchers.ioBlockingThreadPool
import com.fireflysource.common.coroutine.blocking
import com.fireflysource.common.coroutine.blockingAsync
import kotlinx.coroutines.CancellableContinuation
import kotlinx.coroutines.NonCancellable
import kotlinx.coroutines.future.await
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlinx.coroutines.withContext
import java.io.Closeable
import java.net.SocketAddress
import java.nio.ByteBuffer
import java.nio.channels.*
import java.nio.file.Files
import java.nio.file.LinkOption
import java.nio.file.OpenOption
import java.nio.file.Path
import java.nio.file.attribute.BasicFileAttributes
import java.util.concurrent.TimeUnit

/**
 * Performs [AsynchronousFileChannel.lock] without blocking a thread and resumes when asynchronous operation completes.
 * This suspending function is cancellable.
 * If the [Job] of the current coroutine cancelled or completed while this suspending function is waiting, this function
 * *closes the underlying channel* and immediately resumes with [CancellationException].
 */
suspend fun AsynchronousFileChannel.lockAwait() = suspendCancellableCoroutine<FileLock> { cont ->
    lock(cont, asyncIOHandler())
    closeOnCancel(cont)
}

/**
 * Performs [AsynchronousFileChannel.lock] without blocking a thread and resumes when asynchronous operation completes.
 * This suspending function is cancellable.
 * If the [Job] of the current coroutine cancelled or completed while this suspending function is waiting, this function
 * *closes the underlying channel* and immediately resumes with [CancellationException].
 */
suspend fun AsynchronousFileChannel.lockAwait(
    position: Long,
    size: Long,
    shared: Boolean
) = suspendCancellableCoroutine<FileLock> { cont ->
    lock(position, size, shared, cont, asyncIOHandler())
    closeOnCancel(cont)
}

suspend fun <T : Closeable?, R> T.useAwait(block: suspend (T) -> R): R {
    try {
        return block(this)
    } catch (e: Throwable) {
        throw e
    } finally {
        withContext(NonCancellable) {
            this@useAwait?.closeJob()?.join()
        }
    }
}

suspend fun <T : AsyncCloseable?, R> T.useAwait(block: suspend (T) -> R): R {
    try {
        return block(this)
    } catch (e: Throwable) {
        throw e
    } finally {
        withContext(NonCancellable) {
            this@useAwait?.closeFuture()?.await()
        }
    }
}

/**
 * Close in the I/O blocking coroutine dispatcher
 */
fun Closeable.closeJob() = blocking {
    close()
}

fun openFileChannelAsync(file: Path, vararg options: OpenOption) = blockingAsync {
    AsynchronousFileChannel.open(file, setOf(*options), ioBlockingThreadPool)
}

fun openFileChannelAsync(file: Path, options: Set<OpenOption>) = blockingAsync {
    AsynchronousFileChannel.open(file, options, ioBlockingThreadPool)
}

fun readAllBytesAsync(file: Path) = blockingAsync {
    Files.readAllBytes(file)
}
<commit_msg>[feat]: add read all lines method.<commit_after>@file:Suppress("BlockingMethodInNonBlockingContext", "KDocUnresolvedReference")

package com.fireflysource.common.io

import com.fireflysource.common.coroutine.CoroutineDispatchers.ioBlockingThreadPool
import com.fireflysource.common.coroutine.blocking
import com.fireflysource.common.coroutine.blockingAsync
import kotlinx.coroutines.CancellableContinuation
import kotlinx.coroutines.NonCancellable
import kotlinx.coroutines.future.await
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlinx.coroutines.withContext
import java.io.Closeable
import java.net.SocketAddress
import java.nio.ByteBuffer
import java.nio.channels.*
import java.nio.charset.Charset
import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.nio.file.LinkOption
import java.nio.file.OpenOption
import java.nio.file.Path
import java.nio.file.attribute.BasicFileAttributes
import java.util.concurrent.TimeUnit

/**
 * Performs [AsynchronousFileChannel.lock] without blocking a thread and resumes when asynchronous operation completes.
 * This suspending function is cancellable.
 * If the [Job] of the current coroutine cancelled or completed while this suspending function is waiting, this function
 * *closes the underlying channel* and immediately resumes with [CancellationException].
 */
suspend fun AsynchronousFileChannel.lockAwait() = suspendCancellableCoroutine<FileLock> { cont ->
    lock(cont, asyncIOHandler())
    closeOnCancel(cont)
}

/**
 * Performs [AsynchronousFileChannel.lock] without blocking a thread and resumes when asynchronous operation completes.
 * This suspending function is cancellable.
 * If the [Job] of the current coroutine cancelled or completed while this suspending function is waiting, this function
 * *closes the underlying channel* and immediately resumes with [CancellationException].
 */
suspend fun AsynchronousFileChannel.lockAwait(
    position: Long,
    size: Long,
    shared: Boolean
) = suspendCancellableCoroutine<FileLock> { cont ->
    lock(position, size, shared, cont, asyncIOHandler())
    closeOnCancel(cont)
}

suspend fun <T : Closeable?, R> T.useAwait(block: suspend (T) -> R): R {
    try {
        return block(this)
    } catch (e: Throwable) {
        throw e
    } finally {
        withContext(NonCancellable) {
            this@useAwait?.closeJob()?.join()
        }
    }
}

suspend fun <T : AsyncCloseable?, R> T.useAwait(block: suspend (T) -> R): R {
    try {
        return block(this)
    } catch (e: Throwable) {
        throw e
    } finally {
        withContext(NonCancellable) {
            this@useAwait?.closeFuture()?.await()
        }
    }
}

/**
 * Close in the I/O blocking coroutine dispatcher
 */
fun Closeable.closeJob() = blocking {
    close()
}

fun openFileChannelAsync(file: Path, vararg options: OpenOption) = blockingAsync {
    AsynchronousFileChannel.open(file, setOf(*options), ioBlockingThreadPool)
}

fun openFileChannelAsync(file: Path, options: Set<OpenOption>) = blockingAsync {
    AsynchronousFileChannel.open(file, options, ioBlockingThreadPool)
}

fun readAllLinesAsync(file: Path, charset: Charset = StandardCharsets.UTF_8) = blockingAsync {
    Files.readAllLines(file, charset)
}

fun readAllBytesAsync(file: Path) = blockingAsync {
    Files.readAllBytes(file)
}

{'fraction_non_alphanumeric': 0.09353635798632691, 'fraction_numerical': 0.00015537600994406463, 'mean_word_length': 4.544358311800172, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '828565', 'n_tokens_mistral': 1880, 'n_tokens_neox': 1899, 'n_words': 478}