<commit_before>package jp.toastkid.article_viewer.zip

import android.os.Build
import androidx.annotation.RequiresApi
import jp.toastkid.article_viewer.article.Article
import jp.toastkid.article_viewer.article.ArticleRepository
import jp.toastkid.article_viewer.tokenizer.NgramTokenizer
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.launch
import okio.Okio
import timber.log.Timber
import java.io.InputStream
import java.nio.charset.Charset
import java.util.concurrent.TimeUnit
import java.util.zip.ZipEntry
import java.util.zip.ZipInputStream

/**
 * @author toastkidjp
 */
class ZipLoader(private val articleRepository: ArticleRepository) {

    private val tokenizer = NgramTokenizer()

    private val items = mutableListOf<Article>()

    private val disposable = Job()

    @RequiresApi(Build.VERSION_CODES.N)
    operator fun invoke(inputStream: InputStream) {
        ZipInputStream(inputStream, CHARSET)
            .use { zipInputStream ->
                var nextEntry = zipInputStream.nextEntry
                while (nextEntry != null) {
                    if (!nextEntry.name.contains(".")) {
                        nextEntry = zipInputStream.nextEntry
                        continue
                    }

                    extract(zipInputStream, nextEntry)

                    nextEntry = try {
                        zipInputStream.nextEntry
                    } catch (e: IllegalArgumentException) {
                        Timber.e("illegal: ${nextEntry.name}")
                        Timber.e(e)
                        return
                    }
                }
                zipInputStream.closeEntry()
            }
        flush()
    }

    private fun extract(
        zipInputStream: ZipInputStream,
        nextEntry: ZipEntry
    ) {
        // You don't use `use()` because it makes occur "java.io.IOException: Stream closed".
        Okio.buffer(Okio.source(zipInputStream)).let {
            val start = System.currentTimeMillis()
            val content = it.readUtf8()
            val title = extractFileName(nextEntry.name)
            val article = (articleRepository.findFirst(title) ?: Article(0)).also { a ->
                a.title = title
                a.contentText = content
                a.length = a.contentText.length
            }
            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
                article.lastModified = nextEntry.lastModifiedTime.to(TimeUnit.MILLISECONDS)<commit_msg>refactor #580 Use Okio2.<commit_after>package jp.toastkid.article_viewer.zip

import android.os.Build
import androidx.annotation.RequiresApi
import jp.toastkid.article_viewer.article.Article
import jp.toastkid.article_viewer.article.ArticleRepository
import jp.toastkid.article_viewer.tokenizer.NgramTokenizer
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.Job
import kotlinx.coroutines.launch
import okio.buffer
import okio.source
import timber.log.Timber
import java.io.InputStream
import java.nio.charset.Charset
import java.util.concurrent.TimeUnit
import java.util.zip.ZipEntry
import java.util.zip.ZipInputStream

/**
 * @author toastkidjp
 */
class ZipLoader(private val articleRepository: ArticleRepository) {

    private val tokenizer = NgramTokenizer()

    private val items = mutableListOf<Article>()

    private val disposable = Job()

    @RequiresApi(Build.VERSION_CODES.N)
    operator fun invoke(inputStream: InputStream) {
        ZipInputStream(inputStream, CHARSET)
            .use { zipInputStream ->
                var nextEntry = zipInputStream.nextEntry
                while (nextEntry != null) {
                    if (!nextEntry.name.contains(".")) {
                        nextEntry = zipInputStream.nextEntry
                        continue
                    }

                    extract(zipInputStream, nextEntry)

                    nextEntry = try {
                        zipInputStream.nextEntry
                    } catch (e: IllegalArgumentException) {
                        Timber.e("illegal: ${nextEntry.name}")
                        Timber.e(e)
                        return
                    }
                }
                zipInputStream.closeEntry()
            }
        flush()
    }

    private fun extract(
        zipInputStream: ZipInputStream,
        nextEntry: ZipEntry
    ) {
        // You don't use `use()` because it makes occur "java.io.IOException: Stream closed".
        zipInputStream.source().buffer().let {
            val start = System.currentTimeMillis()
            val content = it.readUtf8()
            val title = extractFileName(nextEntry.name)
            val article = (articleRepository.findFirst(title) ?: Article(0)).also { a ->
                a.title = title
                a.contentText = content
                a.length = a.contentText.length
            }
            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
                article.lastModified = nextEntry.lastModifiedTime.to(TimeUnit.MILLISECONDS)
{'fraction_non_alphanumeric': 0.08505254963020631, 'fraction_numerical': 0.001557026080186843, 'mean_word_length': 2.0516627078384797, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3109038', 'n_tokens_mistral': 1425, 'n_tokens_neox': 1333, 'n_words': 266}