<commit_before>package com.github.b3er.idea.plugins.arc.browser.base.sevenzip

import com.github.b3er.idea.plugins.arc.browser.base.BaseArchiveHandler
import com.github.b3er.idea.plugins.arc.browser.base.nest.SupportsStreamForVirtualFile
import com.github.b3er.idea.plugins.arc.browser.util.FSUtils
import com.github.b3er.idea.plugins.arc.browser.util.getAndUse
import com.intellij.openapi.util.io.FileTooBigException
import com.intellij.openapi.util.io.FileUtilRt
import com.intellij.openapi.vfs.VirtualFile
import net.sf.sevenzipjbinding.IInArchive
import net.sf.sevenzipjbinding.simple.ISimpleInArchiveItem
import java.io.ByteArrayOutputStream
import java.io.FileNotFoundException
import java.io.InputStream

class SevenZipArchiveHandler(
  path: String
) : BaseArchiveHandler<SevenZipArchiveHolder>(path), SupportsStreamForVirtualFile {

  override val accessorCache
    get() = cache

  override fun contentsToByteArray(relativePath: String): ByteArray {
    return getFileHandle().getAndUse { holder ->
      val item = holder.getItemForPath(relativePath)
      if (FileUtilRt.isTooLarge(item.size ?: DEFAULT_LENGTH)) {
        throw FileTooBigException("$file!/$relativePath")
      } else {
        ByteArrayOutputStream(item.size?.toInt() ?: SevenZipInputStream.BUFFER_SIZE).use { stream ->
          item.extractSlow {
            stream.write(it)
            it.size
          }
          stream.toByteArray()
        }
      }
    }
  }

  private fun createEntryNameForSingleArchive(entry: ISimpleInArchiveItem): String {
    val path = entry.ideaPath
    if (path.isNotEmpty()) {
      return path
    }
    val archiveFileName = file.name
    val nameWithoutEndExtension = archiveFileName.let { name -> name.substring(0, name.lastIndexOf('.')) }
    return FSUtils.decorateMergedNameWithExtension(file.extension, nameWithoutEndExtension)
  }

  override fun createEntriesMap(): MutableMap<String, EntryInfo> {
    return getFileHandle().getAndUse { holder ->
      if (holder.archive.isSingleFileArchive()) {
        LinkedHashMap<String, EntryInfo>(2).also { map ->
          map[""] = createRootEntry()
          val entry = holder.archiveItems.first()
          val path = createEntryNameForSingleArchive(entry)
          processEntry(map, path) { parent, name ->
            EntryInfo(name, false, entry.size, entry.creationTime?.time ?: DEFAULT_TIMESTAMP, parent)
          }
        }
      } else {
        val entries = holder.archiveItems
        LinkedHashMap<String, EntryInfo>(entries.size).also { map ->
          map[""] = createRootEntry()
          entries.forEach { entry ->
            processEntry(map, entry.ideaPath) { parent, name ->
              EntryInfo(name, false, entry.size, entry.creationTime?.time ?: DEFAULT_TIMESTAMP, parent)
            }
          }
        }
      }<commit_msg>Fix entry size for some archives<commit_after>package com.github.b3er.idea.plugins.arc.browser.base.sevenzip

import com.github.b3er.idea.plugins.arc.browser.base.BaseArchiveHandler
import com.github.b3er.idea.plugins.arc.browser.base.nest.SupportsStreamForVirtualFile
import com.github.b3er.idea.plugins.arc.browser.util.FSUtils
import com.github.b3er.idea.plugins.arc.browser.util.getAndUse
import com.intellij.openapi.util.io.FileTooBigException
import com.intellij.openapi.util.io.FileUtilRt
import com.intellij.openapi.vfs.VirtualFile
import net.sf.sevenzipjbinding.IInArchive
import net.sf.sevenzipjbinding.simple.ISimpleInArchiveItem
import java.io.ByteArrayOutputStream
import java.io.FileNotFoundException
import java.io.InputStream

class SevenZipArchiveHandler(
  path: String
) : BaseArchiveHandler<SevenZipArchiveHolder>(path), SupportsStreamForVirtualFile {

  override val accessorCache
    get() = cache

  override fun contentsToByteArray(relativePath: String): ByteArray {
    return getFileHandle().getAndUse { holder ->
      val item = holder.getItemForPath(relativePath)
      if (FileUtilRt.isTooLarge(item.size ?: DEFAULT_LENGTH)) {
        throw FileTooBigException("$file!/$relativePath")
      } else {
        ByteArrayOutputStream(
          item.size?.toInt()?.coerceAtLeast(DEFAULT_BUFFER_SIZE) ?: SevenZipInputStream.BUFFER_SIZE
        ).use { stream ->
          item.extractSlow {
            stream.write(it)
            it.size
          }
          stream.toByteArray()
        }
      }
    }
  }

  private fun createEntryNameForSingleArchive(entry: ISimpleInArchiveItem): String {
    val path = entry.ideaPath
    if (path.isNotEmpty()) {
      return path
    }
    val archiveFileName = file.name
    val nameWithoutEndExtension = archiveFileName.let { name -> name.substring(0, name.lastIndexOf('.')) }
    return FSUtils.decorateMergedNameWithExtension(file.extension, nameWithoutEndExtension)
  }

  override fun createEntriesMap(): MutableMap<String, EntryInfo> {
    return getFileHandle().getAndUse { holder ->
      if (holder.archive.isSingleFileArchive()) {
        LinkedHashMap<String, EntryInfo>(2).also { map ->
          map[""] = createRootEntry()
          val entry = holder.archiveItems.first()
          val path = createEntryNameForSingleArchive(entry)
          processEntry(map, entry, path)
        }
      } else {
        val entries = holder.archiveItems
        LinkedHashMap<String, EntryInfo>(entries.size).also { map ->
          map[""] = createRootEntry()
          entries.forEach { entry ->
            processEntry(map, entry, entry.ideaPath)
          }
        }
      }
    }
  }

  private fun processEntry(map: Map<String, EntryInfo>, entry: ISimpleInArchiveItem, path: String) {
    processEntry(map, path) { parent, name ->
      EntryInfo(
        name,
        false,
        entry.size ?: DEFAULT_LENGTH,
        entry.creationTime?.time ?: DEFAULT_TIMESTAMP,
        parent
      )
{'fraction_non_alphanumeric': 0.10377521117048785, 'fraction_numerical': 0.0024133770039648336, 'mean_word_length': 3.195227765726681, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1094757', 'n_tokens_mistral': 1718, 'n_tokens_neox': 1678, 'n_words': 288}