<commit_before>package com.wavesplatform

import java.io.{BufferedOutputStream, File, FileOutputStream}
import java.util.concurrent.locks.ReentrantReadWriteLock

import com.google.common.primitives.Ints
import com.typesafe.config.ConfigFactory
import com.wavesplatform.history.HistoryWriterImpl
import com.wavesplatform.settings.{WavesSettings, loadConfig}
import com.wavesplatform.utils._
import org.slf4j.bridge.SLF4JBridgeHandler
import scorex.account.AddressScheme
import scorex.utils.ScorexLogging

import scala.util.{Failure, Success, Try}

object Exporter extends ScorexLogging {
  def main(args: Array[String]): Unit = {
    SLF4JBridgeHandler.removeHandlersForRootLogger()
    SLF4JBridgeHandler.install()

    val configFilename = Try(args(0)).toOption.getOrElse("waves-testnet.conf")
    val outputFilenamePrefix = Try(args(1)).toOption.getOrElse("blockchain")
    val exportHeight = Try(args(2)).toOption.flatMap(s => Try(s.toInt).toOption)

    val settings = WavesSettings.fromConfig(loadConfig(ConfigFactory.parseFile(new File(configFilename))))
    AddressScheme.current = new AddressScheme {
      override val chainId: Byte = settings.blockchainSettings.addressSchemeCharacter.toByte
    }

    val tryHistory = HistoryWriterImpl(settings.blockchainSettings.blockchainFile, new ReentrantReadWriteLock(true), settings.blockchainSettings.functionalitySettings, settings.featuresSettings)
    tryHistory match {
      case Success(history) =>
        val blockchainHeight = history.height()
        val height = Math.min(blockchainHeight, exportHeight.getOrElse(blockchainHeight))
        log.info(s"Blockchain height is $blockchainHeight exporting to $height")
        val outputFilename = s"$outputFilenamePrefix-$height"
        log.info(s"Output file: $outputFilename")

        createOutputStream(outputFilename) match {
          case Success(output) =>
            var exportedBytes = 0
            val bos = new BufferedOutputStream(output)
            val start = System.currentTimeMillis()
            (2 to height).foreach { h =>
              val maybeBlockBytes = history.blockBytes(h)
              maybeBlockBytes.foreach { bytes =>
                val len = bytes.length
                bos.write(Ints.toByteArray(len))
                bos.write(bytes)
                exportedBytes += len + Ints.BYTES
                if (h % (height / 10) == 0)
                  log.info(s"$h blocks exported, ${humanReadableSize(exportedBytes)} written")
              }
            }
            val duration = System.currentTimeMillis() - start
            log.info(s"Finished exporting $height blocks in ${humanReadableDuration(duration)}, ${humanReadableSize(exportedBytes)} written")
            bos.close()
            output.close()
          case Failure(ex) => log.error(s"Failed to create file '$outputFilename': $ex")
        }
      case Failure(ex) => log.error(s"Failed to open history at '${settings.blockchainSettings.blockchainFile}': $ex")
    }
  }

  def createOutputStream(filename: String): Try[FileOutputStream] =
    Try {
      new FileOutputStream(filename)
    }<commit_msg>Export to JSON was implemented<commit_after>package com.wavesplatform

import java.io.{BufferedOutputStream, File, FileOutputStream, OutputStream}
import java.nio.charset.StandardCharsets
import java.util.concurrent.locks.ReentrantReadWriteLock

import com.google.common.primitives.Ints
import com.typesafe.config.ConfigFactory
import com.wavesplatform.history.HistoryWriterImpl
import com.wavesplatform.settings.{WavesSettings, loadConfig}
import com.wavesplatform.utils._
import org.slf4j.bridge.SLF4JBridgeHandler
import scorex.account.AddressScheme
import scorex.transaction.History
import scorex.utils.ScorexLogging

import scala.util.{Failure, Success, Try}

object Exporter extends ScorexLogging {
  def main(args: Array[String]): Unit = {
    SLF4JBridgeHandler.removeHandlersForRootLogger()
    SLF4JBridgeHandler.install()

    val configFilename = Try(args(0)).toOption.getOrElse("waves-testnet.conf")
    val outputFilenamePrefix = Try(args(1)).toOption.getOrElse("blockchain")
    val exportHeight = Try(args(2)).toOption.flatMap(s => Try(s.toInt).toOption)
    val format = Try(args(3)).toOption.filter(s => s.toUpperCase == "JSON").getOrElse("BINARY").toUpperCase

    val settings = WavesSettings.fromConfig(loadConfig(ConfigFactory.parseFile(new File(configFilename))))
    AddressScheme.current = new AddressScheme {
      override val chainId: Byte = settings.blockchainSettings.addressSchemeCharacter.toByte
    }

    val tryHistory = HistoryWriterImpl(settings.blockchainSettings.blockchainFile, new ReentrantReadWriteLock(true), settings.blockchainSettings.functionalitySettings, settings.featuresSettings)
    tryHistory match {
      case Success(history) =>
        val blockchainHeight = history.height()
        val height = Math.min(blockchainHeight, exportHeight.getOrElse(blockchainHeight))
        log.info(s"Blockchain height is $blockchainHeight exporting to $height")
        val outputFilename = s"$outputFilenamePrefix-$height"
        log.info(s"Output file: $outputFilename")

        createOutputStream(outputFilename) match {
          case Success(output) =>
            var exportedBytes = 0L
            val bos = new BufferedOutputStream(output)
            val start = System.currentTimeMillis()
            exportedBytes += writeHeader(bos, format)
            (2 to height).foreach { h =>
              exportedBytes += (if (format == "JSON") exportBlockToJson(bos, history, h) else exportBlockToBinary(bos, history, h))
              if (h % (height / 10) == 0)
                log.info(s"$h blocks exported, ${humanReadableSize(exportedBytes)} written")
            }
            exportedBytes += writeFooter(bos, format)
            val duration = System.currentTimeMillis() - start
            log.info(s"Finished exporting $height blocks in ${humanReadableDuration(duration)}, ${humanReadableSize(exportedBytes)} written")
            bos.close()
            output.close()
          case Failure(ex) => log.error(s"Failed to create file '$outputFilename': $ex")
        }
      case Failure(ex)
      => log.error(s"Failed to open history at '${settings.blockchainSettings.blockchainFile}': $ex")
    }
  }

  private def createOutputStream(filename: String): Try[FileOutputStream] =
    Try {
      new FileOutputStream(filename)
    }

  private def exportBlockToBinary(stream: OutputStream, history: History, height: Int): Int = {
    val maybeBlockBytes = history.blockBytes(height)
    maybeBlockBytes.map {
      bytes =>
        val len = bytes.length
        stream.write(Ints.toByteArray(len))
        stream.write(bytes)
        len + Ints.BYTES
    }.getOrElse(0)
  }

  private def exportBlockToJson(stream: OutputStream, history: History, height: Int): Int = {
    val maybeBlock = history.blockAt(height)
    maybeBlock.map { block =>
      val len = if (height != 2) {
        val bytes = ",\n".getBytes(StandardCharsets.UTF_8)
        stream.write(bytes)
        bytes.length
      } else 0
      val bytes = block.json().toString().getBytes(StandardCharsets.UTF_8)
      stream.write(bytes)
      len + bytes.length
    }.getOrElse(0)
  }

  private def writeHeader(stream: OutputStream, format: String): Int =
    if (format == "JSON") writeString(stream, "[\n") else 0

  private def writeFooter(stream: OutputStream, format: String): Int =
    if (format == "JSON") writeString(stream, "]\n") else 0

  private def writeString(stream: OutputStream, str: String): Int = {
    val bytes = str.getBytes(StandardCharsets.UTF_8)
    stream.write(bytes)
    bytes.length
  }

{'fraction_non_alphanumeric': 0.10506395197076482, 'fraction_numerical': 0.004437483685721744, 'mean_word_length': 3.52094395280236, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2228258', 'n_tokens_mistral': 2189, 'n_tokens_neox': 2107, 'n_words': 456}