<commit_before>/*
 * Isabelle encoding -- based on utf-8
 *
 * @author Makarius
 */

package isabelle.jedit


import org.gjt.sp.jedit.io.Encoding
import org.gjt.sp.jedit.buffer.JEditBuffer

import java.nio.charset.{Charset, CharsetDecoder, CodingErrorAction}
import java.io.{InputStream, OutputStream, Reader, Writer, InputStreamReader, OutputStreamWriter,
  CharArrayReader, ByteArrayOutputStream}

import scala.io.{Source, BufferedSource}


object Isabelle_Encoding
{
  val NAME = "UTF-8-Isabelle"

  def is_active(buffer: JEditBuffer): Boolean =
    buffer.getProperty(JEditBuffer.ENCODING).asInstanceOf[String] == NAME
}

class Isabelle_Encoding extends Encoding
{
  private val charset = Charset.forName(Isabelle_System.charset)
  private val BUFSIZE = 32768

  private def text_reader(in: InputStream, decoder: CharsetDecoder): Reader =
  {
    def source(): Source =
      BufferedSource.fromInputStream(in, decoder, BUFSIZE, { () => source() })
    new CharArrayReader(Isabelle.system.symbols.decode(source.mkString).toArray)
  }

	override def getTextReader(in: InputStream): Reader =
    text_reader(in, charset.newDecoder())<commit_msg>tuned;<commit_after>/*
 * Isabelle encoding -- based on utf-8
 *
 * @author Makarius
 */

package isabelle.jedit


import org.gjt.sp.jedit.io.Encoding
import org.gjt.sp.jedit.buffer.JEditBuffer

import java.nio.charset.{Charset, CharsetDecoder, CodingErrorAction}
import java.io.{InputStream, OutputStream, Reader, Writer, InputStreamReader, OutputStreamWriter,
  CharArrayReader, ByteArrayOutputStream}

import scala.io.{Source, BufferedSource}


object Isabelle_Encoding
{
  val NAME = "UTF-8-Isabelle"

  def is_active(buffer: JEditBuffer): Boolean =
    buffer.getStringProperty(JEditBuffer.ENCODING).asInstanceOf[String] == NAME
}

class Isabelle_Encoding extends Encoding
{
  private val charset = Charset.forName(Isabelle_System.charset)
  private val BUFSIZE = 32768

  private def text_reader(in: InputStream, decoder: CharsetDecoder): Reader =
  {
    def source(): Source =
      BufferedSource.fromInputStream(in, decoder, BUFSIZE, { () => source() })
    new CharArrayReader(Isabelle.system.symbols.decode(source.mkString).toArray)
  }

	override def getTextReader(in: InputStream): Reader =
    text_reader(in, charset.newDecoder())
{'fraction_non_alphanumeric': 0.1077191452245966, 'fraction_numerical': 0.006105538595726123, 'mean_word_length': 5.084880636604774, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3428147', 'n_tokens_mistral': 767, 'n_tokens_neox': 723, 'n_words': 151}