<commit_before>package org.jetbrains.plugins.scala.decompiler

import _root_.scala.tools.nsc.symtab.classfile.ClassfileParser
import _root_.scala.tools.scalap.ByteArrayReader
import _root_.scala.tools.scalap.Classfile
import _root_.scala.tools.scalap.JavaWriter
import com.intellij.lang.impl.PsiBuilderImpl
import com.intellij.lang.{PsiBuilderFactory, ASTNode, PsiBuilder}
import com.intellij.openapi.util.io.FileUtil
import com.intellij.openapi.vfs.{VirtualFileManager, VirtualFile}
import com.intellij.psi.impl.DebugUtil
import com.intellij.psi.stubs.PsiFileStub
import com.intellij.psi.tree.{TokenSet, IElementType}
import com.intellij.psi.{PsiFileFactory, PsiFile}
import com.intellij.util.io.StringRef
import java.io._
import lang.lexer.{ScalaLexer, ScalaTokenTypes}
import lang.parser.{ScalaElementTypes}
import lang.psi.api.{ScalaFile}
import lang.psi.stubs.impl.{ScFileStubImpl, ScExtendsBlockStubImpl, ScTypeDefinitionStubImpl}
import lang.psi.stubs.{ScExtendsBlockStub, ScTypeDefinitionStub, ScFileStub}
import parsec.{Memo, Parser, ParserConversions}

/**
 * @author ilyas
 */

object ScalaDecompiler {

  private val DUMMY = "dummy."

  def isScalaFile(bytes: Array[Byte]) = {
    val reader = new ByteArrayReader(bytes)
    val clazz = new Classfile(reader)
    clazz.attribs.find(a => a.toString() == "ScalaSig") match {
      case Some(sig) => true
      case None => false
    }
  }

  private def getFileText(bytes: Array[Byte]): String = {
    //Write to temp file
    val reader = new ByteArrayReader(bytes)
    val clazz = new Classfile(reader)
    val tempFile = File.createTempFile("scalap_temp", ".scala")
    val out = new FileWriter(tempFile)
    val writer = new JavaWriter(clazz, out)
    writer.printClass
    out.flush

    //reading file to string
    val buffer = new StringBuffer()
    val fis = new FileInputStream(tempFile)
    // Here BufferedInputStream is added for fast reading.
    val bis = new BufferedInputStream(fis);
    val dis = new DataInputStream(bis);

    // dis.available() returns 0 if the file does not have more lines.
    while (dis.available() != 0) {
      buffer.append(dis.readLine).append("\n")
    }
    fis.close();
    bis.close();
    dis.close();
    buffer.toString
  }

  private def declareGrammar = {
    import ScalaTokenTypes._, ScalaElementTypes._, ParserConversions._

    val MODIFIERS = (kFINAL | kSEALED | kPRIVATE | kPROTECTED | kABSTRACT | kCASE) ?

    val REFERENCE_NT = ((tIDENTIFIER > tDOT) *) > tIDENTIFIER |> REFERENCE

    val PACKAGE_NT = kPACKAGE > REFERENCE_NT > tSEMICOLON |> PACKAGE_STMT

    val TYPES_NT = (REFERENCE_NT ?) > ((tCOMMA > REFERENCE_NT) *)
    val TYPE_PARAMS_NT = tLSQBRACKET > REFERENCE_NT > tRSQBRACKET
    val PARAM_NT = REFERENCE_NT > (TYPE_PARAMS_NT ?) |> TYPE //Parameter type
    val PARAMETERS_NT = (PARAM_NT ?) > ((tCOMMA > PARAM_NT) *) |> PARAM_CLAUSES // Function parameters

    val FUNCTION_NT = MODIFIERS > kDEF > tIDENTIFIER > tLPARENTHESIS > PARAMETERS_NT > tRPARENTHESIS >
        tCOLON > PARAM_NT |> FUNCTION_DECLARATION // Function declaration

    val VAR_NT = MODIFIERS > kVAR > tIDENTIFIER > tCOLON > PARAM_NT |> VARIABLE_DECLARATION
    val VAL_NT = MODIFIERS > kVAL > tIDENTIFIER > tCOLON > PARAM_NT |> VALUE_DECLARATION

    val MEMBER_DECL_NT = (FUNCTION_NT | VAR_NT | VAL_NT) > tSEMICOLON

    val mockParser = (new Parser {
      def do_parse(builder: PsiBuilder) = {
        if (builder.getTokenType != tRBRACE) {
          builder.advanceLexer
          true
        } else false
      }
    }) *

    val EXTENDS_BLOCK_NT = ((kEXTENDS > REFERENCE_NT) ?) > ((kWITH > REFERENCE_NT) *) |> EXTENDS_BLOCK
    val TYPE_DEFINITION_BODY_NT = tLBRACE > mockParser > tRBRACE // Type definition body

    val TYPE_DEFINITION_NT = MODIFIERS > (kCLASS | kTRAIT | kOBJECT) > tIDENTIFIER >
        (EXTENDS_BLOCK_NT ?) > // 'extends' block
        TYPE_DEFINITION_BODY_NT |> TYPE_DEFINITION

    val OTHER = PLAIN_PARSER
    val FILE = PACKAGE_NT > (TYPE_DEFINITION_NT > (tLINE_TERMINATOR ?) *) > OTHER |> ScalaElementTypes.FILE
    FILE
  }<commit_msg>decompiler modified<commit_after>package org.jetbrains.plugins.scala.decompiler

import _root_.scala.tools.nsc.symtab.classfile.ClassfileParser
import _root_.scala.tools.scalap.ByteArrayReader
import _root_.scala.tools.scalap.Classfile
import _root_.scala.tools.scalap.JavaWriter
import com.intellij.lang.impl.PsiBuilderImpl
import com.intellij.lang.{PsiBuilderFactory, ASTNode, PsiBuilder}
import com.intellij.openapi.util.io.FileUtil
import com.intellij.openapi.vfs.{VirtualFileManager, VirtualFile}
import com.intellij.psi.impl.DebugUtil
import com.intellij.psi.stubs.PsiFileStub
import com.intellij.psi.tree.{TokenSet, IElementType}
import com.intellij.psi.{PsiFileFactory, PsiFile}
import com.intellij.util.io.StringRef
import java.io._
import lang.lexer.{ScalaLexer, ScalaTokenTypes}
import lang.parser.{ScalaElementTypes}
import lang.psi.api.{ScalaFile}
import lang.psi.stubs.impl.{ScFileStubImpl, ScExtendsBlockStubImpl, ScTypeDefinitionStubImpl}
import lang.psi.stubs.{ScExtendsBlockStub, ScTypeDefinitionStub, ScFileStub}
import parsec.{Memo, Parser, ParserConversions}

/**
 * @author ilyas
 */

object ScalaDecompiler {

  private val DUMMY = "dummy."

  def isScalaFile(bytes: Array[Byte]) = {
    val reader = new ByteArrayReader(bytes)
    val clazz = new Classfile(reader)
    clazz.attribs.find(a => a.toString() == "ScalaSig") match {
      case Some(sig) => true
      case None => false
    }
  }

  private def getFileText(bytes: Array[Byte]): String = {
    //Write to temp file
    val reader = new ByteArrayReader(bytes)
    val clazz = new Classfile(reader)
    val tempFile = File.createTempFile("scalap_temp", ".scala")
    val out = new FileWriter(tempFile)
    val writer = new JavaWriter(clazz, out)
    writer.printClass
    out.flush

    //reading file to string
    val buffer = new StringBuffer()
    val fis = new FileInputStream(tempFile)
    // Here BufferedInputStream is added for fast reading.
    val bis = new BufferedInputStream(fis);
    val dis = new DataInputStream(bis);

    // dis.available() returns 0 if the file does not have more lines.
    while (dis.available() != 0) {
      buffer.append(dis.readLine).append("\n")
    }
    fis.close();
    bis.close();
    dis.close();
    buffer.toString
  }

  private def declareGrammar = {
    import ScalaTokenTypes._, ScalaElementTypes._, ParserConversions._

    val MODIFIERS = (kFINAL | kSEALED | kPRIVATE | kPROTECTED | kABSTRACT | kCASE) ?

    val REFERENCE_NT = ((tIDENTIFIER > tDOT) *) > tIDENTIFIER |> REFERENCE

    val PACKAGE_NT = kPACKAGE > REFERENCE_NT > tSEMICOLON |> PACKAGE_STMT

    val mockParser = (new Parser {
      def do_parse(builder: PsiBuilder) = {
        if (builder.getTokenType != tSEMICOLON) {
          builder.advanceLexer
          true
        } else false
      }
    }) *


    val TYPE_PARAMETERS_NT = (REFERENCE_NT ?) > ((tCOMMA > REFERENCE_NT) *) |> PARAM_CLAUSES // Type params
    val TYPES_NT = (REFERENCE_NT ?) > ((tCOMMA > REFERENCE_NT) *)
    val TYPE_PARAMS_NT = tLSQBRACKET > (TYPE_PARAMETERS_NT ?) > tRSQBRACKET
    val PARAM_NT = REFERENCE_NT > (TYPE_PARAMS_NT ?) |> TYPE //Parameter type
    val PARAMETERS_NT = (PARAM_NT ?) > ((tCOMMA > PARAM_NT) *) |> PARAM_CLAUSES // Function parameters
    val THROWS_NT = tIDENTIFIER > REFERENCE_NT > tLINE_TERMINATOR |> THROW_STMT

    val FUNCTION_NT = MODIFIERS > kDEF > (tIDENTIFIER | kTHIS) > tLPARENTHESIS > PARAMETERS_NT > tRPARENTHESIS >
        tCOLON > PARAM_NT > tSEMICOLON > (THROWS_NT ?) |> FUNCTION_DECLARATION // Function declaration

    val VAR_NT = MODIFIERS > kVAR > tIDENTIFIER > tCOLON > PARAM_NT > tSEMICOLON |> VARIABLE_DECLARATION
    val VAL_NT = MODIFIERS > kVAL > tIDENTIFIER > tCOLON > PARAM_NT > tSEMICOLON |> VALUE_DECLARATION

    val MEMBER_DECL_NT = (FUNCTION_NT | VAR_NT | VAL_NT)

    val EXTENDS_BLOCK_NT = ((kEXTENDS > REFERENCE_NT) ?) > ((kWITH > REFERENCE_NT) *) |> EXTENDS_BLOCK
    val TYPE_DEFINITION_BODY_NT = tLBRACE > (MEMBER_DECL_NT *) > tRBRACE // Type definition body

    val TYPE_DEFINITION_NT = MODIFIERS > (kCLASS | kTRAIT | kOBJECT) > tIDENTIFIER >
        (EXTENDS_BLOCK_NT ?) > // 'extends' block
        TYPE_DEFINITION_BODY_NT |> TYPE_DEFINITION

    val OTHER = PLAIN_PARSER
    val FILE = PACKAGE_NT > ((TYPE_DEFINITION_NT > (tLINE_TERMINATOR ?)) *) > OTHER |> ScalaElementTypes.FILE
    FILE
  }
{'fraction_non_alphanumeric': 0.10508192828306816, 'fraction_numerical': 0.0004749465685110425, 'mean_word_length': 3.756070016939582, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 116, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3318489', 'n_tokens_mistral': 3011, 'n_tokens_neox': 2955, 'n_words': 619}