<commit_before>/* NSC -- new Scala compiler
 * Copyright 2005-2011 LAMP/EPFL
 * @author  <NAME>
 */


package scala.tools.nsc
package util

import io.{ AbstractFile, VirtualFile }
import scala.collection.mutable.ArrayBuffer
import annotation.tailrec
import java.util.regex.Pattern
import java.io.IOException
import scala.reflect.internal.Chars._

/** abstract base class of a source file used in the compiler */
abstract class SourceFile {
  def content : Array[Char]         // normalized, must end in SU
  def file    : AbstractFile
  def isLineBreak(idx : Int) : Boolean
  def isSelfContained: Boolean
  def length : Int
  def position(offset: Int) : Position = {
    assert(offset < length, file + ": " + offset + " >= " + length)
    new OffsetPosition(this, offset)
  }
  def position(line: Int, column: Int) : Position = new OffsetPosition(this, lineToOffset(line) + column)

  def offsetToLine(offset: Int): Int
  def lineToOffset(index : Int): Int

  /** Map a position to a position in the underlying source file.
   *  For regular source files, simply return the argument.
   */
  def positionInUltimateSource(position: Position) = position
  override def toString(): String = file.name /* + ":" + content.length */
  def dbg(offset: Int) = (new OffsetPosition(this, offset)).dbgString
  def path = file.path

  def beginsWith(offset: Int, text: String): Boolean =
    (content drop offset) startsWith text

  def lineToString(index: Int): String =
    content drop lineToOffset(index) takeWhile (c => !isLineBreakChar(c.toChar)) mkString

  @tailrec
  final def skipWhitespace(offset: Int): Int =
    if (content(offset).isWhitespace) skipWhitespace(offset + 1) else offset

  def identifier(pos: Position): Option[String] = None
}

/** An object representing a missing source file.
 */
object NoSourceFile extends SourceFile {
  def content                   = Array()
  def file                      = NoFile
  def isLineBreak(idx: Int)     = false
  def isSelfContained           = true
  def length                    = -1
  def offsetToLine(offset: Int) = -1
  def lineToOffset(index : Int) = -1
  override def toString = "NoSourceFile"
}

object NoFile extends VirtualFile("<no file>", "<no file>")

object ScriptSourceFile {
  /** Length of the script header from the given content, if there is one.
   *  The header begins with "#!" or "::#!" and ends with a line starting
   *  with "!#" or "::!#".
   */
  def headerLength(cs: Array[Char]): Int = {
    val headerPattern = Pattern.compile("""^(::)?!#.*(\r|\n|\r\n)""", Pattern.MULTILINE)
    val headerStarts  = List("#!", "::#!")

    if (headerStarts exists (cs startsWith _)) {
      val matcher = headerPattern matcher cs.mkString
      if (matcher.find) matcher.end
      else throw new IOException("script file does not close its header with !# or ::!#")
    }
    else 0
  }
  def stripHeader(cs: Array[Char]): Array[Char] = cs drop headerLength(cs)<commit_msg>Makes it possible to use env(1) to run scala scripts, by dropping the requirement of a !# line when using it. This is how some other languages, such as ruby, are run.<commit_after>/* NSC -- new Scala compiler
 * Copyright 2005-2011 LAMP/EPFL
 * @author  <NAME>
 */


package scala.tools.nsc
package util

import io.{ AbstractFile, VirtualFile }
import scala.collection.mutable.ArrayBuffer
import annotation.tailrec
import java.util.regex.Pattern
import java.io.IOException
import scala.reflect.internal.Chars._

/** abstract base class of a source file used in the compiler */
abstract class SourceFile {
  def content : Array[Char]         // normalized, must end in SU
  def file    : AbstractFile
  def isLineBreak(idx : Int) : Boolean
  def isSelfContained: Boolean
  def length : Int
  def position(offset: Int) : Position = {
    assert(offset < length, file + ": " + offset + " >= " + length)
    new OffsetPosition(this, offset)
  }
  def position(line: Int, column: Int) : Position = new OffsetPosition(this, lineToOffset(line) + column)

  def offsetToLine(offset: Int): Int
  def lineToOffset(index : Int): Int

  /** Map a position to a position in the underlying source file.
   *  For regular source files, simply return the argument.
   */
  def positionInUltimateSource(position: Position) = position
  override def toString(): String = file.name /* + ":" + content.length */
  def dbg(offset: Int) = (new OffsetPosition(this, offset)).dbgString
  def path = file.path

  def beginsWith(offset: Int, text: String): Boolean =
    (content drop offset) startsWith text

  def lineToString(index: Int): String =
    content drop lineToOffset(index) takeWhile (c => !isLineBreakChar(c.toChar)) mkString

  @tailrec
  final def skipWhitespace(offset: Int): Int =
    if (content(offset).isWhitespace) skipWhitespace(offset + 1) else offset

  def identifier(pos: Position): Option[String] = None
}

/** An object representing a missing source file.
 */
object NoSourceFile extends SourceFile {
  def content                   = Array()
  def file                      = NoFile
  def isLineBreak(idx: Int)     = false
  def isSelfContained           = true
  def length                    = -1
  def offsetToLine(offset: Int) = -1
  def lineToOffset(index : Int) = -1
  override def toString = "NoSourceFile"
}

object NoFile extends VirtualFile("<no file>", "<no file>")

object ScriptSourceFile {
  /** Length of the script header from the given content, if there is one.
   *  The header begins with "#!" or "::#!" and ends with a line starting
   *  with "!#" or "::!#".
   */
  def headerLength(cs: Array[Char]): Int = {
    val headerPattern = Pattern.compile("""((?m)^(::)?!#.*|^.*/env .*)(\r|\n|\r\n)""")
    val headerStarts  = List("#!", "::#!")

    if (headerStarts exists (cs startsWith _)) {
      val matcher = headerPattern matcher cs.mkString
      if (matcher.find) matcher.end
      else throw new IOException("script file does not close its header with !# or ::!#")
    }
    else 0
  }
  def stripHeader(cs: Array[Char]): Array[Char] = cs drop headerLength(cs)
{'fraction_non_alphanumeric': 0.11789089703200133, 'fraction_numerical': 0.004476869507544354, 'mean_word_length': 3.352092352092352, 'pattern_counts': {'":': 10, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4817634', 'n_tokens_mistral': 1827, 'n_tokens_neox': 1773, 'n_words': 607}