<commit_before>/* sxr -- Scala X-Ray
 * Copyright 2009, 2010 <NAME>, <NAME>
 */

package sxr

import scala.tools.nsc.{ast, plugins, symtab, util, Global}
import ast.parser.Tokens
import plugins.Plugin
import symtab.Flags
import reflect.internal.util.SourceFile

import java.io.{File, Reader, Writer}
import java.net.URL

import OutputFormat.{OutputFormat, getWriter}
import Browse._

object Browse
{
	/** The path to a link index, which allows linking between runs.  This applies for both incremental and remote linking.*/
	val LinkIndexRelativePath = "link.index"
	/** The path to a compressed link index, which allows linking between runs.  This applies for both incremental and remote linking.*/
	val CompressedLinkIndexRelativePath = LinkIndexRelativePath + ".gz"
	/** The name of the directory containing cached remote `link.index`es*/
	val CacheRelativePath = "cache.sxr"
}

/** The actual work extracting symbols and types is done here. */
abstract class Browse extends Plugin
{
	def classDirectory: File
	/** The output formats to write */
	def outputFormats: List[OutputFormat]
	/** The URLs of external sxr locations */
	def externalLinkURLs: List[URL]
	/** Relativizes the path to the given Scala source file against the base directories. */
	def getRelativeSourcePath(source: File): String
	/** The compiler.*/
	val global: Global

	import global._

	lazy val outputDirectory = new File(classDirectory.getParentFile, classDirectory.getName + ".sxr")
	outputDirectory.mkdirs

	private val linkIndexFile = new File(outputDirectory, LinkIndexRelativePath)
	private def linkStore = new LinkMapStore(linkIndexFile)

	/** The entry method for invoking the configured writers to generate the output.*/
	def generateOutput(externalLinks: List[LinkMap])
	{
		val sourceFiles = currentRun.units.toList.flatMap(getSourceFile(_))
		if (sourceFiles.size > 0) {
			val links = new CompoundLinkMap(linkStore.read(None), externalLinks)
			links.clear(sourceFiles.map(getRelativeSourcePath(_)).toList)

			val context = new OutputWriterContext(sourceFiles, outputDirectory, settings.encoding.value, externalLinkURLs)
			val writers = outputFormats.map(getWriter(_, context))
			writers.foreach(_.writeStart())

			for(unit <- currentRun.units ; sourceFile <- getSourceFile(unit))
			{
				// generate the tokens
				val tokens = scan(unit)
				val traverser = new Traverse(tokens, unit.source, links)
				traverser(unit.body)
				val tokenList = tokens.toList
				Collapse(tokenList, links)

				writers.foreach(_.writeUnit(sourceFile, getRelativeSourcePath(sourceFile), tokenList))
			}
			writers.foreach(_.writeEnd())
			linkStore.write(links)
		}
	}
	private def getSourceFile(unit: CompilationUnit): Option[File] = unit.source.file.file match {
		case null => None // code compiled from the repl has no source file
		case f: File => Some(f.getAbsoluteFile)
	}
	/** Tokenizes the given source.  The tokens are put into an ordered set by the start position of the token.
	* Symbols will be mapped back to these tokens by the offset of the symbol.*/
	private def scan(unit: CompilationUnit) =
	{
		val tokens = wrap.Wrappers.treeSet[Token]
		def addComment(start: Int, end: Int) { tokens += new Token(start, end - start + 1, Tokens.COMMENT) }

		class Scan extends syntaxAnalyzer.UnitScanner(unit)
		{
			override def deprecationWarning(off: Int, msg: String) {}
			override def error(off: Int, msg: String) {}
			override def incompleteInputError(off: Int, msg: String) {}

			override def foundComment(value: String, start: Int, end: Int) {
				addComment(start, end)
				super.foundComment(value, start, end)
		   }
			override def foundDocComment(value: String, start: Int, end: Int) {
				addComment(start, end)
				super.foundDocComment(value, start, end)
			}
			override def nextToken() {
				val offset0 = offset
				val code = token

				super.nextToken()

				if(includeToken(code)) {
					val length = (lastOffset - offset0) max 1
					tokens += new Token(offset0, length, code)
				}
			}
		}
		val parser = new syntaxAnalyzer.UnitParser(unit) {
			override def newScanner = new Scan
		}
		parser.parse()

		tokens
	}
	/** Filters out unwanted tokens such as whitespace and commas.  Braces are currently
	* included because () is annotated as Unit, and a partial function created by
	* { case ... } is associated with the opening brace.  */
	private def includeToken(code: Int) =
	{
		import Tokens.{COMMENT, USCORE, isBrace, isKeyword, isIdentifier, isLiteral}
		code match
		{
			case COMMENT | USCORE => true
			case _ => isKeyword(code) || isIdentifier(code) || isLiteral(code) || isBrace(code)
		}
	}
	/** Gets the token for the given offset.*/
	private def tokenAt(tokens: wrap.SortedSetWrapper[Token], offset: Int): Option[Token] =
	{
		// create artificial tokens to get a subset of the tokens starting at the given offset
		// then, take the first token in the range
		tokens.range(new Token(offset, 1, 0), new Token(offset+1, 1, 0)).first
	}

	/** Filters unwanted symbols, such as packages.*/
	private def ignore(s: Symbol): Boolean =
		ignoreBase(s) ||
		s.isModuleClass || // ignore the generated class for modules
		s.isPrimaryConstructor // the primary constructor overlaps with the class type, so just use the class type
	private def ignoreBase(s: Symbol): Boolean =
		!s.exists ||
		s.isPackage || // nothing done with packages
		s.isImplClass
		
	private class Traverse(tokens: wrap.SortedSetWrapper[Token], source: SourceFile, links: LinkMap) extends Traverser
	{
		// magic method #1
		override def traverse(tree: Tree)
		{
			def handleDefault()
			{
				process(tree)
				super.traverse(tree)<commit_msg>method to get all tokens at a position<commit_after>/* sxr -- Scala X-Ray
 * Copyright 2009, 2010 <NAME>, <NAME>
 */

package sxr

import scala.tools.nsc.{ast, plugins, symtab, util, Global}
import ast.parser.Tokens
import plugins.Plugin
import symtab.Flags
import reflect.internal.util.SourceFile

import java.io.{File, Reader, Writer}
import java.net.URL

import OutputFormat.{OutputFormat, getWriter}
import Browse._

object Browse
{
	/** The path to a link index, which allows linking between runs.  This applies for both incremental and remote linking.*/
	val LinkIndexRelativePath = "link.index"
	/** The path to a compressed link index, which allows linking between runs.  This applies for both incremental and remote linking.*/
	val CompressedLinkIndexRelativePath = LinkIndexRelativePath + ".gz"
	/** The name of the directory containing cached remote `link.index`es*/
	val CacheRelativePath = "cache.sxr"
}

/** The actual work extracting symbols and types is done here. */
abstract class Browse extends Plugin
{
	def classDirectory: File
	/** The output formats to write */
	def outputFormats: List[OutputFormat]
	/** The URLs of external sxr locations */
	def externalLinkURLs: List[URL]
	/** Relativizes the path to the given Scala source file against the base directories. */
	def getRelativeSourcePath(source: File): String
	/** The compiler.*/
	val global: Global

	import global._

	lazy val outputDirectory = new File(classDirectory.getParentFile, classDirectory.getName + ".sxr")
	outputDirectory.mkdirs

	private val linkIndexFile = new File(outputDirectory, LinkIndexRelativePath)
	private def linkStore = new LinkMapStore(linkIndexFile)

	/** The entry method for invoking the configured writers to generate the output.*/
	def generateOutput(externalLinks: List[LinkMap])
	{
		val sourceFiles = currentRun.units.toList.flatMap(getSourceFile(_))
		if (sourceFiles.size > 0) {
			val links = new CompoundLinkMap(linkStore.read(None), externalLinks)
			links.clear(sourceFiles.map(getRelativeSourcePath(_)).toList)

			val context = new OutputWriterContext(sourceFiles, outputDirectory, settings.encoding.value, externalLinkURLs)
			val writers = outputFormats.map(getWriter(_, context))
			writers.foreach(_.writeStart())

			for(unit <- currentRun.units ; sourceFile <- getSourceFile(unit))
			{
				// generate the tokens
				val tokens = scan(unit)
				val traverser = new Traverse(tokens, unit.source, links)
				traverser(unit.body)
				val tokenList = tokens.toList
				Collapse(tokenList, links)

				writers.foreach(_.writeUnit(sourceFile, getRelativeSourcePath(sourceFile), tokenList))
			}
			writers.foreach(_.writeEnd())
			linkStore.write(links)
		}
	}
	private def getSourceFile(unit: CompilationUnit): Option[File] = unit.source.file.file match {
		case null => None // code compiled from the repl has no source file
		case f: File => Some(f.getAbsoluteFile)
	}
	/** Tokenizes the given source.  The tokens are put into an ordered set by the start position of the token.
	* Symbols will be mapped back to these tokens by the offset of the symbol.*/
	private def scan(unit: CompilationUnit) =
	{
		val tokens = wrap.Wrappers.treeSet[Token]
		def addComment(start: Int, end: Int) { tokens += new Token(start, end - start + 1, Tokens.COMMENT) }

		class Scan extends syntaxAnalyzer.UnitScanner(unit)
		{
			override def deprecationWarning(off: Int, msg: String) {}
			override def error(off: Int, msg: String) {}
			override def incompleteInputError(off: Int, msg: String) {}

			override def foundComment(value: String, start: Int, end: Int) {
				addComment(start, end)
				super.foundComment(value, start, end)
		   }
			override def foundDocComment(value: String, start: Int, end: Int) {
				addComment(start, end)
				super.foundDocComment(value, start, end)
			}
			override def nextToken() {
				val offset0 = offset
				val code = token

				super.nextToken()

				if(includeToken(code)) {
					val length = (lastOffset - offset0) max 1
					tokens += new Token(offset0, length, code)
				}
			}
		}
		val parser = new syntaxAnalyzer.UnitParser(unit) {
			override def newScanner = new Scan
		}
		parser.parse()

		tokens
	}
	/** Filters out unwanted tokens such as whitespace and commas.  Braces are currently
	* included because () is annotated as Unit, and a partial function created by
	* { case ... } is associated with the opening brace.  */
	private def includeToken(code: Int) =
	{
		import Tokens.{COMMENT, USCORE, isBrace, isKeyword, isIdentifier, isLiteral}
		code match
		{
			case COMMENT | USCORE => true
			case _ => isKeyword(code) || isIdentifier(code) || isLiteral(code) || isBrace(code)
		}
	}
	/** Gets the token for the given offset.*/
	private def tokenAt(tokens: wrap.SortedSetWrapper[Token], offset: Int): Option[Token] =
		tokensAt(tokens, offset).headOption
	/** Gets the token for the given offset.*/
	private def tokensAt(tokens: wrap.SortedSetWrapper[Token], offset: Int): List[Token] =
	{
		// create artificial tokens to get a subset of the tokens starting at the given offset
		// then, take the first token in the range
		tokens.range(new Token(offset, 1, 0), new Token(offset+1, 1, 0)).toList
	}

	/** Filters unwanted symbols, such as packages.*/
	private def ignore(s: Symbol): Boolean =
		ignoreBase(s) ||
		s.isModuleClass || // ignore the generated class for modules
		s.isPrimaryConstructor // the primary constructor overlaps with the class type, so just use the class type
	private def ignoreBase(s: Symbol): Boolean =
		!s.exists ||
		s.isPackage || // nothing done with packages
		s.isImplClass
		
	private class Traverse(tokens: wrap.SortedSetWrapper[Token], source: SourceFile, links: LinkMap) extends Traverser
	{
		// magic method #1
		override def traverse(tree: Tree)
		{
			def handleDefault()
			{
				process(tree)
				super.traverse(tree)
{'fraction_non_alphanumeric': 0.10115102895012208, 'fraction_numerical': 0.003487966515521451, 'mean_word_length': 4.063576158940397, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5296022', 'n_tokens_mistral': 3744, 'n_tokens_neox': 3347, 'n_words': 1144}