<commit_before>package controllers


import util.parsing.combinator.RegexParsers
import scala.collection.mutable.Buffer
import javax.inject._
import play.api._
import play.api.mvc._
import play.twirl.api._
import play.api.libs.json._
import scala.io.Source
import java.awt.Color
import services.Logger


trait ImportantTextPart {
	def content: String
}
case class Word (val content: String, nbOccur: Int) extends ImportantTextPart 
case class Punctuations (val content: String) extends ImportantTextPart 
case class Spaces(val content: String) extends ImportantTextPart

case class InputText(text: String)


object ColorUtils {
	import java.lang.Math._

	val blueHue = 0.7f
	def greenToRedGiver(maxNbOccurs: Int)(nbOccur: Int): Color = 
		redToColderColorGiverLogarithmic(0.2f)(maxNbOccurs)(nbOccur)

	def blueToRedGiver(maxNbOccurs: Int)(nbOccur: Int): Color = 
		redToColderColorGiverLogarithmic(blueHue)(maxNbOccurs)(nbOccur)

	def redToColderColorGiverLogarithmic(colderColorHue: Float)(maxNbOccurs: Int)(nbOccur: Int): Color = {
		val k: Float = (colderColorHue / log(maxNbOccurs)).toFloat
		
		val h = {
			val formula = colderColorHue - k * log(nbOccur)
			if (formula < 0) 0
			else formula
		}.toFloat
		//finding the word color from the hue and arbitrary brightness and saturation parameter 
		
		Color.getHSBColor(h, 1f, .9f)
	}

	def redToColderColorGiverLinear(colderColorHue: Float)(nbOccur: Int, maxNbOccurs: Int): Color = {
		/*
		* the more occurrences there is, the redder the word becomes.
		* the words with the least occurrences are blue.
		*/
		val colderColorRatio = 
			(maxNbOccurs - (nbOccur-1)).toFloat / maxNbOccurs
		val h = colderColorRatio * colderColorHue
		//finding the word color from the hue and arbitrary brightness and saturation parameter 
		Color.getHSBColor(h, 1f, .9f)
	}


}

class TextParser extends RegexParsers {

	override def skipWhitespace = false 

	lazy val spacesParser1: Parser[Spaces] = "\\s+".r ^^ { s => Spaces(s) }
	lazy val punctuation: Parser[Punctuations] = "\\p{Punct}+".r ^^ { s => Punctuations(s) }
	lazy val word: Parser[Word] = "[^\\s\\p{Punct}]+".r ^^ { s => Word(s, 1) }
	lazy val token: Parser[ImportantTextPart] = spacesParser1 ||| word ||| punctuation
	lazy val textParser: Parser[List[ImportantTextPart]] = rep(token)

        def wordOccurrenceMap(text: String): Seq[ImportantTextPart] = {
		val parts: List[ImportantTextPart] = parseAll(textParser, text) match {
			case Success(e, _) => 
				e
			case f: NoSuccess => 
				System.err.println(f)
				List[ImportantTextPart]()	
		}

		//the different words to the number of their occurrences
                val wordsOccurrences: Map[String, Int] =
			parts
			.filter{ 
				case w: Word => true 
				case _ => false
			}
			.groupBy(w => w.content.toLowerCase)
			.mapValues(_.size)

                parts.map { 
			case Word(content, _) =>
				val wNbOcc = 
					wordsOccurrences
					.getOrElse(content.toLowerCase, 0)
				Word(content, wNbOcc)
			case other => other
                }
        }




	def wordInfoDiv(w: Word, wordColor: Color): String = {
		val r = wordColor.getRed()
		val g = wordColor.getGreen()
		val b = wordColor.getBlue()
		val strRgb = s"""color:rgb($r,$g,$b)""" ;

		s"""<div class="wordInfo">""" +
			s"""<div style="$strRgb" class="wordDiv">""" +
				s"""${w.content}""" +
			s"""</div>""" +
			s"""<div class="nbOcc coolBorder">""" +
				s"""<div style="$strRgb">${w.nbOccur} occurrences</div>""" +
			s"""</div>""" +
		s"""</div>"""
	}

	def convertNewlinesToBr(ls: List[String]): String =  
		ls.foldLeft(""){ case (str,_) => str+"<br>" }
	def convertTabs(ls: List[String]): String =
		ls.foldLeft(""){ case(str,_) => str+"&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;" }
	def convertWhiteSpaces(ls: List[String]): String = 
		ls.foldLeft("") { case(str,_) => str + "&nbsp;" }
	//could do a parser for vertical tab but who cares about this character?	
	def convertFormFeed(ls: List[String]): String = 
		ls.foldLeft("") { case (str,_) => str + "<br><br>" }

	lazy val lf: Parser[String] = rep1("\n") ^^ { convertNewlinesToBr }
	lazy val crlf: Parser[String] = rep1("\r\n") ^^ { convertNewlinesToBr }
	lazy val cr: Parser[String] = rep1("\r") ^^ { convertNewlinesToBr } // for old macintoshs
	lazy val tabs: Parser[String] = rep1("\t") ^^ { convertTabs }
	lazy val whiteSpaces: Parser[String] = rep1(" ") ^^ { convertWhiteSpaces }
	lazy val formFeeds: Parser[String] = rep1("\f") ^^ { convertFormFeed }
	
	lazy val spacesParser2: Parser[String] = 
		rep1(lf ||| crlf ||| cr ||| tabs ||| whiteSpaces ||| formFeeds) ^^ {
			ls => ls.mkString
		}


	def spaceConverter(s: String): String = {
		parseAll(spacesParser2, s) match {
			case Success(e, rest) => 
				System.out.println("spaces:"+e)
				e
			case f: NoSuccess => 
				System.err.println(f)
				""
		}
	}

	/*this function builds the resulting coloured text from<commit_msg>remove all println for debugging<commit_after>package controllers


import util.parsing.combinator.RegexParsers
import scala.collection.mutable.Buffer
import javax.inject._
import play.api._
import play.api.mvc._
import play.twirl.api._
import play.api.libs.json._
import scala.io.Source
import java.awt.Color
import services.Logger


trait ImportantTextPart {
	def content: String
}
case class Word (val content: String, nbOccur: Int) extends ImportantTextPart 
case class Punctuations (val content: String) extends ImportantTextPart 
case class Spaces(val content: String) extends ImportantTextPart

case class InputText(text: String)


object ColorUtils {
	import java.lang.Math._

	val blueHue = 0.7f
	def greenToRedGiver(maxNbOccurs: Int)(nbOccur: Int): Color = 
		redToColderColorGiverLogarithmic(0.2f)(maxNbOccurs)(nbOccur)

	def blueToRedGiver(maxNbOccurs: Int)(nbOccur: Int): Color = 
		redToColderColorGiverLogarithmic(blueHue)(maxNbOccurs)(nbOccur)

	def redToColderColorGiverLogarithmic(colderColorHue: Float)(maxNbOccurs: Int)(nbOccur: Int): Color = {
		val k: Float = (colderColorHue / log(maxNbOccurs)).toFloat
		
		val h = {
			val formula = colderColorHue - k * log(nbOccur)
			if (formula < 0) 0
			else formula
		}.toFloat
		//finding the word color from the hue and arbitrary brightness and saturation parameter 
		
		Color.getHSBColor(h, 1f, .9f)
	}

	def redToColderColorGiverLinear(colderColorHue: Float)(nbOccur: Int, maxNbOccurs: Int): Color = {
		/*
		* the more occurrences there is, the redder the word becomes.
		* the words with the least occurrences are blue.
		*/
		val colderColorRatio = 
			(maxNbOccurs - (nbOccur-1)).toFloat / maxNbOccurs
		val h = colderColorRatio * colderColorHue
		//finding the word color from the hue and arbitrary brightness and saturation parameter 
		Color.getHSBColor(h, 1f, .9f)
	}


}

class TextParser extends RegexParsers {

	override def skipWhitespace = false 

	lazy val spacesParser1: Parser[Spaces] = "\\s+".r ^^ { s => Spaces(s) }
	lazy val punctuation: Parser[Punctuations] = "\\p{Punct}+".r ^^ { s => Punctuations(s) }
	lazy val word: Parser[Word] = "[^\\s\\p{Punct}]+".r ^^ { s => Word(s, 1) }
	lazy val token: Parser[ImportantTextPart] = spacesParser1 ||| word ||| punctuation
	lazy val textParser: Parser[List[ImportantTextPart]] = rep(token)

        def wordOccurrenceMap(text: String): Seq[ImportantTextPart] = {
		val parts: List[ImportantTextPart] = parseAll(textParser, text) match {
			case Success(e, _) => 
				e
			case f: NoSuccess => 
				System.err.println(f)
				List[ImportantTextPart]()	
		}

		//the different words to the number of their occurrences
                val wordsOccurrences: Map[String, Int] =
			parts
			.filter{ 
				case w: Word => true 
				case _ => false
			}
			.groupBy(w => w.content.toLowerCase)
			.mapValues(_.size)

                parts.map { 
			case Word(content, _) =>
				val wNbOcc = 
					wordsOccurrences
					.getOrElse(content.toLowerCase, 0)
				Word(content, wNbOcc)
			case other => other
                }
        }




	def wordInfoDiv(w: Word, wordColor: Color): String = {
		val r = wordColor.getRed()
		val g = wordColor.getGreen()
		val b = wordColor.getBlue()
		val strRgb = s"""color:rgb($r,$g,$b)""" ;

		s"""<div class="wordInfo">""" +
			s"""<div style="$strRgb" class="wordDiv">""" +
				s"""${w.content}""" +
			s"""</div>""" +
			s"""<div class="nbOcc coolBorder">""" +
				s"""<div style="$strRgb">${w.nbOccur} occurrences</div>""" +
			s"""</div>""" +
		s"""</div>"""
	}

	def convertNewlinesToBr(ls: List[String]): String =  
		ls.foldLeft(""){ case (str,_) => str+"<br>" }
	def convertTabs(ls: List[String]): String =
		ls.foldLeft(""){ case(str,_) => str+"&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;" }
	def convertWhiteSpaces(ls: List[String]): String = 
		ls.foldLeft("") { case(str,_) => str + "&nbsp;" }
	//could do a parser for vertical tab but who cares about this character?	
	def convertFormFeed(ls: List[String]): String = 
		ls.foldLeft("") { case (str,_) => str + "<br><br>" }

	lazy val lf: Parser[String] = rep1("\n") ^^ { convertNewlinesToBr }
	lazy val crlf: Parser[String] = rep1("\r\n") ^^ { convertNewlinesToBr }
	lazy val cr: Parser[String] = rep1("\r") ^^ { convertNewlinesToBr } // for old macintoshs
	lazy val tabs: Parser[String] = rep1("\t") ^^ { convertTabs }
	lazy val whiteSpaces: Parser[String] = rep1(" ") ^^ { convertWhiteSpaces }
	lazy val formFeeds: Parser[String] = rep1("\f") ^^ { convertFormFeed }
	
	lazy val spacesParser2: Parser[String] = 
		rep1(lf ||| crlf ||| cr ||| tabs ||| whiteSpaces ||| formFeeds) ^^ {
			ls => ls.mkString
		}


	def spaceConverter(s: String): String = {
		parseAll(spacesParser2, s) match {
			case Success(e, rest) => 
				e
			case f: NoSuccess => 
				System.err.println(f)
				""
		}
	}

	/*this function builds the resulting coloured text from
{'fraction_non_alphanumeric': 0.1462210796915167, 'fraction_numerical': 0.00493573264781491, 'mean_word_length': 3.8339960238568587, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1350352', 'n_tokens_mistral': 3814, 'n_tokens_neox': 3406, 'n_words': 834}