<commit_before>/*  Title:      Pure/PIDE/state.scala
    Author:     <NAME>, TU Munich
    Author:     Makarius

Accumulating results from prover.
*/

package isabelle


class State(
  val command: Command,
  val status: Command.Status.Value,
  val reverse_results: List[XML.Tree],
  val markup_root: Markup_Text)
{
  def this(command: Command) =
    this(command, Command.Status.UNPROCESSED, Nil, command.empty_markup)


  /* content */

  private def set_status(st: Command.Status.Value): State =
    new State(command, st, reverse_results, markup_root)

  private def add_result(res: XML.Tree): State =
    new State(command, status, res :: reverse_results, markup_root)

  private def add_markup(node: Markup_Tree): State =
    new State(command, status, reverse_results, markup_root + node)

  lazy val results = reverse_results.reverse


  /* markup */

  lazy val highlight: Markup_Text =
  {
    markup_root.filter(_.info match {
      case Command.HighlightInfo(_) => true
      case _ => false
    })
  }

  private lazy val types: List[Markup_Node] =
    markup_root.filter(_.info match {
      case Command.TypeInfo(_) => true
      case _ => false }).flatten

  def type_at(pos: Int): Option[String] =
  {
    types.find(t => t.start <= pos && pos < t.stop) match {
      case Some(t) =>
        t.info match {
          case Command.TypeInfo(ty) => Some(command.source(t.start, t.stop) + ": " + ty)
          case _ => None
        }
      case None => None
    }
  }

  private lazy val refs: List[Markup_Node] =
    markup_root.filter(_.info match {
      case Command.RefInfo(_, _, _, _) => true
      case _ => false }).flatten

  def ref_at(pos: Int): Option[Markup_Node] =
    refs.find(t => t.start <= pos && pos < t.stop)


  /* message dispatch */

  def + (message: XML.Tree): State =
    message match {
      case XML.Elem(Markup.STATUS, _, elems) =>
        (this /: elems)((state, elem) =>
          elem match {
            case XML.Elem(Markup.UNPROCESSED, _, _) => state.set_status(Command.Status.UNPROCESSED)
            case XML.Elem(Markup.FINISHED, _, _) => state.set_status(Command.Status.FINISHED)
            case XML.Elem(Markup.FAILED, _, _) => state.set_status(Command.Status.FAILED)
            case XML.Elem(kind, atts, body) =>
              atts match {
                case Position.Range(begin, end) =>
                  if (kind == Markup.ML_TYPING) {
                    val info = Pretty.string_of(body, margin = 40)
                    state.add_markup(
                      command.markup_node(begin - 1, end - 1, Command.TypeInfo(info)))
                  }
                  else if (kind == Markup.ML_REF) {
                    body match {
                      case List(XML.Elem(Markup.ML_DEF, atts, _)) =>
                        state.add_markup(command.markup_node(
                          begin - 1, end - 1,
                          Command.RefInfo(
                            Position.get_file(atts),
                            Position.get_line(atts),
                            Position.get_id(atts),
                            Position.get_offset(atts))))
                      case _ => state
                    }
                  }
                  else {
                    state.add_markup(
                      command.markup_node(begin - 1, end - 1, Command.HighlightInfo(kind)))
                  }
                case _ => state
              }
            case _ =>
              System.err.println("ignored status report: " + elem)
              state
          })
      case _ => add_result(message)
    }
}<commit_msg>slightly odd workaround to ignore markup that is typically displaced;<commit_after>/*  Title:      Pure/PIDE/state.scala
    Author:     <NAME>, TU Munich
    Author:     Makarius

Accumulating results from prover.
*/

package isabelle


class State(
  val command: Command,
  val status: Command.Status.Value,
  val reverse_results: List[XML.Tree],
  val markup_root: Markup_Text)
{
  def this(command: Command) =
    this(command, Command.Status.UNPROCESSED, Nil, command.empty_markup)


  /* content */

  private def set_status(st: Command.Status.Value): State =
    new State(command, st, reverse_results, markup_root)

  private def add_result(res: XML.Tree): State =
    new State(command, status, res :: reverse_results, markup_root)

  private def add_markup(node: Markup_Tree): State =
    new State(command, status, reverse_results, markup_root + node)

  lazy val results = reverse_results.reverse


  /* markup */

  lazy val highlight: Markup_Text =
  {
    markup_root.filter(_.info match {
      case Command.HighlightInfo(_) => true
      case _ => false
    })
  }

  private lazy val types: List[Markup_Node] =
    markup_root.filter(_.info match {
      case Command.TypeInfo(_) => true
      case _ => false }).flatten

  def type_at(pos: Int): Option[String] =
  {
    types.find(t => t.start <= pos && pos < t.stop) match {
      case Some(t) =>
        t.info match {
          case Command.TypeInfo(ty) => Some(command.source(t.start, t.stop) + ": " + ty)
          case _ => None
        }
      case None => None
    }
  }

  private lazy val refs: List[Markup_Node] =
    markup_root.filter(_.info match {
      case Command.RefInfo(_, _, _, _) => true
      case _ => false }).flatten

  def ref_at(pos: Int): Option[Markup_Node] =
    refs.find(t => t.start <= pos && pos < t.stop)


  /* message dispatch */

  def + (message: XML.Tree): State =
    message match {
      case XML.Elem(Markup.STATUS, _, elems) =>
        (this /: elems)((state, elem) =>
          elem match {
            case XML.Elem(Markup.UNPROCESSED, _, _) => state.set_status(Command.Status.UNPROCESSED)
            case XML.Elem(Markup.FINISHED, _, _) => state.set_status(Command.Status.FINISHED)
            case XML.Elem(Markup.FAILED, _, _) => state.set_status(Command.Status.FAILED)
            case XML.Elem(kind, atts, body) =>
              atts match {
                case Position.Range(begin, end) =>
                  if (kind == Markup.ML_TYPING) {
                    val info = Pretty.string_of(body, margin = 40)
                    state.add_markup(
                      command.markup_node(begin - 1, end - 1, Command.TypeInfo(info)))
                  }
                  else if (kind == Markup.ML_REF) {
                    body match {
                      case List(XML.Elem(Markup.ML_DEF, atts, _)) =>
                        state.add_markup(command.markup_node(
                          begin - 1, end - 1,
                          Command.RefInfo(
                            Position.get_file(atts),
                            Position.get_line(atts),
                            Position.get_id(atts),
                            Position.get_offset(atts))))
                      case _ => state
                    }
                  }
                  else if (kind == Markup.FACT_DECL || kind == Markup.LOCAL_FACT_DECL ||
                      kind == Markup.ATTRIBUTE || kind == Markup.FIXED_DECL) {
                    // FIXME usually displaced due to lack of full history support
                    state
                  }
                  else {
                    state.add_markup(
                      command.markup_node(begin - 1, end - 1, Command.HighlightInfo(kind)))
                  }
                case _ => state
              }
            case _ =>
              System.err.println("ignored status report: " + elem)
              state
          })
      case _ => add_result(message)
    }
}
{'fraction_non_alphanumeric': 0.11207920792079208, 'fraction_numerical': 0.002112211221122112, 'mean_word_length': 1.7028184088476632, 'pattern_counts': {'":': 2, '<': 13, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4195487', 'n_tokens_mistral': 2291, 'n_tokens_neox': 2184, 'n_words': 490}