<commit_before>﻿#region License
/* **********************************************************************************
 * Copyright (c) <NAME>
 * This source code is subject to terms and conditions of the MIT License
 * for Irony. A copy of the license can be found in the License.txt file
 * at the root of this distribution. 
 * By using this source code in any fashion, you are agreeing to be bound by the terms of the 
 * MIT License.
 * You must not remove this notice from this software.
 * **********************************************************************************/
#endregion

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using Irony.Parsing;

namespace Irony.GrammarExplorer {
  public delegate void ColorizeMethod(); 
  public interface IUIThreadInvoker {
    void InvokeOnUIThread(ColorizeMethod colorize);
  }

  public class ColorizeEventArgs : EventArgs {
    public readonly TokenList Tokens; 
    public ColorizeEventArgs(TokenList tokens) {
      Tokens = tokens;
    }
  }

  //Container for two numbers representing visible range of the source text (min...max)
  // we use it to allow replacing two numbers in atomic operation
  public class ViewRange {
    public readonly int Min, Max;
    public ViewRange(int min, int max) {
      Min = min;
      Max = max; 
    }
    public bool Equals(ViewRange other) {
      return other.Min == Min && other.Max == Max; 
    }
  }

  public class ViewData {
    // ColoredTokens + NotColoredTokens == Source.Tokens
    public readonly TokenList ColoredTokens = new TokenList();
    public readonly TokenList NotColoredTokens = new TokenList(); //tokens not colored yet
    public ParseTree Tree;
    public ViewData(ParseTree tree) {
      this.Tree = tree;
      if (tree == null) return; 
      NotColoredTokens.AddRange(tree.Tokens); 
    }
  }

  //Two scenarios:
  // 1. Colorizing in current view range. We colorize only those tokens in current view range that were not colorized yet.
  //    For this we keep two lists (colorized and not colorized) tokens, and move tokens from one list to another when 
  //    we actually colorize them. 
  // 2. Typing/Editing - new editor content is being pushed from EditorAdapter. We try to avoid recoloring all visible tokens, when 
  //     user just typed a single char. What we do is try to identify "already-colored" tokens in new token list by matching 
  //     old viewData.ColoredTokens to newly scanned token list - initially in new-viewData.NonColoredTokens. If we find a "match", 
  //     we move the token from NonColored to Colored in new viewData. This all happens on background thread.

  public class EditorViewAdapterList : List<EditorViewAdapter> { }

  public class EditorViewAdapter {
    public readonly EditorAdapter Adapter;
    private IUIThreadInvoker _invoker;
    //public readonly Control Control;
    ViewData _data;
    ViewRange _range; 
    bool _wantsColorize;
    int _colorizing; 
    public event EventHandler<ColorizeEventArgs> ColorizeTokens; 

    public EditorViewAdapter(EditorAdapter adapter, IUIThreadInvoker invoker) {
      Adapter = adapter;
      _invoker = invoker;
      Adapter.AddView(this);
      _range = new ViewRange(-1, -1);
    }

    //SetViewRange and SetNewText are called by text box's event handlers to notify adapter that user did something edit box
    public void SetViewRange(int min, int max) {
      _range = new ViewRange(min, max);
      _wantsColorize = true; 
    }
    //The new text is passed directly to EditorAdapter instance (possibly shared by several view adapters).
    // EditorAdapter parses the text on a separate background thread, and notifies back this and other 
    // view adapters and provides them with newly parsed source through UpdateParsedSource method (see below) 
    public void SetNewText(string newText) {
      //TODO: fix this
      //hack, temp solution for more general problem
      //When we load/replace/clear entire text, clear out colored tokens to force recoloring from scratch 
      if (string.IsNullOrEmpty(newText))
        _data = null;
      Adapter.SetNewText(newText);
    }

    //Called by EditorAdapter to provide the latest parsed source 
    public void UpdateParsedSource(ParseTree newTree) {
      lock (this) {
        var oldData = _data;
        _data = new ViewData(newTree);
        //Now try to figure out tokens that match old Colored tokens
        if (oldData != null) {
          DetectAlreadyColoredTokens(oldData.ColoredTokens, _data.Tree.SourceText.Length - oldData.Tree.SourceText.Length);<commit_msg>Minor fix in EditorViewAdapter to prevent occasional crush<commit_after>﻿#region License
/* **********************************************************************************
 * Copyright (c) <NAME>
 * This source code is subject to terms and conditions of the MIT License
 * for Irony. A copy of the license can be found in the License.txt file
 * at the root of this distribution. 
 * By using this source code in any fashion, you are agreeing to be bound by the terms of the 
 * MIT License.
 * You must not remove this notice from this software.
 * **********************************************************************************/
#endregion

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using Irony.Parsing;

namespace Irony.GrammarExplorer {
  public delegate void ColorizeMethod(); 
  public interface IUIThreadInvoker {
    void InvokeOnUIThread(ColorizeMethod colorize);
  }

  public class ColorizeEventArgs : EventArgs {
    public readonly TokenList Tokens; 
    public ColorizeEventArgs(TokenList tokens) {
      Tokens = tokens;
    }
  }

  //Container for two numbers representing visible range of the source text (min...max)
  // we use it to allow replacing two numbers in atomic operation
  public class ViewRange {
    public readonly int Min, Max;
    public ViewRange(int min, int max) {
      Min = min;
      Max = max; 
    }
    public bool Equals(ViewRange other) {
      return other.Min == Min && other.Max == Max; 
    }
  }

  public class ViewData {
    // ColoredTokens + NotColoredTokens == Source.Tokens
    public readonly TokenList ColoredTokens = new TokenList();
    public readonly TokenList NotColoredTokens = new TokenList(); //tokens not colored yet
    public ParseTree Tree;
    public ViewData(ParseTree tree) {
      this.Tree = tree;
      if (tree == null) return; 
      NotColoredTokens.AddRange(tree.Tokens); 
    }
  }

  //Two scenarios:
  // 1. Colorizing in current view range. We colorize only those tokens in current view range that were not colorized yet.
  //    For this we keep two lists (colorized and not colorized) tokens, and move tokens from one list to another when 
  //    we actually colorize them. 
  // 2. Typing/Editing - new editor content is being pushed from EditorAdapter. We try to avoid recoloring all visible tokens, when 
  //     user just typed a single char. What we do is try to identify "already-colored" tokens in new token list by matching 
  //     old viewData.ColoredTokens to newly scanned token list - initially in new-viewData.NonColoredTokens. If we find a "match", 
  //     we move the token from NonColored to Colored in new viewData. This all happens on background thread.

  public class EditorViewAdapterList : List<EditorViewAdapter> { }

  public class EditorViewAdapter {
    public readonly EditorAdapter Adapter;
    private IUIThreadInvoker _invoker;
    //public readonly Control Control;
    ViewData _data;
    ViewRange _range; 
    bool _wantsColorize;
    int _colorizing; 
    public event EventHandler<ColorizeEventArgs> ColorizeTokens; 

    public EditorViewAdapter(EditorAdapter adapter, IUIThreadInvoker invoker) {
      Adapter = adapter;
      _invoker = invoker;
      Adapter.AddView(this);
      _range = new ViewRange(-1, -1);
    }

    //SetViewRange and SetNewText are called by text box's event handlers to notify adapter that user did something edit box
    public void SetViewRange(int min, int max) {
      _range = new ViewRange(min, max);
      _wantsColorize = true; 
    }
    //The new text is passed directly to EditorAdapter instance (possibly shared by several view adapters).
    // EditorAdapter parses the text on a separate background thread, and notifies back this and other 
    // view adapters and provides them with newly parsed source through UpdateParsedSource method (see below) 
    public void SetNewText(string newText) {
      //TODO: fix this
      //hack, temp solution for more general problem
      //When we load/replace/clear entire text, clear out colored tokens to force recoloring from scratch 
      if (string.IsNullOrEmpty(newText))
        _data = null;
      Adapter.SetNewText(newText);
    }

    //Called by EditorAdapter to provide the latest parsed source 
    public void UpdateParsedSource(ParseTree newTree) {
      lock (this) {
        var oldData = _data;
        _data = new ViewData(newTree);
        //Now try to figure out tokens that match old Colored tokens
        if (oldData != null && oldData.Tree != null) {
          DetectAlreadyColoredTokens(oldData.ColoredTokens, _data.Tree.SourceText.Length - oldData.Tree.SourceText.Length);
{'fraction_non_alphanumeric': 0.09953827982390208, 'fraction_numerical': 0.0008590142811124234, 'mean_word_length': 3.2941447671738127, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '822752', 'n_tokens_mistral': 2488, 'n_tokens_neox': 2401, 'n_words': 1004}