<commit_before>﻿using N2.Internal;

using Nemerle.Collections;

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

using RecoveryStack = Nemerle.Core.list<N2.Internal.RecoveryStackFrame>.Cons;
using System.Diagnostics;

#if N2RUNTIME
namespace N2.Strategies
#else
namespace N2.DebugStrategies
#endif
{
  using PrseData = Tuple<int, int, List<ParsedStateInfo>>;

  public sealed class Recovery
  {
    List<RecoveryResult> _candidats = new List<RecoveryResult>();
    RecoveryResult       _bestResult;
    List<RecoveryResult> _bestResults = new List<RecoveryResult>();
    int                  _parseCount;
    int                  _recCount;
    int                  _bestResultsCount;
    int                  _nestedLevel;
    Dictionary<int, int> _allacetionsInfo = new Dictionary<int, int>();
    Dictionary<object, PrseData> _visited = new Dictionary<object, PrseData>();
    Dictionary<string, int> _parsedRules = new Dictionary<string, int>();
    RecoveryStack        _recoveryStack;

    void Reset()
    {
      _candidats.Clear();
      _bestResult = null;
      _bestResults.Clear();
      _parseCount = 0;
      _recCount = 0;
      _bestResultsCount = 0;
      _nestedLevel = 0;
      _allacetionsInfo.Clear();
      _visited = new Dictionary<object, PrseData>();
      _parsedRules = new Dictionary<string, int>();
    }

    public void Strategy(int startTextPos, Parser parser)
    {
      Reset();
      var maxFailPos = parser.MaxFailPos;
      var timer = System.Diagnostics.Stopwatch.StartNew();
      _recoveryStack     = parser.RecoveryStack.NToList() as RecoveryStack;
      var curTextPos    = startTextPos;
      var text          = parser.Text;

      parser.ParsingMode = ParsingMode.Parsing;

      var before = parser.Text.Substring(0, startTextPos);

      if (before == "[\r\n  { : 1},\r\n  { a },\r\n  { a: },\r\n  { a:, },\r\n  { \r\n  'a':, \r\n  a:1\r\n  },\r\n  {a::2,:},\r\n  {a# :1}, \r\n  {a")
      {
        Debug.Assert(true);
      }
        
      do
      {
        //for (var stack = _recoveryStack; stack != null; stack = stack.Tail as RecoveryStack)
        //  ProcessStackFrame(startTextPos, parser, stack, curTextPos, text, 0);
        ProcessStackFrame(startTextPos, parser, _recoveryStack, curTextPos, text, 0);
        if (_recoveryStack.hd.RuleParser.IsTokenRule)
          break;
        curTextPos++;
      }
      while (curTextPos - startTextPos < 800 && /*_bestResult == null && _bestResult == null && (res.Count == 0 || curTextPos - startTextPos < 10) &&*/ curTextPos <= text.Length);

      timer.Stop();

      if (_bestResult != null)
      {
        FixAst(parser);
        parser.ParsingMode = ParsingMode.EndRecovery;
        parser.MaxFailPos = _bestResult.EndPos; // HACK!!!
      }
      else
      {
        parser.ParsingMode = ParsingMode.Recovery;
        parser.MaxFailPos = maxFailPos;
      }

      Reset();
    }

    private void ProcessStackFrame(int startTextPos, Parser parser, RecoveryStack recoveryStack, int curTextPos, string text, int subruleLevel)
    {
      var stackFrame = recoveryStack.hd;
      var ruleParser = stackFrame.RuleParser;
      var isPrefixParsed = !ruleParser.IsStartState(stackFrame.FailState);
      var isOptional = ruleParser.IsLoopSeparatorStart(stackFrame.FailState);
      List<ParsedStateInfo> parsedStates;

      int nextState;
      for (var state = stackFrame.FailState; state >= 0; state = nextState) //subruleLevel > 0 ? ruleParser.GetNextState(stackFrame.FailState) : 
      {
        parser.MaxFailPos = startTextPos;
        nextState = ruleParser.GetNextState(state);
        //var needSkip = nextState < 0 && ruleParser.IsVoidState(state);
        //if (nextState < 0 && ruleParser.IsVoidState(state))
        //  continue;
        int pos = TryParse(parser, recoveryStack, curTextPos, ruleParser, state, out parsedStates);<commit_msg>Work on recovery/ Support multi-stack.<commit_after>﻿using N2.Internal;

using Nemerle.Collections;

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

using RecoveryStack = Nemerle.Core.list<N2.Internal.RecoveryStackFrame>.Cons;
using System.Diagnostics;

#if N2RUNTIME
namespace N2.Strategies
#else
namespace N2.DebugStrategies
#endif
{
  using PrseData = Tuple<int, int, List<ParsedStateInfo>>;

  public sealed class Recovery
  {
    List<RecoveryResult> _candidats = new List<RecoveryResult>();
    RecoveryResult       _bestResult;
    List<RecoveryResult> _bestResults = new List<RecoveryResult>();
    int                  _parseCount;
    int                  _recCount;
    int                  _bestResultsCount;
    int                  _nestedLevel;
    Dictionary<int, int> _allacetionsInfo = new Dictionary<int, int>();
    Dictionary<object, PrseData> _visited = new Dictionary<object, PrseData>();
    Dictionary<string, int> _parsedRules = new Dictionary<string, int>();
    RecoveryStack        _recoveryStack;

    void Reset()
    {
      _candidats.Clear();
      _bestResult = null;
      _bestResults.Clear();
      _parseCount = 0;
      _recCount = 0;
      _bestResultsCount = 0;
      _nestedLevel = 0;
      _allacetionsInfo.Clear();
      _visited = new Dictionary<object, PrseData>();
      _parsedRules = new Dictionary<string, int>();
    }

    public void Strategy(int startTextPos, Parser parser)
    {
      Reset();
      var maxFailPos = parser.MaxFailPos;
      var timer = System.Diagnostics.Stopwatch.StartNew();
      var curTextPos = startTextPos;
      var text = parser.Text;
      var stacks = PrepareStacks(parser);

      //var infos = parser.ParserHost.Reflection(parser, startTextPos);
      //foreach (var info in infos)
      //{
      //  if (info.AstPointer > 0)
      //  {
      //    var state = parser.ast[info.AstPointer + 2];
      //    var size1 = parser.ast[info.AstPointer + 3];
      //    var size2 = parser.ast[info.AstPointer + 4];
      //  }
      //  Debug.WriteLine(info.ToString());
      //}
      parser.ParsingMode = ParsingMode.Parsing;

      do
      {
        foreach (var stack in stacks)
        {
          _recoveryStack = stack as RecoveryStack;

          var before = parser.Text.Substring(0, startTextPos); // DEBUG
          if (before == "[\r\n  { : 1},\r\n  { a },\r\n  { a: },\r\n  { a:, },\r\n  { \r\n  'a':, \r\n  a:1\r\n  },\r\n  {a::2,:},\r\n  {a# :1}, \r\n  {a") // DEBUG
          {
            Debug.Assert(true);
          }

          ProcessStackFrame(startTextPos, parser, _recoveryStack, curTextPos, text, 0);
        }
        // Здес нужно прерывать цикл, если _bestResult != null. 
        // Но при этом _bestResult не должен содержать вариантов где в _bestResult.Recovered находятся только пробелы (void-правила).
        curTextPos++;
      }
      while (curTextPos <= text.Length); // curTextPos - startTextPos < 800 && 

      timer.Stop();

      if (_bestResult != null)
      {
        FixAst(parser);
        parser.ParsingMode = ParsingMode.EndRecovery;
        parser.MaxFailPos = _bestResult.EndPos;
      }
      else
      {
        // Этого вхождения быть не должно. Если мы не вычислили продолжение, значит нужно записывать весь хвост в грязь.
        parser.ParsingMode = ParsingMode.Recovery;
        parser.MaxFailPos = maxFailPos;
      }

      Reset();
    }

    private static List<RecoveryStack> PrepareStacks(Parser parser)
    {
      var stacks = new List<RecoveryStack>();

      foreach (var stack in parser.RecoveryStacks)
        UpdateStacks(stacks, stack as RecoveryStack);

      return stacks;
    }

    private static void UpdateStacks(List<RecoveryStack> stacks, RecoveryStack stack)
    {
      var index = stacks.FindIndex(s => IsSubStack(stack, s)); // is nStack SubStack of s
      if (index >= 0)
        stacks[index] = stack;
      else if (stacks.FindIndex(s => IsSubStack(s, stack)) < 0) // is new stack?
        stacks.Add(stack);
      // else -> better stack in list
    }

    private static bool IsSubStack(RecoveryStack stack1, RecoveryStack stack2)
    {
      if (stack2.Length > stack1.Length)
        return false;

      var ary1 = stack1.ToArray();
      var ary2 = stack2.ToArray();

      Array.Reverse(ary1);
      Array.Reverse(ary2);

      for (int i = 0; i < ary2.Length; i++)
      {
        var head1 = ary1[i];
        var head2 = ary2[i];
        if (head1.RuleParser != head2.RuleParser || head1.FailState != head2.FailState)
          return false;
      }

      return true;
    }

    private void ProcessStackFrame(int startTextPos, Parser parser, RecoveryStack recoveryStack, int curTextPos, string text, int subruleLevel)
    {
      var stackFrame = recoveryStack.hd;
      var ruleParser = stackFrame.RuleParser;
      var isPrefixParsed = !ruleParser.IsStartState(stackFrame.FailState);
      var isOptional = ruleParser.IsLoopSeparatorStart(stackFrame.FailState);
      List<ParsedStateInfo> parsedStates;

      int nextState;
      for (var state = stackFrame.FailState; state >= 0; state = nextState) //subruleLevel > 0 ? ruleParser.GetNextState(stackFrame.FailState) : 
      {
        parser.MaxFailPos = startTextPos;
        nextState = ruleParser.GetNextState(state);
        //var needSkip = nextState < 0 && ruleParser.IsVoidState(state);
        //if (nextState < 0 && ruleParser.IsVoidState(state))
        //  continue;
        int pos = TryParse(parser, recoveryStack, curTextPos, ruleParser, state, out parsedStates);
{'fraction_non_alphanumeric': 0.11288447310009513, 'fraction_numerical': 0.008032977486523624, 'mean_word_length': 2.5358744394618835, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '780937', 'n_tokens_mistral': 3019, 'n_tokens_neox': 2946, 'n_words': 681}