<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Diagnostics;

namespace VBF.Compilers.Scanners
{
    class ForkNode
    {
        private Scanner m_masterScanner;
        private CacheQueue<Lexeme> m_lookAheadQueue;

        private ForkNode m_parent;
        private List<ForkNode> m_activeChildren;

        private int m_offset;
        private int TokenIndex
        {
            get
            {
                if (m_parent == null)
                {
                    return m_offset;
                }

                return m_parent.TokenIndex + m_offset;
            }
        }

        private ForkNode(ForkNode parent)
        {
            Debug.Assert(parent != null);
            m_parent = parent;
            m_masterScanner = parent.m_masterScanner;
            m_lookAheadQueue = parent.m_lookAheadQueue;

            m_activeChildren = new List<ForkNode>();
        }

        public ForkNode CreateChildNode()
        {
            ForkNode child = new ForkNode(this);
            m_activeChildren.Add(child);

            return child;
        }

        private bool IsClosed
        {
            get
            {
                return m_masterScanner != null;
            }
        }

        private bool HasForked
        {
            get
            {
                return m_activeChildren.Count > 0;
            }
        }

        public void Close()
        {
            //step 1: remove scanner references
            m_masterScanner = null;
            m_lookAheadQueue = null;

            //step 2: check 
            
        }
    }
}<commit_msg>Implements ForkNode.cs, part 1.<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Diagnostics;

namespace VBF.Compilers.Scanners
{
    class ForkNode1
    {
        private Scanner m_masterScanner;
        private CacheQueue<Lexeme> m_lookAheadQueue;

        private ForkNode1 m_parent;
        private List<ForkNode1> m_activeChildren;

        private int m_offset;
        private int TokenIndex
        {
            get
            {
                if (m_parent == null)
                {
                    return m_offset;
                }

                return m_parent.TokenIndex + m_offset;
            }
        }

        private ForkNode1(ForkNode1 parent)
        {
            Debug.Assert(parent != null);
            m_parent = parent;
            m_masterScanner = parent.m_masterScanner;
            m_lookAheadQueue = parent.m_lookAheadQueue;

            m_activeChildren = new List<ForkNode1>();
        }

        public ForkNode1 CreateChildNode()
        {
            ForkNode1 child = new ForkNode1(this);
            m_activeChildren.Add(child);

            return child;
        }

        private bool IsClosed
        {
            get
            {
                return m_masterScanner != null;
            }
        }

        private bool HasForked
        {
            get
            {
                return m_activeChildren.Count > 0;
            }
        }

        public void Close()
        {
            //step 1: remove scanner references
            m_masterScanner = null;
            m_lookAheadQueue = null;

            //step 2: check 

        }
    }

    class ForkNode
    {
        public Scanner MasterScanner { get; set; }
        public CacheQueue<Lexeme> LookAheadQueue { get; private set; }
        public List<ForkNode> Children { get; private set; }
        public ForkNode Parent { get; private set; }
        public int Offset { get; set; }

        private NodeState m_state;

        public ForkNode(ForkNode parent)
        {
            if (parent == null)
            {
                LookAheadQueue = new CacheQueue<Lexeme>();
            }
            else
            {
                LookAheadQueue = parent.LookAheadQueue;
                MasterScanner = parent.MasterScanner;
            }

            Children = new List<ForkNode>();
            Parent = parent;
        }

        public int Position
        {
            get
            {
                if (Parent == null)
                {
                    return Offset;
                }

                return Parent.Position + Offset;
            }
        }


    }

    abstract class NodeState
    {
        protected ForkNode Node { get; private set; }
        protected NodeState(ForkNode node)
        {
            Node = node;
        }

        public abstract ForkNode[] Fork(int count);
        public abstract void Close();
        public abstract Lexeme Read();
    }

    class TailState
    {
        void Forward(ForkNode child) { }
    }

    class HeadState : NodeState
    {
        public HeadState(ForkNode node) : base(node) { }


        public override ForkNode[] Fork(int count)
        {
            throw new NotImplementedException();
        }

        public override void Close()
        {
            throw new NotImplementedException();
        }

        public override Lexeme Read()
        {
            throw new NotImplementedException();
        }
    }

    class BranchState
    {

    }

    class TailHeadState
    {

    }
}
{'fraction_non_alphanumeric': 0.06820346732710993, 'fraction_numerical': 0.0030481996570775388, 'mean_word_length': 1.2816166883963493, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4254292', 'n_tokens_mistral': 1438, 'n_tokens_neox': 1354, 'n_words': 320}