<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Windows;
using System.Windows.Media;
using CommonMark;
using CommonMark.Syntax;
using ICSharpCode.AvalonEdit.Document;
using ICSharpCode.AvalonEdit.Rendering;

namespace MarkdownEdit.Models
{
    public class MarkdownHighlightingColorizer : DocumentColorizingTransformer
    {
        private readonly CommonMarkSettings _commonMarkSettings;
        private Block _abstractSyntaxTree;
        private Theme _theme;

        public MarkdownHighlightingColorizer()
        {
            _commonMarkSettings = CommonMarkSettings.Default.Clone();
            _commonMarkSettings.TrackSourcePosition = true;
        }

        private static readonly Dictionary<BlockTag, Func<Theme, Highlight>> BlockHighlighter = new Dictionary<BlockTag, Func<Theme, Highlight>>
        {
            {BlockTag.AtxHeader, t => t.HighlightHeading},
            {BlockTag.SETextHeader, t => t.HighlightHeading},
            {BlockTag.BlockQuote, t => t.HighlightBlockQuote},
            {BlockTag.ListItem, t => t.HighlightStrongEmphasis},
            {BlockTag.FencedCode, t => t.HighlightBlockCode},
            {BlockTag.IndentedCode, t => t.HighlightBlockCode},
            {BlockTag.HtmlBlock, t => t.HighlightBlockCode},
            {BlockTag.ReferenceDefinition, t => t.HighlightLink}
        };

        private static readonly Dictionary<InlineTag, Func<Theme, Highlight>> InlineHighlighter = new Dictionary<InlineTag, Func<Theme, Highlight>>
        {
            {InlineTag.Code, t => t.HighlightInlineCode},
            {InlineTag.Emphasis, t => t.HighlightEmphasis},
            {InlineTag.Strong, t => t.HighlightStrongEmphasis},
            {InlineTag.Link, t => t.HighlightLink},
            {InlineTag.Image, t => t.HighlightImage},
            {InlineTag.RawHtml, t => t.HighlightBlockCode}
        };

        protected override void ColorizeLine(DocumentLine line)
        {
            var ast = _abstractSyntaxTree;
            if (ast == null) return;

            var theme = _theme;
            if (theme == null) return;

            var start = line.Offset;
            var end = line.EndOffset;
            var leadingSpaces = CurrentContext.GetText(start, end - start).Text.TakeWhile(char.IsWhiteSpace).Count();
            Func<Theme, Highlight> highlighter;

            foreach (var block in EnumerateSpanningBlocks(ast, start, end))
            {
                if (BlockHighlighter.TryGetValue(block.Tag, out highlighter))
                {
                    var magnify = double.NaN;
                    if (block.HeaderLevel == 1) magnify = theme.Header1Height;
                    if (block.HeaderLevel == 2) magnify = theme.Header2Height;

                    var length = (block.Tag == BlockTag.ListItem)
                        ? Math.Min(block.SourceLength, block.ListData.Padding)
                        : block.SourceLength;

                    ApplyLinePart(highlighter(theme), block.SourcePosition, length, start, end, leadingSpaces, magnify);
                }

                foreach (var inline in EnumerateInlines(block.InlineContent)
                    .TakeWhile(il => il.SourcePosition < end)
                    .Where(il => InlineHighlighter.TryGetValue(il.Tag, out highlighter)))
                {
                    var position = inline.SourcePosition;
                    var length = inline.SourceLength;

                    if ((inline.Tag == InlineTag.Link || inline.Tag == InlineTag.Image)
                        && inline.FirstChild?.LiteralContent != null
                        && inline.FirstChild.LiteralContent != inline.TargetUrl)
                    {
                        var literal = inline.FirstChild.LastSibling;
                        var urlPosition = literal.SourcePosition + literal.SourceLength + 1;
                        var urlLength = inline.SourcePosition + inline.SourceLength - urlPosition;
                        if (urlLength > 0) // check for <name@domain.ext> style links
                        {
                            position = urlPosition;
                            length = urlLength;
                        }
                    }

                    // inlines don't magnify
                    ApplyLinePart(highlighter(theme), position, length, start, end, leadingSpaces, double.NaN);
                }
            }
        }

        private static IEnumerable<Block> EnumerateSpanningBlocks(Block ast, int startOffset, int endOffset)
        {
            return EnumerateBlocks(ast.FirstChild)
                .Where(b => (b.SourcePosition + b.SourceLength) >= startOffset)
                .TakeWhile(b => b.SourcePosition < endOffset);
        }

        private static IEnumerable<Block> EnumerateBlocks(Block block)
        {
            if (block == null) yield break;
            var stack = new Stack<Block>();
            stack.Push(block);
            while (stack.Any())
            {
                var next = stack.Pop();
                yield return next;
                if (next.NextSibling != null) stack.Push(next.NextSibling);
                if (next.FirstChild != null) stack.Push(next.FirstChild);
            }
        }

        private static IEnumerable<Inline> EnumerateInlines(Inline inline)
        {
            if (inline == null) yield break;
            var stack = new Stack<Inline>();
            stack.Push(inline);
            while (stack.Any())
            {<commit_msg>set spanning blocks back to 1 based offset<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Windows;
using System.Windows.Media;
using CommonMark;
using CommonMark.Syntax;
using ICSharpCode.AvalonEdit.Document;
using ICSharpCode.AvalonEdit.Rendering;

namespace MarkdownEdit.Models
{
    public class MarkdownHighlightingColorizer : DocumentColorizingTransformer
    {
        private readonly CommonMarkSettings _commonMarkSettings;
        private Block _abstractSyntaxTree;
        private Theme _theme;

        public MarkdownHighlightingColorizer()
        {
            _commonMarkSettings = CommonMarkSettings.Default.Clone();
            _commonMarkSettings.TrackSourcePosition = true;
        }

        private static readonly Dictionary<BlockTag, Func<Theme, Highlight>> BlockHighlighter = new Dictionary<BlockTag, Func<Theme, Highlight>>
        {
            {BlockTag.AtxHeader, t => t.HighlightHeading},
            {BlockTag.SETextHeader, t => t.HighlightHeading},
            {BlockTag.BlockQuote, t => t.HighlightBlockQuote},
            {BlockTag.ListItem, t => t.HighlightStrongEmphasis},
            {BlockTag.FencedCode, t => t.HighlightBlockCode},
            {BlockTag.IndentedCode, t => t.HighlightBlockCode},
            {BlockTag.HtmlBlock, t => t.HighlightBlockCode},
            {BlockTag.ReferenceDefinition, t => t.HighlightLink}
        };

        private static readonly Dictionary<InlineTag, Func<Theme, Highlight>> InlineHighlighter = new Dictionary<InlineTag, Func<Theme, Highlight>>
        {
            {InlineTag.Code, t => t.HighlightInlineCode},
            {InlineTag.Emphasis, t => t.HighlightEmphasis},
            {InlineTag.Strong, t => t.HighlightStrongEmphasis},
            {InlineTag.Link, t => t.HighlightLink},
            {InlineTag.Image, t => t.HighlightImage},
            {InlineTag.RawHtml, t => t.HighlightBlockCode}
        };

        protected override void ColorizeLine(DocumentLine line)
        {
            var ast = _abstractSyntaxTree;
            if (ast == null) return;

            var theme = _theme;
            if (theme == null) return;

            var start = line.Offset;
            var end = line.EndOffset;
            var leadingSpaces = CurrentContext.GetText(start, end - start).Text.TakeWhile(char.IsWhiteSpace).Count();
            Func<Theme, Highlight> highlighter;

            foreach (var block in EnumerateSpanningBlocks(ast, start, end))
            {
                if (BlockHighlighter.TryGetValue(block.Tag, out highlighter))
                {
                    var magnify = double.NaN;
                    if (block.HeaderLevel == 1) magnify = theme.Header1Height;
                    if (block.HeaderLevel == 2) magnify = theme.Header2Height;

                    var length = (block.Tag == BlockTag.ListItem)
                        ? Math.Min(block.SourceLength, block.ListData.Padding)
                        : block.SourceLength;

                    ApplyLinePart(highlighter(theme), block.SourcePosition, length, start, end, leadingSpaces, magnify);
                }

                foreach (var inline in EnumerateInlines(block.InlineContent)
                    .TakeWhile(il => il.SourcePosition < end)
                    .Where(il => InlineHighlighter.TryGetValue(il.Tag, out highlighter)))
                {
                    var position = inline.SourcePosition;
                    var length = inline.SourceLength;

                    if ((inline.Tag == InlineTag.Link || inline.Tag == InlineTag.Image)
                        && inline.FirstChild?.LiteralContent != null
                        && inline.FirstChild.LiteralContent != inline.TargetUrl)
                    {
                        var literal = inline.FirstChild.LastSibling;
                        var urlPosition = literal.SourcePosition + literal.SourceLength + 1;
                        var urlLength = inline.SourcePosition + inline.SourceLength - urlPosition;
                        if (urlLength > 0) // check for <name<EMAIL>> style links
                        {
                            position = urlPosition;
                            length = urlLength;
                        }
                    }

                    // inlines don't magnify
                    ApplyLinePart(highlighter(theme), position, length, start, end, leadingSpaces, double.NaN);
                }
            }
        }

        private static IEnumerable<Block> EnumerateSpanningBlocks(Block ast, int startOffset, int endOffset)
        {
            return EnumerateBlocks(ast.FirstChild)
                .Where(b => (b.SourcePosition + b.SourceLength) > startOffset)
                .TakeWhile(b => b.SourcePosition < endOffset);
        }

        private static IEnumerable<Block> EnumerateBlocks(Block block)
        {
            if (block == null) yield break;
            var stack = new Stack<Block>();
            stack.Push(block);
            while (stack.Any())
            {
                var next = stack.Pop();
                yield return next;
                if (next.NextSibling != null) stack.Push(next.NextSibling);
                if (next.FirstChild != null) stack.Push(next.FirstChild);
            }
        }

        private static IEnumerable<Inline> EnumerateInlines(Inline inline)
        {
            if (inline == null) yield break;
            var stack = new Stack<Inline>();
            stack.Push(inline);
            while (stack.Any())
            {
{'fraction_non_alphanumeric': 0.09265608083724287, 'fraction_numerical': 0.0011728617827499097, 'mean_word_length': 1.8198931569575172, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 74, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6957501', 'n_tokens_mistral': 2863, 'n_tokens_neox': 2722, 'n_words': 617}