<commit_before>﻿// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.

using System.Collections.Immutable;
using System.Diagnostics;
using System.Linq;
using Microsoft.CodeAnalysis.PooledObjects;
using Microsoft.CodeAnalysis.Text;

namespace Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars
{
    internal abstract class AbstractVirtualCharService : IVirtualCharService
    {
        protected abstract bool IsStringLiteralToken(SyntaxToken token);
        protected abstract ImmutableArray<VirtualChar> TryConvertToVirtualCharsWorker(SyntaxToken token);

        protected static bool TryAddBraceEscape(
            ArrayBuilder<VirtualChar> result, string tokenText, int offset, int index)
        {
            if (index + 1 < tokenText.Length)
            {
                var ch = tokenText[index];
                var next = tokenText[index + 1];
                if ((ch == '{' && next == '{') ||
                    (ch == '}' && next == '}'))
                {
                    result.Add(new VirtualChar(ch, new TextSpan(offset + index, 2)));
                    return true;
                }
            }

            return false;
        }

        public ImmutableArray<VirtualChar> TryConvertToVirtualChars(SyntaxToken token)
        {
            // We don't process any strings that contain diagnostics in it.  That means that we can 
            // trust that all the string's contents (most importantly, the escape sequences) are well
            // formed.
            if (token.ContainsDiagnostics)
            {
                return default;
            }

            var result = TryConvertToVirtualCharsWorker(token);

#if DEBUG
            // Do some invariant checking to make sure we processed the string token the same
            // way the C# and VB compilers did.
            if (!result.IsDefault)
            {
                // Ensure that we properly broke up the token into a sequence of characters that
                // matches what the compiler did.
                if (IsStringLiteralToken(token))
                {
                    var expectedValueText = token.ValueText;
                    var actualValueText = result.CreateString();
                    Debug.Assert(expectedValueText == actualValueText);
                }

                if (result.Length > 0)
                {
                    var currentVC = result[0];
                    Debug.Assert(currentVC.Span.Start >= token.SpanStart, "First span has to start after the start of the string token");
                    if (IsStringLiteralToken(token))
                    {
                        Debug.Assert(currentVC.Span.Start == token.SpanStart + 1 ||
                                     currentVC.Span.Start == token.SpanStart + 2, "First span should start on the second or third char of the string.");
                    }
                    else
                    {
                        Debug.Assert(currentVC.Span.Start == token.SpanStart, "First span should start on the first char of the string.");
                    }

                    for (var i = 1; i < result.Length; i++)
                    {
                        var nextVC = result[i];
                        Debug.Assert(currentVC.Span.End == nextVC.Span.Start, "Virtual character spans have to be touching.");
                        currentVC = nextVC;
                    }

                    var lastVC = result.Last();

                    if (IsStringLiteralToken(token))
                    {
                        Debug.Assert(lastVC.Span.End == token.Span.End - 1, "Last span has to end right before the end of the string token.");
                    }
                    else
                    {
                        Debug.Assert(lastVC.Span.End == token.Span.End, "Last span has to end right before the end of the string token.");
                    }
                }
            }
#endif

            return result;
        }
<commit_msg>Move checking into helper method.<commit_after>﻿// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.

using System.Collections.Immutable;
using System.Diagnostics;
using System.Linq;
using Microsoft.CodeAnalysis.PooledObjects;
using Microsoft.CodeAnalysis.Text;

namespace Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars
{
    internal abstract class AbstractVirtualCharService : IVirtualCharService
    {
        protected abstract bool IsStringLiteralToken(SyntaxToken token);
        protected abstract ImmutableArray<VirtualChar> TryConvertToVirtualCharsWorker(SyntaxToken token);

        protected static bool TryAddBraceEscape(
            ArrayBuilder<VirtualChar> result, string tokenText, int offset, int index)
        {
            if (index + 1 < tokenText.Length)
            {
                var ch = tokenText[index];
                var next = tokenText[index + 1];
                if ((ch == '{' && next == '{') ||
                    (ch == '}' && next == '}'))
                {
                    result.Add(new VirtualChar(ch, new TextSpan(offset + index, 2)));
                    return true;
                }
            }

            return false;
        }

        public ImmutableArray<VirtualChar> TryConvertToVirtualChars(SyntaxToken token)
        {
            // We don't process any strings that contain diagnostics in it.  That means that we can 
            // trust that all the string's contents (most importantly, the escape sequences) are well
            // formed.
            if (token.ContainsDiagnostics)
            {
                return default;
            }

            var result = TryConvertToVirtualCharsWorker(token);

            CheckInvariants(token, result);

            return result;
        }

        [Conditional("DEBUG")]
        private void CheckInvariants(SyntaxToken token, ImmutableArray<VirtualChar> result)
        {
            // Do some invariant checking to make sure we processed the string token the same
            // way the C# and VB compilers did.
            if (!result.IsDefault)
            {
                // Ensure that we properly broke up the token into a sequence of characters that
                // matches what the compiler did.
                if (IsStringLiteralToken(token))
                {
                    var expectedValueText = token.ValueText;
                    var actualValueText = result.CreateString();
                    Debug.Assert(expectedValueText == actualValueText);
                }

                if (result.Length > 0)
                {
                    var currentVC = result[0];
                    Debug.Assert(currentVC.Span.Start >= token.SpanStart, "First span has to start after the start of the string token");
                    if (IsStringLiteralToken(token))
                    {
                        Debug.Assert(currentVC.Span.Start == token.SpanStart + 1 ||
                                     currentVC.Span.Start == token.SpanStart + 2, "First span should start on the second or third char of the string.");
                    }
                    else
                    {
                        Debug.Assert(currentVC.Span.Start == token.SpanStart, "First span should start on the first char of the string.");
                    }

                    for (var i = 1; i < result.Length; i++)
                    {
                        var nextVC = result[i];
                        Debug.Assert(currentVC.Span.End == nextVC.Span.Start, "Virtual character spans have to be touching.");
                        currentVC = nextVC;
                    }

                    var lastVC = result.Last();

                    if (IsStringLiteralToken(token))
                    {
                        Debug.Assert(lastVC.Span.End == token.Span.End - 1, "Last span has to end right before the end of the string token.");
                    }
                    else
                    {
                        Debug.Assert(lastVC.Span.End == token.Span.End, "Last span has to end right before the end of the string token.");
                    }
                }
            }
        }

{'fraction_non_alphanumeric': 0.07860470672560028, 'fraction_numerical': 0.002628120893561104, 'mean_word_length': 1.463077375698735, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5541865', 'n_tokens_mistral': 1978, 'n_tokens_neox': 1835, 'n_words': 606}