<commit_before>﻿using System;
using System.Collections.Generic;
using System.Text;

namespace FullJson {
    /// <summary>
    /// A simple recursive descent parser for JSON.
    /// </summary>
    public class JsonParser {
        private int _start;
        private string _input;

        private JsonFailure MakeFailure(string message) {
            int start = Math.Max(0, _start - 10);
            int length = Math.Min(20, _input.Length - start);

            string error = "Error while parsing: " + message + "; context = \"" +
                _input.Substring(start, length) + "\"";
            return JsonFailure.Fail(error);
        }

        private bool TryMoveNext() {
            if (_start < _input.Length) {
                ++_start;
                return true;
            }

            return false;
        }

        private bool HasValue(int offset = 0) {
            return (_start + offset) >= 0 && (_start + offset) < _input.Length;
        }

        private char Character(int offset = 0) {
            return _input[_start + offset];
        }

        /// <summary>
        /// Skips input such that Character() will return a non-whitespace character
        /// </summary>
        private void SkipSpace() {
            while (HasValue()) {
                char c = Character();

                // whitespace; fine to skip
                if (char.IsWhiteSpace(c)) {
                    TryMoveNext();
                    continue;
                }

                // comment? they begin with //
                if (HasValue(1) &&
                    (Character(0) == '/' && Character(1) == '/')) {

                    // skip the rest of the line
                    while (HasValue() && Environment.NewLine.Contains("" + Character()) == false) {
                        TryMoveNext();
                    }

                    // we still need to skip whitespace on the next line
                    continue;
                }

                break;
            }
        }

        #region Escaping
        private bool IsHex(char c) {
            return ((c >= '0' && c <= '9') ||
                     (c >= 'a' && c <= 'f') ||
                     (c >= 'A' && c <= 'F'));
        }

        private uint ParseSingleChar(char c1, uint multipliyer) {
            uint p1 = 0;
            if (c1 >= '0' && c1 <= '9')
                p1 = (uint)(c1 - '0') * multipliyer;
            else if (c1 >= 'A' && c1 <= 'F')
                p1 = (uint)((c1 - 'A') + 10) * multipliyer;
            else if (c1 >= 'a' && c1 <= 'f')
                p1 = (uint)((c1 - 'a') + 10) * multipliyer;
            return p1;
        }

        private uint ParseUnicode(char c1, char c2, char c3, char c4) {
            uint p1 = ParseSingleChar(c1, 0x1000);
            uint p2 = ParseSingleChar(c2, 0x100);
            uint p3 = ParseSingleChar(c3, 0x10);
            uint p4 = ParseSingleChar(c4, 1);

            return p1 + p2 + p3 + p4;
        }

        private JsonFailure TryUnescapeChar(out char escaped) {
            /* skip leading backslash '\' */
            switch (Character()) {
                case '\\': TryMoveNext(); escaped = '\\'; return JsonFailure.Success;
                case '/': TryMoveNext(); escaped = '/'; return JsonFailure.Success;
                case '\'': TryMoveNext(); escaped = '\''; return JsonFailure.Success;
                case '"': TryMoveNext(); escaped = '\"'; return JsonFailure.Success;
                case 'a': TryMoveNext(); escaped = '\a'; return JsonFailure.Success;
                case 'b': TryMoveNext(); escaped = '\b'; return JsonFailure.Success;
                case 'f': TryMoveNext(); escaped = '\f'; return JsonFailure.Success;
                case 'n': TryMoveNext(); escaped = '\n'; return JsonFailure.Success;
                case 'r': TryMoveNext(); escaped = '\r'; return JsonFailure.Success;
                case 't': TryMoveNext(); escaped = '\t'; return JsonFailure.Success;
                case '0': TryMoveNext(); escaped = '\0'; return JsonFailure.Success;
                case 'u':
                    TryMoveNext();
                    if (IsHex(Character(0))
                     && IsHex(Character(1))
                     && IsHex(Character(2))
                     && IsHex(Character(3))) {
                        TryMoveNext();
                        TryMoveNext();
                        TryMoveNext();
                        TryMoveNext();
                        uint codePoint = ParseUnicode(Character(0), Character(1), Character(2), Character(3));
                        escaped = (char)codePoint;
                        return JsonFailure.Success;
                    }

                    // invalid escape sequence
                    escaped = (char)0;
                    return MakeFailure(
                        string.Format("invalid escape sequence '\\u{0}{1}{2}{3}'\n",
                            Character(0),
                            Character(1),
                            Character(2),
                            Character(3)));
                default:
                    escaped = (char)0;
                    return MakeFailure(string.Format("Invalid escape sequence \\{0}", Character()));
            }
        }
        #endregion

        private JsonFailure TryParseExact(string content) {
            for (int i = 0; i < content.Length; ++i) {
                if (Character() != content[i]) {
                    return MakeFailure("Expected " + content[i]);
                }

                if (TryMoveNext() == false) {
                    return MakeFailure("Unexpected end of content when parsing " + content);
                }
            }
<commit_msg>Fix parsing for escaped values<commit_after>﻿using System;
using System.Collections.Generic;
using System.Text;

namespace FullJson {
    /// <summary>
    /// A simple recursive descent parser for JSON.
    /// </summary>
    public class JsonParser {
        private int _start;
        private string _input;

        private JsonFailure MakeFailure(string message) {
            int start = Math.Max(0, _start - 10);
            int length = Math.Min(20, _input.Length - start);

            string error = "Error while parsing: " + message + "; context = \"" +
                _input.Substring(start, length) + "\"";
            return JsonFailure.Fail(error);
        }

        private bool TryMoveNext() {
            if (_start < _input.Length) {
                ++_start;
                return true;
            }

            return false;
        }

        private bool HasValue(int offset = 0) {
            return (_start + offset) >= 0 && (_start + offset) < _input.Length;
        }

        private char Character(int offset = 0) {
            return _input[_start + offset];
        }

        /// <summary>
        /// Skips input such that Character() will return a non-whitespace character
        /// </summary>
        private void SkipSpace() {
            while (HasValue()) {
                char c = Character();

                // whitespace; fine to skip
                if (char.IsWhiteSpace(c)) {
                    TryMoveNext();
                    continue;
                }

                // comment? they begin with //
                if (HasValue(1) &&
                    (Character(0) == '/' && Character(1) == '/')) {

                    // skip the rest of the line
                    while (HasValue() && Environment.NewLine.Contains("" + Character()) == false) {
                        TryMoveNext();
                    }

                    // we still need to skip whitespace on the next line
                    continue;
                }

                break;
            }
        }

        #region Escaping
        private bool IsHex(char c) {
            return ((c >= '0' && c <= '9') ||
                     (c >= 'a' && c <= 'f') ||
                     (c >= 'A' && c <= 'F'));
        }

        private uint ParseSingleChar(char c1, uint multipliyer) {
            uint p1 = 0;
            if (c1 >= '0' && c1 <= '9')
                p1 = (uint)(c1 - '0') * multipliyer;
            else if (c1 >= 'A' && c1 <= 'F')
                p1 = (uint)((c1 - 'A') + 10) * multipliyer;
            else if (c1 >= 'a' && c1 <= 'f')
                p1 = (uint)((c1 - 'a') + 10) * multipliyer;
            return p1;
        }

        private uint ParseUnicode(char c1, char c2, char c3, char c4) {
            uint p1 = ParseSingleChar(c1, 0x1000);
            uint p2 = ParseSingleChar(c2, 0x100);
            uint p3 = ParseSingleChar(c3, 0x10);
            uint p4 = ParseSingleChar(c4, 0x1);

            return p1 + p2 + p3 + p4;
        }

        private JsonFailure TryUnescapeChar(out char escaped) {
            // skip leading backslash '\'
            TryMoveNext();
            if (HasValue() == false) {
                escaped = ' ';
                return MakeFailure("Unexpected end of input after \\");
            }

            switch (Character()) {
                case '\\': TryMoveNext(); escaped = '\\'; return JsonFailure.Success;
                case '/': TryMoveNext(); escaped = '/'; return JsonFailure.Success;
                case '\'': TryMoveNext(); escaped = '\''; return JsonFailure.Success;
                case '"': TryMoveNext(); escaped = '\"'; return JsonFailure.Success;
                case 'a': TryMoveNext(); escaped = '\a'; return JsonFailure.Success;
                case 'b': TryMoveNext(); escaped = '\b'; return JsonFailure.Success;
                case 'f': TryMoveNext(); escaped = '\f'; return JsonFailure.Success;
                case 'n': TryMoveNext(); escaped = '\n'; return JsonFailure.Success;
                case 'r': TryMoveNext(); escaped = '\r'; return JsonFailure.Success;
                case 't': TryMoveNext(); escaped = '\t'; return JsonFailure.Success;
                case '0': TryMoveNext(); escaped = '\0'; return JsonFailure.Success;
                case 'u':
                    TryMoveNext();
                    if (IsHex(Character(0))
                     && IsHex(Character(1))
                     && IsHex(Character(2))
                     && IsHex(Character(3))) {

                        uint codePoint = ParseUnicode(Character(0), Character(1), Character(2), Character(3));

                        TryMoveNext();
                        TryMoveNext();
                        TryMoveNext();

                        escaped = (char)codePoint;
                        return JsonFailure.Success;
                    }

                    // invalid escape sequence
                    escaped = (char)0;
                    return MakeFailure(
                        string.Format("invalid escape sequence '\\u{0}{1}{2}{3}'\n",
                            Character(0),
                            Character(1),
                            Character(2),
                            Character(3)));
                default:
                    escaped = (char)0;
                    return MakeFailure(string.Format("Invalid escape sequence \\{0}", Character()));
            }
        }
        #endregion

        private JsonFailure TryParseExact(string content) {
            for (int i = 0; i < content.Length; ++i) {
                if (Character() != content[i]) {
                    return MakeFailure("Expected " + content[i]);
                }

                if (TryMoveNext() == false) {
                    return MakeFailure("Unexpected end of content when parsing " + content);
                }
            }

{'fraction_non_alphanumeric': 0.12575522181943724, 'fraction_numerical': 0.015104436388745037, 'mean_word_length': 1.2822532991924365, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6672831', 'n_tokens_mistral': 3060, 'n_tokens_neox': 2846, 'n_words': 790}