<commit_before>﻿#region LICENSE
// --------------------------------------------------------------------------------------------------------------------
// <copyright file="SKEMA.cs" company="SpaceKrakens">
//   MIT License
//   Copyright (c) 2016 SpaceKrakens
// </copyright>
// --------------------------------------------------------------------------------------------------------------------
#endregion

namespace SKON.SKEMA
{
    using System;
    using System.Collections.Generic;
    using System.IO;
    using System.Text;

    using Internal;
    using global::SKON.Internal.Utils;

    public class SKEMA
    {
        public static SKEMAObject LoadFile(string path, TextWriter errorStream = null)
        {
            if (File.Exists(path))
            {
                Scanner sc = new Scanner(path);

                return Parse(sc, errorStream);
            }
            else
            {
                throw new FileNotFoundException("File not found!", path);
            }
        }

        public static SKEMAObject Parse(string skon, TextWriter errorStream = null)
        {
            using (MemoryStream stream = ParserUtils.GenerateStreamFromString(skon))
            {
                Scanner sc = new Scanner(stream);

                return Parse(sc, errorStream);
            }
        }

        public static SKEMAObject Parse(Stream stream, TextWriter errorStream = null)
        {
            Scanner sc = new Scanner(stream);

            return Parse(sc, errorStream);
        }

        private static SKEMAObject Parse(Scanner sc, TextWriter errorStream)
        {
            Parser parser = new Parser(sc);

            if (errorStream != null)
            {
                parser.errors.errorStream = errorStream;
            }

            parser.Parse();

            if (parser.errors.count > 0)
            {
                throw new FormatException(string.Format("Could not parse file! Got {0} errors!", parser.errors.count));
            }

            ReferenceSolver rs = new ReferenceSolver(parser.definitions);

            // The strongly connected components. Should be used for better error messages.
            List<LinkedList<SKEMAObject>> components;

            if (rs.ResolveReferences(parser.data, out components) == false)
            {
                // TODO: Better exception
                throw new FormatException("Could not solve SKEMA references!");
            }
            
            return parser.data;
        }

        public static void WriteToFile(string filepath, SKEMAObject obj)
        {
            File.WriteAllText(filepath, Write(obj), Encoding.UTF8);
        }

        /// <summary>
        /// Writes a SKEMAObject.
        /// </summary>
        /// <param name="obj">The object to write.</param>
        /// <returns>A string to write into a file.</returns>
        public static string Write(SKEMAObject obj)
        {
            if (obj.Type != SKEMAType.MAP)
            {
                throw new ArgumentException("SKEMAObject to write must be of type map!");
            }

            StringBuilder sb = new StringBuilder();

            foreach (string key in obj.Keys)
            {
                sb.Append(key + ": " + WriteObject(obj[key], 0) + ",\n");
            }

            return sb.ToString();
        }

        /// <summary>
        /// Writes a SKEMAObject value.
        /// </summary>
        /// <param name="obj">The object to write.</param>
        /// <param name="indent">The amount of indentation.</param>
        /// <returns>A string to write into a file.</returns>
        private static string WriteObject(SKEMAObject obj, int indent)
        {
            if (obj == null)
            {
                return string.Empty;
            }

            string indentString = string.Empty;

            for (int i = 0; i < indent; i++)
            {
                indentString += SKON.IndentString;
            }

            switch (obj.Type)
            {
                case SKEMAType.REFERENCE:
                    return "#" + obj.Reference;
                case SKEMAType.ANY:
                    return "Any";
                case SKEMAType.STRING:
                    return "String";
                case SKEMAType.INTEGER:
                    return "Integer";
                case SKEMAType.FLOAT:
                    return "Float";
                case SKEMAType.BOOLEAN:
                    return "Boolean";
                case SKEMAType.DATETIME:
                    return "DateTime";
                case SKEMAType.MAP:
                    StringBuilder mapsb = new StringBuilder();

                    if (obj.Keys.Count <= 0)
                    {
                        mapsb.Append("{  }");
                        return mapsb.ToString();
                    }

                    mapsb.Append("\n" + indentString + "{\n");

                    foreach (string key in obj.Keys)
                    {
                        mapsb.Append(indentString + SKON.IndentString + $"{key}: {WriteObject(obj[key], indent + 1)},\n");
                    }

                    mapsb.Append(indentString + "}");

                    return mapsb.ToString();
                case SKEMAType.ARRAY:
                    if (obj.ArrayElementSKEMA == null)
                    {
                        return "[  ]";
                    }
                    return "\n[" + WriteObject(obj.ArrayElementSKEMA, indent + 1) + "\n]";
                default:
                    return null;
            }
        }
<commit_msg>Made writing SKEMA prettier<commit_after>﻿#region LICENSE
// --------------------------------------------------------------------------------------------------------------------
// <copyright file="SKEMA.cs" company="SpaceKrakens">
//   MIT License
//   Copyright (c) 2016 SpaceKrakens
// </copyright>
// --------------------------------------------------------------------------------------------------------------------
#endregion

namespace SKON.SKEMA
{
    using System;
    using System.Collections.Generic;
    using System.IO;
    using System.Text;

    using Internal;
    using global::SKON.Internal.Utils;

    public class SKEMA
    {
        public static SKEMAObject LoadFile(string path, TextWriter errorStream = null)
        {
            if (File.Exists(path))
            {
                Scanner sc = new Scanner(path);

                return Parse(sc, errorStream);
            }
            else
            {
                throw new FileNotFoundException("File not found!", path);
            }
        }

        public static SKEMAObject Parse(string skon, TextWriter errorStream = null)
        {
            using (MemoryStream stream = ParserUtils.GenerateStreamFromString(skon))
            {
                Scanner sc = new Scanner(stream);

                return Parse(sc, errorStream);
            }
        }

        public static SKEMAObject Parse(Stream stream, TextWriter errorStream = null)
        {
            Scanner sc = new Scanner(stream);

            return Parse(sc, errorStream);
        }

        private static SKEMAObject Parse(Scanner sc, TextWriter errorStream)
        {
            Parser parser = new Parser(sc);

            if (errorStream != null)
            {
                parser.errors.errorStream = errorStream;
            }

            parser.Parse();

            if (parser.errors.count > 0)
            {
                throw new FormatException(string.Format("Could not parse file! Got {0} errors!", parser.errors.count));
            }

            ReferenceSolver rs = new ReferenceSolver(parser.definitions);

            // The strongly connected components. Should be used for better error messages.
            List<LinkedList<SKEMAObject>> components;

            if (rs.ResolveReferences(parser.data, out components) == false)
            {
                // TODO: Better exception
                throw new FormatException("Could not solve SKEMA references!");
            }
            
            return parser.data;
        }

        public static void WriteToFile(string filepath, SKEMAObject obj)
        {
            File.WriteAllText(filepath, Write(obj), Encoding.UTF8);
        }

        /// <summary>
        /// Writes a SKEMAObject.
        /// </summary>
        /// <param name="obj">The object to write.</param>
        /// <returns>A string to write into a file.</returns>
        public static string Write(SKEMAObject obj)
        {
            if (obj.Type != SKEMAType.MAP)
            {
                throw new ArgumentException("SKEMAObject to write must be of type map!");
            }

            StringBuilder sb = new StringBuilder();

            foreach (string key in obj.Keys)
            {
                sb.Append(key + ": " + WriteObject(obj[key], 0) + ",\n");
            }

            return sb.ToString();
        }

        /// <summary>
        /// Writes a SKEMAObject value.
        /// </summary>
        /// <param name="obj">The object to write.</param>
        /// <param name="indent">The amount of indentation.</param>
        /// <returns>A string to write into a file.</returns>
        private static string WriteObject(SKEMAObject obj, int indent)
        {
            if (obj == null)
            {
                return string.Empty;
            }

            string indentString = string.Empty;

            for (int i = 0; i < indent; i++)
            {
                indentString += SKON.IndentString;
            }

            switch (obj.Type)
            {
                case SKEMAType.REFERENCE:
                    return "#" + obj.Reference;
                case SKEMAType.ANY:
                    return "Any";
                case SKEMAType.STRING:
                    return "String";
                case SKEMAType.INTEGER:
                    return "Integer";
                case SKEMAType.FLOAT:
                    return "Float";
                case SKEMAType.BOOLEAN:
                    return "Boolean";
                case SKEMAType.DATETIME:
                    return "DateTime";
                case SKEMAType.MAP:
                    StringBuilder mapsb = new StringBuilder();

                    if (obj.Keys.Count <= 0)
                    {
                        mapsb.Append("{  }");
                        return mapsb.ToString();
                    }

                    mapsb.Append("\n" + indentString + "{\n");

                    foreach (string key in obj.Keys)
                    {
                        mapsb.Append(indentString + SKON.IndentString + $"{key}: {WriteObject(obj[key], indent + 1)},\n");
                    }

                    mapsb.Append(indentString + "}");

                    return mapsb.ToString();
                case SKEMAType.ARRAY:
                    if (obj.ArrayElementSKEMA == null)
                    {
                        return "[  ]";
                    }
                    return "\n[\n" + WriteObject(obj.ArrayElementSKEMA, indent + 1) + "\n]";
                default:
                    return null;
            }
        }

{'fraction_non_alphanumeric': 0.13621708758731865, 'fraction_numerical': 0.0021493820526598604, 'mean_word_length': 1.162052274927396, 'pattern_counts': {'":': 2, '<': 43, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3080947', 'n_tokens_mistral': 2783, 'n_tokens_neox': 2570, 'n_words': 669}