<commit_before>/*
 * Copyright 2014 - 2016 Real Logic Ltd.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package uk.co.real_logic.sbe.generation.cpp;

import uk.co.real_logic.sbe.PrimitiveType;
import uk.co.real_logic.sbe.generation.CodeGenerator;
import org.agrona.generation.OutputManager;
import uk.co.real_logic.sbe.ir.*;
import org.agrona.Verify;

import java.io.IOException;
import java.io.Writer;
import java.util.ArrayList;
import java.util.List;

import static uk.co.real_logic.sbe.generation.cpp.CppUtil.*;
import static uk.co.real_logic.sbe.ir.GenerationUtil.collectVarData;
import static uk.co.real_logic.sbe.ir.GenerationUtil.collectGroups;
import static uk.co.real_logic.sbe.ir.GenerationUtil.collectFields;

public class CppGenerator implements CodeGenerator
{
    private static final String BASE_INDENT = "";
    private static final String INDENT = "    ";

    private final Ir ir;
    private final OutputManager outputManager;

    public CppGenerator(final Ir ir, final OutputManager outputManager)
        throws IOException
    {
        Verify.notNull(ir, "ir");
        Verify.notNull(outputManager, "outputManager");

        this.ir = ir;
        this.outputManager = outputManager;
    }

    public void generateMessageHeaderStub() throws IOException
    {
        final String messageHeader = "MessageHeader";

        try (Writer out = outputManager.createOutput(messageHeader))
        {
            final List<Token> tokens = ir.headerStructure().tokens();
            out.append(generateFileHeader(ir.namespaces(), messageHeader, null));
            out.append(generateClassDeclaration(messageHeader));
            out.append(generateFixedFlyweightCode(messageHeader, tokens.get(0).encodedLength()));
            out.append(generateCompositePropertyElements(
                messageHeader, tokens.subList(1, tokens.size() - 1), BASE_INDENT));
            out.append("};\n");
            out.append(CppUtil.closingBraces(ir.namespaces().length)).append("#endif\n");
        }
    }

    public List<String> generateTypeStubs() throws IOException
    {
        final List<String> typesToInclude = new ArrayList<>();

        for (final List<Token> tokens : ir.types())
        {
            switch (tokens.get(0).signal())
            {
                case BEGIN_ENUM:
                    generateEnum(tokens);
                    break;

                case BEGIN_SET:
                    generateChoiceSet(tokens);
                    break;

                case BEGIN_COMPOSITE:
                    generateComposite(tokens);
                    break;
            }

            typesToInclude.add(tokens.get(0).name());
        }

        return typesToInclude;
    }

    public List<String> generateTypesToIncludes(final List<Token> tokens)
    {
        final List<String> typesToInclude = new ArrayList<>();

        for (final Token token : tokens)
        {
            switch (token.signal())
            {
                case BEGIN_ENUM:
                case BEGIN_SET:
                case BEGIN_COMPOSITE:
                    typesToInclude.add(token.name());
                    break;
            }
        }

        return typesToInclude;
    }

    public void generate() throws IOException
    {
        generateMessageHeaderStub();
        final List<String> typesToInclude = generateTypeStubs();

        for (final List<Token> tokens : ir.messages())
        {
            final Token msgToken = tokens.get(0);
            final String className = formatClassName(msgToken.name());

            try (Writer out = outputManager.createOutput(className))
            {
                out.append(generateFileHeader(ir.namespaces(), className, typesToInclude));
                out.append(generateClassDeclaration(className));
                out.append(generateMessageFlyweightCode(className, msgToken));

                final List<Token> messageBody = tokens.subList(1, tokens.size() - 1);
                int i = 0;

                final List<Token> fields = new ArrayList<>();
                i = collectFields(messageBody, i, fields);

                final List<Token> groups = new ArrayList<>();
                i = collectGroups(messageBody, i, groups);

                final List<Token> varData = new ArrayList<>();
                collectVarData(messageBody, i, varData);

                final StringBuilder sb = new StringBuilder();
                out.append(generateFields(className, fields, BASE_INDENT));
                generateGroups(sb, groups, BASE_INDENT);
                out.append(sb);
                out.append(generateVarData(className, varData, BASE_INDENT));
                out.append("};\n");
                out.append(CppUtil.closingBraces(ir.namespaces().length)).append("#endif\n");
            }
        }
    }

    private void generateGroups(final StringBuilder sb, final List<Token> tokens, final String indent)
    {
        for (int i = 0, size = tokens.size(); i < size; i++)
        {
            final Token groupToken = tokens.get(i);
            if (groupToken.signal() != Signal.BEGIN_GROUP)
            {
                throw new IllegalStateException("tokens must begin with BEGIN_GROUP: token=" + groupToken);
            }

            final String groupName = groupToken.name();
            final String cppTypeForNumInGroup = cppTypeName(tokens.get(i + 3).encoding().primitiveType());

            generateGroupClassHeader(sb, groupName, tokens, i, indent + INDENT);

            ++i;
            final int groupHeaderTokenCount = tokens.get(i).componentTokenCount();
            i += groupHeaderTokenCount;

            final List<Token> fields = new ArrayList<>();
            i = collectFields(tokens, i, fields);
            sb.append(generateFields(groupName, fields, indent + INDENT));

            final List<Token> groups = new ArrayList<>();
            i = collectGroups(tokens, i, groups);
            generateGroups(sb, groups, indent + INDENT);

            final List<Token> varData = new ArrayList<>();
            i = collectVarData(tokens, i, varData);
            sb.append(generateVarData(formatClassName(groupName), varData, indent + INDENT));

            sb.append(indent).append("    };\n");
            sb.append(generateGroupProperty(groupName, groupToken, cppTypeForNumInGroup, indent));
        }
    }

    private static void generateGroupClassHeader(
        final StringBuilder sb, final String groupName, final List<Token> tokens, final int index, final String indent)
    {
        final String dimensionsClassName = formatClassName(tokens.get(index + 1).name());
        final int dimensionHeaderLength = tokens.get(index + 1).encodedLength();

        final int blockLength = tokens.get(index).encodedLength();
        final Token numInGroupToken = tokens.get(index + 3);
        final String cppTypeForBlockLength = cppTypeName(tokens.get(index + 2).encoding().primitiveType());
        final String cppTypeForNumInGroup = cppTypeName(numInGroupToken.encoding().primitiveType());

        sb.append(String.format(
            "\n" +
            indent + "class %1$s\n" +
            indent + "{\n" +
            indent + "private:\n" +
            indent + "    char *m_buffer;\n" +
            indent + "    std::uint64_t m_bufferLength;\n" +
            indent + "    std::uint64_t *m_positionPtr;\n" +
            indent + "    std::uint64_t m_blockLength;\n" +
            indent + "    std::uint64_t m_count;\n" +
            indent + "    std::uint64_t m_index;\n" +
            indent + "    std::uint64_t m_offset;\n" +
            indent + "    std::uint64_t m_actingVersion;\n" +
            indent + "    %2$s m_dimensions;\n\n" +
            indent + "public:\n\n",
            formatClassName(groupName), dimensionsClassName));

        sb.append(String.format(
            indent + "    inline void wrapForDecode(char *buffer, std::uint64_t *pos," +
                " const std::uint64_t actingVersion, const std::uint64_t bufferLength)\n" +
            indent + "    {\n" +
            indent + "        m_buffer = buffer;\n" +
            indent + "        m_bufferLength = bufferLength;\n" +
            indent + "        m_dimensions.wrap(m_buffer, *pos, actingVersion, bufferLength);\n" +
            indent + "        m_blockLength = m_dimensions.blockLength();\n" +
            indent + "        m_count = m_dimensions.numInGroup();\n" +
            indent + "        m_index = -1;\n" +
            indent + "        m_actingVersion = actingVersion;\n" +
            indent + "        m_positionPtr = pos;\n" +
            indent + "        *m_positionPtr = *m_positionPtr + %1$d;\n" +
            indent + "    }\n\n",
            dimensionHeaderLength));

        sb.append(String.format(
            indent + "    inline void wrapForEncode(char *buffer, const %3$s count," +
                " std::uint64_t *pos, const std::uint64_t actingVersion, const std::uint64_t bufferLength)\n" +
            indent + "    {\n" +
            indent + "        if (count < %5$d || count > %6$d)\n" +
            indent + "        {\n" +
            indent + "            throw std::runtime_error(\"count outside of allowed range [E110]\");\n" +
            indent + "        }\n" +
            indent + "        m_buffer = buffer;\n" +
            indent + "        m_bufferLength = bufferLength;\n" +
            indent + "        m_dimensions.wrap(m_buffer, *pos, actingVersion, bufferLength);\n" +
            indent + "        m_dimensions.blockLength((%1$s)%2$d);\n" +
            indent + "        m_dimensions.numInGroup((%3$s)count);\n" +
            indent + "        m_index = -1;\n" +
            indent + "        m_count = count;\n" +
            indent + "        m_blockLength = %2$d;\n" +
            indent + "        m_actingVersion = actingVersion;\n" +
            indent + "        m_positionPtr = pos;\n" +
            indent + "        *m_positionPtr = *m_positionPtr + %4$d;\n" +
            indent + "    }\n\n",
            cppTypeForBlockLength, blockLength, cppTypeForNumInGroup, dimensionHeaderLength,
            numInGroupToken.encoding().applicableMinValue().longValue(),
            numInGroupToken.encoding().applicableMaxValue().longValue()));

        sb.append(String.format(
            indent + "    static SBE_CONSTEXPR const std::uint64_t sbeHeaderSize()\n" +
            indent + "    {\n" +
            indent + "        return %1$d;\n" +
            indent + "    }\n\n" +
            indent + "    static SBE_CONSTEXPR const std::uint64_t sbeBlockLength()\n" +
            indent + "    {\n" +
            indent + "        return %2$d;\n" +
            indent + "    }\n\n" +
            indent + "    std::uint64_t position(void) const\n" +
            indent + "    {\n" +
            indent + "        return *m_positionPtr;\n" +
            indent + "    }\n\n" +
            indent + "    void position(const std::uint64_t position)\n" +
            indent + "    {\n" +
            indent + "        if (SBE_BOUNDS_CHECK_EXPECT((position > m_bufferLength), false))\n" +
            indent + "        {\n" +
            indent + "             throw std::runtime_error(\"buffer too short [E100]\");\n" +
            indent + "        }\n" +
            indent + "        *m_positionPtr = position;\n" +
            indent + "    }\n\n" +
            indent + "    inline std::uint64_t count(void) const\n" +
            indent + "    {\n" +
            indent + "        return m_count;\n" +
            indent + "    }\n\n" +
            indent + "    inline bool hasNext(void) const\n" +
            indent + "    {\n" +
            indent + "        return m_index + 1 < m_count;\n" +
            indent + "    }\n\n" +
            indent + "    inline %3$s &next(void)\n" +
            indent + "    {\n" +
            indent + "        m_offset = *m_positionPtr;\n" +
            indent + "        if (SBE_BOUNDS_CHECK_EXPECT(( (m_offset + m_blockLength) > m_bufferLength ), false))\n" +
            indent + "        {\n" +
            indent + "            throw std::runtime_error(\"" +
                "buffer too short to support next group index [E108]\");\n" +
            indent + "        }\n" +
            indent + "        *m_positionPtr = m_offset + m_blockLength;\n" +
            indent + "        ++m_index;\n\n" +
            indent + "        return *this;\n" +
            indent + "    }\n\n",
            dimensionHeaderLength, blockLength, formatClassName(groupName)));

        sb.append(String.format(
            indent + "#if __cplusplus < 201103L\n" +
            indent + "    template<class Func> inline void forEach(Func& func)\n" +
            indent + "    {\n" +
            indent + "        while (hasNext())\n" +
            indent + "        {\n" +
            indent + "            next(); func(*this);\n" +
            indent + "        }\n" +
            indent + "    }\n\n" +
            indent + "#else\n" +
            indent + "    template<class Func> inline void forEach(Func&& func)\n" +
            indent + "    {\n" +
            indent + "        while (hasNext())\n" +
            indent + "        {\n" +
            indent + "            next(); func(*this);\n" +
            indent + "        }\n" +
            indent + "    }\n\n" +
            indent + "#endif\n\n",
            formatClassName(groupName)));
    }

    private static CharSequence generateGroupProperty(
        final String groupName, final Token token, final String cppTypeForNumInGroup, final String indent)
    {
        final StringBuilder sb = new StringBuilder();

        final String className = formatClassName(groupName);
        final String propertyName = formatPropertyName(groupName);

        sb.append(String.format(
            "\n" +
            "private:\n" +
            indent + "    %1$s m_%2$s;\n\n" +
            "public:\n",
            className,
            propertyName));

        sb.append(String.format(
            "\n" +
            indent + "    static SBE_CONSTEXPR const std::uint16_t %1$sId(void)\n" +
            indent + "    {\n" +
            indent + "        return %2$d;\n" +
            indent + "    }\n\n",
            groupName,
            token.id()));

        sb.append(String.format(
            "\n" +
            indent + "    inline %1$s &%2$s(void)\n" +
            indent + "    {\n" +
            indent + "        m_%2$s.wrapForDecode(m_buffer, m_positionPtr, m_actingVersion, m_bufferLength);\n" +
            indent + "        return m_%2$s;\n" +
            indent + "    }\n",
            className,
            propertyName));

        sb.append(String.format(
            "\n" +
            indent + "    %1$s &%2$sCount(const %3$s count)\n" +
            indent + "    {\n" +
            indent + "        m_%2$s.wrapForEncode(" +
                "m_buffer, count, m_positionPtr, m_actingVersion, m_bufferLength);\n" +
            indent + "        return m_%2$s;\n" +
            indent + "    }\n\n",
            className,
            propertyName,
            cppTypeForNumInGroup));

        sb.append(String.format(
            indent + "    static SBE_CONSTEXPR const std::uint64_t %1$sSinceVersion(void)\n" +
            indent + "    {\n" +
            indent + "         return %2$d;\n" +
            indent + "    }\n\n" +
            indent + "    bool %1$sInActingVersion(void)\n" +
            indent + "    {\n" +
            indent + "#pragma GCC diagnostic push\n" +
            indent + "#pragma GCC diagnostic ignored \"-Wtautological-compare\"\n" +
            indent + "        return m_actingVersion >= %1$sSinceVersion();\n" +
            indent + "#pragma GCC diagnostic pop\n" +
            indent + "    }\n",
            propertyName,
            token.version()));

        return sb;
    }

    private CharSequence generateVarData(final String className, final List<Token> tokens, final String indent)
    {
        final StringBuilder sb = new StringBuilder();

        for (int i = 0, size = tokens.size(); i < size;)
        {
            final Token token = tokens.get(i);
            if (token.signal() != Signal.BEGIN_VAR_DATA)
            {
                throw new IllegalStateException("tokens must begin with BEGIN_VAR_DATA: token=" + token);
            }

            final String propertyName = toUpperFirstChar(token.name());
            final String characterEncoding = tokens.get(i + 3).encoding().characterEncoding();
            final Token lengthToken = tokens.get(i + 2);
            final int lengthOfLengthField = lengthToken.encodedLength();
            final String lengthCppType = cppTypeName(lengthToken.encoding().primitiveType());

            generateFieldMetaAttributeMethod(sb, token, indent);

            generateVarDataDescriptors(
                sb, token, propertyName, characterEncoding, lengthToken, lengthOfLengthField, lengthCppType, indent);

            sb.append(String.format(
                indent + "    const char *%1$s(void)\n" +
                indent + "    {\n" +
                    "%2$s" +
                indent + "         const char *fieldPtr = (m_buffer + position() + %3$d);\n" +
                indent + "         position(position() + %3$d + *((%4$s *)(m_buffer + position())));\n" +
                indent + "         return fieldPtr;\n" +
                indent + "    }\n\n",
                formatPropertyName(propertyName),
                generateTypeFieldNotPresentCondition(token.version(), BASE_INDENT),
                lengthOfLengthField,
                lengthCppType));

            sb.append(String.format(
                indent + "    std::uint64_t get%1$s(char *dst, const std::uint64_t length)\n" +
                indent + "    {\n" +
                    "%2$s" +
                indent + "        std::uint64_t lengthOfLengthField = %3$d;\n" +
                indent + "        std::uint64_t lengthPosition = position();\n" +
                indent + "        position(lengthPosition + lengthOfLengthField);\n" +
                indent + "        std::uint64_t dataLength = %4$s(*((%5$s *)(m_buffer + lengthPosition)));\n" +
                indent + "        std::uint64_t bytesToCopy = (length < dataLength) ? length : dataLength;\n" +
                indent + "        std::uint64_t pos = position();\n" +
                indent + "        position(position() + dataLength);\n" +
                indent + "        std::memcpy(dst, m_buffer + pos, bytesToCopy);\n" +
                indent + "        return bytesToCopy;\n" +
                indent + "    }\n\n",
                propertyName,
                generateArrayFieldNotPresentCondition(token.version(), BASE_INDENT),
                lengthOfLengthField,
                formatByteOrderEncoding(lengthToken.encoding().byteOrder(), lengthToken.encoding().primitiveType()),
                lengthCppType));

            sb.append(String.format(
                indent + "    %5$s &put%1$s(const char *src, const %3$s length)\n" +
                indent + "    {\n" +
                indent + "        std::uint64_t lengthOfLengthField = %2$d;\n" +
                indent + "        std::uint64_t lengthPosition = position();\n" +
                indent + "        position(lengthPosition + lengthOfLengthField);\n" +
                indent + "        *((%3$s *)(m_buffer + lengthPosition)) = %4$s(length);\n" +
                indent + "        std::uint64_t pos = position();\n" +
                indent + "        position(position() + length);\n" +
                indent + "        std::memcpy(m_buffer + pos, src, length);\n" +
                indent + "        return *this;\n" +
                indent + "    }\n\n",
                propertyName,
                lengthOfLengthField,
                lengthCppType,
                formatByteOrderEncoding(lengthToken.encoding().byteOrder(), lengthToken.encoding().primitiveType()),
                className));

            sb.append(String.format(
                indent + "    const std::string get%1$sAsString()\n" +
                indent + "    {\n" +
                "%2$s" +
                indent + "        std::uint64_t lengthOfLengthField = %3$d;\n" +
                indent + "        std::uint64_t lengthPosition = position();\n" +
                indent + "        position(lengthPosition + lengthOfLengthField);\n" +
                indent + "        std::uint64_t dataLength = %4$s(*((%5$s *)(m_buffer + lengthPosition)));\n" +
                indent + "        std::uint64_t pos = position();\n" +
                indent + "        const std::string result(m_buffer + pos, dataLength);\n" +
                indent + "        position(position() + dataLength);\n" +
                indent + "        return result;\n" +
                indent + "    }\n\n",
                propertyName,
                generateStringNotPresentCondition(token.version(), BASE_INDENT),
                lengthOfLengthField,
                formatByteOrderEncoding(lengthToken.encoding().byteOrder(), lengthToken.encoding().primitiveType()),
                lengthCppType));

            sb.append(String.format(
                indent + "    %1$s &put%2$s(const std::string& str)\n" +
                indent + "    {\n" +
                indent + "        if (str.length() > %6$d)\n" +
                indent + "        {\n" +
                indent + "             throw std::runtime_error(\"std::string too long for length type [E109]\");\n" +
                indent + "        }\n" +
                indent + "        std::uint64_t lengthOfLengthField = %3$d;\n" +
                indent + "        std::uint64_t lengthPosition = position();\n" +
                indent + "        position(lengthPosition + lengthOfLengthField);\n" +
                indent + "        *((%4$s *)(m_buffer + lengthPosition)) = %5$s((%4$s)str.length());\n" +
                indent + "        std::uint64_t pos = position();\n" +
                indent + "        position(position() + str.length());\n" +
                indent + "        std::memcpy(m_buffer + pos, str.c_str(), str.length());\n" +
                indent + "        return *this;\n" +
                indent + "    }\n",
                className,
                propertyName,
                lengthOfLengthField,
                lengthCppType,
                formatByteOrderEncoding(lengthToken.encoding().byteOrder(), lengthToken.encoding().primitiveType()),
                lengthToken.encoding().applicableMaxValue().longValue()));

            i += token.componentTokenCount();
        }

        return sb;
    }

    private void generateVarDataDescriptors(
        final StringBuilder sb,
        final Token token,
        final String propertyName,
        final String characterEncoding,
        final Token lengthToken,
        final Integer sizeOfLengthField,
        final String lengthCppType,
        final String indent)
    {
        sb.append(String.format(
            "\n"  +
            indent + "    static const char *%1$sCharacterEncoding()\n" +
            indent + "    {\n" +
            indent + "        return \"%2$s\";\n" +
            indent + "    }\n\n",
            toLowerFirstChar(propertyName),
            characterEncoding));

        sb.append(String.format(
            indent + "    static SBE_CONSTEXPR const std::uint64_t %1$sSinceVersion(void)\n" +
            indent + "    {\n" +
            indent + "         return %2$d;\n" +
            indent + "    }\n\n" +
            indent + "    bool %1$sInActingVersion(void)\n" +
            indent + "    {\n" +
            indent + "#pragma GCC diagnostic push\n" +
            indent + "#pragma GCC diagnostic ignored \"-Wtautological-compare\"\n" +
            indent + "        return m_actingVersion >= %1$sSinceVersion();\n" +
            indent + "#pragma GCC diagnostic pop\n" +
            indent + "    }\n\n" +
            indent + "    static SBE_CONSTEXPR const std::uint16_t %1$sId(void)\n" +
            indent + "    {\n" +
            indent + "        return %3$d;\n" +
            indent + "    }\n\n",
            toLowerFirstChar(propertyName),
            token.version(),
            token.id()));

        sb.append(String.format(
            "\n" +
            indent + "    static SBE_CONSTEXPR const std::uint64_t %sHeaderLength()\n" +
            indent + "    {\n" +
            indent + "        return %d;\n" +
            indent + "    }\n",
            toLowerFirstChar(propertyName),
            sizeOfLengthField));

        sb.append(String.format(
            "\n" +
            indent + "    %4$s %1$sLength(void) const\n" +
            indent + "    {\n" +
            "%2$s" +
            indent + "        return %3$s(*((%4$s *)(m_buffer + position())));\n" +
            indent + "    }\n\n",
            toLowerFirstChar(propertyName),
            generateArrayFieldNotPresentCondition(token.version(), BASE_INDENT),
            formatByteOrderEncoding(lengthToken.encoding().byteOrder(), lengthToken.encoding().primitiveType()),
            lengthCppType));
    }

    private void generateChoiceSet(final List<Token> tokens) throws IOException
    {
        final String bitSetName = formatClassName(tokens.get(0).name());

        try (Writer out = outputManager.createOutput(bitSetName))
        {
            out.append(generateFileHeader(ir.namespaces(), bitSetName, null));
            out.append(generateClassDeclaration(bitSetName));
            out.append(generateFixedFlyweightCode(bitSetName, tokens.get(0).encodedLength()));

            out.append(String.format(
                "\n" +
                "    %1$s &clear(void)\n" +
                "    {\n" +
                "        *((%2$s *)(m_buffer + m_offset)) = 0;\n" +
                "        return *this;\n" +
                "    }\n\n",
                bitSetName,
                cppTypeName(tokens.get(0).encoding().primitiveType())));

            out.append(generateChoices(bitSetName, tokens.subList(1, tokens.size() - 1)));
            out.append("};\n");
            out.append(CppUtil.closingBraces(ir.namespaces().length)).append("#endif\n");
        }
    }

    private void generateEnum(final List<Token> tokens) throws IOException
    {
        final Token enumToken = tokens.get(0);
        final String enumName = formatClassName(tokens.get(0).name());

        try (Writer out = outputManager.createOutput(enumName))
        {
            out.append(generateFileHeader(ir.namespaces(), enumName, null));
            out.append(generateEnumDeclaration(enumName));

            out.append(generateEnumValues(tokens.subList(1, tokens.size() - 1), enumToken));

            out.append(generateEnumLookupMethod(tokens.subList(1, tokens.size() - 1), enumToken));

            out.append("};\n");
            out.append(CppUtil.closingBraces(ir.namespaces().length)).append("#endif\n");
        }
    }

    private void generateComposite(final List<Token> tokens) throws IOException
    {
        final String compositeName = formatClassName(tokens.get(0).name());
<commit_msg>[C++]: use contextualTypeName for enums, bitsets, and composites.<commit_after>/*
 * Copyright 2014 - 2016 Real Logic Ltd.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package uk.co.real_logic.sbe.generation.cpp;

import uk.co.real_logic.sbe.PrimitiveType;
import uk.co.real_logic.sbe.generation.CodeGenerator;
import org.agrona.generation.OutputManager;
import uk.co.real_logic.sbe.ir.*;
import org.agrona.Verify;

import java.io.IOException;
import java.io.Writer;
import java.util.ArrayList;
import java.util.List;

import static uk.co.real_logic.sbe.generation.cpp.CppUtil.*;
import static uk.co.real_logic.sbe.ir.GenerationUtil.collectVarData;
import static uk.co.real_logic.sbe.ir.GenerationUtil.collectGroups;
import static uk.co.real_logic.sbe.ir.GenerationUtil.collectFields;

public class CppGenerator implements CodeGenerator
{
    private static final String BASE_INDENT = "";
    private static final String INDENT = "    ";

    private final Ir ir;
    private final OutputManager outputManager;

    public CppGenerator(final Ir ir, final OutputManager outputManager)
        throws IOException
    {
        Verify.notNull(ir, "ir");
        Verify.notNull(outputManager, "outputManager");

        this.ir = ir;
        this.outputManager = outputManager;
    }

    public void generateMessageHeaderStub() throws IOException
    {
        final String messageHeader = "MessageHeader";

        try (Writer out = outputManager.createOutput(messageHeader))
        {
            final List<Token> tokens = ir.headerStructure().tokens();
            out.append(generateFileHeader(ir.namespaces(), messageHeader, null));
            out.append(generateClassDeclaration(messageHeader));
            out.append(generateFixedFlyweightCode(messageHeader, tokens.get(0).encodedLength()));
            out.append(generateCompositePropertyElements(
                messageHeader, tokens.subList(1, tokens.size() - 1), BASE_INDENT));
            out.append("};\n");
            out.append(CppUtil.closingBraces(ir.namespaces().length)).append("#endif\n");
        }
    }

    public List<String> generateTypeStubs() throws IOException
    {
        final List<String> typesToInclude = new ArrayList<>();

        for (final List<Token> tokens : ir.types())
        {
            switch (tokens.get(0).signal())
            {
                case BEGIN_ENUM:
                    generateEnum(tokens);
                    break;

                case BEGIN_SET:
                    generateChoiceSet(tokens);
                    break;

                case BEGIN_COMPOSITE:
                    generateComposite(tokens);
                    break;
            }

            typesToInclude.add(tokens.get(0).name());
        }

        return typesToInclude;
    }

    public List<String> generateTypesToIncludes(final List<Token> tokens)
    {
        final List<String> typesToInclude = new ArrayList<>();

        for (final Token token : tokens)
        {
            switch (token.signal())
            {
                case BEGIN_ENUM:
                case BEGIN_SET:
                case BEGIN_COMPOSITE:
                    typesToInclude.add(token.name());
                    break;
            }
        }

        return typesToInclude;
    }

    public void generate() throws IOException
    {
        generateMessageHeaderStub();
        final List<String> typesToInclude = generateTypeStubs();

        for (final List<Token> tokens : ir.messages())
        {
            final Token msgToken = tokens.get(0);
            final String className = formatClassName(msgToken.name());

            try (Writer out = outputManager.createOutput(className))
            {
                out.append(generateFileHeader(ir.namespaces(), className, typesToInclude));
                out.append(generateClassDeclaration(className));
                out.append(generateMessageFlyweightCode(className, msgToken));

                final List<Token> messageBody = tokens.subList(1, tokens.size() - 1);
                int i = 0;

                final List<Token> fields = new ArrayList<>();
                i = collectFields(messageBody, i, fields);

                final List<Token> groups = new ArrayList<>();
                i = collectGroups(messageBody, i, groups);

                final List<Token> varData = new ArrayList<>();
                collectVarData(messageBody, i, varData);

                final StringBuilder sb = new StringBuilder();
                out.append(generateFields(className, fields, BASE_INDENT));
                generateGroups(sb, groups, BASE_INDENT);
                out.append(sb);
                out.append(generateVarData(className, varData, BASE_INDENT));
                out.append("};\n");
                out.append(CppUtil.closingBraces(ir.namespaces().length)).append("#endif\n");
            }
        }
    }

    private void generateGroups(final StringBuilder sb, final List<Token> tokens, final String indent)
    {
        for (int i = 0, size = tokens.size(); i < size; i++)
        {
            final Token groupToken = tokens.get(i);
            if (groupToken.signal() != Signal.BEGIN_GROUP)
            {
                throw new IllegalStateException("tokens must begin with BEGIN_GROUP: token=" + groupToken);
            }

            final String groupName = groupToken.name();
            final String cppTypeForNumInGroup = cppTypeName(tokens.get(i + 3).encoding().primitiveType());

            generateGroupClassHeader(sb, groupName, tokens, i, indent + INDENT);

            ++i;
            final int groupHeaderTokenCount = tokens.get(i).componentTokenCount();
            i += groupHeaderTokenCount;

            final List<Token> fields = new ArrayList<>();
            i = collectFields(tokens, i, fields);
            sb.append(generateFields(groupName, fields, indent + INDENT));

            final List<Token> groups = new ArrayList<>();
            i = collectGroups(tokens, i, groups);
            generateGroups(sb, groups, indent + INDENT);

            final List<Token> varData = new ArrayList<>();
            i = collectVarData(tokens, i, varData);
            sb.append(generateVarData(formatClassName(groupName), varData, indent + INDENT));

            sb.append(indent).append("    };\n");
            sb.append(generateGroupProperty(groupName, groupToken, cppTypeForNumInGroup, indent));
        }
    }

    private static void generateGroupClassHeader(
        final StringBuilder sb, final String groupName, final List<Token> tokens, final int index, final String indent)
    {
        final String dimensionsClassName = formatClassName(tokens.get(index + 1).name());
        final int dimensionHeaderLength = tokens.get(index + 1).encodedLength();

        final int blockLength = tokens.get(index).encodedLength();
        final Token numInGroupToken = tokens.get(index + 3);
        final String cppTypeForBlockLength = cppTypeName(tokens.get(index + 2).encoding().primitiveType());
        final String cppTypeForNumInGroup = cppTypeName(numInGroupToken.encoding().primitiveType());

        sb.append(String.format(
            "\n" +
            indent + "class %1$s\n" +
            indent + "{\n" +
            indent + "private:\n" +
            indent + "    char *m_buffer;\n" +
            indent + "    std::uint64_t m_bufferLength;\n" +
            indent + "    std::uint64_t *m_positionPtr;\n" +
            indent + "    std::uint64_t m_blockLength;\n" +
            indent + "    std::uint64_t m_count;\n" +
            indent + "    std::uint64_t m_index;\n" +
            indent + "    std::uint64_t m_offset;\n" +
            indent + "    std::uint64_t m_actingVersion;\n" +
            indent + "    %2$s m_dimensions;\n\n" +
            indent + "public:\n\n",
            formatClassName(groupName), dimensionsClassName));

        sb.append(String.format(
            indent + "    inline void wrapForDecode(char *buffer, std::uint64_t *pos," +
                " const std::uint64_t actingVersion, const std::uint64_t bufferLength)\n" +
            indent + "    {\n" +
            indent + "        m_buffer = buffer;\n" +
            indent + "        m_bufferLength = bufferLength;\n" +
            indent + "        m_dimensions.wrap(m_buffer, *pos, actingVersion, bufferLength);\n" +
            indent + "        m_blockLength = m_dimensions.blockLength();\n" +
            indent + "        m_count = m_dimensions.numInGroup();\n" +
            indent + "        m_index = -1;\n" +
            indent + "        m_actingVersion = actingVersion;\n" +
            indent + "        m_positionPtr = pos;\n" +
            indent + "        *m_positionPtr = *m_positionPtr + %1$d;\n" +
            indent + "    }\n\n",
            dimensionHeaderLength));

        sb.append(String.format(
            indent + "    inline void wrapForEncode(char *buffer, const %3$s count," +
                " std::uint64_t *pos, const std::uint64_t actingVersion, const std::uint64_t bufferLength)\n" +
            indent + "    {\n" +
            indent + "        if (count < %5$d || count > %6$d)\n" +
            indent + "        {\n" +
            indent + "            throw std::runtime_error(\"count outside of allowed range [E110]\");\n" +
            indent + "        }\n" +
            indent + "        m_buffer = buffer;\n" +
            indent + "        m_bufferLength = bufferLength;\n" +
            indent + "        m_dimensions.wrap(m_buffer, *pos, actingVersion, bufferLength);\n" +
            indent + "        m_dimensions.blockLength((%1$s)%2$d);\n" +
            indent + "        m_dimensions.numInGroup((%3$s)count);\n" +
            indent + "        m_index = -1;\n" +
            indent + "        m_count = count;\n" +
            indent + "        m_blockLength = %2$d;\n" +
            indent + "        m_actingVersion = actingVersion;\n" +
            indent + "        m_positionPtr = pos;\n" +
            indent + "        *m_positionPtr = *m_positionPtr + %4$d;\n" +
            indent + "    }\n\n",
            cppTypeForBlockLength, blockLength, cppTypeForNumInGroup, dimensionHeaderLength,
            numInGroupToken.encoding().applicableMinValue().longValue(),
            numInGroupToken.encoding().applicableMaxValue().longValue()));

        sb.append(String.format(
            indent + "    static SBE_CONSTEXPR const std::uint64_t sbeHeaderSize()\n" +
            indent + "    {\n" +
            indent + "        return %1$d;\n" +
            indent + "    }\n\n" +
            indent + "    static SBE_CONSTEXPR const std::uint64_t sbeBlockLength()\n" +
            indent + "    {\n" +
            indent + "        return %2$d;\n" +
            indent + "    }\n\n" +
            indent + "    std::uint64_t position(void) const\n" +
            indent + "    {\n" +
            indent + "        return *m_positionPtr;\n" +
            indent + "    }\n\n" +
            indent + "    void position(const std::uint64_t position)\n" +
            indent + "    {\n" +
            indent + "        if (SBE_BOUNDS_CHECK_EXPECT((position > m_bufferLength), false))\n" +
            indent + "        {\n" +
            indent + "             throw std::runtime_error(\"buffer too short [E100]\");\n" +
            indent + "        }\n" +
            indent + "        *m_positionPtr = position;\n" +
            indent + "    }\n\n" +
            indent + "    inline std::uint64_t count(void) const\n" +
            indent + "    {\n" +
            indent + "        return m_count;\n" +
            indent + "    }\n\n" +
            indent + "    inline bool hasNext(void) const\n" +
            indent + "    {\n" +
            indent + "        return m_index + 1 < m_count;\n" +
            indent + "    }\n\n" +
            indent + "    inline %3$s &next(void)\n" +
            indent + "    {\n" +
            indent + "        m_offset = *m_positionPtr;\n" +
            indent + "        if (SBE_BOUNDS_CHECK_EXPECT(( (m_offset + m_blockLength) > m_bufferLength ), false))\n" +
            indent + "        {\n" +
            indent + "            throw std::runtime_error(\"" +
                "buffer too short to support next group index [E108]\");\n" +
            indent + "        }\n" +
            indent + "        *m_positionPtr = m_offset + m_blockLength;\n" +
            indent + "        ++m_index;\n\n" +
            indent + "        return *this;\n" +
            indent + "    }\n\n",
            dimensionHeaderLength, blockLength, formatClassName(groupName)));

        sb.append(String.format(
            indent + "#if __cplusplus < 201103L\n" +
            indent + "    template<class Func> inline void forEach(Func& func)\n" +
            indent + "    {\n" +
            indent + "        while (hasNext())\n" +
            indent + "        {\n" +
            indent + "            next(); func(*this);\n" +
            indent + "        }\n" +
            indent + "    }\n\n" +
            indent + "#else\n" +
            indent + "    template<class Func> inline void forEach(Func&& func)\n" +
            indent + "    {\n" +
            indent + "        while (hasNext())\n" +
            indent + "        {\n" +
            indent + "            next(); func(*this);\n" +
            indent + "        }\n" +
            indent + "    }\n\n" +
            indent + "#endif\n\n",
            formatClassName(groupName)));
    }

    private static CharSequence generateGroupProperty(
        final String groupName, final Token token, final String cppTypeForNumInGroup, final String indent)
    {
        final StringBuilder sb = new StringBuilder();

        final String className = formatClassName(groupName);
        final String propertyName = formatPropertyName(groupName);

        sb.append(String.format(
            "\n" +
            "private:\n" +
            indent + "    %1$s m_%2$s;\n\n" +
            "public:\n",
            className,
            propertyName));

        sb.append(String.format(
            "\n" +
            indent + "    static SBE_CONSTEXPR const std::uint16_t %1$sId(void)\n" +
            indent + "    {\n" +
            indent + "        return %2$d;\n" +
            indent + "    }\n\n",
            groupName,
            token.id()));

        sb.append(String.format(
            "\n" +
            indent + "    inline %1$s &%2$s(void)\n" +
            indent + "    {\n" +
            indent + "        m_%2$s.wrapForDecode(m_buffer, m_positionPtr, m_actingVersion, m_bufferLength);\n" +
            indent + "        return m_%2$s;\n" +
            indent + "    }\n",
            className,
            propertyName));

        sb.append(String.format(
            "\n" +
            indent + "    %1$s &%2$sCount(const %3$s count)\n" +
            indent + "    {\n" +
            indent + "        m_%2$s.wrapForEncode(" +
                "m_buffer, count, m_positionPtr, m_actingVersion, m_bufferLength);\n" +
            indent + "        return m_%2$s;\n" +
            indent + "    }\n\n",
            className,
            propertyName,
            cppTypeForNumInGroup));

        sb.append(String.format(
            indent + "    static SBE_CONSTEXPR const std::uint64_t %1$sSinceVersion(void)\n" +
            indent + "    {\n" +
            indent + "         return %2$d;\n" +
            indent + "    }\n\n" +
            indent + "    bool %1$sInActingVersion(void)\n" +
            indent + "    {\n" +
            indent + "#pragma GCC diagnostic push\n" +
            indent + "#pragma GCC diagnostic ignored \"-Wtautological-compare\"\n" +
            indent + "        return m_actingVersion >= %1$sSinceVersion();\n" +
            indent + "#pragma GCC diagnostic pop\n" +
            indent + "    }\n",
            propertyName,
            token.version()));

        return sb;
    }

    private CharSequence generateVarData(final String className, final List<Token> tokens, final String indent)
    {
        final StringBuilder sb = new StringBuilder();

        for (int i = 0, size = tokens.size(); i < size;)
        {
            final Token token = tokens.get(i);
            if (token.signal() != Signal.BEGIN_VAR_DATA)
            {
                throw new IllegalStateException("tokens must begin with BEGIN_VAR_DATA: token=" + token);
            }

            final String propertyName = toUpperFirstChar(token.name());
            final String characterEncoding = tokens.get(i + 3).encoding().characterEncoding();
            final Token lengthToken = tokens.get(i + 2);
            final int lengthOfLengthField = lengthToken.encodedLength();
            final String lengthCppType = cppTypeName(lengthToken.encoding().primitiveType());

            generateFieldMetaAttributeMethod(sb, token, indent);

            generateVarDataDescriptors(
                sb, token, propertyName, characterEncoding, lengthToken, lengthOfLengthField, lengthCppType, indent);

            sb.append(String.format(
                indent + "    const char *%1$s(void)\n" +
                indent + "    {\n" +
                    "%2$s" +
                indent + "         const char *fieldPtr = (m_buffer + position() + %3$d);\n" +
                indent + "         position(position() + %3$d + *((%4$s *)(m_buffer + position())));\n" +
                indent + "         return fieldPtr;\n" +
                indent + "    }\n\n",
                formatPropertyName(propertyName),
                generateTypeFieldNotPresentCondition(token.version(), BASE_INDENT),
                lengthOfLengthField,
                lengthCppType));

            sb.append(String.format(
                indent + "    std::uint64_t get%1$s(char *dst, const std::uint64_t length)\n" +
                indent + "    {\n" +
                    "%2$s" +
                indent + "        std::uint64_t lengthOfLengthField = %3$d;\n" +
                indent + "        std::uint64_t lengthPosition = position();\n" +
                indent + "        position(lengthPosition + lengthOfLengthField);\n" +
                indent + "        std::uint64_t dataLength = %4$s(*((%5$s *)(m_buffer + lengthPosition)));\n" +
                indent + "        std::uint64_t bytesToCopy = (length < dataLength) ? length : dataLength;\n" +
                indent + "        std::uint64_t pos = position();\n" +
                indent + "        position(position() + dataLength);\n" +
                indent + "        std::memcpy(dst, m_buffer + pos, bytesToCopy);\n" +
                indent + "        return bytesToCopy;\n" +
                indent + "    }\n\n",
                propertyName,
                generateArrayFieldNotPresentCondition(token.version(), BASE_INDENT),
                lengthOfLengthField,
                formatByteOrderEncoding(lengthToken.encoding().byteOrder(), lengthToken.encoding().primitiveType()),
                lengthCppType));

            sb.append(String.format(
                indent + "    %5$s &put%1$s(const char *src, const %3$s length)\n" +
                indent + "    {\n" +
                indent + "        std::uint64_t lengthOfLengthField = %2$d;\n" +
                indent + "        std::uint64_t lengthPosition = position();\n" +
                indent + "        position(lengthPosition + lengthOfLengthField);\n" +
                indent + "        *((%3$s *)(m_buffer + lengthPosition)) = %4$s(length);\n" +
                indent + "        std::uint64_t pos = position();\n" +
                indent + "        position(position() + length);\n" +
                indent + "        std::memcpy(m_buffer + pos, src, length);\n" +
                indent + "        return *this;\n" +
                indent + "    }\n\n",
                propertyName,
                lengthOfLengthField,
                lengthCppType,
                formatByteOrderEncoding(lengthToken.encoding().byteOrder(), lengthToken.encoding().primitiveType()),
                className));

            sb.append(String.format(
                indent + "    const std::string get%1$sAsString()\n" +
                indent + "    {\n" +
                "%2$s" +
                indent + "        std::uint64_t lengthOfLengthField = %3$d;\n" +
                indent + "        std::uint64_t lengthPosition = position();\n" +
                indent + "        position(lengthPosition + lengthOfLengthField);\n" +
                indent + "        std::uint64_t dataLength = %4$s(*((%5$s *)(m_buffer + lengthPosition)));\n" +
                indent + "        std::uint64_t pos = position();\n" +
                indent + "        const std::string result(m_buffer + pos, dataLength);\n" +
                indent + "        position(position() + dataLength);\n" +
                indent + "        return result;\n" +
                indent + "    }\n\n",
                propertyName,
                generateStringNotPresentCondition(token.version(), BASE_INDENT),
                lengthOfLengthField,
                formatByteOrderEncoding(lengthToken.encoding().byteOrder(), lengthToken.encoding().primitiveType()),
                lengthCppType));

            sb.append(String.format(
                indent + "    %1$s &put%2$s(const std::string& str)\n" +
                indent + "    {\n" +
                indent + "        if (str.length() > %6$d)\n" +
                indent + "        {\n" +
                indent + "             throw std::runtime_error(\"std::string too long for length type [E109]\");\n" +
                indent + "        }\n" +
                indent + "        std::uint64_t lengthOfLengthField = %3$d;\n" +
                indent + "        std::uint64_t lengthPosition = position();\n" +
                indent + "        position(lengthPosition + lengthOfLengthField);\n" +
                indent + "        *((%4$s *)(m_buffer + lengthPosition)) = %5$s((%4$s)str.length());\n" +
                indent + "        std::uint64_t pos = position();\n" +
                indent + "        position(position() + str.length());\n" +
                indent + "        std::memcpy(m_buffer + pos, str.c_str(), str.length());\n" +
                indent + "        return *this;\n" +
                indent + "    }\n",
                className,
                propertyName,
                lengthOfLengthField,
                lengthCppType,
                formatByteOrderEncoding(lengthToken.encoding().byteOrder(), lengthToken.encoding().primitiveType()),
                lengthToken.encoding().applicableMaxValue().longValue()));

            i += token.componentTokenCount();
        }

        return sb;
    }

    private void generateVarDataDescriptors(
        final StringBuilder sb,
        final Token token,
        final String propertyName,
        final String characterEncoding,
        final Token lengthToken,
        final Integer sizeOfLengthField,
        final String lengthCppType,
        final String indent)
    {
        sb.append(String.format(
            "\n"  +
            indent + "    static const char *%1$sCharacterEncoding()\n" +
            indent + "    {\n" +
            indent + "        return \"%2$s\";\n" +
            indent + "    }\n\n",
            toLowerFirstChar(propertyName),
            characterEncoding));

        sb.append(String.format(
            indent + "    static SBE_CONSTEXPR const std::uint64_t %1$sSinceVersion(void)\n" +
            indent + "    {\n" +
            indent + "         return %2$d;\n" +
            indent + "    }\n\n" +
            indent + "    bool %1$sInActingVersion(void)\n" +
            indent + "    {\n" +
            indent + "#pragma GCC diagnostic push\n" +
            indent + "#pragma GCC diagnostic ignored \"-Wtautological-compare\"\n" +
            indent + "        return m_actingVersion >= %1$sSinceVersion();\n" +
            indent + "#pragma GCC diagnostic pop\n" +
            indent + "    }\n\n" +
            indent + "    static SBE_CONSTEXPR const std::uint16_t %1$sId(void)\n" +
            indent + "    {\n" +
            indent + "        return %3$d;\n" +
            indent + "    }\n\n",
            toLowerFirstChar(propertyName),
            token.version(),
            token.id()));

        sb.append(String.format(
            "\n" +
            indent + "    static SBE_CONSTEXPR const std::uint64_t %sHeaderLength()\n" +
            indent + "    {\n" +
            indent + "        return %d;\n" +
            indent + "    }\n",
            toLowerFirstChar(propertyName),
            sizeOfLengthField));

        sb.append(String.format(
            "\n" +
            indent + "    %4$s %1$sLength(void) const\n" +
            indent + "    {\n" +
            "%2$s" +
            indent + "        return %3$s(*((%4$s *)(m_buffer + position())));\n" +
            indent + "    }\n\n",
            toLowerFirstChar(propertyName),
            generateArrayFieldNotPresentCondition(token.version(), BASE_INDENT),
            formatByteOrderEncoding(lengthToken.encoding().byteOrder(), lengthToken.encoding().primitiveType()),
            lengthCppType));
    }

    private void generateChoiceSet(final List<Token> tokens) throws IOException
    {
        final String bitSetName = formatClassName(tokens.get(0).contextualTypeName());

        try (Writer out = outputManager.createOutput(bitSetName))
        {
            out.append(generateFileHeader(ir.namespaces(), bitSetName, null));
            out.append(generateClassDeclaration(bitSetName));
            out.append(generateFixedFlyweightCode(bitSetName, tokens.get(0).encodedLength()));

            out.append(String.format(
                "\n" +
                "    %1$s &clear(void)\n" +
                "    {\n" +
                "        *((%2$s *)(m_buffer + m_offset)) = 0;\n" +
                "        return *this;\n" +
                "    }\n\n",
                bitSetName,
                cppTypeName(tokens.get(0).encoding().primitiveType())));

            out.append(generateChoices(bitSetName, tokens.subList(1, tokens.size() - 1)));
            out.append("};\n");
            out.append(CppUtil.closingBraces(ir.namespaces().length)).append("#endif\n");
        }
    }

    private void generateEnum(final List<Token> tokens) throws IOException
    {
        final Token enumToken = tokens.get(0);
        final String enumName = formatClassName(tokens.get(0).contextualTypeName());

        try (Writer out = outputManager.createOutput(enumName))
        {
            out.append(generateFileHeader(ir.namespaces(), enumName, null));
            out.append(generateEnumDeclaration(enumName));

            out.append(generateEnumValues(tokens.subList(1, tokens.size() - 1), enumToken));

            out.append(generateEnumLookupMethod(tokens.subList(1, tokens.size() - 1), enumToken));

            out.append("};\n");
            out.append(CppUtil.closingBraces(ir.namespaces().length)).append("#endif\n");
        }
    }

    private void generateComposite(final List<Token> tokens) throws IOException
    {
        final String compositeName = formatClassName(tokens.get(0).contextualTypeName());

{'fraction_non_alphanumeric': 0.12799190107744596, 'fraction_numerical': 0.007881987128498084, 'mean_word_length': 1.7173453848798939, 'pattern_counts': {'":': 0, '<': 79, '<?xml version=': 0, '>': 79, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3636075', 'n_tokens_mistral': 15652, 'n_tokens_neox': 14988, 'n_words': 3087}