<commit_before>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.document.json.readers;

import com.fasterxml.jackson.core.JsonToken;
import com.yahoo.document.Field;
import com.yahoo.document.datatypes.FieldValue;
import com.yahoo.document.datatypes.StructuredFieldValue;
import com.yahoo.document.json.JsonReaderException;
import com.yahoo.document.json.TokenBuffer;

import static com.yahoo.document.json.readers.SingleValueReader.readSingleValue;

public class StructReader {

    public static void fillStruct(TokenBuffer buffer, StructuredFieldValue parent) {
        // do note the order of initializing initNesting and token is relevant for empty docs
        int initNesting = buffer.nesting();
        buffer.next();

        while (buffer.nesting() >= initNesting) {
            Field f = getField(buffer, parent);
            try {
                // skip fields set to null
                if (buffer.currentToken() != JsonToken.VALUE_NULL) {
                    FieldValue v = readSingleValue(buffer, f.getDataType());
                    parent.setFieldValue(f, v);
                }
                buffer.next();
            } catch (IllegalArgumentException e) {
                throw new JsonReaderException(f, e);
            }
        }
    }

    public static Field getField(TokenBuffer buffer, StructuredFieldValue parent) {
        Field field = parent.getField(buffer.currentName());
        if (field == null)
            throw new IllegalArgumentException("No field '" + buffer.currentName() + "' in the structure of type '" +
                                               parent.getDataType().getDataTypeName() + "'");
        return field;
    }

}<commit_msg>Dump more information to help debugging.<commit_after>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.document.json.readers;

import com.fasterxml.jackson.core.JsonToken;
import com.yahoo.document.Field;
import com.yahoo.document.datatypes.FieldValue;
import com.yahoo.document.datatypes.StructuredFieldValue;
import com.yahoo.document.json.JsonReaderException;
import com.yahoo.document.json.TokenBuffer;

import static com.yahoo.document.json.readers.SingleValueReader.readSingleValue;

public class StructReader {

    public static void fillStruct(TokenBuffer buffer, StructuredFieldValue parent) {
        // do note the order of initializing initNesting and token is relevant for empty docs
        int initNesting = buffer.nesting();
        buffer.next();

        while (buffer.nesting() >= initNesting) {
            Field f = getField(buffer, parent);
            try {
                // skip fields set to null
                if (buffer.currentToken() != JsonToken.VALUE_NULL) {
                    FieldValue v = readSingleValue(buffer, f.getDataType());
                    parent.setFieldValue(f, v);
                }
                buffer.next();
            } catch (IllegalArgumentException e) {
                throw new JsonReaderException(f, e);
            }
        }
    }

    public static Field getField(TokenBuffer buffer, StructuredFieldValue parent) {
        Field field = parent.getField(buffer.currentName());
        if (field == null) {
            throw new IllegalArgumentException("No field '" + buffer.currentName() + "' in the structure of type '" +
                    parent.getDataType().getDataTypeName() + "', which has the fields:" + parent.getDataType().getFields());
        }
        return field;
    }

}
{'fraction_non_alphanumeric': 0.0943188086045229, 'fraction_numerical': 0.003309431880860452, 'mean_word_length': 2.4875, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '983800', 'n_tokens_mistral': 964, 'n_tokens_neox': 903, 'n_words': 252}