<commit_before>package net.masterthought.cucumber.json.deserializers;

import java.io.IOException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;

import org.codehaus.plexus.util.Base64;

import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;

import net.masterthought.cucumber.Configuration;
import net.masterthought.cucumber.ValidationException;
import net.masterthought.cucumber.json.Embedding;

/**
 * Deserializes embedding and stores it in attachment directory.
 * 
 * @author <NAME> (<EMAIL>)
 */
public class EmbeddingDeserializer extends CucumberJsonDeserializer<Embedding> {

    @Override
    public Embedding deserialize(JsonParser parser, Configuration configuration)
            throws IOException, JsonProcessingException {
        JsonNode rootNode = parser.getCodec().readTree(parser);
        String data = rootNode.get("data").asText();
        String mimeType = rootNode.get("mime_type").asText();

        Embedding embedding = new Embedding(mimeType, data);
        storeEmbedding(embedding, configuration);

        return embedding;
    }

    private void storeEmbedding(Embedding embedding, Configuration configuration) {
        Path file = FileSystems.getDefault().getPath(configuration.getEmbeddingDirectory().getAbsolutePath(),
                embedding.getFileId() + "." + embedding.getExtension());
        try {
            Files.write(file, Base64.decodeBase64(embedding.getData().getBytes()));
        } catch (IOException e) {
            throw new ValidationException(e);
        }
    }

}<commit_msg>Fixing embedding deserialization charset.<commit_after>package net.masterthought.cucumber.json.deserializers;

import java.io.IOException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.charset.StandardCharsets;
import org.codehaus.plexus.util.Base64;

import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;

import net.masterthought.cucumber.Configuration;
import net.masterthought.cucumber.ValidationException;
import net.masterthought.cucumber.json.Embedding;

/**
 * Deserializes embedding and stores it in attachment directory.
 * 
 * @author <NAME> (<EMAIL>)
 */
public class EmbeddingDeserializer extends CucumberJsonDeserializer<Embedding> {

    @Override
    public Embedding deserialize(JsonParser parser, Configuration configuration)
            throws IOException, JsonProcessingException {
        JsonNode rootNode = parser.getCodec().readTree(parser);
        String data = rootNode.get("data").asText();
        String mimeType = rootNode.get("mime_type").asText();

        Embedding embedding = new Embedding(mimeType, data);
        storeEmbedding(embedding, configuration);

        return embedding;
    }

    private void storeEmbedding(Embedding embedding, Configuration configuration) {
        Path file = FileSystems.getDefault().getPath(configuration.getEmbeddingDirectory().getAbsolutePath(),
                embedding.getFileId() + "." + embedding.getExtension());
        try {
            Files.write(file, Base64.decodeBase64(embedding.getData().getBytes(StandardCharsets.UTF_8)));
        } catch (IOException e) {
            throw new ValidationException(e);
        }
    }

}
{'fraction_non_alphanumeric': 0.1032183241519281, 'fraction_numerical': 0.0037692084662220935, 'mean_word_length': 4.029154518950437, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2409173', 'n_tokens_mistral': 990, 'n_tokens_neox': 922, 'n_words': 181}