<commit_before>package io.mandrel.common.export;

import io.mandrel.common.content.FieldExtractor;
import io.mandrel.common.store.Document;

import java.io.IOException;
import java.io.Writer;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import lombok.Data;

import org.springframework.util.CollectionUtils;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;

@Data
public class JsonExporter implements Exporter {

	private static final long serialVersionUID = -410119107553820985L;

	@Override
	public void export(Stream<Document> documents, List<FieldExtractor> fields, Writer writer) throws IOException {
		ObjectMapper mapper = new ObjectMapper();

		ArrayNode arrayNode = mapper.createArrayNode();

		List<String> headers = fields.stream().map(field -> field.getName()).collect(Collectors.toList());

		documents.forEach(doc -> {
			ObjectNode rootNode = arrayNode.objectNode();
			for (String header : headers) {
				List<? extends Object> values = doc.get(header);
				if (!CollectionUtils.isEmpty(values)) {
					rootNode.putPOJO(header, values);
				}
			}
		});
	}

	@Override
	public String contentType() {
		return "application/json";
	}
}<commit_msg>Write data in dedicated writer<commit_after>package io.mandrel.common.export;

import io.mandrel.common.content.FieldExtractor;
import io.mandrel.common.store.Document;

import java.io.IOException;
import java.io.Writer;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import lombok.Data;

import org.springframework.util.CollectionUtils;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;

@Data
public class JsonExporter implements Exporter {

	private static final long serialVersionUID = -410119107553820985L;

	@Override
	public void export(Stream<Document> documents, List<FieldExtractor> fields, Writer writer) throws IOException {
		ObjectMapper mapper = new ObjectMapper();

		ArrayNode arrayNode = mapper.createArrayNode();

		List<String> headers = fields.stream().map(field -> field.getName()).collect(Collectors.toList());

		documents.forEach(doc -> {
			ObjectNode rootNode = arrayNode.objectNode();
			for (String header : headers) {
				List<? extends Object> values = doc.get(header);
				if (!CollectionUtils.isEmpty(values)) {
					rootNode.putPOJO(header, values);
				}
			}
		});

		mapper.writeValue(writer, arrayNode);
	}

	@Override
	public String contentType() {
		return "application/json";
	}
}
{'fraction_non_alphanumeric': 0.114529280948851, 'fraction_numerical': 0.013343217197924388, 'mean_word_length': 5.09255079006772, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4936694', 'n_tokens_mistral': 904, 'n_tokens_neox': 786, 'n_words': 172}