<commit_before>package org.openlca.git.find;

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonToken;

class MetaDataParser {

	private static final Logger log = LoggerFactory.getLogger(MetaDataParser.class);

	static Map<String, Object> parse(InputStream json, String... fields) {
		var defs = Stream.of(fields).map(field -> new FieldDefinition(field)).toList();
		return parse(json, defs);
	}

	static Map<String, Object> parse(InputStream json, List<FieldDefinition> fields) {
		if (fields == null || fields.size() == 0)
			return new HashMap<>();
		var matched = 0;
		try {
			var factory = new JsonFactory();
			var parser = factory.createParser(json);
			var parents = new ArrayList<String>();
			var map = new HashMap<String, Object>();
			String fieldName = null;
			while (matched < fields.size() && !parser.isClosed()) {
				var jsonToken = parser.nextToken();
				if (JsonToken.START_OBJECT.equals(jsonToken)) {
					if (fieldName != null) {
						parents.add(fieldName);
					}
					continue;
				}
				if (JsonToken.END_OBJECT.equals(jsonToken)) {
					if (!parents.isEmpty()) {
						fieldName = parents.remove(parents.size() - 1);
					}
					continue;
				}
				if (!JsonToken.FIELD_NAME.equals(jsonToken))
					continue;
				fieldName = parser.getCurrentName();
				var match = findMatch(fields, parents, fieldName);
				if (match != null) {
					jsonToken = parser.nextToken();
					map.put(match.name, match.parser.apply(parser.getValueAsString()));
					matched++;
				}
			}
			return map;
		} catch (IOException e) {
			log.error("Error parsing dataset", e);
			return new HashMap<>();
		}<commit_msg>Added support for arrays<commit_after>package org.openlca.git.find;

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonToken;

class MetaDataParser {

	private static final Logger log = LoggerFactory.getLogger(MetaDataParser.class);

	static Map<String, Object> parse(InputStream json, String... fields) {
		var defs = Stream.of(fields).map(field -> new FieldDefinition(field)).toList();
		return parse(json, defs);
	}

	static Map<String, Object> parse(InputStream json, List<FieldDefinition> fields) {
		if (fields == null || fields.size() == 0)
			return new HashMap<>();
		var matched = 0;
		try {
			var factory = new JsonFactory();
			var parser = factory.createParser(json);
			var parents = new ArrayList<String>();
			var map = new HashMap<String, Object>();
			String fieldName = null;
			var arrayDepth = 0;
			while ((matched < fields.size() || arrayDepth > 0) && !parser.isClosed()) {
				var jsonToken = parser.nextToken();
				if (JsonToken.START_ARRAY.equals(jsonToken)) {
					arrayDepth++;
					continue;
				}
				if (JsonToken.END_ARRAY.equals(jsonToken)) {
					arrayDepth--;
					continue;
				}
				if (JsonToken.START_OBJECT.equals(jsonToken)) {
					if (fieldName != null) {
						parents.add(fieldName);
					}
					continue;
				}
				if (JsonToken.END_OBJECT.equals(jsonToken)) {
					if (!parents.isEmpty()) {
						fieldName = parents.remove(parents.size() - 1);
					}
					continue;
				}
				if (!JsonToken.FIELD_NAME.equals(jsonToken))
					continue;
				fieldName = parser.getCurrentName();
				var match = findMatch(fields, parents, fieldName);
				if (match != null) {
					jsonToken = parser.nextToken();
					if (arrayDepth > 0) {
						@SuppressWarnings("unchecked")
						var list = (List<String>) map.computeIfAbsent(match.name, n -> new ArrayList<String>());
						list.add(parser.getValueAsString());
					} else {
						map.put(match.name, match.parser.apply(parser.getValueAsString()));
					}
					matched++;
				}
			}
			return map;
		} catch (IOException e) {
			log.error("Error parsing dataset", e);
			return new HashMap<>();
		}
{'fraction_non_alphanumeric': 0.13847235238987815, 'fraction_numerical': 0.0030459231490159327, 'mean_word_length': 3.5903225806451613, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '827801', 'n_tokens_mistral': 1601, 'n_tokens_neox': 1300, 'n_words': 303}