<commit_before>package org.wikidata.wdtk.dumpfiles;

/*
 * #%L
 * Wikidata Toolkit Dump File Handling
 * %%
 * Copyright (C) 2014 Wikidata Toolkit Developers
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */

import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang3.Validate;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.wikidata.wdtk.datamodel.interfaces.Claim;
import org.wikidata.wdtk.datamodel.interfaces.DataObjectFactory;
import org.wikidata.wdtk.datamodel.interfaces.DatatypeIdValue;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
import org.wikidata.wdtk.datamodel.interfaces.ItemDocument;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.NoValueSnak;
import org.wikidata.wdtk.datamodel.interfaces.PropertyDocument;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.QuantityValue;
import org.wikidata.wdtk.datamodel.interfaces.Reference;
import org.wikidata.wdtk.datamodel.interfaces.SiteLink;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.SomeValueSnak;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.StatementGroup;
import org.wikidata.wdtk.datamodel.interfaces.StatementRank;
import org.wikidata.wdtk.datamodel.interfaces.StringValue;
import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
import org.wikidata.wdtk.datamodel.interfaces.Value;
import org.wikidata.wdtk.datamodel.interfaces.ValueSnak;

// TODO logging support
/**
 * This class provides methods to convert dump-file JSON objects into
 * representations according to the WDTK data model. Since the converted JSON
 * normally belongs to the same domain, the base IRI is represented as an
 * attribute.
 * 
 * @author <NAME>
 * 
 */
public class JsonConverter {

	public static final String PREFIX_ITEM = "Q";
	public static final String PREFIX_PROPERTY = "P";

	// Keys that might occur in the documents
	private static final String KEY_LABEL = "label";
	private static final String KEY_ENTITY = "entity";
	private static final String KEY_DESCRIPTION = "description";
	private static final String KEY_ALIAS = "aliases";
	private static final String KEY_DATATYPE = "datatype";
	private static final String KEY_CLAIM = "claims";
	private static final String KEY_LINK = "links";

	private final DataObjectFactory factory;
	private String baseIri = "";
	private final MonolingualTextValueHandler mltvHandler;

	static final Logger logger = LoggerFactory.getLogger(JsonConverter.class);

	/**
	 * Creates a new instance of the JsonConverter. For the <i>baseIri</i> see
	 * also {@link org.wikidata.wdtk.datamodel.interfaces.ItemIdValue}.
	 * 
	 * @param baseIri
	 *            the base IRI to be used for entities
	 * @param factory
	 *            the DataObjectFactory to be used to construct objects of the
	 *            data model
	 */
	public JsonConverter(String baseIri, DataObjectFactory factory) {
		this.setBaseIri(baseIri);
		this.factory = factory;

		mltvHandler = new MonolingualTextValueHandler(this.factory);
	}

	/**
	 * Attempts to parse a given JSON object into an instance of
	 * PropertyDocument.
	 * 
	 * @param jsonObject
	 *            the JSON object to convert. Must represent a property document
	 *            and therefore contain the keys "entity", "label",
	 *            "description", "aliases" and "datatype".
	 * @param propertyIdString
	 *            is a string containing the id of the property provided by
	 *            external sources; if null is given, the information will be
	 *            extracted from the JSON
	 * @return the PropertyDocument as described in the JSON.
	 * @throws NullPointerException
	 *             if toConvert was null.
	 * @throws JSONException
	 *             if the JSON object did not contain a key it should have had.
	 */
	public PropertyDocument convertToPropertyDocument(JSONObject jsonObject,
			String propertyIdString) throws JSONException {

		if (jsonObject.length() == 0) { // if the JSON object is empty
			throw new JSONException("The JSON to convert was empty");
		}

		PropertyDocument result;

		PropertyIdValue propertyId;

		if (propertyIdString != null) {
			propertyId = this.getPropertyIdValue(propertyIdString);
		} else {
			propertyId = this.getPropertyIdFromTopLevel(jsonObject);
		}

		List<MonolingualTextValue> labels = this.getMltv(KEY_LABEL, jsonObject);

		List<MonolingualTextValue> descriptions = this.getMltv(KEY_DESCRIPTION,
				jsonObject);

		List<MonolingualTextValue> aliases = this
				.getMltv(KEY_ALIAS, jsonObject);

		String jsonDataTypeId = jsonObject.getString(KEY_DATATYPE);
		DatatypeIdValue datatypeId = this.getDatatypeIdValue(jsonDataTypeId);

		result = this.factory.getPropertyDocument(propertyId, labels,
				descriptions, aliases, datatypeId);
		return result;
	}

	/**
	 * Attempts to parse a given JSON object into an instance of ItemDocument.
	 * 
	 * @param jsonObject
	 *            the JSON object to convert. Must represent an item document
	 *            and therefore might contain the keys "entity", "label",
	 *            "description", "aliases", "claims" and "links".
	 * @param propertyIdString
	 *            is a string containing the id of the item provided by external
	 *            sources; if null is given, the information will be extracted
	 *            from the JSON
	 * @return the ItemDocument as described in the JSON.
	 * @throws NullPointerException
	 *             if toConvert was null.
	 * @throws JSONException
	 *             if the JSON object did not contain a key it should have had.
	 */
	public ItemDocument convertToItemDocument(JSONObject jsonObject,
			String itemIdString) throws JSONException, NullPointerException {

		if (jsonObject.length() == 0) { // if the JSON object is empty
			throw new JSONException("The JSON to convert was empty");
		}

		ItemIdValue itemId;
		if (itemIdString != null) {
			itemId = this.getItemIdValue(itemIdString);
		} else {
			itemId = this.getItemIdFromTopLevel(jsonObject);
		}
		List<MonolingualTextValue> labels = this.getMltv(KEY_LABEL, jsonObject);

		List<MonolingualTextValue> descriptions = this.getMltv(KEY_DESCRIPTION,
				jsonObject);

		List<MonolingualTextValue> aliases = this
				.getMltv(KEY_ALIAS, jsonObject);

		List<StatementGroup> statements = new LinkedList<>();
		if (jsonObject.has(KEY_CLAIM)) {
			JSONArray jsonStatements = jsonObject.getJSONArray(KEY_CLAIM);
			statements = this.getStatementGroups(jsonStatements, itemId);
		}

		Map<String, SiteLink> siteLinks = new HashMap<>();

		if (jsonObject.has(KEY_LINK)) {
			JSONArray linkArray = jsonObject.optJSONArray(KEY_LINK);
			if (linkArray == null) {
				JSONObject jsonLinks = jsonObject.getJSONObject(KEY_LINK);
				siteLinks = this.getSiteLinks(jsonLinks);
			}
		}

		ItemDocument result = factory.getItemDocument(itemId, labels,
				descriptions, aliases, statements, siteLinks);
		return result;
	}

	/**
	 * Attempts to get the item id value from the given JSON-object. Note that
	 * in old dumps the entity is not an array but a string with appropriate
	 * prefix in lower case.
	 * 
	 * @param topLevel
	 *            is the JSON object describing the whole entity
	 * @return the items id as ItemIdValue
	 * @throws JSONException
	 *             if the topLevel does not contain the key "entity"
	 */
	private ItemIdValue getItemIdFromTopLevel(JSONObject topLevel)
			throws JSONException {

		if (!topLevel.has(KEY_ENTITY)) {
			throw new JSONException("No entity entry found.");
		}

		ItemIdValue itemId;
		JSONArray entityJsonArray = topLevel.optJSONArray(KEY_ENTITY);

		if (entityJsonArray != null) { // it is an array
			itemId = this.getItemIdValue(entityJsonArray);
		} else { // it is a String
			String stringItemId = topLevel.getString(KEY_ENTITY);
			itemId = this.getItemIdValue(stringItemId);
		}
		return itemId;
	}

	/**
	 * Attempts to get the property id value from the given JSON-object. Note
	 * that in old dumps the entity is not an array but a string with
	 * appropriate prefix in lower case.
	 * 
	 * @param topLevel
	 *            is the JSON object describing the whole entity
	 * @return the properties id as PropertyIdValue
	 * @throws JSONException
	 *             if the topLevel does not contain the key "entity"
	 */
	private PropertyIdValue getPropertyIdFromTopLevel(JSONObject topLevel)
			throws JSONException {

		if (!topLevel.has(KEY_ENTITY)) {
			throw new JSONException("No entity entry found.");
		}

		PropertyIdValue propertyId;
		JSONArray entityJsonArray = topLevel.optJSONArray(KEY_ENTITY);

		if (entityJsonArray != null) { // it is an array
			propertyId = this.getPropertyIdValue(entityJsonArray);
		} else { // it is a String
			String stringItemId = topLevel.getString(KEY_ENTITY);
			propertyId = this.getPropertyIdValue(stringItemId);
		}
		return propertyId;
	}

	/**
	 * Creates an ItemIdValue from a JSON array that represents an item in JSON.
	 * 
	 * @param jsonEntity
	 *            a JSON array containing information about the item; the array
	 *            should have the structure ["item", itemId] where itemId is an
	 *            integer
	 * @return the corresponding ItemIdValue
	 * @throws JSONException
	 *             if the entity does not contain an "item" entry or the id does
	 *             not have the correct format
	 */
	private ItemIdValue getItemIdValue(JSONArray jsonEntity)
			throws JSONException {

		String entityTypeIndicator = jsonEntity.getString(0);
		if (!entityTypeIndicator.equalsIgnoreCase("item")) {
			throw new JSONException("Entity type indicator \""
					+ entityTypeIndicator
					+ "\" did not match expected type indicator \"item\".");
		}

		int idValue = jsonEntity.getInt(1);
		return this.factory.getItemIdValue(PREFIX_ITEM + idValue, this.baseIri);
	}

	/**
	 * Creates a PropertyIdValue from a JSON array that represents a property in
	 * JSON.
	 * 
	 * @param jsonEntity
	 *            a JSON array containing information about the property; the
	 *            array should have the structure ["property", propertyId] where
	 *            propertyId is an integer
	 * @return the corresponding PropertyIdValue
	 * @throws JSONException
	 *             if the entity does not contain an "property" entry or the id
	 *             does not have the correct format
	 */
	private PropertyIdValue getPropertyIdValue(JSONArray jsonEntity)
			throws JSONException {

		String entityTypeIndicator = jsonEntity.getString(0);
		if (!entityTypeIndicator.equalsIgnoreCase("property")) {
			throw new JSONException("Entity type indicator \""
					+ entityTypeIndicator
					+ "\" did not match expected type indicator \"property\".");
		}

		int idValue = jsonEntity.getInt(1);
		return this.factory.getPropertyIdValue(PREFIX_PROPERTY + idValue,
				this.baseIri);
	}

	/**
	 * Creates an ItemIdValue from a string id given in JSON, which may have the
	 * form Q12345 or q12345. The lower case version is not a valid id but may
	 * occur in the JSON dump for historic reasons.
	 * 
	 * @param id
	 *            JSON string id of the item
	 * @return the corresponding ItemIdValue
	 * @throws JSONException
	 */
	private ItemIdValue getItemIdValue(String id) throws JSONException {
		try {
			return this.factory.getItemIdValue(id.toUpperCase(), this.baseIri);
		} catch (IllegalArgumentException e) { // invalid id format
			throw new JSONException(e);
		}
	}

	/**
	 * Creates a PropertyIdValue from a string id given in JSON, which may have
	 * the form P12345 or p12345. The lower case version is not a valid id but
	 * may occur in the JSON dump for historic reasons.
	 * 
	 * @param id
	 *            JSON string id of the property
	 * @return the corresponding PropertyIdValue
	 * @throws JSONException
	 */
	private PropertyIdValue getPropertyIdValue(String id) throws JSONException {
		try {
			return this.factory.getPropertyIdValue(id.toUpperCase(),
					this.baseIri);
		} catch (IllegalArgumentException e) { // invalid id format
			throw new JSONException(e);
		}
	}

	/**
	 * Converts a JSONObject into a list of mono-lingual text values. The object
	 * to be converted is the value associated with the given key in the top
	 * level object. So if there is a JSONObject <i>topLevel</i> and the key
	 * "label" are given, only the JSONObject found in the <i>topLevel</i> under
	 * the key "label" will be converted, not the whole <i>topLevel</i>.
	 * 
	 * MLTV is the abbreviation for MonoLingualTextValue.
	 * 
	 * @param key
	 *            is the key of the object to be converted in the topLevel. If
	 *            the key is not present or not an object an empty list will be
	 *            returned.
	 * @param topLevel
	 *            is the JSONObject that contains the object to be converted
	 *            under a given key.
	 * @return a list of extracted mono-lingual text values. Might be empty, but
	 *         not null.
	 */
	private List<MonolingualTextValue> getMltv(String key, JSONObject topLevel) {

		JSONObject toConvert = topLevel.optJSONObject(key);
		if (toConvert != null) {
			return this.mltvHandler.convertToMltv(toConvert);
		} // else…

		return new LinkedList<>();
	}

	/**
	 * Converts the given JSON array into a list of Reference-objects. A
	 * reference is an array of reference statements which in turn are value
	 * snaks. Invalid references will be skipped.
	 * 
	 * @param jsonReferences
	 *            is an JSON array of JSON arrays of value snaks
	 * @return the appropriate List of references.
	 */
	private List<? extends Reference> getReferences(JSONArray jsonReferences) {
		// References are [singeRef]
		// singleRef are [refStatements]
		// refStatements are value snaks

		List<Reference> result = new LinkedList<>();

		// process the single references
		for (int i = 0; i < jsonReferences.length(); i++) {
			try {
				JSONArray jsonSingleRef = jsonReferences.getJSONArray(i);
				Reference singleReference = this
						.getSingleReference(jsonSingleRef);
				result.add(singleReference);

			} catch (JSONException e) {
				// skip over invalid references
				continue;
			}

		}
		return result;
	}

	/**
	 * Converts a given JSON-array into a single Reference. The array must be
	 * taken from the JSON-array of references.
	 * 
	 * @param jsonSingleRef
	 *            is a JSON-array describing a single reference, containing
	 *            value snaks
	 * @return the appropriate reference
	 * @throws JSONException
	 *             in case the conversion of the value snaks fails
	 */
	private Reference getSingleReference(JSONArray jsonSingleRef)
			throws JSONException {
		List<ValueSnak> valueSnaks = new LinkedList<>();

		// process the reference statements
		// do not recover from broken snaks, skip the reference
		for (int j = 0; j < jsonSingleRef.length(); j++) {

			JSONArray jsonValueSnak = jsonSingleRef.getJSONArray(j);
			ValueSnak currentValueSnak = this.getValueSnak(jsonValueSnak);
			valueSnaks.add(currentValueSnak);
		}

		Reference singleReference = factory.getReference(valueSnaks);
		return singleReference;
	}

	/**
	 * Converts a JSON object into a mapping from site keys to
	 * SiteLink-instances.
	 * 
	 * @param jsonLinks
	 *            a JSON object representing the site links.
	 * @return A mapping with a String representing a site key e.g. "enwiki" as
	 *         key and a SiteLink-object as value.
	 * @throws JSONException
	 */
	private Map<String, SiteLink> getSiteLinks(JSONObject jsonLinks)
			throws JSONException {
		// assert jsonLinks != null : "Link JSON object was null";

		// links are siteKey:{"name":string,"badges":[string] }
		// the siteKey is the key for the returned map
		// or they are siteKey:string

		Map<String, SiteLink> result = new HashMap<String, SiteLink>();

		// TODO where to get the site IRI from
		String siteIri = "";

		@SuppressWarnings("unchecked")
		Iterator<String> linkIterator = jsonLinks.keys();

		while (linkIterator.hasNext()) {

			String title;
			List<String> badges;

			String siteKey = linkIterator.next();

			JSONObject currentLink = jsonLinks.optJSONObject(siteKey);

			if (currentLink != null) {

				title = currentLink.getString("name");
				JSONArray badgeArray = currentLink.getJSONArray("badges");
				badges = new ArrayList<>(badgeArray.length());

				// convert badges to List<String>
				for (int i = 0; i < badgeArray.length(); i++) {
					badges.add(badgeArray.getString(i));
				}
			} else {
				String stringLink = jsonLinks.optString(siteKey);
				if (stringLink != null) { // its a string
					title = stringLink;
					// initialize the badges as empty list
					// since they are needed for the factory
					badges = new ArrayList<>();
				} else { // none of the above, skip
					logger.info("Site link is neither a JSON object nor a String. Skipped.");
					continue;
				}
			}

			// create the SiteLink instance
			SiteLink siteLink = factory.getSiteLink(title, siteKey, siteIri,
					badges);
			result.put(siteKey, siteLink);
		}

		return result;
	}

	/**
	 * Creates a list of StatementGroup objects from the given JSON array, which
	 * represents a list of statements (without any groups).
	 * 
	 * @param jsonStatements
	 *            a JSON list of JSON representations for Statements
	 * @param entityIdValue
	 *            the subject to which this statement refers
	 * 
	 * @return the corresponding list of StatementGroups
	 * @throws JSONException
	 *             if the given JSON did not have the expected form
	 */
	private List<StatementGroup> getStatementGroups(JSONArray jsonStatements,
			EntityIdValue entityIdValue) throws JSONException {

		List<StatementGroup> result;
		List<Statement> statementsFromJson = new ArrayList<Statement>(
				jsonStatements.length());

		// iterate over all the statements in the item and decompose them
		for (int i = 0; i < jsonStatements.length(); i++) {
			JSONObject statementJson = jsonStatements.getJSONObject(i);
			Statement statement = this.getStatement(statementJson,
					entityIdValue);
			statementsFromJson.add(statement);
		}

		// process the list of statements into a list of statement groups
		StatementGroupBuilder builder = new StatementGroupBuilder(this.factory);
		result = builder.buildFromStatementList(statementsFromJson);

		return result;
	}

	/**
	 * Creates a Statement object for the given entity id from the given JSON
	 * object. The JSON needs to have the following structure:
	 * 
	 * <pre>
	 * {"m":object,
	 *  "q":[],
	 *  "g":string,
	 *  "rank":int,
	 *  "refs":[...]}
	 * </pre>
	 * 
	 * Here, "m" denotes the main snak, "q" the qualifiers, and "g" the
	 * statement id.
	 * 
	 * @param jsonObject
	 *            a JSON object representing a statement
	 * @param entityIdValue
	 *            the subject to which this statement refers
	 * @return the corresponding Statement<commit_msg>Use one global StatementGroupBuilder<commit_after>package org.wikidata.wdtk.dumpfiles;

/*
 * #%L
 * Wikidata Toolkit Dump File Handling
 * %%
 * Copyright (C) 2014 Wikidata Toolkit Developers
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * #L%
 */

import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang3.Validate;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.wikidata.wdtk.datamodel.interfaces.Claim;
import org.wikidata.wdtk.datamodel.interfaces.DataObjectFactory;
import org.wikidata.wdtk.datamodel.interfaces.DatatypeIdValue;
import org.wikidata.wdtk.datamodel.interfaces.EntityIdValue;
import org.wikidata.wdtk.datamodel.interfaces.GlobeCoordinatesValue;
import org.wikidata.wdtk.datamodel.interfaces.ItemDocument;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.NoValueSnak;
import org.wikidata.wdtk.datamodel.interfaces.PropertyDocument;
import org.wikidata.wdtk.datamodel.interfaces.PropertyIdValue;
import org.wikidata.wdtk.datamodel.interfaces.QuantityValue;
import org.wikidata.wdtk.datamodel.interfaces.Reference;
import org.wikidata.wdtk.datamodel.interfaces.SiteLink;
import org.wikidata.wdtk.datamodel.interfaces.Snak;
import org.wikidata.wdtk.datamodel.interfaces.SomeValueSnak;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.StatementGroup;
import org.wikidata.wdtk.datamodel.interfaces.StatementRank;
import org.wikidata.wdtk.datamodel.interfaces.StringValue;
import org.wikidata.wdtk.datamodel.interfaces.TimeValue;
import org.wikidata.wdtk.datamodel.interfaces.Value;
import org.wikidata.wdtk.datamodel.interfaces.ValueSnak;

// TODO logging support
/**
 * This class provides methods to convert dump-file JSON objects into
 * representations according to the WDTK data model. Since the converted JSON
 * normally belongs to the same domain, the base IRI is represented as an
 * attribute.
 * 
 * @author <NAME>
 * 
 */
public class JsonConverter {

	public static final String PREFIX_ITEM = "Q";
	public static final String PREFIX_PROPERTY = "P";

	// Keys that might occur in the documents
	private static final String KEY_LABEL = "label";
	private static final String KEY_ENTITY = "entity";
	private static final String KEY_DESCRIPTION = "description";
	private static final String KEY_ALIAS = "aliases";
	private static final String KEY_DATATYPE = "datatype";
	private static final String KEY_CLAIM = "claims";
	private static final String KEY_LINK = "links";

	private final DataObjectFactory factory;
	private String baseIri = "";
	private final MonolingualTextValueHandler mltvHandler;
	private final StatementGroupBuilder statementGroupBuilder;

	static final Logger logger = LoggerFactory.getLogger(JsonConverter.class);

	/**
	 * Creates a new instance of the JsonConverter. For the <i>baseIri</i> see
	 * also {@link org.wikidata.wdtk.datamodel.interfaces.ItemIdValue}.
	 * 
	 * @param baseIri
	 *            the base IRI to be used for entities
	 * @param factory
	 *            the DataObjectFactory to be used to construct objects of the
	 *            data model
	 */
	public JsonConverter(String baseIri, DataObjectFactory factory) {
		this.setBaseIri(baseIri);
		this.factory = factory;

		mltvHandler = new MonolingualTextValueHandler(this.factory);
		statementGroupBuilder = new StatementGroupBuilder(this.factory);
	}

	/**
	 * Attempts to parse a given JSON object into an instance of
	 * PropertyDocument.
	 * 
	 * @param jsonObject
	 *            the JSON object to convert. Must represent a property document
	 *            and therefore contain the keys "entity", "label",
	 *            "description", "aliases" and "datatype".
	 * @param propertyIdString
	 *            is a string containing the id of the property provided by
	 *            external sources; if null is given, the information will be
	 *            extracted from the JSON
	 * @return the PropertyDocument as described in the JSON.
	 * @throws NullPointerException
	 *             if toConvert was null.
	 * @throws JSONException
	 *             if the JSON object did not contain a key it should have had.
	 */
	public PropertyDocument convertToPropertyDocument(JSONObject jsonObject,
			String propertyIdString) throws JSONException {

		if (jsonObject.length() == 0) { // if the JSON object is empty
			throw new JSONException("The JSON to convert was empty");
		}

		PropertyDocument result;

		PropertyIdValue propertyId;

		if (propertyIdString != null) {
			propertyId = this.getPropertyIdValue(propertyIdString);
		} else {
			propertyId = this.getPropertyIdFromTopLevel(jsonObject);
		}

		List<MonolingualTextValue> labels = this.getMltv(KEY_LABEL, jsonObject);

		List<MonolingualTextValue> descriptions = this.getMltv(KEY_DESCRIPTION,
				jsonObject);

		List<MonolingualTextValue> aliases = this
				.getMltv(KEY_ALIAS, jsonObject);

		String jsonDataTypeId = jsonObject.getString(KEY_DATATYPE);
		DatatypeIdValue datatypeId = this.getDatatypeIdValue(jsonDataTypeId);

		result = this.factory.getPropertyDocument(propertyId, labels,
				descriptions, aliases, datatypeId);
		return result;
	}

	/**
	 * Attempts to parse a given JSON object into an instance of ItemDocument.
	 * 
	 * @param jsonObject
	 *            the JSON object to convert. Must represent an item document
	 *            and therefore might contain the keys "entity", "label",
	 *            "description", "aliases", "claims" and "links".
	 * @param propertyIdString
	 *            is a string containing the id of the item provided by external
	 *            sources; if null is given, the information will be extracted
	 *            from the JSON
	 * @return the ItemDocument as described in the JSON.
	 * @throws NullPointerException
	 *             if toConvert was null.
	 * @throws JSONException
	 *             if the JSON object did not contain a key it should have had.
	 */
	public ItemDocument convertToItemDocument(JSONObject jsonObject,
			String itemIdString) throws JSONException, NullPointerException {

		if (jsonObject.length() == 0) { // if the JSON object is empty
			throw new JSONException("The JSON to convert was empty");
		}

		ItemIdValue itemId;
		if (itemIdString != null) {
			itemId = this.getItemIdValue(itemIdString);
		} else {
			itemId = this.getItemIdFromTopLevel(jsonObject);
		}
		List<MonolingualTextValue> labels = this.getMltv(KEY_LABEL, jsonObject);

		List<MonolingualTextValue> descriptions = this.getMltv(KEY_DESCRIPTION,
				jsonObject);

		List<MonolingualTextValue> aliases = this
				.getMltv(KEY_ALIAS, jsonObject);

		List<StatementGroup> statements = new LinkedList<>();
		if (jsonObject.has(KEY_CLAIM)) {
			JSONArray jsonStatements = jsonObject.getJSONArray(KEY_CLAIM);
			statements = this.getStatementGroups(jsonStatements, itemId);
		}

		Map<String, SiteLink> siteLinks = new HashMap<>();

		if (jsonObject.has(KEY_LINK)) {
			JSONArray linkArray = jsonObject.optJSONArray(KEY_LINK);
			if (linkArray == null) {
				JSONObject jsonLinks = jsonObject.getJSONObject(KEY_LINK);
				siteLinks = this.getSiteLinks(jsonLinks);
			}
		}

		ItemDocument result = factory.getItemDocument(itemId, labels,
				descriptions, aliases, statements, siteLinks);
		return result;
	}

	/**
	 * Attempts to get the item id value from the given JSON-object. Note that
	 * in old dumps the entity is not an array but a string with appropriate
	 * prefix in lower case.
	 * 
	 * @param topLevel
	 *            is the JSON object describing the whole entity
	 * @return the items id as ItemIdValue
	 * @throws JSONException
	 *             if the topLevel does not contain the key "entity"
	 */
	private ItemIdValue getItemIdFromTopLevel(JSONObject topLevel)
			throws JSONException {

		if (!topLevel.has(KEY_ENTITY)) {
			throw new JSONException("No entity entry found.");
		}

		ItemIdValue itemId;
		JSONArray entityJsonArray = topLevel.optJSONArray(KEY_ENTITY);

		if (entityJsonArray != null) { // it is an array
			itemId = this.getItemIdValue(entityJsonArray);
		} else { // it is a String
			String stringItemId = topLevel.getString(KEY_ENTITY);
			itemId = this.getItemIdValue(stringItemId);
		}
		return itemId;
	}

	/**
	 * Attempts to get the property id value from the given JSON-object. Note
	 * that in old dumps the entity is not an array but a string with
	 * appropriate prefix in lower case.
	 * 
	 * @param topLevel
	 *            is the JSON object describing the whole entity
	 * @return the properties id as PropertyIdValue
	 * @throws JSONException
	 *             if the topLevel does not contain the key "entity"
	 */
	private PropertyIdValue getPropertyIdFromTopLevel(JSONObject topLevel)
			throws JSONException {

		if (!topLevel.has(KEY_ENTITY)) {
			throw new JSONException("No entity entry found.");
		}

		PropertyIdValue propertyId;
		JSONArray entityJsonArray = topLevel.optJSONArray(KEY_ENTITY);

		if (entityJsonArray != null) { // it is an array
			propertyId = this.getPropertyIdValue(entityJsonArray);
		} else { // it is a String
			String stringItemId = topLevel.getString(KEY_ENTITY);
			propertyId = this.getPropertyIdValue(stringItemId);
		}
		return propertyId;
	}

	/**
	 * Creates an ItemIdValue from a JSON array that represents an item in JSON.
	 * 
	 * @param jsonEntity
	 *            a JSON array containing information about the item; the array
	 *            should have the structure ["item", itemId] where itemId is an
	 *            integer
	 * @return the corresponding ItemIdValue
	 * @throws JSONException
	 *             if the entity does not contain an "item" entry or the id does
	 *             not have the correct format
	 */
	private ItemIdValue getItemIdValue(JSONArray jsonEntity)
			throws JSONException {

		String entityTypeIndicator = jsonEntity.getString(0);
		if (!entityTypeIndicator.equalsIgnoreCase("item")) {
			throw new JSONException("Entity type indicator \""
					+ entityTypeIndicator
					+ "\" did not match expected type indicator \"item\".");
		}

		int idValue = jsonEntity.getInt(1);
		return this.factory.getItemIdValue(PREFIX_ITEM + idValue, this.baseIri);
	}

	/**
	 * Creates a PropertyIdValue from a JSON array that represents a property in
	 * JSON.
	 * 
	 * @param jsonEntity
	 *            a JSON array containing information about the property; the
	 *            array should have the structure ["property", propertyId] where
	 *            propertyId is an integer
	 * @return the corresponding PropertyIdValue
	 * @throws JSONException
	 *             if the entity does not contain an "property" entry or the id
	 *             does not have the correct format
	 */
	private PropertyIdValue getPropertyIdValue(JSONArray jsonEntity)
			throws JSONException {

		String entityTypeIndicator = jsonEntity.getString(0);
		if (!entityTypeIndicator.equalsIgnoreCase("property")) {
			throw new JSONException("Entity type indicator \""
					+ entityTypeIndicator
					+ "\" did not match expected type indicator \"property\".");
		}

		int idValue = jsonEntity.getInt(1);
		return this.factory.getPropertyIdValue(PREFIX_PROPERTY + idValue,
				this.baseIri);
	}

	/**
	 * Creates an ItemIdValue from a string id given in JSON, which may have the
	 * form Q12345 or q12345. The lower case version is not a valid id but may
	 * occur in the JSON dump for historic reasons.
	 * 
	 * @param id
	 *            JSON string id of the item
	 * @return the corresponding ItemIdValue
	 * @throws JSONException
	 */
	private ItemIdValue getItemIdValue(String id) throws JSONException {
		try {
			return this.factory.getItemIdValue(id.toUpperCase(), this.baseIri);
		} catch (IllegalArgumentException e) { // invalid id format
			throw new JSONException(e);
		}
	}

	/**
	 * Creates a PropertyIdValue from a string id given in JSON, which may have
	 * the form P12345 or p12345. The lower case version is not a valid id but
	 * may occur in the JSON dump for historic reasons.
	 * 
	 * @param id
	 *            JSON string id of the property
	 * @return the corresponding PropertyIdValue
	 * @throws JSONException
	 */
	private PropertyIdValue getPropertyIdValue(String id) throws JSONException {
		try {
			return this.factory.getPropertyIdValue(id.toUpperCase(),
					this.baseIri);
		} catch (IllegalArgumentException e) { // invalid id format
			throw new JSONException(e);
		}
	}

	/**
	 * Converts a JSONObject into a list of mono-lingual text values. The object
	 * to be converted is the value associated with the given key in the top
	 * level object. So if there is a JSONObject <i>topLevel</i> and the key
	 * "label" are given, only the JSONObject found in the <i>topLevel</i> under
	 * the key "label" will be converted, not the whole <i>topLevel</i>.
	 * 
	 * MLTV is the abbreviation for MonoLingualTextValue.
	 * 
	 * @param key
	 *            is the key of the object to be converted in the topLevel. If
	 *            the key is not present or not an object an empty list will be
	 *            returned.
	 * @param topLevel
	 *            is the JSONObject that contains the object to be converted
	 *            under a given key.
	 * @return a list of extracted mono-lingual text values. Might be empty, but
	 *         not null.
	 */
	private List<MonolingualTextValue> getMltv(String key, JSONObject topLevel) {

		JSONObject toConvert = topLevel.optJSONObject(key);
		if (toConvert != null) {
			return this.mltvHandler.convertToMltv(toConvert);
		} // else…

		return new LinkedList<>();
	}

	/**
	 * Converts the given JSON array into a list of Reference-objects. A
	 * reference is an array of reference statements which in turn are value
	 * snaks. Invalid references will be skipped.
	 * 
	 * @param jsonReferences
	 *            is an JSON array of JSON arrays of value snaks
	 * @return the appropriate List of references.
	 */
	private List<? extends Reference> getReferences(JSONArray jsonReferences) {
		// References are [singeRef]
		// singleRef are [refStatements]
		// refStatements are value snaks

		List<Reference> result = new LinkedList<>();

		// process the single references
		for (int i = 0; i < jsonReferences.length(); i++) {
			try {
				JSONArray jsonSingleRef = jsonReferences.getJSONArray(i);
				Reference singleReference = this
						.getSingleReference(jsonSingleRef);
				result.add(singleReference);

			} catch (JSONException e) {
				// skip over invalid references
				continue;
			}

		}
		return result;
	}

	/**
	 * Converts a given JSON-array into a single Reference. The array must be
	 * taken from the JSON-array of references.
	 * 
	 * @param jsonSingleRef
	 *            is a JSON-array describing a single reference, containing
	 *            value snaks
	 * @return the appropriate reference
	 * @throws JSONException
	 *             in case the conversion of the value snaks fails
	 */
	private Reference getSingleReference(JSONArray jsonSingleRef)
			throws JSONException {
		List<ValueSnak> valueSnaks = new LinkedList<>();

		// process the reference statements
		// do not recover from broken snaks, skip the reference
		for (int j = 0; j < jsonSingleRef.length(); j++) {

			JSONArray jsonValueSnak = jsonSingleRef.getJSONArray(j);
			ValueSnak currentValueSnak = this.getValueSnak(jsonValueSnak);
			valueSnaks.add(currentValueSnak);
		}

		Reference singleReference = factory.getReference(valueSnaks);
		return singleReference;
	}

	/**
	 * Converts a JSON object into a mapping from site keys to
	 * SiteLink-instances.
	 * 
	 * @param jsonLinks
	 *            a JSON object representing the site links.
	 * @return A mapping with a String representing a site key e.g. "enwiki" as
	 *         key and a SiteLink-object as value.
	 * @throws JSONException
	 */
	private Map<String, SiteLink> getSiteLinks(JSONObject jsonLinks)
			throws JSONException {
		// assert jsonLinks != null : "Link JSON object was null";

		// links are siteKey:{"name":string,"badges":[string] }
		// the siteKey is the key for the returned map
		// or they are siteKey:string

		Map<String, SiteLink> result = new HashMap<String, SiteLink>();

		// TODO where to get the site IRI from
		String siteIri = "";

		@SuppressWarnings("unchecked")
		Iterator<String> linkIterator = jsonLinks.keys();

		while (linkIterator.hasNext()) {

			String title;
			List<String> badges;

			String siteKey = linkIterator.next();

			JSONObject currentLink = jsonLinks.optJSONObject(siteKey);

			if (currentLink != null) {

				title = currentLink.getString("name");
				JSONArray badgeArray = currentLink.getJSONArray("badges");
				badges = new ArrayList<>(badgeArray.length());

				// convert badges to List<String>
				for (int i = 0; i < badgeArray.length(); i++) {
					badges.add(badgeArray.getString(i));
				}
			} else {
				String stringLink = jsonLinks.optString(siteKey);
				if (stringLink != null) { // its a string
					title = stringLink;
					// initialize the badges as empty list
					// since they are needed for the factory
					badges = new ArrayList<>();
				} else { // none of the above, skip
					logger.info("Site link is neither a JSON object nor a String. Skipped.");
					continue;
				}
			}

			// create the SiteLink instance
			SiteLink siteLink = factory.getSiteLink(title, siteKey, siteIri,
					badges);
			result.put(siteKey, siteLink);
		}

		return result;
	}

	/**
	 * Creates a list of StatementGroup objects from the given JSON array, which
	 * represents a list of statements (without any groups).
	 * 
	 * @param jsonStatements
	 *            a JSON list of JSON representations for Statements
	 * @param entityIdValue
	 *            the subject to which this statement refers
	 * 
	 * @return the corresponding list of StatementGroups
	 * @throws JSONException
	 *             if the given JSON did not have the expected form
	 */
	private List<StatementGroup> getStatementGroups(JSONArray jsonStatements,
			EntityIdValue entityIdValue) throws JSONException {

		List<StatementGroup> result;
		List<Statement> statementsFromJson = new ArrayList<Statement>(
				jsonStatements.length());

		// iterate over all the statements in the item and decompose them
		for (int i = 0; i < jsonStatements.length(); i++) {
			JSONObject statementJson = jsonStatements.getJSONObject(i);
			Statement statement = this.getStatement(statementJson,
					entityIdValue);
			statementsFromJson.add(statement);
		}

		// process the list of statements into a list of statement groups
		result = this.statementGroupBuilder
				.buildFromStatementList(statementsFromJson);

		return result;
	}

	/**
	 * Creates a Statement object for the given entity id from the given JSON
	 * object. The JSON needs to have the following structure:
	 * 
	 * <pre>
	 * {"m":object,
	 *  "q":[],
	 *  "g":string,
	 *  "rank":int,
	 *  "refs":[...]}
	 * </pre>
	 * 
	 * Here, "m" denotes the main snak, "q" the qualifiers, and "g" the
	 * statement id.
	 * 
	 * @param jsonObject
	 *            a JSON object representing a statement
	 * @param entityIdValue
	 *            the subject to which this statement refers
	 * @return the corresponding Statement
{'fraction_non_alphanumeric': 0.08532500566650716, 'fraction_numerical': 0.002065127055682877, 'mean_word_length': 3.4565656565656564, 'pattern_counts': {'":': 14, '<': 91, '<?xml version=': 0, '>': 83, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '19336', 'n_tokens_mistral': 12383, 'n_tokens_neox': 11017, 'n_words': 3776}