<commit_before>package org.wikidata.wdtk.dumpfiles;

import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.wikidata.wdtk.datamodel.implementation.DataObjectFactoryImpl;
import org.wikidata.wdtk.datamodel.interfaces.Claim;
import org.wikidata.wdtk.datamodel.interfaces.DataObjectFactory;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.ItemDocument;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.Reference;
import org.wikidata.wdtk.datamodel.interfaces.SiteLink;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.StatementGroup;
import org.wikidata.wdtk.datamodel.interfaces.StatementRank;

// TODO add @link to documentation where needed
// TODO permanent: check if documentation is up-to-date
/**
 * This class provides methods to convert dump-file JSON objects into
 * representations according to the WDTK data model. Since the converted JSON
 * normally belongs to the same domain, the base IRI is represented as an
 * attribute.
 * 
 * @author <NAME>
 * 
 */
public class JsonConverter {

	private DataObjectFactory factory = new DataObjectFactoryImpl();
	private String baseIri = "";

	/**
	 * For the <i>baseIri</i> see also
	 * {@link org.wikidata.wdtk.datamodel.interfaces.ItemId}
	 * 
	 * @param baseIri
	 *            the initial IRI to be used for the processed JSON.
	 */
	public JsonConverter(String baseIri) {
		this.setBaseIri(baseIri);
		;
	}

	/**
	 * Attempts to parse a given JSON object into an instance of ItemRecord.
	 * 
	 * @param toConvert
	 *            the JSON object to convert. Must represent an item record.
	 * @param baseIri
	 *            he first part of the IRI of the site this belongs to.
	 * @return the ItemRecord parsed from JSON. Might be <b>null</b>.
	 * @throws NullPointerException
	 *             if toParse was null.
	 * @throws JSONException
	 *             if the JSON object did not contain a key it should have had.
	 */
	public ItemDocument convertToItemRecord(JSONObject toConvert)
			throws JSONException, NullPointerException {

		// initialize variables for the things we need to get
		// TODO check if it would not be better to initialize…
		// …with empty maps/lists
		ItemDocument result = null;
		ItemIdValue itemId = null;
		List<MonolingualTextValue> labels = null;
		List<MonolingualTextValue> descriptions = null;
		List<MonolingualTextValue> aliases = null;
		List<StatementGroup> statements = null;
		Map<String, SiteLink> siteLinks = null;

		// sanity check
		if (toConvert == null) {
			throw new NullPointerException();
		}

		if (toConvert.length() == 0) { // if the JSON object is empty
			throw new JSONException("The JSON to convert was empty");
		}
		// get the item Id
		JSONArray jsonEntity = toConvert.getJSONArray("entity");
		itemId = getItemId(jsonEntity);

		// get the labels
		JSONObject jsonLabels = toConvert.getJSONObject("label");
		labels = this.getLabels(jsonLabels);

		// get the description
		JSONObject jsonDescriptions = toConvert.getJSONObject("description");
		descriptions = this.getDescriptions(jsonDescriptions);

		// get the aliases
		JSONObject jsonAliases = toConvert.getJSONObject("aliases");
		aliases = this.getAliases(jsonAliases);

		// get the statements
		JSONArray jsonStatements = toConvert.getJSONArray("claims");
		statements = this.getStatements(jsonStatements);

		// get the site links
		JSONObject jsonLinks = toConvert.getJSONObject("links");
		siteLinks = this.getSiteLinks(jsonLinks);

		// now put it all together
		result = factory.getItemDocument(itemId, labels, descriptions, aliases,
				statements, siteLinks);
		return result;
	}

	/**
	 * Converts a JSON array containing statements into a list of statement
	 * groups a represented by the WDTK data model.
	 * 
	 * @param jsonStatements
	 *            contains all the statements about an item. must consist of
	 *            JSON objects containing the keys "m", "q", "g", "refs" and
	 *            "rank" each.
	 * 
	 * @return a list of statement groups as specified by the WDTK data model.
	 * @throws JSONException
	 *             if one of the JSON objects in the array did not contain all
	 *             required keys.
	 */
	private List<StatementGroup> getStatements(JSONArray jsonStatements)
			throws JSONException {

		assert jsonStatements != null : "statements JSON array was null";
		// structure is [{"m":object, "q":[], "g":string, "rank":int,
		// "refs":[…]},…]
		// "q" => qualifiers
		// "m" => main snak
		// "g" => statement id

		List<StatementGroup> result = new LinkedList<StatementGroup>();
		List<Statement> statementsFromJson = new LinkedList<Statement>();

		// iterate over all the statements in the item and decompose them
		for (int i = 0; i < jsonStatements.length(); i++) {
			JSONObject currentStatement = jsonStatements.getJSONObject(i);

			// get a list of statements in the order they are in the JSON
			// get the claim
			Claim currentClaim = this.getClaim(currentStatement);

			// get the references
			JSONArray jsonRefs = currentStatement.getJSONArray("refs");
			List<? extends Reference> references = this.getReferences(jsonRefs);

			// get the statement rank
			int rankAsInt = currentStatement.getInt("rank");
			StatementRank rank = this.getStatementRank(rankAsInt);

			// get the statement id
			String statementId = currentStatement.getString("g");

			// combine into statement
			Statement statement = factory.getStatement(currentClaim,
					references, rank, statementId);

			statementsFromJson.add(statement);

		}

		// process the list of statements into a list of statement groups
		// TODO complete

		return result;
	}

	private List<? extends Reference> getReferences(JSONArray jsonArray) {
		// TODO Auto-generated method stub
		return null;
	}

	/**
	 * Transforms a statement rank from an integer representation to an
	 * enumerated value as requested by the WDTK data model. <br/>
	 * The number 0 maps to DEPRECATED. <br/>
	 * The number 1 maps to NORMAL. <br/>
	 * The number 2 maps to PREFERRED. <br/>
	 * 
	 * To accommodate for possible other values that may occur any number below
	 * 0 also maps to DEPRECATED and any number above 2 also maps to PREFERRED.
	 * 
	 * @param intRank
	 *            the rank as integer.
	 * @return an appropriate StatementRank-value
	 */
	private StatementRank getStatementRank(int intRank) {

		// this is the default case
		StatementRank result = StatementRank.NORMAL;

		if (intRank < 1) {
			result = StatementRank.DEPRECATED;<commit_msg>Minor additions of obvious source code.<commit_after>package org.wikidata.wdtk.dumpfiles;

import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.wikidata.wdtk.datamodel.implementation.DataObjectFactoryImpl;
import org.wikidata.wdtk.datamodel.interfaces.Claim;
import org.wikidata.wdtk.datamodel.interfaces.DataObjectFactory;
import org.wikidata.wdtk.datamodel.interfaces.ItemIdValue;
import org.wikidata.wdtk.datamodel.interfaces.ItemDocument;
import org.wikidata.wdtk.datamodel.interfaces.MonolingualTextValue;
import org.wikidata.wdtk.datamodel.interfaces.Reference;
import org.wikidata.wdtk.datamodel.interfaces.SiteLink;
import org.wikidata.wdtk.datamodel.interfaces.Statement;
import org.wikidata.wdtk.datamodel.interfaces.StatementGroup;
import org.wikidata.wdtk.datamodel.interfaces.StatementRank;

// TODO add @link to documentation where needed
// TODO permanent: check if documentation is up-to-date
/**
 * This class provides methods to convert dump-file JSON objects into
 * representations according to the WDTK data model. Since the converted JSON
 * normally belongs to the same domain, the base IRI is represented as an
 * attribute.
 * 
 * @author <NAME>
 * 
 */
public class JsonConverter {

	private DataObjectFactory factory = new DataObjectFactoryImpl();
	private String baseIri = "";

	/**
	 * For the <i>baseIri</i> see also
	 * {@link org.wikidata.wdtk.datamodel.interfaces.ItemId}
	 * 
	 * @param baseIri
	 *            the initial IRI to be used for the processed JSON.
	 */
	public JsonConverter(String baseIri) {
		this.setBaseIri(baseIri);
		;
	}

	/**
	 * Attempts to parse a given JSON object into an instance of ItemRecord.
	 * 
	 * @param toConvert
	 *            the JSON object to convert. Must represent an item record.
	 * @param baseIri
	 *            he first part of the IRI of the site this belongs to.
	 * @return the ItemRecord parsed from JSON. Might be <b>null</b>.
	 * @throws NullPointerException
	 *             if toParse was null.
	 * @throws JSONException
	 *             if the JSON object did not contain a key it should have had.
	 */
	public ItemDocument convertToItemRecord(JSONObject toConvert)
			throws JSONException, NullPointerException {

		// initialize variables for the things we need to get
		// TODO check if it would not be better to initialize…
		// …with empty maps/lists
		ItemDocument result = null;
		ItemIdValue itemId = null;
		List<MonolingualTextValue> labels = null;
		List<MonolingualTextValue> descriptions = null;
		List<MonolingualTextValue> aliases = null;
		List<StatementGroup> statements = null;
		Map<String, SiteLink> siteLinks = null;

		// sanity check
		if (toConvert == null) {
			throw new NullPointerException();
		}

		if (toConvert.length() == 0) { // if the JSON object is empty
			throw new JSONException("The JSON to convert was empty");
		}
		// get the item Id
		JSONArray jsonEntity = toConvert.getJSONArray("entity");
		itemId = getItemId(jsonEntity);

		// get the labels
		JSONObject jsonLabels = toConvert.getJSONObject("label");
		labels = this.getLabels(jsonLabels);

		// get the description
		JSONObject jsonDescriptions = toConvert.getJSONObject("description");
		descriptions = this.getDescriptions(jsonDescriptions);

		// get the aliases
		JSONObject jsonAliases = toConvert.getJSONObject("aliases");
		aliases = this.getAliases(jsonAliases);

		// get the statements
		JSONArray jsonStatements = toConvert.getJSONArray("claims");
		statements = this.getStatements(jsonStatements);

		// get the site links
		JSONObject jsonLinks = toConvert.getJSONObject("links");
		siteLinks = this.getSiteLinks(jsonLinks);

		// now put it all together
		result = factory.getItemDocument(itemId, labels, descriptions, aliases,
				statements, siteLinks);
		return result;
	}

	/**
	 * Converts a JSON array containing statements into a list of statement
	 * groups a represented by the WDTK data model.
	 * 
	 * @param jsonStatements
	 *            contains all the statements about an item. must consist of
	 *            JSON objects containing the keys "m", "q", "g", "refs" and
	 *            "rank" each.
	 * 
	 * @return a list of statement groups as specified by the WDTK data model.
	 * @throws JSONException
	 *             if one of the JSON objects in the array did not contain all
	 *             required keys.
	 */
	private List<StatementGroup> getStatements(JSONArray jsonStatements)
			throws JSONException {

		assert jsonStatements != null : "statements JSON array was null";
		// structure is [{"m":object, "q":[], "g":string, "rank":int,
		// "refs":[…]},…]
		// "q" => qualifiers
		// "m" => main snak
		// "g" => statement id

		List<StatementGroup> result = new LinkedList<StatementGroup>();
		List<Statement> statementsFromJson = new LinkedList<Statement>();

		// iterate over all the statements in the item and decompose them
		for (int i = 0; i < jsonStatements.length(); i++) {
			JSONObject currentStatement = jsonStatements.getJSONObject(i);

			// get a list of statements in the order they are in the JSON
			// get the claim
			Claim currentClaim = this.getClaim(currentStatement);

			// get the references
			JSONArray jsonRefs = currentStatement.getJSONArray("refs");
			List<? extends Reference> references = this.getReferences(jsonRefs);

			// get the statement rank
			int rankAsInt = currentStatement.getInt("rank");
			StatementRank rank = this.getStatementRank(rankAsInt);

			// get the statement id
			String statementId = currentStatement.getString("g");

			// combine into statement
			Statement statement = factory.getStatement(currentClaim,
					references, rank, statementId);

			statementsFromJson.add(statement);

		}

		// process the list of statements into a list of statement groups
		// TODO complete

		return result;
	}

	/**
	 * 
	 * @param jsonReferences
	 * @return
	 */
	private List<? extends Reference> getReferences(JSONArray jsonReferences) {
		// References are [singeRef]
		// singleRef are [refStatements]
		// refStatements are ["value", int, …]
		
		List<? extends Reference> result = new LinkedList<>(); 
		// TODO Auto-generated method stub
		return result;
	}

	/**
	 * Transforms a statement rank from an integer representation to an
	 * enumerated value as requested by the WDTK data model. <br/>
	 * The number 0 maps to DEPRECATED. <br/>
	 * The number 1 maps to NORMAL. <br/>
	 * The number 2 maps to PREFERRED. <br/>
	 * 
	 * To accommodate for possible other values that may occur any number below
	 * 0 also maps to DEPRECATED and any number above 2 also maps to PREFERRED.
	 * 
	 * @param intRank
	 *            the rank as integer.
	 * @return an appropriate StatementRank-value
	 */
	private StatementRank getStatementRank(int intRank) {

		// this is the default case
		StatementRank result = StatementRank.NORMAL;

		if (intRank < 1) {
			result = StatementRank.DEPRECATED;
{'fraction_non_alphanumeric': 0.09431094383323682, 'fraction_numerical': 0.0011580775911986102, 'mean_word_length': 3.630361930294906, 'pattern_counts': {'":': 10, '<': 51, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '378265', 'n_tokens_mistral': 4361, 'n_tokens_neox': 3908, 'n_words': 1308}