<commit_before>/*******************************************************************************
 * Copyright (c) 2008-2011 Chair for Applied Software Engineering,
 * Technische Universitaet Muenchen.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 * 
 * Contributors:
 * <NAME>
 * <NAME>
 * <NAME>
 ******************************************************************************/
package org.eclipse.emf.emfstore.internal.common.model.util;

import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.eclipse.core.runtime.IStatus;
import org.eclipse.core.runtime.Platform;
import org.eclipse.core.runtime.Status;
import org.eclipse.emf.common.util.ECollections;
import org.eclipse.emf.common.util.EList;
import org.eclipse.emf.common.util.TreeIterator;
import org.eclipse.emf.common.util.URI;
import org.eclipse.emf.ecore.EClass;
import org.eclipse.emf.ecore.EClassifier;
import org.eclipse.emf.ecore.EObject;
import org.eclipse.emf.ecore.EPackage;
import org.eclipse.emf.ecore.EReference;
import org.eclipse.emf.ecore.EStructuralFeature;
import org.eclipse.emf.ecore.EStructuralFeature.Setting;
import org.eclipse.emf.ecore.EcoreFactory;
import org.eclipse.emf.ecore.InternalEObject;
import org.eclipse.emf.ecore.resource.Resource;
import org.eclipse.emf.ecore.resource.Resource.Diagnostic;
import org.eclipse.emf.ecore.resource.ResourceSet;
import org.eclipse.emf.ecore.resource.URIConverter;
import org.eclipse.emf.ecore.resource.impl.ResourceImpl;
import org.eclipse.emf.ecore.resource.impl.ResourceSetImpl;
import org.eclipse.emf.ecore.util.EcoreUtil;
import org.eclipse.emf.ecore.xmi.XMIResource;
import org.eclipse.emf.ecore.xmi.XMLResource;
import org.eclipse.emf.ecore.xmi.impl.XMLParserPoolImpl;
import org.eclipse.emf.emfstore.common.extensionpoint.ESExtensionElement;
import org.eclipse.emf.emfstore.common.extensionpoint.ESExtensionPoint;
import org.eclipse.emf.emfstore.common.extensionpoint.ESExtensionPointException;
import org.eclipse.emf.emfstore.common.extensionpoint.ESPriorityComparator;
import org.eclipse.emf.emfstore.common.extensionpoint.ESResourceSetProvider;
import org.eclipse.emf.emfstore.common.model.ESSingletonIdResolver;
import org.eclipse.emf.emfstore.internal.common.model.AssociationClassElement;
import org.eclipse.emf.emfstore.internal.common.model.IdEObjectCollection;
import org.eclipse.emf.emfstore.internal.common.model.ModelElementId;
import org.eclipse.emf.emfstore.internal.common.model.ModelFactory;
import org.eclipse.emf.emfstore.internal.common.model.Project;
import org.eclipse.emf.emfstore.internal.common.model.impl.ESModelElementIdImpl;
import org.eclipse.emf.emfstore.internal.common.model.impl.IdEObjectCollectionImpl;
import org.eclipse.emf.emfstore.internal.common.model.impl.ProjectImpl;

/**
 * Utility class for ModelElements.
 * 
 * @author koegel
 * @author emueller
 * @author ottovonwesen
 */
public final class ModelUtil {

	/**
	 * Constant that may be used in case no checksum computation has taken place.
	 */
	public static final long NO_CHECKSUM = -1;

	/**
	 * URI used to serialize EObject with the model util.
	 */
	public static final URI VIRTUAL_URI = URI.createURI("virtualUri");

	private static final String ORG_ECLIPSE_EMF_EMFSTORE_COMMON_MODEL = "org.eclipse.emf.emfstore.common.model";

	public static final String DISCARD_DANGLING_HREF_ID = "org.eclipse.emf.emfstore.common.discardDanglingHREFs";

	private static IResourceLogger resourceLogger = new IResourceLogger() {

		public void logWarning(String msg) {
			ModelUtil.logWarning(msg);
		}

		/**
		 * {@inheritDoc}
		 * 
		 * @see org.eclipse.emf.emfstore.internal.common.model.util.IResourceLogger#logError(java.lang.String)
		 */
		public void logError(String msg) {
			ModelUtil.logError(msg);
		}
	};

	/**
	 * Contains the canonical names of classes which will be ignored.
	 */
	private static Set<String> ignoredDataTypes;

	/**
	 * Contains all ID resolvers for singleton datatypes.
	 */
	private static Set<ESSingletonIdResolver> singletonIdResolvers;
	private static HashMap<Object, Object> resourceLoadOptions;
	private static HashMap<Object, Object> resourceSaveOptions;

	/**
	 * Private constructor.
	 */
	private ModelUtil() {
		// nothing to do
	}

	/**
	 * Creates a ModelElementId object from a string.
	 * 
	 * @param id
	 *            as string
	 * @return id as object
	 */
	public static ModelElementId createModelElementId(String id) {
		ModelElementId modelElementId = ModelFactory.eINSTANCE.createModelElementId();
		modelElementId.setId(id);
		return modelElementId;
	}

	/**
	 * Compares two {@link IdEObjectCollection}s. Order of the model elements at root level
	 * of a collection does not influence the outcome of this operations
	 * 
	 * @param collectionA
	 *            the first collection
	 * @param collectionB
	 *            the second collection
	 * @return true if the two collections are equal, false otherwise
	 */
	public static boolean areEqual(IdEObjectCollection collectionA, IdEObjectCollection collectionB) {
		long a;
		long b;

		try {
			// collections are treated specially because the order
			// of root elements should not matter
			a = computeChecksum(collectionA);
			b = computeChecksum(collectionB);
		} catch (SerializationException e) {
			return false;
		}

		return a == b;
	}

	/**
	 * Copies the given EObject and converts it to a string.
	 * 
	 * @param object
	 *            the eObject
	 * @return the string representation of the EObject
	 * @throws SerializationException
	 *             if a serialization problem occurs
	 */
	public static String eObjectToString(EObject object) throws SerializationException {

		if (object == null) {
			return null;
		}

		ResourceSetImpl resourceSetImpl = new ResourceSetImpl();
		XMIResource res = (XMIResource) resourceSetImpl.createResource(VIRTUAL_URI);
		((ResourceImpl) res).setIntrinsicIDToEObjectMap(new HashMap<String, EObject>());

		EObject copy;
		if (object instanceof IdEObjectCollection) {
			copy = copyIdEObjectCollection((IdEObjectCollection) object, res);
		} else {
			copy = copyEObject(ModelUtil.getProject(object), object, res);
		}

		return copiedEObjectToString(copy, res);
	}

	/**
	 * Converts the given {@link EObject} to a string.
	 * 
	 * @param copy The copied {@link EObject}.
	 * @param resource The resource for the {@link EObject}.
	 * @return The string representing the {@link EObject}.
	 * @throws SerializationException If a serialization problem occurs.
	 */
	private static String copiedEObjectToString(EObject copy, XMIResource resource) throws SerializationException {
		int step = 200;
		int initialSize = step;
		resource.getContents().add(copy);

		StringWriter stringWriter = new StringWriter(initialSize);
		URIConverter.WriteableOutputStream uws = new URIConverter.WriteableOutputStream(stringWriter, "UTF-8");

		String lineSeparator = System.getProperty("line.separator");
		try {
			System.setProperty("line.separator", "\r\n");
			resource.save(uws, getResourceSaveOptions());
		} catch (IOException e) {
			throw new SerializationException(e);
		} finally {
			System.setProperty("line.separator", lineSeparator);
			try {
				stringWriter.close();
				uws.close();
			} catch (IOException e) {
				throw new SerializationException(e);
			}
		}

		return stringWriter.toString();
	}

	/**
	 * Computes the checksum for a given string representing an {@link EObject}.
	 * 
	 * @param eObjectString
	 *            the string representing the {@link EObject}.
	 * @return the computed checksum
	 * 
	 * @throws SerializationException
	 *             in case any errors occur during computation of the checksum
	 */
	private static long computeChecksum(String eObjectString) {
		long h = 1125899906842597L; // prime
		int len = eObjectString.length();

		for (int i = 0; i < len; i++) {
			h = 31 * h + eObjectString.charAt(i);
		}

		return h;
	}

	/**
	 * Computes the checksum for a given {@link EObject}.
	 * 
	 * @param eObject
	 *            the EObject for which to compute a checksum
	 * @return the computed checksum
	 * 
	 * @throws SerializationException
	 *             in case any errors occur during computation of the checksum
	 */
	public static long computeChecksum(EObject eObject) throws SerializationException {
		return (computeChecksum(eObjectToString(eObject)));
	}

	/**
	 * Computes the checksum for a given {@link IdEObjectCollection}.
	 * The checksum for a collection is independent of the order of the
	 * collection's elements at the root level.
	 * 
	 * @param collection
	 *            the collection for which to compute a checksum
	 * @return the computed checksum
	 * 
	 * @throws SerializationException
	 *             in case any errors occur during computation of the checksum
	 */
	public static long computeChecksum(IdEObjectCollection collection) throws SerializationException {

		ResourceSetImpl resourceSetImpl = new ResourceSetImpl();
		XMIResource res = (XMIResource) resourceSetImpl.createResource(VIRTUAL_URI);
		((ResourceImpl) res).setIntrinsicIDToEObjectMap(new HashMap<String, EObject>());
		final IdEObjectCollection copy = copyIdEObjectCollection(collection, res);

		ECollections.sort(copy.getModelElements(), new Comparator<EObject>() {
			public int compare(EObject o1, EObject o2) {
				return copy.getModelElementId(o1).getId().compareTo(copy.getModelElementId(o2).getId());
			}
		});

		return computeChecksum(copiedEObjectToString(copy, res));
	}

	/**
	 * Returns the resource logger.
	 * 
	 * @return the resource logger
	 */
	public static IResourceLogger getResourceLogger() {
		return resourceLogger;
	}

	/**
	 * Copies the given {@link IdEObjectCollection} and writes the IDs it contains into the given {@link XMIResource}.
	 * 
	 * @param collection
	 *            the collection to be copied
	 * @param res
	 *            the resource into which the collection's IDs should be written into
	 * @return the copied collection
	 */
	public static IdEObjectCollection copyIdEObjectCollection(IdEObjectCollection collection, XMIResource res) {
		IdEObjectCollection copiedCollection = clone(collection);

		for (EObject modelElement : copiedCollection.getAllModelElements()) {
			if (isIgnoredDatatype(modelElement)) {
				continue;
			}
			ModelElementId modelElementId = copiedCollection.getModelElementId(modelElement);
			res.setID(modelElement, modelElementId.getId());
		}

		for (EObject modelElement : ((Project) copiedCollection).getCutElements()) {
			if (isIgnoredDatatype(modelElement)) {
				continue;
			}
			ModelElementId modelElementId = ((IdEObjectCollectionImpl) copiedCollection)
				.getModelElementId(modelElement);
			res.setID(modelElement, modelElementId.getId());
		}

		return copiedCollection;
	}

	// TODO: javadoc
	private static EObject copyEObject(IdEObjectCollection collection, EObject object, XMIResource res) {
		IdEObjectCollection copiedCollection = copyIdEObjectCollection(collection, res);
		EObject copiedEObject = copiedCollection.getModelElement(collection.getModelElementId(object));
		return copiedEObject;
	}

	/**
	 * Determines whether the type of an EObject is an ignored one.
	 * 
	 * @param eObject
	 *            the EObject which is to be checked
	 * @return true, if the EObject will be ignored, false otherwise
	 */
	public static synchronized boolean isIgnoredDatatype(EObject eObject) {

		if (ignoredDataTypes == null) {
			ignoredDataTypes = new LinkedHashSet<String>();
			for (ESExtensionElement element : new ESExtensionPoint(
				"org.eclipse.emf.emfstore.common.model.ignoreDatatype",
				true).getExtensionElements()) {
				try {
					ignoredDataTypes.add(element.getAttribute("type"));
				} catch (ESExtensionPointException e) {
				}
			}
		}

		return ignoredDataTypes.contains(eObject.eClass().getInstanceClassName());
	}

	/**
	 * Delivers a map of options for loading resources. Especially {@link XMLResource#OPTION_DEFER_IDREF_RESOLUTION}
	 * which speeds up loading
	 * due to our id based resources.
	 * 
	 * @return map of options for {@link XMIResource} or {@link XMLResource}.
	 */
	@SuppressWarnings("rawtypes")
	public static synchronized Map<Object, Object> getResourceLoadOptions() {
		if (resourceLoadOptions == null) {
			resourceLoadOptions = new LinkedHashMap<Object, Object>();
			resourceLoadOptions.put(XMLResource.OPTION_DEFER_ATTACHMENT, Boolean.TRUE);
			resourceLoadOptions.put(XMLResource.OPTION_DEFER_IDREF_RESOLUTION, Boolean.TRUE);
			resourceLoadOptions.put(XMLResource.OPTION_USE_DEPRECATED_METHODS, Boolean.FALSE);
			resourceLoadOptions.put(XMLResource.OPTION_USE_PARSER_POOL, new XMLParserPoolImpl());
			resourceLoadOptions.put(XMLResource.OPTION_USE_XML_NAME_TO_FEATURE_MAP, new HashMap());
			resourceLoadOptions.put(XMLResource.OPTION_USE_ENCODED_ATTRIBUTE_STYLE, Boolean.TRUE);
			resourceLoadOptions.put(XMLResource.OPTION_ENCODING, "UTF-8");
		}
		return resourceLoadOptions;
	}

	/**
	 * Delivers a map of mandatory options for saving resources.
	 * 
	 * @return map of options for {@link XMIResource} or {@link XMLResource}.
	 */
	public static synchronized Map<Object, Object> getResourceSaveOptions() {

		if (resourceSaveOptions == null) {

			resourceSaveOptions = new LinkedHashMap<Object, Object>();
			resourceSaveOptions.put(XMLResource.OPTION_USE_ENCODED_ATTRIBUTE_STYLE, Boolean.TRUE);
			resourceSaveOptions.put(XMLResource.OPTION_USE_CACHED_LOOKUP_TABLE, new ArrayList<Object>());
			resourceSaveOptions.put(XMLResource.OPTION_ENCODING, "UTF-8");
			resourceSaveOptions.put(XMLResource.OPTION_FLUSH_THRESHOLD, 100000);
			resourceSaveOptions.put(XMLResource.OPTION_USE_FILE_BUFFER, Boolean.TRUE);

			ESExtensionPoint extensionPoint = new ESExtensionPoint(DISCARD_DANGLING_HREF_ID);
			Boolean discardDanglingHREFs = extensionPoint.getBoolean("value", Boolean.FALSE);
			if (discardDanglingHREFs) {
				resourceSaveOptions.put(XMLResource.OPTION_PROCESS_DANGLING_HREF,
					XMLResource.OPTION_PROCESS_DANGLING_HREF_RECORD);
			}

			logInfo("Resource save options initialized:");
			for (Map.Entry<Object, Object> entry : resourceSaveOptions.entrySet()) {
				logInfo("\t" + entry.getKey() + ": " + entry.getValue());
			}
		}
		return resourceSaveOptions;
	}

	/**
	 * Saves a given resource and logs any warning and/or errors.
	 * 
	 * @param resource
	 *            the resource to be saved
	 * @param logger
	 *            a logger instance which will be used to log warnings and errors on resources
	 * @throws IOException
	 *             in case an exception occurs during save
	 */
	public static void saveResource(Resource resource, IResourceLogger logger) throws IOException {
		try {
			resource.save(ModelUtil.getResourceSaveOptions());
		} catch (IOException e) {
			// rethrow exception
			throw e;
		} finally {
			logWarningsAndErrors(resource, logger);
		}
	}

	/**
	 * Loads a given resource and logs any warning and/or errors.
	 * 
	 * @param resource
	 *            the resource to be loaded
	 * @param logger
	 *            a logger instance which will be used to log warnings and errors on resources
	 * @throws IOException
	 *             in case an exception occurs during load
	 */
	public static void loadResource(Resource resource, IResourceLogger logger) throws IOException {
		try {
			resource.load(ModelUtil.getResourceLoadOptions());
		} catch (IOException e) {
			// rethrow exception
			throw e;
		} finally {
			logWarningsAndErrors(resource, logger);
		}
	}

	private static void logWarningsAndErrors(Resource resource, IResourceLogger logger) {

		if (resource.getWarnings().size() > 0) {
			for (Diagnostic diagnostic : resource.getErrors()) {
				logger.logWarning(logDiagnostic(diagnostic).toString());
			}
		}

		if (resource.getErrors().size() > 0) {
			for (Diagnostic diagnostic : resource.getErrors()) {
				logger.logError(logDiagnostic(diagnostic).toString());
			}
		}

	}

	private static StringWriter logDiagnostic(Diagnostic diagnostic) {

		StringWriter error = new StringWriter();
		error.append(diagnostic.getLocation() + "\n");
		error.append(diagnostic.getMessage() + "\n");

		if (diagnostic instanceof Exception) {
			StringWriter stringWriter = new StringWriter();
			PrintWriter printWriter = new PrintWriter(stringWriter);
			((Throwable) diagnostic).printStackTrace(printWriter);
			error.append(stringWriter.toString() + "\n");
		}

		return error;
	}

	private static boolean canHaveInstances(EClass eClass) {
		return !(eClass.isAbstract() || eClass.isInterface());
	}

	/**
	 * Recursively goes through model and create a list of all non-Abstract
	 * classes.
	 * 
	 * @param ePackage
	 *            the package to start with.
	 * @return list of all non-Abstract model element classes in starting
	 *         package and its sub-packages
	 */
	public static Set<EClass> getNonAbstractMETypes(EPackage ePackage) {

		Set<EClass> nonAbstractMETypes = new LinkedHashSet<EClass>();
		Set<EClass> allMETypes = getAllMETypes(ePackage);

		Iterator<EClass> iterator = allMETypes.iterator();
		while (iterator.hasNext()) {
			EClass eClass = iterator.next();
			if (canHaveInstances(eClass)) {
				nonAbstractMETypes.add(eClass);
			}
		}

		return nonAbstractMETypes;

	}

	/**
	 * Recursively goes through package and returns a list of all EClasses
	 * inheriting ModelElement (abstract classes and interfaces are also
	 * include).
	 * 
	 * @param ePackage
	 *            starting package
	 * @return a list of all EClasses inheriting ModelElement (inclusive
	 *         abstract classes and interfaces) in starting package and all its
	 *         sub-packages.
	 */
	public static Set<EClass> getAllMETypes(EPackage ePackage) {
		Set<EClass> meTypes = new LinkedHashSet<EClass>();

		for (EObject eObject : ePackage.eContents()) {
			if (eObject instanceof EClass) {
				EClass eClass = (EClass) eObject;
				meTypes.add(eClass);
			} else if (eObject instanceof EPackage) {
				EPackage eSubPackage = (EPackage) eObject;
				meTypes.addAll(getAllMETypes(eSubPackage));
			}
		}

		return meTypes;
	}

	/**
	 * This will add a new entry to error log view of eclipse.
	 * 
	 * @param message
	 *            message
	 * @param exception
	 *            exception
	 * @param statusInt
	 *            severity. Use one of constants in
	 *            org.eclipse.core.runtime.Status class.
	 * @throws LoggedException
	 */
	public static void log(String message, Throwable exception, int statusInt) {
		Status status = new Status(statusInt, Platform.getBundle(ORG_ECLIPSE_EMF_EMFSTORE_COMMON_MODEL)
			.getSymbolicName(), statusInt, message, exception);
		Platform.getLog(Platform.getBundle(ORG_ECLIPSE_EMF_EMFSTORE_COMMON_MODEL)).log(status);
	}

	/**
	 * Log an exception to the platform log. This will create a popup in the ui.
	 * 
	 * @param message
	 *            the message
	 * @param exception
	 *            the exception
	 */
	public static void logException(String message, Throwable exception) {
		log(message, exception, IStatus.ERROR);
	}

	/**
	 * Log an exception to the platform log. This will create a popup in the ui.
	 * 
	 * @param exception
	 *            the exception
	 */
	public static void logException(Throwable exception) {
		logException(exception.getMessage(), exception);
	}

	/**
	 * Log a warning to the platform log. This will NOT create a popup in the UI.
	 * 
	 * @param message
	 *            the message
	 * @param exception
	 *            the exception
	 */
	public static void logWarning(String message, Throwable exception) {
		log(message, exception, IStatus.WARNING);
	}

	/**
	 * Log a warning to the platform log. This will NOT create a popup in the UI.
	 * 
	 * @param message
	 *            the message being logged
	 */
	public static void logWarning(String message) {
		log(message, null, IStatus.WARNING);
	}

	/**
	 * Log a error to the platform log. This will NOT create a popup in the UI.
	 * 
	 * @param message
	 *            the message being logged
	 */
	public static void logError(String message) {
		log(message, null, IStatus.ERROR);
	}

	/**
	 * Log an exception to the platform log. This will create a popup in the ui.
	 * 
	 * @param message
	 *            the message
	 */
	public static void logInfo(String message) {
		log(message, null, IStatus.INFO);
	}

	/**
	 * Clone any EObject.
	 * 
	 * @param <T>
	 *            the Eobject sub type
	 * @param eObject
	 *            the Eobject instance
	 * @return a clone of the Eobject instance
	 */
	@SuppressWarnings("unchecked")
	public static <T extends EObject> T clone(T eObject) {
		if (eObject instanceof ProjectImpl) {
			return (T) ((ProjectImpl) eObject).copy();
		}
		EObject clone = EcoreUtil.copy(eObject);
		return (T) clone;
	}

	/**
	 * Clone a list of EObjects.
	 * 
	 * @param <T>
	 *            the EObject sub type the list consists of
	 * @param list
	 *            the list instance
	 * @return a clone of the list and its contents instance
	 */
	@SuppressWarnings("unchecked")
	public static <T extends EObject> List<T> clone(List<T> list) {
		ArrayList<T> result = new ArrayList<T>();
		for (EObject eObject : list) {
			T clone = (T) ModelUtil.clone(eObject);
			result.add(clone);
		}
		return result;
	}

	/**
	 * Create a flat clone of the list, the list if cloned but ot its content.
	 * 
	 * @param <T>
	 *            the list type parameter
	 * @param originalList
	 *            the original list
	 * @return a flat copy
	 */
	public static <T extends EObject> List<T> flatCloneList(List<T> originalList) {
		List<T> clonedList = new ArrayList<T>(originalList.size());
		for (T element : originalList) {
			clonedList.add(element);
		}
		return clonedList;
	}

	/**
	 * Load an EObject from a resource, the resource is supposed to contain only
	 * one root object of the given EClass type. Type T must match EClass type.
	 * 
	 * @param <T>
	 *            Type of the EObject
	 * @param eClass
	 *            the EClass of the EObject
	 * @param resourceURI
	 *            the resources URI
	 * @param checkConstraints
	 *            whether to perform additional sanity checks. These checks
	 *            basically try to enforce that a resource contains exactly one
	 *            object.
	 * @return the object loaded from the resource
	 * @throws IOException
	 *             if loading the object from the resource fails.
	 */
	@SuppressWarnings("unchecked")
	public static <T extends EObject> T loadEObjectFromResource(EClass eClass, URI resourceURI, boolean checkConstraints)
		throws IOException {

		ResourceSet resourceSet = null;

		if (resourceURI != null && resourceURI.scheme().equals("emfstore")) {
			ESExtensionPoint extensionPoint = null;
			if (resourceURI.segment(0).equals("workspaces")) {
				extensionPoint = new ESExtensionPoint("org.eclipse.emf.emfstore.client.resourceSetProvider",
					false);
			} else {
				extensionPoint = new ESExtensionPoint("org.eclipse.emf.emfstore.server.resourceSetProvider",
					false);
			}

			extensionPoint.setComparator(new ESPriorityComparator("priority", true));
			extensionPoint.reload();

			ESResourceSetProvider resourceSetProvider = extensionPoint.getElementWithHighestPriority().getClass(
				"class",
				ESResourceSetProvider.class);

			if (resourceSetProvider == null) {
				resourceSet = new ResourceSetImpl();<commit_msg>fixed loading resources<commit_after>/*******************************************************************************
 * Copyright (c) 2008-2011 Chair for Applied Software Engineering,
 * Technische Universitaet Muenchen.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 * 
 * Contributors:
 * <NAME>
 * <NAME>
 * <NAME>
 ******************************************************************************/
package org.eclipse.emf.emfstore.internal.common.model.util;

import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.eclipse.core.runtime.IStatus;
import org.eclipse.core.runtime.Platform;
import org.eclipse.core.runtime.Status;
import org.eclipse.emf.common.util.ECollections;
import org.eclipse.emf.common.util.EList;
import org.eclipse.emf.common.util.TreeIterator;
import org.eclipse.emf.common.util.URI;
import org.eclipse.emf.ecore.EClass;
import org.eclipse.emf.ecore.EClassifier;
import org.eclipse.emf.ecore.EObject;
import org.eclipse.emf.ecore.EPackage;
import org.eclipse.emf.ecore.EReference;
import org.eclipse.emf.ecore.EStructuralFeature;
import org.eclipse.emf.ecore.EStructuralFeature.Setting;
import org.eclipse.emf.ecore.EcoreFactory;
import org.eclipse.emf.ecore.InternalEObject;
import org.eclipse.emf.ecore.resource.Resource;
import org.eclipse.emf.ecore.resource.Resource.Diagnostic;
import org.eclipse.emf.ecore.resource.ResourceSet;
import org.eclipse.emf.ecore.resource.URIConverter;
import org.eclipse.emf.ecore.resource.impl.ResourceImpl;
import org.eclipse.emf.ecore.resource.impl.ResourceSetImpl;
import org.eclipse.emf.ecore.util.EcoreUtil;
import org.eclipse.emf.ecore.xmi.XMIResource;
import org.eclipse.emf.ecore.xmi.XMLResource;
import org.eclipse.emf.ecore.xmi.impl.XMLParserPoolImpl;
import org.eclipse.emf.emfstore.common.extensionpoint.ESExtensionElement;
import org.eclipse.emf.emfstore.common.extensionpoint.ESExtensionPoint;
import org.eclipse.emf.emfstore.common.extensionpoint.ESExtensionPointException;
import org.eclipse.emf.emfstore.common.extensionpoint.ESPriorityComparator;
import org.eclipse.emf.emfstore.common.extensionpoint.ESResourceSetProvider;
import org.eclipse.emf.emfstore.common.model.ESSingletonIdResolver;
import org.eclipse.emf.emfstore.internal.common.model.AssociationClassElement;
import org.eclipse.emf.emfstore.internal.common.model.IdEObjectCollection;
import org.eclipse.emf.emfstore.internal.common.model.ModelElementId;
import org.eclipse.emf.emfstore.internal.common.model.ModelFactory;
import org.eclipse.emf.emfstore.internal.common.model.Project;
import org.eclipse.emf.emfstore.internal.common.model.impl.ESModelElementIdImpl;
import org.eclipse.emf.emfstore.internal.common.model.impl.IdEObjectCollectionImpl;
import org.eclipse.emf.emfstore.internal.common.model.impl.ProjectImpl;

/**
 * Utility class for ModelElements.
 * 
 * @author koegel
 * @author emueller
 * @author ottovonwesen
 */
public final class ModelUtil {

	/**
	 * Constant that may be used in case no checksum computation has taken place.
	 */
	public static final long NO_CHECKSUM = -1;

	/**
	 * URI used to serialize EObject with the model util.
	 */
	public static final URI VIRTUAL_URI = URI.createURI("virtualUri");

	private static final String ORG_ECLIPSE_EMF_EMFSTORE_COMMON_MODEL = "org.eclipse.emf.emfstore.common.model";

	public static final String DISCARD_DANGLING_HREF_ID = "org.eclipse.emf.emfstore.common.discardDanglingHREFs";

	private static IResourceLogger resourceLogger = new IResourceLogger() {

		public void logWarning(String msg) {
			ModelUtil.logWarning(msg);
		}

		/**
		 * {@inheritDoc}
		 * 
		 * @see org.eclipse.emf.emfstore.internal.common.model.util.IResourceLogger#logError(java.lang.String)
		 */
		public void logError(String msg) {
			ModelUtil.logError(msg);
		}
	};

	/**
	 * Contains the canonical names of classes which will be ignored.
	 */
	private static Set<String> ignoredDataTypes;

	/**
	 * Contains all ID resolvers for singleton datatypes.
	 */
	private static Set<ESSingletonIdResolver> singletonIdResolvers;
	private static HashMap<Object, Object> resourceLoadOptions;
	private static HashMap<Object, Object> resourceSaveOptions;

	/**
	 * Private constructor.
	 */
	private ModelUtil() {
		// nothing to do
	}

	/**
	 * Creates a ModelElementId object from a string.
	 * 
	 * @param id
	 *            as string
	 * @return id as object
	 */
	public static ModelElementId createModelElementId(String id) {
		ModelElementId modelElementId = ModelFactory.eINSTANCE.createModelElementId();
		modelElementId.setId(id);
		return modelElementId;
	}

	/**
	 * Compares two {@link IdEObjectCollection}s. Order of the model elements at root level
	 * of a collection does not influence the outcome of this operations
	 * 
	 * @param collectionA
	 *            the first collection
	 * @param collectionB
	 *            the second collection
	 * @return true if the two collections are equal, false otherwise
	 */
	public static boolean areEqual(IdEObjectCollection collectionA, IdEObjectCollection collectionB) {
		long a;
		long b;

		try {
			// collections are treated specially because the order
			// of root elements should not matter
			a = computeChecksum(collectionA);
			b = computeChecksum(collectionB);
		} catch (SerializationException e) {
			return false;
		}

		return a == b;
	}

	/**
	 * Copies the given EObject and converts it to a string.
	 * 
	 * @param object
	 *            the eObject
	 * @return the string representation of the EObject
	 * @throws SerializationException
	 *             if a serialization problem occurs
	 */
	public static String eObjectToString(EObject object) throws SerializationException {

		if (object == null) {
			return null;
		}

		ResourceSetImpl resourceSetImpl = new ResourceSetImpl();
		XMIResource res = (XMIResource) resourceSetImpl.createResource(VIRTUAL_URI);
		((ResourceImpl) res).setIntrinsicIDToEObjectMap(new HashMap<String, EObject>());

		EObject copy;
		if (object instanceof IdEObjectCollection) {
			copy = copyIdEObjectCollection((IdEObjectCollection) object, res);
		} else {
			copy = copyEObject(ModelUtil.getProject(object), object, res);
		}

		return copiedEObjectToString(copy, res);
	}

	/**
	 * Converts the given {@link EObject} to a string.
	 * 
	 * @param copy The copied {@link EObject}.
	 * @param resource The resource for the {@link EObject}.
	 * @return The string representing the {@link EObject}.
	 * @throws SerializationException If a serialization problem occurs.
	 */
	private static String copiedEObjectToString(EObject copy, XMIResource resource) throws SerializationException {
		int step = 200;
		int initialSize = step;
		resource.getContents().add(copy);

		StringWriter stringWriter = new StringWriter(initialSize);
		URIConverter.WriteableOutputStream uws = new URIConverter.WriteableOutputStream(stringWriter, "UTF-8");

		String lineSeparator = System.getProperty("line.separator");
		try {
			System.setProperty("line.separator", "\r\n");
			resource.save(uws, getResourceSaveOptions());
		} catch (IOException e) {
			throw new SerializationException(e);
		} finally {
			System.setProperty("line.separator", lineSeparator);
			try {
				stringWriter.close();
				uws.close();
			} catch (IOException e) {
				throw new SerializationException(e);
			}
		}

		return stringWriter.toString();
	}

	/**
	 * Computes the checksum for a given string representing an {@link EObject}.
	 * 
	 * @param eObjectString
	 *            the string representing the {@link EObject}.
	 * @return the computed checksum
	 * 
	 * @throws SerializationException
	 *             in case any errors occur during computation of the checksum
	 */
	private static long computeChecksum(String eObjectString) {
		long h = 1125899906842597L; // prime
		int len = eObjectString.length();

		for (int i = 0; i < len; i++) {
			h = 31 * h + eObjectString.charAt(i);
		}

		return h;
	}

	/**
	 * Computes the checksum for a given {@link EObject}.
	 * 
	 * @param eObject
	 *            the EObject for which to compute a checksum
	 * @return the computed checksum
	 * 
	 * @throws SerializationException
	 *             in case any errors occur during computation of the checksum
	 */
	public static long computeChecksum(EObject eObject) throws SerializationException {
		return (computeChecksum(eObjectToString(eObject)));
	}

	/**
	 * Computes the checksum for a given {@link IdEObjectCollection}.
	 * The checksum for a collection is independent of the order of the
	 * collection's elements at the root level.
	 * 
	 * @param collection
	 *            the collection for which to compute a checksum
	 * @return the computed checksum
	 * 
	 * @throws SerializationException
	 *             in case any errors occur during computation of the checksum
	 */
	public static long computeChecksum(IdEObjectCollection collection) throws SerializationException {

		ResourceSetImpl resourceSetImpl = new ResourceSetImpl();
		XMIResource res = (XMIResource) resourceSetImpl.createResource(VIRTUAL_URI);
		((ResourceImpl) res).setIntrinsicIDToEObjectMap(new HashMap<String, EObject>());
		final IdEObjectCollection copy = copyIdEObjectCollection(collection, res);

		ECollections.sort(copy.getModelElements(), new Comparator<EObject>() {
			public int compare(EObject o1, EObject o2) {
				return copy.getModelElementId(o1).getId().compareTo(copy.getModelElementId(o2).getId());
			}
		});

		return computeChecksum(copiedEObjectToString(copy, res));
	}

	/**
	 * Returns the resource logger.
	 * 
	 * @return the resource logger
	 */
	public static IResourceLogger getResourceLogger() {
		return resourceLogger;
	}

	/**
	 * Copies the given {@link IdEObjectCollection} and writes the IDs it contains into the given {@link XMIResource}.
	 * 
	 * @param collection
	 *            the collection to be copied
	 * @param res
	 *            the resource into which the collection's IDs should be written into
	 * @return the copied collection
	 */
	public static IdEObjectCollection copyIdEObjectCollection(IdEObjectCollection collection, XMIResource res) {
		IdEObjectCollection copiedCollection = clone(collection);

		for (EObject modelElement : copiedCollection.getAllModelElements()) {
			if (isIgnoredDatatype(modelElement)) {
				continue;
			}
			ModelElementId modelElementId = copiedCollection.getModelElementId(modelElement);
			res.setID(modelElement, modelElementId.getId());
		}

		for (EObject modelElement : ((Project) copiedCollection).getCutElements()) {
			if (isIgnoredDatatype(modelElement)) {
				continue;
			}
			ModelElementId modelElementId = ((IdEObjectCollectionImpl) copiedCollection)
				.getModelElementId(modelElement);
			res.setID(modelElement, modelElementId.getId());
		}

		return copiedCollection;
	}

	// TODO: javadoc
	private static EObject copyEObject(IdEObjectCollection collection, EObject object, XMIResource res) {
		IdEObjectCollection copiedCollection = copyIdEObjectCollection(collection, res);
		EObject copiedEObject = copiedCollection.getModelElement(collection.getModelElementId(object));
		return copiedEObject;
	}

	/**
	 * Determines whether the type of an EObject is an ignored one.
	 * 
	 * @param eObject
	 *            the EObject which is to be checked
	 * @return true, if the EObject will be ignored, false otherwise
	 */
	public static synchronized boolean isIgnoredDatatype(EObject eObject) {

		if (ignoredDataTypes == null) {
			ignoredDataTypes = new LinkedHashSet<String>();
			for (ESExtensionElement element : new ESExtensionPoint(
				"org.eclipse.emf.emfstore.common.model.ignoreDatatype",
				true).getExtensionElements()) {
				try {
					ignoredDataTypes.add(element.getAttribute("type"));
				} catch (ESExtensionPointException e) {
				}
			}
		}

		return ignoredDataTypes.contains(eObject.eClass().getInstanceClassName());
	}

	/**
	 * Delivers a map of options for loading resources. Especially {@link XMLResource#OPTION_DEFER_IDREF_RESOLUTION}
	 * which speeds up loading
	 * due to our id based resources.
	 * 
	 * @return map of options for {@link XMIResource} or {@link XMLResource}.
	 */
	@SuppressWarnings("rawtypes")
	public static synchronized Map<Object, Object> getResourceLoadOptions() {
		if (resourceLoadOptions == null) {
			resourceLoadOptions = new LinkedHashMap<Object, Object>();
			resourceLoadOptions.put(XMLResource.OPTION_DEFER_ATTACHMENT, Boolean.TRUE);
			resourceLoadOptions.put(XMLResource.OPTION_DEFER_IDREF_RESOLUTION, Boolean.TRUE);
			resourceLoadOptions.put(XMLResource.OPTION_USE_DEPRECATED_METHODS, Boolean.FALSE);
			resourceLoadOptions.put(XMLResource.OPTION_USE_PARSER_POOL, new XMLParserPoolImpl());
			resourceLoadOptions.put(XMLResource.OPTION_USE_XML_NAME_TO_FEATURE_MAP, new HashMap());
			resourceLoadOptions.put(XMLResource.OPTION_USE_ENCODED_ATTRIBUTE_STYLE, Boolean.TRUE);
			resourceLoadOptions.put(XMLResource.OPTION_ENCODING, "UTF-8");
		}
		return resourceLoadOptions;
	}

	/**
	 * Delivers a map of mandatory options for saving resources.
	 * 
	 * @return map of options for {@link XMIResource} or {@link XMLResource}.
	 */
	public static synchronized Map<Object, Object> getResourceSaveOptions() {

		if (resourceSaveOptions == null) {

			resourceSaveOptions = new LinkedHashMap<Object, Object>();
			resourceSaveOptions.put(XMLResource.OPTION_USE_ENCODED_ATTRIBUTE_STYLE, Boolean.TRUE);
			resourceSaveOptions.put(XMLResource.OPTION_USE_CACHED_LOOKUP_TABLE, new ArrayList<Object>());
			resourceSaveOptions.put(XMLResource.OPTION_ENCODING, "UTF-8");
			resourceSaveOptions.put(XMLResource.OPTION_FLUSH_THRESHOLD, 100000);
			resourceSaveOptions.put(XMLResource.OPTION_USE_FILE_BUFFER, Boolean.TRUE);

			ESExtensionPoint extensionPoint = new ESExtensionPoint(DISCARD_DANGLING_HREF_ID);
			Boolean discardDanglingHREFs = extensionPoint.getBoolean("value", Boolean.FALSE);
			if (discardDanglingHREFs) {
				resourceSaveOptions.put(XMLResource.OPTION_PROCESS_DANGLING_HREF,
					XMLResource.OPTION_PROCESS_DANGLING_HREF_RECORD);
			}

			logInfo("Resource save options initialized:");
			for (Map.Entry<Object, Object> entry : resourceSaveOptions.entrySet()) {
				logInfo("\t" + entry.getKey() + ": " + entry.getValue());
			}
		}
		return resourceSaveOptions;
	}

	/**
	 * Saves a given resource and logs any warning and/or errors.
	 * 
	 * @param resource
	 *            the resource to be saved
	 * @param logger
	 *            a logger instance which will be used to log warnings and errors on resources
	 * @throws IOException
	 *             in case an exception occurs during save
	 */
	public static void saveResource(Resource resource, IResourceLogger logger) throws IOException {
		try {
			resource.save(ModelUtil.getResourceSaveOptions());
		} catch (IOException e) {
			// rethrow exception
			throw e;
		} finally {
			logWarningsAndErrors(resource, logger);
		}
	}

	/**
	 * Loads a given resource and logs any warning and/or errors.
	 * 
	 * @param resource
	 *            the resource to be loaded
	 * @param logger
	 *            a logger instance which will be used to log warnings and errors on resources
	 * @throws IOException
	 *             in case an exception occurs during load
	 */
	public static void loadResource(Resource resource, IResourceLogger logger) throws IOException {
		try {
			resource.load(ModelUtil.getResourceLoadOptions());
		} catch (IOException e) {
			// rethrow exception
			throw e;
		} finally {
			logWarningsAndErrors(resource, logger);
		}
	}

	private static void logWarningsAndErrors(Resource resource, IResourceLogger logger) {

		if (resource.getWarnings().size() > 0) {
			for (Diagnostic diagnostic : resource.getErrors()) {
				logger.logWarning(logDiagnostic(diagnostic).toString());
			}
		}

		if (resource.getErrors().size() > 0) {
			for (Diagnostic diagnostic : resource.getErrors()) {
				logger.logError(logDiagnostic(diagnostic).toString());
			}
		}

	}

	private static StringWriter logDiagnostic(Diagnostic diagnostic) {

		StringWriter error = new StringWriter();
		error.append(diagnostic.getLocation() + "\n");
		error.append(diagnostic.getMessage() + "\n");

		if (diagnostic instanceof Exception) {
			StringWriter stringWriter = new StringWriter();
			PrintWriter printWriter = new PrintWriter(stringWriter);
			((Throwable) diagnostic).printStackTrace(printWriter);
			error.append(stringWriter.toString() + "\n");
		}

		return error;
	}

	private static boolean canHaveInstances(EClass eClass) {
		return !(eClass.isAbstract() || eClass.isInterface());
	}

	/**
	 * Recursively goes through model and create a list of all non-Abstract
	 * classes.
	 * 
	 * @param ePackage
	 *            the package to start with.
	 * @return list of all non-Abstract model element classes in starting
	 *         package and its sub-packages
	 */
	public static Set<EClass> getNonAbstractMETypes(EPackage ePackage) {

		Set<EClass> nonAbstractMETypes = new LinkedHashSet<EClass>();
		Set<EClass> allMETypes = getAllMETypes(ePackage);

		Iterator<EClass> iterator = allMETypes.iterator();
		while (iterator.hasNext()) {
			EClass eClass = iterator.next();
			if (canHaveInstances(eClass)) {
				nonAbstractMETypes.add(eClass);
			}
		}

		return nonAbstractMETypes;

	}

	/**
	 * Recursively goes through package and returns a list of all EClasses
	 * inheriting ModelElement (abstract classes and interfaces are also
	 * include).
	 * 
	 * @param ePackage
	 *            starting package
	 * @return a list of all EClasses inheriting ModelElement (inclusive
	 *         abstract classes and interfaces) in starting package and all its
	 *         sub-packages.
	 */
	public static Set<EClass> getAllMETypes(EPackage ePackage) {
		Set<EClass> meTypes = new LinkedHashSet<EClass>();

		for (EObject eObject : ePackage.eContents()) {
			if (eObject instanceof EClass) {
				EClass eClass = (EClass) eObject;
				meTypes.add(eClass);
			} else if (eObject instanceof EPackage) {
				EPackage eSubPackage = (EPackage) eObject;
				meTypes.addAll(getAllMETypes(eSubPackage));
			}
		}

		return meTypes;
	}

	/**
	 * This will add a new entry to error log view of eclipse.
	 * 
	 * @param message
	 *            message
	 * @param exception
	 *            exception
	 * @param statusInt
	 *            severity. Use one of constants in
	 *            org.eclipse.core.runtime.Status class.
	 * @throws LoggedException
	 */
	public static void log(String message, Throwable exception, int statusInt) {
		Status status = new Status(statusInt, Platform.getBundle(ORG_ECLIPSE_EMF_EMFSTORE_COMMON_MODEL)
			.getSymbolicName(), statusInt, message, exception);
		Platform.getLog(Platform.getBundle(ORG_ECLIPSE_EMF_EMFSTORE_COMMON_MODEL)).log(status);
	}

	/**
	 * Log an exception to the platform log. This will create a popup in the ui.
	 * 
	 * @param message
	 *            the message
	 * @param exception
	 *            the exception
	 */
	public static void logException(String message, Throwable exception) {
		log(message, exception, IStatus.ERROR);
	}

	/**
	 * Log an exception to the platform log. This will create a popup in the ui.
	 * 
	 * @param exception
	 *            the exception
	 */
	public static void logException(Throwable exception) {
		logException(exception.getMessage(), exception);
	}

	/**
	 * Log a warning to the platform log. This will NOT create a popup in the UI.
	 * 
	 * @param message
	 *            the message
	 * @param exception
	 *            the exception
	 */
	public static void logWarning(String message, Throwable exception) {
		log(message, exception, IStatus.WARNING);
	}

	/**
	 * Log a warning to the platform log. This will NOT create a popup in the UI.
	 * 
	 * @param message
	 *            the message being logged
	 */
	public static void logWarning(String message) {
		log(message, null, IStatus.WARNING);
	}

	/**
	 * Log a error to the platform log. This will NOT create a popup in the UI.
	 * 
	 * @param message
	 *            the message being logged
	 */
	public static void logError(String message) {
		log(message, null, IStatus.ERROR);
	}

	/**
	 * Log an exception to the platform log. This will create a popup in the ui.
	 * 
	 * @param message
	 *            the message
	 */
	public static void logInfo(String message) {
		log(message, null, IStatus.INFO);
	}

	/**
	 * Clone any EObject.
	 * 
	 * @param <T>
	 *            the Eobject sub type
	 * @param eObject
	 *            the Eobject instance
	 * @return a clone of the Eobject instance
	 */
	@SuppressWarnings("unchecked")
	public static <T extends EObject> T clone(T eObject) {
		if (eObject instanceof ProjectImpl) {
			return (T) ((ProjectImpl) eObject).copy();
		}
		EObject clone = EcoreUtil.copy(eObject);
		return (T) clone;
	}

	/**
	 * Clone a list of EObjects.
	 * 
	 * @param <T>
	 *            the EObject sub type the list consists of
	 * @param list
	 *            the list instance
	 * @return a clone of the list and its contents instance
	 */
	@SuppressWarnings("unchecked")
	public static <T extends EObject> List<T> clone(List<T> list) {
		ArrayList<T> result = new ArrayList<T>();
		for (EObject eObject : list) {
			T clone = (T) ModelUtil.clone(eObject);
			result.add(clone);
		}
		return result;
	}

	/**
	 * Create a flat clone of the list, the list if cloned but ot its content.
	 * 
	 * @param <T>
	 *            the list type parameter
	 * @param originalList
	 *            the original list
	 * @return a flat copy
	 */
	public static <T extends EObject> List<T> flatCloneList(List<T> originalList) {
		List<T> clonedList = new ArrayList<T>(originalList.size());
		for (T element : originalList) {
			clonedList.add(element);
		}
		return clonedList;
	}

	/**
	 * Load an EObject from a resource, the resource is supposed to contain only
	 * one root object of the given EClass type. Type T must match EClass type.
	 * 
	 * @param <T>
	 *            Type of the EObject
	 * @param eClass
	 *            the EClass of the EObject
	 * @param resourceURI
	 *            the resources URI
	 * @param checkConstraints
	 *            whether to perform additional sanity checks. These checks
	 *            basically try to enforce that a resource contains exactly one
	 *            object.
	 * @return the object loaded from the resource
	 * @throws IOException
	 *             if loading the object from the resource fails.
	 */
	@SuppressWarnings("unchecked")
	public static <T extends EObject> T loadEObjectFromResource(EClass eClass, URI resourceURI, boolean checkConstraints)
		throws IOException {

		ResourceSet resourceSet = null;

		if (resourceURI != null && resourceURI.scheme().equals("emfstore")) {
			ESExtensionPoint extensionPoint = null;
			if (resourceURI.authority().equals("workspaces")) {
				extensionPoint = new ESExtensionPoint("org.eclipse.emf.emfstore.client.resourceSetProvider",
					false);
			} else {
				extensionPoint = new ESExtensionPoint("org.eclipse.emf.emfstore.server.resourceSetProvider",
					false);
			}

			extensionPoint.setComparator(new ESPriorityComparator("priority", true));
			extensionPoint.reload();

			ESResourceSetProvider resourceSetProvider = extensionPoint.getElementWithHighestPriority().getClass(
				"class",
				ESResourceSetProvider.class);

			if (resourceSetProvider == null) {
				resourceSet = new ResourceSetImpl();
{'fraction_non_alphanumeric': 0.10253492635438397, 'fraction_numerical': 0.0021282424089175463, 'mean_word_length': 3.8659899518096994, 'pattern_counts': {'":': 2, '<': 87, '<?xml version=': 0, '>': 89, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1225974', 'n_tokens_mistral': 15006, 'n_tokens_neox': 13485, 'n_words': 3741}