<commit_before>package org.openforis.collect.relational.model;

import java.util.List;

import org.openforis.idm.metamodel.NodeDefinition;
import org.openforis.idm.model.Attribute;
import org.openforis.idm.model.Date;
import org.openforis.idm.model.DateAttribute;
import org.openforis.idm.model.Field;
import org.openforis.idm.model.Node;
import org.openforis.idm.model.Time;
import org.openforis.idm.model.TimeAttribute;
import org.openforis.idm.path.Path;

/**
 * 
 * @author <NAME>
 *
 */
public class DataColumn extends AbstractColumn<Node<?>> {
	
	private Object defaultValue;
	private NodeDefinition nodeDefinition;
	private Path relativePath;

	DataColumn(String name, int type, String typeName, NodeDefinition defn, Path relPath, Integer length, boolean nullable) {
		this(name, type, typeName, defn, relPath, length, nullable, null);
	}

	DataColumn(String name, int type, String typeName, NodeDefinition defn, Path relPath, Integer length, boolean nullable, Object defaultValue) {
		super(name, type, typeName, length, nullable);
		this.nodeDefinition = defn;
		this.relativePath = relPath;
		this.defaultValue = defaultValue;
	}

	public NodeDefinition getNodeDefinition() {
		return nodeDefinition;
	}
	
	public Path getRelativePath() {
		return relativePath;
	}
	
	public Object getDefaultValue() {
		return defaultValue;
	}

	@Override
	public Object extractValue(Node<?> context) {
		Node<?> valNode = extractValueNode(context);
		Object val = convert(valNode);
		if ( getTypeName().equals("varchar") && val != null && val.toString().length() > getLength()) {
			System.out.println("Record: " + context.getRecord().getId() + ". Value of node " + context.getPath() + 
					" : length " + val.toString().length() + " exceeds max allowed (" + getLength() + "): " + val );
			val = ((String) val).substring(0, getLength());
		}
		if ( val == null && defaultValue != null) {
			return defaultValue;
		} else {
			return val;
		}
	}

	protected Node<?> extractValueNode(Node<?> context) {
		List<Node<?>> vals = relativePath.evaluate(context);
		if ( vals.size() > 1 ) {
			throw new RuntimeException("Path "+relativePath+" returned more than one value");
		}
		if ( vals.isEmpty() ) {
			return null;
		} else {
			Node<?> valNode = vals.get(0);
			return valNode;
		}
	}
	
	private Object convert(Node<?> valNode) {
		if ( valNode == null ) {
			return null;
		} else if ( valNode instanceof Field ) {
			return ((Field<?>) valNode).getValue();
		} else if ( valNode instanceof DateAttribute ) {
			Date date = ((DateAttribute) valNode).getValue();
			return date.toJavaDate();
		} else if ( valNode instanceof TimeAttribute ) {
			Time time = ((TimeAttribute) valNode).getValue();
			return time.toXmlTime();
		} else if ( valNode instanceof Attribute ) {
			return ((Attribute<?,?>) valNode).getValue();
		} else {
			throw new RuntimeException("Unknown data node type "+valNode.getClass());
		}
	}
	
}
<commit_msg>Skip invalid values during export<commit_after>package org.openforis.collect.relational.model;

import java.util.List;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.openforis.idm.metamodel.NodeDefinition;
import org.openforis.idm.model.Attribute;
import org.openforis.idm.model.Date;
import org.openforis.idm.model.DateAttribute;
import org.openforis.idm.model.Field;
import org.openforis.idm.model.Node;
import org.openforis.idm.model.Time;
import org.openforis.idm.model.TimeAttribute;
import org.openforis.idm.path.Path;

/**
 * 
 * @author <NAME>
 * @author <NAME>
 *
 */
public class DataColumn extends AbstractColumn<Node<?>> {
	
	private final Log log = LogFactory.getLog(DataColumn.class);
	
	private Object defaultValue;
	private NodeDefinition nodeDefinition;
	private Path relativePath;

	DataColumn(String name, int type, String typeName, NodeDefinition defn, Path relPath, Integer length, boolean nullable) {
		this(name, type, typeName, defn, relPath, length, nullable, null);
	}

	DataColumn(String name, int type, String typeName, NodeDefinition defn, Path relPath, Integer length, boolean nullable, Object defaultValue) {
		super(name, type, typeName, length, nullable);
		this.nodeDefinition = defn;
		this.relativePath = relPath;
		this.defaultValue = defaultValue;
	}

	public NodeDefinition getNodeDefinition() {
		return nodeDefinition;
	}
	
	public Path getRelativePath() {
		return relativePath;
	}
	
	public Object getDefaultValue() {
		return defaultValue;
	}

	@Override
	public Object extractValue(Node<?> context) {
		Node<?> valNode = extractValueNode(context);
		Object val = convert(valNode);
		if ( getTypeName().equals("varchar") && val != null && val.toString().length() > getLength()) {
			System.out.println("Record: " + context.getRecord().getId() + ". Value of node " + context.getPath() + 
					" : length " + val.toString().length() + " exceeds max allowed (" + getLength() + "): " + val );
			val = ((String) val).substring(0, getLength());
		}
		if ( val == null && defaultValue != null) {
			return defaultValue;
		} else {
			return val;
		}
	}

	protected Node<?> extractValueNode(Node<?> context) {
		List<Node<?>> vals = relativePath.evaluate(context);
		if ( vals.size() > 1 ) {
			throw new RuntimeException("Path "+relativePath+" returned more than one value");
		}
		if ( vals.isEmpty() ) {
			return null;
		} else {
			Node<?> valNode = vals.get(0);
			return valNode;
		}
	}
	
	private Object convert(Node<?> valNode) {
		if ( valNode == null ) {
			return null;
		} else {
			try {	
				if ( valNode instanceof Field ) {
					return ((Field<?>) valNode).getValue();
				} else if ( valNode instanceof DateAttribute ) {
					Date date = ((DateAttribute) valNode).getValue();
					return date.toJavaDate();
				} else if ( valNode instanceof TimeAttribute ) {
					Time time = ((TimeAttribute) valNode).getValue();
					return time.toXmlTime();
				} else if ( valNode instanceof Attribute ) {
					return ((Attribute<?,?>) valNode).getValue();
				} else {
					throw new RuntimeException("Unknown data node type "+valNode.getClass());
				}
			} catch ( Exception e) {
				//ERRORS in data?
				String messageFormat = "Error converting attribute value in record: %d - node: %s";
				String message = String.format(messageFormat, valNode.getRecord().getId(), valNode.getPath());
				log.error(message);
				System.out.println(message);
				return null;
			}
		}
	}
	
}

{'fraction_non_alphanumeric': 0.13649851632047477, 'fraction_numerical': 0.0009370607527721381, 'mean_word_length': 4.110933758978452, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4444807', 'n_tokens_mistral': 2180, 'n_tokens_neox': 1977, 'n_words': 502}