<commit_before>package org.pentaho.di.core.row;

import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.EOFException;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.math.BigDecimal;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.text.NumberFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;

import org.pentaho.di.compatibility.Value;
import org.pentaho.di.core.Const;
import org.pentaho.di.core.exception.KettleEOFException;
import org.pentaho.di.core.exception.KettleFileException;
import org.pentaho.di.core.exception.KettleValueException;


public class ValueMeta implements ValueMetaInterface
{
    public static final String DEFAULT_DATE_FORMAT_MASK = "yyyy/MM/dd HH:mm:ss.SSS";
    
    private String   name;
    private int      length;
    private int      precision;
    private int      type;
    private int      trimType;
    private int      storageType;
    private String   origin;
    private String   comments;
    private Object[] index;
    private String   conversionMask;
    private String   stringEncoding;
    private String   decimalSymbol;
    private String   groupingSymbol;
    private String   currencySymbol;
    private boolean  caseInsensitive;
    private boolean  sortedDescending;
    private boolean  outputPaddingEnabled;
    private boolean  largeTextField;
    private Locale   dateFormatLocale;
    private boolean  dateFormatLenient;
    
    private SimpleDateFormat dateFormat;
    private boolean dateFormatChanged;
    
    private DecimalFormat    decimalFormat;
    private boolean decimalFormatChanged;
    
    private ValueMetaInterface storageMetadata;
    private ValueMetaInterface stringMetadata;
    private boolean identicalFormat;

	private ValueMetaInterface nativeMetadata;
    
    public ValueMeta()
    {
        this(null, ValueMetaInterface.TYPE_NONE, -1, -1);
    }
    
    public ValueMeta(String name)
    {
        this(name, ValueMetaInterface.TYPE_NONE, -1, -1);
    }

    public ValueMeta(String name, int type)
    {
        this(name, type, -1, -1);
    }
    
    public ValueMeta(String name, int type, int storageType)
    {
        this(name, type, -1, -1);
        this.storageType = storageType;
    }
    
    public ValueMeta(String name, int type, int length, int precision)
    {
        this.name = name;
        this.type = type;
        this.length = length;
        this.precision = precision;
        this.storageType=STORAGE_TYPE_NORMAL;
        this.sortedDescending=false;
        this.outputPaddingEnabled=false;
        this.decimalSymbol = ""+Const.DEFAULT_DECIMAL_SEPARATOR;
        this.groupingSymbol = ""+Const.DEFAULT_GROUPING_SEPARATOR;
        this.dateFormatLocale = Locale.getDefault();
        this.identicalFormat = true;
    }
    
    public Object clone()
    {
        try
        {
            ValueMeta valueMeta = (ValueMeta) super.clone();
            valueMeta.dateFormat = null;
            valueMeta.decimalFormat = null;
            if (dateFormatLocale!=null) valueMeta.dateFormatLocale = (Locale) dateFormatLocale.clone();
            if (storageMetadata!=null) valueMeta.storageMetadata = (ValueMetaInterface) storageMetadata.clone();
            return valueMeta;
        }
        catch (CloneNotSupportedException e)
        {
            return null;
        }
    }

    /**
     * @return the comments
     */
    public String getComments()
    {
        return comments;
    }
    
    /**
     * @param comments the comments to set
     */
    public void setComments(String comments)
    {
        this.comments = comments;
    }
    
    /**
     * @return the index
     */
    public Object[] getIndex()
    {
        return index;
    }
    
    /**
     * @param index the index to set
     */
    public void setIndex(Object[] index)
    {
        this.index = index;
    }
    
    /**
     * @return the length
     */
    public int getLength()
    {
        return length;
    }
    
    /**
     * @param length the length to set
     */
    public void setLength(int length)
    {
        this.length = length;
    }

    /**
     * @param length the length to set
     */
    public void setLength(int length, int precision)
    {
        this.length = length;
        this.precision = precision;
    }

    /**
     * @return the name
     */
    public String getName()
    {
        return name;
    }
    
    /**
     * @param name the name to set
     */
    public void setName(String name)
    {
        this.name = name;
    }
    
    /**
     * @return the origin
     */
    public String getOrigin()
    {
        return origin;
    }
    
    /**
     * @param origin the origin to set
     */
    public void setOrigin(String origin)
    {
        this.origin = origin;
    }
    
    /**
     * @return the precision
     */
    public int getPrecision()
    {
        return precision;
    }
    
    /**
     * @param precision the precision to set
     */
    public void setPrecision(int precision)
    {
        this.precision = precision;
    }
    
    /**
     * @return the storageType
     */
    public int getStorageType()
    {
        return storageType;
    }
    
    /**
     * @param storageType the storageType to set
     */
    public void setStorageType(int storageType)
    {
        this.storageType = storageType;
    }

    public boolean isStorageNormal()
    {
        return storageType == STORAGE_TYPE_NORMAL;
    }

    public boolean isStorageIndexed()
    {
        return storageType == STORAGE_TYPE_INDEXED;
    }

    public boolean isStorageBinaryString()
    {
        return storageType == STORAGE_TYPE_BINARY_STRING;
    }


    /**
     * @return the type
     */
    public int getType()
    {
        return type;
    }
    
    /**
     * @param type the type to set
     */
    public void setType(int type)
    {
        this.type = type;
    }
    
    /**
     * @return the conversionMask
     */
    public String getConversionMask()
    {
        return conversionMask;
    }
    
    /**
     * @param conversionMask the conversionMask to set
     */
    public void setConversionMask(String conversionMask)
    {
        this.conversionMask = conversionMask;
        dateFormatChanged = true;
        decimalFormatChanged = true;
        compareStorageAndActualFormat();
    }
    
    /**
     * @return the encoding
     */
    public String getStringEncoding()
    {
        return stringEncoding;
    }

    /**
     * @param encoding the encoding to set
     */
    public void setStringEncoding(String encoding)
    {
        this.stringEncoding = encoding;
        compareStorageAndActualFormat();
    }
    
    /**
     * @return the decimalSymbol
     */
    public String getDecimalSymbol()
    {
        return decimalSymbol;
    }

    /**
     * @param decimalSymbol the decimalSymbol to set
     */
    public void setDecimalSymbol(String decimalSymbol)
    {
        this.decimalSymbol = decimalSymbol;
        decimalFormatChanged = true;
        compareStorageAndActualFormat();
    }

    /**
     * @return the groupingSymbol
     */
    public String getGroupingSymbol()
    {
        return groupingSymbol;
    }

    /**
     * @param groupingSymbol the groupingSymbol to set
     */
    public void setGroupingSymbol(String groupingSymbol)
    {
        this.groupingSymbol = groupingSymbol;
        decimalFormatChanged = true;
        compareStorageAndActualFormat();
    }
    

    /**
     * @return the currencySymbol
     */
    public String getCurrencySymbol()
    {
        return currencySymbol;
    }

    /**
     * @param currencySymbol the currencySymbol to set
     */
    public void setCurrencySymbol(String currencySymbol)
    {
        this.currencySymbol = currencySymbol;
        decimalFormatChanged = true;
    }
    
    /**
     * @return the caseInsensitive
     */
    public boolean isCaseInsensitive()
    {
        return caseInsensitive;
    }

    /**
     * @param caseInsensitive the caseInsensitive to set
     */
    public void setCaseInsensitive(boolean caseInsensitive)
    {
        this.caseInsensitive = caseInsensitive;
    }

    
    /**
     * @return the sortedDescending
     */
    public boolean isSortedDescending()
    {
        return sortedDescending;
    }

    /**
     * @param sortedDescending the sortedDescending to set
     */
    public void setSortedDescending(boolean sortedDescending)
    {
        this.sortedDescending = sortedDescending;
    }
    

    /**
     * @return true if output padding is enabled (padding to specified length)
     */
    public boolean isOutputPaddingEnabled()
    {
        return outputPaddingEnabled;
    }

    /**
     * @param outputPaddingEnabled Set to true if output padding is to be enabled (padding to specified length)
     */
    public void setOutputPaddingEnabled(boolean outputPaddingEnabled)
    {
        this.outputPaddingEnabled = outputPaddingEnabled;
    }

    /**
     * @return true if this is a large text field (CLOB, TEXT) with arbitrary length.
     */
    public boolean isLargeTextField()
    {
        return largeTextField;
    }

    /**
     * @param largeTextField Set to true if this is to be a large text field (CLOB, TEXT) with arbitrary length.
     */
    public void setLargeTextField(boolean largeTextField)
    {
        this.largeTextField = largeTextField;
    }
    
    /**
     * @return the dateFormatLenient
     */
    public boolean isDateFormatLenient()
    {
        return dateFormatLenient;
    }

    /**
     * @param dateFormatLenient the dateFormatLenient to set
     */
    public void setDateFormatLenient(boolean dateFormatLenient)
    {
        this.dateFormatLenient = dateFormatLenient;
        dateFormatChanged=true;
    }

    /**
     * @return the dateFormatLocale
     */
    public Locale getDateFormatLocale()
    {
        return dateFormatLocale;
    }

    /**
     * @param dateFormatLocale the dateFormatLocale to set
     */
    public void setDateFormatLocale(Locale dateFormatLocale)
    {
        this.dateFormatLocale = dateFormatLocale;
        dateFormatChanged=true;
    }
    
    

    // DATE + STRING


    private synchronized String convertDateToString(Date date)
    {
        if (date==null) return null;
        
        return getDateFormat().format(date);
    }

    private synchronized Date convertStringToDate(String string) throws KettleValueException
    {
        if (string==null) return null;
        
        try
        {
            return getDateFormat().parse(string);
        }
        catch (ParseException e)
        {
            throw new KettleValueException("Unable to convert string ["+string+"] to a date", e);
        }
    }

    // DATE + NUMBER 

    private Double convertDateToNumber(Date date)
    {
        return new Double( date.getTime() );
    }

    private Date convertNumberToDate(Double number)
    {
        return new Date( number.longValue() );
    }

    // DATE + INTEGER
    
    private Long convertDateToInteger(Date date)
    {
        return new Long( date.getTime() );
    }

    private Date convertIntegerToDate(Long number)
    {
        return new Date( number.longValue() );
    }

    // DATE + BIGNUMBER
    
    private BigDecimal convertDateToBigNumber(Date date)
    {
        return new BigDecimal( date.getTime() );
    }

    private Date convertBigNumberToDate(BigDecimal number)
    {
        return new Date( number.longValue() );
    }

    private synchronized String convertNumberToString(Double number) throws KettleValueException
    {
        if (number==null) return null;
        
        try
        {
            if (Const.isEmpty(conversionMask) && Const.isEmpty(decimalSymbol) && Const.isEmpty(groupingSymbol)) 
            {
                return number.toString();
            }
            return getDecimalFormat().format(number);
        }
        catch(Exception e)
        {
            throw new KettleValueException("Couldn't convert Number to String ", e);
        }
    }
    
    private synchronized Double convertStringToNumber(String string) throws KettleValueException
    {
        if (string==null) return null;
        
        try
        {
            return new Double( getDecimalFormat().parse(string).doubleValue() );
        }
        catch(Exception e)
        {
            throw new KettleValueException("Couldn't convert String to number ", e);
        }
    }
    
    public synchronized SimpleDateFormat getDateFormat()
    {
        if (dateFormat==null || dateFormatChanged)
        {
            dateFormat = new SimpleDateFormat();
            String mask;
            if (Const.isEmpty(conversionMask))
            {
                mask = DEFAULT_DATE_FORMAT_MASK;
            }
            else
            {
                mask = conversionMask;
            }
            
            if (dateFormatLocale==null || dateFormatLocale.equals(Locale.getDefault()))
            {
                dateFormat = new SimpleDateFormat(mask);
            }
            else
            {
                dateFormat = new SimpleDateFormat(mask, dateFormatLocale);
            }
            
            dateFormatChanged=false;
        }
        return dateFormat;
    }

    public synchronized DecimalFormat getDecimalFormat()
    {
        if (decimalFormat==null || decimalFormatChanged)
        {
            decimalFormat        = (DecimalFormat)NumberFormat.getInstance();
            DecimalFormatSymbols decimalFormatSymbols = decimalFormat.getDecimalFormatSymbols();
        
            if (!Const.isEmpty(currencySymbol)) decimalFormatSymbols.setCurrencySymbol( currencySymbol );
            if (!Const.isEmpty(groupingSymbol)) decimalFormatSymbols.setGroupingSeparator( groupingSymbol.charAt(0) );
            if (!Const.isEmpty(decimalSymbol)) decimalFormatSymbols.setDecimalSeparator( decimalSymbol.charAt(0) );
            decimalFormat.setDecimalFormatSymbols(decimalFormatSymbols);
            if (!Const.isEmpty(conversionMask)) decimalFormat.applyPattern(conversionMask);
            
            decimalFormatChanged=false;
        }
        return decimalFormat;
    }

    private synchronized String convertIntegerToString(Long number) throws KettleValueException
    {
        if (number==null) return null;

        try
        {
            if (Const.isEmpty(conversionMask) && Const.isEmpty(decimalSymbol) && Const.isEmpty(groupingSymbol)) 
            {
                return number.toString();
            }
            return getDecimalFormat().format(number);
        }
        catch(Exception e)
        {
            throw new KettleValueException("Couldn't convert Long to String ", e);
        }
    }
    
    private synchronized Long convertStringToInteger(String string) throws KettleValueException
    {
        if (Const.isEmpty(string)) return null;

        try
        {
            return new Long( getDecimalFormat().parse(string).longValue() );
        }
        catch(Exception e)
        {
            throw new KettleValueException("Couldn't convert String to Integer", e);
        }
    }
    
    private synchronized String convertBigNumberToString(BigDecimal number) throws KettleValueException
    {
        if (number==null) return null;

        String string = number.toString();
        /*
        if ( !Const.isEmpty(decimalSymbol) && !".".equalsIgnoreCase(decimalSymbol) )
        {
            string = Const.replace(string, ".", decimalSymbol.substring(0, 1));
        }
        */
        return string;
    }
    
    private synchronized BigDecimal convertStringToBigNumber(String string) throws KettleValueException
    {
        if (string==null) return null;

        /*
        if (!".".equalsIgnoreCase(decimalSymbol))
        {
            string = Const.replace(string, decimalSymbol.substring(0, 1), ".");
        }
        */
        
        return new BigDecimal( string );
    }

    // BOOLEAN + STRING
    
    private String convertBooleanToString(Boolean bool)
    {
        if (length>=3)
        {
            return bool.booleanValue()?"true":"false";
        }
        else
        {
            return bool.booleanValue()?"Y":"N";
        }
    }
    
    public static Boolean convertStringToBoolean(String string)
    {
        return new Boolean( "Y".equalsIgnoreCase(string) || "TRUE".equalsIgnoreCase(string) || "YES".equalsIgnoreCase(string) );
    }
    
    // BOOLEAN + NUMBER
    
    private Double convertBooleanToNumber(Boolean bool)
    {
        return new Double( bool.booleanValue() ? 1.0 : 0.0 );
    }
    
    private Boolean convertNumberToBoolean(Double number)
    {
        return new Boolean( number.intValue() != 0 );
    }

    // BOOLEAN + INTEGER<commit_msg>Adding "1" to being recognized as boolean to be backward compatible with 2.5.<commit_after>package org.pentaho.di.core.row;

import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.EOFException;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.math.BigDecimal;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.text.NumberFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;

import org.pentaho.di.compatibility.Value;
import org.pentaho.di.core.Const;
import org.pentaho.di.core.exception.KettleEOFException;
import org.pentaho.di.core.exception.KettleFileException;
import org.pentaho.di.core.exception.KettleValueException;


public class ValueMeta implements ValueMetaInterface
{
    public static final String DEFAULT_DATE_FORMAT_MASK = "yyyy/MM/dd HH:mm:ss.SSS";
    
    private String   name;
    private int      length;
    private int      precision;
    private int      type;
    private int      trimType;
    private int      storageType;
    private String   origin;
    private String   comments;
    private Object[] index;
    private String   conversionMask;
    private String   stringEncoding;
    private String   decimalSymbol;
    private String   groupingSymbol;
    private String   currencySymbol;
    private boolean  caseInsensitive;
    private boolean  sortedDescending;
    private boolean  outputPaddingEnabled;
    private boolean  largeTextField;
    private Locale   dateFormatLocale;
    private boolean  dateFormatLenient;
    
    private SimpleDateFormat dateFormat;
    private boolean dateFormatChanged;
    
    private DecimalFormat    decimalFormat;
    private boolean decimalFormatChanged;
    
    private ValueMetaInterface storageMetadata;
    private ValueMetaInterface stringMetadata;
    private boolean identicalFormat;

	private ValueMetaInterface nativeMetadata;
    
    public ValueMeta()
    {
        this(null, ValueMetaInterface.TYPE_NONE, -1, -1);
    }
    
    public ValueMeta(String name)
    {
        this(name, ValueMetaInterface.TYPE_NONE, -1, -1);
    }

    public ValueMeta(String name, int type)
    {
        this(name, type, -1, -1);
    }
    
    public ValueMeta(String name, int type, int storageType)
    {
        this(name, type, -1, -1);
        this.storageType = storageType;
    }
    
    public ValueMeta(String name, int type, int length, int precision)
    {
        this.name = name;
        this.type = type;
        this.length = length;
        this.precision = precision;
        this.storageType=STORAGE_TYPE_NORMAL;
        this.sortedDescending=false;
        this.outputPaddingEnabled=false;
        this.decimalSymbol = ""+Const.DEFAULT_DECIMAL_SEPARATOR;
        this.groupingSymbol = ""+Const.DEFAULT_GROUPING_SEPARATOR;
        this.dateFormatLocale = Locale.getDefault();
        this.identicalFormat = true;
    }
    
    public Object clone()
    {
        try
        {
            ValueMeta valueMeta = (ValueMeta) super.clone();
            valueMeta.dateFormat = null;
            valueMeta.decimalFormat = null;
            if (dateFormatLocale!=null) valueMeta.dateFormatLocale = (Locale) dateFormatLocale.clone();
            if (storageMetadata!=null) valueMeta.storageMetadata = (ValueMetaInterface) storageMetadata.clone();
            return valueMeta;
        }
        catch (CloneNotSupportedException e)
        {
            return null;
        }
    }

    /**
     * @return the comments
     */
    public String getComments()
    {
        return comments;
    }
    
    /**
     * @param comments the comments to set
     */
    public void setComments(String comments)
    {
        this.comments = comments;
    }
    
    /**
     * @return the index
     */
    public Object[] getIndex()
    {
        return index;
    }
    
    /**
     * @param index the index to set
     */
    public void setIndex(Object[] index)
    {
        this.index = index;
    }
    
    /**
     * @return the length
     */
    public int getLength()
    {
        return length;
    }
    
    /**
     * @param length the length to set
     */
    public void setLength(int length)
    {
        this.length = length;
    }

    /**
     * @param length the length to set
     */
    public void setLength(int length, int precision)
    {
        this.length = length;
        this.precision = precision;
    }

    /**
     * @return the name
     */
    public String getName()
    {
        return name;
    }
    
    /**
     * @param name the name to set
     */
    public void setName(String name)
    {
        this.name = name;
    }
    
    /**
     * @return the origin
     */
    public String getOrigin()
    {
        return origin;
    }
    
    /**
     * @param origin the origin to set
     */
    public void setOrigin(String origin)
    {
        this.origin = origin;
    }
    
    /**
     * @return the precision
     */
    public int getPrecision()
    {
        return precision;
    }
    
    /**
     * @param precision the precision to set
     */
    public void setPrecision(int precision)
    {
        this.precision = precision;
    }
    
    /**
     * @return the storageType
     */
    public int getStorageType()
    {
        return storageType;
    }
    
    /**
     * @param storageType the storageType to set
     */
    public void setStorageType(int storageType)
    {
        this.storageType = storageType;
    }

    public boolean isStorageNormal()
    {
        return storageType == STORAGE_TYPE_NORMAL;
    }

    public boolean isStorageIndexed()
    {
        return storageType == STORAGE_TYPE_INDEXED;
    }

    public boolean isStorageBinaryString()
    {
        return storageType == STORAGE_TYPE_BINARY_STRING;
    }


    /**
     * @return the type
     */
    public int getType()
    {
        return type;
    }
    
    /**
     * @param type the type to set
     */
    public void setType(int type)
    {
        this.type = type;
    }
    
    /**
     * @return the conversionMask
     */
    public String getConversionMask()
    {
        return conversionMask;
    }
    
    /**
     * @param conversionMask the conversionMask to set
     */
    public void setConversionMask(String conversionMask)
    {
        this.conversionMask = conversionMask;
        dateFormatChanged = true;
        decimalFormatChanged = true;
        compareStorageAndActualFormat();
    }
    
    /**
     * @return the encoding
     */
    public String getStringEncoding()
    {
        return stringEncoding;
    }

    /**
     * @param encoding the encoding to set
     */
    public void setStringEncoding(String encoding)
    {
        this.stringEncoding = encoding;
        compareStorageAndActualFormat();
    }
    
    /**
     * @return the decimalSymbol
     */
    public String getDecimalSymbol()
    {
        return decimalSymbol;
    }

    /**
     * @param decimalSymbol the decimalSymbol to set
     */
    public void setDecimalSymbol(String decimalSymbol)
    {
        this.decimalSymbol = decimalSymbol;
        decimalFormatChanged = true;
        compareStorageAndActualFormat();
    }

    /**
     * @return the groupingSymbol
     */
    public String getGroupingSymbol()
    {
        return groupingSymbol;
    }

    /**
     * @param groupingSymbol the groupingSymbol to set
     */
    public void setGroupingSymbol(String groupingSymbol)
    {
        this.groupingSymbol = groupingSymbol;
        decimalFormatChanged = true;
        compareStorageAndActualFormat();
    }
    

    /**
     * @return the currencySymbol
     */
    public String getCurrencySymbol()
    {
        return currencySymbol;
    }

    /**
     * @param currencySymbol the currencySymbol to set
     */
    public void setCurrencySymbol(String currencySymbol)
    {
        this.currencySymbol = currencySymbol;
        decimalFormatChanged = true;
    }
    
    /**
     * @return the caseInsensitive
     */
    public boolean isCaseInsensitive()
    {
        return caseInsensitive;
    }

    /**
     * @param caseInsensitive the caseInsensitive to set
     */
    public void setCaseInsensitive(boolean caseInsensitive)
    {
        this.caseInsensitive = caseInsensitive;
    }

    
    /**
     * @return the sortedDescending
     */
    public boolean isSortedDescending()
    {
        return sortedDescending;
    }

    /**
     * @param sortedDescending the sortedDescending to set
     */
    public void setSortedDescending(boolean sortedDescending)
    {
        this.sortedDescending = sortedDescending;
    }
    

    /**
     * @return true if output padding is enabled (padding to specified length)
     */
    public boolean isOutputPaddingEnabled()
    {
        return outputPaddingEnabled;
    }

    /**
     * @param outputPaddingEnabled Set to true if output padding is to be enabled (padding to specified length)
     */
    public void setOutputPaddingEnabled(boolean outputPaddingEnabled)
    {
        this.outputPaddingEnabled = outputPaddingEnabled;
    }

    /**
     * @return true if this is a large text field (CLOB, TEXT) with arbitrary length.
     */
    public boolean isLargeTextField()
    {
        return largeTextField;
    }

    /**
     * @param largeTextField Set to true if this is to be a large text field (CLOB, TEXT) with arbitrary length.
     */
    public void setLargeTextField(boolean largeTextField)
    {
        this.largeTextField = largeTextField;
    }
    
    /**
     * @return the dateFormatLenient
     */
    public boolean isDateFormatLenient()
    {
        return dateFormatLenient;
    }

    /**
     * @param dateFormatLenient the dateFormatLenient to set
     */
    public void setDateFormatLenient(boolean dateFormatLenient)
    {
        this.dateFormatLenient = dateFormatLenient;
        dateFormatChanged=true;
    }

    /**
     * @return the dateFormatLocale
     */
    public Locale getDateFormatLocale()
    {
        return dateFormatLocale;
    }

    /**
     * @param dateFormatLocale the dateFormatLocale to set
     */
    public void setDateFormatLocale(Locale dateFormatLocale)
    {
        this.dateFormatLocale = dateFormatLocale;
        dateFormatChanged=true;
    }
    
    

    // DATE + STRING


    private synchronized String convertDateToString(Date date)
    {
        if (date==null) return null;
        
        return getDateFormat().format(date);
    }

    private synchronized Date convertStringToDate(String string) throws KettleValueException
    {
        if (string==null) return null;
        
        try
        {
            return getDateFormat().parse(string);
        }
        catch (ParseException e)
        {
            throw new KettleValueException("Unable to convert string ["+string+"] to a date", e);
        }
    }

    // DATE + NUMBER 

    private Double convertDateToNumber(Date date)
    {
        return new Double( date.getTime() );
    }

    private Date convertNumberToDate(Double number)
    {
        return new Date( number.longValue() );
    }

    // DATE + INTEGER
    
    private Long convertDateToInteger(Date date)
    {
        return new Long( date.getTime() );
    }

    private Date convertIntegerToDate(Long number)
    {
        return new Date( number.longValue() );
    }

    // DATE + BIGNUMBER
    
    private BigDecimal convertDateToBigNumber(Date date)
    {
        return new BigDecimal( date.getTime() );
    }

    private Date convertBigNumberToDate(BigDecimal number)
    {
        return new Date( number.longValue() );
    }

    private synchronized String convertNumberToString(Double number) throws KettleValueException
    {
        if (number==null) return null;
        
        try
        {
            if (Const.isEmpty(conversionMask) && Const.isEmpty(decimalSymbol) && Const.isEmpty(groupingSymbol)) 
            {
                return number.toString();
            }
            return getDecimalFormat().format(number);
        }
        catch(Exception e)
        {
            throw new KettleValueException("Couldn't convert Number to String ", e);
        }
    }
    
    private synchronized Double convertStringToNumber(String string) throws KettleValueException
    {
        if (string==null) return null;
        
        try
        {
            return new Double( getDecimalFormat().parse(string).doubleValue() );
        }
        catch(Exception e)
        {
            throw new KettleValueException("Couldn't convert String to number ", e);
        }
    }
    
    public synchronized SimpleDateFormat getDateFormat()
    {
        if (dateFormat==null || dateFormatChanged)
        {
            dateFormat = new SimpleDateFormat();
            String mask;
            if (Const.isEmpty(conversionMask))
            {
                mask = DEFAULT_DATE_FORMAT_MASK;
            }
            else
            {
                mask = conversionMask;
            }
            
            if (dateFormatLocale==null || dateFormatLocale.equals(Locale.getDefault()))
            {
                dateFormat = new SimpleDateFormat(mask);
            }
            else
            {
                dateFormat = new SimpleDateFormat(mask, dateFormatLocale);
            }
            
            dateFormatChanged=false;
        }
        return dateFormat;
    }

    public synchronized DecimalFormat getDecimalFormat()
    {
        if (decimalFormat==null || decimalFormatChanged)
        {
            decimalFormat        = (DecimalFormat)NumberFormat.getInstance();
            DecimalFormatSymbols decimalFormatSymbols = decimalFormat.getDecimalFormatSymbols();
        
            if (!Const.isEmpty(currencySymbol)) decimalFormatSymbols.setCurrencySymbol( currencySymbol );
            if (!Const.isEmpty(groupingSymbol)) decimalFormatSymbols.setGroupingSeparator( groupingSymbol.charAt(0) );
            if (!Const.isEmpty(decimalSymbol)) decimalFormatSymbols.setDecimalSeparator( decimalSymbol.charAt(0) );
            decimalFormat.setDecimalFormatSymbols(decimalFormatSymbols);
            if (!Const.isEmpty(conversionMask)) decimalFormat.applyPattern(conversionMask);
            
            decimalFormatChanged=false;
        }
        return decimalFormat;
    }

    private synchronized String convertIntegerToString(Long number) throws KettleValueException
    {
        if (number==null) return null;

        try
        {
            if (Const.isEmpty(conversionMask) && Const.isEmpty(decimalSymbol) && Const.isEmpty(groupingSymbol)) 
            {
                return number.toString();
            }
            return getDecimalFormat().format(number);
        }
        catch(Exception e)
        {
            throw new KettleValueException("Couldn't convert Long to String ", e);
        }
    }
    
    private synchronized Long convertStringToInteger(String string) throws KettleValueException
    {
        if (Const.isEmpty(string)) return null;

        try
        {
            return new Long( getDecimalFormat().parse(string).longValue() );
        }
        catch(Exception e)
        {
            throw new KettleValueException("Couldn't convert String to Integer", e);
        }
    }
    
    private synchronized String convertBigNumberToString(BigDecimal number) throws KettleValueException
    {
        if (number==null) return null;

        String string = number.toString();
        /*
        if ( !Const.isEmpty(decimalSymbol) && !".".equalsIgnoreCase(decimalSymbol) )
        {
            string = Const.replace(string, ".", decimalSymbol.substring(0, 1));
        }
        */
        return string;
    }
    
    private synchronized BigDecimal convertStringToBigNumber(String string) throws KettleValueException
    {
        if (string==null) return null;

        /*
        if (!".".equalsIgnoreCase(decimalSymbol))
        {
            string = Const.replace(string, decimalSymbol.substring(0, 1), ".");
        }
        */
        
        return new BigDecimal( string );
    }

    // BOOLEAN + STRING
    
    private String convertBooleanToString(Boolean bool)
    {
        if (length>=3)
        {
            return bool.booleanValue()?"true":"false";
        }
        else
        {
            return bool.booleanValue()?"Y":"N";
        }
    }
    
    public static Boolean convertStringToBoolean(String string)
    {
        return new Boolean( "Y".equalsIgnoreCase(string) || "TRUE".equalsIgnoreCase(string) || "YES".equalsIgnoreCase(string) || "1".equals(string) );
    }
    
    // BOOLEAN + NUMBER
    
    private Double convertBooleanToNumber(Boolean bool)
    {
        return new Double( bool.booleanValue() ? 1.0 : 0.0 );
    }
    
    private Boolean convertNumberToBoolean(Double number)
    {
        return new Boolean( number.intValue() != 0 );
    }

    // BOOLEAN + INTEGER
{'fraction_non_alphanumeric': 0.0859462651313847, 'fraction_numerical': 0.0012990847357543548, 'mean_word_length': 2.0350358422939068, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4902603', 'n_tokens_mistral': 9127, 'n_tokens_neox': 8942, 'n_words': 2132}