<commit_before>/*******************************************************************************
 * Copyright (c) 2004, 2008Actuate Corporation.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *  Actuate Corporation  - initial API and implementation
 *******************************************************************************/

package org.eclipse.birt.report.engine.emitter.excel.layout;

import java.io.IOException;
import java.sql.Time;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Stack;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.eclipse.birt.report.engine.content.ICellContent;
import org.eclipse.birt.report.engine.content.IContainerContent;
import org.eclipse.birt.report.engine.content.IDataContent;
import org.eclipse.birt.report.engine.content.IHyperlinkAction;
import org.eclipse.birt.report.engine.content.IImageContent;
import org.eclipse.birt.report.engine.content.IStyle;
import org.eclipse.birt.report.engine.emitter.EmitterUtil;
import org.eclipse.birt.report.engine.emitter.excel.BlankData;
import org.eclipse.birt.report.engine.emitter.excel.BookmarkDef;
import org.eclipse.birt.report.engine.emitter.excel.Data;
import org.eclipse.birt.report.engine.emitter.excel.DataCache;
import org.eclipse.birt.report.engine.emitter.excel.DateTimeUtil;
import org.eclipse.birt.report.engine.emitter.excel.ExcelEmitter;
import org.eclipse.birt.report.engine.emitter.excel.ExcelUtil;
import org.eclipse.birt.report.engine.emitter.excel.HyperlinkDef;
import org.eclipse.birt.report.engine.emitter.excel.ImageData;
import org.eclipse.birt.report.engine.emitter.excel.RowData;
import org.eclipse.birt.report.engine.emitter.excel.SheetData;
import org.eclipse.birt.report.engine.emitter.excel.StyleBuilder;
import org.eclipse.birt.report.engine.emitter.excel.StyleConstant;
import org.eclipse.birt.report.engine.emitter.excel.StyleEngine;
import org.eclipse.birt.report.engine.emitter.excel.StyleEntry;
import org.eclipse.birt.report.engine.emitter.excel.BlankData.Type;
import org.eclipse.birt.report.engine.i18n.EngineResourceHandle;
import org.eclipse.birt.report.engine.i18n.MessageConstants;
import org.eclipse.birt.report.engine.layout.emitter.Image;
import org.eclipse.birt.report.engine.layout.pdf.util.PropertyUtil;
import org.eclipse.birt.report.engine.util.FlashFile;

import com.ibm.icu.util.ULocale;


public class ExcelLayoutEngine
{

	protected static Logger logger = Logger.getLogger( ExcelLayoutEngine.class
			.getName( ) );

	public final static String EMPTY = "";
	
	public static final float DEFAULT_ROW_HEIGHT = 15;

	// Excel 2007 can support 1048576 rows and 16384 columns.
	private int autoBookmarkIndex = 0;

	public static final String AUTO_GENERATED_BOOKMARK = "auto_generated_bookmark_";

	public final static int MAX_ROW_OFFICE2007 = 1048576;
	
	public final static int MAX_COL_OFFICE2007 = 16384;
	
	public final static int MAX_ROW_OFFICE2003 = 65535;
	
	public final static int MAX_COLUMN_OFFICE2003 = 255;
	
	private int maxRow = MAX_ROW_OFFICE2003;

	private int maxCol = MAX_COLUMN_OFFICE2003;
	
	private HashMap<String, String> cachedBookmarks = new HashMap<String, String>( );

	protected DataCache cache;

	private AxisProcessor axis;

	protected StyleEngine engine;
	
	private ExcelEmitter emitter;

	private Stack<XlsContainer> containers = new Stack<XlsContainer>( );

	private Stack<XlsTable> tables = new Stack<XlsTable>( );

	private ExcelContext context = null;

	private String messageFlashObjectNotSupported;
	
	private ULocale locale;

	private HashMap<String, BookmarkDef> bookmarkList = new HashMap<String, BookmarkDef>( );

	public ExcelLayoutEngine( ExcelContext context,
			ExcelEmitter emitter )
	{
		this.context = context;
		this.emitter = emitter;
		this.locale = context.getLocale( );
		EngineResourceHandle resourceHandle = new EngineResourceHandle( locale );
		messageFlashObjectNotSupported = resourceHandle
				.getMessage( MessageConstants.FLASH_OBJECT_NOT_SUPPORTED_PROMPT );
	}
	
	public void initalize( int contentWidth, IStyle style )
	{
		axis = new AxisProcessor( );		
		axis.addCoordinate( contentWidth );

		setCacheSize();
		
		ContainerSizeInfo rule = new ContainerSizeInfo( 0, contentWidth );
		cache = createDataCache( maxCol, maxRow );
		engine = new StyleEngine( this );
		containers.push( createContainer( rule, style, null ) );
	}

	protected DataCache createDataCache( int maxColumn, int maxRow )
	{
		return new DataCache( maxColumn, maxRow );
	}
	
	private void setCacheSize()
	{
		if ( context.getOfficeVersion( ).equals( "office2007" ) )
		{
			maxCol = MAX_COL_OFFICE2007;
		}
	}

	public XlsContainer getCurrentContainer( )
	{
		return (XlsContainer) containers.peek( );
	}

	public Stack<XlsContainer> getContainers( )
	{
		return containers;
	}

	public void setPageStyle( IStyle style )
	{
		XlsContainer topContainer = containers.peek( );
		topContainer.setStyle( StyleBuilder.createStyleEntry( style ) );
	}
	
	public void addTable( IContainerContent content, ColumnsInfo table,
			ContainerSizeInfo size )
	{
		IStyle style = content.getComputedStyle( );
		XlsContainer currentContainer = getCurrentContainer( );
		ContainerSizeInfo parentSizeInfo = currentContainer.getSizeInfo( );
		int[] columnStartCoordinates = splitColumns( table, parentSizeInfo );
		createTable( table, style, currentContainer, columnStartCoordinates );
	}

	protected int[] splitColumns( ColumnsInfo columnsInfo,
			ContainerSizeInfo parentSizeInfo )
	{
		int startCoordinate = parentSizeInfo.getStartCoordinate( );
		int endCoordinate = parentSizeInfo.getEndCoordinate( );

		int[] columnStartCoordinates = calculateColumnCoordinates( columnsInfo,
				startCoordinate, endCoordinate );

		splitColumns( startCoordinate, endCoordinate, columnStartCoordinates );
		return columnStartCoordinates;
	}

	private void createTable( ColumnsInfo tableInfo, IStyle style,
			XlsContainer currentContainer, int[] columnStartCoordinates )
	{
		int leftCordinate = columnStartCoordinates[0];
		int width = columnStartCoordinates[columnStartCoordinates.length - 1]
				- leftCordinate;
		ContainerSizeInfo sizeInfo = new ContainerSizeInfo( leftCordinate,
				width );
		StyleEntry styleEntry = engine.createEntry( sizeInfo, style,
													getParentStyle( ) );
		XlsTable table = new XlsTable( tableInfo, styleEntry,
				sizeInfo, currentContainer );
		tables.push( table );
		addContainer( table );
	}

	protected StyleEntry getParentStyle( )
	{
		return getParentStyle( getCurrentContainer( ) );
	}

	private void splitColumns( int startCoordinate, int endCoordinate,
			int[] columnStartCoordinates )
	{
		int[] scale = axis.getColumnCoordinatesInRange( startCoordinate,
				endCoordinate );

		for ( int i = 0; i < scale.length - 1; i++ )
		{
			int startPosition = scale[i];
			int endPostion = scale[i + 1];

			int[] range = inRange( startPosition, endPostion, columnStartCoordinates );

			if ( range.length > 0 )
			{				
				int pos = axis.getColumnIndexByCoordinate( startPosition );
				cache.insertColumns( pos, range.length );

				for ( int j = 0; j < range.length; j++ )
				{
					axis.addCoordinate( range[j] );
				}
			}
		}
	}

	private int[] calculateColumnCoordinates( ColumnsInfo table,
			int startCoordinate,
			int endCoordinate )
	{
		int columnCount = table.getColumnCount( );
		int[] columnStartCoordinates = new int[ columnCount + 1 ];
		columnStartCoordinates[0] = startCoordinate;
		for ( int i = 1; i <= columnCount; i++ )
		{
			if ( ( columnStartCoordinates[i - 1] + table.getColumnWidth( i - 1 ) ) > endCoordinate )
			{
				columnStartCoordinates[i] = endCoordinate;
			}
			else
				columnStartCoordinates[i] = columnStartCoordinates[i - 1]
				+ table.getColumnWidth( i - 1 );
		}
		return columnStartCoordinates;
	}

	private int[] inRange( int start, int end, int[] data )
	{
		int[] range = new int[data.length];
		int count = 0;

		for ( int i = 0; i < data.length; i++ )
		{
			if ( ( data[i] > start ) && ( data[i] < end ) )
			{
				count++;
				range[count] = data[i];
			}
		}

		int[] result = new int[count];

		int j = 0;
		for ( int i = 0; i < range.length; i++ )
		{
			if ( range[i] != 0 )
			{
				result[j] = range[i];
				j++;
			}
		}

		return result;
	}

	public void addCell( int col, int colSpan, int rowSpan, IStyle style )
	{
		XlsTable table = tables.peek( );
		ContainerSizeInfo cellSizeInfo = table.getColumnSizeInfo( col, colSpan );
		XlsCell cell = new XlsCell( engine.createEntry( cellSizeInfo, style,
														getParentStyle( ) ),
				cellSizeInfo, getCurrentContainer( ), rowSpan );
		addContainer( cell );
	}

	public void addCell( ICellContent cellcontent, int col, int colSpan,
			int rowSpan, IStyle style )
	{
		XlsTable table = tables.peek( );
		ContainerSizeInfo cellSizeInfo = table.getColumnSizeInfo( col, colSpan );
		int diagonalNumber = cellcontent.getDiagonalNumber( );
		StyleEntry cellStyleEntry = null;
		if ( diagonalNumber != 0 )
		{
			String diagonalColor = cellcontent.getDiagonalColor( );
			String diagonalStyle = cellcontent.getDiagonalStyle( );
			int diagonalWidth = PropertyUtil.getDimensionValue( cellcontent,
					cellcontent.getDiagonalWidth( ), cellSizeInfo.getWidth( ) );
			cellStyleEntry = engine.createCellEntry( cellSizeInfo, style,
														diagonalColor,
														diagonalStyle,
														diagonalWidth,
														getParentStyle( ) );
		}
		else
		{
			cellStyleEntry = engine.createEntry( cellSizeInfo, style,
													getParentStyle( ) );
		}
		XlsCell cell = new XlsCell( cellStyleEntry, cellSizeInfo,
				getCurrentContainer( ), rowSpan );
		addContainer( cell );
	}

	public void endCell( )
	{
		endNormalContainer( );
	}

	public void addRow( IStyle style )
	{
		XlsContainer parent = getCurrentContainer( );
		ContainerSizeInfo sizeInfo = parent.getSizeInfo( );
		XlsContainer container = createContainer( sizeInfo, style,
				parent );
		container.setEmpty( false );
		addContainer( container );
	}

	public void endRow( float rowHeight )
	{
		synchronize( rowHeight );
		endContainer( );
	}

	private void synchronize( float height )
	{
		XlsContainer rowContainer = getCurrentContainer( );
		ContainerSizeInfo rowSizeInfo = rowContainer.getSizeInfo( );
		int startCoordinate = rowSizeInfo.getStartCoordinate( );
		int endCoordinate = rowSizeInfo.getEndCoordinate( );
		int startColumnIndex = axis.getColumnIndexByCoordinate( startCoordinate );
		int endColumnIndex = axis.getColumnIndexByCoordinate( endCoordinate );

		int maxRowIndex = 0;
		int rowIndexes[] = new int[endColumnIndex - startColumnIndex];

		for ( int currentColumnIndex = startColumnIndex; currentColumnIndex < endColumnIndex; currentColumnIndex++ )
		{
			int rowIndex = cache.getMaxRowIndex( currentColumnIndex );
			SheetData lastData = cache.getColumnLastData( currentColumnIndex );
			rowIndexes[currentColumnIndex - startColumnIndex] = rowIndex;
			int span = lastData != null ? lastData.getRowSpanInDesign( ) : 0;
			if ( span == 0
					|| ( span == 1 && !isInContainer( lastData, rowContainer ) ) )
			{
				maxRowIndex = maxRowIndex > rowIndex ? maxRowIndex : rowIndex;
			}
		}
		int startRowIndex = rowContainer.getRowIndex( );
		if ( maxRowIndex == startRowIndex )
		{
			maxRowIndex++;
		}
		rowContainer.setRowIndex( maxRowIndex );
		float resize = height / ( maxRowIndex - startRowIndex );
		for ( int i = startRowIndex; i < maxRowIndex; i++ )
		{
			cache.setRowHeight( i, resize );
		}

		for ( int currentColumnIndex = startColumnIndex; currentColumnIndex < endColumnIndex; currentColumnIndex++ )
		{
			int rowspan = maxRowIndex - rowIndexes[currentColumnIndex - startColumnIndex];
			SheetData upstair = cache.getColumnLastData( currentColumnIndex );
			if ( rowspan > 0 )
			{
				if ( upstair != null && canSpan( upstair, rowContainer ) )
				{
					Type blankType = Type.VERTICAL;
					if ( upstair.isBlank( ) )
					{
						BlankData blankData = (BlankData) upstair;
						if ( blankData.getType( ) == Type.VERTICAL )
						{
							upstair
									.setRowSpan( upstair.getRowSpan( )
											+ rowspan );
							if ( !isInContainer( blankData, rowContainer ) )
							{
								upstair.decreasRowSpanInDesign( );
							}
						}
						blankType = blankData.getType( );
					}
					else
					{
						upstair.setRowSpan( upstair.getRowSpan( ) + rowspan );
						if ( !isInContainer( upstair, rowContainer ) )
						{
							upstair.decreasRowSpanInDesign( );
						}
					}
					int rowIndex = upstair.getRowIndex( );
					for ( int p = 1; p <= rowspan; p++ )
					{
						BlankData blank = new BlankData( upstair );
						blank.setRowIndex( rowIndex + p );
						blank.setType( blankType );
						cache.addData( currentColumnIndex, blank );
					}
				}
			}
			else if ( upstair.getRowSpanInDesign( ) > 0
					&& !isInContainer( upstair, rowContainer ) )
			{
				upstair.decreasRowSpanInDesign( );
			}
		}
	}

	private void calculateRowHeight( SheetData[] rowData, boolean isAuto )
	{
		float rowHeight = 0;
		int rowIndex = getRowIndex( rowData );
		float lastRowHeight = rowIndex > 0
				? cache.getRowHeight( rowIndex - 1 )
				: 0;
		boolean hasCurrentRowHeight = cache.hasRowHeight( rowIndex );
		if ( !hasCurrentRowHeight || isAuto )
		{
			for ( int i = 0; i < rowData.length; i++ )
			{
				SheetData data = rowData[i];
				if ( data != null )
				{
					if ( data.isBlank( ) )
					{
						// if the data spans last row,then recalculate data
						// height.
						// if current row is the last row of real data, then
						// adjust
						// row height.
						BlankData blankData = (BlankData) data;
						if ( blankData.getType( ) == Type.VERTICAL )
						{
							data.setHeight( data.getHeight( ) - lastRowHeight );
						}
					}
					SheetData realData = getRealData( data );
					if ( realData != null )
					{
						int realDataRowEnd = realData.getRowIndex( )
								+ realData.getRowSpan( );
						if ( realDataRowEnd == data.getRowIndex( ) )
						{
							rowHeight = data.getHeight( ) > rowHeight ? data
									.getHeight( ) : rowHeight;
						}
					}
				}
			}
			cache.setRowHeight( rowIndex, rowHeight );
		}
	}

	private int getRowIndex( SheetData[] rowData )
	{
		for ( int j = 0; j < rowData.length; j++ )
		{
			SheetData data = rowData[j];
			if ( data != null )
			{
				return data.getRowIndex( ) - 1;
			}
		}
		return 0;
	}

	private boolean canSpan( SheetData data, XlsContainer rowContainer )
	{
		SheetData realData = getRealData(data);
		if ( realData == null )
			return false;
		if ( isInContainer( realData, rowContainer ) )
		{
			return true;
		}
		return realData.getRowSpanInDesign( ) > 0;
	}

	private SheetData getRealData( SheetData data )
	{
		while ( data != null && data.isBlank( ) )
		{
			data = ( (BlankData) data ).getData( );
		}
		return data;
	}

	private boolean isInContainer( SheetData data, XlsContainer rowContainer )
	{
		return data.getRowIndex( ) > rowContainer.getStartRowId( );
	}

	public void endTable( )
	{
		if ( !tables.isEmpty( ) )
		{
			tables.pop( );
			endContainer( );
		}
	}

	public void addContainer( IStyle style, HyperlinkDef link )
	{
		XlsContainer parent = getCurrentContainer( );
		ContainerSizeInfo sizeInfo = parent.getSizeInfo( );
		StyleEntry entry = engine.createEntry( sizeInfo, style,
												getParentStyle( parent ) );
		addContainer( new XlsContainer( entry, sizeInfo, parent ) );
	}

	private StyleEntry getParentStyle( XlsContainer parent )
	{
		return parent == null ? null : parent.getStyle( );
	}

	private void addContainer( XlsContainer child )
	{
		XlsContainer parent = child.getParent( );
		if ( parent instanceof XlsCell )
		{
			addEmptyDataToContainer( child, parent );
		}
		if ( parent != null )
		{
			parent.setEmpty( false );
		}
		containers.push( child);
	}

	private void addEmptyDataToContainer( XlsContainer child,
			XlsContainer parent )
	{
		ContainerSizeInfo childSizeInfo = child.getSizeInfo( );
		int childStartCoordinate = childSizeInfo.getStartCoordinate( );
		int childEndCoordinate = childSizeInfo.getEndCoordinate( );
		ContainerSizeInfo parentSizeInfo = parent.getSizeInfo( );
		int parentStartCoordinate = parentSizeInfo.getStartCoordinate( );
		int parentEndCoordinate = parent.getSizeInfo( ).getEndCoordinate( );

		if ( childEndCoordinate < parentEndCoordinate )
		{
			StyleEntry style = parent.getStyle( );
			removeLeftBorder( style );
			removeDiagonalLine( style );
			addEmptyDataToContainer( style, parent, childEndCoordinate,
					parentEndCoordinate - childEndCoordinate );
		}
		if ( childStartCoordinate > parentStartCoordinate )
		{
			StyleEntry style = parent.getStyle( );
			removeRightBorder( style );
			removeDiagonalLine( style );
			addEmptyDataToContainer( style, parent, childStartCoordinate,
					parentStartCoordinate - childStartCoordinate );
		}
	}

	private void addEmptyDataToContainer( StyleEntry style,
			XlsContainer parent, int startCoordinate, int width )
	{
		Data data = createEmptyData( style );
		data.setStartX( startCoordinate );
		data.setEndX( startCoordinate + width );
		addData( data );
	}

	public void endContainer( )
	{
		setParentContainerIndex( );
		endNormalContainer( );
	}

	private void setParentContainerIndex( )
	{
		XlsContainer container = getCurrentContainer( );
		XlsContainer parent = container.getParent( );
		if ( parent != null )
			parent.setRowIndex( container.getRowIndex( ) );
	}

	public void endNormalContainer( )
	{
		XlsContainer container = getCurrentContainer( );
		if ( container.isEmpty( ) )
		{
			Data data = createData( EMPTY, container.getStyle( ) );
			ContainerSizeInfo containerSize = container.getSizeInfo( );
			data.setStartX( containerSize.getStartCoordinate( ) );
			data.setEndX( containerSize.getEndCoordinate( ) );
			addData( data );
		}
		engine.applyContainerBottomStyle( );
		containers.pop( );
	}

	public Data addData( Object txt, IStyle style, HyperlinkDef link,
			BookmarkDef bookmark, float height )
	{
		return addData( txt, style, link, bookmark, null, height );
	}

	public Data addData( Object value, IStyle style, HyperlinkDef link,
			BookmarkDef bookmark, String locale, float height )
	{
		XlsContainer container = getCurrentContainer( );
		ContainerSizeInfo containerSize = container.getSizeInfo( );
		StyleEntry entry = engine.getStyle( style, containerSize,
											getParentStyle( container ) );
		setDataType( entry, value, locale );
		setlinkStyle( entry, link );
		Data data = createData( value, entry );
		data.setHeight( height );
		data.setHyperlinkDef( link );
		data.setBookmark( bookmark );
		data.setStartX( containerSize.getStartCoordinate( ) );
		data.setEndX( containerSize.getEndCoordinate( ) );
		addData( data );
		return data;
	}

	protected void setlinkStyle( StyleEntry entry, HyperlinkDef link )
	{
		if ( link != null )
		{
			entry.setProperty( StyleConstant.COLOR_PROP,
								StyleConstant.HYPERLINK_COLOR );
			entry.setProperty( StyleConstant.TEXT_UNDERLINE_PROP, true );
			entry.setName( StyleEntry.ENTRYNAME_HYPERLINK );
		}
	}

	public void addImageData( IImageContent image, IStyle style,
			HyperlinkDef link, BookmarkDef bookmark )
	{
		XlsContainer container = getCurrentContainer( );
		ContainerSizeInfo parentSizeInfo = container.getSizeInfo( );
		ColumnsInfo imageColumnsInfo = LayoutUtil
				.createImage( image, parentSizeInfo.getWidth( ) );
		splitColumns( imageColumnsInfo, parentSizeInfo );
		ContainerSizeInfo imageSize = new ContainerSizeInfo( parentSizeInfo
				.getStartCoordinate( ), imageColumnsInfo.getTotalWidth( ) );
		StyleEntry entry = engine.getStyle( style, imageSize, parentSizeInfo,
											getParentStyle( container ) );
		setlinkStyle( entry, link );
		SheetData data = createImageData( image, entry, container );
		data.setHyperlinkDef( link );
		data.setBookmark( bookmark );
		data.setStartX( imageSize.getStartCoordinate( ) );
		data.setEndX( imageSize.getEndCoordinate( ) );
		addData( data );
	}

	private SheetData createImageData( IImageContent image, StyleEntry entry, XlsContainer container )
	{
		int type = SheetData.IMAGE;
		entry.setProperty( StyleConstant.DATA_TYPE_PROP, type );
		String uri = image.getURI( );
		String mimeType = image.getMIMEType( );
		String extension = image.getExtension( );
		String altText = image.getAltText( );
		if ( FlashFile.isFlash( mimeType, uri, extension ) )
		{
			if ( null == altText )
			{
				altText = messageFlashObjectNotSupported; 
			}
			entry.setProperty( StyleConstant.DATA_TYPE_PROP, SheetData.STRING );
			return createData( altText, entry );
		}

		try
		{
			Image imageInfo = EmitterUtil.parseImage( image, image
					.getImageSource( ), image.getURI( ), image.getMIMEType( ),
					image.getExtension( ) );

			byte[] data = imageInfo.getData( );
			if ( data != null )
			{
				return createData( image, entry, container, type, imageInfo );
			}
			else
			{
				return createData( image.getAltText( ), entry );
			}
		}
		catch ( IOException e )
		{
			logger.log( Level.WARNING, e.getLocalizedMessage( ) );
			return createData( image.getAltText( ), entry );
		}
	}

	protected SheetData createData( IImageContent image, StyleEntry entry,
			XlsContainer container, int type, Image imageInfo )
	{
		int styleId = engine.getStyleId( entry );
		SheetData imageData = new ImageData( image, styleId, type, imageInfo,
				container );
		return imageData;
	}

	public Data addDateTime( Object txt, IStyle style, HyperlinkDef link,
			BookmarkDef bookmark, String dateTimeLocale, float height )
	{
		XlsContainer currentContainer = getCurrentContainer( );
		ContainerSizeInfo containerSize = currentContainer.getSizeInfo( );
		StyleEntry entry = engine.getStyle( style, containerSize,
											getParentStyle( currentContainer ) );
		setlinkStyle( entry, link );
		Data data = null;
		
		IDataContent dataContent = (IDataContent)txt;
		Object value = dataContent.getValue( );
		Date date = ExcelUtil.getDate( value );
		//If date time is before 1900, it must be output as string, otherwise, excel can't format the date.
		if ( date != null<commit_msg>Fix issue 20415.<commit_after>/*******************************************************************************
 * Copyright (c) 2004, 2008Actuate Corporation.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *  Actuate Corporation  - initial API and implementation
 *******************************************************************************/

package org.eclipse.birt.report.engine.emitter.excel.layout;

import java.io.IOException;
import java.sql.Time;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Stack;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.eclipse.birt.report.engine.content.ICellContent;
import org.eclipse.birt.report.engine.content.IContainerContent;
import org.eclipse.birt.report.engine.content.IDataContent;
import org.eclipse.birt.report.engine.content.IHyperlinkAction;
import org.eclipse.birt.report.engine.content.IImageContent;
import org.eclipse.birt.report.engine.content.IStyle;
import org.eclipse.birt.report.engine.emitter.EmitterUtil;
import org.eclipse.birt.report.engine.emitter.excel.BlankData;
import org.eclipse.birt.report.engine.emitter.excel.BookmarkDef;
import org.eclipse.birt.report.engine.emitter.excel.Data;
import org.eclipse.birt.report.engine.emitter.excel.DataCache;
import org.eclipse.birt.report.engine.emitter.excel.DateTimeUtil;
import org.eclipse.birt.report.engine.emitter.excel.ExcelEmitter;
import org.eclipse.birt.report.engine.emitter.excel.ExcelUtil;
import org.eclipse.birt.report.engine.emitter.excel.HyperlinkDef;
import org.eclipse.birt.report.engine.emitter.excel.ImageData;
import org.eclipse.birt.report.engine.emitter.excel.RowData;
import org.eclipse.birt.report.engine.emitter.excel.SheetData;
import org.eclipse.birt.report.engine.emitter.excel.StyleBuilder;
import org.eclipse.birt.report.engine.emitter.excel.StyleConstant;
import org.eclipse.birt.report.engine.emitter.excel.StyleEngine;
import org.eclipse.birt.report.engine.emitter.excel.StyleEntry;
import org.eclipse.birt.report.engine.emitter.excel.BlankData.Type;
import org.eclipse.birt.report.engine.i18n.EngineResourceHandle;
import org.eclipse.birt.report.engine.i18n.MessageConstants;
import org.eclipse.birt.report.engine.layout.emitter.Image;
import org.eclipse.birt.report.engine.layout.pdf.util.PropertyUtil;
import org.eclipse.birt.report.engine.util.FlashFile;

import com.ibm.icu.util.ULocale;


public class ExcelLayoutEngine
{

	protected static Logger logger = Logger.getLogger( ExcelLayoutEngine.class
			.getName( ) );

	public final static String EMPTY = "";
	
	public static final float DEFAULT_ROW_HEIGHT = 15;

	// Excel 2007 can support 1048576 rows and 16384 columns.
	private int autoBookmarkIndex = 0;

	public static final String AUTO_GENERATED_BOOKMARK = "auto_generated_bookmark_";

	public final static int MAX_ROW_OFFICE2007 = 1048576;
	
	public final static int MAX_COL_OFFICE2007 = 16384;
	
	public final static int MAX_ROW_OFFICE2003 = 65535;
	
	public final static int MAX_COLUMN_OFFICE2003 = 255;
	
	private int maxRow = MAX_ROW_OFFICE2003;

	private int maxCol = MAX_COLUMN_OFFICE2003;
	
	private HashMap<String, String> cachedBookmarks = new HashMap<String, String>( );

	protected DataCache cache;

	private AxisProcessor axis;

	protected StyleEngine engine;
	
	private ExcelEmitter emitter;

	private Stack<XlsContainer> containers = new Stack<XlsContainer>( );

	private Stack<XlsTable> tables = new Stack<XlsTable>( );

	private ExcelContext context = null;

	private String messageFlashObjectNotSupported;
	
	private ULocale locale;

	private HashMap<String, BookmarkDef> bookmarkList = new HashMap<String, BookmarkDef>( );

	public ExcelLayoutEngine( ExcelContext context,
			ExcelEmitter emitter )
	{
		this.context = context;
		this.emitter = emitter;
		this.locale = context.getLocale( );
		EngineResourceHandle resourceHandle = new EngineResourceHandle( locale );
		messageFlashObjectNotSupported = resourceHandle
				.getMessage( MessageConstants.FLASH_OBJECT_NOT_SUPPORTED_PROMPT );
	}
	
	public void initalize( int contentWidth, IStyle style )
	{
		axis = new AxisProcessor( );		
		axis.addCoordinate( contentWidth );

		setCacheSize();
		
		ContainerSizeInfo rule = new ContainerSizeInfo( 0, contentWidth );
		cache = createDataCache( maxCol, maxRow );
		engine = new StyleEngine( this );
		containers.push( createContainer( rule, style, null ) );
	}

	protected DataCache createDataCache( int maxColumn, int maxRow )
	{
		return new DataCache( maxColumn, maxRow );
	}
	
	private void setCacheSize()
	{
		if ( context.getOfficeVersion( ).equals( "office2007" ) )
		{
			maxCol = MAX_COL_OFFICE2007;
		}
	}

	public XlsContainer getCurrentContainer( )
	{
		return (XlsContainer) containers.peek( );
	}

	public Stack<XlsContainer> getContainers( )
	{
		return containers;
	}

	public void setPageStyle( IStyle style )
	{
		XlsContainer topContainer = containers.peek( );
		topContainer.setStyle( StyleBuilder.createStyleEntry( style ) );
	}
	
	public void addTable( IContainerContent content, ColumnsInfo table,
			ContainerSizeInfo size )
	{
		IStyle style = content.getComputedStyle( );
		XlsContainer currentContainer = getCurrentContainer( );
		ContainerSizeInfo parentSizeInfo = currentContainer.getSizeInfo( );
		int[] columnStartCoordinates = splitColumns( table, parentSizeInfo );
		createTable( table, style, currentContainer, columnStartCoordinates );
	}

	protected int[] splitColumns( ColumnsInfo columnsInfo,
			ContainerSizeInfo parentSizeInfo )
	{
		int startCoordinate = parentSizeInfo.getStartCoordinate( );
		int endCoordinate = parentSizeInfo.getEndCoordinate( );

		int[] columnStartCoordinates = calculateColumnCoordinates( columnsInfo,
				startCoordinate, endCoordinate );

		splitColumns( startCoordinate, endCoordinate, columnStartCoordinates );
		return columnStartCoordinates;
	}

	private void createTable( ColumnsInfo tableInfo, IStyle style,
			XlsContainer currentContainer, int[] columnStartCoordinates )
	{
		int leftCordinate = columnStartCoordinates[0];
		int width = columnStartCoordinates[columnStartCoordinates.length - 1]
				- leftCordinate;
		ContainerSizeInfo sizeInfo = new ContainerSizeInfo( leftCordinate,
				width );
		StyleEntry styleEntry = engine.createEntry( sizeInfo, style,
													getParentStyle( ) );
		XlsTable table = new XlsTable( tableInfo, styleEntry,
				sizeInfo, currentContainer );
		tables.push( table );
		addContainer( table );
	}

	protected StyleEntry getParentStyle( )
	{
		return getParentStyle( getCurrentContainer( ) );
	}

	private void splitColumns( int startCoordinate, int endCoordinate,
			int[] columnStartCoordinates )
	{
		int[] scale = axis.getColumnCoordinatesInRange( startCoordinate,
				endCoordinate );

		for ( int i = 0; i < scale.length - 1; i++ )
		{
			int startPosition = scale[i];
			int endPostion = scale[i + 1];

			int[] range = inRange( startPosition, endPostion, columnStartCoordinates );

			if ( range.length > 0 )
			{				
				int pos = axis.getColumnIndexByCoordinate( startPosition );
				cache.insertColumns( pos, range.length );

				for ( int j = 0; j < range.length; j++ )
				{
					axis.addCoordinate( range[j] );
				}
			}
		}
	}

	private int[] calculateColumnCoordinates( ColumnsInfo table,
			int startCoordinate,
			int endCoordinate )
	{
		int columnCount = table.getColumnCount( );
		int[] columnStartCoordinates = new int[ columnCount + 1 ];
		columnStartCoordinates[0] = startCoordinate;
		for ( int i = 1; i <= columnCount; i++ )
		{
			if ( ( columnStartCoordinates[i - 1] + table.getColumnWidth( i - 1 ) ) > endCoordinate )
			{
				columnStartCoordinates[i] = endCoordinate;
			}
			else
				columnStartCoordinates[i] = columnStartCoordinates[i - 1]
				+ table.getColumnWidth( i - 1 );
		}
		return columnStartCoordinates;
	}

	private int[] inRange( int start, int end, int[] data )
	{
		int[] range = new int[data.length];
		int count = 0;

		for ( int i = 0; i < data.length; i++ )
		{
			if ( ( data[i] > start ) && ( data[i] < end ) )
			{
				count++;
				range[count] = data[i];
			}
		}

		int[] result = new int[count];

		int j = 0;
		for ( int i = 0; i < range.length; i++ )
		{
			if ( range[i] != 0 )
			{
				result[j] = range[i];
				j++;
			}
		}

		return result;
	}

	public void addCell( int col, int colSpan, int rowSpan, IStyle style )
	{
		XlsTable table = tables.peek( );
		ContainerSizeInfo cellSizeInfo = table.getColumnSizeInfo( col, colSpan );
		XlsCell cell = new XlsCell( engine.createEntry( cellSizeInfo, style,
														getParentStyle( ) ),
				cellSizeInfo, getCurrentContainer( ), rowSpan );
		addContainer( cell );
	}

	public void addCell( ICellContent cellcontent, int col, int colSpan,
			int rowSpan, IStyle style )
	{
		XlsTable table = tables.peek( );
		ContainerSizeInfo cellSizeInfo = table.getColumnSizeInfo( col, colSpan );
		int diagonalNumber = cellcontent.getDiagonalNumber( );
		StyleEntry cellStyleEntry = null;
		if ( diagonalNumber != 0 )
		{
			String diagonalColor = cellcontent.getDiagonalColor( );
			String diagonalStyle = cellcontent.getDiagonalStyle( );
			int diagonalWidth = PropertyUtil.getDimensionValue( cellcontent,
					cellcontent.getDiagonalWidth( ), cellSizeInfo.getWidth( ) );
			cellStyleEntry = engine.createCellEntry( cellSizeInfo, style,
														diagonalColor,
														diagonalStyle,
														diagonalWidth,
														getParentStyle( ) );
		}
		else
		{
			cellStyleEntry = engine.createEntry( cellSizeInfo, style,
													getParentStyle( ) );
		}
		XlsCell cell = new XlsCell( cellStyleEntry, cellSizeInfo,
				getCurrentContainer( ), rowSpan );
		addContainer( cell );
	}

	public void endCell( )
	{
		endNormalContainer( );
	}

	public void addRow( IStyle style )
	{
		XlsContainer parent = getCurrentContainer( );
		ContainerSizeInfo sizeInfo = parent.getSizeInfo( );
		XlsContainer container = createContainer( sizeInfo, style,
				parent );
		container.setEmpty( false );
		addContainer( container );
	}

	public void endRow( float rowHeight )
	{
		synchronize( rowHeight );
		endContainer( );
	}

	private void synchronize( float height )
	{
		XlsContainer rowContainer = getCurrentContainer( );
		ContainerSizeInfo rowSizeInfo = rowContainer.getSizeInfo( );
		int startCoordinate = rowSizeInfo.getStartCoordinate( );
		int endCoordinate = rowSizeInfo.getEndCoordinate( );
		int startColumnIndex = axis.getColumnIndexByCoordinate( startCoordinate );
		int endColumnIndex = axis.getColumnIndexByCoordinate( endCoordinate );

		int maxRowIndex = 0;
		int rowIndexes[] = new int[endColumnIndex - startColumnIndex];

		for ( int currentColumnIndex = startColumnIndex; currentColumnIndex < endColumnIndex; currentColumnIndex++ )
		{
			int rowIndex = cache.getMaxRowIndex( currentColumnIndex );
			SheetData lastData = cache.getColumnLastData( currentColumnIndex );
			rowIndexes[currentColumnIndex - startColumnIndex] = rowIndex;
			int span = lastData != null ? lastData.getRowSpanInDesign( ) : 0;
			if ( span == 0
					|| ( span == 1 && !isInContainer( lastData, rowContainer ) ) )
			{
				maxRowIndex = maxRowIndex > rowIndex ? maxRowIndex : rowIndex;
			}
		}
		int startRowIndex = rowContainer.getRowIndex( );
		if ( maxRowIndex == startRowIndex )
		{
			maxRowIndex++;
		}
		rowContainer.setRowIndex( maxRowIndex );
		float resize = height / ( maxRowIndex - startRowIndex );
		for ( int i = startRowIndex; i < maxRowIndex; i++ )
		{
			cache.setRowHeight( i, resize );
		}

		for ( int currentColumnIndex = startColumnIndex; currentColumnIndex < endColumnIndex; currentColumnIndex++ )
		{
			int rowspan = maxRowIndex - rowIndexes[currentColumnIndex - startColumnIndex];
			SheetData upstair = cache.getColumnLastData( currentColumnIndex );
			if ( rowspan > 0 )
			{
				if ( upstair != null && canSpan( upstair, rowContainer ) )
				{
					Type blankType = Type.VERTICAL;
					if ( upstair.isBlank( ) )
					{
						BlankData blankData = (BlankData) upstair;
						if ( blankData.getType( ) == Type.VERTICAL )
						{
							upstair
									.setRowSpan( upstair.getRowSpan( )
											+ rowspan );
							if ( !isInContainer( blankData, rowContainer ) )
							{
								upstair.decreasRowSpanInDesign( );
							}
						}
						blankType = blankData.getType( );
					}
					else
					{
						upstair.setRowSpan( upstair.getRowSpan( ) + rowspan );
						if ( !isInContainer( upstair, rowContainer ) )
						{
							upstair.decreasRowSpanInDesign( );
						}
					}
					int rowIndex = upstair.getRowIndex( );
					for ( int p = 1; p <= rowspan; p++ )
					{
						BlankData blank = new BlankData( upstair );
						blank.setRowIndex( rowIndex + p );
						blank.setType( blankType );
						cache.addData( currentColumnIndex, blank );
					}
				}
			}
			else if ( upstair.getRowSpanInDesign( ) > 0
					&& !isInContainer( upstair, rowContainer ) )
			{
				upstair.decreasRowSpanInDesign( );
			}
		}
	}

	private void calculateRowHeight( SheetData[] rowData, boolean isAuto )
	{
		float rowHeight = 0;
		int rowIndex = getRowIndex( rowData );
		float lastRowHeight = rowIndex > 0
				? cache.getRowHeight( rowIndex - 1 )
				: 0;
		boolean hasCurrentRowHeight = cache.hasRowHeight( rowIndex );
		if ( !hasCurrentRowHeight || isAuto )
		{
			for ( int i = 0; i < rowData.length; i++ )
			{
				SheetData data = rowData[i];
				if ( data != null )
				{
					if ( data.isBlank( ) )
					{
						// if the data spans last row,then recalculate data
						// height.
						// if current row is the last row of real data, then
						// adjust
						// row height.
						BlankData blankData = (BlankData) data;
						if ( blankData.getType( ) == Type.VERTICAL )
						{
							data.setHeight( data.getHeight( ) - lastRowHeight );
						}
					}
					SheetData realData = getRealData( data );
					if ( realData != null )
					{
						int realDataRowEnd = realData.getRowIndex( )
								+ realData.getRowSpan( );
						if ( realDataRowEnd == data.getRowIndex( ) )
						{
							rowHeight = data.getHeight( ) > rowHeight ? data
									.getHeight( ) : rowHeight;
						}
					}
				}
			}
			cache.setRowHeight( rowIndex, rowHeight );
		}
	}

	private int getRowIndex( SheetData[] rowData )
	{
		for ( int j = 0; j < rowData.length; j++ )
		{
			SheetData data = rowData[j];
			if ( data != null )
			{
				return data.getRowIndex( ) - 1;
			}
		}
		return 0;
	}

	private boolean canSpan( SheetData data, XlsContainer rowContainer )
	{
		SheetData realData = getRealData(data);
		if ( realData == null )
			return false;
		if ( isInContainer( realData, rowContainer ) )
		{
			return true;
		}
		return realData.getRowSpanInDesign( ) > 0;
	}

	private SheetData getRealData( SheetData data )
	{
		while ( data != null && data.isBlank( ) )
		{
			data = ( (BlankData) data ).getData( );
		}
		return data;
	}

	private boolean isInContainer( SheetData data, XlsContainer rowContainer )
	{
		return data.getRowIndex( ) > rowContainer.getStartRowId( );
	}

	public void endTable( )
	{
		if ( !tables.isEmpty( ) )
		{
			tables.pop( );
			endContainer( );
		}
	}

	public void addContainer( IStyle style, HyperlinkDef link )
	{
		XlsContainer parent = getCurrentContainer( );
		ContainerSizeInfo sizeInfo = parent.getSizeInfo( );
		StyleEntry entry = engine.createEntry( sizeInfo, style,
												getParentStyle( parent ) );
		addContainer( new XlsContainer( entry, sizeInfo, parent ) );
	}

	private StyleEntry getParentStyle( XlsContainer parent )
	{
		return parent == null ? null : parent.getStyle( );
	}

	private void addContainer( XlsContainer child )
	{
		XlsContainer parent = child.getParent( );
		if ( parent instanceof XlsCell )
		{
			addEmptyDataToContainer( child, parent );
		}
		if ( parent != null )
		{
			parent.setEmpty( false );
		}
		containers.push( child);
	}

	private void addEmptyDataToContainer( XlsContainer child,
			XlsContainer parent )
	{
		ContainerSizeInfo childSizeInfo = child.getSizeInfo( );
		int childStartCoordinate = childSizeInfo.getStartCoordinate( );
		int childEndCoordinate = childSizeInfo.getEndCoordinate( );
		ContainerSizeInfo parentSizeInfo = parent.getSizeInfo( );
		int parentStartCoordinate = parentSizeInfo.getStartCoordinate( );
		int parentEndCoordinate = parent.getSizeInfo( ).getEndCoordinate( );

		if ( childEndCoordinate < parentEndCoordinate )
		{
			StyleEntry style = parent.getStyle( );
			removeLeftBorder( style );
			removeDiagonalLine( style );
			addEmptyDataToContainer( style, parent, childEndCoordinate,
					parentEndCoordinate - childEndCoordinate );
		}
		if ( childStartCoordinate > parentStartCoordinate )
		{
			StyleEntry style = parent.getStyle( );
			removeRightBorder( style );
			removeDiagonalLine( style );
			addEmptyDataToContainer( style, parent, childStartCoordinate,
					parentStartCoordinate - childStartCoordinate );
		}
	}

	private void addEmptyDataToContainer( StyleEntry style,
			XlsContainer parent, int startCoordinate, int width )
	{
		Data data = createEmptyData( style );
		data.setStartX( startCoordinate );
		data.setEndX( startCoordinate + width );
		addData( data );
	}

	public void endContainer( )
	{
		setParentContainerIndex( );
		endNormalContainer( );
	}

	private void setParentContainerIndex( )
	{
		XlsContainer container = getCurrentContainer( );
		XlsContainer parent = container.getParent( );
		if ( parent != null )
			parent.setRowIndex( container.getRowIndex( ) );
	}

	public void endNormalContainer( )
	{
		XlsContainer container = getCurrentContainer( );
		if ( container.isEmpty( ) )
		{
			Data data = createData( EMPTY, container.getStyle( ) );
			ContainerSizeInfo containerSize = container.getSizeInfo( );
			data.setStartX( containerSize.getStartCoordinate( ) );
			data.setEndX( containerSize.getEndCoordinate( ) );
			addData( data );
		}
		engine.applyContainerBottomStyle( );
		containers.pop( );
	}

	public Data addData( Object txt, IStyle style, HyperlinkDef link,
			BookmarkDef bookmark, float height )
	{
		return addData( txt, style, link, bookmark, null, height );
	}

	public Data addData( Object value, IStyle style, HyperlinkDef link,
			BookmarkDef bookmark, String locale, float height )
	{
		XlsContainer container = getCurrentContainer( );
		ContainerSizeInfo containerSize = container.getSizeInfo( );
		StyleEntry entry = engine.getStyle( style, containerSize,
											getParentStyle( container ) );
		setDataType( entry, value, locale );
		setlinkStyle( entry, link );
		Data data = createData( value, entry );
		data.setHeight( height );
		data.setHyperlinkDef( link );
		data.setBookmark( bookmark );
		data.setStartX( containerSize.getStartCoordinate( ) );
		data.setEndX( containerSize.getEndCoordinate( ) );
		addData( data );
		return data;
	}

	protected void setlinkStyle( StyleEntry entry, HyperlinkDef link )
	{
		if ( link != null )
		{
			entry.setProperty( StyleConstant.COLOR_PROP,
								StyleConstant.HYPERLINK_COLOR );
			entry.setProperty( StyleConstant.TEXT_UNDERLINE_PROP, true );
			entry.setName( StyleEntry.ENTRYNAME_HYPERLINK );
		}
	}

	public void addImageData( IImageContent image, IStyle style,
			HyperlinkDef link, BookmarkDef bookmark )
	{
		XlsContainer container = getCurrentContainer( );
		ContainerSizeInfo parentSizeInfo = container.getSizeInfo( );
		ColumnsInfo imageColumnsInfo = LayoutUtil
				.createImage( image, parentSizeInfo.getWidth( ) );
		splitColumns( imageColumnsInfo, parentSizeInfo );
		ContainerSizeInfo imageSize = new ContainerSizeInfo( parentSizeInfo
				.getStartCoordinate( ), imageColumnsInfo.getTotalWidth( ) );
		StyleEntry entry = engine.getStyle( style, imageSize, parentSizeInfo,
											getParentStyle( container ) );
		setlinkStyle( entry, link );
		SheetData data = createImageData( image, entry, container );
		data.setHyperlinkDef( link );
		data.setBookmark( bookmark );
		data.setStartX( imageSize.getStartCoordinate( ) );
		data.setEndX( imageSize.getEndCoordinate( ) );
		addData( data );
	}

	private SheetData createImageData( IImageContent image, StyleEntry entry, XlsContainer container )
	{
		int type = SheetData.IMAGE;
		entry.setProperty( StyleConstant.DATA_TYPE_PROP, type );
		String uri = image.getURI( );
		String mimeType = image.getMIMEType( );
		String extension = image.getExtension( );
		String altText = image.getAltText( );
		if ( FlashFile.isFlash( mimeType, uri, extension ) )
		{
			if ( null == altText )
			{
				altText = messageFlashObjectNotSupported; 
			}
			entry.setProperty( StyleConstant.DATA_TYPE_PROP, SheetData.STRING );
			return createData( altText, entry );
		}

		try
		{
			Image imageInfo = EmitterUtil.parseImage( image, image
					.getImageSource( ), image.getURI( ), image.getMIMEType( ),
					image.getExtension( ) );

			byte[] data = imageInfo.getData( );
			if ( data != null )
			{
				return createData( image, entry, container, type, imageInfo );
			}
			else
			{
				entry.setProperty( StyleConstant.DATA_TYPE_PROP,
						SheetData.STRING );
				return createData( image.getAltText( ), entry );
			}
		}
		catch ( IOException e )
		{
			logger.log( Level.WARNING, e.getLocalizedMessage( ) );
			entry.setProperty( StyleConstant.DATA_TYPE_PROP, SheetData.STRING );
			return createData( image.getAltText( ), entry );
		}
	}

	protected SheetData createData( IImageContent image, StyleEntry entry,
			XlsContainer container, int type, Image imageInfo )
	{
		int styleId = engine.getStyleId( entry );
		SheetData imageData = new ImageData( image, styleId, type, imageInfo,
				container );
		return imageData;
	}

	public Data addDateTime( Object txt, IStyle style, HyperlinkDef link,
			BookmarkDef bookmark, String dateTimeLocale, float height )
	{
		XlsContainer currentContainer = getCurrentContainer( );
		ContainerSizeInfo containerSize = currentContainer.getSizeInfo( );
		StyleEntry entry = engine.getStyle( style, containerSize,
											getParentStyle( currentContainer ) );
		setlinkStyle( entry, link );
		Data data = null;
		
		IDataContent dataContent = (IDataContent)txt;
		Object value = dataContent.getValue( );
		Date date = ExcelUtil.getDate( value );
		//If date time is before 1900, it must be output as string, otherwise, excel can't format the date.
		if ( date != null
{'fraction_non_alphanumeric': 0.10861899660774861, 'fraction_numerical': 0.00582485270487413, 'mean_word_length': 3.8332434473088126, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5458107', 'n_tokens_mistral': 16229, 'n_tokens_neox': 13673, 'n_words': 3195}