<commit_before>/**
 * Copyright (c) 2014 by <NAME>.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 * 
 * Any modifications to this file must keep this entire header intact.
 */

package cafe.image.reader;

import java.awt.Transparency;
import java.awt.color.ColorSpace;
import java.awt.image.BufferedImage;
import java.awt.image.ColorModel;
import java.awt.image.ComponentColorModel;
import java.awt.image.DataBuffer;
import java.awt.image.DataBufferByte;
import java.awt.image.IndexColorModel;
import java.awt.image.Raster;
import java.awt.image.WritableRaster;
import java.io.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

import cafe.image.compression.deflate.DeflateDecoder;
import cafe.image.compression.lzw.LZWTreeDecoder;
import cafe.image.tiff.ASCIIField;
import cafe.image.tiff.ByteField;
import cafe.image.tiff.IFD;
import cafe.image.tiff.LongField;
import cafe.image.tiff.RationalField;
import cafe.image.tiff.ShortField;
import cafe.image.tiff.Tag;
import cafe.image.tiff.TiffField;
import cafe.image.tiff.TiffFieldEnum;
import cafe.image.tiff.TiffTag;
import cafe.image.tiff.FieldType;
import cafe.image.tiff.UndefinedField;
import cafe.io.FileCacheRandomAccessInputStream;
import cafe.io.IOUtils;
import cafe.io.RandomAccessInputStream;
import cafe.io.ReadStrategyII;
import cafe.io.ReadStrategyMM;
import cafe.string.StringUtils;

/** 
 * Decodes and shows TIFF images. 
 * 
 * @author <NAME>, <EMAIL>
 * @version 1.0 03/09/2012  
 */
public class TIFFReader extends ImageReader {
	private RandomAccessInputStream randIS = null;
	private List<IFD> list = new ArrayList<IFD>();
	 
	public BufferedImage read(InputStream is) throws Exception
	{		  
		randIS = new FileCacheRandomAccessInputStream(is);
		if(!readHeader(randIS)) return null;
		 
		int offset = randIS.readInt();
		  	
		int ifd = 0;
				
		while (offset != 0)
		{
			offset = readIFD(ifd++, offset);
		}
		BufferedImage bi =  decode(list.get(0));
		randIS.close();
		
		return bi;
	}
	 
	private BufferedImage decode(IFD ifd) throws Exception {
		TiffField<?> field = ifd.getField(TiffTag.COMPRESSION.getValue());
		short[] data = (short[])field.getData();
		TiffFieldEnum.Compression compression = TiffFieldEnum.Compression.fromValue(data[0]&0xffff);
		System.out.println("Compression type: " + compression.getDescription());
		// Forget about tiled TIFF for now
		TiffField<?> f_stripOffsets = ifd.getField(TiffTag.STRIP_OFFSETS.getValue());
		TiffField<?> f_stripByteCounts = ifd.getField(TiffTag.STRIP_BYTE_COUNTS.getValue());
		TiffField<?> f_rowsPerStrip = ifd.getField(TiffTag.ROWS_PER_STRIP.getValue());
		int[] stripOffsets = f_stripOffsets.getDataAsLong();
		int[] stripByteCounts = f_stripByteCounts.getDataAsLong();
		System.out.println(Arrays.toString(stripOffsets));
		System.out.println(Arrays.toString(stripByteCounts));
		int rowsPerStrip = -1;
		if(f_rowsPerStrip != null)
			rowsPerStrip = f_rowsPerStrip.getDataAsLong()[0];
		System.out.println(rowsPerStrip);
		int imageWidth = ifd.getField(TiffTag.IMAGE_WIDTH.getValue()).getDataAsLong()[0];
		int imageHeight = ifd.getField(TiffTag.IMAGE_LENGTH.getValue()).getDataAsLong()[0];
		System.out.println("Image width: " + imageWidth);
		System.out.println("Image height: " + imageHeight);
		TiffField<?> f_photoMetric = ifd.getField(TiffTag.PHOTOMETRIC_INTERPRETATION.getValue());
		int photoMetric = f_photoMetric.getDataAsLong()[0];
		TiffFieldEnum.PhotoMetric e_photoMetric = TiffFieldEnum.PhotoMetric.fromValue(photoMetric);
		System.out.println("PhotoMetric: " + e_photoMetric);
		TiffField<?> f_bitsPerSample = ifd.getField(TiffTag.BITS_PER_SAMPLE.getValue());
		int bitsPerSample = f_bitsPerSample.getDataAsLong()[0];
		System.out.println("Bits per sample: " + bitsPerSample);
		
		if(rowsPerStrip < 0) rowsPerStrip = imageHeight;
		int rowsRemain = imageHeight;
		int offset = 0;
		
		switch(e_photoMetric) {
			case PALETTE_COLOR:
				short[] colorMap = (short[])ifd.getField(TiffTag.COLORMAP.getValue()).getData();
				rgbColorPalette = new int[colorMap.length/3];
				int numOfColors = (1<<bitsPerSample);
				int numOfColors2 = (numOfColors<<1);
				for(int i = 0, index = 0; i < colorMap.length/3;i++) {
					rgbColorPalette[index++] = 0xff000000|((colorMap[i]&0xff00)<<8)|((colorMap[i+numOfColors]&0xff00))|((colorMap[i+numOfColors2]&0xff00)>>8) ;
				}				
				byte[] pixels = new byte[imageWidth*imageHeight];
				
				if(compression == TiffFieldEnum.Compression.LZW) {
					LZWTreeDecoder decoder = new LZWTreeDecoder(8, true);					
					for(int i = 0; i < stripByteCounts.length; i++) {
						byte[] temp = new byte[stripByteCounts[i]];
						randIS.seek(stripOffsets[i]);
						randIS.readFully(temp);
						decoder.setInput(temp);
						int numOfBytes = decoder.decode(pixels, offset, Math.min(rowsPerStrip,rowsRemain)*imageWidth);
						offset += numOfBytes;
						rowsRemain -= Math.min(rowsPerStrip, rowsRemain);
					}
				}  else if(compression == TiffFieldEnum.Compression.DEFLATE || compression == TiffFieldEnum.Compression.DEFLATE_ADOBE) {
					DeflateDecoder decoder = new DeflateDecoder();
					for(int i = 0; i < stripByteCounts.length; i++) {
						byte[] temp = new byte[stripByteCounts[i]];
						randIS.seek(stripOffsets[i]);
						randIS.readFully(temp);
						decoder.setInput(temp);
						int numOfBytes = decoder.decode(pixels, offset, Math.min(rowsPerStrip,rowsRemain)*imageWidth);
						offset += numOfBytes;
						rowsRemain -= Math.min(rowsPerStrip, rowsRemain);
					}
				}
				//Create a BufferedImage
				int[] off = {0};//band offset, we have only one band start at 0
				DataBuffer db = new DataBufferByte(pixels, pixels.length);
				WritableRaster raster = Raster.createInterleavedRaster(db, imageWidth, imageHeight, imageWidth, 1, off, null);
				ColorModel cm = new IndexColorModel(bitsPerSample, rgbColorPalette.length, rgbColorPalette, 0, false, -1, DataBuffer.TYPE_BYTE);
		   	
				return new BufferedImage(cm, raster, false, null);
			case RGB:
				pixels = new byte[imageWidth*imageHeight*3];				
				if(compression == TiffFieldEnum.Compression.NONE) {
					for(int i = 0; i < stripByteCounts.length; i++) {					
						randIS.seek(stripOffsets[i]);
						randIS.readFully(pixels, offset, Math.min(rowsPerStrip, rowsRemain)*imageWidth*3);
						offset += Math.min(rowsPerStrip, rowsRemain)*imageWidth*3;
						rowsRemain -= Math.min(rowsPerStrip, rowsRemain);					
					}
				} else if(compression == TiffFieldEnum.Compression.LZW) {
					LZWTreeDecoder decoder = new LZWTreeDecoder(8, true);
					for(int i = 0; i < stripByteCounts.length; i++) {
						byte[] temp = new byte[stripByteCounts[i]];
						randIS.seek(stripOffsets[i]);
						randIS.readFully(temp);
						decoder.setInput(temp);
						int bytes2Read = Math.min(rowsPerStrip, rowsRemain)*imageWidth*3;
						int numOfBytes = decoder.decode(pixels, offset, bytes2Read);							
						offset += numOfBytes;
						rowsRemain -= Math.min(rowsPerStrip, rowsRemain);					
					}					
				} else if(compression == TiffFieldEnum.Compression.DEFLATE || compression == TiffFieldEnum.Compression.DEFLATE_ADOBE) {
					DeflateDecoder decoder = new DeflateDecoder();
					for(int i = 0; i < stripByteCounts.length; i++) {
						byte[] temp = new byte[stripByteCounts[i]];
						randIS.seek(stripOffsets[i]);
						randIS.readFully(temp);
						decoder.setInput(temp);
						int bytes2Read = Math.min(rowsPerStrip, rowsRemain)*imageWidth*3;
						int numOfBytes = decoder.decode(pixels, offset, bytes2Read);							
						offset += numOfBytes;
						rowsRemain -= Math.min(rowsPerStrip, rowsRemain);					
					}					
				}
				//Create a BufferedImage
				 db = new DataBufferByte(pixels, pixels.length);
				 int[] bandoff = {0, 1, 2}; //band offset, we have 3 bands
				 int numOfBands = 3;
				 int trans = Transparency.OPAQUE;
				 int[] nBits = {8, 8, 8};						
				 raster = Raster.createInterleavedRaster(db, imageWidth, imageHeight, imageWidth*numOfBands, numOfBands, bandoff, null);
				 cm = new ComponentColorModel(ColorSpace.getInstance(ColorSpace.CS_sRGB), nBits, false, false,
			                trans, DataBuffer.TYPE_BYTE);
					
		 	   	 return new BufferedImage(cm, raster, false, null);
						
			default:
		 		break;
		}	
		
		return null;
	}
	 
	private boolean readHeader(RandomAccessInputStream randIS) throws IOException {
		// First 2 bytes determine the byte order of the file
		short endian = randIS.readShort();
		
		if(endian == IOUtils.BIG_ENDIAN)
		{
			System.out.println("Byte order: Motorola BIG_ENDIAN");
			this.randIS.setReadStrategy(ReadStrategyMM.getInstance());
		} else if(endian == IOUtils.LITTLE_ENDIAN) {
			System.out.println("Byte order: Intel LITTLE_ENDIAN");
			this.randIS.setReadStrategy(ReadStrategyII.getInstance());
		} else {
			System.out.println("Warning: invalid TIFF byte order!");
			return false;
		} 
		
		// Read TIFF identifier
		short tiff_id = randIS.readShort();
		  
		if(tiff_id!=0x2a)//"*" 42 decimal<commit_msg>Non-8bit support for palette color and packbits support<commit_after>/**
 * Copyright (c) 2014 by <NAME>.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 * 
 * Any modifications to this file must keep this entire header intact.
 */

package cafe.image.reader;

import java.awt.Transparency;
import java.awt.color.ColorSpace;
import java.awt.image.BufferedImage;
import java.awt.image.ColorModel;
import java.awt.image.ComponentColorModel;
import java.awt.image.DataBuffer;
import java.awt.image.DataBufferByte;
import java.awt.image.IndexColorModel;
import java.awt.image.Raster;
import java.awt.image.WritableRaster;
import java.io.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

import cafe.image.compression.deflate.DeflateDecoder;
import cafe.image.compression.lzw.LZWTreeDecoder;
import cafe.image.compression.packbits.Packbits;
import cafe.image.tiff.ASCIIField;
import cafe.image.tiff.ByteField;
import cafe.image.tiff.IFD;
import cafe.image.tiff.LongField;
import cafe.image.tiff.RationalField;
import cafe.image.tiff.ShortField;
import cafe.image.tiff.Tag;
import cafe.image.tiff.TiffField;
import cafe.image.tiff.TiffFieldEnum;
import cafe.image.tiff.TiffTag;
import cafe.image.tiff.FieldType;
import cafe.image.tiff.UndefinedField;
import cafe.io.FileCacheRandomAccessInputStream;
import cafe.io.IOUtils;
import cafe.io.RandomAccessInputStream;
import cafe.io.ReadStrategyII;
import cafe.io.ReadStrategyMM;
import cafe.string.StringUtils;

/** 
 * Decodes and shows TIFF images. 
 * 
 * @author <NAME>, <EMAIL>
 * @version 1.0 03/09/2012  
 */
public class TIFFReader extends ImageReader {
	private RandomAccessInputStream randIS = null;
	private List<IFD> list = new ArrayList<IFD>();
	 
	public BufferedImage read(InputStream is) throws Exception
	{		  
		randIS = new FileCacheRandomAccessInputStream(is);
		if(!readHeader(randIS)) return null;
		 
		int offset = randIS.readInt();
		  	
		int ifd = 0;
				
		while (offset != 0)
		{
			offset = readIFD(ifd++, offset);
		}
		BufferedImage bi =  decode(list.get(0));
		randIS.close();
		
		return bi;
	}
	 
	private BufferedImage decode(IFD ifd) throws Exception {
		TiffField<?> field = ifd.getField(TiffTag.COMPRESSION.getValue());
		short[] data = (short[])field.getData();
		TiffFieldEnum.Compression compression = TiffFieldEnum.Compression.fromValue(data[0]&0xffff);
		System.out.println("Compression type: " + compression.getDescription());
		// Forget about tiled TIFF for now
		TiffField<?> f_stripOffsets = ifd.getField(TiffTag.STRIP_OFFSETS.getValue());
		TiffField<?> f_stripByteCounts = ifd.getField(TiffTag.STRIP_BYTE_COUNTS.getValue());
		TiffField<?> f_rowsPerStrip = ifd.getField(TiffTag.ROWS_PER_STRIP.getValue());
		int[] stripOffsets = f_stripOffsets.getDataAsLong();
		int[] stripByteCounts = f_stripByteCounts.getDataAsLong();
		System.out.println(Arrays.toString(stripOffsets));
		System.out.println(Arrays.toString(stripByteCounts));
		int rowsPerStrip = -1;
		if(f_rowsPerStrip != null)
			rowsPerStrip = f_rowsPerStrip.getDataAsLong()[0];
		System.out.println(rowsPerStrip);
		int imageWidth = ifd.getField(TiffTag.IMAGE_WIDTH.getValue()).getDataAsLong()[0];
		int imageHeight = ifd.getField(TiffTag.IMAGE_LENGTH.getValue()).getDataAsLong()[0];
		System.out.println("Image width: " + imageWidth);
		System.out.println("Image height: " + imageHeight);
		TiffField<?> f_photoMetric = ifd.getField(TiffTag.PHOTOMETRIC_INTERPRETATION.getValue());
		int photoMetric = f_photoMetric.getDataAsLong()[0];
		TiffFieldEnum.PhotoMetric e_photoMetric = TiffFieldEnum.PhotoMetric.fromValue(photoMetric);
		System.out.println("PhotoMetric: " + e_photoMetric);
		TiffField<?> f_bitsPerSample = ifd.getField(TiffTag.BITS_PER_SAMPLE.getValue());
		int bitsPerSample = f_bitsPerSample.getDataAsLong()[0];
		System.out.println("Bits per sample: " + bitsPerSample);
		
		if(rowsPerStrip < 0) rowsPerStrip = imageHeight;
		int rowsRemain = imageHeight;
		int offset = 0;
		
		switch(e_photoMetric) {
			case PALETTE_COLOR:
				short[] colorMap = (short[])ifd.getField(TiffTag.COLORMAP.getValue()).getData();
				rgbColorPalette = new int[colorMap.length/3];
				int numOfColors = (1<<bitsPerSample);
				int numOfColors2 = (numOfColors<<1);
				for(int i = 0, index = 0; i < colorMap.length/3;i++) {
					rgbColorPalette[index++] = 0xff000000|((colorMap[i]&0xff00)<<8)|((colorMap[i+numOfColors]&0xff00))|((colorMap[i+numOfColors2]&0xff00)>>8) ;
				}
				int bytesPerScanLine = (imageWidth*bitsPerSample +7)/8;
				byte[] pixels = new byte[bytesPerScanLine*imageHeight];				
				if(compression == TiffFieldEnum.Compression.LZW) {
					LZWTreeDecoder decoder = new LZWTreeDecoder(8, true);					
					for(int i = 0; i < stripByteCounts.length; i++) {
						byte[] temp = new byte[stripByteCounts[i]];
						randIS.seek(stripOffsets[i]);
						randIS.readFully(temp);
						decoder.setInput(temp);
						int numOfBytes = decoder.decode(pixels, offset, Math.min(rowsPerStrip,rowsRemain)*bytesPerScanLine);
						offset += numOfBytes;
						rowsRemain -= Math.min(rowsPerStrip, rowsRemain);
					}
				}  else if(compression == TiffFieldEnum.Compression.DEFLATE || compression == TiffFieldEnum.Compression.DEFLATE_ADOBE) {
					DeflateDecoder decoder = new DeflateDecoder();
					for(int i = 0; i < stripByteCounts.length; i++) {
						byte[] temp = new byte[stripByteCounts[i]];
						randIS.seek(stripOffsets[i]);
						randIS.readFully(temp);
						decoder.setInput(temp);
						int numOfBytes = decoder.decode(pixels, offset, Math.min(rowsPerStrip,rowsRemain)*bytesPerScanLine);
						offset += numOfBytes;
						rowsRemain -= Math.min(rowsPerStrip, rowsRemain);
					}
				}
				//Create a BufferedImage
				int[] off = {0};//band offset, we have only one band start at 0
				DataBuffer db = new DataBufferByte(pixels, pixels.length);
				WritableRaster raster = Raster.createInterleavedRaster(db, imageWidth, imageHeight, bytesPerScanLine, 1, off, null);
				ColorModel cm = new IndexColorModel(bitsPerSample, rgbColorPalette.length, rgbColorPalette, 0, false, -1, DataBuffer.TYPE_BYTE);
		   	
				return new BufferedImage(cm, raster, false, null);
			case RGB:
				pixels = new byte[imageWidth*imageHeight*3];				
				if(compression == TiffFieldEnum.Compression.NONE) {
					for(int i = 0; i < stripByteCounts.length; i++) {					
						randIS.seek(stripOffsets[i]);
						randIS.readFully(pixels, offset, Math.min(rowsPerStrip, rowsRemain)*imageWidth*3);
						offset += Math.min(rowsPerStrip, rowsRemain)*imageWidth*3;
						rowsRemain -= Math.min(rowsPerStrip, rowsRemain);					
					}
				} else if(compression == TiffFieldEnum.Compression.LZW) {
					LZWTreeDecoder decoder = new LZWTreeDecoder(8, true);
					for(int i = 0; i < stripByteCounts.length; i++) {
						byte[] temp = new byte[stripByteCounts[i]];
						randIS.seek(stripOffsets[i]);
						randIS.readFully(temp);
						decoder.setInput(temp);
						int bytes2Read = Math.min(rowsPerStrip, rowsRemain)*imageWidth*3;
						int numOfBytes = decoder.decode(pixels, offset, bytes2Read);							
						offset += numOfBytes;
						rowsRemain -= Math.min(rowsPerStrip, rowsRemain);					
					}					
				} else if(compression == TiffFieldEnum.Compression.DEFLATE || compression == TiffFieldEnum.Compression.DEFLATE_ADOBE) {
					DeflateDecoder decoder = new DeflateDecoder();
					for(int i = 0; i < stripByteCounts.length; i++) {
						byte[] temp = new byte[stripByteCounts[i]];
						randIS.seek(stripOffsets[i]);
						randIS.readFully(temp);
						decoder.setInput(temp);
						int bytes2Read = Math.min(rowsPerStrip, rowsRemain)*imageWidth*3;
						int numOfBytes = decoder.decode(pixels, offset, bytes2Read);							
						offset += numOfBytes;
						rowsRemain -= Math.min(rowsPerStrip, rowsRemain);					
					}					
				} else if(compression == TiffFieldEnum.Compression.PACKBITS) {
					for(int i = 0; i < stripByteCounts.length; i++) {
						byte[] temp = new byte[stripByteCounts[i]];
						randIS.seek(stripOffsets[i]);
						randIS.readFully(temp);
						int bytes2Read = Math.min(rowsPerStrip, rowsRemain)*imageWidth*3;
						byte[] temp2 = new byte[bytes2Read];
						Packbits.unpackbits(temp, temp2);
						System.arraycopy(temp2, 0, pixels, offset, bytes2Read);							
						offset += bytes2Read;
						rowsRemain -= Math.min(rowsPerStrip, rowsRemain);					
					}					
				}
				//Create a BufferedImage
				db = new DataBufferByte(pixels, pixels.length);
				int[] bandoff = {0, 1, 2}; //band offset, we have 3 bands
				int numOfBands = 3;
				int trans = Transparency.OPAQUE;
				int[] nBits = {8, 8, 8};						
				raster = Raster.createInterleavedRaster(db, imageWidth, imageHeight, imageWidth*numOfBands, numOfBands, bandoff, null);
				cm = new ComponentColorModel(ColorSpace.getInstance(ColorSpace.CS_sRGB), nBits, false, false,
			                trans, DataBuffer.TYPE_BYTE);
					
				return new BufferedImage(cm, raster, false, null);
						
			default:
		 		break;
		}	
		
		return null;
	}
	 
	private boolean readHeader(RandomAccessInputStream randIS) throws IOException {
		// First 2 bytes determine the byte order of the file
		short endian = randIS.readShort();
		
		if(endian == IOUtils.BIG_ENDIAN)
		{
			System.out.println("Byte order: Motorola BIG_ENDIAN");
			this.randIS.setReadStrategy(ReadStrategyMM.getInstance());
		} else if(endian == IOUtils.LITTLE_ENDIAN) {
			System.out.println("Byte order: Intel LITTLE_ENDIAN");
			this.randIS.setReadStrategy(ReadStrategyII.getInstance());
		} else {
			System.out.println("Warning: invalid TIFF byte order!");
			return false;
		} 
		
		// Read TIFF identifier
		short tiff_id = randIS.readShort();
		  
		if(tiff_id!=0x2a)//"*" 42 decimal
{'fraction_non_alphanumeric': 0.12414046506745052, 'fraction_numerical': 0.010235683166237993, 'mean_word_length': 4.110515021459228, 'pattern_counts': {'":': 0, '<': 52, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1869505', 'n_tokens_mistral': 7520, 'n_tokens_neox': 6321, 'n_words': 1247}