<commit_before>package org.vast.decompression;

import java.awt.Color;
import java.awt.Image;
import java.awt.MediaTracker;
import java.awt.image.BufferedImage;
import java.awt.image.DataBuffer;
import java.awt.image.DataBufferByte;
import java.awt.image.DataBufferInt;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile;

import javax.imageio.ImageIO;
import javax.imageio.stream.ImageInputStream;
import javax.swing.JLabel;

import org.vast.cdm.common.BinaryBlock;
import org.vast.cdm.common.CDMException;
import org.vast.cdm.common.DataBlock;
import org.vast.cdm.common.DataComponent;
import org.vast.cdm.common.DataType;
import org.vast.data.DataBlockFactory;
import org.vast.data.DataValue;
import org.vast.sweCommon.DataInputExt;

import sun.awt.image.ToolkitImage;


/**
 * <p><b>Title:</b>
 * CommonImageDecoder
 * </p>
 *
 * <p><b>Description:</b><br/>
 * decoder to be used by the BinaryDataParser to decode a binary block 
 * that is compressed in JPEG and TIFF, PNG, ... 
 * </p>
 *
 * <p>Copyright (c) 2007</p>
 * @author <NAME>
 * @date Jul 16, 2008
 * @version 1.0
 */
public class CommonImageIODecoder extends CompressedStreamReader
{
	protected String imageUrl;
	protected BufferedImage image;  
	protected DataBlock imageBlock;
	protected boolean fileBased;
	protected ByteArrayInputStream byteArrayInputStream;
	protected byte[] byteArray;
	
	
	public CommonImageIODecoder() 
	{
	}

	
	@Override
	public void init(BinaryBlock binaryBlock, DataComponent blockComponent) throws CDMException {
		
		DataComponent primitiveRecord = blockComponent.getComponent(0).getComponent(0);
		for(int i = 0; i<primitiveRecord.getComponentCount(); i++)
		{
			((DataValue)primitiveRecord.getComponent(i)).setDataType(DataType.BYTE);
		}
	}


	@Override
	protected void parse(DataInputExt inputStream, DataComponent blockInfo)
			throws CDMException {

		int byteSize = 0;
    	
    	try
    	{
    		byteSize = (int)inputStream.readLong();
    	}
    	catch (IOException e)
		{
			throw new CDMException("Error while reading binary stream", e);
		}
    	
		
		byte [] block = new byte[byteSize];
		try {
			inputStream.readFully(block);
		} catch (IOException e) {
			throw new CDMException("error when reading binary block of DataBlock " + blockInfo.getName(), e);
		}
		
		ByteArrayInputStream byteArrayStream = new ByteArrayInputStream(block);
		InputStream stream = (InputStream)byteArrayStream;
		//ImageInputStream imageInputStream = (ImageInputStream) byteArrayStream;
		BufferedImage image;
		try {
			image = ImageIO.read(stream);
		} catch (IOException e) {
			throw new CDMException("error when generating the compressed from the binary block of DataBlock " + blockInfo.getName(), e);
		}
		if(blockInfo.getData().getAtomCount() != image.getWidth()*image.getHeight()*3)
		{
			throw new CDMException("the size of the decoded image is not that " +
								   "described in the Swe Common description");
		}
		int [] rgbArray = new int[image.getWidth()*image.getHeight()];
		image.getRGB(0, 0, image.getWidth(), image.getHeight(), rgbArray, 0, 0);
		Color color = null;
			
		for (int i = 0; i<rgbArray.length; i++){
			color = new Color(rgbArray[i]);
		    short red = (short)color.getRed();
		    short green = (short)color.getGreen();
		    short blue = (short)color.getBlue();
		    blockInfo.getData().setShortValue(i, red);
			blockInfo.getData().setShortValue(i+1, green);
			blockInfo.getData().setShortValue(i+2, blue);
		}
	}
	
}<commit_msg>Gregoire's fixes to support JPEG decompression in web video stream<commit_after>package org.vast.decompression;

import java.awt.Color;
import java.awt.Image;
import java.awt.MediaTracker;
import java.awt.image.BufferedImage;
import java.awt.image.DataBuffer;
import java.awt.image.DataBufferByte;
import java.awt.image.DataBufferInt;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile;

import javax.imageio.ImageIO;
import javax.imageio.stream.ImageInputStream;
import javax.swing.JLabel;
import jj2000.j2k.decoder.Decoder;
import jj2000.j2k.util.ParameterList;

import org.vast.cdm.common.BinaryBlock;
import org.vast.cdm.common.CDMException;
import org.vast.cdm.common.DataBlock;
import org.vast.cdm.common.DataComponent;
import org.vast.cdm.common.DataType;
import org.vast.data.DataBlockFactory;
import org.vast.data.DataValue;
import org.vast.sweCommon.DataInputExt;

import sun.awt.image.ToolkitImage;


/**
 * <p><b>Title:</b>
 * CommonImageDecoder
 * </p>
 *
 * <p><b>Description:</b><br/>
 * decoder to be used by the BinaryDataParser to decode a binary block 
 * that is compressed in JPEG and TIFF, PNG, ... 
 * </p>
 *
 * <p>Copyright (c) 2007</p>
 * @author <NAME>
 * @date Jul 16, 2008
 * @version 1.0
 */
public class CommonImageIODecoder extends CompressedStreamReader
{
	protected String imageUrl;
	protected BufferedImage image;  
	protected DataBlock imageBlock;
	protected ParameterList list;
	protected Decoder dec;
	protected boolean fileBased;
	protected ByteArrayInputStream byteArrayInputStream;
	protected byte[] byteArray;
	
	
	public CommonImageIODecoder() 
	{
	}

	
	@Override
	public void init(BinaryBlock binaryBlock, DataComponent blockComponent) throws CDMException {
		
		DataComponent primitiveRecord = blockComponent.getComponent(0).getComponent(0);
		for(int i = 0; i<primitiveRecord.getComponentCount(); i++)
		{
			((DataValue)primitiveRecord.getComponent(i)).setDataType(DataType.BYTE);
		}
	}


	@Override
	protected void parse(DataInputExt inputStream, DataComponent blockInfo)
			throws CDMException {

		int byteSize = 0;
    	
    	try
    	{
    		byteSize = (int)inputStream.readLong();
    	}
    	catch (IOException e)
		{
			throw new CDMException("Error while reading binary stream", e);
		}
    	
		
		byte [] block = new byte[byteSize];
		try {
			inputStream.readFully(block);
		} catch (IOException e) {
			throw new CDMException("error when reading binary block of DataBlock " + blockInfo.getName(), e);
		}
		
		ByteArrayInputStream byteArrayStream = new ByteArrayInputStream(block);
		InputStream stream = (InputStream)byteArrayStream;
		//ImageInputStream imageInputStream = (ImageInputStream) byteArrayStream;
		BufferedImage image;
		try {
			image = ImageIO.read(stream);
		} catch (IOException e) {
			throw new CDMException("error when generating the compressed from the binary block of DataBlock " + blockInfo.getName(), e);
		}
		if(blockInfo.getData().getAtomCount() != image.getWidth()*image.getHeight()*3)
		{
			throw new CDMException("the size of the decoded image is not that " +
								   "described in the Swe Common description");
		}
		//int [] rgbArray = new int[image.getWidth()*image.getHeight()];
		//image.getRGB(0, 0, image.getWidth(), image.getHeight(), rgbArray, 0, 0);
		Color color = null;
		
		int m=0;
		for (int j = 0; j<image.getHeight(); j++){
			for (int i = 0; i<image.getWidth(); i++){
				color = new Color(image.getRGB(i, j));
				short red = (short)color.getRed();
				short green = (short)color.getGreen();
				short blue = (short)color.getBlue();
				//System.out.println("red " + red +  " green " + green + " blue " + blue);
				blockInfo.getData().setShortValue(m, red);
				blockInfo.getData().setShortValue(m+1, green);
				blockInfo.getData().setShortValue(m+2, blue);
				m+=3;
			}
		}
	}
	
}
{'fraction_non_alphanumeric': 0.1211430104641803, 'fraction_numerical': 0.008183525623826134, 'mean_word_length': 4.283486888731396, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '525664', 'n_tokens_mistral': 2606, 'n_tokens_neox': 2335, 'n_words': 567}