<commit_before>package vu.nl.lsde.group01;

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;

import javax.xml.parsers.ParserConfigurationException;

import org.apache.commons.compress.compressors.CompressorException;
import org.apache.commons.compress.compressors.CompressorInputStream;
import org.apache.commons.compress.compressors.CompressorStreamFactory;
import org.apache.poi.ss.util.ImageUtils;
import org.apache.sanselan.ImageReadException;
import org.openimaj.image.DisplayUtilities;
import org.openimaj.image.FImage;
import org.openimaj.image.ImageUtilities;
import org.openimaj.image.MBFImage;
import org.openimaj.image.colour.RGBColour;
import org.openimaj.image.colour.Transforms;
import org.openimaj.image.processing.face.detection.CCDetectedFace;
import org.openimaj.image.processing.face.detection.DetectedFace;
import org.openimaj.image.processing.face.detection.FaceDetector;
import org.openimaj.image.processing.face.detection.HaarCascadeDetector;
import org.openimaj.image.processing.face.detection.IdentityFaceDetector;
import org.openimaj.image.processing.face.detection.SandeepFaceDetector;
import org.openimaj.image.processing.face.detection.keypoints.FKEFaceDetector;
import org.openimaj.image.processing.face.detection.keypoints.KEDetectedFace;
import org.xml.sax.SAXException;

import cern.colt.Arrays;

import com.flickr4java.flickr.FlickrException;



/**
 * OpenIMAJ Hello world!
 *
 */
public class App {
    public static void main( String[] args ) throws CompressorException, FileNotFoundException {
        readImages();
        
        FaceDetector<DetectedFace,FImage> fd = new HaarCascadeDetector(80);
        //FaceDetector<KEDetectedFace, FImage> fdSandeep = new FKEFaceDetector();

        int faceCounter = 0;
        MBFImage image = null;
        
        for(int i = 0; i< 1000; i++){
            try{
                image = ImageUtilities.readMBF(new File("resources/"+i+"_b.jpg"));
            }catch(IOException e){
                continue;
            }
            List<DetectedFace> faces = fd.detectFaces(Transforms.calculateIntensity(image));
            
            int j = 0;
            for( DetectedFace face : faces ) {
                //image.drawShape(face.getBounds(), RGBColour.RED);
                //faceCounter++;
                MBFImage faceImage = image.extractROI(face.getBounds());
                
                try {
                    ImageUtilities.write(faceImage, new File("resources/face"+i+"_"+j+"_b.jpg"));
                } catch (IOException e) {
                    continue;
                }
                j++;
                
            }
            if( faces.size() > 0 ){
                System.out.println("Detected Faces: "+(faceCounter++)+" (current url: "+i+")");
            }
        }
        
    }
    
    public static void readImages() throws FileNotFoundException, CompressorException{
        FileInputStream fin = new FileInputStream(new File("resources/yfcc100m_dataset-0.bz2"));
        BufferedInputStream bis = new BufferedInputStream(fin);
        CompressorInputStream input = new CompressorStreamFactory().createCompressorInputStream(bis);
        BufferedReader br2 = new BufferedReader(new InputStreamReader(input));

        boolean end = false;
        InputStream in = null;
        String line = null;
        String[] columns = null;
        ArrayList<String> errors = new ArrayList<String>();
        int i = 0;
        while(!end){
            try {
                line = br2.readLine();
            } catch (IOException e1) {
                // TODO Auto-generated catch block
                e1.printStackTrace();
            }
            columns = line.split("\t");
            String url = "https://farm"+columns[18]+".staticflickr.com/"+columns[17]+"/"+columns[0]+"_"+columns[19]+"_b."+columns[21];//columns[14];
            //System.out.println(Arrays.toString(url));

            if(!columns[22].equals("0")){
                continue;
            }
            URL link = null;
            try {
                link = new URL(url);
            } catch (MalformedURLException e1) {
                errors.add("Image at URL: "+url+" not found.");
                continue;
            } //The file that you want to download
            try {
                in = new BufferedInputStream(link.openStream());
            } catch (IOException e1) {
                errors.add("Could not load Image at URL: "+url+".");
                continue;
            }

            try {
                OutputStream out = new FileOutputStream(new File("resources/"+i+"_b.jpg"));
                byte[] buf = new byte[1024];
                int len;
                while((len=in.read(buf))>0){
                    out.write(buf,0,len);
                }
                out.close();<commit_msg>imporoved implementation with advanced HaarCascade configuration<commit_after>package vu.nl.lsde.group01;

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;

import javax.xml.parsers.ParserConfigurationException;

import org.apache.commons.compress.compressors.CompressorException;
import org.apache.commons.compress.compressors.CompressorInputStream;
import org.apache.commons.compress.compressors.CompressorStreamFactory;
import org.apache.poi.ss.util.ImageUtils;
import org.apache.sanselan.ImageReadException;
import org.openimaj.image.DisplayUtilities;
import org.openimaj.image.FImage;
import org.openimaj.image.ImageUtilities;
import org.openimaj.image.MBFImage;
import org.openimaj.image.colour.RGBColour;
import org.openimaj.image.colour.Transforms;
import org.openimaj.image.processing.face.detection.CCDetectedFace;
import org.openimaj.image.processing.face.detection.DetectedFace;
import org.openimaj.image.processing.face.detection.FaceDetector;
import org.openimaj.image.processing.face.detection.FaceDetectorFeatures;
import org.openimaj.image.processing.face.detection.HaarCascadeDetector;
import org.openimaj.image.processing.face.detection.HaarCascadeDetector.BuiltInCascade;
import org.openimaj.image.processing.face.detection.IdentityFaceDetector;
import org.openimaj.image.processing.face.detection.SandeepFaceDetector;
import org.openimaj.image.processing.face.detection.keypoints.FKEFaceDetector;
import org.openimaj.image.processing.face.detection.keypoints.KEDetectedFace;
import org.openimaj.image.processing.resize.ResizeProcessor;
import org.xml.sax.SAXException;

import cern.colt.Arrays;

import com.flickr4java.flickr.FlickrException;



/**
 * OpenIMAJ Hello world!
 *
 */
public class App {
    public static void main( String[] args ) throws CompressorException, FileNotFoundException {
        readImages();
        
        //FaceDetector<DetectedFace,FImage> fd = new HaarCascadeDetector(80);
        //FaceDetector<KEDetectedFace, FImage> fdSandeep = new FKEFaceDetector();
        ResizeProcessor resize = new ResizeProcessor(100, 100);
        BuiltInCascade cascade = BuiltInCascade.frontalface_alt_tree;
        HaarCascadeDetector fd = cascade.load();
        fd.setScale(1.15f);
        
        int faceCounter = 0;
        MBFImage image = null;
        
        for(int i = 0; i< 1000; i++){
            try{
                image = ImageUtilities.readMBF(new File("resources/"+i+".jpg"));
            }catch(IOException e){
                continue;
            }
            
            
            List<DetectedFace> faces = fd.detectFaces(Transforms.calculateIntensity(image));
            //FaceDetector<KEDetectedFace, FImage> fdSandeep = new FKEFaceDetector();

            int j = 0;
            for( DetectedFace face : faces ) {
                //image.drawShape(face.getBounds(), RGBColour.RED);
                //faceCounter++;
                MBFImage faceImage = image.extractROI(face.getBounds());
                faceImage = faceImage.process(resize);
                try {
                    ImageUtilities.write(faceImage, new File("resources/new_run/face"+i+"_"+j+".jpg"));
                } catch (IOException e) {
                    continue;
                }
                j++;
                
            }
            if( faces.size() > 0 ){
                System.out.println("Detected Faces: "+(faceCounter++)+" (current url: "+i+")");
            }
        }
        
    }
    
    public static void readImages() throws FileNotFoundException, CompressorException{
        FileInputStream fin = new FileInputStream(new File("resources/yfcc100m_dataset-0.bz2"));
        BufferedInputStream bis = new BufferedInputStream(fin);
        CompressorInputStream input = new CompressorStreamFactory().createCompressorInputStream(bis);
        BufferedReader br2 = new BufferedReader(new InputStreamReader(input));

        boolean end = false;
        InputStream in = null;
        String line = null;
        String[] columns = null;
        ArrayList<String> errors = new ArrayList<String>();
        int i = 0;
        while(!end){
            try {
                line = br2.readLine();
            } catch (IOException e1) {
                // TODO Auto-generated catch block
                e1.printStackTrace();
            }
            
            columns = line.split("\t");
            String url = columns[14];

            if(!columns[22].equals("0")){
                continue;
            }
            URL link = null;
            try {
                link = new URL(url);
            } catch (MalformedURLException e1) {
                errors.add("Image at URL: "+url+" not found.");
                continue;
            } //The file that you want to download
            try {
                in = new BufferedInputStream(link.openStream());
            } catch (IOException e1) {
                errors.add("Could not load Image at URL: "+url+".");
                continue;
            }

            try {
                OutputStream out = new FileOutputStream(new File("resources/"+i+"_b.jpg"));
                byte[] buf = new byte[1024];
                int len;
                while((len=in.read(buf))>0){
                    out.write(buf,0,len);
                }
                out.close();
{'fraction_non_alphanumeric': 0.1097100782328578, 'fraction_numerical': 0.008283479061205707, 'mean_word_length': 2.363045496750232, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5827748', 'n_tokens_mistral': 3154, 'n_tokens_neox': 2965, 'n_words': 572}