<commit_before>/*

Copyright (C) 2015 - <NAME> / Rapid Information Systems

<EMAIL>


This file is part of the Rapid Application Platform

RapidSOA is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as 
published by the Free Software Foundation, either version 3 of the 
License, or (at your option) any later version. The terms require you 
to include the original copyright, and the license notice in all redistributions.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
in a file named "COPYING".  If not, see <http://www.gnu.org/licenses/>.

*/

package com.rapid.utils;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.Deflater;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;

public class ZipFile {
	
	public static class ZipSource {
		
		private File _file;
		private String _path;
		
		public ZipSource(File file) {
			_file = file;
			_path = "";
		}
		
		public ZipSource(File file, String path) {
			_file = file;
			_path = path;
		}
		
		public File getFile() { return _file; }
		public String getPath() { return _path; }
		
	}
	
	public static class ZipSources extends ArrayList<ZipSource> {

		// overrides
		
		@Override
		public boolean contains(Object o) {
			if (o.getClass() == ZipSource.class) {
				ZipSource zs = (ZipSource) o;
				return contains(zs);
			}
			return false;
		}

		@Override
		public boolean add(ZipSource zipSource) {
			if (contains(zipSource)) {
				return false;
			} else {
				return super.add(zipSource);
			}
		}

		@Override
		public void add(int index, ZipSource zipSource) {
			if (!contains(zipSource)) super.add(index, zipSource);
		}
		
		// instance methods
		
		public boolean contains(ZipSource zipSource) {
			for (ZipSource zs : this) {
				if (zipSource.getFile().getName().equals(zs.getFile().getName()) && zipSource.getPath().equals(zs.getPath())) return true;
			}
			return false;
		}
		
		public boolean add(File file) {
			ZipSource zipSource = new ZipSource(file);
			if (contains(zipSource)) {
				return false;
			} else {
				return add(zipSource);
			}
		}
		
		public boolean add(File file, String path) {
			ZipSource zipSource = new ZipSource(file, path);
			if (contains(zipSource)) {
				return false;
			} else {
				return add(zipSource);
			}
		}					
	}
	
	final int BUFFER = 1024;
	
	File _file;
	
	public ZipFile(File file) throws FileNotFoundException {
		_file = file;			
	}
	
	private String buildPath(String path, String file) {
        if (path == null || path.isEmpty()) {
            return file;
        } else {
            return path + "/" + file;
        }
    }

    private void zipDir(ZipOutputStream zos, String path, File dir) throws IOException {

        File[] files = dir.listFiles();
        path = buildPath(path, dir.getName());

        for (File source : files) {
            if (source.isDirectory()) {
                zipDir(zos, path, source);
            } else {
                zipFile(zos, path, source);
            }
        }
    }

    private void zipFile(ZipOutputStream zos, String path, File file) throws IOException {

        zos.putNextEntry(new ZipEntry(buildPath(path, file.getName())));

        FileInputStream fis = new FileInputStream(file);

        byte[] buffer = new byte[BUFFER];
        int byteCount = 0;
        while ((byteCount = fis.read(buffer)) != -1) {
            zos.write(buffer, 0, byteCount);
        }
        
        fis.close();
        zos.closeEntry();
    }
	
    public void zipFiles(ZipSources sources, List<String> ignoreFiles) throws IOException {
        
    	ZipOutputStream zipOut = new ZipOutputStream(new FileOutputStream(_file));
        zipOut.setLevel(Deflater.DEFAULT_COMPRESSION);

        for (ZipSource source : sources) {
        	
        	File sourceFile = source.getFile();
        	
        	boolean ignore = false;
        	
        	if (ignoreFiles != null) {
        		for (String ignoreFile : ignoreFiles) {
        			if (sourceFile.getName().equals(ignoreFile)) {
        				ignore = true;
        				break;
        			}<commit_msg>Improvement to ZipFile to avoid duplicate entries of items in subfolders that we already have.<commit_after>/*

Copyright (C) 2015 - <NAME> / Rapid Information Systems

<EMAIL>


This file is part of the Rapid Application Platform

RapidSOA is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as 
published by the Free Software Foundation, either version 3 of the 
License, or (at your option) any later version. The terms require you 
to include the original copyright, and the license notice in all redistributions.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
in a file named "COPYING".  If not, see <http://www.gnu.org/licenses/>.

*/

package com.rapid.utils;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.Deflater;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;

public class ZipFile {
	
	public static class ZipSource {
		
		private File _file;
		private String _path;
		
		public ZipSource(File file) {
			_file = file;
			_path = "";
		}
		
		public ZipSource(File file, String path) {
			_file = file;
			_path = path;
		}
		
		public File getFile() { return _file; }
		public String getPath() { return _path; }
		
	}
	
	public static class ZipSources extends ArrayList<ZipSource> {

		// overrides
		
		@Override
		public boolean contains(Object o) {
			if (o.getClass() == ZipSource.class) {
				ZipSource zs = (ZipSource) o;
				return contains(zs);
			}
			return false;
		}

		@Override
		public boolean add(ZipSource zipSource) {
			if (contains(zipSource)) {
				return false;
			} else {
				return super.add(zipSource);
			}
		}

		@Override
		public void add(int index, ZipSource zipSource) {
			if (!contains(zipSource)) super.add(index, zipSource);
		}
		
		// instance methods

		public boolean contains(ZipSource zipSource) {
			for (ZipSource zs : this) {
				if (zs.getFile().isDirectory()) {
					if (zipSource.getPath().startsWith(zs.getPath())) {
						if (zipSource.getFile().isDirectory()) {
							if (zipSource.getFile().getName().startsWith(zs.getFile().getName())) {
								return true;
							}
						} else {
							if (zipSource.getFile().getParentFile().getName().startsWith(zs.getFile().getName())) {
								return true;
							}
						}
					}
				} else 	if (zipSource.getFile().getName().equals(zs.getFile().getName()) && zipSource.getPath().equals(zs.getPath())) return true;
			}
			return false;
		}
		
		public boolean add(File file) {
			ZipSource zipSource = new ZipSource(file);
			if (contains(zipSource)) {
				return false;
			} else {
				return add(zipSource);
			}
		}
		
		public boolean add(File file, String path) {
			ZipSource zipSource = new ZipSource(file, path);
			if (contains(zipSource)) {
				return false;
			} else {
				return add(zipSource);
			}
		}					
	}
	
	final int BUFFER = 1024;
	
	File _file;
	
	public ZipFile(File file) throws FileNotFoundException {
		_file = file;			
	}
	
	private String buildPath(String path, String file) {
        if (path == null || path.isEmpty()) {
            return file;
        } else {
            return path + "/" + file;
        }
    }

    private void zipDir(ZipOutputStream zos, String path, File dir) throws IOException {

        File[] files = dir.listFiles();
        path = buildPath(path, dir.getName());

        for (File source : files) {
            if (source.isDirectory()) {
                zipDir(zos, path, source);
            } else {
                zipFile(zos, path, source);
            }
        }
    }

    private void zipFile(ZipOutputStream zos, String path, File file) throws IOException {
    	
        zos.putNextEntry(new ZipEntry(buildPath(path, file.getName())));

        FileInputStream fis = new FileInputStream(file);

        byte[] buffer = new byte[BUFFER];
        int byteCount = 0;
        while ((byteCount = fis.read(buffer)) != -1) {
            zos.write(buffer, 0, byteCount);
        }
        
        fis.close();
        zos.closeEntry();
    }
	
    public void zipFiles(ZipSources sources, List<String> ignoreFiles) throws IOException {
        
    	ZipOutputStream zipOut = new ZipOutputStream(new FileOutputStream(_file));
        zipOut.setLevel(Deflater.DEFAULT_COMPRESSION);

        for (ZipSource source : sources) {
        	
        	File sourceFile = source.getFile();
        	
        	boolean ignore = false;
        	
        	if (ignoreFiles != null) {
        		for (String ignoreFile : ignoreFiles) {
        			if (sourceFile.getName().equals(ignoreFile)) {
        				ignore = true;
        				break;
        			}
{'fraction_non_alphanumeric': 0.10052963943776737, 'fraction_numerical': 0.002444489712772459, 'mean_word_length': 2.708081570996979, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6416876', 'n_tokens_mistral': 3284, 'n_tokens_neox': 2801, 'n_words': 880}