<commit_before>/*
 *  Licensed to the Apache Software Foundation (ASF) under one or more
 *  contributor license agreements.  See the NOTICE file distributed with
 *  this work for additional information regarding copyright ownership.
 *  The ASF licenses this file to You under the Apache License, Version 2.0
 *  (the "License"); you may not use this file except in compliance with
 *  the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 */

package org.apache.commons.compress.archivers.zip;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URI;
import java.net.URL;
import java.util.Random;

import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assume.assumeNotNull;
import static org.junit.Assume.assumeTrue;

public class Zip64SupportTest {

    private static final long FIVE_BILLION = 5000000000l;
    private static final int ONE_HUNDRED_THOUSAND = 100000;

    @Test public void read5GBOfZerosUsingInputStream() throws Throwable {
        FileInputStream fin = new FileInputStream(get5GBZerosFile());
        ZipArchiveInputStream zin = null;
        try {
            zin = new ZipArchiveInputStream(fin);
            ZipArchiveEntry zae = zin.getNextZipEntry();
            assertEquals("5GB_of_Zeros", zae.getName());
            assertEquals(FIVE_BILLION, zae.getSize());
            byte[] buf = new byte[1024 * 1024];
            long read = 0;
            Random r = new Random(System.currentTimeMillis());
            int readNow;
            while ((readNow = zin.read(buf, 0, buf.length)) > 0) {
                // testing all bytes for a value of 0 is going to take
                // too long, just pick a few ones randomly
                for (int i = 0; i < 1024; i++) {
                    int idx = r.nextInt(readNow);
                    assertEquals("testing byte " + (read + idx), 0, buf[idx]);
                }
                read += readNow;
            }
            assertEquals(FIVE_BILLION, read);
            assertNull(zin.getNextZipEntry());
        } finally {
            if (zin != null) {
                zin.close();
            }
            if (fin != null) {
                fin.close();
            }
        }
    }

    @Test public void read100KFilesUsingInputStream() throws Throwable {
        FileInputStream fin = new FileInputStream(get100KFileFile());
        ZipArchiveInputStream zin = null;
        try {
            zin = new ZipArchiveInputStream(fin);
            int files = 0;
            ZipArchiveEntry zae = null;
            while ((zae = zin.getNextZipEntry()) != null) {
                if (!zae.isDirectory()) {
                    files++;
                    assertEquals(0, zae.getSize());
                }
            }
            assertEquals(ONE_HUNDRED_THOUSAND, files);
        } finally {
            if (zin != null) {
                zin.close();
            }
            fin.close();
        }
    }

    @Test public void write100KFiles() throws Throwable {
        withTemporaryArchive("write100KFiles", new ZipOutputTest() {
                public void test(File f, ZipArchiveOutputStream zos)
                    throws IOException {
                    for (int i = 0; i < ONE_HUNDRED_THOUSAND; i++) {
                        ZipArchiveEntry zae =
                            new ZipArchiveEntry(String.valueOf(i));
                        zae.setSize(0);
                        zos.putArchiveEntry(zae);
                        zos.closeArchiveEntry();
                    }
                }
            });
    }

    static interface ZipOutputTest {
        void test(File f, ZipArchiveOutputStream zos) throws IOException;
    }

    private static void withTemporaryArchive(String testName,
                                             ZipOutputTest test)
        throws Throwable {
        File f = getTempFile(testName);
        ZipArchiveOutputStream zos = new ZipArchiveOutputStream(f);
        try {
            test.test(f, zos);
        } catch (IOException ex) {
            System.err.println("Failed to write archive because of: "
                               + ex.getMessage()
                               + " - likely not enough disk space.");
            assumeTrue(false);
        } finally {
            zos.close();
        }
    }

    private static File getFile(String name) throws Throwable {
        URL url = Zip64SupportTest.class.getResource(name);
        assumeNotNull(url);
        File file = new File(new URI(url.toString()));
        assumeTrue(file.exists());
        return file;
    }

    private static File get5GBZerosFile() throws Throwable {
        return getFile("/5GB_of_Zeros.zip");
    }

    private static File get100KFileFile() throws Throwable {
        return getFile("/100k_Files.zip");
    }

    private static File getTempFile(String testName) throws Throwable {
        File f = File.createTempFile("commons-compress-" + testName, ".zip");
        f.deleteOnExit();
        return f;
    }
}
<commit_msg>start validating file structure, incomplete, need to run now<commit_after>/*
 *  Licensed to the Apache Software Foundation (ASF) under one or more
 *  contributor license agreements.  See the NOTICE file distributed with
 *  this work for additional information regarding copyright ownership.
 *  The ASF licenses this file to You under the Apache License, Version 2.0
 *  (the "License"); you may not use this file except in compliance with
 *  the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 */

package org.apache.commons.compress.archivers.zip;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.net.URI;
import java.net.URL;
import java.util.Random;

import org.junit.Test;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assume.assumeNotNull;
import static org.junit.Assume.assumeTrue;

public class Zip64SupportTest {

    private static final long FIVE_BILLION = 5000000000l;
    private static final int ONE_HUNDRED_THOUSAND = 100000;

    @Test public void read5GBOfZerosUsingInputStream() throws Throwable {
        FileInputStream fin = new FileInputStream(get5GBZerosFile());
        ZipArchiveInputStream zin = null;
        try {
            zin = new ZipArchiveInputStream(fin);
            ZipArchiveEntry zae = zin.getNextZipEntry();
            assertEquals("5GB_of_Zeros", zae.getName());
            assertEquals(FIVE_BILLION, zae.getSize());
            byte[] buf = new byte[1024 * 1024];
            long read = 0;
            Random r = new Random(System.currentTimeMillis());
            int readNow;
            while ((readNow = zin.read(buf, 0, buf.length)) > 0) {
                // testing all bytes for a value of 0 is going to take
                // too long, just pick a few ones randomly
                for (int i = 0; i < 1024; i++) {
                    int idx = r.nextInt(readNow);
                    assertEquals("testing byte " + (read + idx), 0, buf[idx]);
                }
                read += readNow;
            }
            assertEquals(FIVE_BILLION, read);
            assertNull(zin.getNextZipEntry());
        } finally {
            if (zin != null) {
                zin.close();
            }
            if (fin != null) {
                fin.close();
            }
        }
    }

    @Test public void read100KFilesUsingInputStream() throws Throwable {
        FileInputStream fin = new FileInputStream(get100KFileFile());
        ZipArchiveInputStream zin = null;
        try {
            zin = new ZipArchiveInputStream(fin);
            int files = 0;
            ZipArchiveEntry zae = null;
            while ((zae = zin.getNextZipEntry()) != null) {
                if (!zae.isDirectory()) {
                    files++;
                    assertEquals(0, zae.getSize());
                }
            }
            assertEquals(ONE_HUNDRED_THOUSAND, files);
        } finally {
            if (zin != null) {
                zin.close();
            }
            fin.close();
        }
    }

    @Test public void write100KFiles() throws Throwable {
        withTemporaryArchive("write100KFiles", new ZipOutputTest() {
                public void test(File f, ZipArchiveOutputStream zos)
                    throws IOException {
                    for (int i = 0; i < ONE_HUNDRED_THOUSAND; i++) {
                        ZipArchiveEntry zae =
                            new ZipArchiveEntry(String.valueOf(i));
                        zae.setSize(0);
                        zos.putArchiveEntry(zae);
                        zos.closeArchiveEntry();
                    }
                    zos.close();
                    RandomAccessFile a = new RandomAccessFile(f, "r");
                    try {
                        final long end = a.length();

                        // validate "end of central directory" is at
                        // the end of the file and contains the magic
                        // value 0xFFFF as "number of entries".
                        a.seek(end
                               - 22 /* length of EOCD without file comment */);
                        byte[] eocd = new byte[12];
                        a.readFully(eocd);
                        assertArrayEquals(new byte[] {
                                // sig
                                (byte) 0x50, (byte) 0x4b, 5, 6,
                                // disk numbers
                                0, 0, 0, 0,
                                // entries
                                (byte) 0xff, (byte) 0xff,
                                (byte) 0xff, (byte) 0xff,
                            }, eocd); 
                    } finally {
                        a.close();
                    }
                }
            });
    }

    static interface ZipOutputTest {
        void test(File f, ZipArchiveOutputStream zos) throws IOException;
    }

    private static void withTemporaryArchive(String testName,
                                             ZipOutputTest test)
        throws Throwable {
        File f = getTempFile(testName);
        ZipArchiveOutputStream zos = new ZipArchiveOutputStream(f);
        try {
            test.test(f, zos);
        } catch (IOException ex) {
            System.err.println("Failed to write archive because of: "
                               + ex.getMessage()
                               + " - likely not enough disk space.");
            assumeTrue(false);
        } finally {
            zos.close();
        }
    }

    private static File getFile(String name) throws Throwable {
        URL url = Zip64SupportTest.class.getResource(name);
        assumeNotNull(url);
        File file = new File(new URI(url.toString()));
        assumeTrue(file.exists());
        return file;
    }

    private static File get5GBZerosFile() throws Throwable {
        return getFile("/5GB_of_Zeros.zip");
    }

    private static File get100KFileFile() throws Throwable {
        return getFile("/100k_Files.zip");
    }

    private static File getTempFile(String testName) throws Throwable {
        File f = File.createTempFile("commons-compress-" + testName, ".zip");
        f.deleteOnExit();
        return f;
    }
}

{'fraction_non_alphanumeric': 0.08841812530672338, 'fraction_numerical': 0.01292327825944708, 'mean_word_length': 1.6908010563380282, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6723253', 'n_tokens_mistral': 3334, 'n_tokens_neox': 3057, 'n_words': 922}