<commit_before>// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.application;

import com.google.common.collect.ImmutableList;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UncheckedIOException;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.List;
import java.util.function.Predicate;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

/**
 * @author bratseth
 */
public class ZipStreamReader {

    private final ImmutableList<ZipEntryWithContent> entries;
    private final int maxEntrySizeInBytes;

    public ZipStreamReader(InputStream input, Predicate<String> entryNameMatcher, int maxEntrySizeInBytes) {
        this.maxEntrySizeInBytes = maxEntrySizeInBytes;
        try (ZipInputStream zipInput = new ZipInputStream(input)) {
            ImmutableList.Builder<ZipEntryWithContent> builder = new ImmutableList.Builder<>();
            ZipEntry zipEntry;
            while (null != (zipEntry = zipInput.getNextEntry())) {
                if (!entryNameMatcher.test(requireName(zipEntry.getName()))) continue;
                builder.add(new ZipEntryWithContent(zipEntry, readContent(zipInput)));
            }
            entries = builder.build();
        } catch (IOException e) {
            throw new UncheckedIOException("IO error reading zip content", e);
        }
    }

    private byte[] readContent(ZipInputStream zipInput) {
        try (ByteArrayOutputStream bis = new ByteArrayOutputStream()) {
            byte[] buffer = new byte[2048];
            int read;
            int size = 0;
            while ( -1 != (read = zipInput.read(buffer))) {
                size += read;
                if (size > maxEntrySizeInBytes) {
                    throw new IllegalArgumentException("Entry in zip content exceeded size limit of " +
                                                       maxEntrySizeInBytes + " bytes");
                }
                bis.write(buffer, 0, read);
            }
            return bis.toByteArray();
        } catch (IOException e) {
            throw new UncheckedIOException("Failed reading from zipped content", e);
        }
    }

    public List<ZipEntryWithContent> entries() { return entries; }

    private static String requireName(String name) {
        IllegalArgumentException e = new IllegalArgumentException("Unexpected non-normalized path found in zip content");<commit_msg>Use long for total size<commit_after>// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.application;

import com.google.common.collect.ImmutableList;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UncheckedIOException;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.List;
import java.util.function.Predicate;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

/**
 * @author bratseth
 */
public class ZipStreamReader {

    private final ImmutableList<ZipEntryWithContent> entries;
    private final int maxEntrySizeInBytes;

    public ZipStreamReader(InputStream input, Predicate<String> entryNameMatcher, int maxEntrySizeInBytes) {
        this.maxEntrySizeInBytes = maxEntrySizeInBytes;
        try (ZipInputStream zipInput = new ZipInputStream(input)) {
            ImmutableList.Builder<ZipEntryWithContent> builder = new ImmutableList.Builder<>();
            ZipEntry zipEntry;
            while (null != (zipEntry = zipInput.getNextEntry())) {
                if (!entryNameMatcher.test(requireName(zipEntry.getName()))) continue;
                builder.add(new ZipEntryWithContent(zipEntry, readContent(zipInput)));
            }
            entries = builder.build();
        } catch (IOException e) {
            throw new UncheckedIOException("IO error reading zip content", e);
        }
    }

    private byte[] readContent(ZipInputStream zipInput) {
        try (ByteArrayOutputStream bis = new ByteArrayOutputStream()) {
            byte[] buffer = new byte[2048];
            int read;
            long size = 0;
            while ( -1 != (read = zipInput.read(buffer))) {
                size += read;
                if (size > maxEntrySizeInBytes) {
                    throw new IllegalArgumentException("Entry in zip content exceeded size limit of " +
                                                       maxEntrySizeInBytes + " bytes");
                }
                bis.write(buffer, 0, read);
            }
            return bis.toByteArray();
        } catch (IOException e) {
            throw new UncheckedIOException("Failed reading from zipped content", e);
        }
    }

    public List<ZipEntryWithContent> entries() { return entries; }

    private static String requireName(String name) {
        IllegalArgumentException e = new IllegalArgumentException("Unexpected non-normalized path found in zip content");
{'fraction_non_alphanumeric': 0.0906781317178034, 'fraction_numerical': 0.005081102208325191, 'mean_word_length': 2.6169611307420495, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2775398', 'n_tokens_mistral': 1321, 'n_tokens_neox': 1252, 'n_words': 353}