<commit_before>/*  
 * Copyright 2008 CoreMedia AG, Hamburg
 *
 * Licensed under the Apache License, Version 2.0 (the License); 
 * you may not use this file except in compliance with the License. 
 * You may obtain a copy of the License at 
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0 
 * 
 * Unless required by applicable law or agreed to in writing, software 
 * distributed under the License is distributed on an AS IS BASIS, 
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
 * See the License for the specific language governing permissions and 
 * limitations under the License. 
 */

package com.coremedia.iso;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.List;

/**
 * A <code>FilterInputStream</code> enriched with helper methods to ease writing of
 * Iso specific numbers and strings.
 */
public class IsoBufferWrapper {
    ByteBuffer[] parents;
    int activeParent = 0;

    public IsoBufferWrapper(ByteBuffer parent) {
        this.parents = new ByteBuffer[]{parent};
    }

    public IsoBufferWrapper(ByteBuffer[] parents) {
        this.parents = parents;
    }

    public IsoBufferWrapper(List<ByteBuffer> parents) {
        this.parents = parents.toArray(new ByteBuffer[parents.size()]);
    }

    public IsoBufferWrapper(File file) throws IOException {
        long filelength = file.length();
        int sliceSize = Integer.MAX_VALUE;

        RandomAccessFile raf = new RandomAccessFile(file, "r");
        ArrayList<ByteBuffer> buffers = new ArrayList<ByteBuffer>();
        long i = 0;
        while (i < filelength) {
            if ((filelength - i) > sliceSize) {
                buffers.add(raf.getChannel().map(FileChannel.MapMode.READ_ONLY, i, sliceSize).slice());
                i += sliceSize;
            } else {
                buffers.add(raf.getChannel().map(FileChannel.MapMode.READ_ONLY, i, filelength - i).slice());
                i += filelength - i;
            }
        }
        parents = buffers.toArray(new ByteBuffer[buffers.size()]);
        raf.close();
    }

    public long position() {
        if (activeParent >= 0) {
            long pos = 0;
            for (int i = 0; i < activeParent; i++) {
                pos += parents[i].limit();
            }
            pos += parents[activeParent].position();
            return pos;
        } else {
            return size();
        }
    }
<commit_msg>Don't use 2Gb buffers. user smaller buffers.  IsoViewer now uses IsoBufferWrapper(file) constructor instead of splitting file by itself.<commit_after>/*  
 * Copyright 2008 CoreMedia AG, Hamburg
 *
 * Licensed under the Apache License, Version 2.0 (the License); 
 * you may not use this file except in compliance with the License. 
 * You may obtain a copy of the License at 
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0 
 * 
 * Unless required by applicable law or agreed to in writing, software 
 * distributed under the License is distributed on an AS IS BASIS, 
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
 * See the License for the specific language governing permissions and 
 * limitations under the License. 
 */

package com.coremedia.iso;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.List;

/**
 * A <code>FilterInputStream</code> enriched with helper methods to ease writing of
 * Iso specific numbers and strings.
 */
public class IsoBufferWrapper {
    ByteBuffer[] parents;
    int activeParent = 0;

    public IsoBufferWrapper(ByteBuffer parent) {
        this.parents = new ByteBuffer[]{parent};
    }

    public IsoBufferWrapper(ByteBuffer[] parents) {
        this.parents = parents;
    }

    public IsoBufferWrapper(List<ByteBuffer> parents) {
        this.parents = parents.toArray(new ByteBuffer[parents.size()]);
    }

    public IsoBufferWrapper(File file) throws IOException {
        long filelength = file.length();
        int sliceSize = 1024 * 1024 * 128;

        RandomAccessFile raf = new RandomAccessFile(file, "r");
        ArrayList<ByteBuffer> buffers = new ArrayList<ByteBuffer>();
        long i = 0;
        while (i < filelength) {
            if ((filelength - i) > sliceSize) {
                ByteBuffer bb;
                try {
                    bb = raf.getChannel().map(FileChannel.MapMode.READ_ONLY, i, sliceSize);//.slice();
                } catch (IOException e1) {

                    try {
                        bb = raf.getChannel().map(FileChannel.MapMode.READ_ONLY, i, sliceSize);//.slice();
                    } catch (IOException e2) {

                        try {
                            bb = raf.getChannel().map(FileChannel.MapMode.READ_ONLY, i, sliceSize);//.slice();
                        } catch (IOException e3) {
                            bb = raf.getChannel().map(FileChannel.MapMode.READ_ONLY, i, sliceSize);//.slice();
                        }
                    }
                }
                buffers.add(bb);
                i += sliceSize;
                System.err.println(i / 1024 / 1024);
            } else {
                buffers.add(raf.getChannel().map(FileChannel.MapMode.READ_ONLY, i, filelength - i).slice());
                i += filelength - i;
            }
        }
        parents = buffers.toArray(new ByteBuffer[buffers.size()]);
        raf.close();
    }

    public long position() {
        if (activeParent >= 0) {
            long pos = 0;
            for (int i = 0; i < activeParent; i++) {
                pos += parents[i].limit();
            }
            pos += parents[activeParent].position();
            return pos;
        } else {
            return size();
        }
    }

{'fraction_non_alphanumeric': 0.10387065164135229, 'fraction_numerical': 0.008002613098154499, 'mean_word_length': 2.214698162729659, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7553928', 'n_tokens_mistral': 1709, 'n_tokens_neox': 1603, 'n_words': 482}