<commit_before>package org.zeroxlab.momodict.archive;

import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.io.FileUtils;

import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;

import rx.functions.Action0;

public class TarExtractor implements Extractor {

    @Override
    public FileSet extract(final File outputDir, InputStream inputStream) throws Exception {
        FileSet archive = new FileSet();
        archive.setCleanCallback(new Action0() {
            @Override
            public void call() {
                try {
                    if (outputDir.isDirectory()) {
                        FileUtils.deleteDirectory(outputDir);
                    }
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        });

        TarArchiveInputStream tis = new TarArchiveInputStream(inputStream);
        while (tis.getNextEntry() != null) {
            onEntryFound(outputDir, tis, tis.getCurrentEntry(), archive);
        }

        if (!archive.isSane()) {
            throw new Exception("RealmBook is malformed");
        }
        return archive;
    }

    private void onEntryFound(File parent,
                              InputStream is,
                              TarArchiveEntry entry,
                              FileSet archive)
            throws Exception {

        final String fileName = entry.getName();
        if (entry.isDirectory()) {
            File dir = new File(parent, fileName);
            if (!dir.mkdirs()) {
                throw new Exception("Create dir fail");
            }
        } else {
            // TODO: support .dz file. issue #2
            if (fileName.endsWith(".dz")) {
                StringBuilder msg = new StringBuilder();
                msg.append("Haven't support .dz (dictzip) file so far.\n");
                msg.append("please provide '.dict' to instead of '.dict.dz'");
                throw new RuntimeException(msg.toString());
            }
            // write file
            File out = new File(parent, fileName);

            byte[] buf = new byte[65535];
            FileOutputStream fos = new FileOutputStream(out);
            BufferedOutputStream bos = new BufferedOutputStream(fos);
            int count;
            while ((count = is.read(buf)) != -1) {
                bos.write(buf, 0, count);
            }
            bos.close();

            // cache file path
            if (fileName.endsWith(".idx")) {
                archive.set(FileSet.Type.IDX, out.getAbsolutePath());
            } else if (fileName.endsWith(".ifo")) {
                archive.set(FileSet.Type.IFO, out.getAbsolutePath());
            } else if (fileName.endsWith(".dict.dz") || fileName.endsWith(".dict")) {
                archive.set(FileSet.Type.DICT, out.getAbsolutePath());
            }
        }
        System.out.println(entry.getName());
    }
}
<commit_msg>fix #2: extract dz file by using Gzip stream<commit_after>package org.zeroxlab.momodict.archive;

import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.io.FileUtils;

import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;

import rx.functions.Action0;

public class TarExtractor implements Extractor {

    @Override
    public FileSet extract(final File outputDir, InputStream inputStream) throws Exception {
        FileSet archive = new FileSet();
        archive.setCleanCallback(new Action0() {
            @Override
            public void call() {
                try {
                    if (outputDir.isDirectory()) {
                        FileUtils.deleteDirectory(outputDir);
                    }
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        });

        TarArchiveInputStream tis = new TarArchiveInputStream(inputStream);
        while (tis.getNextEntry() != null) {
            onEntryFound(outputDir, tis, tis.getCurrentEntry(), archive);
        }

        if (!archive.isSane()) {
            throw new Exception("RealmBook is malformed");
        }
        return archive;
    }

    private void onEntryFound(File parent,
                              InputStream is,
                              TarArchiveEntry entry,
                              FileSet archive)
            throws Exception {

        final String fileName = entry.getName();
        if (entry.isDirectory()) {
            File dir = new File(parent, fileName);
            if (!dir.mkdirs()) {
                throw new Exception("Create dir fail");
            }
        } else {
            // write file
            File out = new File(parent, fileName);

            byte[] buf = new byte[65535];
            FileOutputStream fos = new FileOutputStream(out);
            BufferedOutputStream bos = new BufferedOutputStream(fos);
            int count;
            while ((count = is.read(buf)) != -1) {
                bos.write(buf, 0, count);
            }
            bos.close();

            // cache file path
            if (fileName.endsWith(".idx")) {
                archive.set(FileSet.Type.IDX, out.getAbsolutePath());
            } else if (fileName.endsWith(".ifo")) {
                archive.set(FileSet.Type.IFO, out.getAbsolutePath());
            } else if (fileName.endsWith(".dict.dz")) {
                // found a dz file, use gzip to extract again
                String extractFileName = out.getAbsolutePath().replace(".dict.dz", ".dict");
                FileInputStream fis = new FileInputStream(out);
                GzipCompressorInputStream gis = new GzipCompressorInputStream(fis);
                FileOutputStream fos2 = new FileOutputStream(extractFileName);
                byte[] buffer = new byte[2048];
                int r = 0;
                while ((r = gis.read(buffer))!= -1) {
                    fos2.write(buffer, 0, r);
                }
                fis.close();
                fos2.close();
                archive.set(FileSet.Type.DICT, extractFileName);
            } else if (fileName.endsWith(".dict")) {
                archive.set(FileSet.Type.DICT, out.getAbsolutePath());
            }
        }
        System.out.println(entry.getName());
    }
}

{'fraction_non_alphanumeric': 0.10664283586535597, 'fraction_numerical': 0.004468275245755138, 'mean_word_length': 1.7385807504078303, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3955969', 'n_tokens_mistral': 1746, 'n_tokens_neox': 1681, 'n_words': 390}