<commit_before>package org.yi.happy.archive;

import java.io.InputStream;
import java.io.PrintStream;
import java.util.List;

import org.yi.happy.annotate.DuplicatedLogic;
import org.yi.happy.archive.block.BlobEncodedBlock;
import org.yi.happy.archive.block.ContentEncodedBlock;
import org.yi.happy.archive.block.EncodedBlock;
import org.yi.happy.archive.block.parser.EncodedBlockParse;
import org.yi.happy.archive.commandLine.UsesArgs;
import org.yi.happy.archive.commandLine.UsesInput;
import org.yi.happy.archive.commandLine.UsesOutput;
import org.yi.happy.archive.crypto.DigestFactory;
import org.yi.happy.archive.crypto.DigestProvider;
import org.yi.happy.archive.crypto.Digests;

/**
 * Check that the files on a volume are still readable. Do this by building a
 * new index based on the file names of an existing index.
 */
@UsesArgs({ "image-path" })
@UsesInput("index")
@UsesOutput("index")
public class IndexCheckMain implements MainCommand {
    private final FileStore fs;
    private final InputStream in;
    private final PrintStream out;
    private final PrintStream err;
    private final List<String> args;

    /**
     * setup the command.
     * 
     * @param fs
     *            the file system to use.
     * @param in
     *            the input stream to use.
     * @param out
     *            the output stream to use.
     * @param err
     *            the error stream to use.
     * @param args
     *            the arguments to use.
     */
    public IndexCheckMain(FileStore fs, InputStream in, PrintStream out,
            PrintStream err, List<String> args) {
        this.fs = fs;
        this.in = in;
        this.out = out;
        this.err = err;
        this.args = args;
    }

    @Override
    @DuplicatedLogic("with IndexVolumeMain.process, and internally")
    public void run() throws Exception {
        String imagePath = args.get(0);

        DigestProvider digest = DigestFactory.getProvider("sha-256");

        String prevName = null;
        for (String line : new LineIterator(in)) {
            try {
                String name = line.split("\t")[0];
                String path = imagePath + "/" + name;
                if (name.equals(prevName)) {
                    continue;
                }
                prevName = name;

                byte[] data = fs.get(path, Blocks.MAX_SIZE);
                EncodedBlock block = EncodedBlockParse.parse(data);

                String key = block.getKey().toString();

                data = block.asBytes();

                String hash = Base16.encode(Digests.digestData(digest, data));
                String size = Integer.toString(data.length);

                out.println(name + "\t" + "plain" + "\t" + key + "\t" + hash
                        + "\t" + size);

                /*
                 * to-blob
                 */
                if (block instanceof ContentEncodedBlock) {
                    block = new BlobEncodedBlock(block.getDigest(),
                            block.getCipher(), block.getBody());

                    key = block.getKey().toString();

                    data = block.asBytes();

                    hash = Base16.encode(Digests.digestData(digest, data));
                    size = Integer.toString(data.length);

                    out.println(name + "\t" + "to-blob" + "\t" + key + "\t"
                            + hash + "\t" + size);
                }
            } catch (Exception e) {<commit_msg>friendlier IOError message in index-check<commit_after>package org.yi.happy.archive;

import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.util.List;

import org.yi.happy.annotate.DuplicatedLogic;
import org.yi.happy.archive.block.BlobEncodedBlock;
import org.yi.happy.archive.block.ContentEncodedBlock;
import org.yi.happy.archive.block.EncodedBlock;
import org.yi.happy.archive.block.parser.EncodedBlockParse;
import org.yi.happy.archive.commandLine.UsesArgs;
import org.yi.happy.archive.commandLine.UsesInput;
import org.yi.happy.archive.commandLine.UsesOutput;
import org.yi.happy.archive.crypto.DigestFactory;
import org.yi.happy.archive.crypto.DigestProvider;
import org.yi.happy.archive.crypto.Digests;

/**
 * Check that the files on a volume are still readable. Do this by building a
 * new index based on the file names of an existing index.
 */
@UsesArgs({ "image-path" })
@UsesInput("index")
@UsesOutput("index")
public class IndexCheckMain implements MainCommand {
    private final FileStore fs;
    private final InputStream in;
    private final PrintStream out;
    private final PrintStream err;
    private final List<String> args;

    /**
     * setup the command.
     * 
     * @param fs
     *            the file system to use.
     * @param in
     *            the input stream to use.
     * @param out
     *            the output stream to use.
     * @param err
     *            the error stream to use.
     * @param args
     *            the arguments to use.
     */
    public IndexCheckMain(FileStore fs, InputStream in, PrintStream out,
            PrintStream err, List<String> args) {
        this.fs = fs;
        this.in = in;
        this.out = out;
        this.err = err;
        this.args = args;
    }

    @Override
    @DuplicatedLogic("with IndexVolumeMain.process, and internally")
    public void run() throws Exception {
        String imagePath = args.get(0);

        DigestProvider digest = DigestFactory.getProvider("sha-256");

        String prevName = null;
        for (String line : new LineIterator(in)) {
            try {
                String name = line.split("\t")[0];
                String path = imagePath + "/" + name;
                if (name.equals(prevName)) {
                    continue;
                }
                prevName = name;

                byte[] data = fs.get(path, Blocks.MAX_SIZE);
                EncodedBlock block = EncodedBlockParse.parse(data);

                String key = block.getKey().toString();

                data = block.asBytes();

                String hash = Base16.encode(Digests.digestData(digest, data));
                String size = Integer.toString(data.length);

                out.println(name + "\t" + "plain" + "\t" + key + "\t" + hash
                        + "\t" + size);

                /*
                 * to-blob
                 */
                if (block instanceof ContentEncodedBlock) {
                    block = new BlobEncodedBlock(block.getDigest(),
                            block.getCipher(), block.getBody());

                    key = block.getKey().toString();

                    data = block.asBytes();

                    hash = Base16.encode(Digests.digestData(digest, data));
                    size = Integer.toString(data.length);

                    out.println(name + "\t" + "to-blob" + "\t" + key + "\t"
                            + hash + "\t" + size);
                }
            } catch (IOException e) {
                err.println("Unable to read: " + prevName);
            } catch (Exception e) {
{'fraction_non_alphanumeric': 0.11001410437235543, 'fraction_numerical': 0.002538787023977433, 'mean_word_length': 1.8364, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3424362', 'n_tokens_mistral': 2053, 'n_tokens_neox': 1912, 'n_words': 491}