<commit_before>/* This file is part of VoltDB.
 * Copyright (C) 2008-2010 VoltDB Inc.
 *
 * VoltDB is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * VoltDB is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with VoltDB.  If not, see <http://www.gnu.org/licenses/>.
 */

package org.voltdb.sysprocs.saverestore;

import java.io.BufferedInputStream;
import java.io.CharArrayWriter;
import java.io.EOFException;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.zip.CRC32;

import org.voltdb.VoltDB;
import org.voltdb.ExecutionSite.SystemProcedureExecutionContext;
import org.voltdb.catalog.*;
import org.voltdb.utils.CatalogUtil;
import org.voltdb.utils.Pair;
import org.voltdb.utils.DBBPool.BBContainer;

public class SnapshotUtil {

    /**
     * Create a digest for a snapshot containing the time of the snapshot and the list of tables included.
     * The first item in the comma separated list is the time in milliseconds as a string.
     * @param snapshotTime
     * @param path
     * @param nonce
     * @param tables
     * @throws IOException
     */
    public static void
        recordSnapshotTableList(
            long snapshotTime,
            String path,
            String nonce,
            List<Table> tables) throws IOException {
        final File f = new File(path, constructDigestFilenameForNonce(nonce));
        if (f.exists()) {
            if (!f.delete()) {
                throw new IOException("Unable to write table list file " + f);
            }
        }
        FileOutputStream fos = new FileOutputStream(f);
        StringWriter sw = new StringWriter();
        sw.append(Long.toString(snapshotTime));
        if (!tables.isEmpty()) {
            sw.append(',');
        }
        for (int ii = 0; ii < tables.size(); ii++) {
            sw.append(tables.get(ii).getTypeName());
            if (!(ii == (tables.size() - 1))) {
                sw.append(',');
            } else {
                sw.append('\n');
            }
        }

        final byte tableListBytes[] = sw.getBuffer().toString().getBytes("UTF-8");
        final CRC32 crc = new CRC32();
        crc.update(tableListBytes);
        ByteBuffer fileBuffer = ByteBuffer.allocate(tableListBytes.length + 4);
        fileBuffer.putInt((int)crc.getValue());
        fileBuffer.put(tableListBytes);
        fileBuffer.flip();
        fos.getChannel().write(fileBuffer);
        fos.getChannel().force(false);
        fos.close();
    }

    public static List<String> retrieveRelevantTableNames(String path,
            String nonce) throws Exception {
        return retrieveRelevantTableNamesAndTime(new File(path, constructDigestFilenameForNonce(nonce))).getSecond();
    }

    /**
     * Retrieve a list of tables from a digest. Doesn't return the snapshot time
     * value that is stashed in the digest file.
     * @param f
     * @throws Exception
     */
    public static Pair<Long, List<String>> retrieveRelevantTableNamesAndTime(File f) throws Exception {
        String tableList = CRCCheck(f);
        String tableNames[] = tableList.split(",");
        String actualTableNames[] = new String[tableNames.length - 1];
        System.arraycopy( tableNames, 1, actualTableNames, 0, tableNames.length - 1);
        return Pair.of(Long.valueOf(tableNames[0]),
                       java.util.Arrays.asList(actualTableNames));
    }

    /**
     * Check if the CRC of the snapshot file matches the digest.
     * @param f The snapshot file object
     * @return The table list as a string
     * @throws IOException If CRC does not match
     */
    public static String CRCCheck(File f) throws IOException {
        final FileInputStream fis = new FileInputStream(f);
        try {
            final BufferedInputStream bis = new BufferedInputStream(fis);
            ByteBuffer crcBuffer = ByteBuffer.allocate(4);
            if (4 != bis.read(crcBuffer.array())) {
                throw new EOFException("EOF while attempting to read CRC from snapshot digest");
            }
            final int crc = crcBuffer.getInt();
            final InputStreamReader isr = new InputStreamReader(bis, "UTF-8");
            CharArrayWriter caw = new CharArrayWriter();
            while (true) {
                int nextChar = isr.read();<commit_msg>Change the comment of the CRCCheck() method in SnapshotUtil to be more accurate.<commit_after>/* This file is part of VoltDB.
 * Copyright (C) 2008-2010 VoltDB Inc.
 *
 * VoltDB is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * VoltDB is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with VoltDB.  If not, see <http://www.gnu.org/licenses/>.
 */

package org.voltdb.sysprocs.saverestore;

import java.io.BufferedInputStream;
import java.io.CharArrayWriter;
import java.io.EOFException;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.zip.CRC32;

import org.voltdb.VoltDB;
import org.voltdb.ExecutionSite.SystemProcedureExecutionContext;
import org.voltdb.catalog.*;
import org.voltdb.utils.CatalogUtil;
import org.voltdb.utils.Pair;
import org.voltdb.utils.DBBPool.BBContainer;

public class SnapshotUtil {

    /**
     * Create a digest for a snapshot containing the time of the snapshot and the list of tables included.
     * The first item in the comma separated list is the time in milliseconds as a string.
     * @param snapshotTime
     * @param path
     * @param nonce
     * @param tables
     * @throws IOException
     */
    public static void
        recordSnapshotTableList(
            long snapshotTime,
            String path,
            String nonce,
            List<Table> tables) throws IOException {
        final File f = new File(path, constructDigestFilenameForNonce(nonce));
        if (f.exists()) {
            if (!f.delete()) {
                throw new IOException("Unable to write table list file " + f);
            }
        }
        FileOutputStream fos = new FileOutputStream(f);
        StringWriter sw = new StringWriter();
        sw.append(Long.toString(snapshotTime));
        if (!tables.isEmpty()) {
            sw.append(',');
        }
        for (int ii = 0; ii < tables.size(); ii++) {
            sw.append(tables.get(ii).getTypeName());
            if (!(ii == (tables.size() - 1))) {
                sw.append(',');
            } else {
                sw.append('\n');
            }
        }

        final byte tableListBytes[] = sw.getBuffer().toString().getBytes("UTF-8");
        final CRC32 crc = new CRC32();
        crc.update(tableListBytes);
        ByteBuffer fileBuffer = ByteBuffer.allocate(tableListBytes.length + 4);
        fileBuffer.putInt((int)crc.getValue());
        fileBuffer.put(tableListBytes);
        fileBuffer.flip();
        fos.getChannel().write(fileBuffer);
        fos.getChannel().force(false);
        fos.close();
    }

    public static List<String> retrieveRelevantTableNames(String path,
            String nonce) throws Exception {
        return retrieveRelevantTableNamesAndTime(new File(path, constructDigestFilenameForNonce(nonce))).getSecond();
    }

    /**
     * Retrieve a list of tables from a digest. Doesn't return the snapshot time
     * value that is stashed in the digest file.
     * @param f
     * @throws Exception
     */
    public static Pair<Long, List<String>> retrieveRelevantTableNamesAndTime(File f) throws Exception {
        String tableList = CRCCheck(f);
        String tableNames[] = tableList.split(",");
        String actualTableNames[] = new String[tableNames.length - 1];
        System.arraycopy( tableNames, 1, actualTableNames, 0, tableNames.length - 1);
        return Pair.of(Long.valueOf(tableNames[0]),
                       java.util.Arrays.asList(actualTableNames));
    }

    /**
     * Check if the CRC of the snapshot digest. Note that this only checks if
     * the CRC at the beginning of the digest file matches the CRC of the digest
     * file itself.
     *
     * @param f
     *            The snapshot digest file object
     * @return The table list as a string
     * @throws IOException
     *             If CRC does not match
     */
    public static String CRCCheck(File f) throws IOException {
        final FileInputStream fis = new FileInputStream(f);
        try {
            final BufferedInputStream bis = new BufferedInputStream(fis);
            ByteBuffer crcBuffer = ByteBuffer.allocate(4);
            if (4 != bis.read(crcBuffer.array())) {
                throw new EOFException("EOF while attempting to read CRC from snapshot digest");
            }
            final int crc = crcBuffer.getInt();
            final InputStreamReader isr = new InputStreamReader(bis, "UTF-8");
            CharArrayWriter caw = new CharArrayWriter();
            while (true) {
                int nextChar = isr.read();
{'fraction_non_alphanumeric': 0.09776063229205871, 'fraction_numerical': 0.005080918328942416, 'mean_word_length': 2.6588640275387263, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6815211', 'n_tokens_mistral': 2938, 'n_tokens_neox': 2881, 'n_words': 926}