<commit_before>package net.q3aiml.dbdata.cli;

import com.github.rvesse.airline.Arguments;
import com.github.rvesse.airline.Command;
import com.google.common.io.Files;
import com.google.common.io.Resources;
import net.q3aiml.dbdata.DataSpec;
import net.q3aiml.dbdata.config.DatabaseConfig;
import net.q3aiml.dbdata.introspect.DefaultDatabaseIntrospector;
import net.q3aiml.dbdata.verify.VerificationError;
import net.q3aiml.dbdata.verify.Verifier;

import javax.sql.DataSource;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.sql.SQLException;
import java.util.List;
import java.util.concurrent.Callable;

@Command(name = "verify", description = "Verify database matches data in a file")
public class VerifyCli implements Callable<Void> {
    SqlConnectionOption sqlConnectionOption = new SqlConnectionOption();

    @Arguments
    public File file;

    @Override
    public Void call() throws IOException, SQLException {
        String yaml;
        try {
            yaml = Files.toString(file, Charset.defaultCharset());
        } catch (IOException e) {
            throw new IOException("error reading data file " + file.getAbsolutePath() + ": " + e.getMessage(), e);
        }
        DataSpec dataSpec = DataSpec.fromYaml(yaml);

        DataSource dataSource = sqlConnectionOption.dataSource();

        DatabaseConfig serializeConfig = DatabaseConfig
                .fromYaml(Resources.toString(Resources.getResource("config.yaml"), StandardCharsets.UTF_8));

        DefaultDatabaseIntrospector introspector = new DefaultDatabaseIntrospector(serializeConfig.schema);<commit_msg>fix verify<commit_after>package net.q3aiml.dbdata.cli;

import com.github.rvesse.airline.Arguments;
import com.github.rvesse.airline.Command;
import com.google.common.io.Files;
import com.google.common.io.Resources;
import net.q3aiml.dbdata.DataSpec;
import net.q3aiml.dbdata.config.DatabaseConfig;
import net.q3aiml.dbdata.introspect.DefaultDatabaseIntrospector;
import net.q3aiml.dbdata.verify.VerificationError;
import net.q3aiml.dbdata.verify.Verifier;

import javax.inject.Inject;
import javax.sql.DataSource;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.sql.SQLException;
import java.util.List;
import java.util.concurrent.Callable;

@Command(name = "verify", description = "Verify database matches data in a file")
public class VerifyCli implements Callable<Void> {
    @Inject
    SqlConnectionOption sqlConnectionOption = new SqlConnectionOption();

    @Arguments
    public File file;

    @Override
    public Void call() throws IOException, SQLException {
        String yaml;
        try {
            yaml = Files.toString(file, Charset.defaultCharset());
        } catch (IOException e) {
            throw new IOException("error reading data file " + file.getAbsolutePath() + ": " + e.getMessage(), e);
        }
        DataSpec dataSpec = DataSpec.fromYaml(yaml);

        DataSource dataSource = sqlConnectionOption.dataSource();

        DatabaseConfig serializeConfig = DatabaseConfig
                .fromYaml(Resources.toString(Resources.getResource("config.yaml"), StandardCharsets.UTF_8));

        DefaultDatabaseIntrospector introspector = new DefaultDatabaseIntrospector(serializeConfig.schema);
{'fraction_non_alphanumeric': 0.09961685823754789, 'fraction_numerical': 0.00412614205717654, 'mean_word_length': 4.229583975346687, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1210613', 'n_tokens_mistral': 988, 'n_tokens_neox': 965, 'n_words': 201}