<commit_before>package org.verapdf.integration;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.file.Path;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.Set;

import javax.xml.bind.JAXBException;

import org.junit.BeforeClass;
import org.junit.Test;
import org.verapdf.core.ValidationException;
import org.verapdf.model.ModelLoader;
import org.verapdf.pdfa.qa.CorpusDirectory;
import org.verapdf.pdfa.qa.CorpusItemId;
import org.verapdf.pdfa.qa.RuleDirectory;
import org.verapdf.pdfa.results.ValidationResult;
import org.verapdf.pdfa.results.ValidationResults;
import org.verapdf.pdfa.validation.ProfileDetails;
import org.verapdf.pdfa.validation.Profiles;
import org.verapdf.pdfa.validation.Rule;
import org.verapdf.pdfa.validation.RuleId;
import org.verapdf.pdfa.validation.ValidationProfile;
import org.verapdf.pdfa.validation.Validator;

public class ITVeraPDFTestSuite {
    private static RuleDirectory RULES;
    private static CorpusDirectory CORPUS;

    @BeforeClass
    public static void initialise() {
        try {
            RULES = RuleDirectory.loadFromDir(new File(
                    "../../veraPDF-validation-profiles/PDF_A/1b/"));
        } catch (IOException e) {
            e.printStackTrace();
            throw new IllegalStateException("Error loading rules", e);
        }
        try {
            CORPUS = CorpusDirectory.loadFromDir(new File(
                    "../../veraPDF-corpus/PDF_A-1b/"));
        } catch (IOException e) {
            e.printStackTrace();
            throw new IllegalStateException("Error loading corpus", e);
        }
    }

    @Test
    public void performValidation() throws URISyntaxException, IOException {
        for (RuleId ruleId : RULES.getKeys()) {
            Set<Rule> rules = new HashSet<>();
            rules.add(RULES.getItem(ruleId));
            System.out.println();
            System.out.println("RULE =============================================================");
            System.out.println("Rule: " + ruleId);
            for (CorpusItemId corpusId : CORPUS.getCorpusIdForRule(ruleId)){
                System.out.println();
                System.out.println("CORPUS -------------------------------------------------------");
                System.out.println("Corpus: " + corpusId);
                ProfileDetails details = Profiles.profileDetailsFromValues(CORPUS.getFlavour().getPart().getName(), CORPUS.getFlavour().getPart().getDescription(), "testsuite", new Date());
                ValidationProfile profile = Profiles.profileFromValues(CORPUS.getFlavour(), details, "", rules, Collections.EMPTY_SET);
                Path path = CORPUS.getItem(corpusId);
                try (ModelLoader loader = new ModelLoader(new FileInputStream(path.toFile()))) {
                    ValidationResult result = Validator.validate(profile, loader.getRoot(), false);
                    ValidationResults.toXml(result, System.out, Boolean.TRUE);
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }
    }
}<commit_msg>New ITs driven direct from GitHub.<commit_after>package org.verapdf.integration;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.Enumeration;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;

import org.junit.BeforeClass;
import org.junit.Test;
import org.verapdf.core.ProfileException;
import org.verapdf.model.ModelLoader;
import org.verapdf.pdfa.flavours.PDFAFlavour;
import org.verapdf.pdfa.results.ValidationResult;
import org.verapdf.pdfa.results.ValidationResults;
import org.verapdf.pdfa.validation.ValidationProfile;
import org.verapdf.pdfa.validation.Validator;
import org.verapdf.validation.profile.parser.LegacyProfileConverter;

public class ITVeraPDFTestSuite {
    private static ValidationProfile PROFILE_1B;

    @BeforeClass
    public static void initialise() throws ProfileException {
        try {
            URL profileURL = new URL(
                    "https://raw.githubusercontent.com/veraPDF/veraPDF-validation-profiles/integration/PDF_A/PDFA-1B.xml");
            PROFILE_1B = LegacyProfileConverter.fromLegacyStream(
                    profileURL.openStream(), PDFAFlavour.PDFA_1_B);
            
        } catch (IOException e) {
            e.printStackTrace();
            throw new IllegalStateException("Error loading corpus", e);
        }
    }

    @Test
    public void performValidation() throws URISyntaxException, IOException {
        URL corpusURL = new URL(
                "https://github.com/veraPDF/veraPDF-corpus/archive/staging.zip");
        File tempZip = File.createTempFile("veraCorpus", "zip");
        try (OutputStream output = new FileOutputStream(tempZip);
                InputStream corpusInput = corpusURL.openStream();) {
            byte[] buffer = new byte[8 * 1024];
            try {
                int bytesRead;
                while ((bytesRead = corpusInput.read(buffer)) != -1) {
                    output.write(buffer, 0, bytesRead);
                }
            } finally {
                output.close();
            }
        }
        ZipFile zipIn = new ZipFile(tempZip);
        Enumeration<? extends ZipEntry> entries = zipIn.entries();
        while (entries.hasMoreElements()) {
            ZipEntry entry = entries.nextElement();
            if (entry.isDirectory())
                continue;
            if (entry.getName().endsWith(".pdf")) {
                try (ModelLoader loader = new ModelLoader(
                        zipIn.getInputStream(entry))) {
                    ValidationResult result = Validator.validate(PROFILE_1B,
                            loader.getRoot(), false);
                    System.out.println();
                    System.out.println(entry.getName());
                    ValidationResults.toXml(result, System.out, Boolean.TRUE);
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }
    }
}
{'fraction_non_alphanumeric': 0.124282982791587, 'fraction_numerical': 0.0022307202039515616, 'mean_word_length': 2.3856526429341964, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2894539', 'n_tokens_mistral': 1767, 'n_tokens_neox': 1682, 'n_words': 321}