<commit_before>package uk.gov.register.functional;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.Before;
import org.junit.Test;

import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.*;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

import static org.hamcrest.CoreMatchers.*;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertTrue;

public class DataDownloadFunctionalTest extends FunctionalTestBase {

    private final String item1 = "{\"address\":\"12345\",\"street\":\"ellis\"}";
    private final String item2 = "{\"address\":\"6789\",\"street\":\"presley\"}";
    private final String item3 =  "{\"address\":\"12345\",\"street\":\"foo\"}";
    private final String item4 =  "{\"address\":\"145678\",\"street\":\"ellis\"}";

    @Before
    public void publishTestMessages() {
        mintItems(item1, item2, item3, item4, item1);
    }

    @Test
    public void downloadRegister_shouldReturnAZipfile() throws IOException {
        Response response = getRequest("/download-register");

        assertThat(response.getHeaderString("Content-Type"), equalTo(MediaType.APPLICATION_OCTET_STREAM));
        assertThat(response.getHeaderString("Content-Disposition"), startsWith("attachment; filename="));
        assertThat(response.getHeaderString("Content-Disposition"), endsWith(".zip"));

        InputStream is = response.readEntity(InputStream.class);
        Set<String> zipEntryNames = getEntries(is).keySet();

        assertThat(zipEntryNames, hasItem("register.json"));
        assertThat(zipEntryNames.stream().filter(e -> e.matches("(entry/)(\\d)(.json)")).count(), is(5L));
        assertThat(zipEntryNames.stream().filter(e -> e.matches("(item/)(\\w+)(.json)")).count(), is(4L));
    }

    @Test
    public void downloadRegister_shouldUseCorrectEntryAndItemJsonFormat() throws IOException {
        Response response = getRequest("/download-register");
        InputStream is = response.readEntity(InputStream.class);

        List<String> itemJson = getEntries(is).entrySet().stream()
                .filter(j -> j.getKey().startsWith("item"))
                .map(j -> j.getValue().toString())
                .collect(Collectors.toList());

        assertThat(itemJson, hasItems(item1, item2, item3, item4));

        List<JsonNode> entryJson = getEntries(is).entrySet().stream()
                .filter(j -> j.getKey().startsWith("entry"))
                .map(Map.Entry::getValue)
                .collect(Collectors.toList());

        entryJson.forEach(j -> {
            assertTrue(j.has("entry-number"));
            assertTrue(j.has("entry-timestamp"));
            assertTrue(j.has("item-hash"));
        });
    }

    @Test
    public void downloadRegister_shouldUseCorrectRegisterJsonFormat() throws IOException {
        Response response = getRequest("/download-register");
        InputStream is = response.readEntity(InputStream.class);

        JsonNode registerJson = getEntries(is).get("register.json");

        assertThat(registerJson.get("total-entries").asInt(), is(5));
        assertThat(registerJson.get("total-records").asInt(), is(3));
        assertTrue(registerJson.has("last-updated"));
        assertTrue(registerJson.has("domain"));
        assertTrue(registerJson.has("register-record"));
    }

    @Test
    public void downloadRSF_shouldReturnRegisterAsRsfStream() throws IOException {
        Response response = getRequest("/download-rsf");

        assertThat(response.getHeaderString("Content-Type"), equalTo(MediaType.APPLICATION_OCTET_STREAM));
        assertThat(response.getHeaderString("Content-Disposition"), startsWith("attachment; filename="));
        assertThat(response.getHeaderString("Content-Disposition"), endsWith(".tsv"));

        String[] rsfLines = getRsfLinesFrom(response);

        assertThat(rsfLines[0], equalTo("add-item\t{\"address\":\"12345\",\"street\":\"ellis\"}"));
        assertThat(rsfLines[1], equalTo("add-item\t{\"address\":\"6789\",\"street\":\"presley\"}"));
        assertThat(rsfLines[2], equalTo("add-item\t{\"address\":\"12345\",\"street\":\"foo\"}"));
        assertThat(rsfLines[3], equalTo("add-item\t{\"address\":\"145678\",\"street\":\"ellis\"}"));

        assertFormattedEntry(rsfLines[4], "sha-256:19205fafe65406b9b27fce1b689abc776df4ddcf150c28b29b73b4ea054af6b9");
        assertFormattedEntry(rsfLines[5], "sha-256:bd239db51960376826b937a615f0f3397485f00611d35bb7e951e357bf73b934");
        assertFormattedEntry(rsfLines[6], "sha-256:cc8a7c42275c84b94c6e282ae88b3dbcc06319156fc4539a2f39af053bf30592");
        assertFormattedEntry(rsfLines[7], "sha-256:8ac926428ee49fb83c02bdd2556e62e84cfd9e636cd35eb1306ac8cb661e4983");
        assertFormattedEntry(rsfLines[8], "sha-256:19205fafe65406b9b27fce1b689abc776df4ddcf150c28b29b73b4ea054af6b9");
    }

    @Test
    public void downloadPartialRSF_shouldReturnAPartOfRegisterAsRsfStream() throws IOException {
        Response response = getRequest("/download-rsf/3/4");

        assertThat(response.getHeaderString("Content-Type"), equalTo(MediaType.APPLICATION_OCTET_STREAM));
        assertThat(response.getHeaderString("Content-Disposition"), startsWith("attachment; filename="));
        assertThat(response.getHeaderString("Content-Disposition"), endsWith(".tsv"));

        String[] rsfLines = getRsfLinesFrom(response);

        assertThat(rsfLines[0], equalTo("add-item\t{\"address\":\"12345\",\"street\":\"foo\"}"));
        assertThat(rsfLines[1], equalTo("add-item\t{\"address\":\"145678\",\"street\":\"ellis\"}"));

        assertFormattedEntry(rsfLines[2], "sha-256:cc8a7c42275c84b94c6e282ae88b3dbcc06319156fc4539a2f39af053bf30592");
        assertFormattedEntry(rsfLines[3], "sha-256:8ac926428ee49fb83c02bdd2556e62e84cfd9e636cd35eb1306ac8cb661e4983");
    }

    private String[] getRsfLinesFrom(Response response){
        InputStream is = response.readEntity(InputStream.class);
        return new BufferedReader(new InputStreamReader(is)).lines().toArray(String[]::new);
    }

    private void assertFormattedEntry(String actualEntry, String expectedEntrySha){
        assertThat(actualEntry, startsWith("append-entry"));
        assertThat(actualEntry, endsWith(expectedEntrySha));
    }

    private Map<String, JsonNode> getEntries(InputStream inputStream) throws IOException {
        ZipInputStream zis = new ZipInputStream(inputStream);

        Map<String, JsonNode> entries = new HashMap<>();
        byte[] buffer = new byte[1024];
        int read = 0;
        for (ZipEntry entry; (entry = zis.getNextEntry()) != null; ) {
            StringBuilder sb = new StringBuilder();
            while ((read = zis.read(buffer, 0, 1024)) >= 0) {
                sb.append(new String(buffer, 0, read));
            }

            entries.put(entry.getName(), new ObjectMapper().readTree(sb.toString()));
        }

        return entries;
    }<commit_msg>Add functional test rejecting invalid boundaries for rsf<commit_after>package uk.gov.register.functional;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.Before;
import org.junit.Test;

import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.*;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

import static org.hamcrest.CoreMatchers.*;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertTrue;

public class DataDownloadFunctionalTest extends FunctionalTestBase {

    private final String item1 = "{\"address\":\"12345\",\"street\":\"ellis\"}";
    private final String item2 = "{\"address\":\"6789\",\"street\":\"presley\"}";
    private final String item3 =  "{\"address\":\"12345\",\"street\":\"foo\"}";
    private final String item4 =  "{\"address\":\"145678\",\"street\":\"ellis\"}";

    @Before
    public void publishTestMessages() {
        mintItems(item1, item2, item3, item4, item1);
    }

    @Test
    public void downloadRegister_shouldReturnAZipfile() throws IOException {
        Response response = getRequest("/download-register");

        assertThat(response.getHeaderString("Content-Type"), equalTo(MediaType.APPLICATION_OCTET_STREAM));
        assertThat(response.getHeaderString("Content-Disposition"), startsWith("attachment; filename="));
        assertThat(response.getHeaderString("Content-Disposition"), endsWith(".zip"));

        InputStream is = response.readEntity(InputStream.class);
        Set<String> zipEntryNames = getEntries(is).keySet();

        assertThat(zipEntryNames, hasItem("register.json"));
        assertThat(zipEntryNames.stream().filter(e -> e.matches("(entry/)(\\d)(.json)")).count(), is(5L));
        assertThat(zipEntryNames.stream().filter(e -> e.matches("(item/)(\\w+)(.json)")).count(), is(4L));
    }

    @Test
    public void downloadRegister_shouldUseCorrectEntryAndItemJsonFormat() throws IOException {
        Response response = getRequest("/download-register");
        InputStream is = response.readEntity(InputStream.class);

        List<String> itemJson = getEntries(is).entrySet().stream()
                .filter(j -> j.getKey().startsWith("item"))
                .map(j -> j.getValue().toString())
                .collect(Collectors.toList());

        assertThat(itemJson, hasItems(item1, item2, item3, item4));

        List<JsonNode> entryJson = getEntries(is).entrySet().stream()
                .filter(j -> j.getKey().startsWith("entry"))
                .map(Map.Entry::getValue)
                .collect(Collectors.toList());

        entryJson.forEach(j -> {
            assertTrue(j.has("entry-number"));
            assertTrue(j.has("entry-timestamp"));
            assertTrue(j.has("item-hash"));
        });
    }

    @Test
    public void downloadRegister_shouldUseCorrectRegisterJsonFormat() throws IOException {
        Response response = getRequest("/download-register");
        InputStream is = response.readEntity(InputStream.class);

        JsonNode registerJson = getEntries(is).get("register.json");

        assertThat(registerJson.get("total-entries").asInt(), is(5));
        assertThat(registerJson.get("total-records").asInt(), is(3));
        assertTrue(registerJson.has("last-updated"));
        assertTrue(registerJson.has("domain"));
        assertTrue(registerJson.has("register-record"));
    }

    @Test
    public void downloadRSF_shouldReturnRegisterAsRsfStream() throws IOException {
        Response response = getRequest("/download-rsf");

        assertThat(response.getHeaderString("Content-Type"), equalTo(MediaType.APPLICATION_OCTET_STREAM));
        assertThat(response.getHeaderString("Content-Disposition"), startsWith("attachment; filename="));
        assertThat(response.getHeaderString("Content-Disposition"), endsWith(".tsv"));

        String[] rsfLines = getRsfLinesFrom(response);

        assertThat(rsfLines[0], equalTo("add-item\t{\"address\":\"12345\",\"street\":\"ellis\"}"));
        assertThat(rsfLines[1], equalTo("add-item\t{\"address\":\"6789\",\"street\":\"presley\"}"));
        assertThat(rsfLines[2], equalTo("add-item\t{\"address\":\"12345\",\"street\":\"foo\"}"));
        assertThat(rsfLines[3], equalTo("add-item\t{\"address\":\"145678\",\"street\":\"ellis\"}"));

        assertFormattedEntry(rsfLines[4], "sha-256:19205fafe65406b9b27fce1b689abc776df4ddcf150c28b29b73b4ea054af6b9");
        assertFormattedEntry(rsfLines[5], "sha-256:bd239db51960376826b937a615f0f3397485f00611d35bb7e951e357bf73b934");
        assertFormattedEntry(rsfLines[6], "sha-256:cc8a7c42275c84b94c6e282ae88b3dbcc06319156fc4539a2f39af053bf30592");
        assertFormattedEntry(rsfLines[7], "sha-256:8ac926428ee49fb83c02bdd2556e62e84cfd9e636cd35eb1306ac8cb661e4983");
        assertFormattedEntry(rsfLines[8], "sha-256:19205fafe65406b9b27fce1b689abc776df4ddcf150c28b29b73b4ea054af6b9");
    }

    @Test
    public void downloadPartialRSF_shouldReturnAPartOfRegisterAsRsfStream() throws IOException {
        Response response = getRequest("/download-rsf/3/4");

        assertThat(response.getHeaderString("Content-Type"), equalTo(MediaType.APPLICATION_OCTET_STREAM));
        assertThat(response.getHeaderString("Content-Disposition"), startsWith("attachment; filename="));
        assertThat(response.getHeaderString("Content-Disposition"), endsWith(".tsv"));

        String[] rsfLines = getRsfLinesFrom(response);

        assertThat(rsfLines[0], equalTo("add-item\t{\"address\":\"12345\",\"street\":\"foo\"}"));
        assertThat(rsfLines[1], equalTo("add-item\t{\"address\":\"145678\",\"street\":\"ellis\"}"));

        assertFormattedEntry(rsfLines[2], "sha-256:cc8a7c42275c84b94c6e282ae88b3dbcc06319156fc4539a2f39af053bf30592");
        assertFormattedEntry(rsfLines[3], "sha-256:8ac926428ee49fb83c02bdd2556e62e84cfd9e636cd35eb1306ac8cb661e4983");
    }

    @Test
    public void downloadPartialRSF_shouldReturnReturn404ForIncorectRsfBoundaries() throws IOException {
        Response response = getRequest("/download-rsf/4/1");
        
        assertThat(response.getStatus(), equalTo(400));
    }

    private String[] getRsfLinesFrom(Response response){
        InputStream is = response.readEntity(InputStream.class);
        return new BufferedReader(new InputStreamReader(is)).lines().toArray(String[]::new);
    }

    private void assertFormattedEntry(String actualEntry, String expectedEntrySha){
        assertThat(actualEntry, startsWith("append-entry"));
        assertThat(actualEntry, endsWith(expectedEntrySha));
    }

    private Map<String, JsonNode> getEntries(InputStream inputStream) throws IOException {
        ZipInputStream zis = new ZipInputStream(inputStream);

        Map<String, JsonNode> entries = new HashMap<>();
        byte[] buffer = new byte[1024];
        int read = 0;
        for (ZipEntry entry; (entry = zis.getNextEntry()) != null; ) {
            StringBuilder sb = new StringBuilder();
            while ((read = zis.read(buffer, 0, 1024)) >= 0) {
                sb.append(new String(buffer, 0, read));
            }

            entries.put(entry.getName(), new ObjectMapper().readTree(sb.toString()));
        }

        return entries;
    }
{'fraction_non_alphanumeric': 0.15064365927143247, 'fraction_numerical': 0.05464256368118324, 'mean_word_length': 3.979543129901125, 'pattern_counts': {'":': 40, '<': 15, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4941903', 'n_tokens_mistral': 4822, 'n_tokens_neox': 4323, 'n_words': 606}