<commit_before>
/*
 * Copyright (C) 2015 <NAME>. All rights reserved.
 */

package org.jsimpledb.core.type;

import com.google.common.net.InetAddresses;

import java.io.File;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.net.Inet4Address;
import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.URI;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.MonthDay;
import java.time.OffsetDateTime;
import java.time.OffsetTime;
import java.time.Period;
import java.time.Year;
import java.time.YearMonth;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.util.Arrays;
import java.util.BitSet;
import java.util.Date;
import java.util.UUID;
import java.util.function.Supplier;
import java.util.regex.Pattern;

import javax.mail.internet.InternetAddress;

import org.jsimpledb.core.CoreAPITestSupport;
import org.jsimpledb.core.FieldType;
import org.jsimpledb.core.FieldTypeRegistry;
import org.jsimpledb.util.ByteReader;
import org.jsimpledb.util.ByteUtil;
import org.jsimpledb.util.ByteWriter;
import org.jsimpledb.util.ParseContext;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;

public class FieldTypeTest extends CoreAPITestSupport {

    private final FieldTypeRegistry registry = new FieldTypeRegistry();

    @Test(dataProvider = "cases")
    public void testFieldType(String typeName, Object[] values) throws Exception {
        final FieldType<?> fieldType = registry.getFieldType(typeName);
        assert fieldType != null : "didn't find `" + typeName + "'";
        this.testFieldType2(fieldType, values);
    }

    @SuppressWarnings("unchecked")
    private <T> void testFieldType2(FieldType<T> fieldType, Object[] values) throws Exception {
        this.testFieldType3(fieldType, (T[])values);
    }

    private <T> void testFieldType3(FieldType<T> fieldType, T[] values) throws Exception {
        final byte[][] encodings = new byte[values.length][];
        for (int i = 0; i < values.length; i++) {
            final T value = values[i];

            // Binary encoding
            final ByteWriter writer = new ByteWriter();
            fieldType.write(writer, value);
            encodings[i] = writer.getBytes();
            final T value2 = fieldType.read(new ByteReader(encodings[i]));
            this.assertEquals(fieldType, value2, value);

            // String encoding
            if (value != null) {
                Assert.assertEquals(fieldType.toString(value2), fieldType.toString(value));
                final String s = fieldType.toString(value);
                final T value3 = fieldType.fromString(s);
                this.assertEquals(fieldType, value3, value);
            } else {
                try {
                    fieldType.toString(null);
                    assert false;
                } catch (IllegalArgumentException e) {
                    // expected
                }
            }

            // "list" style string encoding for some primitive arrays
            if (fieldType instanceof Base64ArrayType) {
                final Base64ArrayType<T, ?> arrayType = (Base64ArrayType<T, ?>)fieldType;

                // String encoding
                if (value != null) {
                    Assert.assertEquals(arrayType.toString(value2, false), arrayType.toString(value, false));
                    final String s = arrayType.toString(value, false);
                    final T value3 = arrayType.fromString(s);
                    this.assertEquals(arrayType, value3, value);
                } else {
                    try {
                        arrayType.toString(null, false);
                        assert false;
                    } catch (IllegalArgumentException e) {
                        // expected
                    }
                }
            }

            // Parseable string encoding
            Assert.assertEquals(fieldType.toParseableString(value2), fieldType.toParseableString(value));
            final String s2 = fieldType.toParseableString(value);
            final ParseContext ctx = new ParseContext(s2 + ",abcd");
            final T value4 = fieldType.fromParseableString(ctx);
            this.assertEquals(fieldType, value4, value);
            Assert.assertEquals(ctx.getInput(), ",abcd");

            // "list" style string encoding for some primitive arrays
            if (fieldType instanceof Base64ArrayType) {
                final Base64ArrayType<T, ?> arrayType = (Base64ArrayType<T, ?>)fieldType;

                // Parseable string encoding
                Assert.assertEquals(arrayType.toParseableString(value2, false), arrayType.toParseableString(value, false));
                final String s3 = arrayType.toParseableString(value, false);
                final ParseContext ctx2 = new ParseContext(s3 + ",abcd");
                final T value5 = arrayType.fromParseableString(ctx2);
                this.assertEquals(arrayType, value5, value);
                Assert.assertEquals(ctx2.getInput(), ",abcd");
            }

            // Check sort order
            if (i > 0) {
                final T previous = values[i - 1];
                final boolean bytesEqual = ByteUtil.compare(encodings[i - 1], encodings[i]) == 0;
                final boolean bytesLessThan = ByteUtil.compare(encodings[i - 1], encodings[i]) < 0;
                final boolean fieldEqual = fieldType.compare(previous, value) == 0;
                final boolean fieldLessThan = fieldType.compare(previous, value) < 0;

                Assert.assertTrue(bytesLessThan || bytesEqual, "Binary sort failure @ " + i + ": expected "
                  + fieldType.toParseableString(previous) + " [" + ByteUtil.toString(encodings[i - 1]) + "] <= "
                  + fieldType.toParseableString(value) + " [" + ByteUtil.toString(encodings[i]) + "]");
                Assert.assertTrue(fieldLessThan || fieldEqual, "Java sort failure @ " + i + ": expected "
                  + fieldType.toParseableString(previous) + " <= " + fieldType.toParseableString(value));

                Assert.assertEquals(bytesEqual, fieldEqual, "equality mismatch @ " + i + ": "
                  + fieldType.toParseableString(previous) + " and " + fieldType.toParseableString(value));
                Assert.assertEquals(bytesLessThan, fieldLessThan, "less-than mismatch @ " + i + ": "
                  + fieldType.toParseableString(previous) + " and " + fieldType.toParseableString(value));
            }
        }
    }

    private <T> void assertEquals(FieldType<T> fieldType, T actual, T expected) {
        FieldTypeTest.assertEquals(fieldType, actual, expected, "equals check failed: " + actual + " != " + expected);
    }<commit_msg>Verify FieldType string encodings don't contain invalid chars.<commit_after>
/*
 * Copyright (C) 2015 <NAME>. All rights reserved.
 */

package org.jsimpledb.core.type;

import com.google.common.net.InetAddresses;

import java.io.File;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.net.Inet4Address;
import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.URI;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.MonthDay;
import java.time.OffsetDateTime;
import java.time.OffsetTime;
import java.time.Period;
import java.time.Year;
import java.time.YearMonth;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.util.Arrays;
import java.util.BitSet;
import java.util.Date;
import java.util.UUID;
import java.util.function.Supplier;
import java.util.regex.Pattern;

import javax.mail.internet.InternetAddress;

import org.jsimpledb.core.CoreAPITestSupport;
import org.jsimpledb.core.FieldType;
import org.jsimpledb.core.FieldTypeRegistry;
import org.jsimpledb.util.ByteReader;
import org.jsimpledb.util.ByteUtil;
import org.jsimpledb.util.ByteWriter;
import org.jsimpledb.util.ParseContext;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;

public class FieldTypeTest extends CoreAPITestSupport {

    private final FieldTypeRegistry registry = new FieldTypeRegistry();

    @Test(dataProvider = "cases")
    public void testFieldType(String typeName, Object[] values) throws Exception {
        final FieldType<?> fieldType = registry.getFieldType(typeName);
        assert fieldType != null : "didn't find `" + typeName + "'";
        this.testFieldType2(fieldType, values);
    }

    @SuppressWarnings("unchecked")
    private <T> void testFieldType2(FieldType<T> fieldType, Object[] values) throws Exception {
        this.testFieldType3(fieldType, (T[])values);
    }

    private <T> void testFieldType3(FieldType<T> fieldType, T[] values) throws Exception {
        final byte[][] encodings = new byte[values.length][];
        for (int i = 0; i < values.length; i++) {
            final T value = values[i];

            // Binary encoding
            final ByteWriter writer = new ByteWriter();
            fieldType.write(writer, value);
            encodings[i] = writer.getBytes();
            final T value2 = fieldType.read(new ByteReader(encodings[i]));
            this.assertEquals(fieldType, value2, value);

            // String encoding
            if (value != null) {
                Assert.assertEquals(fieldType.toString(value2), fieldType.toString(value));
                final String s = fieldType.toString(value);
                this.checkValidString(value, s);
                final T value3 = fieldType.fromString(s);
                this.assertEquals(fieldType, value3, value);
            } else {
                try {
                    fieldType.toString(null);
                    assert false;
                } catch (IllegalArgumentException e) {
                    // expected
                }
            }

            // "list" style string encoding for some primitive arrays
            if (fieldType instanceof Base64ArrayType) {
                final Base64ArrayType<T, ?> arrayType = (Base64ArrayType<T, ?>)fieldType;

                // String encoding
                if (value != null) {
                    Assert.assertEquals(arrayType.toString(value2, false), arrayType.toString(value, false));
                    final String s = arrayType.toString(value, false);
                    final T value3 = arrayType.fromString(s);
                    this.assertEquals(arrayType, value3, value);
                } else {
                    try {
                        arrayType.toString(null, false);
                        assert false;
                    } catch (IllegalArgumentException e) {
                        // expected
                    }
                }
            }

            // Parseable string encoding
            Assert.assertEquals(fieldType.toParseableString(value2), fieldType.toParseableString(value));
            final String s2 = fieldType.toParseableString(value);
            this.checkValidString(value, s2);
            final ParseContext ctx = new ParseContext(s2 + ",abcd");
            final T value4 = fieldType.fromParseableString(ctx);
            this.assertEquals(fieldType, value4, value);
            Assert.assertEquals(ctx.getInput(), ",abcd");

            // "list" style string encoding for some primitive arrays
            if (fieldType instanceof Base64ArrayType) {
                final Base64ArrayType<T, ?> arrayType = (Base64ArrayType<T, ?>)fieldType;

                // Parseable string encoding
                Assert.assertEquals(arrayType.toParseableString(value2, false), arrayType.toParseableString(value, false));
                final String s3 = arrayType.toParseableString(value, false);
                this.checkValidString(value, s3);
                final ParseContext ctx2 = new ParseContext(s3 + ",abcd");
                final T value5 = arrayType.fromParseableString(ctx2);
                this.assertEquals(arrayType, value5, value);
                Assert.assertEquals(ctx2.getInput(), ",abcd");
            }

            // Check sort order
            if (i > 0) {
                final T previous = values[i - 1];
                final boolean bytesEqual = ByteUtil.compare(encodings[i - 1], encodings[i]) == 0;
                final boolean bytesLessThan = ByteUtil.compare(encodings[i - 1], encodings[i]) < 0;
                final boolean fieldEqual = fieldType.compare(previous, value) == 0;
                final boolean fieldLessThan = fieldType.compare(previous, value) < 0;

                Assert.assertTrue(bytesLessThan || bytesEqual, "Binary sort failure @ " + i + ": expected "
                  + fieldType.toParseableString(previous) + " [" + ByteUtil.toString(encodings[i - 1]) + "] <= "
                  + fieldType.toParseableString(value) + " [" + ByteUtil.toString(encodings[i]) + "]");
                Assert.assertTrue(fieldLessThan || fieldEqual, "Java sort failure @ " + i + ": expected "
                  + fieldType.toParseableString(previous) + " <= " + fieldType.toParseableString(value));

                Assert.assertEquals(bytesEqual, fieldEqual, "equality mismatch @ " + i + ": "
                  + fieldType.toParseableString(previous) + " and " + fieldType.toParseableString(value));
                Assert.assertEquals(bytesLessThan, fieldLessThan, "less-than mismatch @ " + i + ": "
                  + fieldType.toParseableString(previous) + " and " + fieldType.toParseableString(value));
            }
        }
    }

    private void checkValidString(Object value, String s) {
        for (int i = 0; i < s.length(); i++) {
            final int ch = s.charAt(i);
            switch (ch) {
            case '\t':
            case '\n':
            case '\r':
                break;
            default:
                if (ch >= '\u0020' && ch <= '\ud7ff')
                    break;
                if (ch >= '\ue000' && ch <= '\uffdf')
                    break;
                assert false : String.format(
                  "string encoding \"%s\" of value %s contains illegal character 0x%04x at index %d", s, value, ch, i);
            }
        }
    }

    private <T> void assertEquals(FieldType<T> fieldType, T actual, T expected) {
        FieldTypeTest.assertEquals(fieldType, actual, expected, "equals check failed: " + actual + " != " + expected);
    }
{'fraction_non_alphanumeric': 0.10921478218980112, 'fraction_numerical': 0.008258206592801597, 'mean_word_length': 2.2759242560865647, 'pattern_counts': {'":': 8, '<': 40, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1567542', 'n_tokens_mistral': 4026, 'n_tokens_neox': 3839, 'n_words': 890}