<commit_before>package com.akiban.ais.ddl;

import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicInteger;

import org.antlr.runtime.ANTLRFileStream;
import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.CharStream;
import org.antlr.runtime.CommonTokenStream;
import org.antlr.runtime.RecognitionException;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.PosixParser;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import com.akiban.ais.ddl.SchemaDef.CName;
import com.akiban.ais.ddl.SchemaDef.ColumnDef;
import com.akiban.ais.ddl.SchemaDef.IndexDef;
import com.akiban.ais.ddl.SchemaDef.UserTableDef;
import com.akiban.ais.io.MessageTarget;
import com.akiban.ais.io.Reader;
import com.akiban.ais.io.Writer;
import com.akiban.ais.metamodel.MetaModel;
import com.akiban.ais.metamodel.ModelObject;
import com.akiban.ais.model.AISBuilder;
import com.akiban.ais.model.AkibaInformationSchema;
import com.akiban.ais.model.Column;
import com.akiban.ais.model.Index;
import com.akiban.ais.model.IndexColumn;
import com.akiban.ais.model.Source;
import com.akiban.ais.model.Type;
import com.akiban.ais.model.UserTable;
import com.akiban.util.Strings;

/**
 * This class reads the CREATE TABLE statements in a mysqldump file, plus
 * annotations to denote the group structure. There is neither an attempt to
 * fully parse the DDL, nor to handle syntactic variations. The purpose of this
 * class is to facilitate creating AIS instances from existing MySQL databases
 * prior to the arrival of the Control Center's implementation.
 * 
 * There is no error handling, and this is not a general-purpose parser. The
 * format of the text file must be exact, especially with respect to spaces,
 * back ticks, etc.
 * 
 * See the xxxxxxxx_schema.sql file in src/test/resources for an example of the
 * syntax.
 * 
 * @author peter
 * 
 */
public class DDLSource extends Source {

    public class ParseException extends Exception {
        private ParseException(SchemaDef.SchemaDefException cause) {
            super(cause.getMessage(), cause);
        }
    }

    private static final Log LOG = LogFactory.getLog(DDLSource.class.getName());

    private final static String SCHEMA_FILE_NAME = "src/test/resources/xxxxxxxx_schema.ddl";
    private final static int MAX_AIS_SIZE = 1048576;
    private final static String BINARY_FORMAT = "binary";
    private final static String SQL_FORMAT = "sql";

    private String ddlSourceName = "<not specified>";
    private SchemaDef schemaDef;
    private Map<String, ColumnDef> columnDefMap = new HashMap<String, ColumnDef>();
    private final Map<CName, String> groupNames = new HashMap<CName, String>();
    private final Map<JoinName, String> joinNames = new HashMap<JoinName, String>();
    private final int baseId;

    public static class StringStream extends ANTLRStringStream {

        public StringStream(final String string) {
            super(string);
        }

        @Override
        public int LA(int i) {
            if (i == 0) {
                return 0; // undefined
            }
            if (i < 0) {
                i++; // e.g., translate LA(-1) to use offset 0
                if ((p + i - 1) < 0) {
                    return CharStream.EOF; // invalid; no char
                    // before first
                    // char
                }
            }

            if ((p + i - 1) >= n) {

                return CharStream.EOF;
            }
            return Character.toLowerCase(data[p + i - 1]);
        }
    }

    public static class FileStream extends ANTLRFileStream {

        public FileStream(final String fileName) throws IOException {
            super(fileName);
        }

        @Override
        public int LA(int i) {
            if (i == 0) {
                return 0; // undefined
            }
            if (i < 0) {
                i++; // e.g., translate LA(-1) to use offset 0
                if ((p + i - 1) < 0) {
                    return CharStream.EOF; // invalid; no char
                    // before first
                    // char
                }
            }

            if ((p + i - 1) >= n) {

                return CharStream.EOF;
            }
            return Character.toLowerCase(data[p + i - 1]);
        }
    }

    public static void main(final String[] args) throws Exception {

        String iFileName = SCHEMA_FILE_NAME;
        String oFileName = "/tmp/"
                + new File(iFileName).getName().split("\\.")[0] + ".out";
        String format = SQL_FORMAT;

        CommandLineParser parser = new PosixParser();
        Options options = new Options();
        options.addOption("i", "input-file", true, "default input file = "
                + iFileName);
        options.addOption("o", "output-file", true, "default output file = "
                + oFileName);
        options.addOption("f", "format", true,
                "valid values are sql and binary; the default is sql");
        options.addOption("h", "help", false, "print this message");

        try {
            CommandLine line = parser.parse(options, args);

            if (line.hasOption("help")) {
                HelpFormatter formatter = new HelpFormatter();
                formatter.printHelp("DDLSource", options);
                return;
            }

            if (line.hasOption("input-file")) {
                iFileName = line.getOptionValue("input-file");
            }

            if (line.hasOption("output-file")) {
                oFileName = line.getOptionValue("output-file");
            }

            if (line.hasOption("format")) {
                format = line.getOptionValue("format");
                format = format.toLowerCase();
                if (format.compareTo(BINARY_FORMAT) != 0
                        && format.compareTo(SQL_FORMAT) != 0) {
                    System.out.println("invald format option " + format
                            + "; using default = " + SQL_FORMAT);
                    format = SQL_FORMAT;
                }
            }
        } catch (org.apache.commons.cli.ParseException exp) {
            System.out.println("Unexpected exception:" + exp.getMessage());
        }

        final DDLSource source = new DDLSource();
        final AkibaInformationSchema ais = source.buildAIS(iFileName);
        // AISPrinter.print(ais);

        if (format.compareTo(SQL_FORMAT) == 0) {
            final PrintWriter pw = new PrintWriter(new FileWriter(oFileName));
            SqlTextTarget target = new SqlTextTarget(pw);
            new Writer(target).save(ais);
            target.writeGroupTableDDL(ais);
            pw.close();
        } else {
            assert format.compareTo(BINARY_FORMAT) == 0;

            ByteBuffer rawAis = ByteBuffer.allocate(MAX_AIS_SIZE);
            rawAis.order(ByteOrder.LITTLE_ENDIAN);
            new Writer(new MessageTarget(rawAis)).save(ais);
            rawAis.flip();

            boolean append = false;
            File file = new File(oFileName);
            try {
                FileChannel wChannel = new FileOutputStream(file, append)
                        .getChannel();
                wChannel.write(rawAis);
                wChannel.close();
            } catch (IOException e) {
                throw new Exception("rarrrgh");
            }
        }
    }

    private static class JoinName {
        private final CName parentName;
        private final CName childName;

        private JoinName(CName parentName, CName childName) {
            this.parentName = parentName;
            this.childName = childName;
        }

        @Override
        public boolean equals(Object o) {
            if (this == o) {
                return true;
            }
            if (o == null || getClass() != o.getClass()) {
                return false;
            }
            JoinName joinName = (JoinName) o;
            if (!childName.equals(joinName.childName)) {
                return false;
            }
            if (!parentName.equals(joinName.parentName)) {
                return false;
            }
            return true;
        }

        @Override
        public int hashCode() {
            int result = parentName.hashCode();
            result = 31 * result + childName.hashCode();
            return result;
        }

        @Override
        public String toString() {
            return "Join[parent: " + parentName + " -> child: " + childName
                    + ']';
        }
    }

    public DDLSource() {
        this(0);
    }

    public DDLSource(int baseId) {
        this.baseId = baseId;
    }

    public AkibaInformationSchema buildAIS(final String fileName)
            throws Exception {
        ddlSourceName = fileName;
        return buildAIS(new FileStream(fileName));
    }

    public AkibaInformationSchema buildAISFromString(final String string)
            throws Exception {
        return buildAIS(new StringStream(string));
    }

    private String constructFKJoinName(UserTableDef childTable, IndexDef fkIndex)
    {
        String ret = (fkIndex.getParentSchema() + "/" + fkIndex.getParentTable() + "/" + Strings.join(fkIndex.getParentColumns(), ",") + "/" + childTable.getCName().getSchema() + "/" + childTable.name + "/" + Strings.join(fkIndex.getChildColumns(), ",")).toLowerCase();
        return ret.replace(',','_');
    }
    
    public static SchemaDef parseSchemaDef(final String string)
            throws Exception {
        DDLSource instance = new DDLSource();
        instance.parseSchemaDef(new StringStream(string));
        return instance.schemaDef;
    }

    private void parseSchemaDef(final ANTLRStringStream stringStream)
            throws RecognitionException {
        DDLSourceLexer lex = new DDLSourceLexer(stringStream);
        CommonTokenStream tokens = new CommonTokenStream(lex);
        final DDLSourceParser tsparser = new DDLSourceParser(tokens);
        this.schemaDef = new SchemaDef();
        tsparser.schema(schemaDef);
        if (tsparser.getNumberOfSyntaxErrors() > 0) {
            throw new RuntimeException("DDLSource reported a syntax error in: "
                    + ddlSourceName);
        }
        addImpliedGroups(schemaDef.getMasterSchemaName());
        computeColumnMapAndPositions();
    }

    private AkibaInformationSchema buildAIS(final ANTLRStringStream stringStream)
            throws Exception {
        parseSchemaDef(stringStream);
        AkibaInformationSchema ais = new Reader(this)
                .load(new AkibaInformationSchema());
        return ais;
    }

    public UserTableDef parseCreateTable(final String createTableStatement)
            throws Exception {
        DDLSourceLexer lex = new DDLSourceLexer(new StringStream(
                createTableStatement));
        CommonTokenStream tokens = new CommonTokenStream(lex);
        final DDLSourceParser tsparser = new DDLSourceParser(tokens);
        try {
            final SchemaDef schemaDef = new SchemaDef();
            tsparser.table_spec(schemaDef);
            if (tsparser.getNumberOfSyntaxErrors() > 0) {
                throw new RuntimeException("DDLSource reported a syntax error in: "
                        + ddlSourceName);
            }
            return schemaDef.getCurrentTable();
        } catch (SchemaDef.SchemaDefException e) {
            throw new ParseException(e);
        }
    }

    @Override
    public void close() throws Exception {
    }

    private Long longValue(final String s) {
        return s == null ? null : Long.valueOf(Long.parseLong(s));
    }

    /**
     * Name generator for Group tables.
     * 
     * @param groupName
     * @return
     */
    private String groupTableName(final CName groupName) {
        return "_akiba_" + groupName.getName();
    }

    private CName groupName(final CName rootTable) {
        String ret = groupNames.get(rootTable);
        if (ret == null) {
            ret = rootTable.getName();
            for (int i = 0; groupNames.containsValue(ret); ++i) {
                ret = rootTable.getName() + '$' + i;
            }
            groupNames.put(rootTable, ret);
        }
        return new CName("akiba_objects", ret);
    }

    /**
     * Schema name generator for groups
     * 
     * @return "akiba_objects"
     */
    private String groupSchemaName() {
        return "akiba_objects";
    }

    /**
     * Group column name generator for group columns. TODO: Needs to handle long
     * schemaName / tableName / columnName combos.
     * 
     * @param tableName
     * @param columnName
     * @return
     */
    private String mangledColumnName(final CName tableName,
            final String columnName) {
        return tableName.getName() + "$" + columnName;
    }

    /**
     * Index name generator.
     * 
     * @param utdef
     * @param indexDef
     * @return
     */
    private String mangledIndexName(final UserTableDef utdef,
            final IndexDef indexDef) {
        return utdef.name.getName() + "$" + indexDef.name;
    }

    /**
     * Tests an FOREIGN KEY index definition to determine whether it represents
     * a group-defining relationship.
     * 
     * @param indexDef
     * @return
     */
    private static boolean isAkiban(final IndexDef indexDef) {
        return SchemaDef.isAkiban(indexDef);
    }

    /**
     * Index name generator. This code actually has to (and does) emulate the
     * MySQL algorithm for assigning unspecified names.
     * 
     * @param namingMap
     * @param indexDef
     * @return
     */
    private String generateIndexName(
            final Map<String, AtomicInteger> namingMap, final IndexDef indexDef) {
        final String firstColumnName = indexDef.columns.get(0).columnName;
        AtomicInteger count = namingMap.get(firstColumnName);
        if (count == null) {
            count = new AtomicInteger(1);
            namingMap.put(firstColumnName, count);
            return firstColumnName;
        } else {
            return firstColumnName + "_" + (count.incrementAndGet());
        }
    }

    /**
     * Return List of names of user tables in a specified group sorted by
     * depth-first traversal of the parent-child relationships between those
     * tables.
     * 
     * @param groupName
     * @return sorted List of table names
     */
    List<CName> depthFirstSortedUserTables(final CName groupName) {
        final CName root = new CName(schemaDef, "", "");
        final Map<CName, SortedSet<CName>> hierarchy = new HashMap<CName, SortedSet<CName>>();
        final List<CName> tableList = new ArrayList<CName>();
        for (final CName tableName : schemaDef.getGroupMap().get(groupName)) {
            final UserTableDef utdef = schemaDef.getUserTableMap().get(
                    tableName);
            CName parent = utdef.parent == null ? root : utdef.parent.name;
            SortedSet<CName> children = hierarchy.get(parent);
            if (children == null) {
                children = new TreeSet<CName>();
                hierarchy.put(parent, children);
            }
            children.add(utdef.name);
        }
        traverseDepthFirstSortedTableMap(root, hierarchy, tableList);
        if (tableList.isEmpty()) {
            throw new IllegalStateException("Broken user table hiearchy: "
                    + hierarchy);
        }
        return tableList;
    }

    void traverseDepthFirstSortedTableMap(final CName parent,
            final Map<CName, SortedSet<CName>> hierarchy,
            final List<CName> tableList) {
        SortedSet<CName> siblings = hierarchy.get(parent);
        if (siblings != null) {
            for (final CName tableName : siblings) {
                tableList.add(tableName);
                traverseDepthFirstSortedTableMap(tableName, hierarchy,
                        tableList);
            }
        }
    }

    void computeColumnMapAndPositions() {
        for (final CName groupName : schemaDef.getGroupMap().keySet()) {
            int gposition = 0;
            final List<CName> tableList = depthFirstSortedUserTables(groupName);
            for (final CName tableName : tableList) {
                int uposition = 0;
                final UserTableDef utdef = schemaDef.getUserTableMap().get(
                        tableName);
                List<ColumnDef> columns = utdef.columns;
                for (final ColumnDef def : columns) {
                    def.gposition = gposition++;
                    def.uposition = uposition++;
                    columnDefMap.put(utdef.name + "." + def.name, def);
                }
            }
        }
    }

    @Override
    protected final void read(String typename, Receiver receiver)
            throws Exception {
        throw new UnsupportedOperationException();
    }

    @Override
    public void readColumns(Receiver columnReceiver) throws Exception {
        for (final CName groupName : schemaDef.getGroupMap().keySet()) {
            final List<CName> tableList = depthFirstSortedUserTables(groupName);
            for (final CName tableName : tableList) {
                final UserTableDef utdef = schemaDef.getUserTableMap().get(
                        tableName);
                List<ColumnDef> columns = utdef.columns;
                for (final ColumnDef def : columns) {
                    String groupTableName = groupTableName(groupName);
                    String groupColumnName = mangledColumnName(tableName,
                            def.name);
                    columnReceiver.receive(map(column, tableName.getSchema(),
                            tableName.getName(), def.name, def.uposition,
                            def.typeName, longValue(def.typeParam1),
                            longValue(def.typeParam2), def.nullable,
                            def.defaultAutoIncrement(), groupSchemaName(),
                            groupTableName, groupColumnName, null, null,
                            // TODO: This isn't really correct: if collation is specified but not 
                            // charset, the collation's default charset should be used.
                            // But to do this we need to add a charset/collation database.
                            def.charset == null ? AkibaInformationSchema.DEFAULT_CHARSET : def.charset,
                            def.collate == null ? AkibaInformationSchema.DEFAULT_COLLATION : def.collate,
                            true)); // realColumn
                    columnReceiver.receive(map(column, groupSchemaName(),
                            groupTableName, groupColumnName, def.gposition,
                            def.typeName, longValue(def.typeParam1),
                            longValue(def.typeParam2), def.nullable,
                            def.defaultAutoIncrement(), null, null, null,
                            null, null,
                            // TODO: This isn't really correct: if collation is specified but not 
                            // charset, the collation's default charset should be used.
                            // But to do this we need to add a charset/collation database.
                            def.charset == null ? AkibaInformationSchema.DEFAULT_CHARSET : def.charset,
                            def.collate == null ? AkibaInformationSchema.DEFAULT_COLLATION : def.collate,
                            true)); // realColumn
                }
            }
        }
    }

    @Override
    public void readJoins(Receiver joinReceiver) throws Exception {
        for (final CName groupName : schemaDef.getGroupMap().keySet()) {
            for (final CName tableName : schemaDef.getGroupMap().get(groupName)) {
                final UserTableDef utdef = schemaDef.getUserTableMap().get(
                        tableName);
                if (utdef.parent != null) {<commit_msg>Addressing review comments.<commit_after>package com.akiban.ais.ddl;

import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicInteger;

import org.antlr.runtime.ANTLRFileStream;
import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.CharStream;
import org.antlr.runtime.CommonTokenStream;
import org.antlr.runtime.RecognitionException;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.PosixParser;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import com.akiban.ais.ddl.SchemaDef.CName;
import com.akiban.ais.ddl.SchemaDef.ColumnDef;
import com.akiban.ais.ddl.SchemaDef.IndexDef;
import com.akiban.ais.ddl.SchemaDef.UserTableDef;
import com.akiban.ais.io.MessageTarget;
import com.akiban.ais.io.Reader;
import com.akiban.ais.io.Writer;
import com.akiban.ais.metamodel.MetaModel;
import com.akiban.ais.metamodel.ModelObject;
import com.akiban.ais.model.AISBuilder;
import com.akiban.ais.model.AkibaInformationSchema;
import com.akiban.ais.model.Column;
import com.akiban.ais.model.Index;
import com.akiban.ais.model.IndexColumn;
import com.akiban.ais.model.Source;
import com.akiban.ais.model.Type;
import com.akiban.ais.model.UserTable;
import com.akiban.util.Strings;

/**
 * This class reads the CREATE TABLE statements in a mysqldump file, plus
 * annotations to denote the group structure. There is neither an attempt to
 * fully parse the DDL, nor to handle syntactic variations. The purpose of this
 * class is to facilitate creating AIS instances from existing MySQL databases
 * prior to the arrival of the Control Center's implementation.
 * 
 * There is no error handling, and this is not a general-purpose parser. The
 * format of the text file must be exact, especially with respect to spaces,
 * back ticks, etc.
 * 
 * See the xxxxxxxx_schema.sql file in src/test/resources for an example of the
 * syntax.
 * 
 * @author peter
 * 
 */
public class DDLSource extends Source {

    public class ParseException extends Exception {
        private ParseException(SchemaDef.SchemaDefException cause) {
            super(cause.getMessage(), cause);
        }
    }

    private static final Log LOG = LogFactory.getLog(DDLSource.class.getName());

    private final static String SCHEMA_FILE_NAME = "src/test/resources/xxxxxxxx_schema.ddl";
    private final static int MAX_AIS_SIZE = 1048576;
    private final static String BINARY_FORMAT = "binary";
    private final static String SQL_FORMAT = "sql";

    private String ddlSourceName = "<not specified>";
    private SchemaDef schemaDef;
    private Map<String, ColumnDef> columnDefMap = new HashMap<String, ColumnDef>();
    private final Map<CName, String> groupNames = new HashMap<CName, String>();
    private final Map<JoinName, String> joinNames = new HashMap<JoinName, String>();
    private final int baseId;

    public static class StringStream extends ANTLRStringStream {

        public StringStream(final String string) {
            super(string);
        }

        @Override
        public int LA(int i) {
            if (i == 0) {
                return 0; // undefined
            }
            if (i < 0) {
                i++; // e.g., translate LA(-1) to use offset 0
                if ((p + i - 1) < 0) {
                    return CharStream.EOF; // invalid; no char
                    // before first
                    // char
                }
            }

            if ((p + i - 1) >= n) {

                return CharStream.EOF;
            }
            return Character.toLowerCase(data[p + i - 1]);
        }
    }

    public static class FileStream extends ANTLRFileStream {

        public FileStream(final String fileName) throws IOException {
            super(fileName);
        }

        @Override
        public int LA(int i) {
            if (i == 0) {
                return 0; // undefined
            }
            if (i < 0) {
                i++; // e.g., translate LA(-1) to use offset 0
                if ((p + i - 1) < 0) {
                    return CharStream.EOF; // invalid; no char
                    // before first
                    // char
                }
            }

            if ((p + i - 1) >= n) {

                return CharStream.EOF;
            }
            return Character.toLowerCase(data[p + i - 1]);
        }
    }

    public static void main(final String[] args) throws Exception {

        String iFileName = SCHEMA_FILE_NAME;
        String oFileName = "/tmp/"
                + new File(iFileName).getName().split("\\.")[0] + ".out";
        String format = SQL_FORMAT;

        CommandLineParser parser = new PosixParser();
        Options options = new Options();
        options.addOption("i", "input-file", true, "default input file = "
                + iFileName);
        options.addOption("o", "output-file", true, "default output file = "
                + oFileName);
        options.addOption("f", "format", true,
                "valid values are sql and binary; the default is sql");
        options.addOption("h", "help", false, "print this message");

        try {
            CommandLine line = parser.parse(options, args);

            if (line.hasOption("help")) {
                HelpFormatter formatter = new HelpFormatter();
                formatter.printHelp("DDLSource", options);
                return;
            }

            if (line.hasOption("input-file")) {
                iFileName = line.getOptionValue("input-file");
            }

            if (line.hasOption("output-file")) {
                oFileName = line.getOptionValue("output-file");
            }

            if (line.hasOption("format")) {
                format = line.getOptionValue("format");
                format = format.toLowerCase();
                if (format.compareTo(BINARY_FORMAT) != 0
                        && format.compareTo(SQL_FORMAT) != 0) {
                    System.out.println("invald format option " + format
                            + "; using default = " + SQL_FORMAT);
                    format = SQL_FORMAT;
                }
            }
        } catch (org.apache.commons.cli.ParseException exp) {
            System.out.println("Unexpected exception:" + exp.getMessage());
        }

        final DDLSource source = new DDLSource();
        final AkibaInformationSchema ais = source.buildAIS(iFileName);
        // AISPrinter.print(ais);

        if (format.compareTo(SQL_FORMAT) == 0) {
            final PrintWriter pw = new PrintWriter(new FileWriter(oFileName));
            SqlTextTarget target = new SqlTextTarget(pw);
            new Writer(target).save(ais);
            target.writeGroupTableDDL(ais);
            pw.close();
        } else {
            assert format.compareTo(BINARY_FORMAT) == 0;

            ByteBuffer rawAis = ByteBuffer.allocate(MAX_AIS_SIZE);
            rawAis.order(ByteOrder.LITTLE_ENDIAN);
            new Writer(new MessageTarget(rawAis)).save(ais);
            rawAis.flip();

            boolean append = false;
            File file = new File(oFileName);
            try {
                FileChannel wChannel = new FileOutputStream(file, append)
                        .getChannel();
                wChannel.write(rawAis);
                wChannel.close();
            } catch (IOException e) {
                throw new Exception("rarrrgh");
            }
        }
    }

    private static class JoinName {
        private final CName parentName;
        private final CName childName;

        private JoinName(CName parentName, CName childName) {
            this.parentName = parentName;
            this.childName = childName;
        }

        @Override
        public boolean equals(Object o) {
            if (this == o) {
                return true;
            }
            if (o == null || getClass() != o.getClass()) {
                return false;
            }
            JoinName joinName = (JoinName) o;
            if (!childName.equals(joinName.childName)) {
                return false;
            }
            if (!parentName.equals(joinName.parentName)) {
                return false;
            }
            return true;
        }

        @Override
        public int hashCode() {
            int result = parentName.hashCode();
            result = 31 * result + childName.hashCode();
            return result;
        }

        @Override
        public String toString() {
            return "Join[parent: " + parentName + " -> child: " + childName
                    + ']';
        }
    }

    public DDLSource() {
        this(0);
    }

    public DDLSource(int baseId) {
        this.baseId = baseId;
    }

    public AkibaInformationSchema buildAIS(final String fileName)
            throws Exception {
        ddlSourceName = fileName;
        return buildAIS(new FileStream(fileName));
    }

    public AkibaInformationSchema buildAISFromString(final String string)
            throws Exception {
        return buildAIS(new StringStream(string));
    }

    private String constructFKJoinName(UserTableDef childTable, IndexDef fkIndex)
    {
        String ret = (fkIndex.getParentSchema() + "/" + fkIndex.getParentTable() + "/" + Strings.join(fkIndex.getParentColumns(), ",") + "/" + childTable.getCName().getSchema() + "/" + childTable.name + "/" + Strings.join(fkIndex.getChildColumns(), ",")).toLowerCase();
        return ret.replace(',','_');
    }
    
    public static SchemaDef parseSchemaDef(final String string)
            throws Exception {
        DDLSource instance = new DDLSource();
        instance.parseSchemaDef(new StringStream(string));
        return instance.schemaDef;
    }

    private void parseSchemaDef(final ANTLRStringStream stringStream)
            throws RecognitionException {
        DDLSourceLexer lex = new DDLSourceLexer(stringStream);
        CommonTokenStream tokens = new CommonTokenStream(lex);
        final DDLSourceParser tsparser = new DDLSourceParser(tokens);
        this.schemaDef = new SchemaDef();
        tsparser.schema(schemaDef);
        if (tsparser.getNumberOfSyntaxErrors() > 0) {
            throw new RuntimeException("DDLSource reported a syntax error in: "
                    + ddlSourceName);
        }
        addImpliedGroups(schemaDef.getMasterSchemaName());
        computeColumnMapAndPositions();
    }

    private AkibaInformationSchema buildAIS(final ANTLRStringStream stringStream)
            throws Exception {
        parseSchemaDef(stringStream);
        AkibaInformationSchema ais = new Reader(this)
                .load(new AkibaInformationSchema());
        return ais;
    }

    public UserTableDef parseCreateTable(final String createTableStatement)
            throws Exception {
        DDLSourceLexer lex = new DDLSourceLexer(new StringStream(
                createTableStatement));
        CommonTokenStream tokens = new CommonTokenStream(lex);
        final DDLSourceParser tsparser = new DDLSourceParser(tokens);
        try {
            final SchemaDef schemaDef = new SchemaDef();
            tsparser.table_spec(schemaDef);
            if (tsparser.getNumberOfSyntaxErrors() > 0) {
                throw new RuntimeException("DDLSource reported a syntax error in: "
                        + ddlSourceName);
            }
            return schemaDef.getCurrentTable();
        } catch (SchemaDef.SchemaDefException e) {
            throw new ParseException(e);
        }
    }

    @Override
    public void close() throws Exception {
    }

    private Long longValue(final String s) {
        return s == null ? null : Long.valueOf(Long.parseLong(s));
    }

    /**
     * Name generator for Group tables.
     * 
     * @param groupName
     * @return
     */
    private String groupTableName(final CName groupName) {
        return "_akiba_" + groupName.getName();
    }

    private CName groupName(final CName rootTable) {
        String ret = groupNames.get(rootTable);
        if (ret == null) {
            ret = rootTable.getName();
            for (int i = 0; groupNames.containsValue(ret); ++i) {
                ret = rootTable.getName() + '$' + i;
            }
            groupNames.put(rootTable, ret);
        }
        return new CName("akiba_objects", ret);
    }

    /**
     * Schema name generator for groups
     * 
     * @return "akiba_objects"
     */
    private String groupSchemaName() {
        return "akiba_objects";
    }

    /**
     * Group column name generator for group columns. TODO: Needs to handle long
     * schemaName / tableName / columnName combos.
     * 
     * @param tableName
     * @param columnName
     * @return
     */
    private String mangledColumnName(final CName tableName,
            final String columnName) {
        return tableName.getName() + "$" + columnName;
    }

    /**
     * Index name generator.
     * 
     * @param utdef
     * @param indexDef
     * @return
     */
    private String mangledIndexName(final UserTableDef utdef,
            final IndexDef indexDef) {
        return utdef.name.getName() + "$" + indexDef.name;
    }

    /**
     * Tests an FOREIGN KEY index definition to determine whether it represents
     * a group-defining relationship.
     * 
     * @param indexDef
     * @return
     */
    private static boolean isAkiban(final IndexDef indexDef) {
        return SchemaDef.isAkiban(indexDef);
    }

    /**
     * Index name generator. This code actually has to (and does) emulate the
     * MySQL algorithm for assigning unspecified names.
     * 
     * @param namingMap
     * @param indexDef
     * @return
     */
    private String generateIndexName(
            final Map<String, AtomicInteger> namingMap, final IndexDef indexDef) {
        final String firstColumnName = indexDef.columns.get(0).columnName;
        AtomicInteger count = namingMap.get(firstColumnName);
        if (count == null) {
            count = new AtomicInteger(1);
            namingMap.put(firstColumnName, count);
            return firstColumnName;
        } else {
            return firstColumnName + "_" + (count.incrementAndGet());
        }
    }

    /**
     * Return List of names of user tables in a specified group sorted by
     * depth-first traversal of the parent-child relationships between those
     * tables.
     * 
     * @param groupName
     * @return sorted List of table names
     */
    List<CName> depthFirstSortedUserTables(final CName groupName) {
        final CName root = new CName(schemaDef, "", "");
        final Map<CName, SortedSet<CName>> hierarchy = new HashMap<CName, SortedSet<CName>>();
        final List<CName> tableList = new ArrayList<CName>();
        for (final CName tableName : schemaDef.getGroupMap().get(groupName)) {
            final UserTableDef utdef = schemaDef.getUserTableMap().get(
                    tableName);
            CName parent = utdef.parent == null ? root : utdef.parent.name;
            SortedSet<CName> children = hierarchy.get(parent);
            if (children == null) {
                children = new TreeSet<CName>();
                hierarchy.put(parent, children);
            }
            children.add(utdef.name);
        }
        traverseDepthFirstSortedTableMap(root, hierarchy, tableList);
        if (tableList.isEmpty()) {
            throw new IllegalStateException("Broken user table hiearchy: "
                    + hierarchy);
        }
        return tableList;
    }

    void traverseDepthFirstSortedTableMap(final CName parent,
            final Map<CName, SortedSet<CName>> hierarchy,
            final List<CName> tableList) {
        SortedSet<CName> siblings = hierarchy.get(parent);
        if (siblings != null) {
            for (final CName tableName : siblings) {
                tableList.add(tableName);
                traverseDepthFirstSortedTableMap(tableName, hierarchy,
                        tableList);
            }
        }
    }

    void computeColumnMapAndPositions() {
        for (final CName groupName : schemaDef.getGroupMap().keySet()) {
            int gposition = 0;
            final List<CName> tableList = depthFirstSortedUserTables(groupName);
            for (final CName tableName : tableList) {
                int uposition = 0;
                final UserTableDef utdef = schemaDef.getUserTableMap().get(
                        tableName);
                List<ColumnDef> columns = utdef.columns;
                for (final ColumnDef def : columns) {
                    def.gposition = gposition++;
                    def.uposition = uposition++;
                    columnDefMap.put(utdef.name + "." + def.name, def);
                }
            }
        }
    }

    @Override
    protected final void read(String typename, Receiver receiver)
            throws Exception {
        throw new UnsupportedOperationException();
    }

    @Override
    public void readColumns(Receiver columnReceiver) throws Exception {
        for (final CName groupName : schemaDef.getGroupMap().keySet()) {
            final List<CName> tableList = depthFirstSortedUserTables(groupName);
            for (final CName tableName : tableList) {
                final UserTableDef utdef = schemaDef.getUserTableMap().get(
                        tableName);
                List<ColumnDef> columns = utdef.columns;
                for (final ColumnDef def : columns) {
                    String groupTableName = groupTableName(groupName);
                    String groupColumnName = mangledColumnName(tableName,
                            def.name);
                    columnReceiver.receive(map(column, tableName.getSchema(),
                            tableName.getName(), def.name, def.uposition,
                            def.typeName, longValue(def.typeParam1),
                            longValue(def.typeParam2), def.nullable,
                            def.defaultAutoIncrement(), groupSchemaName(),
                            groupTableName, groupColumnName, null, null,
                            // TODO: This isn't really correct: if collation is specified but not 
                            // charset, the collation's default charset should be used.
                            // But to do this we need to add a charset/collation database.
                            def.charset == null ? AkibaInformationSchema.DEFAULT_CHARSET : def.charset,
                            def.collate == null ? AkibaInformationSchema.DEFAULT_COLLATION : def.collate));
                    columnReceiver.receive(map(column, groupSchemaName(),
                            groupTableName, groupColumnName, def.gposition,
                            def.typeName, longValue(def.typeParam1),
                            longValue(def.typeParam2), def.nullable,
                            def.defaultAutoIncrement(), null, null, null,
                            null, null,
                            // TODO: This isn't really correct: if collation is specified but not 
                            // charset, the collation's default charset should be used.
                            // But to do this we need to add a charset/collation database.
                            def.charset == null ? AkibaInformationSchema.DEFAULT_CHARSET : def.charset,
                            def.collate == null ? AkibaInformationSchema.DEFAULT_COLLATION : def.collate));
                }
            }
        }
    }

    @Override
    public void readJoins(Receiver joinReceiver) throws Exception {
        for (final CName groupName : schemaDef.getGroupMap().keySet()) {
            for (final CName tableName : schemaDef.getGroupMap().get(groupName)) {
                final UserTableDef utdef = schemaDef.getUserTableMap().get(
                        tableName);
                if (utdef.parent != null) {
{'fraction_non_alphanumeric': 0.094994617868676, 'fraction_numerical': 0.002152852529601722, 'mean_word_length': 1.9509818076812013, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6533135', 'n_tokens_mistral': 11042, 'n_tokens_neox': 10543, 'n_words': 2710}