<commit_before>"""Uses VARBINARY and binary encoding and jams UTF-8 into it.

Needed because GoDaddy uses MySQL 5.0, and even if it were using 5.5+,
the various MySQL connectors are in various states of working or
not.

"""

import sys
import gzip
import json

import xml.sax
import xml.sax.handler
import xml.sax.xmlreader


# Custom expat parser...  Based on
# http://www.vultaire.net/blog/2009/07/20/python-snippet-sax-parser-with-internal-entity-expansion-disabled/.
# (Thanks, me!  Guess I worked on this a lot awhile back and
# forgot...)
from xml.sax.expatreader import ExpatParser
from xml.parsers import expat


class ExpatParserNoEntityExp(ExpatParser):

    """An overridden Expat parser class which disables entity expansion."""

    def __init__(self, *args, **kwargs):
        ExpatParser.__init__(self, *args, **kwargs)

    def reset(self):
        ExpatParser.reset(self)
        self._parser.DefaultHandler = self.__default_handler

    def __default_handler(self, *args):
        pass


class FirstPassContentHandler(xml.sax.handler.ContentHandler):

    """Purpose: to identify required lengths of data and attributes for each table."""

    def __init__(self, conn):
        xml.sax.handler.ContentHandler.__init__(self)
        self.conn = conn
        self.cursor = conn.cursor()
        self.stack = []
        self.elem_len_d = {}
        self.attr_len_d = {}
        self.fk_needed = set()

    def startDocument(self):
        print "Parsing file to determine data structure..."

    def startElement(self, name, attrs):
        # Track whether we need a foreign key.
        # Since we don't record docroot in the db, we check if the
        # stack has more than just docroot.  (i.e. > 1)
        if len(self.stack) > 1:
            self.fk_needed.add(name)

        self.stack.append("")
        for k, v in attrs.items():
            self.attr_len_d.setdefault(name, {})
            self.attr_len_d[name][k] = max(len(v) * 4, self.attr_len_d[name].get(k, 0))

    def endElement(self, name):
        data = self.stack.pop()
        # Again, not doing anything for docroot, so only do something
        # if docroot is still on the stack.
        if len(self.stack) > 0:
            self.elem_len_d[name] = max(len(data) * 4, self.elem_len_d.get(name, 0))

    def characters(self, content):
        self.stack[-1] += content.strip()

    def skippedEntity(self, name):
        self.stack[-1] += "&{0};".format(name)

    def endDocument(self):
        print "Creating database tables..."
        try:
            # Create tables... *to do*
            for elem, data_len in self.elem_len_d.iteritems():
                cols = ["id INTEGER AUTO_INCREMENT PRIMARY KEY"]
                if elem in self.fk_needed:
                    cols.append("fk INTEGER")
                else:
                    cols.append("json MEDIUMBLOB")
                if data_len > 0:
                    cols.append("data VARBINARY({0})".format(data_len))
                attr_d = self.attr_len_d.get(elem, {})
                for attr, data_len in sorted(attr_d.iteritems()):
                    attr = attr.replace(":", "_")
                    cols.append("{0} VARBINARY({1})".format(attr, data_len))
                query = """\
CREATE TABLE `{0}` (
    {1}
) CHARACTER SET binary;""".format(elem, ",\n    ".join(cols))
                try:
                    self.cursor.execute(query)
                except:
                    print query
                    raise
        finally:
            self.cursor.close()


class ContentHandler(xml.sax.handler.ContentHandler):

    """Purpose: populating the tables"""

    def __init__(self, conn, commit_interval):
        xml.sax.handler.ContentHandler.__init__(self)
        self.conn = conn
        self.cursor = conn.cursor()
        self.stack = []
        self.count = 0
        self.commit_interval = commit_interval

    def startDocument(self):
        print "Parsing file again, this time populating the database..."

    def startElement(self, name, attrs):
        if len(self.stack) == 0:
            # docroot; id is dummy value since we don't create a record for this.
            _id = None
        else:
            cols = []
            vals = []
            fk = self.stack[-1]["_id"]
            if fk is not None:  # Parent is JMdict, which we ignore...
                cols.append("fk")
                vals.append(fk)
            for key, value in attrs.items():
                key = key.replace(":", "_")
                cols.append(key)
                vals.append(value.encode('utf-8'))
            cols = ", ".join(cols)
            val_template = ", ".join(["%s" for val in vals])
            query = "INSERT INTO `{0}` ({1}) VALUES ({2});".format(name, cols, val_template)
            try:
                self.cursor.execute(query, vals)
            except:
                print query, vals
                raise
            _id = self.cursor.lastrowid
        self.stack.append({"_id": _id, "_attrs": dict(attrs), "_data": ""})

    def endElement(self, name):
        record = self.stack.pop()
        if len(record["_data"]) > 0:
            query = "UPDATE `{0}` SET DATA=%s WHERE ID=%s".format(name)
            try:
                self.cursor.execute(query, (record["_data"].encode('utf-8'), record["_id"]))
            except:
                print query, (record["_data"], record["_id"])
                raise
        if len(self.stack) == 1:
            json_blob = json.dumps(record, indent=4, sort_keys=True)
            query = "UPDATE `{0}` SET json=%s WHERE ID=%s".format(name)
            try:<commit_msg>Removed JSON pretty-printing from json dump saved to DB.<commit_after>"""Uses VARBINARY and binary encoding and jams UTF-8 into it.

Needed because GoDaddy uses MySQL 5.0, and even if it were using 5.5+,
the various MySQL connectors are in various states of working or
not.

"""

import sys
import gzip
import json

import xml.sax
import xml.sax.handler
import xml.sax.xmlreader


# Custom expat parser...  Based on
# http://www.vultaire.net/blog/2009/07/20/python-snippet-sax-parser-with-internal-entity-expansion-disabled/.
# (Thanks, me!  Guess I worked on this a lot awhile back and
# forgot...)
from xml.sax.expatreader import ExpatParser
from xml.parsers import expat


class ExpatParserNoEntityExp(ExpatParser):

    """An overridden Expat parser class which disables entity expansion."""

    def __init__(self, *args, **kwargs):
        ExpatParser.__init__(self, *args, **kwargs)

    def reset(self):
        ExpatParser.reset(self)
        self._parser.DefaultHandler = self.__default_handler

    def __default_handler(self, *args):
        pass


class FirstPassContentHandler(xml.sax.handler.ContentHandler):

    """Purpose: to identify required lengths of data and attributes for each table."""

    def __init__(self, conn):
        xml.sax.handler.ContentHandler.__init__(self)
        self.conn = conn
        self.cursor = conn.cursor()
        self.stack = []
        self.elem_len_d = {}
        self.attr_len_d = {}
        self.fk_needed = set()

    def startDocument(self):
        print "Parsing file to determine data structure..."

    def startElement(self, name, attrs):
        # Track whether we need a foreign key.
        # Since we don't record docroot in the db, we check if the
        # stack has more than just docroot.  (i.e. > 1)
        if len(self.stack) > 1:
            self.fk_needed.add(name)

        self.stack.append("")
        for k, v in attrs.items():
            self.attr_len_d.setdefault(name, {})
            self.attr_len_d[name][k] = max(len(v) * 4, self.attr_len_d[name].get(k, 0))

    def endElement(self, name):
        data = self.stack.pop()
        # Again, not doing anything for docroot, so only do something
        # if docroot is still on the stack.
        if len(self.stack) > 0:
            self.elem_len_d[name] = max(len(data) * 4, self.elem_len_d.get(name, 0))

    def characters(self, content):
        self.stack[-1] += content.strip()

    def skippedEntity(self, name):
        self.stack[-1] += "&{0};".format(name)

    def endDocument(self):
        print "Creating database tables..."
        try:
            # Create tables... *to do*
            for elem, data_len in self.elem_len_d.iteritems():
                cols = ["id INTEGER AUTO_INCREMENT PRIMARY KEY"]
                if elem in self.fk_needed:
                    cols.append("fk INTEGER")
                else:
                    cols.append("json MEDIUMBLOB")
                if data_len > 0:
                    cols.append("data VARBINARY({0})".format(data_len))
                attr_d = self.attr_len_d.get(elem, {})
                for attr, data_len in sorted(attr_d.iteritems()):
                    attr = attr.replace(":", "_")
                    cols.append("{0} VARBINARY({1})".format(attr, data_len))
                query = """\
CREATE TABLE `{0}` (
    {1}
) CHARACTER SET binary;""".format(elem, ",\n    ".join(cols))
                try:
                    self.cursor.execute(query)
                except:
                    print query
                    raise
        finally:
            self.cursor.close()


class ContentHandler(xml.sax.handler.ContentHandler):

    """Purpose: populating the tables"""

    def __init__(self, conn, commit_interval):
        xml.sax.handler.ContentHandler.__init__(self)
        self.conn = conn
        self.cursor = conn.cursor()
        self.stack = []
        self.count = 0
        self.commit_interval = commit_interval

    def startDocument(self):
        print "Parsing file again, this time populating the database..."

    def startElement(self, name, attrs):
        if len(self.stack) == 0:
            # docroot; id is dummy value since we don't create a record for this.
            _id = None
        else:
            cols = []
            vals = []
            fk = self.stack[-1]["_id"]
            if fk is not None:  # Parent is JMdict, which we ignore...
                cols.append("fk")
                vals.append(fk)
            for key, value in attrs.items():
                key = key.replace(":", "_")
                cols.append(key)
                vals.append(value.encode('utf-8'))
            cols = ", ".join(cols)
            val_template = ", ".join(["%s" for val in vals])
            query = "INSERT INTO `{0}` ({1}) VALUES ({2});".format(name, cols, val_template)
            try:
                self.cursor.execute(query, vals)
            except:
                print query, vals
                raise
            _id = self.cursor.lastrowid
        self.stack.append({"_id": _id, "_attrs": dict(attrs), "_data": ""})

    def endElement(self, name):
        record = self.stack.pop()
        if len(record["_data"]) > 0:
            query = "UPDATE `{0}` SET DATA=%s WHERE ID=%s".format(name)
            try:
                self.cursor.execute(query, (record["_data"].encode('utf-8'), record["_id"]))
            except:
                print query, (record["_data"], record["_id"])
                raise
        if len(self.stack) == 1:
            json_blob = json.dumps(record)
            query = "UPDATE `{0}` SET json=%s WHERE ID=%s".format(name)
            try:
{'fraction_non_alphanumeric': 0.12514443160607946, 'fraction_numerical': 0.007377122033597014, 'mean_word_length': 1.9525059039622146, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 20}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1638202', 'n_tokens_mistral': 3483, 'n_tokens_neox': 3286, 'n_words': 889}