<commit_before>from xml.dom import minidom
from types import *
import datetime
import csv
import os
import re
import sqlite3
import copy
import unicodedata


class XMLPatent(object):
    """
    Base class for handling parsing of xml fields
    """

    def __init__(self, xmlstring):
        """
        [xmlstring]: string containing xml document. Should be provided by 
        parse.py
        """
        # lowercase all tags
        self.xmlstring = re.sub(r"<[/]?[A-Za-z-]+?[ >]", lambda x: x.group().lower(), xmlstring)
        # store the minidom parsed xml doc
        self.xmldoc = minidom.parseString(self.xmlstring)
        
        # country, patent, kind, date_grant
        self.country, self.patent, self.kind, self.date_grant = self.__tagNme__(xmldoc, ["publication-reference", ["country", "doc-number", "kind", "date"]])
        # pat_type
        self.pat_type = self.__tagNme__(xmldoc, ["application-reference"], iHTML=False)[0].attributes["appl-type"].value
        # date_app, country_app, patent_app
        self.date_app, self.country_app, self.patent_app = self.__tagNme__(xmldoc, ["application-reference", ["date", "country", "doc-number"]])
        # code_app
        self.code_app = self.__tagNme__(xmldoc, ["us-application-series-code"])
        # clm_num
        self.clm_num = self.__tagNme__(xmldoc, ["number-of-claims"])
        # classes
        self.classes = [[x[:3].replace(' ',''), x[3:].replace(' ','')] for x in self.__tagNme__(xmldoc, ["classification-national", ["main-classification", "further-classification"]], idx=1, listType=True)]
        # abstract
        self.abstract = self.__allHTML__(xmldoc, ["abstract", "p"])
        # invention_title
        self.invention_title = self.__allHTML__(xmldoc, ["invention-title"])
        # asg_list
        self.asg_list = self.__asg_detail__(self.__tagNme__(xmldoc, ["assignees", "assignee"], iHTML=False))
        # cit_list
        self.cit_list = self.__cit_detail__(self.__tagNme__(xmldoc, ["references-cited", "citation"], iHTML=False))
        # rel_list
        self.rel_list = self.__rel_detail__(self.__tagNme__(xmldoc, ["us-related-documents"], iHTML=False))
        # inv_list
        self.inv_list = self.__tagSplit__(xmldoc, ["parties", "applicant"], [["addressbook", ["last-name", "first-name"]], ["addressbook", "address", ["street", "city", "state", "country", "postcode"]], [["nationality", "residence"], "country"]], blank=True)
        # law_list
        self.law_list = self.__tagSplit__(xmldoc, ["parties", "agents", "agent"], [["addressbook", ["last-name", "first-name", "country", "orgname"]]], blank=True)

        """
        Callback structure to handle the build_table specifics
        """
        self.tablecallbacks = {
            'assignee': self.build_assignee,
            'citation': self.build_citation,
            'class': self.build_class,
            'inventor': self.build_inventor,
            'patent': self.build_patent,
            'patdesc': self.build_patdesc,
            'lawyer': self.build_lawyer,
            'sciref': self.build_sciref,
            'usreldoc': self.build_usreldoc
            }


    def build_table(self, tablename):
        """
        Return list of requisite fields for inserting this patent into [tablename]
        """
        return self.tablecallbacks[tablename]()

    def build_assignee(self):
        ack = []
        for i,y in enumerate(self.asg_list):
            if not y[0]:
                ack.extend([[self, y[2], y[1], y[4], y[5], y[7], y[8], i]])
            else:
                ack.extend([[self, "00", "%s, %s" % (y[2], y[1]), y[4], y[5], y[6], y[7], y[8], i]])
        return ack

    def build_citation(self):
        ack = []
        for i,y in enumerate([x for x in self.cit_list if x[1] != ""]):
            ack.extend([[self, y[3], y[5], y[4], y[1], y[2], y[0], i]])
        return ack

    def build_class(self):
        ack = []
        for i,y in enumerate(self.classes):
            ack.extend([[self, (i==0)*1, y[0], y[1]]])
        return ack

    def build_inventor(self):
        ack = []
        for i,y in enumerate(self.inv_list):
            ack.extend([[self, y[1], y[0], y[2], y[3], y[4], y[5], y[6], y[8], i]])
        return ack

    def build_self(self):
        return [[self, self.kind, self.clm_num, self.code_app, self_app, self.date_grant, self.date_grant[:4], self.date_app, self.date_app[:4], self.pat_type]]

    def build_patdesc(self):
        return [[self, self.abstract, self.invention_title]]

    def build_lawyer(self):
        ack = []
        for i,y in enumerate(self.law_list):
            ack.extend([[self, y[1], y[0], y[2], y[3], i]])
        return ack

    def build_sciref(self):
        ack = []
        for i,y in enumerate([y for y in self.cit_list if y[1] == ""]):
            ack.extend([[self, y[-1], i]])
        return ack

    def build_usreldoc(self):
        ack = []
        for i,y in enumerate(self.rel_list):
            if y[1] == 1:
                ack.extend([[self, y[0], y[1], y[3], y[2], y[4], y[5], y[6]]])
            else:
                ack.extend([[self, y[0], y[1], y[3], y[2], y[4], "", ""]])
        return ack


    """
    These are all methods from the old XMLPatent class. I'm keeping them
    in here to assist in the carry-over of parsing the xml fields
    """
    def __allHTML__(self, xmldoc, tagList):
        """
        Replaces the html tags for everything returned by __tagName__(taglist)
        with nothing
        """
        for x in self.__tagNme__(xmldoc, tagList, iHTML=False):
            return re.sub(r"<[/]?%s( .*?)?>" % (tagList[-1]), "", x.toxml())
        return ""

    def __innerHTML__(self, dom_element):
        """
        Gets the text out of all child nodes for a certain<commit_msg>add insert methods for db<commit_after>from xml.dom import minidom
from types import *
import datetime
import csv
import os
import re
import sqlite3
import copy
import unicodedata


class XMLPatent(object):
    """
    Base class for handling parsing of xml fields
    """

    def __init__(self, xmlstring):
        """
        [xmlstring]: string containing xml document. Should be provided by 
        parse.py
        """
        # lowercase all tags
        self.xmlstring = re.sub(r"<[/]?[A-Za-z-]+?[ >]", lambda x: x.group().lower(), xmlstring)
        # store the minidom parsed xml doc
        self.xmldoc = minidom.parseString(self.xmlstring)
        
        # country, patent, kind, date_grant
        self.country, self.patent, self.kind, self.date_grant = self.__tagNme__(xmldoc, ["publication-reference", ["country", "doc-number", "kind", "date"]])
        # pat_type
        self.pat_type = self.__tagNme__(xmldoc, ["application-reference"], iHTML=False)[0].attributes["appl-type"].value
        # date_app, country_app, patent_app
        self.date_app, self.country_app, self.patent_app = self.__tagNme__(xmldoc, ["application-reference", ["date", "country", "doc-number"]])
        # code_app
        self.code_app = self.__tagNme__(xmldoc, ["us-application-series-code"])
        # clm_num
        self.clm_num = self.__tagNme__(xmldoc, ["number-of-claims"])
        # classes
        self.classes = [[x[:3].replace(' ',''), x[3:].replace(' ','')] for x in self.__tagNme__(xmldoc, ["classification-national", ["main-classification", "further-classification"]], idx=1, listType=True)]
        # abstract
        self.abstract = self.__allHTML__(xmldoc, ["abstract", "p"])
        # invention_title
        self.invention_title = self.__allHTML__(xmldoc, ["invention-title"])
        # asg_list
        self.asg_list = self.__asg_detail__(self.__tagNme__(xmldoc, ["assignees", "assignee"], iHTML=False))
        # cit_list
        self.cit_list = self.__cit_detail__(self.__tagNme__(xmldoc, ["references-cited", "citation"], iHTML=False))
        # rel_list
        self.rel_list = self.__rel_detail__(self.__tagNme__(xmldoc, ["us-related-documents"], iHTML=False))
        # inv_list
        self.inv_list = self.__tagSplit__(xmldoc, ["parties", "applicant"], [["addressbook", ["last-name", "first-name"]], ["addressbook", "address", ["street", "city", "state", "country", "postcode"]], [["nationality", "residence"], "country"]], blank=True)
        # law_list
        self.law_list = self.__tagSplit__(xmldoc, ["parties", "agents", "agent"], [["addressbook", ["last-name", "first-name", "country", "orgname"]]], blank=True)

        """
        Callback structure to handle the build_table specifics
        """
        self.tablecallbacks = {
            'assignee': self.build_assignee,
            'citation': self.build_citation,
            'class': self.build_class,
            'inventor': self.build_inventor,
            'patent': self.build_patent,
            'patdesc': self.build_patdesc,
            'lawyer': self.build_lawyer,
            'sciref': self.build_sciref,
            'usreldoc': self.build_usreldoc
            }


    def build_table(self, tablename):
        """
        Return list of requisite fields for inserting this patent into [tablename]
        """
        return self.tablecallbacks[tablename]()

    def build_assignee(self):
        ack = []
        for i,y in enumerate(self.asg_list):
            if not y[0]:
                ack.extend([[self, y[2], y[1], y[4], y[5], y[7], y[8], i]])
            else:
                ack.extend([[self, "00", "%s, %s" % (y[2], y[1]), y[4], y[5], y[6], y[7], y[8], i]])
        return ack

    def build_citation(self):
        ack = []
        for i,y in enumerate([x for x in self.cit_list if x[1] != ""]):
            ack.extend([[self, y[3], y[5], y[4], y[1], y[2], y[0], i]])
        return ack

    def build_class(self):
        ack = []
        for i,y in enumerate(self.classes):
            ack.extend([[self, (i==0)*1, y[0], y[1]]])
        return ack

    def build_inventor(self):
        ack = []
        for i,y in enumerate(self.inv_list):
            ack.extend([[self, y[1], y[0], y[2], y[3], y[4], y[5], y[6], y[8], i]])
        return ack

    def build_self(self):
        return [[self, self.kind, self.clm_num, self.code_app, self_app, self.date_grant, self.date_grant[:4], self.date_app, self.date_app[:4], self.pat_type]]

    def build_patdesc(self):
        return [[self, self.abstract, self.invention_title]]

    def build_lawyer(self):
        ack = []
        for i,y in enumerate(self.law_list):
            ack.extend([[self, y[1], y[0], y[2], y[3], i]])
        return ack

    def build_sciref(self):
        ack = []
        for i,y in enumerate([y for y in self.cit_list if y[1] == ""]):
            ack.extend([[self, y[-1], i]])
        return ack

    def build_usreldoc(self):
        ack = []
        for i,y in enumerate(self.rel_list):
            if y[1] == 1:
                ack.extend([[self, y[0], y[1], y[3], y[2], y[4], y[5], y[6]]])
            else:
                ack.extend([[self, y[0], y[1], y[3], y[2], y[4], "", ""]])
        return ack

    def insert_assignee(self):
        conn = sqlite3.connect("assignee.sqlite3")
        c = conn.cursor()
        c.executescript("""
            CREATE TABLE IF NOT EXISTS assignee (
                Patent VARCHAR(8),      AsgType INTEGER,        Assignee VARCHAR(30),
                City VARCHAR(10),       State VARCHAR(2),       Country VARCHAR(2),
                Nationality VARCHAR(2), Residence VARCHAR(2),   AsgSeq INTEGER);
            CREATE UNIQUE INDEX IF NOT EXISTS uqAsg ON assignee (Patent, AsgSeq);
            """)
        c.executemany("""INSERT OR IGNORE INTO %s VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""" % tbl, \
            self.tablecallbacks["assignee"]())
    
    def insert_citation(self):
        conn = sqlite3.connect("citation.sqlite3")
        c = conn.cursor()
        c.executescript("""
            CREATE TABLE IF NOT EXISTS citation (
                Patent VARCHAR(8),      Cit_Date INTEGER,       Cit_Name VARCHAR(10),
                Cit_Kind VARCHAR(1),    Cit_Country VARCHAR(2), Citation VARCHAR(8),
                Category VARCHAR(15),   CitSeq INTEGER);
            CREATE UNIQUE INDEX IF NOT EXISTS uqCit ON citation (Patent, CitSeq);
            """)
        c.executemany("""INSERT OR IGNORE INTO %s VALUES (?, ?, ?, ?, ?, ?, ?, ?)""" % tbl, \
            self.tablecallbacks["citation"]())

    def insert_class(self):
        conn = sqlite3.connect("class.sqlite3")
        c = conn.cursor()
        c.executescript("""
            CREATE TABLE IF NOT EXISTS class (
                Patent VARCHAR(8),      Prim INTEGER,
                Class VARCHAR(3),       SubClass VARCHAR(3));
            CREATE UNIQUE INDEX IF NOT EXISTS uqClass ON class (Patent, Class, SubClass);
            """)
        c.executemany("""INSERT OR IGNORE INTO %s VALUES (?, ?, ?, ?)""" % tbl, \
            self.tablecallbacks["class"]())

    def insert_inventor(self):
        conn = sqlite3.connect("inventor.sqlite3")
        c = conn.cursor()
        c.executescript("""
            CREATE TABLE IF NOT EXISTS inventor (
                Patent VARCHAR(8),      Firstname VARCHAR(15),  Lastname VARCHAR(15),
                Street VARCHAR(15),     City VARCHAR(10),
                State VARCHAR(2),       Country VARCHAR(12),
                Zipcode VARCHAR(5),     Nationality VARCHAR(2), InvSeq INTEGER);
            CREATE UNIQUE INDEX IF NOT EXISTS uqInv ON inventor (Patent, InvSeq);
            """)
        c.executemany("""INSERT OR IGNORE INTO %s VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""" % tbl, \
            self.tablecallbacks["inventor"]())


    def insert_patent(self):
        conn = sqlite3.connect("patent.sqlite3")
        c = conn.cursor()
        c.executescript("""
            CREATE TABLE IF NOT EXISTS patent (
                Patent VARCHAR(8),      Kind VARCHAR(3),        Claims INTEGER,
                AppType INTEGER,        AppNum VARCHAR(8),
                GDate INTEGER,          GYear INTEGER,
                AppDate INTEGER,        AppYear INTEGER, PatType VARCHAR(15) );
            CREATE UNIQUE INDEX IF NOT EXISTS uqPat on patent (Patent);
            """)
        c.executemany("""INSERT OR IGNORE INTO %s VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""" % tbl, \
            self.tablecallbacks["patent"]())

    def insert_patdesc(self):
        conn = sqlite3.connect("patdesc.sqlite3")
        c = conn.cursor()
        c.executescript("""
            CREATE TABLE IF NOT EXISTS patdesc (
                Patent VARCHAR(8),
                Abstract VARCHAR(50),   Title VARCHAR(20));
            CREATE UNIQUE INDEX IF NOT EXISTS uqPatDesc ON patdesc (Patent);
            """)
        c.executemany("""INSERT OR IGNORE INTO %s VALUES (?, ?, ?)""" % tbl, \
            self.tablecallbacks["patdesc"]())


    def insert_lawyer(self):
        conn = sqlite3.connect("lawyer.sqlite3")
        c = conn.cursor()
        c.executescript("""
            CREATE TABLE IF NOT EXISTS lawyer (
                Patent VARCHAR(8),      Firstname VARCHAR(15),  Lastname VARCHAR(15),
                LawCountry VARCHAR(2),  OrgName VARCHAR(20),    LawSeq INTEGER);
            CREATE UNIQUE INDEX IF NOT EXISTS uqLawyer ON lawyer (Patent, LawSeq);
            """)
        c.executemany("""INSERT OR IGNORE INTO %s VALUES (?, ?, ?, ?, ?, ?)""" % tbl, \
            self.tablecallbacks["lawyer"]())


    def insert_sciref(self):
        conn = sqlite3.connect("sciref.sqlite3")
        c = conn.cursor()
        c.executescript("""
            CREATE TABLE IF NOT EXISTS sciref (
                Patent VARCHAR(8),      Descrip VARCHAR(20),    CitSeq INTEGER);
            CREATE UNIQUE INDEX IF NOT EXISTS uqSciref ON sciref (Patent, CitSeq);
            """)
        c.executemany("""INSERT OR IGNORE INTO %s VALUES (?, ?, ?)""" % tbl, \
            self.tablecallbacks["sciref"]())


    def insert_usreldoc(self):
        conn = sqlite3.connect("usreldoc.sqlite3")
        c = conn.cursor()
        c.executescript("""
            CREATE TABLE IF NOT EXISTS usreldoc (
                Patent VARCHAR(8),      DocType VARCHAR(10),    OrderSeq INTEGER,
                Country VARCHAR(2),     RelPatent VARCHAR(8),   Kind VARCHAR(2),
                RelDate INTEGER,        Status VARCHAR(10));
            CREATE UNIQUE INDEX IF NOT EXISTS uqUSRelDoc ON usreldoc (Patent, OrderSeq);
            """)
        c.executemany("""INSERT OR IGNORE INTO %s VALUES (?, ?, ?, ?, ?, ?, ?, ?)""" % tbl, \
            self.tablecallbacks["usreldoc"]())


    """
    These are all methods from the old XMLPatent class. I'm keeping them
    in here to assist in the carry-over of parsing the xml fields
    """
    def __allHTML__(self, xmldoc, tagList):
        """
        Replaces the html tags for everything returned by __tagName__(taglist)
        with nothing
        """
        for x in self.__tagNme__(xmldoc, tagList, iHTML=False):
            return re.sub(r"<[/]?%s( .*?)?>" % (tagList[-1]), "", x.toxml())
        return ""

    def __innerHTML__(self, dom_element):
        """
        Gets the text out of all child nodes for a certain
{'fraction_non_alphanumeric': 0.1532607423018382, 'fraction_numerical': 0.012118018967334035, 'mean_word_length': 2.3074540174249756, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 54}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '453748', 'n_tokens_mistral': 5847, 'n_tokens_neox': 5434, 'n_words': 1296}