<jupyter_start><jupyter_text>This is demonstration only. (AND now out of date. See semantics-preprocessing for current code.)<jupyter_code>%reload_ext autoreload
%autoreload 2

import json as js  # name conflict with sqla
import sqlalchemy as sqla
from sqlalchemy.orm import sessionmaker

from mpp.models import Response
# from semproc.unique_identifiers import IdentifierExtractor

import re
from semproc.bag_parser import BagParser
from semproc.nlp_utils import load_token_list
from semproc.utils import unquote, break_url, match
from semproc.xml_utils import extract_items
import dateutil.parser as dateparser
from datetime import datetime
from itertools import chain, izip
from rfc3987 import parse as uparse

import warnings
warnings.filterwarnings('ignore')
# for sorting out excludes

_pattern_set = [
    ('doi', re.compile(ur"((?:https?:\/\/){1}(?:dx\.)?doi\.org\/(10[.][0-9]{4,}(?:[/][0-9]+)*/(?:(?![\"&\\'])\S)+))", re.IGNORECASE)),
    # from a hdl.handle.net registry; don't need this.
    # ('hdl', re.compile(ur'((?:https?:\/\/){1}(?:hdl\.)?handle\.net\/(?:(?:\w)*:#!\/)?(?:[\w\-]*\/)*([\w\-\.]*)/{0,1})', re.IGNORECASE)),
    #('url', re.compile(ur"""(?i)\b((?:https?:(?:/{1,3}|[a-z0-9%])|[a-z0-9.\-]+[.](?:com|net|org|edu|gov|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|name|post|pro|tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|Ja|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw)/)(?:[^\s()<>{}\[\]]+|\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\))+(?:\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\)|[^\s`!()\[\]{};:'".,<>?«»“”‘’])|(?:(?<!@)[a-z0-9]+(?:[.\-][a-z0-9]+)*[.](?:com|net|org|edu|gov|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|name|post|pro|tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|Ja|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw)\b/?(?!@)))""", re.IGNORECASE)),
    ('urn', re.compile(ur'^([a-z0-9.#]{0,}:([a-z0-9][a-z0-9.-]{0,31}):[a-z0-9A-Z_()+,-.:=@;$!*\'%/?#\[\]]+)', re.IGNORECASE)),
    ('uuid', re.compile(ur'([a-f\d]{8}(-[a-f\d]{4}){3}-[a-f\d]{12}?)', re.IGNORECASE)),
    ('doi', re.compile(ur"((doi:){0,1}10[.][0-9]{4,}(?:[/][0-9]+)*/(?:(?![\"&\\'])\S)+)", re.IGNORECASE)),
    ('md5', re.compile(ur"(\b([A-Fa-f0-9]{32}))\b", re.IGNORECASE))
]
_rule_set = [
    ('uri', 'fileIdentifier/CharacterString'),  # ISO
    ('uri', 'identifier/'),
    ('uri', 'dataSetURI/CharacterString'),
    ('uri', 'parentIdentifier/CharacterString'),
    ('uri', 'Entry_ID'),  # DIF
    ('uri', 'dc/identifier'),  # DC
    ('basic', 'Layer/Name'),  # WMS
    ('basic', 'dataset/ID'),  # THREDDS
    ('uri', '@URI'),  # ddi
    ('uri', '@IDNo'),  # ddi
    ('uri', '@ID'),
    ('uri', 'datsetid')
]


class Identifier(object):
    def __init__(
            self,
            tag,
            extraction_type,
            match_type,
            original_text,
            potential_identifier):
        self.tag = tag
        self.extraction_type = extraction_type
        self.match_type = match_type
        self.original_text = original_text
        self.potential_identifier = potential_identifier

    def __eq__(self, other):
        return (isinstance(other, self.__class__)
                and self.__dict__ == other.__dict__)

    def __ne__(self, other):
        return not self.__eq__(other)

    def has(self, comparison_identifier):
        return self.potential_identifier == comparison_identifier

    def __repr__(self):
        return '<Identifier %s>' % js.dumps(self.__dict__)

    def to_json(self):
        return {
            "tag": self.tag,
            "extraction_type": self.extraction_type,
            "match_type": self.match_type,
            "original_text": self.original_text,
            "potential_identifier": self.potential_identifier
        }


class IdentifierExtractor(object):
    def __init__(
            self,
            source_url,
            source_xml_as_str,
            equality_excludes=[],
            contains_excludes=[],
            tag_excludes=[]
        ):
        self.source_url = source_url
        self.source_xml_as_str = source_xml_as_str
        self.identifieds = []
        self.texts = [('', self.source_url)]
        self.seen_texts = []
        self.equality_excludes = equality_excludes
        self.contains_excludes = contains_excludes
        self.tag_excludes = tag_excludes

        self._parse()

    def _parse(self):
        try:
            parser = BagParser(self.source_xml_as_str, True, False)
        except Exception as ex:
            print ex
            raise ex
        if not parser or parser.parser.xml is None:
            raise Exception('failed to parse')

        for tag, txt in parser.strip_text():
            # if it's from a tag we know we nee to exclude
            if any(t in tag for t in self.tag_excludes):
                continue
                
            if not txt.strip():
                continue
                
            # do not split if it comes form an identifier field
            self.texts += ((tag, t) for t in txt.split()) if not any(r[1] in tag for r in _rule_set) else [(tag, txt)]           
            
    def _strip_punctuation(self, text):
        terminal_punctuation = '(){}[],~|":&-<>.'
        text = text.strip(terminal_punctuation)
        return text.strip()

    def _strip_dates(self, text):
        # this should still make it an invalid date
        # text = text[3:] if text.startswith('NaN') else text
        def try_format(fmt):
            try:
                d = datetime.strptime(text, fmt)
            except:
                return False
            return True
                
        try:
            d = dateparser.parse(text)
            return ''
        except ValueError:
            pass
        
        known_formats = ['%d/%m/%Y [%H:%M:%S:%f]', '%H:%M:%S%f']
        tests = [try_format(kf) for kf in known_formats]
        return '' if sum(tests) > 0 else text

    def _strip_scales(self, text):
        #'1:100:000'
        scale_pttn = ur"(1:[\d]{0,}(,[\d]{3}){1,})"
        m = match(text, scale_pttn)
        if m:
            return ''
        return text

    def _strip_excludes(self, match_type, tag, text):
        # TODO: how to handle the excludes for skipping
        #       by tag and skipping by value and what was
        #       the thinking a few months ago?
        
        # if it came from any known elem/attrib
        if any([t.lower() in tag.lower() for t in self.tag_excludes]):
            return ''
        
        # if it's a url, must equal an item
        if match_type == 'url':
            if text in self.equality_excludes:
                # equality only
                return ''
            return text

        # if it's *not* a url and contains any of these patterns
        if any(e.lower() in text.lower() for e in self.contains_excludes):
            return ''
        return text

    def _strip_whitespace(self, text):
        space_pattern = re.compile(' ')
        if space_pattern.subn(' ', text)[1] > 0:
            # i actually don't know if this is the right index. huh.
            return text.split(' ')[0]
        return text

    def _tidy_text(self, text):
        text = self._strip_punctuation(text)
        if not text:
            return ''

        text = self._strip_scales(text)
        if not text:
            return ''

        text = self._strip_dates(text)
        if not text:
            return ''

        text = self._strip_whitespace(text)
        if not text:
            return ''

        return text

    def _verify_cool_uri(self, potential_uri):
        parts = potential_uri.split('/')
        if len(parts) < 2:
            return ''

        # it should start with a slash or end with a filename
        # that's not really an accurate understanding of the things
        # they are route paths, non-unique and are difficult
        # to categorize given how badly we put things in the xml
        starts_with = '/' == potential_uri[0]
        ends_with = len(parts[-1].split('.')) > 1 or '/' == potential_uri[-1]
        if not starts_with or not ends_with:
            return ''
        return potential_uri
    
    def _verify_urn(self, potential_urn):
        # not much point to this but we're just going to leave it
        pttn = re.compile(ur'^([a-z0-9.#]{0,}:([a-z0-9][a-z0-9.-]{0,31}):[a-z0-9A-Z_()+,-.:=@;$!*\'%/?#\[\]]+)',
            re.IGNORECASE)
        m = match(potential_urn, pttn)
        return potential_urn if m else ''

    def _verify_url(self, potential_url):
        # ugh
        if 'mailto:' in potential_url:
            return ''
        
        # is it a urn?
        pttn = re.compile(ur'^([a-z0-9]{0,}:[a-z0-9][a-z0-9-]{0,31}:[a-z0-9()+,\-.:=@;$_!*\'%/?#\[\]]+)', re.IGNORECASE)
        urn_m = match(potential_url, pttn)
        if urn_m:
            return ''

        # does it even have a scheme?
        try:
            u = uparse(potential_url, rule='URI')
            if not u.get('scheme'):
                # will consider the leading blob of a URN
                # a scheme even if we don't so this is not 
                # 100% reliable
                return ''
            
            parts = potential_url.split(':', 1)
            if not parts[1].startswith('/') or len(parts) < 2:
                # so if it's our false urn scheme, we
                # say it has to be :// or it's not a url
                # and it has to have a split
                return ''
        except Exception as ex:
            return ''

        pttn = re.compile(ur"""(?i)\b((?:https?:(?:/{1,3}|[a-z0-9%])|[a-z0-9.\-]+[.](?:com|net|org|edu|gov|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|name|post|pro|tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|Ja|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw)/)(?:[^\s()<>{}\[\]]+|\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\))+(?:\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\)|[^\s`!()\[\]{};:'".,<>?«»“”‘’])|(?:(?<!@)[a-z0-9]+(?:[.\-][a-z0-9]+)*[.](?:com|net|org|edu|gov|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|name|post|pro|tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|Ja|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw)\b/?(?!@)))""", re.IGNORECASE)
        m = match(potential_url, pttn)
        if not m:
            return ''
        return potential_url

    def _extract_url(self, text):
        # but really first, is it a urn?
        text = self._verify_url(text)
        if not text:
            return '', '', []
        url = self._tidy_text(unquote(text))
        base_url, values = break_url(url)
        values = values.split(' ') + [base_url] if base_url else []
        
        if url == 'http://dx.doi.org':
            return '', '', []
        
        if 'dx.doi.org' in base_url:
            t = 'doi'
        elif 'hdl.handle.net' in base_url:
            t = 'hdl'
        else:
            t = 'url'

        # return the original extracted url, tag, and the values plus 
        # the base_url for more extracting
        return url, t, filter(None, [self._tidy_text(v) for v in values])

    def _extract_identifiers(self, text):
        # make sure it's not a date first
        text = self._strip_dates(text)
        if not text:
            yield '', ''
        for pattern_type, pattern in _pattern_set:                
            m = match(text, pattern)
            if not m:
                continue

            # if the pattern type doesn't match the
            # verification ie urn is not actually urn
            # bounce (this is mostly for urls)
            urn_verified = self._verify_urn(m)
            is_urn = urn_verified != ''
            if urn_verified:
                yield 'urn', m
            
            if pattern_type == 'urn' and not is_urn:
                continue
                
            url_verified = self._verify_url(m)
            is_url = url_verified.strip() != ''
            if is_url and not is_urn:
                if 'dx.doi.org' in m:
                    t = 'doi'
                elif 'hdl.handle.net' in m:
                    t = 'hdl'
                else:
                    t = 'url'
                yield t, m
 
            if pattern_type == 'url' and not is_url:
                continue
            
            m = self._tidy_text(m)
            if not m:
                continue
            yield pattern_type, m

    def _check(self, comparison_identifier, arr):
        return len([a for a in arr if a.has(comparison_identifier)]) > 0

    def process_text(self):
        def _append(potential_identifier):
            # clean it up (make sure it's still a good thing)
            # before checking against knowns and appending
            cleaned_text = self._tidy_text(potential_identifier.potential_identifier) \
                if potential_identifier.extraction_type != 'rule_set' and \
                potential_identifier.match_type != 'text' else potential_identifier.potential_identifier
            if cleaned_text:
                potential_identifier.potential_identifier = cleaned_text
            if cleaned_text and not self._check(
                    cleaned_text, self.identifieds):
                self.identifieds.append(potential_identifier)
            if not self._check(potential_identifier.original_text, self.seen_texts):
                self.texts.append((potential_identifier.tag, cleaned_text))
                self.seen_texts.append(potential_identifier)
        
        def _urlify(potential_url, ext_type='extract'):
            url, utype, values = self._extract_url(potential_url)
            values = [v for v in values if not self._check(v, self.seen_texts)]
            self.texts += list(iter(izip([tag] * len(values), values)))
                
            if url and not self._check(url, self.identifieds) \
                    and not self._check(url, self.seen_texts):
                _append(Identifier(tag, ext_type, utype, text, url))
        
        while self.texts:
            tag, text = self.texts.pop()
            if self._check(text, self.seen_texts) or not text.strip():
                continue

            try:
                j = json.loads(text)
                j.keys()  # it will decode a quoted string without error
                continue
            except:
                pass
            
            # check the tag against the rule set 
            # we don't need to worry about xpaths
            # just a bit of string matching - we've
            # extracted the tags
            if any(r[1] in tag for r in _rule_set):
                # see if it matches a pattern 
                # better to know it's a doi or url
                
                # the url
                _urlify(text, 'rule_set')
                
                # anything else
                for match_type, match_text in self._extract_identifiers(text):
                    if not match_type and not match_text:
                        continue
                    # add it if it matches any pattern
                    _append(Identifier(tag, 'rule_set', match_type, text, match_text))
                    
                # if nothing else is a match, just add the text, as is
                # it is not tracking the type of the rule, fyi
                _append(Identifier(tag, 'rule_set', 'text', text, text))
                    
            # check if the text is a url        
            _urlify(text)
            
            # check if it's a cool uri
            uri_verified = self._verify_cool_uri(text)
            if uri_verified:
                _append(Identifier(tag, 'extract', 'cooluri', text, text))

            # now run the OTHER regex
            for match_type, match_text in self._extract_identifiers(text):
                if not match_type and not match_text:
                    continue
                _append(Identifier(tag, 'regex', match_type, text, match_text))
                    
        # exclude from here
        for i, j in enumerate(self.identifieds):
            # type tag text
            text = self._strip_excludes(j.match_type, j.tag, j.potential_identifier)
            if text:
                yield j
        
# exclude lists

# the equality excludes 
equality = []
files = ['namespaces.txt']
for f in files:
    equality += load_token_list(f)
    
# equality = ['http://standards.iso.org/ittf/PubliclyAvailableStandards/ISO_19139_Schemas/resources/Codelist/gmxCodelists.xml']
    
contains = []
files = ['cat_interop_urns.txt', 'mimetypes.txt', 'namespaces.txt', 'excludes_by_contains.txt']
for f in files:
    contains += load_token_list(f)

tag_paths = ['@codeList', '@schemaLocation', 'identifier/@type', 
             'Details/Licence', 'binding/@transport', '@template',
            '@uri-type', 'rights/@resource', 'digform/digtinfo/formspec',
            'license/@href', 'definition/@href', 'dateTime/formatString',
            "RDF/Description/@about", 'Binary/Thumbnail/Data'
            ]
# grab the clean text from the rds
with open('../local/big_rds.conf', 'r') as f:
    conf = js.loads(f.read())

# our connection
engine = sqla.create_engine(conf.get('connection'))
Session = sessionmaker()
Session.configure(bind=engine)
session = Session()
response_id = 599943
response_url, response_cleaned, = session.query(
    Response.source_url,
    Response.cleaned_content
).filter(Response.id==response_id).first()
extractor = IdentifierExtractor(response_url, response_cleaned, equality, contains, tag_paths)
extractor._parse()
extractor.texts
for i in extractor.process_text():
    print i.to_json()
# let's run some identifiers (a few k to see what borks, etc)

session.rollback()

with open('../local/local_pg.conf', 'r') as f:
    local_conf = js.loads(f.read())

# our connection
local_engine = sqla.create_engine(local_conf.get('connection'))
local_Session = sessionmaker()
local_Session.configure(bind=local_engine)
local_session = local_Session()

sketchy_sql ='''WITH t AS (SELECT *, row_number() OVER () AS rn FROM responses WHERE format = 'xml')
SELECT * FROM (
    SELECT trunc(random() * (SELECT max(rn) FROM t))::int + 1 AS rn
    FROM   generate_series(1, 500) g
    ) r
JOIN   t USING (rn);'''
# the model
import sqlalchemy as sqla
from sqlalchemy import (
    MetaData,
    Column,
    String,
    DateTime,
    Integer,
    Boolean,
    ForeignKey,
)
from sqlalchemy.dialects.postgresql import *
from sqlalchemy.orm import relationship, backref
from sqlalchemy.ext.declarative import declarative_base

local_Base = declarative_base()

class Extracted(local_Base):
    __tablename__ = 'identifiers'
    id = Column(Integer, primary_key=True)
    source_url = Column(String)
    tag = Column(String)
    extraction_type = Column(String)
    match_type = Column(String)
    original_text = Column(String)
    potential_identifier = Column(String)
    response_id = Column(Integer)
fails = []
for result in session.execute(sketchy_sql):
    identifiers = []
    try:
        extractor = IdentifierExtractor(result['source_url'], result['cleaned_content'], equality, contains, tag_paths)
        for i in extractor.process_text():
            identifiers.append(i.to_json())
    except:
        fails.append(result['id'])
        continue
    
    for e in extractor.process_text():
        ex = Extracted(
            source_url=result['source_url'],
            tag=e.tag,
            extraction_type=e.extraction_type,
            match_type=e.match_type,
            original_text=e.original_text,
            potential_identifier=e.potential_identifier,
            response_id=result['id']
        )
        try:
            local_session.add(ex)
            local_session.commit()
        except Exception as a:
            print 'failed identifier:', e.to_json()
            print a
            local_session.rollback()
            continue
        
        
len(fails)<jupyter_output><empty_output><jupyter_text>Clear URN testsor: the random set is good for spotting certain types of things, but it is unclear (ie, the point of this) how frequently urns turn up. so is the sample missing the few urns (that aren't epsg codes or something) or missing a ton?no rule_sets here.<jupyter_code>with open('inputs/identifier_exemplars.xml', 'r') as f:
    response_cleaned = f.read().encode('utf-8')

response_url = "http://noaa.gov/some/path/to/nowhere"

extractor = IdentifierExtractor(response_url, response_cleaned, equality, contains, tag_paths)
extractor._parse()
extractor.texts
for i in extractor.process_text():
    print i.match_type, i.extraction_type, i.tag, i.potential_identifier<jupyter_output>md5 regex start/md5/first e53076e89dfa08991fe7fdfbbd367892
url extract start/urls/sixth https://www.cats.com/asg/asfgery
hdl extract start/urls/fifth http://hdl.handle.net/10022/AC:P:9998
doi extract start/urls/fourth http://dx.doi.org/10.5255/UKDA-SN-2158-1
doi regex start/urls/fourth 10.5255/UKDA-SN-2158-1
url extract start/urls/third http://e4ftl01.cr.usgs.gov/MOLT/MOD13Q1.005/2002.05.25/MOD13Q1.A2002145.h14v04.005.2007156103919.hdf.xml
url extract start/urls/second http://www.nbmg.unr.edu/dox/of0330.pdf
url extract start/urls/first http://pubs.usgs.gov/ds/618/data_files/fs/fs_e410_n3438_16/fs_e410_n3438_16.las.xml
urn regex start/text/urn/fifth urn:uuid:6426c02b-f2b1-4326-a767-2384c303faf3
uuid regex start/text/urn/fifth 6426c02b-f2b1-4326-a767-2384c303faf3
urn regex start/text/urn/fourth clarin.eu:cr1:p_1271859438204
urn regex start/text/urn/third oai:www.mpi.nl:MPI110411
urn regex start/text/urn/first urn:nbn:nl:ui:13-01hb-3b
urn regex start/alone/urn/second oai:easy.dans.knaw.nl[...]
{'fraction_non_alphanumeric': 0.1625530512176027, 'fraction_numerical': 0.01372120812559232, 'mean_word_length': 2.6234696924455063, 'pattern_counts': {'":': 6, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 3, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '126368', 'license': 'MIT', 'n_tokens_mistral': 9226, 'n_tokens_neox': 8738, 'n_words': 1582, 'path': 'notebooks/data_processing/Identifier Extraction.ipynb', 'repo_name': 'b-cube/Response-Identification-Info', 'size': '24269'}