<commit_before>'''
Created on Oct 3, 2010

@author: Mark V Systems Limited
(c) Copyright 2010 Mark V Systems Limited, All rights reserved.
'''
from collections import defaultdict
import sys, traceback

def load(modelManager, url, nextaction, base=None):
    from arelle import (ModelDocument, FileSource)
    modelXbrl = create(modelManager)
    if isinstance(url,FileSource.FileSource):
        modelXbrl.fileSource = url
        url = modelXbrl.fileSource.url
    else:
        modelXbrl.fileSource = FileSource.FileSource(url)
    modelXbrl.modelDocument = ModelDocument.load(modelXbrl, url, base, isEntry=True)
    if modelXbrl.modelDocument is not None and modelXbrl.modelDocument.type < ModelDocument.Type.DTSENTRIES:
        # at this point DTS is fully discovered but schemaLocated xsd's are not yet loaded
        modelDocumentsSchemaLocated = set()
        while True: # need this logic because each new pass may add new urlDocs
            modelDocuments = set(modelXbrl.urlDocs.values()) - modelDocumentsSchemaLocated
            if not modelDocuments:
                break
            modelDocument = modelDocuments.pop()
            modelDocumentsSchemaLocated.add(modelDocument)
            modelDocument.loadSchemalocatedSchemas()
        
    #from arelle import XmlValidate
    #uncomment for trial use of lxml xml schema validation of entry document
    #XmlValidate.xmlValidate(modelXbrl.modelDocument)
    modelManager.cntlr.webCache.saveUrlCheckTimes()
    modelManager.showStatus(_("xbrl loading finished, {0}...").format(nextaction))
    return modelXbrl

def create(modelManager, newDocumentType=None, url=None, schemaRefs=None, createModelDocument=True, isEntry=False):
    from arelle import (ModelDocument, FileSource)
    modelXbrl = ModelXbrl(modelManager)
    modelXbrl.locale = modelManager.locale
    if newDocumentType:
        modelXbrl.fileSource = FileSource.FileSource(url)
        if createModelDocument:
            modelXbrl.modelDocument = ModelDocument.create(modelXbrl, newDocumentType, url, schemaRefs=schemaRefs, isEntry=isEntry)
    return modelXbrl
    
class ModelXbrl:
    
    def __init__(self, modelManager):
        self.modelManager = modelManager
        self.init()
        
    def init(self, keepViews=False):
        self.namespaceDocs = defaultdict(list)
        self.urlDocs = {}
        self.errors = []
        self.logCountErr = 0
        self.logCountWrn = 0
        self.logCountInfo = 0
        self.arcroleTypes = defaultdict(list)
        self.roleTypes = defaultdict(list)
        self.qnameConcepts = {} # contains ModelConcepts by Py key {ns}}localname of schema elements
        self.qnameAttributes = {}
        self.nameConcepts = defaultdict(list) # contains ModelConcepts by name 
        self.qnameTypes = {} # contains ModelTypes by Py key {ns}localname of type
        self.baseSets = defaultdict(list) # contains ModelLinks for keys arcrole, arcrole#linkrole
        self.relationshipSets = {} # contains ModelRelationshipSets by bas set keys
        self.qnameDimensionDefaults = {} # contains qname of dimension (index) and default member(value)
        self.facts = []
        self.factsInInstance = []
        self.contexts = {}
        self.units = {}
        self.modelObjects = []
        self.qnameParameters = {}
        self.modelVariableSets = set()
        self.modelCustomFunctionSignatures = {}
        self.modelCustomFunctionImplementations = set()
        if not keepViews:
            self.views = []
        self.langs = {self.modelManager.defaultLang}
        from arelle.XbrlConst import standardLabel
        self.labelroles = {standardLabel}
        self.hasXDT = False
        self.hasEuRendering = False
        self.hasFormulae = False
        self.formulaOutputInstance = None

    def close(self):
        self.closeViews()
        if self.modelDocument:
            self.modelDocument.close([])
        self.modelDocument = None
        self.xbrlManager = None
        self.namespaceDocs = defaultdict(list)
        self.urlDocs = {}
        self.errors = []
        self.logCountErr = 0
        self.logCountWrn = 0
        self.logCountInfo = 0
        self.arcroleTypes = defaultdict(list)
        self.roleTypes = defaultdict(list)<commit_msg>fix close crash when document wasn't loaded<commit_after>'''
Created on Oct 3, 2010

@author: Mark V Systems Limited
(c) Copyright 2010 Mark V Systems Limited, All rights reserved.
'''
from collections import defaultdict
import sys, traceback

def load(modelManager, url, nextaction, base=None):
    from arelle import (ModelDocument, FileSource)
    modelXbrl = create(modelManager)
    if isinstance(url,FileSource.FileSource):
        modelXbrl.fileSource = url
        url = modelXbrl.fileSource.url
    else:
        modelXbrl.fileSource = FileSource.FileSource(url)
    modelXbrl.modelDocument = ModelDocument.load(modelXbrl, url, base, isEntry=True)
    if modelXbrl.modelDocument is not None and modelXbrl.modelDocument.type < ModelDocument.Type.DTSENTRIES:
        # at this point DTS is fully discovered but schemaLocated xsd's are not yet loaded
        modelDocumentsSchemaLocated = set()
        while True: # need this logic because each new pass may add new urlDocs
            modelDocuments = set(modelXbrl.urlDocs.values()) - modelDocumentsSchemaLocated
            if not modelDocuments:
                break
            modelDocument = modelDocuments.pop()
            modelDocumentsSchemaLocated.add(modelDocument)
            modelDocument.loadSchemalocatedSchemas()
        
    #from arelle import XmlValidate
    #uncomment for trial use of lxml xml schema validation of entry document
    #XmlValidate.xmlValidate(modelXbrl.modelDocument)
    modelManager.cntlr.webCache.saveUrlCheckTimes()
    modelManager.showStatus(_("xbrl loading finished, {0}...").format(nextaction))
    return modelXbrl

def create(modelManager, newDocumentType=None, url=None, schemaRefs=None, createModelDocument=True, isEntry=False):
    from arelle import (ModelDocument, FileSource)
    modelXbrl = ModelXbrl(modelManager)
    modelXbrl.locale = modelManager.locale
    if newDocumentType:
        modelXbrl.fileSource = FileSource.FileSource(url)
        if createModelDocument:
            modelXbrl.modelDocument = ModelDocument.create(modelXbrl, newDocumentType, url, schemaRefs=schemaRefs, isEntry=isEntry)
    return modelXbrl
    
class ModelXbrl:
    
    def __init__(self, modelManager):
        self.modelManager = modelManager
        self.init()
        
    def init(self, keepViews=False):
        self.namespaceDocs = defaultdict(list)
        self.urlDocs = {}
        self.errors = []
        self.logCountErr = 0
        self.logCountWrn = 0
        self.logCountInfo = 0
        self.arcroleTypes = defaultdict(list)
        self.roleTypes = defaultdict(list)
        self.qnameConcepts = {} # contains ModelConcepts by Py key {ns}}localname of schema elements
        self.qnameAttributes = {}
        self.nameConcepts = defaultdict(list) # contains ModelConcepts by name 
        self.qnameTypes = {} # contains ModelTypes by Py key {ns}localname of type
        self.baseSets = defaultdict(list) # contains ModelLinks for keys arcrole, arcrole#linkrole
        self.relationshipSets = {} # contains ModelRelationshipSets by bas set keys
        self.qnameDimensionDefaults = {} # contains qname of dimension (index) and default member(value)
        self.facts = []
        self.factsInInstance = []
        self.contexts = {}
        self.units = {}
        self.modelObjects = []
        self.qnameParameters = {}
        self.modelVariableSets = set()
        self.modelCustomFunctionSignatures = {}
        self.modelCustomFunctionImplementations = set()
        if not keepViews:
            self.views = []
        self.langs = {self.modelManager.defaultLang}
        from arelle.XbrlConst import standardLabel
        self.labelroles = {standardLabel}
        self.hasXDT = False
        self.hasEuRendering = False
        self.hasFormulae = False
        self.formulaOutputInstance = None

    def close(self):
        self.closeViews()
        if hasattr(self,"modelDocument") and self.modelDocument:
            self.modelDocument.close([])
        self.modelDocument = None
        self.xbrlManager = None
        self.namespaceDocs = defaultdict(list)
        self.urlDocs = {}
        self.errors = []
        self.logCountErr = 0
        self.logCountWrn = 0
        self.logCountInfo = 0
        self.arcroleTypes = defaultdict(list)
        self.roleTypes = defaultdict(list)
{'fraction_non_alphanumeric': 0.08133302260545328, 'fraction_numerical': 0.0037287345607084597, 'mean_word_length': 2.850605652759085, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '308005', 'n_tokens_mistral': 2436, 'n_tokens_neox': 2379, 'n_words': 579}