<commit_before>
import os
import json
import uuid
from avro import schema, io

import util
import pkgutil

NAMESPACE = "edu.berkeley.cs.local"
SCHEMA_NAMES = schema.Names(default_namespace=NAMESPACE)
SCHEMAS = [
    # new extension types
    "uuid", 
    "UnitofTime", "Duration", 
    "ReadingType", "ReadingValue",
    
    'InstrumentMetadata', 'LocationMetadata',
    
    # timeseries subobjects
    "Actuator", 
    "Properties", "Metadata", 
    "TimeSeries", "Collection",
    "Reporting"]
SCHEMA_OBJECTS = []

# load all the schemas when we check an object
for sf in SCHEMAS:
    # print "Loading", sf
    obj = json.loads(pkgutil.get_data(__name__, 'schema/' + sf.lower() + ".av"))
    s = schema.make_avsc_object(obj, SCHEMA_NAMES)
    SCHEMA_OBJECTS.append(obj)
        <commit_msg>Check for schema files in multiple locations to support py2exe<commit_after>
import os
import sys
import json
import uuid
from avro import schema, io

import util
import pkgutil

NAMESPACE = "edu.berkeley.cs.local"
SCHEMA_NAMES = schema.Names(default_namespace=NAMESPACE)
SCHEMAS = [
    # new extension types
    "uuid", 
    "UnitofTime", "Duration", 
    "ReadingType", "ReadingValue",
    
    'InstrumentMetadata', 'LocationMetadata',
    
    # timeseries subobjects
    "Actuator", 
    "Properties", "Metadata", 
    "TimeSeries", "Collection",
    "Reporting"]
SCHEMA_OBJECTS = []

# load all the schemas when we check an object
for sf in SCHEMAS:
    # print "Loading", sf
	
    dirs = [os.path.dirname(sys.modules[__name__].__file__), sys.prefix]
    obj = None
    for d in dirs:
        try:
            path = os.path.join(d, 'schema', sf.lower() + '.av')
            obj = json.load(open(path, 'r'))
        except:
            pass
    if obj == None:
        raise Exception('Cannot load schema: ' + sf.lower())
    
    s = schema.make_avsc_object(obj, SCHEMA_NAMES)
    SCHEMA_OBJECTS.append(obj)
        
{'fraction_non_alphanumeric': 0.11626694473409802, 'fraction_numerical': 0.0005213764337851929, 'mean_word_length': 3.109207708779443, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6500426', 'n_tokens_mistral': 641, 'n_tokens_neox': 630, 'n_words': 164}