<jupyter_start><jupyter_text>Configuration<jupyter_code>### RUN THIS CELL TO BEGIN CONFIGURATION ###
%run manage_data_nodes_config.ipynb
### LEGACY CONFIGURATION CELL -- SKIP THIS IF USING THE CONFIGURATION FORM ABOVE ###

# Data Node configurationRawData or ProcessedData

datatype = 'RawData' # string: "RawData" or "ProcessedData"
namespace = 'we1sv1.1'
collection = 'test' # string: The _id of the target collection 

# Optional RawData Node Properties

relationships = [] # list
OCR = True # Boolean
rights = "" # string

# Optional ProcessedData Node Properties

processes = [] # list

# Paths to Data Manifests

source_files = [] # list<jupyter_output><empty_output><jupyter_text>Basic Setup<jupyter_code># If using the configuration form, get the values from the form
try:
    assert config.values['datatype']
    datatype = config.values['datatype']
    collection = config.values['collection']
    source_files = config.values['source_files']
    title = "" # string
    altTitle = "" # string
    date = [] # list
    description = "" # string
    label = "" # string
    notes = "" # list
    if datatype == 'RawData':
        relationships.values['relationships']
        rights.values['rights']
        OCR = config.values['OCR']
    else:
        processes = config.values['processes']
except:
    pass

# Import dependencies

import os, json, pymongo
from pymongo import MongoClient
from jsonschema import validate, FormatChecker

# Set up the MongoDB client, configure the databases, and assign variables to the "collections" 
client = MongoClient('mongodb://localhost:27017')
db = client.we1s
Corpus = db.Corpus

# Define the schema to be used to create the manifest
if datatype == 'RawData':
    schema = [
        { "name": "namespace", "type": "string" },
        { "name": "path", "type": "string" },
        { "name": "collection", "type": "string" },    
        { "name": "relationships", "type": "list" },    
        { "name": "OCR", "type": "bool" },    
        { "name": "rights", "type": "string" },
        { "name": "title", "type": "string" },
        { "name": "altTitle", "type": "string" },
        { "name": "date", "type": "list" },
        { "name": "description", "type": "string" },
        { "name": "label", "type": "string" },
        { "name": "notes", "type": "list" }
    ]
else:
    schema = [
        { "name": "namespace", "type": "string" },
        { "name": "path", "type": "string" },
        { "name": "collection", "type": "string" },    
        { "name": "processes", "type": "list" },
        { "name": "title", "type": "string" },
        { "name": "altTitle", "type": "string" },
        { "name": "date", "type": "list" },
        { "name": "description", "type": "string" },
        { "name": "label", "type": "string" },
        { "name": "notes", "type": "list" }
    ]
    

# Auto-generate `_id` and `path` values for the Data Node
_id = datatype
path = ',Corpus,' + collection + ','

# Pass the configurations to a variable
opts = globals()<jupyter_output><empty_output><jupyter_text>API Methods<jupyter_code>def create_data_manifest(opts, schema):
    """
    Detects if the RawData or ProcessedData node exists along the
    specified path. If not, the node is created.
    """
    try:
        assert Corpus.find({'path': opts['path']})
        pass
    except:
        manifest = {'namespace': namespace}
        for item in schema:
            key = item['name']
            val = opts[key]
            valid_datatype = isinstance(val, item['type'])
            try:
                valid_datatype == True
                manifest[key] = val
            except:
                print(manifest[key] + ' has an invalid data type. Please double check it against the schema.')
        Corpus.insert_one(manifest)


def detect_data_node(manifest, opts, schema):
    """
    Attempts to find the manifest's data node in the database. If it does not exist, creates the node or raises an error if the database does not contain the specified collection.
    """
    try:
        # Check whether the data node already exists
        assert Corpus.find_one(manifest['path'])
        pass
    except:
        try:
            # Check whether the collection exists
            assert Corpus.find_one(manifest['path'].rstrip(opts['datatype'] + ','))
            # Create the data node
            create_data_manifest(opts, schema)
        except:
            msg = 'The database does not contain the collection listed in the `path` property for ' + manifest['_id'] + '. Please run the `create_collection` notebook.'
            print(msg)

        
def insert_doc_manifests(doc_paths, opts, schema):
    """
    Loops through a list of doc manifests, ensures that the collection and data node specified therein exists in the database, and then inserts
    the doc's manifest in the database.
    """
    for path in doc_paths:
        with open(path, 'r') as f:
            manifest = json.load(f)
        detect_data_node(manifest, opts, schema)
        if validate_manifest(manifest) == True:
            Corpus.insert_one(manifest)
        else:
            print("Error: Could not produce a valid manifest.")


def validate_datatype(key, val, datatype):
    datatype = datatype.replace('string', 'str')
    try:
        assert isinstance(val, eval(datatype))
    except:
        msg = 'Error: The value "' + str(val) + '" is of the wrong data type for "' + key + '". '
        msg += 'Change the value to a ' + datatype + '.'
        print(msg)

            
def validate_manifest(manifest):
    """
    Validates a manifest against the online manifest schema.
    """
    schema_file = 'https://raw.githubusercontent.com/whatevery1says/manifest/master/schema/Corpus/collection.json'
    schema = json.loads(requests.get(schema_file).text)
    try:
        validate(manifest, schema, format_checker=FormatChecker())
        return True
    except:
        return False<jupyter_output><empty_output><jupyter_text>Execute Action<jupyter_code>insert_doc_manifests(source_files, opts, schema)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13332237382870293, 'fraction_numerical': 0.001643925694558606, 'mean_word_length': 2.8604060913705585, 'pattern_counts': {'":': 44, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '233813', 'license': 'MIT', 'n_tokens_mistral': 1822, 'n_tokens_neox': 1779, 'n_words': 533, 'path': 'collection-manager/import_collection_data.ipynb', 'repo_name': 'whatevery1says/jupyter-notebooks', 'size': '6083'}