<commit_before>'use strict';
var macros = require('../macros')
var classesDB = require('../databases/classesDB')
var ellucianRequisitesParser = require('../parsers/ellucianRequisitesParser')

var queue = require('d3-queue').queue

// This file converts prereq classIds to ClassUids by looking up the classes in the db and replacing classIds with classUids
// if there are multiple results, it creates a 'or' prereq tree, much like Class.js does in the frontend. 

function PrereqClassUids() {


	// runs second, after addClassUids
	this.priority = 1;
}

// runs on all hosts
PrereqClassUids.prototype.supportsHost = function (host) {
	return true;
};

PrereqClassUids.prototype.updatePrereqs = function (prereqs, host, termId, keyToRows) {
	for (var i = prereqs.values.length - 1; i >= 0; i--) {
		var prereqEntry = prereqs.values[i]

		// prereqEntry could be Object{subject:classId:} or string i think
		if (typeof prereqEntry == 'string') {
			continue;
		}
		else if (prereqEntry.classId && prereqEntry.subject) {
			// multiple classes could have same key
			var key = host + termId + prereqEntry.subject + prereqEntry.classId

			var newPrereqs = keyToRows[key];

			// not in db, this is possible and causes those warnings in the frontend 
			// unable to find class even though its a prereq of another class????
			if (!newPrereqs) {
				prereqs.values[i].missing = true;
			}
			// only one match, just swap classId for classUid
			else if (newPrereqs.length == 1) {
				prereqs.values[i] = {
					subject: newPrereqs[0].subject,
					classUid: newPrereqs[0].classUid
				}
			}
			// the fun part - make the 'or' split for multiple classes
			else {
				prereqs.values[i] = {
					type: 'or',
					values: newPrereqs
				}
			}
		}
		else if (prereqEntry.type && prereqEntry.values) {
			prereqs.values[i] = this.updatePrereqs(prereqEntry, host, termId, keyToRows)
		}
		else if (prereqEntry.classUid && prereqEntry.subject) {
			// don't do anything, this is already fixed
		}
		else {
			elog('wtf is ', prereqEntry, prereqs)
		}
	}
	return prereqs;

};



// base query is the key shared by all classes that need to be updated
// if an entire college needs to be updated, it could be just {host:'neu.edu'}
// at minimum it will be a host
// or if just one class {host, termId, subject, classId}
PrereqClassUids.prototype.go = function (baseQuery, callback) {
	if (!baseQuery.host) {
		elog('no host in PrereqClassUids?')
		return callback('no')
	}

	var q = queue();
	var classesToUpdate = [];

	// find classes that need to be updated
	q.defer(function (callback) {
		classesDB.find(baseQuery, {
			skipValidation: true
		}, function (err, results) {
			if (err) {
				console.log(err);
				return callback(err)
			}
			classesToUpdate = results;
			callback()
		}.bind(this));
	}.bind(this))

	// and find all classes that could be matched
	var matchingQuery = {
		host: baseQuery.host
	}

	// if base query specified term, we can specify it here too and still find all the classes needed
	if (baseQuery.termId) {
		matchingQuery.termId = baseQuery.termId
	}


	//make obj to find results here quickly
	var keyToRows = {};

	q.defer(function (callback) {
		classesDB.find(matchingQuery, {
			skipValidation: true
		}, function (err, results) {
			if (err) {
				console.log(err);
				return callback(err)
			}

			results.forEach(function (aClass) {
				if (!aClass.host || !aClass.termId || !aClass.subject || !aClass.classUid) {
					elog("ERROR class dosent have required fields??", aClass);
					return;
				}

				// multiple classes could have same key
				var key = aClass.host + aClass.termId + aClass.subject + aClass.classId

				if (!keyToRows[key]) {
					keyToRows[key] = []
				}

				// only need to keep subject and classUid
				keyToRows[key].push({
					subject: aClass.subject,
					classUid: aClass.classUid
				})
			}.bind(this));
			callback()
		}.bind(this))
	}.bind(this))

	q.awaitAll(function (err) {
		if (err) {
			return callback(err)
		}


		var updateQueue = queue()

		// loop through classes to update, and get the new data from all the classes<commit_msg>little abstraction for keyToRow for processors<commit_after>'use strict';
var macros = require('../macros')
var classesDB = require('../databases/classesDB')
var ellucianRequisitesParser = require('../parsers/ellucianRequisitesParser')

var queue = require('d3-queue').queue

// This file converts prereq classIds to ClassUids by looking up the classes in the db and replacing classIds with classUids
// if there are multiple results, it creates a 'or' prereq tree, much like Class.js does in the frontend. 

function PrereqClassUids() {


	// runs second, after addClassUids
	this.priority = 1;
}

// runs on all hosts
PrereqClassUids.prototype.supportsHost = function (host) {
	return true;
};


PrereqClassUids.prototype.getClassHash = function(query, config, callback) {
	
	// and find all classes that could be matched
	var matchingQuery = {
		host: query.host
	}

	// if base query specified term, we can specify it here too and still find all the classes needed
	if (query.termId) {
		matchingQuery.termId = query.termId
	}


	//make obj to find results here quickly
	var keyToRows = {};

	classesDB.find(matchingQuery, {
		skipValidation: true
	}, function (err, results) {
		if (err) {
			console.log(err);
			return callback(err)
		}

		results.forEach(function (aClass) {
			if (!aClass.host || !aClass.termId || !aClass.subject || !aClass.classUid) {
				elog("ERROR class dosent have required fields??", aClass);
				return;
			}

			// multiple classes could have same key
			var key = aClass.host + aClass.termId + aClass.subject;
			if (config.useClassId) {
				key += aClass.classId
			}
			else if (aClass.classUid) {
				key += aClass.classUid	
			}
			else {
				elog('Cant use classUid if dont have classUid!',aClass)
			}
			

			if (!keyToRows[key]) {
				keyToRows[key] = []
			}

			// only need to keep subject and classUid
			keyToRows[key].push(aClass)
			
		}.bind(this));
		
		return callback(null, keyToRows);
		callback()
	}.bind(this))
}

PrereqClassUids.prototype.updatePrereqs = function (prereqs, host, termId, keyToRows) {
	for (var i = prereqs.values.length - 1; i >= 0; i--) {
		var prereqEntry = prereqs.values[i]

		// prereqEntry could be Object{subject:classId:} or string i think
		if (typeof prereqEntry == 'string') {
			continue;
		}
		else if (prereqEntry.classId && prereqEntry.subject) {
			// multiple classes could have same key
			var key = host + termId + prereqEntry.subject + prereqEntry.classId

			var newPrereqs = {
				subject: keyToRows[key].subject,
				classUid: keyToRows[key].classUid
			}
			

			// not in db, this is possible and causes those warnings in the frontend 
			// unable to find class even though its a prereq of another class????
			if (!newPrereqs) {
				prereqs.values[i].missing = true;
			}
			// only one match, just swap classId for classUid
			else if (newPrereqs.length == 1) {
				prereqs.values[i] = {
					subject: newPrereqs[0].subject,
					classUid: newPrereqs[0].classUid
				}
			}
			// the fun part - make the 'or' split for multiple classes
			else {
				prereqs.values[i] = {
					type: 'or',
					values: newPrereqs
				}
			}
		}
		else if (prereqEntry.type && prereqEntry.values) {
			prereqs.values[i] = this.updatePrereqs(prereqEntry, host, termId, keyToRows)
		}
		else if (prereqEntry.classUid && prereqEntry.subject) {
			// don't do anything, this is already fixed
		}
		else {
			elog('wtf is ', prereqEntry, prereqs)
		}
	}
	return prereqs;

};



// base query is the key shared by all classes that need to be updated
// if an entire college needs to be updated, it could be just {host:'neu.edu'}
// at minimum it will be a host
// or if just one class {host, termId, subject, classId}
PrereqClassUids.prototype.go = function (baseQuery, callback) {
	if (!baseQuery.host) {
		elog('no host in PrereqClassUids?')
		return callback('no')
	}

	var q = queue();
	var classesToUpdate = [];

	// find classes that need to be updated
	q.defer(function (callback) {
		classesDB.find(baseQuery, {
			skipValidation: true
		}, function (err, results) {
			if (err) {
				console.log(err);
				return callback(err)
			}
			classesToUpdate = results;
			callback()
		}.bind(this));
	}.bind(this))

	var keyToRow = {};
	q.defer(function (callback) {
		this.getClassHash(baseQuery, {useClassId:true}, function(err, theKeyToRow) {
			if (err) {
				elog(err)
				return callback(err)
			}
			keyToRow = theKeyToRow;
			callback();
		}.bind(this))
	}.bind(this))


	q.awaitAll(function (err) {
		if (err) {
			return callback(err)
		}


		var updateQueue = queue()

		// loop through classes to update, and get the new data from all the classes
{'fraction_non_alphanumeric': 0.11823899371069183, 'fraction_numerical': 0.0016009148084619783, 'mean_word_length': 3.245631067961165, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '805626', 'n_tokens_mistral': 3358, 'n_tokens_neox': 2877, 'n_words': 875}