<commit_before>// import db from './db'
import Student from '../models/student'
import Schedule from '../models/schedule'
import groupBy from 'lodash/groupBy'
import map from 'lodash/map'
import forEach from 'lodash/forEach'
import uniq from 'lodash/uniq'
import fromPairs from 'lodash/fromPairs'
import filter from 'lodash/filter'
import plur from 'plur'

export default function convertStudent({courses, degrees}) {
	let schedules = processSchedules(courses)
	let info = processDegrees(degrees)
	let fabrications = processFabrications(courses)

	let newStudent = Student({
		...info,
		schedules,
		fabrications,
	})

	return newStudent
}


function isNormalSemester({term='00000'}) {
	term = String(term)
	if (term.length !== 5) {
		return true
	}
	let s = term[4]
	if (s !== '1' && s !== '2' && s !== '3' && s !== '4' && s !== '5') {
		return true
	}
	return false
}

function processFabrications(courses) {
	let onlyNonOlaf = filter(courses, isNormalSemester)
	let fabrications = fromPairs(map(onlyNonOlaf, c => [c.clbid, c]))
	return fabrications
}


function processSchedules(courses) {
	let schedules = groupBy(courses, 'term')
	schedules = map(schedules, (courses, term) => {
		term = String(term)
		return Schedule({
			active: true,
			clbids: map(courses, c => c.clbid),
			year: parseInt(term.substr(0, 4), 10),
			semester: parseInt(term.substr(4, 1), 10),
		})
	})
	schedules = fromPairs(map(schedules, s => [s.id, s]))
	return schedules
}
<commit_msg>check for courses in the database when importing students<commit_after>import {getCourse} from './get-courses'
import Bluebird from 'bluebird'
import Student from '../models/student'
import Schedule from '../models/schedule'
import groupBy from 'lodash/groupBy'
import map from 'lodash/map'
import forEach from 'lodash/forEach'
import uniq from 'lodash/uniq'
import fromPairs from 'lodash/fromPairs'
import plur from 'plur'
import filter from 'lodash/filter'

export default async function convertStudent({courses, degrees}) {
	let {
		schedulesAndFabrications,
		info,
	} = await Bluebird.props({
		schedulesAndFabrications: processSchedules(courses),
		info: processDegrees(degrees),
	})

	let {schedules, fabrications} = schedulesAndFabrications

	let newStudent = Student({
		...info,
		schedules,
		fabrications,
	})

	return newStudent
}


async function processSchedules(courses) {
	courses = await Bluebird.all(map(courses, course => {
		return getCourse(course).then(resolvedCourse => {
			if (resolvedCourse.error) {
				course._fabrication = true
				return course
			}
			return resolvedCourse
		})
	}))

	let fabrications = filter(courses, '_fabrication')

	let schedules = groupBy(courses, 'term')
	schedules = map(schedules, (courses, term) => {
		term = String(term)
		return Schedule({
			courses,
			active: true,
			clbids: map(courses, c => c.clbid),
			year: parseInt(term.substr(0, 4), 10),
			semester: parseInt(term.substr(4, 1), 10),
		})
	})
	schedules = fromPairs(map(schedules, s => [s.id, s]))

	return {schedules, fabrications}
}

{'fraction_non_alphanumeric': 0.1337958374628345, 'fraction_numerical': 0.009250082590023126, 'mean_word_length': 4.586715867158672, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '117619', 'n_tokens_mistral': 1124, 'n_tokens_neox': 1006, 'n_words': 274}