<commit_before>// A script that uses Cheerio to scrape course evaluation information from the Registrar
// At the moment this script does not save the data anywhere

// Load external dependencies
var http = require('https')
var cheerio = require('cheerio')
var prompt = require('prompt')

// Load config variables from the .env file
require('dotenv').config()

// Load internal modules
var courseModel = require('./course.js')
require('./semester.js')
require('./database.js')

// Load a request from the server and call the function externalCallback
var loadPage = function (term, courseID, externalCallback) {
  // Define the HTTP request options
  var options = {
    host: 'registrar.princeton.edu',
    path: '/course-offerings/course_details.xml?' + 'term=' + term + '&courseid=' + courseID
  }

  // Make the request
  var req = http.request(options, function (res) {
    var str = ''

    // Append received data to already received data
    res.on('data', function (chunk) {
      str += chunk
    })

    // Handle data once it has all been received
    res.on('end', function () {
      externalCallback(str)
    })
  })
  req.end()
}

var extractSingle = function ($, container, title) {
  var inside
  var lines = container.first().contents().filter(function () {
    if ($(this).is('strong')) {
      inside = $(this).text().indexOf(title) > -1
    }
    return (this.nodeType === 3 && inside)
  }).text().split('\n')
  for (var lineIndex in lines) {
    var line = lines[lineIndex].trim()
    if (line.length > 1 && line.substring(line.length - 2) === '..') {
      line = line.substring(0, line.length - 1)
    }
    if (line.length > 0) {
      return line
    }
  }
}

// Return the course evaluation data for the given semester/courseID to the function callback
var getCourseListingData = function (semester, courseID, callback) {
  loadPage(semester, courseID, function (data) {
    console.log('\tRecieved data for course %s in semester %s.', courseID, semester)

    // Prepare to Extract Data From Page
    var $ = cheerio.load(data)
    var results = {}
    var detailsContainer = $('#timetable')

    // Get Distrubution Area
    const regex = /\((LA|SA|HA|EM|EC|QR|STN|STL)\)/
    let m
    if ((m = regex.exec(detailsContainer.text())) !== null) {
      results.distribution = m[0].substring(1, m[0].length - 1)
    }

    var attributes = detailsContainer.find('em').first().text()

    // Get PDF Status
    results.pdf = {}
    if (attributes.indexOf('P/D/F Only') > -1) {
      results.pdf.required = true
    } else {
      results.pdf.required = false
    }
    if (attributes.indexOf('P/D/F') > -1) {
      results.pdf.permitted = true
    } else if (attributes.indexOf('npdf') > -1) {
      results.pdf.permitted = false
    }

    // Get Audit Status
    if (attributes.indexOf('Audit') > -1) {
      results.audit = true
    } else if (attributes.indexOf('na') > -1) {
      results.audit = false
    }

    // Get Assignments
    var assignments = extractSingle($, detailsContainer, 'Reading/Writing assignments')
    if (typeof (assignments) !== 'undefined') {
      results.assignments = assignments
    }

    // Get Grading Components
    var insideGrading
    var gradingComponentsRaw = detailsContainer.first().contents().filter(function () {
      if ($(this).is('strong')) {
        insideGrading = $(this).text().indexOf('Requirements/Grading') > -1
      }
      return (this.nodeType === 3 && insideGrading)
    }).text()
    var gradingComponentLines = gradingComponentsRaw.split('\n')
    results.grading = []
    for (var gradingComponentLinesIndex in gradingComponentLines) {
      var thisGradingComponentLine = gradingComponentLines[gradingComponentLinesIndex].trim()
      if (thisGradingComponentLine.length > 0) {
        var thisGradingComponent = thisGradingComponentLine.split('-')
        results.grading.push({
          component: thisGradingComponent[0].trim(),
          weight: parseInt(thisGradingComponent[1].trim())
        })
      }
    }

    // Get Prerequisites
    var prerequisites = extractSingle($, detailsContainer, 'Prerequisites and Restrictions')
    if (typeof (prerequisites) !== 'undefined') {
      results.prerequisites = prerequisites
    }

    // Get Equivalent Courses
    var equivalentcourses = extractSingle($, detailsContainer, 'Equivalent Courses')
    if (typeof (equivalentcourses) !== 'undefined') {
      results.equivalentcourses = equivalentcourses
    }
<commit_msg>Handling cases where the Registrar's page is slightly different from normal<commit_after>// A script that uses Cheerio to scrape course evaluation information from the Registrar
// At the moment this script does not save the data anywhere

// Load external dependencies
var http = require('https')
var cheerio = require('cheerio')
var prompt = require('prompt')

// Load config variables from the .env file
require('dotenv').config()

// Load internal modules
var courseModel = require('./course.js')
require('./semester.js')
require('./database.js')

// Load a request from the server and call the function externalCallback
var loadPage = function (term, courseID, externalCallback) {
  // Define the HTTP request options
  var options = {
    host: 'registrar.princeton.edu',
    path: '/course-offerings/course_details.xml?' + 'term=' + term + '&courseid=' + courseID
  }

  // Make the request
  var req = http.request(options, function (res) {
    var str = ''

    // Append received data to already received data
    res.on('data', function (chunk) {
      str += chunk
    })

    // Handle data once it has all been received
    res.on('end', function () {
      externalCallback(str)
    })
  })
  req.end()
}

var extractSingle = function ($, container, title) {
  var inside
  var lines = container.first().contents().filter(function () {
    if ($(this).is('strong')) {
      inside = $(this).text().indexOf(title) > -1
    }
    return (this.nodeType === 3 && inside)
  }).text().split('\n')
  for (var lineIndex in lines) {
    var line = lines[lineIndex].trim()
    if (line.length > 1 && line.substring(line.length - 2) === '..') {
      line = line.substring(0, line.length - 1)
    }
    if (line.length > 0) {
      return line
    }
  }
}

// Return the course evaluation data for the given semester/courseID to the function callback
var getCourseListingData = function (semester, courseID, callback) {
  loadPage(semester, courseID, function (data) {
    console.log('\tRecieved data for course %s in semester %s.', courseID, semester)

    // Prepare to Extract Data From Page
    var $ = cheerio.load(data)
    var results = {}
    var detailsContainer = $('#timetable')

    // Get Distrubution Area
    const regex = /\((LA|SA|HA|EM|EC|QR|STN|STL)\)/
    let m
    if ((m = regex.exec(detailsContainer.text())) !== null) {
      results.distribution = m[0].substring(1, m[0].length - 1)
    }

    var attributes = detailsContainer.find('em').first().text()

    // Get PDF Status
    results.pdf = {}
    if (attributes.indexOf('P/D/F Only') > -1) {
      results.pdf.required = true
    } else {
      results.pdf.required = false
    }
    if (attributes.indexOf('P/D/F') > -1) {
      results.pdf.permitted = true
    } else if (attributes.indexOf('npdf') > -1) {
      results.pdf.permitted = false
    }

    // Get Audit Status
    if (attributes.indexOf('Audit') > -1) {
      results.audit = true
    } else if (attributes.indexOf('na') > -1) {
      results.audit = false
    }

    // Get Assignments
    var assignments = extractSingle($, detailsContainer, 'Reading/Writing assignments')
    if (typeof (assignments) !== 'undefined') {
      results.assignments = assignments
    }

    // Get Grading Components
    var insideGrading
    var gradingComponentsRaw = detailsContainer.first().contents().filter(function () {
      if ($(this).is('strong, b')) {
        insideGrading = $(this).text().indexOf('Requirements/Grading') > -1
      }
      return (this.nodeType === 3 && insideGrading)
    }).text()
    var gradingComponentLines = gradingComponentsRaw.split('\n')
    results.grading = []
    for (var gradingComponentLinesIndex in gradingComponentLines) {
      var thisGradingComponentLine = gradingComponentLines[gradingComponentLinesIndex].trim()
      if (thisGradingComponentLine.length > 0) {
        var thisGradingComponent = thisGradingComponentLine.split('-')
        results.grading.push({
          component: thisGradingComponent[0].trim(),
          weight: parseInt(thisGradingComponent[1].trim())
        })
      }
    }

    // Get Prerequisites
    var prerequisites = extractSingle($, detailsContainer, 'Prerequisites and Restrictions')
    if (typeof (prerequisites) !== 'undefined') {
      results.prerequisites = prerequisites
    }

    // Get Equivalent Courses
    var equivalentcourses = extractSingle($, detailsContainer, 'Equivalent Courses')
    if (typeof (equivalentcourses) !== 'undefined') {
      results.equivalentcourses = equivalentcourses
    }

{'fraction_non_alphanumeric': 0.13636868406440866, 'fraction_numerical': 0.004664075513603554, 'mean_word_length': 3.045822102425876, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3285517', 'n_tokens_mistral': 2688, 'n_tokens_neox': 2584, 'n_words': 709}