<commit_before>/*
 * This file is part of Search NEU and licensed under AGPL3.
 * See the license file in the root folder for details.
 */

import domutils from 'domutils';
import _ from 'lodash';
import cheerio from 'cheerio';

import cache from '../../cache';
import macros from '../../../macros';
import Request from '../../request';
import EllucianBaseParser from './ellucianBaseParser';
import ellucianClassParser from './ellucianClassParser';
import ellucianRequisitesParser from './ellucianRequisitesParser';


const request = new Request('EllucianCatalogParser');

class EllucianCatalogParser extends EllucianBaseParser.EllucianBaseParser {
  supportsPage(url) {
    return url.indexOf('bwckctlg.p_disp_course_detail') > -1;
  }


  parseClass(element, url) {
    const { termId, classId, subject } = this.parseCatalogUrl(url);

    const depData = {
      desc: '',
      classId: classId,
    };


    depData.prettyUrl = url;


    let value;

    try {
      //get the class name
      value = domutils.getText(element);
    } catch (e) {
      macros.warn(element);
      macros.warn('ERROR Dom utils crashed?', url, e, element);
      return null;
    }


    const match = value.match(/.+?\s-\s*(.+)/i);
    if (!match || match.length < 2 || match[1].length < 2) {
      macros.error('could not find title!', match, value, url);
      return null;
    }
    depData.name = this.standardizeClassName(match[1]);


    //find the box below this row
    let descTR = element.parent.next;
    while (descTR.type !== 'tag') {
      descTR = descTR.next;
    }
    const rows = domutils.getElementsByTagName('td', descTR);
    if (rows.length !== 1) {
      macros.error('td rows !=1??', depData.classId, url);
      return null;
    }

    element = rows[0];


    //get credits from element.parent.text here

    //grab credits
    const text = domutils.getText(element.parent);
    const creditsParsed = this.parseCredits(text);

    if (creditsParsed) {
      depData.maxCredits = creditsParsed.maxCredits;
      depData.minCredits = creditsParsed.minCredits;
    } else {
      macros.log('warning, nothing matched credits', url, text);
    }

    const courseAttributes = this.parseStacysMom(text);

    if (courseAttributes) {
      depData.courseAttributes = courseAttributes;
      macros.log(depData);
    } else {
      macros.log('warning, nothing matched course attributes', url, text);
    }


    //desc
    //list all texts between this and next element, not including <br> or <i>
    //usally stop at <span> or <p>
    for (let i = 0; i < element.children.length; i++) {
      if (element.children[i].type === 'tag' && !['i', 'br', 'a'].includes(element.children[i].name)) {
        break;
      }
      depData.desc += `  ${domutils.getText(element.children[i]).trim()}`;
    }

    depData.desc = depData.desc.replace(/\n|\r/gi, ' ').trim();

    //remove credit hours
    // 0.000 TO 1.000 Credit hours
    depData.desc = depData.desc.replace(/(\d+(\.\d+)?\s+TO\s+)?\d+(.\d+)?\s+credit hours/gi, '').trim();

    depData.desc = depData.desc.replace(/\s+/gi, ' ').trim();

    const invalidDescriptions = ['xml extract', 'new search'];

    if (invalidDescriptions.includes(depData.desc.trim().toLowerCase())) {
      return null;
    }


    //url
    depData.url = this.createClassURL(url, termId, subject, classId);
    if (!depData.url) {
      macros.log('error could not create class url', depData);
      return null;
    }

    //find co and pre reqs and restrictions
    const prereqs = ellucianRequisitesParser.parseRequirementSection(url, element.children, 'prerequisites');

    const coreqs = ellucianRequisitesParser.parseRequirementSection(url, element.children, 'corequisites');

    if (prereqs) {
      depData.prereqs = prereqs;
    }

    if (coreqs) {
      depData.coreqs = coreqs;
    }

    macros.log(depData);
    return depData;
  }


  parse(body, url) {
    // Parse the dom
    const $ = cheerio.load(body);

    const elements = $('td.nttitle');

    let matchingElement;

    for (let i = 0; i < elements.length; i++) {
      const currElement = elements[i];

      if (matchingElement) {
        macros.error('Already have a matching element', elements, elements.length);
      }

      if (currElement.parent.parent.attribs.summary.includes('term')) {
        matchingElement = currElement;
      }
    }
<commit_msg>removed macros logs<commit_after>/*
 * This file is part of Search NEU and licensed under AGPL3.
 * See the license file in the root folder for details.
 */

import domutils from 'domutils';
import _ from 'lodash';
import cheerio from 'cheerio';

import cache from '../../cache';
import macros from '../../../macros';
import Request from '../../request';
import EllucianBaseParser from './ellucianBaseParser';
import ellucianClassParser from './ellucianClassParser';
import ellucianRequisitesParser from './ellucianRequisitesParser';


const request = new Request('EllucianCatalogParser');

class EllucianCatalogParser extends EllucianBaseParser.EllucianBaseParser {
  supportsPage(url) {
    return url.indexOf('bwckctlg.p_disp_course_detail') > -1;
  }


  parseClass(element, url) {
    const { termId, classId, subject } = this.parseCatalogUrl(url);

    const depData = {
      desc: '',
      classId: classId,
    };


    depData.prettyUrl = url;


    let value;

    try {
      //get the class name
      value = domutils.getText(element);
    } catch (e) {
      macros.warn(element);
      macros.warn('ERROR Dom utils crashed?', url, e, element);
      return null;
    }


    const match = value.match(/.+?\s-\s*(.+)/i);
    if (!match || match.length < 2 || match[1].length < 2) {
      macros.error('could not find title!', match, value, url);
      return null;
    }
    depData.name = this.standardizeClassName(match[1]);


    //find the box below this row
    let descTR = element.parent.next;
    while (descTR.type !== 'tag') {
      descTR = descTR.next;
    }
    const rows = domutils.getElementsByTagName('td', descTR);
    if (rows.length !== 1) {
      macros.error('td rows !=1??', depData.classId, url);
      return null;
    }

    element = rows[0];


    //get credits from element.parent.text here

    //grab credits
    const text = domutils.getText(element.parent);
    const creditsParsed = this.parseCredits(text);

    if (creditsParsed) {
      depData.maxCredits = creditsParsed.maxCredits;
      depData.minCredits = creditsParsed.minCredits;
    } else {
      macros.log('warning, nothing matched credits', url, text);
    }

    const courseAttributes = this.parseStacysMom(text);

    if (courseAttributes) {
      depData.courseAttributes = courseAttributes;
    } else {
      macros.log('warning, nothing matched course attributes', url, text);
    }


    //desc
    //list all texts between this and next element, not including <br> or <i>
    //usally stop at <span> or <p>
    for (let i = 0; i < element.children.length; i++) {
      if (element.children[i].type === 'tag' && !['i', 'br', 'a'].includes(element.children[i].name)) {
        break;
      }
      depData.desc += `  ${domutils.getText(element.children[i]).trim()}`;
    }

    depData.desc = depData.desc.replace(/\n|\r/gi, ' ').trim();

    //remove credit hours
    // 0.000 TO 1.000 Credit hours
    depData.desc = depData.desc.replace(/(\d+(\.\d+)?\s+TO\s+)?\d+(.\d+)?\s+credit hours/gi, '').trim();

    depData.desc = depData.desc.replace(/\s+/gi, ' ').trim();

    const invalidDescriptions = ['xml extract', 'new search'];

    if (invalidDescriptions.includes(depData.desc.trim().toLowerCase())) {
      return null;
    }


    //url
    depData.url = this.createClassURL(url, termId, subject, classId);
    if (!depData.url) {
      macros.log('error could not create class url', depData);
      return null;
    }

    //find co and pre reqs and restrictions
    const prereqs = ellucianRequisitesParser.parseRequirementSection(url, element.children, 'prerequisites');

    const coreqs = ellucianRequisitesParser.parseRequirementSection(url, element.children, 'corequisites');

    if (prereqs) {
      depData.prereqs = prereqs;
    }

    if (coreqs) {
      depData.coreqs = coreqs;
    }

    return depData;
  }


  parse(body, url) {
    // Parse the dom
    const $ = cheerio.load(body);

    const elements = $('td.nttitle');

    let matchingElement;

    for (let i = 0; i < elements.length; i++) {
      const currElement = elements[i];

      if (matchingElement) {
        macros.error('Already have a matching element', elements, elements.length);
      }

      if (currElement.parent.parent.attribs.summary.includes('term')) {
        matchingElement = currElement;
      }
    }

{'fraction_non_alphanumeric': 0.14351320321469574, 'fraction_numerical': 0.004362801377726751, 'mean_word_length': 2.8595480726628266, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '977525', 'n_tokens_mistral': 2842, 'n_tokens_neox': 2739, 'n_words': 680}