<commit_before>var fs = require('fs');
var moment = require('moment');
var xmldom = require('xmldom');

const PARSER_FUNCTIONS = 
{
  "MBO": parseMBOPage,
};

function cleanupHtml(htmlString)
{
  return htmlString.replace(/&nbsp;/g,' ');
}

function parseFromData(cell)
{
  if (cell.childNodes !== null &&
      cell.childNodes.length > 0)
  {
    const dataCell = cell.childNodes[0];
    if (dataCell.data !== null &&
        dataCell.data !== undefined)
    {
      return dataCell.data.trim();
    }
  }
}

function parseFromChild(cell, recurseLevel)
{
  const firstData = parseFromData(cell);
  if (firstData !== undefined)
  {
    return firstData;
  }

  if (firstData === undefined && recurseLevel > 0)
  {
    if (cell.childNodes !== null &&
        cell.childNodes !== undefined &&
        cell.childNodes.length > 0)
    {
      return parseFromChild(cell.childNodes[0], recurseLevel - 1);
    }
  }
}

function makeParseFromChildFunction(recurseLevel)
{
  return function(cell) { return parseFromChild(cell, recurseLevel); };
}

const PARSER_MAP =
{
  start: 
  {
    required: true,
    column: [ 'Start time' ],
    parser: parseFromData,
  },
  course:
  {
    required: true,
    column: [ 'Classes' ],
    parser: makeParseFromChildFunction(1),
  },
  teacher:
  {
    required: true,
    column: [ 'Teacher', 'Yoga + Pilates Mat Teacher' ], 
    parser: makeParseFromChildFunction(2),
  },
  duration:
  {
    required: true,
    column: [ 'Duration' ],
    parser: parseFromData,
  },
  room:
  {
    required: false,
    column: [ 'Room' ],
    parser: parseFromData,
  },
  locale:
  {
    required: false,
    column: [ 'Location' ],
    parser: parseFromData,
  },
};

function propNameOfColumnHeader(col)
{
  for (var prop in PARSER_MAP)
  {
    const headers = PARSER_MAP[prop];
    if (PARSER_MAP[prop].column.indexOf(col) !== -1)
    {
      return prop;
    }
  }
  return '';
}

function makeColumnMap(headerRow)
{
  var map = {};

  for (var i = 0; i < headerRow.childNodes[0].childNodes.length; ++i)
  {
    var item = headerRow.childNodes[0].childNodes[i];
    if (item.childNodes !== null &&
        item.childNodes.length > 0)
    {
      var data = item.childNodes[0].data;
      var propName = propNameOfColumnHeader(data);
      if (propName !== undefined && propName !== "")
      {
        console.log('Mapping for "'+data+'",property "'+propName+'", index '+i);
        map[i] = propName;
      }
    }
  }

  return map;
}

function parseDateFromRow(row)
{
  const DATE_LOCATION = 2; // MAGIC NUMBER

  const firstCell = row.childNodes[0];
  if (firstCell.childNodes !== null &&
      firstCell.childNodes.length > 0)
  {
    // Pull out the date
    const firstData = firstCell.childNodes[0];
    if (firstData.childNodes !== null &&
        firstData.childNodes.length > DATE_LOCATION)
    {
      const dateElement = firstData.childNodes[DATE_LOCATION].data.trim();
      const parsedDate = moment(dateElement, 'DD MMM YYYY');
      if (parsedDate.isValid())
      {
        console.log('Current date: ' + parsedDate.format('DD MMM YYYY'));
      }
      else
      {
        console.log('Problem parsing date from data element in:"' + dateElement + '"');
      }
      return parsedDate;
    }
  }
}

function isCourseValid(course)
{
  for (var prop in PARSER_MAP)
  {
    if (PARSER_MAP[prop].required && course.hasOwnProperty(prop) === false)
    {
      return false;
    }
  }
  return true;
}

function parseCourseStart(webCourse, currentDate)
{
  var courseStart = moment(webCourse['start'], 'HH:mm');
  if (courseStart.isValid())
  {
    courseStart.year(currentDate.year());
    courseStart.month(currentDate.month());
    courseStart.date(currentDate.date());
  }
  else
  {
    console.log('Error parsing time from start time: ' + webCourse['start']);
  }
  return courseStart;
}

function parseCourseEnd(webCourse, courseStart)
{
  const NUMBER_LOCATION = 1;
  const TIME_TYPE_LOCATION = 2;
  const SECOND_NUMBER_LOCATION = 4; // needed for "1 hour & 15 minutes"
  const SECOND_TIME_TYPE_LOCATION = 5;
  const durationRegex = /(\d+) *([a-zA-Z]+)( & (\d+) *([a-zA-Z]+))?/;
  const match = webCourse.duration.match(durationRegex);

  var courseEnd = courseStart.clone().add(
      Number(match[NUMBER_LOCATION]),
      match[TIME_TYPE_LOCATION]);
  if (match[SECOND_NUMBER_LOCATION] !== undefined &&
      match[SECOND_TIME_TYPE_LOCATION] !== undefined)
  {
    courseEnd = courseEnd.add(
        Number(match[SECOND_NUMBER_LOCATION]),
        match[SECOND_TIME_TYPE_LOCATION]);
  }
  if (courseEnd.isValid() === false)
  {
    console.log('Error parsing end time from duration: ' + webCourse['Duration']);
  }
  return courseEnd;
}

function dbCourseOfWebCourse(webCourse, currentDate, studio)
{
  var dbCourse = 
  {
    name: webCourse['course'],
    start: null,
    end: null,
    teacher: webCourse['teacher'],
    studio: studio.name,
    room: webCourse['room'],
    location: webCourse['location'],
    url: null,
  };

  dbCourse.start = parseCourseStart(webCourse, currentDate);
  dbCourse.end = parseCourseEnd(webCourse, dbCourse.start);

  return dbCourse;
}

function rowIsValid(row)
{
  const STRIKETHROUGH_TAG_LOCATION = 1;
  if (row.childNodes === undefined || <commit_msg>Add area to course info<commit_after>var fs = require('fs');
var moment = require('moment');
var xmldom = require('xmldom');

const PARSER_FUNCTIONS = 
{
  "MBO": parseMBOPage,
};

function cleanupHtml(htmlString)
{
  return htmlString.replace(/&nbsp;/g,' ');
}

function parseFromData(cell)
{
  if (cell.childNodes !== null &&
      cell.childNodes.length > 0)
  {
    const dataCell = cell.childNodes[0];
    if (dataCell.data !== null &&
        dataCell.data !== undefined)
    {
      return dataCell.data.trim();
    }
  }
}

function parseFromChild(cell, recurseLevel)
{
  const firstData = parseFromData(cell);
  if (firstData !== undefined)
  {
    return firstData;
  }

  if (firstData === undefined && recurseLevel > 0)
  {
    if (cell.childNodes !== null &&
        cell.childNodes !== undefined &&
        cell.childNodes.length > 0)
    {
      return parseFromChild(cell.childNodes[0], recurseLevel - 1);
    }
  }
}

function makeParseFromChildFunction(recurseLevel)
{
  return function(cell) { return parseFromChild(cell, recurseLevel); };
}

const PARSER_MAP =
{
  start: 
  {
    required: true,
    column: [ 'Start time' ],
    parser: parseFromData,
  },
  course:
  {
    required: true,
    column: [ 'Classes' ],
    parser: makeParseFromChildFunction(1),
  },
  teacher:
  {
    required: true,
    column: [ 'Teacher', 'Yoga + Pilates Mat Teacher' ], 
    parser: makeParseFromChildFunction(2),
  },
  duration:
  {
    required: true,
    column: [ 'Duration' ],
    parser: parseFromData,
  },
  room:
  {
    required: false,
    column: [ 'Room' ],
    parser: parseFromData,
  },
  locale:
  {
    required: false,
    column: [ 'Location' ],
    parser: parseFromData,
  },
};

function propNameOfColumnHeader(col)
{
  for (var prop in PARSER_MAP)
  {
    const headers = PARSER_MAP[prop];
    if (PARSER_MAP[prop].column.indexOf(col) !== -1)
    {
      return prop;
    }
  }
  return '';
}

function makeColumnMap(headerRow)
{
  var map = {};

  for (var i = 0; i < headerRow.childNodes[0].childNodes.length; ++i)
  {
    var item = headerRow.childNodes[0].childNodes[i];
    if (item.childNodes !== null &&
        item.childNodes.length > 0)
    {
      var data = item.childNodes[0].data;
      var propName = propNameOfColumnHeader(data);
      if (propName !== undefined && propName !== "")
      {
        console.log('Mapping for "'+data+'",property "'+propName+'", index '+i);
        map[i] = propName;
      }
    }
  }

  return map;
}

function parseDateFromRow(row)
{
  const DATE_LOCATION = 2; // MAGIC NUMBER

  const firstCell = row.childNodes[0];
  if (firstCell.childNodes !== null &&
      firstCell.childNodes.length > 0)
  {
    // Pull out the date
    const firstData = firstCell.childNodes[0];
    if (firstData.childNodes !== null &&
        firstData.childNodes.length > DATE_LOCATION)
    {
      const dateElement = firstData.childNodes[DATE_LOCATION].data.trim();
      const parsedDate = moment(dateElement, 'DD MMM YYYY');
      if (parsedDate.isValid())
      {
        console.log('Current date: ' + parsedDate.format('DD MMM YYYY'));
      }
      else
      {
        console.log('Problem parsing date from data element in:"' + dateElement + '"');
      }
      return parsedDate;
    }
  }
}

function isCourseValid(course)
{
  for (var prop in PARSER_MAP)
  {
    if (PARSER_MAP[prop].required && course.hasOwnProperty(prop) === false)
    {
      return false;
    }
  }
  return true;
}

function parseCourseStart(webCourse, currentDate)
{
  var courseStart = moment(webCourse['start'], 'HH:mm');
  if (courseStart.isValid())
  {
    courseStart.year(currentDate.year());
    courseStart.month(currentDate.month());
    courseStart.date(currentDate.date());
  }
  else
  {
    console.log('Error parsing time from start time: ' + webCourse['start']);
  }
  return courseStart;
}

function parseCourseEnd(webCourse, courseStart)
{
  const NUMBER_LOCATION = 1;
  const TIME_TYPE_LOCATION = 2;
  const SECOND_NUMBER_LOCATION = 4; // needed for "1 hour & 15 minutes"
  const SECOND_TIME_TYPE_LOCATION = 5;
  const durationRegex = /(\d+) *([a-zA-Z]+)( & (\d+) *([a-zA-Z]+))?/;
  const match = webCourse.duration.match(durationRegex);

  var courseEnd = courseStart.clone().add(
      Number(match[NUMBER_LOCATION]),
      match[TIME_TYPE_LOCATION]);
  if (match[SECOND_NUMBER_LOCATION] !== undefined &&
      match[SECOND_TIME_TYPE_LOCATION] !== undefined)
  {
    courseEnd = courseEnd.add(
        Number(match[SECOND_NUMBER_LOCATION]),
        match[SECOND_TIME_TYPE_LOCATION]);
  }
  if (courseEnd.isValid() === false)
  {
    console.log('Error parsing end time from duration: ' + webCourse['Duration']);
  }
  return courseEnd;
}

function dbCourseOfWebCourse(webCourse, currentDate, studio)
{
  var dbCourse = 
  {
    name: webCourse['course'],
    start: null,
    end: null,
    teacher: webCourse['teacher'],
    studio: studio.name,
    room: webCourse['room'],
    location: webCourse['location'],
    url: studio.url,
    area: studio.area,
  };

  dbCourse.start = parseCourseStart(webCourse, currentDate);
  dbCourse.end = parseCourseEnd(webCourse, dbCourse.start);

  return dbCourse;
}

function rowIsValid(row)
{
  const STRIKETHROUGH_TAG_LOCATION = 1;
  if (row.childNodes === undefined || 
{'fraction_non_alphanumeric': 0.1421377672209026, 'fraction_numerical': 0.00494061757719715, 'mean_word_length': 3.168712871287129, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3165405', 'n_tokens_mistral': 3618, 'n_tokens_neox': 3521, 'n_words': 720}