<commit_before>var fs = require('fs'),
    xml = require('xml-object-stream'),
    enums = require('../shared/enums'),
    _ = require('lodash'),
    moment = require('moment'),
    utils = require('../shared/utils')

exports.readPrograms = function (body, callback) {
  var stream = xml.parse(body)
  var programs = []

  stream.each('KUVAOHJELMA', function(xml) {
    var p = validateProgram(xml)
    programs.push(p)
  })

  stream.on('end', function() {
    return callback(null, programs)
  })
}

var validateProgram = compose([
  and(requiredAttr('TYPE', 'type'), function(xml) {
    var type = xml.$.TYPE
    if (_.has(enums.legacyProgramTypes, type)) return ok({'program-type': enums.legacyProgramTypes[type]})
    else return error("Virheellinen attribuutti: TYPE")
  }),
  required('ASIAKKAANTUNNISTE', 'externalId'),
  required('ALKUPERAINENNIMI', 'name'),
  flatMap(requiredAttr('TYPE', 'type'), function(p) {
    var allButTvOrOther = ['01','02','03','04','06','07','08','10','11']
    if (_.contains(allButTvOrOther, p.type)) return required('SUOMALAINENNIMI', 'name-fi')
    else return optional('SUOMALAINENNIMI', 'name-fi')
  }),
  optional('RUOTSALAINENNIMI', 'name-sv'),
  optional('MUUNIMI', 'name-other'),
  optional('JULKAISUVUOSI', 'year'),
  optional('VALMISTUMISVUOSI', 'year'),
  flatMap(optional('MAAT'), function(p) {
    var countries = p.MAAT ? p.MAAT.split(' ') : []
    return function(xml) {
      if (_.all(countries, _.curry(_.has)(enums.countries))) return ok({country: countries})
      else return error('Virheellinen MAAT kenttä: ' + countries)
    }
  }),
  optional('TUOTANTOYHTIO', 'legacy-production-companies'),
  required('SYNOPSIS', 'synopsis'),
  optional('TUOTANTOKAUSI', 'season'),
  optional('OSA', 'episode'),
  map(compose([
    enumList('LAJIT', enums.legacyGenres),
    enumList('TELEVISIO-OHJELMALAJIT', enums.legacyTvGenres),
    inList('PELINLAJIT', enums.legacyGameGenres)
  ]), function(p) {
    return { 'legacy-genre': p.LAJIT.concat(p['TELEVISIO-OHJELMALAJIT']).concat(p.PELINLAJIT) }
  }),
  map(childrenByNameTo('OHJAAJA', 'directors'), function(p) { return {directors: p.directors.map(fullname) }}),
  map(childrenByNameTo('NAYTTELIJA', 'actors'), function(p) { return {actors: p.actors.map(fullname) }}),
  map(required('LUOKITTELIJA', 'author'), function(p) { return { classifications: [{ author: { name: p.author } }] }}),
  map(node('LUOKITTELU', 'classification', [
    and(requiredAttr('REKISTEROINTIPAIVA', 'registration-date'), function(xml) {
      var d = moment(xml.$.REKISTEROINTIPAIVA, "DD.MM.YYYY HH:mm:ss")
      if (d.isValid()) return ok({'registration-date': d.toDate()})
      else return error("Virheellinen aikaformaatti: " + 'REKISTEROINTIPAIVA')
    }),
    mapError(or(required('FORMAATTI', 'format'), required('PELIFORMAATTI', 'gameFormat')), function(errors) {
      return "FORMAATTI tai PELIFORMAATTI puuttuu"
    }),
    and(required('KESTO'), test('KESTO', utils.isValidDuration, "Virheellinen kesto", 'duration')),
    map(childrenByNameTo('VALITTUTERMI', 'criteria'), function(p) {
      var criteriaComments = _.object(p.criteria.map(function (c) {
        return [parseInt(c.$.KRITEERI), c.$.KOMMENTTI]
      }))
      return {safe: _.isEmpty(criteriaComments), criteria: _.keys(criteriaComments), 'criteria-comments': criteriaComments}
    })
  ]), function(p) { return {classifications: [p.classification], gameFormat: p.classification.gameFormat} })
])

// validator = Xml -> Result
// validation = validator -> (program -> validator) -> validator
function flatMap(validator, f) {
  return function(xml) {
    var res = validator(xml)
    var res2 = f(res.program)(xml)
    var errors = _.flatten(res.errors.concat(res2.errors))
    return {program: _.merge(_.cloneDeep(res.program), _.cloneDeep(res2.program)), errors: errors}
  }
}

function map(validator, f) {
  return function(xml) {
    var res = validator(xml)
    if (res.errors.length > 0) return res
    else return {program: f(res.program), errors: []}
  }
}

function mapError(v, f) {
  return function(xml) {
    var res = v(xml)
    if (res.errors.length > 0) return error([f(res.errors)])
    else return res
  }
}

function compose(xs) {
  return _.reduce(xs, function(acc, f) {
    return flatMap(acc, function(_) { return f })
  }, ret({}))
}

function and(v1, v2) {
  return function(xml) {
    var res = v1(xml)
    if (res.errors.length == 0) {
      return v2(xml)
    } else {
      return res
    }
  }
}

function or(v1, v2) {
  return function(xml) {
    var res = v1(xml)
    if (res.errors.length > 0) {
      return v2(xml)
    } else {
      return res
    }
  }
}

function ok(program) {
  return {program: program, errors: []}
}
function error(msg) {
  return {program: {}, errors: [msg]}
}

function ret(program) {
  return function(xml) {
    return {program: program, errors: []}
  }
}

function required(name, toField) {
  return function(xml) {
    toField = toField || name
    var val = xml[name]
    if (val && val.$text !== '' && val.$text !== undefined) return ok(utils.keyValue(toField, xml[name].$text))
    else return error(["Pakollinen kenttä puuttuu: " + name])
  }
}

function requiredNode(name, toField) {
  return function(xml) {
    toField = toField || name
    if (xml[name]) return ok(utils.keyValue(toField, xml[name].$text))
    else return error(["Pakollinen elementti puuttuu: " + name])
  }
}

function requiredAttr(name, toField) {
  return function(xml) {
    if (xml.$[name]) return ok(utils.keyValue(toField, xml.$[name]))
    else return error("Pakollinen attribuutti puuttuu: " + name)
  }
}

function optional(field, toField) {
  return function(xml) {
    toField = toField || field
    if (!xml[field] || !xml[field].$text || xml[field].$text.length == 0) return {program: utils.keyValue(toField, undefined), errors: []}
    else return {program: utils.keyValue(toField, xml[field].$text), errors: []}
  }
}

function childrenByNameTo(field, toField) {
  return function(xml) {
    return {
      program: utils.keyValue(toField, childrenByName(xml, field)),
      errors: []
    }
  }
}

function enumList(field, _enum) {
  return function(xml) {
    var values = optionListToArray(xml[field]).map(function(g) { return _enum[g] })
    if (_.all(values, function(v) { return v !== undefined })) return ok(utils.keyValue(field, values))
    else return error("Virheellinen kenttä " + field)
  }
}

function inList(field, list) {
  return function(xml) {
    var values = optionListToArray(xml[field])
    var exists = values.map(_.curry(_.contains)(list))
    if (_.all(exists)) return ok(utils.keyValue(field, values))
    else return error("Virheellinen kenttä " + field)
  }
}

function node(name, toField, validators) {
  return flatMap(requiredNode(name, toField), function(p) {
    return function (xml) {
      return _.reduce(validators, function(acc, f) {
        return flatMap(acc, function(_) { return map(f, function(p) { return utils.keyValue(toField, p)}) })
      }, ret(p))(xml[name])
    }
  })
}

function test(field, f, msg, toField) {
  return function(xml) {
    var text = xml[field].$text
    if (f(text)) return ok(utils.keyValue(toField || field, text))
    else return error(msg + " " + text)
  }
}
function fullname(node) {
  return node.ETUNIMI.$text + ' ' + node.SUKUNIMI.$text
}
function childrenByName(root, name) {<commit_msg>Rename enum related validators<commit_after>var fs = require('fs'),
    xml = require('xml-object-stream'),
    enums = require('../shared/enums'),
    _ = require('lodash'),
    moment = require('moment'),
    utils = require('../shared/utils')

exports.readPrograms = function (body, callback) {
  var stream = xml.parse(body)
  var programs = []

  stream.each('KUVAOHJELMA', function(xml) {
    var p = validateProgram(xml)
    programs.push(p)
  })

  stream.on('end', function() {
    return callback(null, programs)
  })
}

var validateProgram = compose([
  and(requiredAttr('TYPE', 'type'), function(xml) {
    var type = xml.$.TYPE
    if (_.has(enums.legacyProgramTypes, type)) return ok({'program-type': enums.legacyProgramTypes[type]})
    else return error("Virheellinen attribuutti: TYPE")
  }),
  required('ASIAKKAANTUNNISTE', 'externalId'),
  required('ALKUPERAINENNIMI', 'name'),
  flatMap(requiredAttr('TYPE', 'type'), function(p) {
    var allButTvOrOther = ['01','02','03','04','06','07','08','10','11']
    if (_.contains(allButTvOrOther, p.type)) return required('SUOMALAINENNIMI', 'name-fi')
    else return optional('SUOMALAINENNIMI', 'name-fi')
  }),
  optional('RUOTSALAINENNIMI', 'name-sv'),
  optional('MUUNIMI', 'name-other'),
  optional('JULKAISUVUOSI', 'year'),
  optional('VALMISTUMISVUOSI', 'year'),
  flatMap(optional('MAAT'), function(p) {
    var countries = p.MAAT ? p.MAAT.split(' ') : []
    return function(xml) {
      if (_.all(countries, _.curry(_.has)(enums.countries))) return ok({country: countries})
      else return error('Virheellinen MAAT kenttä: ' + countries)
    }
  }),
  optional('TUOTANTOYHTIO', 'legacy-production-companies'),
  required('SYNOPSIS', 'synopsis'),
  optional('TUOTANTOKAUSI', 'season'),
  optional('OSA', 'episode'),
  map(compose([
    valuesInEnum('LAJIT', enums.legacyGenres),
    valuesInEnum('TELEVISIO-OHJELMALAJIT', enums.legacyTvGenres),
    valuesInList('PELINLAJIT', enums.legacyGameGenres)
  ]), function(p) {
    return { 'legacy-genre': p.LAJIT.concat(p['TELEVISIO-OHJELMALAJIT']).concat(p.PELINLAJIT) }
  }),
  map(childrenByNameTo('OHJAAJA', 'directors'), function(p) { return {directors: p.directors.map(fullname) }}),
  map(childrenByNameTo('NAYTTELIJA', 'actors'), function(p) { return {actors: p.actors.map(fullname) }}),
  map(required('LUOKITTELIJA', 'author'), function(p) { return { classifications: [{ author: { name: p.author } }] }}),
  map(node('LUOKITTELU', 'classification', [
    and(requiredAttr('REKISTEROINTIPAIVA', 'registration-date'), function(xml) {
      var d = moment(xml.$.REKISTEROINTIPAIVA, "DD.MM.YYYY HH:mm:ss")
      if (d.isValid()) return ok({'registration-date': d.toDate()})
      else return error("Virheellinen aikaformaatti: " + 'REKISTEROINTIPAIVA')
    }),
    mapError(or(required('FORMAATTI', 'format'), required('PELIFORMAATTI', 'gameFormat')), function(errors) {
      return "FORMAATTI tai PELIFORMAATTI puuttuu"
    }),
    and(required('KESTO'), test('KESTO', utils.isValidDuration, "Virheellinen kesto", 'duration')),
    map(childrenByNameTo('VALITTUTERMI', 'criteria'), function(p) {
      var criteriaComments = _.object(p.criteria.map(function (c) {
        return [parseInt(c.$.KRITEERI), c.$.KOMMENTTI]
      }))
      return {safe: _.isEmpty(criteriaComments), criteria: _.keys(criteriaComments), 'criteria-comments': criteriaComments}
    })
  ]), function(p) { return {classifications: [p.classification], gameFormat: p.classification.gameFormat} })
])

// validator = Xml -> Result
// validation = validator -> (program -> validator) -> validator
function flatMap(validator, f) {
  return function(xml) {
    var res = validator(xml)
    var res2 = f(res.program)(xml)
    var errors = _.flatten(res.errors.concat(res2.errors))
    return {program: _.merge(_.cloneDeep(res.program), _.cloneDeep(res2.program)), errors: errors}
  }
}

function map(validator, f) {
  return function(xml) {
    var res = validator(xml)
    if (res.errors.length > 0) return res
    else return {program: f(res.program), errors: []}
  }
}

function mapError(v, f) {
  return function(xml) {
    var res = v(xml)
    if (res.errors.length > 0) return error([f(res.errors)])
    else return res
  }
}

function compose(xs) {
  return _.reduce(xs, function(acc, f) {
    return flatMap(acc, function(_) { return f })
  }, ret({}))
}

function and(v1, v2) {
  return function(xml) {
    var res = v1(xml)
    if (res.errors.length == 0) {
      return v2(xml)
    } else {
      return res
    }
  }
}

function or(v1, v2) {
  return function(xml) {
    var res = v1(xml)
    if (res.errors.length > 0) {
      return v2(xml)
    } else {
      return res
    }
  }
}

function ok(program) {
  return {program: program, errors: []}
}
function error(msg) {
  return {program: {}, errors: [msg]}
}

function ret(program) {
  return function(xml) {
    return {program: program, errors: []}
  }
}

function required(name, toField) {
  return function(xml) {
    toField = toField || name
    var val = xml[name]
    if (val && val.$text !== '' && val.$text !== undefined) return ok(utils.keyValue(toField, xml[name].$text))
    else return error(["Pakollinen kenttä puuttuu: " + name])
  }
}

function requiredNode(name, toField) {
  return function(xml) {
    toField = toField || name
    if (xml[name]) return ok(utils.keyValue(toField, xml[name].$text))
    else return error(["Pakollinen elementti puuttuu: " + name])
  }
}

function requiredAttr(name, toField) {
  return function(xml) {
    if (xml.$[name]) return ok(utils.keyValue(toField, xml.$[name]))
    else return error("Pakollinen attribuutti puuttuu: " + name)
  }
}

function optional(field, toField) {
  return function(xml) {
    toField = toField || field
    if (!xml[field] || !xml[field].$text || xml[field].$text.length == 0) return {program: utils.keyValue(toField, undefined), errors: []}
    else return {program: utils.keyValue(toField, xml[field].$text), errors: []}
  }
}

function childrenByNameTo(field, toField) {
  return function(xml) {
    return {
      program: utils.keyValue(toField, childrenByName(xml, field)),
      errors: []
    }
  }
}

function valuesInEnum(field, _enum) {
  return function(xml) {
    var values = optionListToArray(xml[field]).map(function(g) { return _enum[g] })
    if (_.all(values, function(v) { return v !== undefined })) return ok(utils.keyValue(field, values))
    else return error("Virheellinen kenttä " + field)
  }
}

function valuesInList(field, list) {
  return function(xml) {
    var values = optionListToArray(xml[field])
    var exists = values.map(_.curry(_.contains)(list))
    if (_.all(exists)) return ok(utils.keyValue(field, values))
    else return error("Virheellinen kenttä " + field)
  }
}

function node(name, toField, validators) {
  return flatMap(requiredNode(name, toField), function(p) {
    return function (xml) {
      return _.reduce(validators, function(acc, f) {
        return flatMap(acc, function(_) { return map(f, function(p) { return utils.keyValue(toField, p)}) })
      }, ret(p))(xml[name])
    }
  })
}

function test(field, f, msg, toField) {
  return function(xml) {
    var text = xml[field].$text
    if (f(text)) return ok(utils.keyValue(toField || field, text))
    else return error(msg + " " + text)
  }
}
function fullname(node) {
  return node.ETUNIMI.$text + ' ' + node.SUKUNIMI.$text
}
function childrenByName(root, name) {
{'fraction_non_alphanumeric': 0.1698763429961484, 'fraction_numerical': 0.004594905061152781, 'mean_word_length': 4.1211072664359865, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 96}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1162458', 'n_tokens_mistral': 5287, 'n_tokens_neox': 5172, 'n_words': 1037}