<commit_before>import fs from 'fs'
import jsonfile from 'jsonfile'
import path from 'path'

// set indentation for jsonfile
jsonfile.spaces = 2

class ProductValidator {
  constructor () {
    this.orderedKeys = [
      'id',
      'name',
      'linked-id', // added by mcmunder
      'fao-product-id', // added by mcmunder
      'co2-value',
      'tags',
      'specification',
      'synonyms',
      'name-english',
      'name-french',
      'nutrition-id',
      'alternatives',
      'production-names',
      'production-values',
      'processing-names',
      'processing-values',
      'conservation-names',
      'conservation-values',
      'packaging-names',
      'packaging-values',
      'season-begin',
      'season-end',
      'combined-product',
      'density',
      'unit-weight',
      'quantity-comments',
      'quantity-references',
      'foodwaste',
      'foodwaste-comment',
      'co2-calculation',
      'calculation-process-documentation',
      'info-text',
      'references',
      'other-references',
      'comments',
      'co2-calculation-parameters',
      'references-parameters',
      'data-quality',
      'author',
      'delete',
      'allergenes', // added by mcmunder
      'processes', // added by mcmunder
      'waste-id', // added by mcmunder
      'filename', // delete later
      'validationSummary' // delete later
    ]

    this.mandatoryFields = [
      'name',
      'id',
      'co2-value',
      'nutrition-id',
      'tags'
    ]

    // these optional fields will be oulled from parent if possible
    this.optionalFields = [
      'fao-product-id',
      'production-names',
      'production-values',
      'conservation-names',
      'conservation-values',
      'packaging-names',
      'packaging-values',
      'season-begin',
      'season-end',
      'unit-weight',
      'processes',
      'allergenes',
      'density',
      'alternatives'
    ]

    // concat all fields
    this.allFields = this.mandatoryFields.concat(this.optionalFields)
  }

  // method to set dataDir
  setDataDir (dataDir) {
    this.dataDir = dataDir
    this.prodFilenames = fs.readdirSync(`${this.dataDir}/prods`)
    this.nutrsFilenames = fs.readdirSync(`${this.dataDir}/nutrs`)
    this.nutrChangeFilenames = fs.readdirSync(`${this.dataDir}/nutr-change`)
    this.prods = this.prodFilenames
      .filter(filename => {
        return path.extname(filename) === '.json' &&
        !(filename === 'prod.schema.json')
      })
      .map(filename => {
        const product = jsonfile.readFileSync(`${this.dataDir}/prods/${filename}`)
        return Object.assign(product, {filename})
      })
    this.nutrs = this.nutrsFilenames.map(filename => {
      return jsonfile.readFileSync(`${this.dataDir}/nutrs/${filename}`)
    })
    this.nutrChange = this.nutrChangeFilenames.map(filename => {
      return jsonfile.readFileSync(`${this.dataDir}/nutr-change/${filename}`)
    })

    return this
  }

  setProduct (product) {
    this.product = product
    return this
  }

  orderAndCleanupProduct (product = this.product) {
    const orderedPairs = this.orderedKeys
      .filter(key => {
        return product.hasOwnProperty(key) && product[key] !== ''
      })
      .map(key => ({[key]: product[key]}))

    this.orderedProduct = Object.assign({}, ...orderedPairs)

    return this
  }

  orderFixedProducts (fixedProducts = this.fixedProducts) {
    this.orderedFixedProducts = fixedProducts.map(product => {
      // delete product.filename
      // delete product.validationSummary
      return this.orderAndCleanupProduct(product).orderedProduct
    })

    return this
  }

  saveOrderedFixedProducts () {
    // TODO: Save single products to original location! First figure out a way
    // to conserve order of keys...
    this.orderedFixedProducts.forEach(product => {
      const filename = product.filename

      // remove filename and validationSummary fields
      delete product.filename
      delete product.validationSummary

      const orderedProduct = this.orderProduct(product).orderedProduct

      jsonfile.writeFileSync(`${this.dataDir}/prods/${filename}`, orderedProduct)
    })

    jsonfile.writeFileSync(`${this.dataDir}/prods.all.json`,
      this.orderedFixedProducts)

    return this
  }

  // method to validate a product
  validateProduct (product = this.product) {
    this.setProduct(product)

    // define a validationResult
    let validationSummary = {
      parentProduct: '',
      hasNutritionId: false,
      hasNutritionChangeId: false,
      linkedNutritionIdExists: false,
      linkedNutritionChangeIdsExist: false,
      missingFields: []
    }

    const missingFields = this.allFields.filter(field => {
      return !this.product.hasOwnProperty(field)
    })

    const isLinked = this.product.hasOwnProperty('linked-id')
    const fieldsMissing = missingFields.length > 0

    if (fieldsMissing) {
      validationSummary = Object.assign(validationSummary, {missingFields})<commit_msg>Rollback validator to d418a0d6ace3dbb907c96a32a181b553ed445474<commit_after>import fs from 'fs'
import jsonfile from 'jsonfile'
import path from 'path'

// set indentation for jsonfile
jsonfile.spaces = 2

class ProductValidator {
  constructor () {
    this.orderedKeys = [
      'id',
      'name',
      'linked-id', // added by mcmunder
      'co2-value',
      'tags',
      'specification',
      'synonyms',
      'name-english',
      'name-french',
      'nutrition-id',
      'alternatives',
      'production-names',
      'production-values',
      'processing-names',
      'processing-values',
      'conservation-names',
      'conservation-values',
      'packaging-names',
      'packaging-values',
      'season-begin',
      'season-end',
      'combined-product',
      'density',
      'unit-weight',
      'quantity-comments',
      'quantity-references',
      'foodwaste',
      'foodwaste-comment',
      'co2-calculation',
      'calculation-process-documentation',
      'info-text',
      'references',
      'other-references',
      'comments',
      'co2-calculation-parameters',
      'references-parameters',
      'data-quality',
      'author',
      'delete',
      'filename', // delete later
      'validationSummary' // delete later
    ]

    this.mandatoryFields = [
      'name',
      'id',
      'co2-value',
      'nutrition-id',
      'tags'
    ]

    // these optional fields will be oulled from parent if possible
    this.optionalFields = [
      'fao-product-id',
      'production-names',
      'production-values',
      'conservation-names',
      'conservation-values',
      'packaging-names',
      'packaging-values',
      'season-begin',
      'season-end',
      'unit-weight',
      'processes',
      'allergenes',
      'density',
      'alternatives'
    ]

    // concat all fields
    this.allFields = this.mandatoryFields.concat(this.optionalFields)
  }

  // method to set dataDir
  setDataDir (dataDir) {
    this.dataDir = dataDir
    this.prodFilenames = fs.readdirSync(`${this.dataDir}/prods`)
    this.nutrsFilenames = fs.readdirSync(`${this.dataDir}/nutrs`)
    this.nutrChangeFilenames = fs.readdirSync(`${this.dataDir}/nutr-change`)
    this.prods = this.prodFilenames
      .filter(filename => {
        return path.extname(filename) === '.json' &&
        !(filename === 'prod.schema.json')
      })
      .map(filename => {
        const product = jsonfile.readFileSync(`${this.dataDir}/prods/${filename}`)
        return Object.assign(product, {filename})
      })
    this.nutrs = this.nutrsFilenames.map(filename => {
      return jsonfile.readFileSync(`${this.dataDir}/nutrs/${filename}`)
    })
    this.nutrChange = this.nutrChangeFilenames.map(filename => {
      return jsonfile.readFileSync(`${this.dataDir}/nutr-change/${filename}`)
    })

    return this
  }

  setProduct (product) {
    this.product = product
    return this
  }

  orderProduct (product = this.product) {
    const orderedPairs = this.orderedKeys
      .filter(key => !!product[key])
      .map(key => ({[key]: product[key]}))

    this.orderedProduct = Object.assign({}, ...orderedPairs)

    return this
  }

  orderFixedProducts (fixedProducts = this.fixedProducts) {
    this.orderedFixedProducts = fixedProducts.map(product => {
      // delete product.filename
      // delete product.validationSummary
      return this.orderProduct(product).orderedProduct
    })

    return this
  }

  saveOrderedFixedProducts () {
    // TODO: Save single products to original location! First figure out a way
    // to conserve order of keys...
    // this.fixedProducts.forEach(product => {
    //   const filename = product.filename
    //
    //   // remove filename and validationSummary fields
    //   delete product.filename
    //   delete product.validationSummary
    //
    //   const orderedProduct = this.orderProduct(product).orderedProduct
    //
    //   jsonfile.writeFileSync(`${this.dataDir}/prods/${filename}`, orderedProduct)
    // })
    jsonfile.writeFileSync(`${this.dataDir}/prods.all.json`,
      this.orderedFixedProducts)

    return this
  }

  // method to validate a product
  validateProduct (product = this.product) {
    this.setProduct(product)

    // define a validationResult
    let validationSummary = {
      parentProduct: '',
      hasNutritionId: false,
      hasNutritionChangeId: false,
      linkedNutritionIdExists: false,
      linkedNutritionChangeIdsExist: false,
      missingFields: []
    }

    const missingFields = this.allFields.filter(field => {
      return !this.product.hasOwnProperty(field)
    })

    const isLinked = this.product.hasOwnProperty('linked-id')
    const fieldsMissing = missingFields.length > 0

    if (fieldsMissing) {
      validationSummary = Object.assign(validationSummary, {missingFields})
{'fraction_non_alphanumeric': 0.13131929610416032, 'fraction_numerical': 0.0037636049232021157, 'mean_word_length': 2.6307237813884785, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1034276', 'n_tokens_mistral': 2990, 'n_tokens_neox': 2912, 'n_words': 542}