<commit_before>import path from 'path'
import fs from 'fs-extra'
import _ from 'lodash'
import moment from 'moment'
import request from 'request'
import errors from 'feathers-errors'
import logger from 'winston'
import makeDebug from 'debug'
import { Grid } from '../grid'
const debug = makeDebug('weacast:weacast-core')

function getMinMax(grid) {
  let minValue = (grid && grid.length > 0 ? grid[0] : Number.NEGATIVE_INFINITY)
  let maxValue = (grid && grid.length > 0 ? grid[0] : Number.POSITIVE_INFINITY)
  for (let i = 1; i < grid.length; i++) {
    minValue = Math.min(minValue, grid[i])
    maxValue = Math.max(maxValue, grid[i])
  }
  return { minValue, maxValue }
}

export default {

  // Retrieve the path where downloaded/persited data are
  getDataDirectory () {
    return path.join(this.app.get('forecastPath'), this.forecast.name, this.element.name)
  },

  // Generate file name to store temporary output (i.e. converted) data, assume by default a similar name than getForecastTimeFilePath() with a json extension
  getForecastTimeConvertedFilePath (runTime, forecastTime) {
    let filePath = this.getForecastTimeFilePath(runTime, forecastTime)
    return path.join(path.dirname(filePath), path.basename(filePath, path.extname(filePath)) + '.json')
  },

  downloadForecastTime (runTime, forecastTime) {
    let promise = new Promise((resolve, reject) => {
      const filePath = this.getForecastTimeFilePath(runTime, forecastTime)
      if (fs.existsSync(filePath)) {
        logger.verbose('Already downloaded ' + this.forecast.name + '/' + this.element.name + ' forecast at ' + forecastTime.format() + ' for run ' + runTime.format())
        resolve(filePath)
        return
      }
      // Get request options
      logger.verbose('Downloading ' + this.forecast.name + '/' + this.element.name + ' forecast at ' + forecastTime.format() + ' for run ' + runTime.format())
      let errorMessage = 'Could not download ' + this.forecast.name + '/' + this.element.name + ' forecast at ' + forecastTime.format() + ' for run ' + runTime.format()
      // Get request options
      request.get(this.getForecastTimeRequest(runTime, forecastTime))
      .on('error', err => {
        logger.error(errorMessage, err)
        reject(err)
      })
      .on('timeout', err => {
        logger.error(errorMessage + ', provider timed out')
        reject(err)
      })
      .on('response', response => {
        if (response.statusCode !== 200) {
          errorMessage += ', provider responded with HTTP code ' + response.statusCode
          reject(errors.convert({
            name: response.statusCode,
            message: errorMessage
          }))
        } else {
          let file = fs.createWriteStream(filePath)
          response.pipe(file)
          .on('finish', _ => {
            file.close()
            logger.verbose('Written ' + this.forecast.name + '/' + this.element.name + ' forecast at ' + forecastTime.format() + ' for run ' + runTime.format())
            resolve(filePath)
          })
          .on('error', err => {
            logger.error(errorMessage + ', unable to write temporary file', err)
            debug('Output TIFF file was : ' + filePath)
            reject(err)
          })
        }
      })
    })

    return promise
  },

  async processForecastTime (runTime, forecastTime) {
    await this.downloadForecastTime(runTime, forecastTime)
    let grid = await this.convertForecastTime(runTime, forecastTime)
    if (this.element.dataStore === 'gridfs') {
      await this.saveToGridFS(this.getForecastTimeConvertedFilePath(runTime, forecastTime))
    }
    // Compute min/max values
    let forecast = Object.assign({
      runTime: runTime,
      forecastTime: forecastTime
    }, getMinMax(grid))
    // Depending if we keep file as data storage include a link to files or data directly in the object
    if (this.element.dataStore === 'fs' || this.element.dataStore === 'gridfs') {
      return Object.assign(forecast, {
        filePath: this.getForecastTimeFilePath(runTime, forecastTime),
        convertedFilePath: this.getForecastTimeConvertedFilePath(runTime, forecastTime)
      })
    } else {
      return Object.assign(forecast, {
        data: grid
      })
    }
  },

  async updateForecastTimeInDatabase (data, previousData) {
    // Test if we have to remove existing data first
    if (previousData) {
      await this.remove(null, {
        query: {<commit_msg>WIP https://github.com/weacast/weacast/issues/29 (added processing function)<commit_after>import path from 'path'
import fs from 'fs-extra'
import _ from 'lodash'
import moment from 'moment'
import request from 'request'
import errors from 'feathers-errors'
import logger from 'winston'
import makeDebug from 'debug'
import { Grid } from '../grid'
const debug = makeDebug('weacast:weacast-core')

function getMinMax(grid) {
  let minValue = (grid && grid.length > 0 ? grid[0] : Number.NEGATIVE_INFINITY)
  let maxValue = (grid && grid.length > 0 ? grid[0] : Number.POSITIVE_INFINITY)
  for (let i = 1; i < grid.length; i++) {
    minValue = Math.min(minValue, grid[i])
    maxValue = Math.max(maxValue, grid[i])
  }
  return { minValue, maxValue }
}

export default {

  // Retrieve the path where downloaded/persited data are
  getDataDirectory () {
    return path.join(this.app.get('forecastPath'), this.forecast.name, this.element.name)
  },

  // Generate file name to store temporary output (i.e. converted) data, assume by default a similar name than getForecastTimeFilePath() with a json extension
  getForecastTimeConvertedFilePath (runTime, forecastTime) {
    let filePath = this.getForecastTimeFilePath(runTime, forecastTime)
    return path.join(path.dirname(filePath), path.basename(filePath, path.extname(filePath)) + '.json')
  },

  downloadForecastTime (runTime, forecastTime) {
    let promise = new Promise((resolve, reject) => {
      const filePath = this.getForecastTimeFilePath(runTime, forecastTime)
      if (fs.existsSync(filePath)) {
        logger.verbose('Already downloaded ' + this.forecast.name + '/' + this.element.name + ' forecast at ' + forecastTime.format() + ' for run ' + runTime.format())
        resolve(filePath)
        return
      }
      // Get request options
      logger.verbose('Downloading ' + this.forecast.name + '/' + this.element.name + ' forecast at ' + forecastTime.format() + ' for run ' + runTime.format())
      let errorMessage = 'Could not download ' + this.forecast.name + '/' + this.element.name + ' forecast at ' + forecastTime.format() + ' for run ' + runTime.format()
      // Get request options
      request.get(this.getForecastTimeRequest(runTime, forecastTime))
      .on('error', err => {
        logger.error(errorMessage, err)
        reject(err)
      })
      .on('timeout', err => {
        logger.error(errorMessage + ', provider timed out')
        reject(err)
      })
      .on('response', response => {
        if (response.statusCode !== 200) {
          errorMessage += ', provider responded with HTTP code ' + response.statusCode
          reject(errors.convert({
            name: response.statusCode,
            message: errorMessage
          }))
        } else {
          let file = fs.createWriteStream(filePath)
          response.pipe(file)
          .on('finish', _ => {
            file.close()
            logger.verbose('Written ' + this.forecast.name + '/' + this.element.name + ' forecast at ' + forecastTime.format() + ' for run ' + runTime.format())
            resolve(filePath)
          })
          .on('error', err => {
            logger.error(errorMessage + ', unable to write temporary file', err)
            debug('Output TIFF file was : ' + filePath)
            reject(err)
          })
        }
      })
    })

    return promise
  },

  async processForecastTime (runTime, forecastTime) {
    await this.downloadForecastTime(runTime, forecastTime)
    let grid = await this.convertForecastTime(runTime, forecastTime)
    if (this.element.dataStore === 'gridfs') {
      await this.saveToGridFS(this.getForecastTimeConvertedFilePath(runTime, forecastTime))
    }
    // Check for processing function
    if (typeof this.element.transform === 'function') {
      for (let i = 1; i < grid.length; i++) {
        grid[i] = this.element.transform({
          runTime, forecastTime, value: grid[i], index: i, data: grid, forecast: this.forecast, element: this.element
        })
      }
    }
    // Compute min/max values
    let forecast = Object.assign({
      runTime: runTime,
      forecastTime: forecastTime
    }, getMinMax(grid))
    // Depending if we keep file as data storage include a link to files or data directly in the object
    if (this.element.dataStore === 'fs' || this.element.dataStore === 'gridfs') {
      return Object.assign(forecast, {
        filePath: this.getForecastTimeFilePath(runTime, forecastTime),
        convertedFilePath: this.getForecastTimeConvertedFilePath(runTime, forecastTime)
      })
    } else {
      return Object.assign(forecast, {
        data: grid
      })
    }
  },

  async updateForecastTimeInDatabase (data, previousData) {
    // Test if we have to remove existing data first
    if (previousData) {
      await this.remove(null, {
        query: {
{'fraction_non_alphanumeric': 0.12292071721754158, 'fraction_numerical': 0.0020522791099589544, 'mean_word_length': 2.8675856307435255, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 19, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3864369', 'n_tokens_mistral': 2700, 'n_tokens_neox': 2660, 'n_words': 675}