<commit_before>import JSZip from 'jszip'
import fetch from 'isomorphic-fetch'

import { secureFetch } from '../../common/util/util'
import { getConfigProperty } from '../../common/util/config'
import { fetchFeedVersions } from '../../manager/actions/feeds'
import { getFeedId } from '../../common/util/modules'

// EDIT ACTIVE GTFS+ ACTIONS

export function addGtfsPlusRow (tableId) {
  const table = getConfigProperty('modules.gtfsplus.spec').find(t => t.id === tableId)

  const rowData = {}
  for (const field of table.fields) {
    rowData[field.name] = null
  }

  return {
    type: 'ADD_GTFSPLUS_ROW',
    tableId,
    rowData
  }
}

export function updateGtfsPlusField (tableId, rowIndex, fieldName, newValue) {
  return {
    type: 'UPDATE_GTFSPLUS_FIELD',
    tableId,
    rowIndex,
    fieldName,
    newValue
  }
}

export function deleteGtfsPlusRow (tableId, rowIndex) {
  return {
    type: 'DELETE_GTFSPLUS_ROW',
    tableId,
    rowIndex
  }
}

// DOWNLOAD/RECEIVE DATA ACTIONS

export function requestingGtfsPlusContent () {
  return {
    type: 'REQUESTING_GTFSPLUS_CONTENT'
  }
}

export function clearGtfsPlusContent () {
  return {
    type: 'CLEAR_GTFSPLUS_CONTENT'
  }
}

export function receiveGtfsPlusContent (feedVersionId, filenames, fileContent, timestamp) {
  return {
    type: 'RECEIVE_GTFSPLUS_CONTENT',
    feedVersionId,
    filenames,
    fileContent,
    timestamp
  }
}

export function downloadGtfsPlusFeed (feedVersionId) {
  return function (dispatch, getState) {
    dispatch(requestingGtfsPlusContent())

    const fetchFeed = fetch('/api/manager/secure/gtfsplus/' + feedVersionId, {
      method: 'get',
      cache: 'default',
      headers: { 'Authorization': 'Bearer ' + getState().user.token }
    }).then((response) => {
      if (response.status !== 200) {
        console.log('error downloading gtfs+ feed', response.statusCode)
        dispatch(clearGtfsPlusContent())
      }
      return response.blob()
    })

    const fetchTimestamp = secureFetch(`/api/manager/secure/gtfsplus/${feedVersionId}/timestamp`, getState())
    .then(response => response.json())

    Promise.all([fetchFeed, fetchTimestamp]).then(([feed, timestamp]) => {
      JSZip.loadAsync(feed).then((zip) => {
        const filenames = []
        const filePromises = []
        zip.forEach((path, file) => {
          filenames.push(path)
          filePromises.push(file.async('string'))
        })
        Promise.all(filePromises).then(fileContent => {
          dispatch(receiveGtfsPlusContent(feedVersionId, filenames, fileContent, timestamp))
          dispatch(validateGtfsPlusFeed(feedVersionId))
        })
      })
    })
  }
}

// VALIDATION ACTIONS

export function validatingGtfsPlusFeed () {
  return {
    type: 'VALIDATING_GTFSPLUS_FEED'
  }
}

export function receiveGtfsPlusValidation (validationIssues) {
  return {
    type: 'RECEIVE_GTFSPLUS_VALIDATION',
    validationIssues
  }
}

export function validateGtfsPlusFeed (feedVersionId) {
  return function (dispatch, getState) {
    dispatch(validatingGtfsPlusFeed())
    const url = `/api/manager/secure/gtfsplus/${feedVersionId}/validation`
    return secureFetch(url, getState())
      .then(res => res.json())
      .then(validationIssues => {
        dispatch(receiveGtfsPlusValidation(validationIssues))
      })
  }
}

// UPLOAD ACTIONS

export function uploadingGtfsPlusFeed () {
  return {
    type: 'UPLOADING_GTFSPLUS_FEED'
  }
}

export function uploadedGtfsPlusFeed () {
  return {
    type: 'UPLOADED_GTFSPLUS_FEED'
  }
}

export function uploadGtfsPlusFeed (feedVersionId, file) {
  return function (dispatch, getState) {
    dispatch(uploadingGtfsPlusFeed())
    const url = `/api/manager/secure/gtfsplus/${feedVersionId}`
    var data = new window.FormData()
    data.append('file', file)

    return fetch(url, {
      method: 'post',
      headers: { 'Authorization': 'Bearer ' + getState().user.token },
      body: data
    }).then(result => {
      return dispatch(uploadedGtfsPlusFeed())
    })
  }<commit_msg>fix(clean file content of any BOMs):<commit_after>import JSZip from 'jszip'
import fetch from 'isomorphic-fetch'
import stripBom from 'strip-bom'

import { secureFetch } from '../../common/util/util'
import { getConfigProperty } from '../../common/util/config'
import { fetchFeedVersions } from '../../manager/actions/feeds'
import { getFeedId } from '../../common/util/modules'

// EDIT ACTIVE GTFS+ ACTIONS

export function addGtfsPlusRow (tableId) {
  const table = getConfigProperty('modules.gtfsplus.spec').find(t => t.id === tableId)

  const rowData = {}
  for (const field of table.fields) {
    rowData[field.name] = null
  }

  return {
    type: 'ADD_GTFSPLUS_ROW',
    tableId,
    rowData
  }
}

export function updateGtfsPlusField (tableId, rowIndex, fieldName, newValue) {
  return {
    type: 'UPDATE_GTFSPLUS_FIELD',
    tableId,
    rowIndex,
    fieldName,
    newValue
  }
}

export function deleteGtfsPlusRow (tableId, rowIndex) {
  return {
    type: 'DELETE_GTFSPLUS_ROW',
    tableId,
    rowIndex
  }
}

// DOWNLOAD/RECEIVE DATA ACTIONS

export function requestingGtfsPlusContent () {
  return {
    type: 'REQUESTING_GTFSPLUS_CONTENT'
  }
}

export function clearGtfsPlusContent () {
  return {
    type: 'CLEAR_GTFSPLUS_CONTENT'
  }
}

export function receiveGtfsPlusContent (feedVersionId, filenames, fileContent, timestamp) {
  return {
    type: 'RECEIVE_GTFSPLUS_CONTENT',
    feedVersionId,
    filenames,
    fileContent,
    timestamp
  }
}

export function downloadGtfsPlusFeed (feedVersionId) {
  return function (dispatch, getState) {
    dispatch(requestingGtfsPlusContent())

    const fetchFeed = fetch('/api/manager/secure/gtfsplus/' + feedVersionId, {
      method: 'get',
      cache: 'default',
      headers: { 'Authorization': 'Bearer ' + getState().user.token }
    }).then((response) => {
      if (response.status !== 200) {
        console.log('error downloading gtfs+ feed', response.statusCode)
        dispatch(clearGtfsPlusContent())
      }
      return response.blob()
    })

    const fetchTimestamp = secureFetch(`/api/manager/secure/gtfsplus/${feedVersionId}/timestamp`, getState())
    .then(response => response.json())

    Promise.all([fetchFeed, fetchTimestamp]).then(([feed, timestamp]) => {
      JSZip.loadAsync(feed).then((zip) => {
        const filenames = []
        const filePromises = []
        zip.forEach((path, file) => {
          filenames.push(path)
          filePromises.push(file.async('string'))
        })
        Promise.all(filePromises).then(fileContent => {
          const cleanFileContent = fileContent.map(cleanString)
          dispatch(receiveGtfsPlusContent(feedVersionId, filenames, cleanFileContent, timestamp))
          dispatch(validateGtfsPlusFeed(feedVersionId))
        })
      })
    })
  }
}

// clean BOM from string https://en.wikipedia.org/wiki/Byte_order_mark
function cleanString (string) {
  if (string.charCodeAt(0) === 0xFEFF) {
    string = string.slice(1)
  }
  return string
}
// VALIDATION ACTIONS

export function validatingGtfsPlusFeed () {
  return {
    type: 'VALIDATING_GTFSPLUS_FEED'
  }
}

export function receiveGtfsPlusValidation (validationIssues) {
  return {
    type: 'RECEIVE_GTFSPLUS_VALIDATION',
    validationIssues
  }
}

export function validateGtfsPlusFeed (feedVersionId) {
  return function (dispatch, getState) {
    dispatch(validatingGtfsPlusFeed())
    const url = `/api/manager/secure/gtfsplus/${feedVersionId}/validation`
    return secureFetch(url, getState())
      .then(res => res.json())
      .then(validationIssues => {
        dispatch(receiveGtfsPlusValidation(validationIssues))
      })
  }
}

// UPLOAD ACTIONS

export function uploadingGtfsPlusFeed (feedVersionId, file) {
  return {
    type: 'UPLOADING_GTFSPLUS_FEED',
    feedVersionId,
    file
  }
}

export function uploadedGtfsPlusFeed (feedVersionId, file) {
  return {
    type: 'UPLOADED_GTFSPLUS_FEED',
    feedVersionId,
    file
  }
}

export function uploadGtfsPlusFeed (feedVersionId, file) {
  return function (dispatch, getState) {
    dispatch(uploadingGtfsPlusFeed(feedVersionId, file))
    const url = `/api/manager/secure/gtfsplus/${feedVersionId}`
    var data = new window.FormData()
    data.append('file', file)

    return fetch(url, {
      method: 'post',
      headers: { 'Authorization': 'Bearer ' + getState().user.token },
      body: data
    }).then(result => {
      console.log(result)
      return dispatch(uploadedGtfsPlusFeed(feedVersionId, file))
    })
  }
{'fraction_non_alphanumeric': 0.12391660810494261, 'fraction_numerical': 0.0010541110330288123, 'mean_word_length': 3.4289419087136928, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4717144', 'n_tokens_mistral': 2789, 'n_tokens_neox': 2755, 'n_words': 522}