<commit_before>import Promise from 'bluebird'
import log from 'npmlog'
import {partial} from 'lodash/function'
import {find} from 'lodash/collection'
import {assign, get, omitBy, omit} from 'lodash/object'
import getEntityName from './get-entity-name'
import errorBuffer from '../utils/error-buffer'

/**
 * Creates a list of entities
 * Applies to all entities except Entries, as the CMA API for those is slightly different
 * See handleCreationErrors for details on what errors reject the promise or not.
 */
export function createEntities (context, entities, destinationEntities) {
  return Promise.map(entities, (entity) => {
    const destinationEntity = getDestinationEntityForSourceEntity(destinationEntities, entity.transformed)
    const promise = destinationEntity
      ? updateDestinationWithSourceData(destinationEntity, entity.transformed)
      : createInDestination(context, entity.transformed)
    return promise.then(
      partial(creationSuccessNotifier, destinationEntity ? 'update' : 'create'),
      partial(handleCreationErrors, entity)
    )
  })
}

/**
 * Creates a list of entries
 */
export function createEntries (context, entries, destinationEntries) {
  return Promise.map(entries, (entry) => createEntry(entry, context.space, context.skipContentModel, destinationEntries))
  .then(function (entries) {
    return entries.filter((entry) => entry)
  })
}

function createEntry (entry, space, skipContentModel, destinationEntries) {
  const contentTypeId = entry.original.sys.contentType.sys.id
  const destinationEntity = getDestinationEntityForSourceEntity(destinationEntries, entry.transformed)
  const promise = destinationEntity
    ? updateDestinationWithSourceData(destinationEntity, entry.transformed)
    : createEntryInDestination(space, contentTypeId, entry.transformed)
  return promise.then(
    partial(creationSuccessNotifier, destinationEntity ? 'update' : 'create'),
    partial(handleEntryCreationErrors, entry, space, skipContentModel, destinationEntries)
  )
}

function updateDestinationWithSourceData (destinationEntity, sourceEntity) {
  const plainData = getPlainData(sourceEntity)
  assign(destinationEntity, plainData)
  return destinationEntity.update()
}

function createInDestination (context, sourceEntity) {
  const id = get(sourceEntity, 'sys.id')
  const plainData = getPlainData(sourceEntity)
  return id
    ? context.space[`create${context.type}WithId`](id, plainData)
    : context.space[`create${context.type}`](plainData)
}

function createEntryInDestination (space, contentTypeId, sourceEntity) {
  const id = sourceEntity.sys.id
  const plainData = getPlainData(sourceEntity)
  return space.createEntryWithId(contentTypeId, id, plainData)
}

/**
 * Handles entity creation errors.
 * If the error is a VersionMismatch the error is thrown and a message is returned
 * instructing the user on what this situation probably means.
 */
function handleCreationErrors (entity, err) {
  // Handle the case where a locale already exists and skip it
  if (get(err, 'error.sys.id') === 'ValidationFailed') {
    const errors = get(err, 'error.details.errors')
    if (errors && errors.length > 0 && errors[0].name === 'taken') {
      return entity
    }
  }
  if (get(err, 'error.sys.id') === 'VersionMismatch') {
    log.error('Content update error:')
    log.error('Error', err.error)
    log.error('Request', err.request)
    log.error(`
This probably means you are synchronizing over a space with previously existing
content, or that you don't have the sync token for the last sync you performed
to this space.
    `)
  }
  throw err
}

/**
 * Handles entry creation errors.
 * If a field doesn't exist, it means it has been removed from the content types
 * In that case, the field is removed from the entry, and creation is attempted again.
 */
function handleEntryCreationErrors (entry, space, skipContentModel, destinationEntries, err) {
  if (skipContentModel && err.name === 'UnknownField') {
    entry.transformed.fields = cleanupUnknownFields(entry.transformed.fields, err.error.details.errors)
    return createEntry(entry, space, skipContentModel, destinationEntries)
  }
  err.originalEntry = entry.original
  err.transformedEntry = entry.transformed
  err.contentModelWasSkipped = skipContentModel
  errorBuffer.push(err)
  // No need to pass this entry down to publishing if it wasn't created
  return null
}

function cleanupUnknownFields (fields, errors) {
  return omitBy(fields, (field, fieldId) => {
    return find(errors, (error) => {
      const [, errorFieldId] = error.path
      return error.name === 'unknown' && errorFieldId === fieldId
    })
  })
}

function getDestinationEntityForSourceEntity (destinationEntities, sourceEntity) {
  return find(destinationEntities, {
    sys: {id: get(sourceEntity, 'sys.id', null)}
  })
}

function creationSuccessNotifier (method, createdEntity) {
  const verb = method[0].toUpperCase() + method.substr(1, method.length) + 'd'
  log.info(`${verb} ${createdEntity.sys.type} ${getEntityName(createdEntity)}`)
  return createdEntity
}

function getPlainData (entity) {
  const data = entity.toPlainObject ? entity.toPlainObject() : entity
  return omit(data, 'sys')
}
<commit_msg>fix(errors): Obtain errors safely in handleEntryCreationErrors <commit_after>import Promise from 'bluebird'
import log from 'npmlog'
import {partial} from 'lodash/function'
import {find} from 'lodash/collection'
import {assign, get, omitBy, omit} from 'lodash/object'
import getEntityName from './get-entity-name'
import errorBuffer from '../utils/error-buffer'

/**
 * Creates a list of entities
 * Applies to all entities except Entries, as the CMA API for those is slightly different
 * See handleCreationErrors for details on what errors reject the promise or not.
 */
export function createEntities (context, entities, destinationEntities) {
  return Promise.map(entities, (entity) => {
    const destinationEntity = getDestinationEntityForSourceEntity(destinationEntities, entity.transformed)
    const promise = destinationEntity
      ? updateDestinationWithSourceData(destinationEntity, entity.transformed)
      : createInDestination(context, entity.transformed)
    return promise.then(
      partial(creationSuccessNotifier, destinationEntity ? 'update' : 'create'),
      partial(handleCreationErrors, entity)
    )
  })
}

/**
 * Creates a list of entries
 */
export function createEntries (context, entries, destinationEntries) {
  return Promise.map(entries, (entry) => createEntry(entry, context.space, context.skipContentModel, destinationEntries))
  .then(function (entries) {
    return entries.filter((entry) => entry)
  })
}

function createEntry (entry, space, skipContentModel, destinationEntries) {
  const contentTypeId = entry.original.sys.contentType.sys.id
  const destinationEntity = getDestinationEntityForSourceEntity(destinationEntries, entry.transformed)
  const promise = destinationEntity
    ? updateDestinationWithSourceData(destinationEntity, entry.transformed)
    : createEntryInDestination(space, contentTypeId, entry.transformed)
  return promise.then(
    partial(creationSuccessNotifier, destinationEntity ? 'update' : 'create'),
    partial(handleEntryCreationErrors, entry, space, skipContentModel, destinationEntries)
  )
}

function updateDestinationWithSourceData (destinationEntity, sourceEntity) {
  const plainData = getPlainData(sourceEntity)
  assign(destinationEntity, plainData)
  return destinationEntity.update()
}

function createInDestination (context, sourceEntity) {
  const id = get(sourceEntity, 'sys.id')
  const plainData = getPlainData(sourceEntity)
  return id
    ? context.space[`create${context.type}WithId`](id, plainData)
    : context.space[`create${context.type}`](plainData)
}

function createEntryInDestination (space, contentTypeId, sourceEntity) {
  const id = sourceEntity.sys.id
  const plainData = getPlainData(sourceEntity)
  return space.createEntryWithId(contentTypeId, id, plainData)
}

/**
 * Handles entity creation errors.
 * If the error is a VersionMismatch the error is thrown and a message is returned
 * instructing the user on what this situation probably means.
 */
function handleCreationErrors (entity, err) {
  // Handle the case where a locale already exists and skip it
  if (get(err, 'error.sys.id') === 'ValidationFailed') {
    const errors = get(err, 'error.details.errors')
    if (errors && errors.length > 0 && errors[0].name === 'taken') {
      return entity
    }
  }
  if (get(err, 'error.sys.id') === 'VersionMismatch') {
    log.error('Content update error:')
    log.error('Error', err.error)
    log.error('Request', err.request)
    log.error(`
This probably means you are synchronizing over a space with previously existing
content, or that you don't have the sync token for the last sync you performed
to this space.
    `)
  }
  throw err
}

/**
 * Handles entry creation errors.
 * If a field doesn't exist, it means it has been removed from the content types
 * In that case, the field is removed from the entry, and creation is attempted again.
 */
function handleEntryCreationErrors (entry, space, skipContentModel, destinationEntries, err) {
  if (skipContentModel && err.name === 'UnknownField') {
    const errors = get(err, 'error.details.errors')
    entry.transformed.fields = cleanupUnknownFields(entry.transformed.fields, errors)
    return createEntry(entry, space, skipContentModel, destinationEntries)
  }
  err.originalEntry = entry.original
  err.transformedEntry = entry.transformed
  err.contentModelWasSkipped = skipContentModel
  errorBuffer.push(err)
  // No need to pass this entry down to publishing if it wasn't created
  return null
}

function cleanupUnknownFields (fields, errors) {
  return omitBy(fields, (field, fieldId) => {
    return find(errors, (error) => {
      const [, errorFieldId] = error.path
      return error.name === 'unknown' && errorFieldId === fieldId
    })
  })
}

function getDestinationEntityForSourceEntity (destinationEntities, sourceEntity) {
  return find(destinationEntities, {
    sys: {id: get(sourceEntity, 'sys.id', null)}
  })
}

function creationSuccessNotifier (method, createdEntity) {
  const verb = method[0].toUpperCase() + method.substr(1, method.length) + 'd'
  log.info(`${verb} ${createdEntity.sys.type} ${getEntityName(createdEntity)}`)
  return createdEntity
}

function getPlainData (entity) {
  const data = entity.toPlainObject ? entity.toPlainObject() : entity
  return omit(data, 'sys')
}

{'fraction_non_alphanumeric': 0.10344827586206896, 'fraction_numerical': 0.0007620499142693847, 'mean_word_length': 4.908272369161508, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1835184', 'n_tokens_mistral': 2889, 'n_tokens_neox': 2875, 'n_words': 854}