<commit_before>import fs from 'fs'
import path from 'path'
import os from 'os'
import Promise from 'bluebird'
import axios from 'axios'
import Confirm from 'prompt-confirm'
import username from 'username'
import prependFile from 'prepend-file'
import mapValues from 'lodash/mapValues'

import { print } from '../util'
import { REVISIONS_FILE_NAME } from '../ghost-content'

import { login } from './auth'

Promise.promisifyAll(fs)

const writeRevisions = async (revisionsFile, revisions) => {
  const user = await username()
  const host = os.hostname()
  let fullUser = [user, host].filter(Boolean).join('@')
  if (fullUser) {
    fullUser = ' by ' + fullUser
  }
  const preamble = `# Synced ${new Date().toISOString()}${fullUser}`

  return Promise.fromCallback(cb => {
    prependFile(revisionsFile, [preamble, ...revisions, ''].join('\n'), cb)
  })
}

const writeFile = folderPath => async ({ file, content, deleted }) => {
  const filePath = path.join(folderPath, file)

  if (!deleted) {
    return fs.writeFileAsync(filePath, content)
  }

  try {
    return await fs.unlinkAsync(filePath)
  } catch (e) {
    if (e.code === 'ENOENT') {
      return Promise.resolve(true)
    }
    throw e
  }
}

const updateFolder = projectLocation => async ({ files, revisions, binary }, folder) => {
  const folderPath = path.join(projectLocation, folder)
  const revisionsFile = path.join(folderPath, REVISIONS_FILE_NAME)
  await writeRevisions(revisionsFile, revisions)

  if (binary) {
    files.forEach(data => {
      data.content = Buffer.from(data.content, 'base64')
    })
  }

  return Promise.each(files, writeFile(folderPath))
}

module.exports = async botUrl => {
  botUrl = botUrl.replace(/\/+$/, '')<commit_msg>fix(core): create missing files/folder during ghost-sync (resolve #996) (#1031)<commit_after>import fs from 'fs'
import path from 'path'
import os from 'os'
import Promise from 'bluebird'
import axios from 'axios'
import Confirm from 'prompt-confirm'
import username from 'username'
import prependFile from 'prepend-file'
import mapValues from 'lodash/mapValues'

import { print } from '../util'
import { REVISIONS_FILE_NAME } from '../ghost-content'

import { login } from './auth'

Promise.promisifyAll(fs)

const writeRevisions = async (revisionsFile, revisions) => {
  const user = await username()
  const host = os.hostname()
  let fullUser = [user, host].filter(Boolean).join('@')
  if (fullUser) {
    fullUser = ' by ' + fullUser
  }
  const preamble = `# Synced ${new Date().toISOString()}${fullUser}`

  return Promise.fromCallback(cb => {
    prependFile(revisionsFile, [preamble, ...revisions, ''].join('\n'), cb)
  })
}

const writeFile = rootFolderPath => async ({ file, content, deleted }) => {
  const folderPath = path.join(rootFolderPath, ...file.split('/').slice(0, -1))
  const filePath = path.join(rootFolderPath, file)

  if (!deleted) {
    try {
      await fs.statAsync(folderPath)
    } catch (e) {
      await fs.mkdirAsync(folderPath, { recursive: true })
    }
    return fs.writeFileAsync(filePath, content)
  }

  try {
    return await fs.unlinkAsync(filePath)
  } catch (e) {
    if (e.code === 'ENOENT') {
      return Promise.resolve(true)
    }
    throw e
  }
}

const updateFolder = projectLocation => async ({ files, revisions, binary }, folder) => {
  const folderPath = path.join(projectLocation, folder)
  try {
    await fs.statAsync(folderPath)
  } catch (e) {
    await fs.mkdirAsync(folderPath, { recursive: true })
  }
  const revisionsFile = path.join(folderPath, REVISIONS_FILE_NAME)
  await writeRevisions(revisionsFile, revisions)

  if (binary) {
    files.forEach(data => {
      data.content = Buffer.from(data.content, 'base64')
    })
  }

  return Promise.each(files, writeFile(folderPath))
}

module.exports = async botUrl => {
  botUrl = botUrl.replace(/\/+$/, '')
{'fraction_non_alphanumeric': 0.14360041623309053, 'fraction_numerical': 0.0033818938605619147, 'mean_word_length': 3.848675914249685, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2130403', 'n_tokens_mistral': 1284, 'n_tokens_neox': 1239, 'n_words': 319}