<commit_before>{ fail, pretty, debug, log, errorLog } = require '../lib/common'
_ = require 'lodash'
Promise = require('parse/node').Promise
request = require 'request-promise'
WP = require 'wpapi' # https://github.com/WP-API/node-wpapi
Entities = require('html-entities').AllHtmlEntities # https://www.npmjs.com/package/html-entities
parse = require '../lib/pegg-parse'
AWS = require 'aws-sdk'

entities = new Entities()

SQUISHLE_USERNAME = process.env.SQUISHLE_USERNAME or fail "cannot have an empty SQUISHLE_USERNAME"
SQUISHLE_PASSWORD = process.env.SQUISHLE_PASSWORD or fail "cannot have an empty SQUISHLE_PASSWORD"

wp = new WP (
  endpoint: 'http://squishle.me/wp-json'
  username: SQUISHLE_USERNAME
  password: <PASSWORD>
  auth: true
)

gifIdPattern = /[\/-]([^\/?-]+)($|\?)/

class PeggSquishle
  #card =
  #  question: 'Some question'
  #  deck: 'Throwback'
  #  choices: [
  #    {text: "first choice", image: {source:"https://media1.giphy.com/media/lppjX4teaSUnu/giphy.gif", url:""}},
  #    {text: "second choice", image:{source:"https://media1.giphy.com/media/lppjX4teaSUnu/giphy.gif", url:""}},
  #    {text:"third choice", image:{source:"https://media1.giphy.com/media/lppjX4teaSUnu/giphy.gif", url:""}},
  #    {text:"fourth choice", image:{source:"https://media1.giphy.com/media/lppjX4teaSUnu/giphy.gif", url:""}}
  #  ]
  processCard: (postId, categories) =>
    log "creating card from post: #{postId}"
    post = null
    @fetchPostData postId
    .then (_post) => post = _post
    .then @fetchImageData
    .catch (errors) =>
      errorLog errors.toString()
      content = post.content
      if content?.choices?
        for error in errors
          if error?.choice?.num?
            squishleChoice = _.find content.choices, (c) -> c.num is error.choice.num
            squishleChoice.error = error.message
      else
        content = error: errors.toString()
      debug "updating post with error", content
      @updatePost postId, JSON.stringify content
      throw errors
    .then (card) =>
      if categories?
        card.deck = JSON.parse(categories)?[0]
      if card.id?
        @updateCard card
          .then (result) =>
            log "card updated: ", pretty result
          # FIX: updating the post causes an infinite loop.
          # If we need this someday, then find some way to check the request on wp_insert_post action...
          #   to see if update came from this app or the user hitting the publish button.
          #   @updatePost postId, JSON.stringify result
            @backupImages card.choices
      else
        @createCard card
          .then (result) =>
            log "card created: ", pretty result
            @updatePost postId, JSON.stringify result
            @backupImages card.choices
            @incrementDeck card.deck

  backupImages: (choices) =>
    s3 = new AWS.S3()
    for own id, choice of choices
      do (id, choice) =>
        # download image to buffer
        request.get uri: choice.image.url, encoding: null
        .then (body) =>
          log "downloaded image for choice", choice.id
          # upload to s3
          s3.putObject {
            Bucket: 'images.pegg.us'
            Key: choice.id + '.mp4'
            Metadata:
              sourceUrl: choice.image.url
            Body: body
          }, (error, data) =>
            if error? then errorLog "aws error", error, error.stack
            else log "aws success", data<commit_msg>updates now dont spawn another push from squishle, so we can update squishle on card update<commit_after>{ fail, pretty, debug, log, errorLog } = require '../lib/common'
_ = require 'lodash'
Promise = require('parse/node').Promise
request = require 'request-promise'
WP = require 'wpapi' # https://github.com/WP-API/node-wpapi
Entities = require('html-entities').AllHtmlEntities # https://www.npmjs.com/package/html-entities
parse = require '../lib/pegg-parse'
AWS = require 'aws-sdk'

entities = new Entities()

SQUISHLE_USERNAME = process.env.SQUISHLE_USERNAME or fail "cannot have an empty SQUISHLE_USERNAME"
SQUISHLE_PASSWORD = process.env.SQUISHLE_PASSWORD or fail "cannot have an empty SQUISHLE_PASSWORD"

wp = new WP (
  endpoint: 'http://squishle.me/wp-json'
  username: SQUISHLE_USERNAME
  password: <PASSWORD>
  auth: true
)

gifIdPattern = /[\/-]([^\/?-]+)($|\?)/

class PeggSquishle
  #card =
  #  question: 'Some question'
  #  deck: 'Throwback'
  #  choices: [
  #    {text: "first choice", image: {source:"https://media1.giphy.com/media/lppjX4teaSUnu/giphy.gif", url:""}},
  #    {text: "second choice", image:{source:"https://media1.giphy.com/media/lppjX4teaSUnu/giphy.gif", url:""}},
  #    {text:"third choice", image:{source:"https://media1.giphy.com/media/lppjX4teaSUnu/giphy.gif", url:""}},
  #    {text:"fourth choice", image:{source:"https://media1.giphy.com/media/lppjX4teaSUnu/giphy.gif", url:""}}
  #  ]
  processCard: (postId, categories) =>
    log "creating card from post: #{postId}"
    post = null
    @fetchPostData postId
    .then (_post) => post = _post
    .then @fetchImageData
    .catch (errors) =>
      errorLog errors.toString()
      content = post.content
      if content?.choices?
        for error in errors
          if error?.choice?.num?
            squishleChoice = _.find content.choices, (c) -> c.num is error.choice.num
            squishleChoice.error = error.message
      else
        content = error: errors.toString()
      debug "updating post with error", content
      @updatePost postId, JSON.stringify content
      throw errors
    .then (card) =>
      if categories?
        card.deck = JSON.parse(categories)?[0]
      if card.id?
        @updateCard card
          .then (result) =>
            log "card updated: ", pretty result
            @updatePost postId, JSON.stringify result
            @backupImages card.choices
      else
        @createCard card
          .then (result) =>
            log "card created: ", pretty result
            @updatePost postId, JSON.stringify result
            @backupImages card.choices
            @incrementDeck card.deck

  backupImages: (choices) =>
    s3 = new AWS.S3()
    for own id, choice of choices
      do (id, choice) =>
        # download image to buffer
        request.get uri: choice.image.url, encoding: null
        .then (body) =>
          log "downloaded image for choice", choice.id
          # upload to s3
          s3.putObject {
            Bucket: 'images.pegg.us'
            Key: choice.id + '.mp4'
            Metadata:
              sourceUrl: choice.image.url
            Body: body
          }, (error, data) =>
            if error? then errorLog "aws error", error, error.stack
            else log "aws success", data
{'fraction_non_alphanumeric': 0.12840756740652465, 'fraction_numerical': 0.004171011470281543, 'mean_word_length': 2.5115062761506275, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 27, 'https://': 12, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '481710', 'n_tokens_mistral': 2134, 'n_tokens_neox': 2117, 'n_words': 553}