<commit_before>const Rx = require('rxjs')
const Task = require('data.task')
const Either = require('data.either')
const prop = require('propper')
const { cheerio, cheerioFind } = require('./utils/cheerio-utils')

const eitherToTask = require('./utils/either-to-task')
const fetchFirstPageToken = require('./fetch-first-page-token')
const { commentPage } = require('./youtube-api/youtube-api')

const getContentHtml = response =>
  Either.fromNullable(prop(response, 'content_html'))
    .leftMap(_ => 'API response does not contain a "content_html" field')

const getLoadMoreWidgetHtml = response =>
  Either.fromNullable(prop(response, 'load_more_widget_html'))
    .leftMap(_ => 'API response does not contain a "load_more_widget_html" field')

const extractButtonElement = html =>
  cheerioFind(cheerio(html), 'button.comment-section-renderer-paginator')
    .leftMap(_ => 'Comment page HTML does not contain a "Load More" button')

const extractTokenAttribute = $btn =>
  Either.fromNullable($btn.attr('data-uix-load-more-post-body'))
    .leftMap(_ => 'The comment page "Load More" button does not have the token attribute')

const extractNextPageToken = p =>
  getLoadMoreWidgetHtml(p)
    .chain(extractButtonElement)
    .chain(extractTokenAttribute)
    .map(token => token.replace(/^page_token=/i, ''))
    .map(token => decodeURIComponent(token))
    .chain(token => Task.of(token))

const observerEmitPage = observer => page =>
  getContentHtml(page)
    .orElse(Task.rejected)
    .map(html => observer.next(html))
    .chain(() => Task.of(page))

const buildCommentPageStream = videoId =>
  Rx.Observable.create(observer => {
    const emitPage = observerEmitPage(observer)

    const fetchAllPages = pageToken =>
      commentPage(videoId, pageToken)
        .chain(emitPage)
        .chain(p =>
          p.load_more_widget_html
            ? eitherToTask(extractNextPageToken(p).chain(fetchAllPages))
            : Task.of(`${videoId} - complete`))

    fetchFirstPageToken(videoId)
      .chain(fetchAllPages)
      .fork(e => observer.error(e), _ => observer.complete())
  })

module.exports = buildCommentPageStream
<commit_msg>fold either back to task<commit_after>const Rx = require('rxjs')
const Task = require('data.task')
const Either = require('data.either')
const prop = require('propper')
const { cheerio, cheerioFind } = require('./utils/cheerio-utils')

const fetchFirstPageToken = require('./fetch-first-page-token')
const { commentPage } = require('./youtube-api/youtube-api')

const getContentHtml = response =>
  Either.fromNullable(prop(response, 'content_html'))
    .leftMap(_ => 'API response does not contain a "content_html" field')

const getLoadMoreWidgetHtml = response =>
  Either.fromNullable(prop(response, 'load_more_widget_html'))
    .leftMap(_ => 'API response does not contain a "load_more_widget_html" field')

const extractButtonElement = html =>
  cheerioFind(cheerio(html), 'button.comment-section-renderer-paginator')
    .leftMap(_ => 'Comment page HTML does not contain a "Load More" button')

const extractTokenAttribute = $btn =>
  Either.fromNullable($btn.attr('data-uix-load-more-post-body'))
    .leftMap(_ => 'The comment page "Load More" button does not have the token attribute')

const extractNextPageToken = p =>
  getLoadMoreWidgetHtml(p)
    .chain(extractButtonElement)
    .chain(extractTokenAttribute)
    .map(token => token.replace(/^page_token=/i, ''))
    .map(token => decodeURIComponent(token))
    .fold(e => Task.rejected(e),
          t => Task.of(t))

const observerEmitPage = observer => page =>
  getContentHtml(page)
    .map(html => observer.next(html))
    .fold(e => Task.rejected(e),
          _ => Task.of(page))

const buildCommentPageStream = videoId =>
  Rx.Observable.create(observer => {
    const emitPage = observerEmitPage(observer)

    const fetchAllPages = pageToken =>
      commentPage(videoId, pageToken)
        .chain(emitPage)
        .chain(p =>
          p.load_more_widget_html
            ? extractNextPageToken(p).chain(fetchAllPages)
            : Task.of(`${videoId} - complete`))

    fetchFirstPageToken(videoId)
      .chain(fetchAllPages)
      .fork(e => observer.error(e), _ => observer.complete())
  })

module.exports = buildCommentPageStream

{'fraction_non_alphanumeric': 0.13783403656821377, 'fraction_numerical': 0.0, 'mean_word_length': 4.159613059250303, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5599367', 'n_tokens_mistral': 1293, 'n_tokens_neox': 1284, 'n_words': 269}