<commit_before>/* jshint esversion:6, node:true  */

'use strict';

eamModule(module, 'rssRegistrationsFetcher', (
  $_,
  $q,
  logger,
  modelsEntry,
  dbMongooseBinders,
  modelsRssRegistration,
  modelsFetchReport,
  rssRegistrationsFetcherIterationFetch
) => {

  return {
    fetch
  };

  function fetch() {
    const finishedDefer = $q.defer();
    
    const fetchReport = createFetchReport();
    
    fetchRegistrations()
      .then(startRssRegistrationFetch);

    return finishedDefer.promise;

    function fetchRegistrations() {
      return dbMongooseBinders.find(modelsRssRegistration)
        .catch(reason => finishedDefer.reject(reason))
    }

    function startRssRegistrationFetch(registrationsResp) {
      rssRegistrationsFetcherIterationFetch.fetch(
        registrationsResp, 
        onIterationFetchFinished, 
        onFinish
      );
    }

    function onIterationFetchFinished(registrationEntries) {
      const promiseOfEntries = $_.map(registrationEntries, registrationEntry => {
        const rssRegistration = registrationEntry.rssRegistration;
        logger.assert(rssRegistration, 'rssRegistration always exist');

        const majorError = registrationEntry.error;
        const translationErrors = registrationEntry.entriesResp 
                                    && registrationEntry.entriesResp.errors;
        const error = majorError || translationErrors;

        if (error) {
          const failedFetch = {
            error,
            rssRegistration: rssRegistration._id
          };
          fetchReport.failedFetches.push(failedFetch);
        }

        if (majorError) {
          return $q.when();
        }

        const entryModel = modelsEntry.getByCategoryLang(
          rssRegistration.lang,
          rssRegistration.category.name
        );        

        const entries = registrationEntry.entriesResp.entries;
        fetchReport.succeededFetches += $_.size(entries);
        fetchReport.totalFetches += $_.size(translationErrors) + $_.size(entries);

        if ($_.isEmpty(entries)) {
          return $q.when();
        }

        return dbMongooseBinders.create(entryModel, entries);              
      });

      return $q.all(promiseOfEntries);
    }

    function onFinish(error) {
      if (error) {
        return finishedDefer.reject(error);
      }

      fetchReport.finished = new Date();

      dbMongooseBinders.create(modelsFetchReport, fetchReport)
        .then(newFetchReport => finishedDefer.resolve(newFetchReport))
        .catch(reason => finishedDefer.reject(reason))
    }      
  }

  function createFetchReport() {
    return {
      success: false,
      totalFetches: 0,
      succeededFetches: 0,
      started: new Date(),
      finished: undefined,
      log: '',
      failedFetches: []
    };
  }

});<commit_msg>save unique entries on db<commit_after>/* jshint esversion:6, node:true  */

'use strict';

eamModule(module, 'rssRegistrationsFetcher', (
  $_,
  $q,
  logger,
  modelsEntry,
  dbMongooseBinders,
  modelsRssRegistration,
  modelsFetchReport,
  rssRegistrationsFetcherIterationFetch
) => {

  return {
    fetch
  };

  function fetch() {
    const finishedDefer = $q.defer();
    
    const fetchReport = createFetchReport();
    
    fetchRegistrations()
      .then(startRssRegistrationFetch);

    return finishedDefer.promise;

    function fetchRegistrations() {
      return dbMongooseBinders.find(modelsRssRegistration)
        .catch(reason => finishedDefer.reject(reason))
    }

    function startRssRegistrationFetch(registrationsResp) {
      rssRegistrationsFetcherIterationFetch.fetch(
        registrationsResp, 
        onIterationFetchFinished, 
        onFinish
      );
    }

    function onIterationFetchFinished(registrationEntries) {
      const promiseOfEntries = $_.map(registrationEntries, registrationEntry => {
        const rssRegistration = registrationEntry.rssRegistration;
        logger.assert(rssRegistration, 'rssRegistration always exist');

        const majorError = registrationEntry.error;
        const translationErrors = registrationEntry.entriesResp 
                                    && registrationEntry.entriesResp.errors;
        const error = majorError || translationErrors;

        if (!$_.isEmpty(error)) {
          const failedFetch = {
            error,
            rssRegistration: rssRegistration.id
          };
          fetchReport.failedFetches.push(failedFetch);
        }

        if (!$_.isEmpty(majorError)) {
          return $q.when();
        }

        const entryModel = modelsEntry.getByCategoryLang(
          rssRegistration.lang,
          rssRegistration.category.name
        );        

        const entries = registrationEntry.entriesResp.entries;
        fetchReport.succeededFetches += $_.size(entries);
        fetchReport.totalFetches += $_.size(translationErrors) + $_.size(entries);

        if ($_.isEmpty(entries)) {
          return $q.when();
        }

        return entryModel.saveAvoidingDuplications(entries)
          .then(savedEntries => fetchReport.entriesStored += $_.size(savedEntries))           
      });

      return $q.all(promiseOfEntries);
    }

    function onFinish(error) {
      if (error) {
        return finishedDefer.reject(error);
      }

      fetchReport.finished = new Date();

      dbMongooseBinders.create(modelsFetchReport, fetchReport)
        .then(newFetchReport => finishedDefer.resolve(newFetchReport))
        .catch(reason => finishedDefer.reject(reason))
    }      
  }

  function createFetchReport() {
    return {
      success: false,
      totalFetches: 0,
      succeededFetches: 0,
      entriesStored: 0,
      started: new Date(),
      finished: undefined,
      log: '',
      failedFetches: []
    };
  }

});
{'fraction_non_alphanumeric': 0.09681881051175657, 'fraction_numerical': 0.0012102351313969572, 'mean_word_length': 2.5039370078740157, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5617621', 'n_tokens_mistral': 1632, 'n_tokens_neox': 1562, 'n_words': 268}