<commit_before>'use strict';

const request = require('request').defaults({
  jar: true,
});
const moment = require('moment');
const cheerio = require('cheerio');
const baseKonnector = require('../lib/base_konnector');

const filterExisting = require('../lib/filter_existing');
const localization = require('../lib/localization_manager');
const saveDataAndFile = require('../lib/save_data_and_file');
const linkBankOperation = require('../lib/link_bank_operation');

const log = require('printit')({
  prefix: 'Sfr mobile',
  date: true,
});

const fileOptions = {
  vendor: 'SFR',
  dateFormat: 'YYYYMMDD',
};

const Bill = require('../models/bill');

// Konnector
const connector = module.exports = baseKonnector.createNew({
  name: 'SFR Mobile',
  vendorLink: 'espace-client.sfr.fr/facture-mobile/consultation',
  fields: {
    login: 'text',
    password: 'password',
    folderPath: 'folder',
  },
  models: [Bill],
  fetchOperations: [
    getToken,
    logIn,
    fetchBillingInfo,
    parsePage,
    customFilterExisting,
    customSaveDataAndFile,
    linkBankOperation({
      log,
      model: Bill,
      identifier: 'SFR MOBILE',
      minDateDelta: 4,
      maxDateDelta: 20,
      amountDelta: 0.1,
    }),
    buildNotifContent,
  ],
});

// Procedure to get the login token
function getToken(requiredFields, bills, data, next) {
  const url = 'https://www.sfr.fr/bounce?target=//www.sfr.fr/sfr-et-moi/bounce.html&casforcetheme=mire-sfr-et-moi&mire_layer';
  const options = {
    url,
    method: 'GET',
  };

  connector.logger.info('Getting the token on Sfr Website...');

  request(options, (err, res, body) => {
    if (err) {
      connector.logger.info(err);
      return next('token not found');
    }

    const $ = cheerio.load(body);
    data.token = $('input[name=lt]').val();

    connector.logger.info('Token retrieved');
    return next();
  });
}

// Procedure to login to Sfr website.
function logIn(requiredFields, bills, data, next) {
  const options = {
    method: 'POST',
    url: 'https://www.sfr.fr/cas/login?domain=mire-sfr-et-moi&service=https://www.sfr.fr/accueil/j_spring_cas_security_check#sfrclicid=EC_mire_Me-Connecter',
    form: {
      lt: data.token,
      execution: 'e1s1',
      _eventId: 'submit',
      username: requiredFields.login,
      password: <PASSWORD>,
      identifier: '',
    },
  };

  connector.logger.info('Logging in on Sfr website...');

  request(options, (err) => {
    if (err) {
      connector.logger.info(err);
      return next('bad credentials');
    }

    connector.logger.info('Successfully logged in.');
    return next();
  });
}

function fetchBillingInfo(requiredFields, bills, data, next) {
  const url = 'https://espace-client.sfr.fr/facture-mobile/consultation';

  connector.logger.info('Fetch bill info');
  const options = {
    method: 'GET',
    url,
  };
  request(options, (err, res, body) => {
    if (err) {
      log.error('An error occured while fetching bills');
      log.raw(err);
      return next('request error');
    }
    connector.logger.info('Fetch bill info succeeded');

    data.html = body;
    return next();
  });
}

function parsePage(requiredFields, bills, data, next) {
  bills.fetched = [];
  moment.locale('fr');
  const $ = cheerio.load(data.html);
  const baseURL = 'https://espace-client.sfr.fr';

  const firstBill = $('#facture');
  const firstBillUrl = $('#lien-telecharger-pdf').attr('href');
  // The year is not provided, but we assume this is the current year or that
  // it will be provided if different from the current year
  let firstBillDate = firstBill.find('tr.header h3').text().substr(17);
  firstBillDate = moment(firstBillDate, 'D MMM YYYY');

  const price = firstBill.find('tr.total td.prix').text()
                                                .replace('€', '')
                                                .replace(',', '.');

  const bill = {
    date: firstBillDate,
    type: 'Mobile',
    amount: parseFloat(price),
    pdfurl: `${baseURL}${firstBillUrl}`,
    vendor: 'Sfr'
  };

  bills.fetched.push(bill);

  $('#tab tr').each(function each() {
    let date = $(this).find('.date').text();
    let prix = $(this).find('.prix').text()
                                    .replace('€', '')
                                    .replace(',', '.');
    let pdf = $(this).find('.liens a').attr('href');
    date = date.split(' ');
    date.pop();
    date = date.join(' ');
    date = moment(date, 'D MMM YYYY');
    prix = parseFloat(prix);
    pdf = `${baseURL}${pdf}`;

    const bill = {
      date,
      type: 'Mobile',
      amount: prix,
      pdfurl: pdf,
      vendor: 'Sfr',
    };
    bills.fetched.push(bill);
  });

  connector.logger.info('Successfully parsed the page');
  next();
}

function customFilterExisting(requiredFields, bills, data, next) {
  filterExisting(log, Bill)(requiredFields, bills, data, next);
}

function customSaveDataAndFile(requiredFields, bills, data, next) {
  const fnsave = saveDataAndFile(log, Bill, fileOptions, ['bill']);
  fnsave(requiredFields, bills, data, next);
}

function buildNotifContent(requiredFields, bills, data, next) {
  if (bills.filtered.length > 0) {
    const localizationKey = 'notification bills';
    const options = {
      smart_count: bills.filtered.length,
    };
    bills.notifContent = localization.t(localizationKey, options);
  }

  next();
}<commit_msg>Fix SFR mobile undefined urls (#497) (#507)<commit_after>'use strict';

const request = require('request').defaults({
  jar: true,
});
const moment = require('moment');
const cheerio = require('cheerio');
const baseKonnector = require('../lib/base_konnector');

const filterExisting = require('../lib/filter_existing');
const localization = require('../lib/localization_manager');
const saveDataAndFile = require('../lib/save_data_and_file');
const linkBankOperation = require('../lib/link_bank_operation');

const log = require('printit')({
  prefix: 'Sfr mobile',
  date: true,
});

const fileOptions = {
  vendor: 'SFR',
  dateFormat: 'YYYYMMDD',
};

const Bill = require('../models/bill');

// Konnector
const connector = module.exports = baseKonnector.createNew({
  name: 'SFR Mobile',
  vendorLink: 'espace-client.sfr.fr/facture-mobile/consultation',
  fields: {
    login: 'text',
    password: 'password',
    folderPath: 'folder',
  },
  models: [Bill],
  fetchOperations: [
    getToken,
    logIn,
    fetchBillingInfo,
    parsePage,
    customFilterExisting,
    customSaveDataAndFile,
    linkBankOperation({
      log,
      model: Bill,
      identifier: 'SFR MOBILE',
      minDateDelta: 4,
      maxDateDelta: 20,
      amountDelta: 0.1,
    }),
    buildNotifContent,
  ],
});

// Procedure to get the login token
function getToken(requiredFields, bills, data, next) {
  const url = 'https://www.sfr.fr/bounce?target=//www.sfr.fr/sfr-et-moi/bounce.html&casforcetheme=mire-sfr-et-moi&mire_layer';
  const options = {
    url,
    method: 'GET',
  };

  connector.logger.info('Getting the token on Sfr Website...');

  request(options, (err, res, body) => {
    if (err) {
      connector.logger.info(err);
      return next('token not found');
    }

    const $ = cheerio.load(body);
    data.token = $('input[name=lt]').val();

    connector.logger.info('Token retrieved');
    return next();
  });
}

// Procedure to login to Sfr website.
function logIn(requiredFields, bills, data, next) {
  const options = {
    method: 'POST',
    url: 'https://www.sfr.fr/cas/login?domain=mire-sfr-et-moi&service=https://www.sfr.fr/accueil/j_spring_cas_security_check#sfrclicid=EC_mire_Me-Connecter',
    form: {
      lt: data.token,
      execution: 'e1s1',
      _eventId: 'submit',
      username: requiredFields.login,
      password: <PASSWORD>,
      identifier: '',
    },
  };

  connector.logger.info('Logging in on Sfr website...');

  request(options, (err) => {
    if (err) {
      connector.logger.info(err);
      return next('bad credentials');
    }

    connector.logger.info('Successfully logged in.');
    return next();
  });
}

function fetchBillingInfo(requiredFields, bills, data, next) {
  const url = 'https://espace-client.sfr.fr/facture-mobile/consultation';

  connector.logger.info('Fetch bill info');
  const options = {
    method: 'GET',
    url,
  };
  request(options, (err, res, body) => {
    if (err) {
      log.error('An error occured while fetching bills');
      log.raw(err);
      return next('request error');
    }
    connector.logger.info('Fetch bill info succeeded');

    data.html = body;
    return next();
  });
}

function parsePage(requiredFields, bills, data, next) {
  bills.fetched = [];
  moment.locale('fr');
  const $ = cheerio.load(data.html);
  const baseURL = 'https://espace-client.sfr.fr';

  const firstBill = $('#facture');
  const firstBillUrl = $('#lien-telecharger-pdf').attr('href');

  if (firstBillUrl) {
    // The year is not provided, but we assume this is the current year or that
    // it will be provided if different from the current year
    let firstBillDate = firstBill.find('tr.header h3').text().substr(17);
    firstBillDate = moment(firstBillDate, 'D MMM YYYY');

    const price = firstBill.find('tr.total td.prix').text()
                                                    .replace('€', '')
                                                    .replace(',', '.');

    const bill = {
      date: firstBillDate,
      type: 'Mobile',
      amount: parseFloat(price),
      pdfurl: `${baseURL}${firstBillUrl}`,
      vendor: 'Sfr'
    };

    bills.fetched.push(bill);
  } else {
    connector.logger.info('wrong url for first PDF bill.');
  }

  $('#tab tr').each(function each() {
    let date = $(this).find('.date').text();
    let prix = $(this).find('.prix').text()
                                    .replace('€', '')
                                    .replace(',', '.');
    let pdf = $(this).find('.liens a').attr('href');

    if (pdf) {
      date = date.split(' ');
      date.pop();
      date = date.join(' ');
      date = moment(date, 'D MMM YYYY');
      prix = parseFloat(prix);
      pdf = `${baseURL}${pdf}`;

      const bill = {
        date,
        type: 'Mobile',
        amount: prix,
        pdfurl: pdf,
        vendor: 'Sfr',
      };
      bills.fetched.push(bill);
    } else {
      connector.logger.info('wrong url for PDF bill.');
    }
  });

  connector.logger.info('Successfully parsed the page');
  next();
}

function customFilterExisting(requiredFields, bills, data, next) {
  filterExisting(log, Bill)(requiredFields, bills, data, next);
}

function customSaveDataAndFile(requiredFields, bills, data, next) {
  const fnsave = saveDataAndFile(log, Bill, fileOptions, ['bill']);
  fnsave(requiredFields, bills, data, next);
}

function buildNotifContent(requiredFields, bills, data, next) {
  if (bills.filtered.length > 0) {
    const localizationKey = 'notification bills';
    const options = {
      smart_count: bills.filtered.length,
    };
    bills.notifContent = localization.t(localizationKey, options);
  }

  next();
}
{'fraction_non_alphanumeric': 0.1609257752463611, 'fraction_numerical': 0.0025314166892686013, 'mean_word_length': 2.9394586894586894, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 10, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7197832', 'n_tokens_mistral': 3628, 'n_tokens_neox': 3532, 'n_words': 779}