<commit_before>'use strict';

const baseKonnector = require('../lib/base_konnector');
const filterExisting = require('../lib/filter_existing');
const localization = require('../lib/localization_manager');
const saveDataAndFile = require('../lib/save_data_and_file');
const linkBankOperation = require('../lib/link_bank_operation');
const requestJson = require('request-json');
const request = require('request');
const cheerio = require('cheerio');
const moment = require('moment');

const Bill = require('../models/bill');
/* The goal of this connector is to fetch bills from the
service captaintrain.com */

const logger = require('printit')({
  prefix: 'Captaintrain',
  date: true,
});


module.exports = baseKonnector.createNew({
  name: 'Captain Train',

  fields: {
    login: 'email',
    password: 'password',
    folderPath: 'folder',
  },

  models: [Bill],

  fetchOperations: [
    login,
    fetchBills,
    customFilterExisting,
    customSaveDataAndFile,
    linkBankOperation({
      log: logger,
      minDateDelta: 1,
      maxDateDelta: 1,
      model: Bill,
      amountDelta: 0.1,
      identifier: ['CAPITAINE TRAIN', 'CAPTAIN TRAIN', 'OUIGO'],
    }),
    buildNotifContent,
  ],

});

const fileOptions = {
  vendor: 'Captaintrain',
  dateFormat: 'YYYYMMDD',
};

const baseUrl = 'https://www.captaintrain.com/';
const client = requestJson.createClient(baseUrl);
const userAgent = 'Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:37.0) ' +
                  'Gecko/20100101 Firefox/37.0';
// Authenticity token used by the jequest-json client.
let csrfToken;
client.headers['user-agent'] = userAgent;
function login(requiredFields, entries, data, next) {
  const options = {
    method: 'GET',
    url: `${baseUrl}signin`,
    headers: {
      'User-Agent': userAgent,
      Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    },
    jar: true,
  };
  request(options, (err, res, body) => {
    if (err) {
      logger.error(err);
      next(err);
    }
    // Retrieve the authenticity token
    const $ = cheerio.load(body);
    csrfToken = $('meta[name=csrf-token]').attr('content');
    // Retrieve the cookie for the request-json clien
    let cookie = res.headers['set-cookie'][0];
    // Signin form
    const signinForm = {
      concur_auth_code: null,
      concur_migration_type: null,
      concur_new_email: null,
      correlation_key: null,
      email: requiredFields.login,
      facebook_id: null,
      facebook_token: null,
      google_code: null,
      google_id: null,
      password: <PASSWORD>,
      source: null,
      user_itokend: null,
    };
    client.headers.cookie = cookie;
    // Signin
    const signinPath = `${baseUrl}api/v5/account/signin`;
    client.post(signinPath, signinForm, (err, res, body) => {
      if (err) {
        logger.error(err);
        next(err);
      }

      if (res.statusCode === 422) {
        next('bad credentials');
      }

      // Retrieve token for json client
      const token = body.meta.token;

     // Login data for the json client
      const loginData = {
        email: requiredFields.login,
        password: <PASSWORD>,
        authenticity_token: csrfToken,
      };
      options.method = 'POST';
      options.url = `${baseUrl}login`;
      options.form = loginData;
      // Log in
      request(options, (err, res) => {
        // Update cookie
        cookie = res.headers['set-cookie'][0];
        if (err) {
          next(err);
        }

        client.headers.cookie = cookie;
        client.headers.Authorization = `Token token="${token}"`;
        data.authHeader = `Token token="${token}"`;
        // the api/v5/pnrs uri gives all information necessary to get bill
        // information
        client.get(`${baseUrl}api/v5/pnrs`, (err, res, body) => {
          if (err) {
            logger.error(err);
            next(err);
          }
          // We check their are bills
          if (body.proofs && body.proofs.length > 0) {
            saveMetadata(data, body);
            getNextMetaData(computeNextDate(body.pnrs), data, next);
          } else {
            next();
          }
        });<commit_msg>Fix captaintrain konnector. Fixes #360<commit_after>'use strict';

const baseKonnector = require('../lib/base_konnector');
const filterExisting = require('../lib/filter_existing');
const localization = require('../lib/localization_manager');
const saveDataAndFile = require('../lib/save_data_and_file');
const linkBankOperation = require('../lib/link_bank_operation');
const requestJson = require('request-json');
const request = require('request');
const moment = require('moment');

const Bill = require('../models/bill');
/* The goal of this connector is to fetch bills from the
service captaintrain.com */

const logger = require('printit')({
  prefix: 'Captaintrain',
  date: true,
});


module.exports = baseKonnector.createNew({
  name: 'Captain Train',

  fields: {
    login: 'email',
    password: 'password',
    folderPath: 'folder',
  },

  models: [Bill],

  fetchOperations: [
    login,
    fetchBills,
    customFilterExisting,
    customSaveDataAndFile,
    linkBankOperation({
      log: logger,
      minDateDelta: 1,
      maxDateDelta: 1,
      model: Bill,
      amountDelta: 0.1,
      identifier: ['CAPITAINE TRAIN', 'CAPTAIN TRAIN', 'OUIGO'],
    }),
    buildNotifContent,
  ],

});

const fileOptions = {
  vendor: 'Captaintrain',
  dateFormat: 'YYYYMMDD',
};

const baseUrl = 'https://www.captaintrain.com/';
const client = requestJson.createClient(baseUrl);
const userAgent = 'Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:37.0) ' +
                  'Gecko/20100101 Firefox/37.0';

client.headers['user-agent'] = userAgent;
function login(requiredFields, entries, data, next) {
  const options = {
    method: 'GET',
    url: `${baseUrl}signin`,
    headers: {
      'User-Agent': userAgent,
      Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    },
    jar: true,
  };
  request(options, (err) => {
    if (err) {
      logger.error(err);
      next(err);
    }

    // Signin form
    const signinForm = {
      concur_auth_code: null,
      concur_migration_type: null,
      concur_new_email: null,
      correlation_key: null,
      email: requiredFields.login,
      facebook_id: null,
      facebook_token: null,
      google_code: null,
      google_id: null,
      password: <PASSWORD>,
      source: null,
      user_itokend: null,
    };
    // Signin
    const signinPath = `${baseUrl}api/v5/account/signin`;
    client.post(signinPath, signinForm, (err, res, body) => {
      if (err) {
        logger.error(err);
        next(err);
      }

      if (res.statusCode === 422) {
        next('bad credentials');
      }
      // Retrieve token for json client
      const token = body.meta.token;
    //  client.headers.cookie = cookie;
      client.headers.Authorization = `Token token="${token}"`;
      data.authHeader = `Token token="${token}"`;
      // the api/v5/pnrs uri gives all information necessary to get bill
      // information
      client.get(`${baseUrl}api/v5/pnrs`, (err, res, body) => {
        if (err) {
          logger.error(err);
          next(err);
        }
        // We check their are bills
        if (body.proofs && body.proofs.length > 0) {
          saveMetadata(data, body);
          getNextMetaData(computeNextDate(body.pnrs), data, next);
        } else {
          next();
        }
{'fraction_non_alphanumeric': 0.14371980676328502, 'fraction_numerical': 0.010601180891035964, 'mean_word_length': 2.6606090373280944, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7640301', 'n_tokens_mistral': 2413, 'n_tokens_neox': 2348, 'n_words': 532}