<commit_before>cozydb = require 'cozydb'
request = require 'request'
moment = require 'moment'
cheerio = require 'cheerio'
fs = require 'fs'

File = require '../models/file'
fetcher = require '../lib/fetcher'
filterExisting = require '../lib/filter_existing'
saveDataAndFile = require '../lib/save_data_and_file'
linkBankOperation = require '../lib/link_bank_operation'

localization = require '../lib/localization_manager'

log = require('printit')
    prefix: "Free"
    date: true


# Models

InternetBill = cozydb.getModel 'InternetBill',
    date: Date
    vendor: String
    amount: Number
    fileId: String

InternetBill.all = (callback) ->
    InternetBill.request 'byDate', callback

# Konnector

module.exports =

    name: "Free"
    slug: "free"
    description: 'konnector description free'
    vendorLink: "https://www.free.fr/"

    fields:
        login: "text"
        password: "password"
        folderPath: "folder"
    models:
        internetbill: InternetBill

    # Define model requests.
    init: (callback) ->
        map = (doc) -> emit doc.date, doc
        InternetBill.defineRequest 'byDate', map, (err) ->
            callback err

    fetch: (requiredFields, callback) ->

        log.info "Import started"

        fetcher.new()
            .use(logIn)
            .use(parsePage)
            .use(filterExisting log, InternetBill)
            .use(saveDataAndFile log, InternetBill, 'free', ['facture'])
            .use(linkBankOperation
                log: log
                model: InternetBill
                identifier: ['free telecom', ' free hautdebit']
                dateDelta: 10
                amountDelta: 0.1
            )
            .args(requiredFields, {}, {})
            .fetch (err, fields, entries) ->
                log.info "Import finished"

                notifContent = null
                if entries?.filtered?.length > 0
                    localizationKey = 'notification free'
                    options = smart_count: entries.filtered.length
                    notifContent = localization.t localizationKey, options

                callback err, notifContent

# Procedure to login to Free website.
logIn = (requiredFields, billInfos, data, next) ->

    loginUrl = "https://subscribe.free.fr/login/login.pl"
    billUrl = "https://adsl.free.fr/liste-factures.pl"

    form =
        "pass": <PASSWORD>
        "login": requiredFields.login

    options =
        method: 'POST'
        form: form
        jar: true
        url: loginUrl

    request options, (err, res, body) ->

        isNoLocation = not res.headers.location?
        isNot302 = res.statusCode isnt 302
        isError = res.headers.location? and \
            res.headers.location.indexOf("error") isnt -1

        if err or isNoLocation or isNot302 or isError
            log.error "Authentification error"
            next 'bad credentials'

        else
            location = res.headers.location
            parameters = location.split('?')[1]
            url = "#{billUrl}?#{parameters}"
            request.get url, (err, res, body) ->
                if err then next err
                else
                    data.html = body
                    next()


# Parse the fetched page to extract bill data.
parsePage = (requiredFields, bills, data, next) ->

    bills.fetched = []

    return next() if not data.html?

    $ = cheerio.load data.html
    $('.pane li').each ->
        amount = $($(this).find('strong').get(1)).html()
        amount = amount.replace ' Euros', ''
        amount = parseFloat amount

        pdfUrl = $(this).find('.last a').attr 'href'
        pdfUrl = "https://adsl.free.fr/#{pdfUrl}"

        month = pdfUrl.split('&')[2].split('=')[1]
        date = moment month, 'YYYYMM'

        bill =
            amount: amount
            date: date
            vendor: 'Free'<commit_msg>Fix free connector<commit_after>cozydb = require 'cozydb'
request = require 'request'
moment = require 'moment'
cheerio = require 'cheerio'
fs = require 'fs'

File = require '../models/file'
fetcher = require '../lib/fetcher'
filterExisting = require '../lib/filter_existing'
saveDataAndFile = require '../lib/save_data_and_file'
linkBankOperation = require '../lib/link_bank_operation'

localization = require '../lib/localization_manager'

log = require('printit')
    prefix: "Free"
    date: true


# Models

InternetBill = cozydb.getModel 'InternetBill',
    date: Date
    vendor: String
    amount: Number
    fileId: String

InternetBill.all = (callback) ->
    InternetBill.request 'byDate', callback

# Konnector

module.exports =

    name: "Free"
    slug: "free"
    description: 'konnector description free'
    vendorLink: "https://www.free.fr/"

    fields:
        login: "text"
        password: "password"
        folderPath: "folder"
    models:
        internetbill: InternetBill

    # Define model requests.
    init: (callback) ->
        map = (doc) -> emit doc.date, doc
        InternetBill.defineRequest 'byDate', map, (err) ->
            callback err

    fetch: (requiredFields, callback) ->

        log.info "Import started"

        fetcher.new()
            .use(logIn)
            .use(parsePage)
            .use(filterExisting log, InternetBill)
            .use(saveDataAndFile log, InternetBill, 'free', ['facture'])
            .use(linkBankOperation
                log: log
                model: InternetBill
                identifier: ['free telecom', ' free hautdebit']
                dateDelta: 10
                amountDelta: 0.1
            )
            .args(requiredFields, {}, {})
            .fetch (err, fields, entries) ->
                log.info "Import finished"

                notifContent = null
                if entries?.filtered?.length > 0
                    localizationKey = 'notification free'
                    options = smart_count: entries.filtered.length
                    notifContent = localization.t localizationKey, options

                callback err, notifContent

# Procedure to login to Free website.
logIn = (requiredFields, billInfos, data, next) ->

    loginUrl = "https://subscribe.free.fr/login/login.pl"
    billUrl = "https://adsl.free.fr/liste-factures.pl"

    form =
        "pass": <PASSWORD>
        "login": requiredFields.login

    options =
        method: 'POST'
        form: form
        jar: true
        url: loginUrl

    request options, (err, res, body) ->

        isNoLocation = not res.headers.location?
        isNot302 = res.statusCode isnt 302
        isError = res.headers.location? and \
            res.headers.location.indexOf("error") isnt -1

        if err or isNoLocation or isNot302 or isError
            log.error "Authentification error"
            next 'bad credentials'

        else
            location = res.headers.location
            parameters = location.split('?')[1]
            url = "#{billUrl}?#{parameters}"
            request.get url, (err, res, body) ->
                if err then next err
                else
                    data.html = body
                    next()


# Parse the fetched page to extract bill data.
parsePage = (requiredFields, bills, data, next) ->

    bills.fetched = []

    return next() if not data.html?

    $ = cheerio.load data.html
    $('.pane li').each ->
        amount = $($(this).find('strong').get(1)).html()
        amount = amount.replace(' Euros', '').replace('&euro;', '')
        amount = amount.replace(',', '.').trim()
        amount = parseFloat amount

        pdfUrl = $(this).find('.last a').attr 'href'
        pdfUrl = "https://adsl.free.fr/#{pdfUrl}"

        month = pdfUrl.split('&')[2].split('=')[1]
        date = moment month, 'YYYYMM'

        bill =
            amount: amount
            date: date
            vendor: 'Free'
{'fraction_non_alphanumeric': 0.11957495839201127, 'fraction_numerical': 0.004864934067340929, 'mean_word_length': 1.9379465964648364, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 29, 'https://': 8, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4713264', 'n_tokens_mistral': 2278, 'n_tokens_neox': 2158, 'n_words': 571}