<commit_before>https = require 'https'
pocketOAuth = require('./pocketOAuth.js')
db = require('../database.js')
logger = require('../logger.js').logger

HOST = "getpocket.com"
GET_PATH = "/v3/get"
MAX_ARTICLES_PER_CALL = 5000


getItemsSinceWithOffset = (res, offset, since) ->
    options = {
        host : HOST,
        path : GET_PATH,
        method : 'POST',
        headers : {
                'Content-Type' : 'application/json; charset=UTF-8',
                'X-Accept' : 'application/json'
        }
    }
    authReq = https.request options, (response) ->
        str = ''
        response.on 'data', (chunk) ->
            str += chunk
            return
        response.on 'end', () ->
            json = JSON.parse str
            if json.list? and (Object.prototype.toString.call json.list) is "[object Object]"
                for id, obj of json.list
                    db.upsertArticle obj
                getItemsSinceWithOffset res, offset + MAX_ARTICLES_PER_CALL, since
            else
                newList = []
                for id, obj of json.list
                    newList.push obj
                json.list = newList.sort (a,b) -> return a.time_added - b.time_added
                res.charset = 'utf-8'
                res.json json
                db.updateLastSinceTimestamp json.since
            return
        return
    params = {
        "state" : "all"
        "consumer_key" : pocketOAuth.consumerKey,
        "access_token" : pocketOAuth.accessToken,
        "sort" : "oldest",
        "count" : MAX_ARTICLES_PER_CALL,
        "offset" : offset
    }
    if since?
        params.since = since
    authReq.write JSON.stringify params
    authReq.end()
    return    

exports.getItemsSince = (req, res) ->
    db.getLastSinceTimestamp (since) ->
        getItemsSinceWithOffset res, 0, since
        return
    return<commit_msg>remove sort param from api call<commit_after>https = require 'https'
pocketOAuth = require('./pocketOAuth.js')
db = require('../database.js')
logger = require('../logger.js').logger

HOST = "getpocket.com"
GET_PATH = "/v3/get"
MAX_ARTICLES_PER_CALL = 5000


getItemsSinceWithOffset = (res, offset, since) ->
    options = {
        host : HOST,
        path : GET_PATH,
        method : 'POST',
        headers : {
                'Content-Type' : 'application/json; charset=UTF-8',
                'X-Accept' : 'application/json'
        }
    }
    authReq = https.request options, (response) ->
        str = ''
        response.on 'data', (chunk) ->
            str += chunk
            return
        response.on 'end', () ->
            json = JSON.parse str
            if json.list? and (Object.prototype.toString.call json.list) is "[object Object]"
                for id, obj of json.list
                    db.upsertArticle obj
                getItemsSinceWithOffset res, offset + MAX_ARTICLES_PER_CALL, since
            else
                newList = []
                for id, obj of json.list
                    newList.push obj
                json.list = newList.sort (a,b) -> return a.time_added - b.time_added
                res.charset = 'utf-8'
                res.json json
                db.updateLastSinceTimestamp json.since
            return
        return
    params = {
        "state" : "all"
        "consumer_key" : pocketOAuth.consumerKey,
        "access_token" : pocketOAuth.accessToken,
        "count" : MAX_ARTICLES_PER_CALL,
        "offset" : offset
    }
    if since?
        params.since = since
    authReq.write JSON.stringify params
    authReq.end()
    return    

exports.getItemsSince = (req, res) ->
    db.getLastSinceTimestamp (since) ->
        getItemsSinceWithOffset res, 0, since
        return
    return
{'fraction_non_alphanumeric': 0.10947930574098798, 'fraction_numerical': 0.004272363150867824, 'mean_word_length': 1.6930265995686555, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2847809', 'n_tokens_mistral': 1118, 'n_tokens_neox': 1063, 'n_words': 250}