<commit_before>##############################################################################
#
#    CoCalc: Collaborative Calculation in the Cloud
#
#    Copyright (C) 2016 -- 2017, Sagemath Inc.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################

###
PostgreSQL -- basic queries and database interface
###

exports.DEBUG = true

# If database conection is non-responsive but no error raised directly
# by db client, then we will know and fix, rather than just sitting there...
DEFAULT_TIMEOUS_MS = 60000

# Do not test for non-responsiveness until a while after initial connection
# established, since things tend to work initially, *but* may also be much
# slower, due to tons of clients simultaneously connecting to DB.
DEFAULT_TIMEOUT_DELAY_MS = DEFAULT_TIMEOUS_MS * 4

QUERY_ALERT_THRESH_MS=5000

EventEmitter = require('events')

fs      = require('fs')
async   = require('async')

pg      = require('pg').native    # You might have to do: "apt-get install libpq5 libpq-dev"
if not pg?
    throw Error("YOU MUST INSTALL the pg-native npm module")
# You can uncommment this to use the pure javascript driver.
#  However: (1) it can be 5x slower or more!
#           (2) I think it corrupts something somehow in a subtle way, since our whole
#               syncstring system was breaking... until I switched to native.  Not sure.
#pg      = require('pg')

winston      = require('./winston-metrics').get_logger('postgres')

misc_node = require('smc-util-node/misc_node')

{defaults} = misc = require('smc-util/misc')
required = defaults.required

{SCHEMA, client_db} = require('smc-util/schema')

exports.PUBLIC_PROJECT_COLUMNS = ['project_id',  'last_edited', 'title', 'description', 'deleted',  'created']
exports.PROJECT_COLUMNS = ['users'].concat(exports.PUBLIC_PROJECT_COLUMNS)

read_password_from_disk = ->
    filename = (process.env.SMC_ROOT ? '.') + '/data/secrets/postgres'
    try
        #winston.debug("Loading password from '#{filename}'")
        return fs.readFileSync(filename).toString().trim()
    catch
        #winston.debug("NO PASSWORD FILE!")
        # no password file
        return

class exports.PostgreSQL extends EventEmitter    # emits a 'connect' event whenever we successfully connect to the database and 'disconnect' when connection to postgres fails
    constructor: (opts) ->
        super()
        opts = defaults opts,
            host            : process.env['PGHOST'] ? 'localhost'    # or 'hostname:port' or 'host1,host2,...' (multiple hosts) -- TODO -- :port only works for one host.
            database        : process.env['SMC_DB'] ? 'smc'
            user            : process.env['PGUSER'] ? 'smc'
            debug           : exports.DEBUG
            connect         : true
            password        : <PASSWORD>
            pool            : undefined   # IGNORED for now.
            cache_expiry    : 3000  # expire cached queries after this many milliseconds
                                    # keep this very short; it's just meant to reduce impact of a bunch of
                                    # identical permission checks in a single user query.
            cache_size      : 100   # cache this many queries; use @_query(cache:true, ...) to cache result
            concurrent_warn : 500
            ensure_exists   : true  # ensure database exists on startup (runs psql in a shell)
            timeout_ms      : DEFAULT_TIMEOUS_MS # **IMPORTANT: if *any* query takes this long, entire connection is terminated and recreated!**
            timeout_delay_ms : DEFAULT_TIMEOUT_DELAY_MS # Only reconnect on timeout this many ms after connect.  Motivation: on initial startup queries may take much longer due to competition with other clients.
        @setMaxListeners(10000)  # because of a potentially large number of changefeeds
        @_state = 'init'
        @_debug = opts.debug
        @_timeout_ms = opts.timeout_ms
        @_timeout_delay_ms = opts.timeout_delay_ms
        @_ensure_exists = opts.ensure_exists
        @_init_test_query()
        dbg = @_dbg("constructor")  # must be after setting @_debug above
        dbg(opts)
        i = opts.host.indexOf(':')
        if i != -1
            @_host = opts.host.slice(0, i)
            @_port = parseInt(opts.host.slice(i+1))
        else
            @_host = opts.host
            @_port = 5432
        @_concurrent_warn = opts.concurrent_warn
        @_user = opts.user
        @_database = opts.database
        @_password = opts.password ? read_password_from_disk()
        @_init_metrics()

        if opts.cache_expiry and opts.cache_size
            @_query_cache = (new require('expiring-lru-cache'))(size:opts.cache_size, expiry: opts.cache_expiry)
        if opts.connect
            @connect()  # start trying to connect

    clear_cache: =>
        @_query_cache?.reset()

    close: =>
        if @_state == 'closed'
            return  # nothing to do
        @_close_test_query()
        @_state = 'closed'
        @emit('close')
        @removeAllListeners()
        if @_clients?
            for client in @_clients
                client.removeAllListeners()
                client.end()
            delete @_clients

    ###
    If @_timeout_ms is set, then we periodically do a simple test query,
    to ensure that the database connection is working and responding to queries.
    If the query below times out, then the connection will get recreated.
    ###
    _do_test_query: =>
        dbg = @_dbg('test_query')
        dbg('starting')
        @_query
            query : 'SELECT NOW()'
            cb    : (err, result) =>
                dbg("finished", err, result)

    _init_test_query: =>
        if not @_timeout_ms
            return
        @_test_query = setInterval(@_do_test_query, @_timeout_ms)

    _close_test_query: =>
        if @_test_query?
            clearInterval(@_test_query)
            delete @_test_query

    engine: -> 'postgresql'

    connect: (opts) =>
        opts = defaults opts,
            max_time : undefined   # set to something shorter to not try forever
                                   # Only first max_time is used.
            cb       : undefined
        if @_state == 'closed'
            opts.cb?("closed")
            return
        dbg = @_dbg("connect")
        if @_clients?
            dbg("already connected")
            opts.cb?()
            return
        if @_connecting?
            dbg('already trying to connect')
            @_connecting.push(opts.cb)
            return
        dbg('will try to connect')
        @_state = 'init'
        if opts.max_time
            dbg("for up to #{opts.max_time}ms")
        else
            dbg("until successful")
        @_connecting = [opts.cb]
        misc.retry_until_success
            f           : @_connect
            max_delay   : 10000
            max_time    : opts.max_time
            start_delay : 500 + 500*Math.random()
            log         : dbg
            cb          : (err) =>
                v = @_connecting
                delete @_connecting
                for cb in v
                    cb?(err)
                if not err
                    @_state = 'connected'
                    @emit('connect')

    disconnect: () =>
        if @_clients?
            for client in @_clients
                client.end()
                client.removeAllListeners()
        delete @_clients

    _connect: (cb) =>
        dbg = @_dbg("_do_connect")
        dbg("connect to #{@_host}")
        @_clear_listening_state()   # definitely not listening
        if @_clients?
            @disconnect()
        locals =
            clients : []
            hosts   : []
        @_connect_time = 0
        @_concurrent_queries = 0  # can't be any going on now.
        async.series([
            (cb) =>
                if @_ensure_exists
                    dbg("first make sure db exists")
                    @_ensure_database_exists(cb)
                else
                    dbg("assuming database exists")
                    cb()
            (cb) =>
                if not @_host   # undefined if @_host=''
                    locals.hosts = [undefined]
                    cb()
                    return
                if @_host.indexOf('/') != -1
                    dbg("using a local socket file (not a hostname)")
                    locals.hosts = [@_host]
                    cb()
                    return
                f = (host, cb) =>
                    hostname = host.split(':')[0]
                    winston.debug("Looking up ip addresses of #{hostname}")
                    require('dns').lookup hostname, {all:true}, (err, ips) =>
                        if err
                            winston.debug("Got #{hostname} --> err=#{err}")
                            # NON-FATAL -- we just don't include these and hope to
                            # have at least one total working host...
                            cb()
                        else
                            winston.debug("Got #{hostname} --> #{JSON.stringify(ips)}")
                            # In kubernetes the stateful set service just has
                            # lots of ip address.  We connect to *all* of them,
                            # and spread queries across them equally.
                            for x in ips
                                locals.hosts.push(x.address)
                            cb()
                async.map(@_host.split(','), f, cb)
            (cb) =>
                dbg("connecting to #{JSON.stringify(locals.hosts)}...")
                if locals.hosts.length == 0
                    dbg("locals.hosts has length 0 -- no available db")
                    cb("no databases available")
                    return

                dbg("create client and start connecting...")
                locals.clients = []

                # Use a function to initialize the client, to avoid any
                # issues with scope of "client" below.
                init_client = (host) =>
                    client = new pg.Client
                        user     : 'smc'
                        host     : host
                        port     : @_port
                        password : <PASSWORD>
                        database : @_database
                    if @_notification?
                        client.on('notification', @_notification)
                    client.on 'error', (err) =>
                        if @_state == 'init'
                            # already started connecting
                            return
                        @emit('disconnect')
                        dbg("error -- #{err}")
                        @disconnect()
                        @connect()  # start trying to reconnect
                    client.setMaxListeners(1000)  # there is one emitter for each concurrent query... (see query_cb)
                    locals.clients.push(client)

                for host in locals.hosts
                    init_client(host)

                # Connect the clients.  If at least one succeeds, we use this.
                # If none succeed, we declare failure.
                # Obviously, this is NOT optimal -- it's just hopefully sufficiently robust/works.
                # I'm going to redo this with experience.
                locals.clients_that_worked = []
                locals.errors = []
                f = (c) =>
                    try
                        await c.connect()
                        locals.clients_that_worked.push(c)
                    catch err
                        locals.errors.push(err)

                await Promise.all((f(c) for c in locals.clients))

                if locals.clients_that_worked.length == 0
                    dbg("ALL clients failed", locals.errors)
                    cb("ALL clients failed to connect")
                else
                    # take what we got
                    if locals.clients.length == locals.clients_that_worked.length
                        dbg("ALL clients worked")
                    else
                        dbg("ONLY #{locals.clients_that_worked.length} clients worked")
                    locals.clients = locals.clients_that_worked
                    cb()

            (cb) =>
                # CRITICAL!  At scale, this query
                #    SELECT * FROM file_use WHERE project_id = any(select project_id from projects where users ? '25e2cae4-05c7-4c28-ae22-1e6d3d2e8bb3') ORDER BY last_edited DESC limit 100;
                # will take forever due to the query planner using a nestloop scan.  We thus
                # disable doing so!
                @_connect_time = new Date()
                locals.i = 0
                f = (client, cb) =>
                    dbg("now connected; disabling nestloop query planning for client #{locals.i}")
                    locals.i += 1
                    client.query "SET enable_nestloop TO off", (err) =>
                        if err
                            dbg("disabling nestloop failed for client #{locals.i}")
                        else
                            dbg("disabling nestloop done for client #{locals.i}")
                        cb(err)
                async.map(locals.clients, f, cb)
            (cb) =>
                dbg("checking if ANY db server is in recovery, i.e., we are doing standby queries only")
                @is_standby = false
                f = (client, cb) =>
                    # Is this a read/write or read-only connection?
                    client.query "SELECT pg_is_in_recovery()", (err, resp) =>
                        if err
                            cb(err)
                        else
                            # True if ANY db connection is read only.
                            if resp.rows[0].pg_is_in_recovery
                                @is_standby = true
                            cb()
                async.map(locals.clients, f, cb)
        ], (err) =>
            if err
                mesg = "Failed to connect to database -- #{err}"
                dbg(mesg)
                console.warn(mesg)  # make it clear for interactive users with debugging off -- common mistake with env not setup right.
                cb?(err)
            else<commit_msg>hub, etc. -- hopefully workaround bug that prevents reconnect when database dies<commit_after>##############################################################################
#
#    CoCalc: Collaborative Calculation in the Cloud
#
#    Copyright (C) 2016 -- 2017, Sagemath Inc.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################

###
PostgreSQL -- basic queries and database interface
###

exports.DEBUG = true

# If database conection is non-responsive but no error raised directly
# by db client, then we will know and fix, rather than just sitting there...
DEFAULT_TIMEOUS_MS = 60000

# Do not test for non-responsiveness until a while after initial connection
# established, since things tend to work initially, *but* may also be much
# slower, due to tons of clients simultaneously connecting to DB.
DEFAULT_TIMEOUT_DELAY_MS = DEFAULT_TIMEOUS_MS * 4

QUERY_ALERT_THRESH_MS=5000

EventEmitter = require('events')

fs      = require('fs')
async   = require('async')

pg      = require('pg').native    # You might have to do: "apt-get install libpq5 libpq-dev"
if not pg?
    throw Error("YOU MUST INSTALL the pg-native npm module")
# You can uncommment this to use the pure javascript driver.
#  However: (1) it can be 5x slower or more!
#           (2) I think it corrupts something somehow in a subtle way, since our whole
#               syncstring system was breaking... until I switched to native.  Not sure.
#pg      = require('pg')

winston      = require('./winston-metrics').get_logger('postgres')

misc_node = require('smc-util-node/misc_node')

{defaults} = misc = require('smc-util/misc')
required = defaults.required

{SCHEMA, client_db} = require('smc-util/schema')

exports.PUBLIC_PROJECT_COLUMNS = ['project_id',  'last_edited', 'title', 'description', 'deleted',  'created']
exports.PROJECT_COLUMNS = ['users'].concat(exports.PUBLIC_PROJECT_COLUMNS)

read_password_from_disk = ->
    filename = (process.env.SMC_ROOT ? '.') + '/data/secrets/postgres'
    try
        #winston.debug("Loading password from '#{filename}'")
        return fs.readFileSync(filename).toString().trim()
    catch
        #winston.debug("NO PASSWORD FILE!")
        # no password file
        return

class exports.PostgreSQL extends EventEmitter    # emits a 'connect' event whenever we successfully connect to the database and 'disconnect' when connection to postgres fails
    constructor: (opts) ->
        super()
        opts = defaults opts,
            host            : process.env['PGHOST'] ? 'localhost'    # or 'hostname:port' or 'host1,host2,...' (multiple hosts) -- TODO -- :port only works for one host.
            database        : process.env['SMC_DB'] ? 'smc'
            user            : process.env['PGUSER'] ? 'smc'
            debug           : exports.DEBUG
            connect         : true
            password        : <PASSWORD>
            pool            : undefined   # IGNORED for now.
            cache_expiry    : 3000  # expire cached queries after this many milliseconds
                                    # keep this very short; it's just meant to reduce impact of a bunch of
                                    # identical permission checks in a single user query.
            cache_size      : 100   # cache this many queries; use @_query(cache:true, ...) to cache result
            concurrent_warn : 500
            ensure_exists   : true  # ensure database exists on startup (runs psql in a shell)
            timeout_ms      : DEFAULT_TIMEOUS_MS # **IMPORTANT: if *any* query takes this long, entire connection is terminated and recreated!**
            timeout_delay_ms : DEFAULT_TIMEOUT_DELAY_MS # Only reconnect on timeout this many ms after connect.  Motivation: on initial startup queries may take much longer due to competition with other clients.
        @setMaxListeners(10000)  # because of a potentially large number of changefeeds
        @_state = 'init'
        @_debug = opts.debug
        @_timeout_ms = opts.timeout_ms
        @_timeout_delay_ms = opts.timeout_delay_ms
        @_ensure_exists = opts.ensure_exists
        @_init_test_query()
        dbg = @_dbg("constructor")  # must be after setting @_debug above
        dbg(opts)
        i = opts.host.indexOf(':')
        if i != -1
            @_host = opts.host.slice(0, i)
            @_port = parseInt(opts.host.slice(i+1))
        else
            @_host = opts.host
            @_port = 5432
        @_concurrent_warn = opts.concurrent_warn
        @_user = opts.user
        @_database = opts.database
        @_password = opts.password ? read_password_from_disk()
        @_init_metrics()

        if opts.cache_expiry and opts.cache_size
            @_query_cache = (new require('expiring-lru-cache'))(size:opts.cache_size, expiry: opts.cache_expiry)
        if opts.connect
            @connect()  # start trying to connect

    clear_cache: =>
        @_query_cache?.reset()

    close: =>
        if @_state == 'closed'
            return  # nothing to do
        @_close_test_query()
        @_state = 'closed'
        @emit('close')
        @removeAllListeners()
        if @_clients?
            for client in @_clients
                client.removeAllListeners()
                client.end()
            delete @_clients

    ###
    If @_timeout_ms is set, then we periodically do a simple test query,
    to ensure that the database connection is working and responding to queries.
    If the query below times out, then the connection will get recreated.
    ###
    _do_test_query: =>
        dbg = @_dbg('test_query')
        dbg('starting')
        @_query
            query : 'SELECT NOW()'
            cb    : (err, result) =>
                dbg("finished", err, result)

    _init_test_query: =>
        if not @_timeout_ms
            return
        @_test_query = setInterval(@_do_test_query, @_timeout_ms)

    _close_test_query: =>
        if @_test_query?
            clearInterval(@_test_query)
            delete @_test_query

    engine: -> 'postgresql'

    connect: (opts) =>
        opts = defaults opts,
            max_time : undefined   # set to something shorter to not try forever
                                   # Only first max_time is used.
            cb       : undefined
        if @_state == 'closed'
            opts.cb?("closed")
            return
        dbg = @_dbg("connect")
        if @_clients?
            dbg("already connected")
            opts.cb?()
            return
        if @_connecting?
            dbg('already trying to connect')
            @_connecting.push(opts.cb)
            return
        dbg('will try to connect')
        @_state = 'init'
        if opts.max_time
            dbg("for up to #{opts.max_time}ms")
        else
            dbg("until successful")
        @_connecting = [opts.cb]
        misc.retry_until_success
            f           : @_connect
            max_delay   : 10000
            max_time    : opts.max_time
            start_delay : 500 + 500*Math.random()
            log         : dbg
            cb          : (err) =>
                v = @_connecting
                delete @_connecting
                for cb in v
                    cb?(err)
                if not err
                    @_state = 'connected'
                    @emit('connect')

    disconnect: () =>
        if @_clients?
            for client in @_clients
                client.end()
                client.removeAllListeners()
        delete @_clients

    _connect: (cb) =>
        dbg = @_dbg("_do_connect")
        dbg("connect to #{@_host}")
        @_clear_listening_state()   # definitely not listening
        if @_clients?
            @disconnect()
        locals =
            clients : []
            hosts   : []
        @_connect_time = 0
        @_concurrent_queries = 0  # can't be any going on now.
        async.series([
            (cb) =>
                if @_ensure_exists
                    dbg("first make sure db exists")
                    @_ensure_database_exists(cb)
                else
                    dbg("assuming database exists")
                    cb()
            (cb) =>
                if not @_host   # undefined if @_host=''
                    locals.hosts = [undefined]
                    cb()
                    return
                if @_host.indexOf('/') != -1
                    dbg("using a local socket file (not a hostname)")
                    locals.hosts = [@_host]
                    cb()
                    return
                f = (host, cb) =>
                    hostname = host.split(':')[0]
                    winston.debug("Looking up ip addresses of #{hostname}")
                    require('dns').lookup hostname, {all:true}, (err, ips) =>
                        if err
                            winston.debug("Got #{hostname} --> err=#{err}")
                            # NON-FATAL -- we just don't include these and hope to
                            # have at least one total working host...
                            cb()
                        else
                            winston.debug("Got #{hostname} --> #{JSON.stringify(ips)}")
                            # In kubernetes the stateful set service just has
                            # lots of ip address.  We connect to *all* of them,
                            # and spread queries across them equally.
                            for x in ips
                                locals.hosts.push(x.address)
                            cb()
                async.map(@_host.split(','), f, cb)
            (cb) =>
                dbg("connecting to #{JSON.stringify(locals.hosts)}...")
                if locals.hosts.length == 0
                    dbg("locals.hosts has length 0 -- no available db")
                    cb("no databases available")
                    return

                dbg("create client and start connecting...")
                locals.clients = []

                # Use a function to initialize the client, to avoid any
                # issues with scope of "client" below.
                init_client = (host) =>
                    client = new pg.Client
                        user     : 'smc'
                        host     : host
                        port     : @_port
                        password : <PASSWORD>
                        database : @_database
                    if @_notification?
                        client.on('notification', @_notification)
                    client.on 'error', (err) =>
                        if @_state == 'init'
                            # already started connecting
                            return
                        @emit('disconnect')
                        dbg("error -- #{err}")
                        @disconnect()
                        @connect()  # start trying to reconnect
                    client.setMaxListeners(1000)  # there is one emitter for each concurrent query... (see query_cb)
                    locals.clients.push(client)

                for host in locals.hosts
                    init_client(host)

                # Connect the clients.  If at least one succeeds, we use this.
                # If none succeed, we declare failure.
                # Obviously, this is NOT optimal -- it's just hopefully sufficiently robust/works.
                # I'm going to redo this with experience.
                locals.clients_that_worked = []
                locals.errors = []
                f = (c) =>
                    try
                        await c.connect()
                        locals.clients_that_worked.push(c)
                    catch err
                        locals.errors.push(err)

                await Promise.all((f(c) for c in locals.clients))

                if locals.clients_that_worked.length == 0
                    dbg("ALL clients failed", locals.errors)
                    cb("ALL clients failed to connect")
                else
                    # take what we got
                    if locals.clients.length == locals.clients_that_worked.length
                        dbg("ALL clients worked")
                    else
                        dbg("ONLY #{locals.clients_that_worked.length} clients worked")
                    locals.clients = locals.clients_that_worked
                    cb()

            (cb) =>
                # CRITICAL!  At scale, this query
                #    SELECT * FROM file_use WHERE project_id = any(select project_id from projects where users ? '25e2cae4-05c7-4c28-ae22-1e6d3d2e8bb3') ORDER BY last_edited DESC limit 100;
                # will take forever due to the query planner using a nestloop scan.  We thus
                # disable doing so!
                @_connect_time = new Date()
                locals.i = 0

                # Weird and unfortunate fact -- this query can and does **HANG** never returning
                # in some edge cases.  That's why we have to be paranoid about this entire _connect
                # function...
                f = (client, cb) =>
                    it_hung = =>
                        cb("hung")
                        cb = undefined
                    timeout = setTimeout(it_hung, 15000)
                    dbg("now connected; disabling nestloop query planning for client #{locals.i}")
                    locals.i += 1
                    client.query "SET enable_nestloop TO off", (err) =>
                        if err
                            dbg("disabling nestloop failed for client #{locals.i}")
                        else
                            dbg("disabling nestloop done for client #{locals.i}")
                        clearTimeout(timeout)
                        cb(err)
                async.map(locals.clients, f, cb)
            (cb) =>
                dbg("checking if ANY db server is in recovery, i.e., we are doing standby queries only")
                @is_standby = false
                f = (client, cb) =>
                    # Is this a read/write or read-only connection?
                    client.query "SELECT pg_is_in_recovery()", (err, resp) =>
                        if err
                            cb(err)
                        else
                            # True if ANY db connection is read only.
                            if resp.rows[0].pg_is_in_recovery
                                @is_standby = true
                            cb()
                async.map(locals.clients, f, cb)
        ], (err) =>
            if err
                mesg = "Failed to connect to database -- #{err}"
                dbg(mesg)
                console.warn(mesg)  # make it clear for interactive users with debugging off -- common mistake with env not setup right.
                cb?(err)
            else
{'fraction_non_alphanumeric': 0.09618726026948168, 'fraction_numerical': 0.006261679179097138, 'mean_word_length': 1.485050916496945, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 70, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3289175', 'n_tokens_mistral': 8113, 'n_tokens_neox': 7564, 'n_words': 2459}