<commit_before>###

SageMathCloud, Copyright (C) 2015, <NAME>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

---

SYNCHRONIZED TABLE -- defined by an object query

    - Do a query against a RethinkDB table using our object query description.
    - Synchronization with the backend database is done automatically.

   Methods:
      - constructor(query): query = the name of a table (or a more complicated object)

      - set(map):  Set the given keys of map to their values; one key must be
                   the primary key for the table.  NOTE: Computed primary keys will
                   get automatically filled in; these are keys in schema.coffee,
                   where the set query looks like this say:
                      (obj, db) -> db.sha1(obj.project_id, obj.path)
      - get():     Current value of the query, as an immutable.js Map from
                   the primary key to the records, which are also immutable.js Maps.
      - get(key):  The record with given key, as an immutable Map.
      - get(keys): Immutable Map from given keys to the corresponding records.
      - get_one(): Returns one record as an immutable Map (useful if there
                   is only one record)

      - close():   Frees up resources, stops syncing, don't use object further

   Events:
      - 'change', [array of primary keys] : fired any time the value of the query result
                 changes, *including* if changed by calling set on this object.
                 Also, called with empty list on first connection if there happens
                 to be nothing in this table.
###


{EventEmitter} = require('events')

immutable = require('immutable')
async     = require('async')
misc      = require('misc')
schema    = require('schema')

# We represent synchronized tables by an immutable.js mapping from the primary
# key to the object.  Since RethinkDB primary keys can be more than just strings,
# e.g., they can be arrays, so we convert complicated keys to their
# JSON representation.  According to RethinkdB: "The data type of a primary
# key is usually a string (like a UUID) or a number, but it can also be a
# time, binary object, boolean or an array."
# (see https://rethinkdb.com/api/javascript/table_create/)
# A binary object doesn't make sense here in pure javascript, but these do:
#       string, number, time, boolean, or array
# Everything automatically converts fine to a string except array, which is the
# main thing this function deals with below.
# NOTE (1)  RIGHT NOW:  This should be safe to change at
# any time, since the keys aren't stored longterm.
# If we do something with localStorage, this will no longer be safe
# without a version number.
# NOTE (2) Of course you could use both a string and an array as primary keys
# in the same table.  You could evily make the string equal the json of an array,
# and this *would* break things.  We are thus assuming that such mixing
# doesn't happen.  An alternative would be to just *always* use JSON.stringify.
to_key = (x) ->
    if typeof(x) == 'object'
        return JSON.stringify(x)
    else
        return x


class SyncTable extends EventEmitter
    constructor: (@_query, @_options, @_client, @_debounce_interval=2000) ->
        @_init_query()

        # The value of this query locally.
        @_value_local = undefined

        # Our best guess as to the value of this query on the server,
        # according to queries and updates the server pushes to us.
        @_value_server = undefined

        # The changefeed id, when set by doing a change-feed aware query.
        @_id = undefined

        # Whether or not currently successfully connected.
        @_connected = false

        # Reconnect on connect.
        @_client.on 'connected', =>
            @_connected = false
            # We delete @_reconnecting to ensure that it immediately reconnects.
            # This is safe, since if we just connected, the only possibility for
            # outstanding attempts is failure.
            delete @_reconnecting
            @_reconnect()

        # Connect to the server the first time.
        @_reconnect()

    get: (arg) =>
        if arg?
            if misc.is_array(arg)
                x = {}
                for k in arg
                    x[to_key(k)] = @_value_local.get(to_key(k))
                return immutable.fromJS(x)
            else
                return @_value_local.get(to_key(arg))
        else
            return @_value_local

    get_one: =>
        return @_value_local?.toSeq().first()

    _parse_query: (query) =>
        if typeof(query) == 'string'
            # name of a table -- get all fields
            v = misc.copy(schema.SCHEMA[query].user_query.get.fields)
            for k, _ of v
                v[k] = null
            return {"#{query}": [v]}
        else
            keys = misc.keys(query)
            if keys.length != 1
                throw Error("must specify exactly one table")
            table = keys[0]
            x = {}
            if not misc.is_array(query[table])
                return {"#{table}": [query[table]]}
            else
                return {"#{table}": query[table]}

    _init_query: =>
        # first parse the query to allow for some convenient shortcuts
        @_query = @_parse_query(@_query)

        # Check that the query is probably valid, and record the table and schema
        if misc.is_array(@_query)
            throw Error("must be a single query")
        tables = misc.keys(@_query)
        if misc.len(tables) != 1
            throw Error("must query only a single table")
        @_table = tables[0]
        if not misc.is_array(@_query[@_table])
            throw Error("must be a multi-document queries")
        @_schema = schema.SCHEMA[@_table]
        if not @_schema?
            throw Error("unknown schema for table #{@_table}")
        @_primary_key = @_schema.primary_key ? "id"
        # TODO: could put in more checks on validity of query here, using schema...
        if not @_query[@_table][0][@_primary_key]?
            # must include primary key in query
            @_query[@_table][0][@_primary_key] = null

        # Which fields the user is allowed to set.
        @_set_fields = []
        # Which fields *must* be included in any set query
        @_required_set_fields = {}
        for field in misc.keys(@_query[@_table][0])
            if @_schema.user_query?.set?.fields?[field]?
                @_set_fields.push(field)
            if @_schema.user_query?.set?.required_fields?[field]?
                @_required_set_fields[field] = true

        # Is anonymous access to this table allowed?
        @_anonymous = !!@_schema.anonymous

    _reconnect: =>
        if @_closed
            throw Error("object is closed")
        #dbg = (m) -> console.log("_reconnect: #{m}")
        #dbg()
        if not @_client._connected
            # nothing to do -- not connected to server; when reconnect to server, will do proper reconnect
            #dbg("not connected to server")
            return
        if @_connected
            #dbg("already connected to feed")
            return
        if @_reconnecting
            #dbg("_reconnecting right now already")
            return
        if not @_anonymous and not @_client.is_signed_in()
            #console.log("waiting for sign in before connecting")
            @_client.once 'signed_in', =>
                #console.log("sign in triggered connecting")
                @_reconnect()
            return
        @_reconnecting = true
        setTimeout( (() => delete @_reconnecting), 30000 )
        @_run (err) =>
            @_reconnecting = false
            if not @_connected and @_client.connected
                # didn't work, but client is connected to server -- try again in a little while
                setTimeout( @_reconnect, 120*1000 )

    _run: (cb) =>
        if @_closed
            throw Error("object is closed")
        first_resp = true
        #console.log("query #{@_table}: _run")
        @_client.query
            query   : @_query
            changes : true
            timeout : 30
            options : @_options
            cb      : (err, resp) =>
                @_last_err = err
                #console.log("query #{@_table}: -- got result of doing query", resp)
                if first_resp
                    first_resp = false
                    if @_closed
                        cb?("closed")
                    else if err
                        console.warn("query #{@_table}: _run: first error ", err)
                        cb?(err)
                    else if not resp?.query?[@_table]?
                        console.warn("query on #{@_table} returned undefined")
                        cb?("got no data")
                    else
                        @_id = resp.id
                        @_connected = true
                        #console.log("query #{@_table}: query resp = ", resp)
                        @_update_all(resp.query[@_table])
                        cb?()
                else
                    #console.log("changefeed #{@_table} produced: #{err}, ", resp)
                    # changefeed
                    if err
                        @_connected = false
                        if err != 'killfeed'    # killfeed is expected and happens regularly (right now)
                            console.warn("query #{@_table}: _run: not first error -- ", err)
                        @_reconnect()
                    else
                        @_update_change(resp)

    _save: (cb) =>
        #console.log("_save(#{@_table})")
        # Determine which records have changed and what their new values are.
        changed = {}
        if not @_value_server?
            cb?("don't know server yet")
            return
        if not @_value_local?
            cb?("don't know local yet")
            return
        at_start = @_value_local
        @_value_local.map (new_val, key) =>
            old_val = @_value_server.get(key)
            if not new_val.equals(old_val)
                changed[key] = {new_val:new_val, old_val:old_val}

        # Send our changes to the server.
        query = []
        for key in misc.keys(changed).sort()  # sort so that behavior is more predictable = faster (e.g., sync patches are in order)
            c = changed[key]
            obj = {"#{@_primary_key}":key}
            for k in @_set_fields
                v = c.new_val.get(k)
                if v?
                    if @_required_set_fields[k] or not immutable.is(v, c.old_val?.get(k))
                        if immutable.Map.isMap(v)
                            obj[k] = v.toJS()
                        else
                            obj[k] = v
            query.push({"#{@_table}":obj})

        #console.log("sending #{query.length} changes: #{misc.to_json(query)}")
        if query.length == 0
            cb?()
            return
        @_client.query
            query : query
            cb    : (err) =>
                if not err and at_start != @_value_local
                    # keep saving until table doesn't change *during* the save
                    @_save(cb)
                else
                    cb?(err)

    _save0 : (cb) =>
        misc.retry_until_success
            f         : @_save
            max_tries : 100
            #warn      : (m) -> console.warn(m)
            #log       : (m) -> console.log(m)
            cb        : cb

    save: (cb) =>
        if @_saving?
            @_saving.push(cb)
            return
        @_saving = [cb]
        @_save_debounce ?= {}
        misc.async_debounce
            f        : @_save0
            interval : @_debounce_interval
            state    : @_save_debounce
            cb       : (err) =>
                v = @_saving
                delete @_saving
                for cb in v
                    cb?(err)

    # Handle an update of all records from the database.  This happens on
    # initialization, and also if we disconnect and reconnect.
    _update_all: (v) =>
        #console.log("_update_all(#{@_table})", v)
        if not v?
            console.warn("_update_all(#{@_table}) called with v=undefined")
            return

        # Restructure the array of records in v as a mapping from the primary key
        # to the corresponding record.
        x = {}
        for y in v
            x[to_key(y[@_primary_key])] = y

        conflict = false

        # Figure out what to change in our local view of the database query result.
        if not @_value_local? or not @_value_server?
            #console.log("_update_all: easy case -- nothing has been initialized yet, so just set everything.")
            @_value_local = @_value_server = immutable.fromJS(x)
            first_connect = true
            changed_keys = misc.keys(x)  # of course all keys have been changed.
        else
            # Harder case -- everything has already been initialized.
            changed_keys = []
            # DELETE or CHANGED:
            # First check through each key in our local view of the query
            # and if the value differs from what is in the database (i.e., what we just got from DB), make
            # that change.  (Later we will possibly merge in the change
            # using the last known upstream database state.)
            @_value_local.map (local, key) =>
                # x[key] is what we just got from DB, and it's different from what we have locally
                new_val = new_val0 = immutable.fromJS(x[key])
                if not local.equals(new_val)
                    changed_keys.push(key)
                    if not new_val?
                        # delete the record
                        @_value_local = @_value_local.delete(key)
                    else
                        server = @_value_server.get(key)
                        if not local.equals(server)
                            # conflict
                            local.map (v, k) =>
                                if not immutable.is(server.get(k), v)
                                    conflict = true
                                    #console.log("update_all conflict ", k)
                                    new_val0 = new_val0.set(k, v)
                        # set the record to its new server value
                        @_value_local = @_value_local.set(key, new_val0)
            # NEWLY ADDED:
            # Next check through each key in what's on the remote database,
            # and if the corresponding local key isn't defined, set its value.
            # Here we are simply checking for newly added records.
            for key, val of x
                if not @_value_local.get(key)?
                    @_value_local = @_value_local.set(key, immutable.fromJS(val))
                    changed_keys.push(key)

        # It's possibly that nothing changed (e.g., typical case on reconnect!) so we check.
        # If something really did change, we set the server state to what we just got, and
        # also inform listeners of which records changed (by giving keys).
        #console.log("update_all: changed_keys=", changed_keys)
        if changed_keys.length != 0
            @_value_server = immutable.fromJS(x)
            @emit('change', changed_keys)
        else if first_connect
            # First connection and table is empty.
            @emit('change', changed_keys)
        if conflict
            @save()

    _update_change: (change) =>
        #console.log("_update_change", change)
        if not @_value_local?
            console.warn("BUG: tried to call _update_change even though local not yet defined")
            return<commit_msg>synctable: another subtle issue when connection closed but there might still be another message before feed canceled... ?<commit_after>###

SageMathCloud, Copyright (C) 2015, <NAME>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

---

SYNCHRONIZED TABLE -- defined by an object query

    - Do a query against a RethinkDB table using our object query description.
    - Synchronization with the backend database is done automatically.

   Methods:
      - constructor(query): query = the name of a table (or a more complicated object)

      - set(map):  Set the given keys of map to their values; one key must be
                   the primary key for the table.  NOTE: Computed primary keys will
                   get automatically filled in; these are keys in schema.coffee,
                   where the set query looks like this say:
                      (obj, db) -> db.sha1(obj.project_id, obj.path)
      - get():     Current value of the query, as an immutable.js Map from
                   the primary key to the records, which are also immutable.js Maps.
      - get(key):  The record with given key, as an immutable Map.
      - get(keys): Immutable Map from given keys to the corresponding records.
      - get_one(): Returns one record as an immutable Map (useful if there
                   is only one record)

      - close():   Frees up resources, stops syncing, don't use object further

   Events:
      - 'change', [array of primary keys] : fired any time the value of the query result
                 changes, *including* if changed by calling set on this object.
                 Also, called with empty list on first connection if there happens
                 to be nothing in this table.
###


{EventEmitter} = require('events')

immutable = require('immutable')
async     = require('async')
misc      = require('misc')
schema    = require('schema')

# We represent synchronized tables by an immutable.js mapping from the primary
# key to the object.  Since RethinkDB primary keys can be more than just strings,
# e.g., they can be arrays, so we convert complicated keys to their
# JSON representation.  According to RethinkdB: "The data type of a primary
# key is usually a string (like a UUID) or a number, but it can also be a
# time, binary object, boolean or an array."
# (see https://rethinkdb.com/api/javascript/table_create/)
# A binary object doesn't make sense here in pure javascript, but these do:
#       string, number, time, boolean, or array
# Everything automatically converts fine to a string except array, which is the
# main thing this function deals with below.
# NOTE (1)  RIGHT NOW:  This should be safe to change at
# any time, since the keys aren't stored longterm.
# If we do something with localStorage, this will no longer be safe
# without a version number.
# NOTE (2) Of course you could use both a string and an array as primary keys
# in the same table.  You could evily make the string equal the json of an array,
# and this *would* break things.  We are thus assuming that such mixing
# doesn't happen.  An alternative would be to just *always* use JSON.stringify.
to_key = (x) ->
    if typeof(x) == 'object'
        return JSON.stringify(x)
    else
        return x


class SyncTable extends EventEmitter
    constructor: (@_query, @_options, @_client, @_debounce_interval=2000) ->
        @_init_query()

        # The value of this query locally.
        @_value_local = undefined

        # Our best guess as to the value of this query on the server,
        # according to queries and updates the server pushes to us.
        @_value_server = undefined

        # The changefeed id, when set by doing a change-feed aware query.
        @_id = undefined

        # Whether or not currently successfully connected.
        @_connected = false

        # Reconnect on connect.
        @_client.on 'connected', =>
            @_connected = false
            # We delete @_reconnecting to ensure that it immediately reconnects.
            # This is safe, since if we just connected, the only possibility for
            # outstanding attempts is failure.
            delete @_reconnecting
            @_reconnect()

        # Connect to the server the first time.
        @_reconnect()

    get: (arg) =>
        if arg?
            if misc.is_array(arg)
                x = {}
                for k in arg
                    x[to_key(k)] = @_value_local.get(to_key(k))
                return immutable.fromJS(x)
            else
                return @_value_local.get(to_key(arg))
        else
            return @_value_local

    get_one: =>
        return @_value_local?.toSeq().first()

    _parse_query: (query) =>
        if typeof(query) == 'string'
            # name of a table -- get all fields
            v = misc.copy(schema.SCHEMA[query].user_query.get.fields)
            for k, _ of v
                v[k] = null
            return {"#{query}": [v]}
        else
            keys = misc.keys(query)
            if keys.length != 1
                throw Error("must specify exactly one table")
            table = keys[0]
            x = {}
            if not misc.is_array(query[table])
                return {"#{table}": [query[table]]}
            else
                return {"#{table}": query[table]}

    _init_query: =>
        # first parse the query to allow for some convenient shortcuts
        @_query = @_parse_query(@_query)

        # Check that the query is probably valid, and record the table and schema
        if misc.is_array(@_query)
            throw Error("must be a single query")
        tables = misc.keys(@_query)
        if misc.len(tables) != 1
            throw Error("must query only a single table")
        @_table = tables[0]
        if not misc.is_array(@_query[@_table])
            throw Error("must be a multi-document queries")
        @_schema = schema.SCHEMA[@_table]
        if not @_schema?
            throw Error("unknown schema for table #{@_table}")
        @_primary_key = @_schema.primary_key ? "id"
        # TODO: could put in more checks on validity of query here, using schema...
        if not @_query[@_table][0][@_primary_key]?
            # must include primary key in query
            @_query[@_table][0][@_primary_key] = null

        # Which fields the user is allowed to set.
        @_set_fields = []
        # Which fields *must* be included in any set query
        @_required_set_fields = {}
        for field in misc.keys(@_query[@_table][0])
            if @_schema.user_query?.set?.fields?[field]?
                @_set_fields.push(field)
            if @_schema.user_query?.set?.required_fields?[field]?
                @_required_set_fields[field] = true

        # Is anonymous access to this table allowed?
        @_anonymous = !!@_schema.anonymous

    _reconnect: =>
        if @_closed
            throw Error("object is closed")
        #dbg = (m) -> console.log("_reconnect: #{m}")
        #dbg()
        if not @_client._connected
            # nothing to do -- not connected to server; when reconnect to server, will do proper reconnect
            #dbg("not connected to server")
            return
        if @_connected
            #dbg("already connected to feed")
            return
        if @_reconnecting
            #dbg("_reconnecting right now already")
            return
        if not @_anonymous and not @_client.is_signed_in()
            #console.log("waiting for sign in before connecting")
            @_client.once 'signed_in', =>
                #console.log("sign in triggered connecting")
                @_reconnect()
            return
        @_reconnecting = true
        setTimeout( (() => delete @_reconnecting), 30000 )
        @_run (err) =>
            @_reconnecting = false
            if not @_connected and @_client.connected
                # didn't work, but client is connected to server -- try again in a little while
                setTimeout( @_reconnect, 120*1000 )

    _run: (cb) =>
        if @_closed
            throw Error("object is closed")
        first_resp = true
        #console.log("query #{@_table}: _run")
        @_client.query
            query   : @_query
            changes : true
            timeout : 30
            options : @_options
            cb      : (err, resp) =>
                @_last_err = err
                #console.log("query #{@_table}: -- got result of doing query", resp)
                if first_resp
                    first_resp = false
                    if @_closed
                        cb?("closed")
                    else if err
                        console.warn("query #{@_table}: _run: first error ", err)
                        cb?(err)
                    else if not resp?.query?[@_table]?
                        console.warn("query on #{@_table} returned undefined")
                        cb?("got no data")
                    else
                        @_id = resp.id
                        @_connected = true
                        #console.log("query #{@_table}: query resp = ", resp)
                        @_update_all(resp.query[@_table])
                        cb?()
                else
                    #console.log("changefeed #{@_table} produced: #{err}, ", resp)
                    # changefeed
                    if err
                        @_connected = false
                        if err != 'killfeed'    # killfeed is expected and happens regularly (right now)
                            console.warn("query #{@_table}: _run: not first error -- ", err)
                        @_reconnect()
                    else
                        @_update_change(resp)

    _save: (cb) =>
        #console.log("_save(#{@_table})")
        # Determine which records have changed and what their new values are.
        changed = {}
        if not @_value_server?
            cb?("don't know server yet")
            return
        if not @_value_local?
            cb?("don't know local yet")
            return
        at_start = @_value_local
        @_value_local.map (new_val, key) =>
            old_val = @_value_server.get(key)
            if not new_val.equals(old_val)
                changed[key] = {new_val:new_val, old_val:old_val}

        # Send our changes to the server.
        query = []
        for key in misc.keys(changed).sort()  # sort so that behavior is more predictable = faster (e.g., sync patches are in order)
            c = changed[key]
            obj = {"#{@_primary_key}":key}
            for k in @_set_fields
                v = c.new_val.get(k)
                if v?
                    if @_required_set_fields[k] or not immutable.is(v, c.old_val?.get(k))
                        if immutable.Map.isMap(v)
                            obj[k] = v.toJS()
                        else
                            obj[k] = v
            query.push({"#{@_table}":obj})

        #console.log("sending #{query.length} changes: #{misc.to_json(query)}")
        if query.length == 0
            cb?()
            return
        @_client.query
            query : query
            cb    : (err) =>
                if not err and at_start != @_value_local
                    # keep saving until table doesn't change *during* the save
                    @_save(cb)
                else
                    cb?(err)

    _save0 : (cb) =>
        misc.retry_until_success
            f         : @_save
            max_tries : 100
            #warn      : (m) -> console.warn(m)
            #log       : (m) -> console.log(m)
            cb        : cb

    save: (cb) =>
        if @_saving?
            @_saving.push(cb)
            return
        @_saving = [cb]
        @_save_debounce ?= {}
        misc.async_debounce
            f        : @_save0
            interval : @_debounce_interval
            state    : @_save_debounce
            cb       : (err) =>
                v = @_saving
                delete @_saving
                for cb in v
                    cb?(err)

    # Handle an update of all records from the database.  This happens on
    # initialization, and also if we disconnect and reconnect.
    _update_all: (v) =>
        if @_closed
            return
        #console.log("_update_all(#{@_table})", v)
        if not v?
            console.warn("_update_all(#{@_table}) called with v=undefined")
            return

        # Restructure the array of records in v as a mapping from the primary key
        # to the corresponding record.
        x = {}
        for y in v
            x[to_key(y[@_primary_key])] = y

        conflict = false

        # Figure out what to change in our local view of the database query result.
        if not @_value_local? or not @_value_server?
            #console.log("_update_all: easy case -- nothing has been initialized yet, so just set everything.")
            @_value_local = @_value_server = immutable.fromJS(x)
            first_connect = true
            changed_keys = misc.keys(x)  # of course all keys have been changed.
        else
            # Harder case -- everything has already been initialized.
            changed_keys = []
            # DELETE or CHANGED:
            # First check through each key in our local view of the query
            # and if the value differs from what is in the database (i.e., what we just got from DB), make
            # that change.  (Later we will possibly merge in the change
            # using the last known upstream database state.)
            @_value_local.map (local, key) =>
                # x[key] is what we just got from DB, and it's different from what we have locally
                new_val = new_val0 = immutable.fromJS(x[key])
                if not local.equals(new_val)
                    changed_keys.push(key)
                    if not new_val?
                        # delete the record
                        @_value_local = @_value_local.delete(key)
                    else
                        server = @_value_server.get(key)
                        if not local.equals(server)
                            # conflict
                            local.map (v, k) =>
                                if not immutable.is(server.get(k), v)
                                    conflict = true
                                    #console.log("update_all conflict ", k)
                                    new_val0 = new_val0.set(k, v)
                        # set the record to its new server value
                        @_value_local = @_value_local.set(key, new_val0)
            # NEWLY ADDED:
            # Next check through each key in what's on the remote database,
            # and if the corresponding local key isn't defined, set its value.
            # Here we are simply checking for newly added records.
            for key, val of x
                if not @_value_local.get(key)?
                    @_value_local = @_value_local.set(key, immutable.fromJS(val))
                    changed_keys.push(key)

        # It's possibly that nothing changed (e.g., typical case on reconnect!) so we check.
        # If something really did change, we set the server state to what we just got, and
        # also inform listeners of which records changed (by giving keys).
        #console.log("update_all: changed_keys=", changed_keys)
        if changed_keys.length != 0
            @_value_server = immutable.fromJS(x)
            @emit('change', changed_keys)
        else if first_connect
            # First connection and table is empty.
            @emit('change', changed_keys)
        if conflict
            @save()

    _update_change: (change) =>
        if @_closed
            return
        #console.log("_update_change", change)
        if not @_value_local?
            console.warn("BUG: tried to call _update_change even though local not yet defined")
            return
{'fraction_non_alphanumeric': 0.08650118794547955, 'fraction_numerical': 0.002751031636863824, 'mean_word_length': 1.6593233020201181, 'pattern_counts': {'":': 10, '<': 5, '<?xml version=': 0, '>': 59, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4857374', 'n_tokens_mistral': 8635, 'n_tokens_neox': 8261, 'n_words': 3050}