<commit_before>/* gc_lock.h                                                       -*- C++ -*-
   <NAME>, 19 November 2011
   Copyright (c) 2011 Datacratic.  All rights reserved.

   "Lock" that works by deferring the destruction of objects into a garbage collection
   process which is only run when nothing could be using them.
*/

#ifndef __mmap__gc_lock_h__
#define __mmap__gc_lock_h__

#define GC_LOCK_DEBUG 0

#include "jml/utils/exc_assert.h"
#include "jml/arch/atomic_ops.h"
#include "jml/arch/thread_specific.h"
#include <vector>
#include <iostream>

#if GC_LOCK_DEBUG
#  include <iostream>
#endif

/**
   Deterministic memory reclamation is a fundamental problem in
   lock-free algorithms and data structures. 

   When concurrently updating a lock-free data structure, it is
   not safe to immediatly dispose the old value since a bazillon
   of threads might in the same time hold a reference to the old
   value. We then need a safe memory reclamation mechanism.
   That is why GcLock exists. GcLock works by deferring destruction 
   when it is safe to do (when nobody can hold a reference).

   GcLock works by defining "critical sections". When a thread holds
   a reference to a shared object, it enters a critical section.
   
   Speculative critical section should be used when you only want
   to make progress occasionally. Common use case for speculative
   critical section is inside a loop, when you don't need Gc to be unlocked
   everytime but only occasionally :

   GcLock gc;
   for (condition) {
       gc.enterSpeculative();
       // In critical section

       gc.exitSpeculative();
       // After the call, gc might or might not be unlocked
   } 

   gc.forceUnlock();

   forceUnlock makes sure that after the call, the Gc is "unlocked".
   At the end of the loop, Gc might still be in a speculative 
   section, forceUnlock is a way to enforce the unlock.  
   Under heavy contention, speculative "locking" might help improving
   general throughput.
   
*/

namespace Datacratic {

extern int32_t SpeculativeThreshold;

/*****************************************************************************/
/* GC LOCK BASE                                                              */
/*****************************************************************************/

struct GcLockBase : public boost::noncopyable {

public:

    /** Enum for type safe specification of whether or not we run deferrals on
        entry or exit to a critical sections.  Thoss places that are latency
        sensitive should use RD_NO.
    */
    enum RunDefer {
        RD_NO = 0,      ///< Don't run deferred work on this call
        RD_YES = 1      ///< Potentially run deferred work on this call
    };

    /// A thread's bookkeeping info about each GC area
    struct ThreadGcInfoEntry {
        ThreadGcInfoEntry()
            : inEpoch(-1), readLocked(0), writeLocked(0),
              specLocked(0), specUnlocked(0),
              owner(0)
        {
        }

        ~ThreadGcInfoEntry() {
            using namespace std;
            /* We are not in a speculative critical section, check if
             * Gc has been left locked
             */
            if (!specLocked && !specUnlocked && (readLocked || writeLocked))
               cerr << "Thread died but GcLock is still locked" << endl;

            /* We are in a speculative CS but Gc has not beed unlocked,
             * then forceUnlock 
             */
            else if (!specLocked && specUnlocked) {
                unlockShared(RD_YES);
            }
           
        } 


        int inEpoch;  // 0, 1, -1 = not in 
        int readLocked;
        int writeLocked;

        int specLocked;
        int specUnlocked;

        GcLockBase *owner;

        void init(const GcLockBase * const self) {
            if (!owner) 
                owner = const_cast<GcLockBase *>(self);
        }
                

        void lockShared(RunDefer runDefer) {
            if (!readLocked && !writeLocked)
                owner->enterCS(this, runDefer);

            ++readLocked;
        }

        void unlockShared(RunDefer runDefer) {
            if (readLocked <= 0)
                throw ML::Exception("Bad read lock nesting");

            --readLocked;
            if (!readLocked && !writeLocked) 
                owner->exitCS(this, runDefer);
        }

        bool isLockedShared() {
            return readLocked + writeLocked;
        }

        void lockExclusive() {
            if (!writeLocked)
                owner->enterCSExclusive(this);
            
             ++writeLocked;
        }

        void unlockExclusive() {
            if (writeLocked <= 0)
                throw ML::Exception("Bad write lock nesting");

            --writeLocked;
            if (!writeLocked)
                owner->exitCSExclusive(this);
        }

        void lockSpeculative() {
            if (!specLocked && !specUnlocked) 
                lockShared(RD_NO);

            ++specLocked;
        }

        void unlockSpeculative() {
            if (!specLocked) 
                throw ML::Exception("Bad speculative lock nesting");

            --specLocked;
            if (!specLocked) {
                if (++specUnlocked == SpeculativeThreshold) {
                    unlockShared(RD_NO);
                    specUnlocked = 0;
                }
            }
        }

        void forceUnlock() {
            ExcCheckEqual(specLocked, 0, "Bad forceUnlock call");

            if (specUnlocked) {
                unlockShared(RD_YES);
                specUnlocked = 0;
            }
        }


        std::string print() const;
    };

    typedef ML::ThreadSpecificInstanceInfo<ThreadGcInfoEntry, GcLockBase>
        GcInfo;
    typedef typename GcInfo::PerThreadInfo ThreadGcInfo;

    struct Data {
        Data();
        Data(const Data & other);

        Data & operator = (const Data & other);

        typedef uint64_t q2 __attribute__((__vector_size__(16)));
        
        volatile union {
            struct {
                int32_t epoch;       ///< Current epoch number (could be smaller).
                int16_t in[2];       ///< How many threads in each epoch
                int32_t visibleEpoch;///< Lowest epoch number that's visible
                int32_t exclusive;   ///< Mutex value to lock exclusively
            };
            struct {
                uint64_t bits;
                uint64_t bits2;
            };
            struct {
                q2 q;
            };
        } JML_ALIGNED(16);

        int16_t inCurrent() const { return in[epoch & 1]; }
        int16_t inOld() const { return in[(epoch - 1)&1]; }

        void setIn(int32_t epoch, int val)
        {
            //if (epoch != this->epoch && epoch + 1 != this->epoch)
            //    throw ML::Exception("modifying wrong epoch");
            in[epoch & 1] = val;
        }

        void addIn(int32_t epoch, int val)
        {
            //if (epoch != this->epoch && epoch + 1 != this->epoch)
            //    throw ML::Exception("modifying wrong epoch");
            in[epoch & 1] += val;
        }

        /** Check that the invariants all hold.  Throws an exception if not. */
        void validate() const;

        /** Calculate the appropriate value of visibleEpoch from the rest
            of the fields.  Returns true if waiters should be woken up.
        */
        bool calcVisibleEpoch();
        
        /** Human readable string. */
        std::string print() const;

        bool operator == (const Data & other) const
        {
            return bits == other.bits && bits2 == other.bits2;
        }

        bool operator != (const Data & other) const
        {
            return ! operator == (other);
        }

    } JML_ALIGNED(16);


    void enterCS(ThreadGcInfoEntry * entry = 0, RunDefer runDefer = RD_YES);
    void exitCS(ThreadGcInfoEntry * entry = 0, RunDefer runDefer = RD_YES);
    void enterCSExclusive(ThreadGcInfoEntry * entry = 0);
    void exitCSExclusive(ThreadGcInfoEntry * entry = 0);

    int myEpoch(GcInfo::PerThreadInfo * threadInfo = 0) const
    {
        return getEntry(threadInfo).inEpoch;
    }

    int currentEpoch() const
    {
        return data->epoch;
    }

    JML_ALWAYS_INLINE ThreadGcInfoEntry &
    getEntry(GcInfo::PerThreadInfo * info = 0) const
    {
        ThreadGcInfoEntry *entry = gcInfo.get(info);
        entry->init(this);
        return *entry;

        //return *gcInfo.get(info);
    }

    GcLockBase();

    virtual ~GcLockBase();

    /** Permanently deletes any resources associated with this lock. */
    virtual void unlink() = 0;

    void lockShared(GcInfo::PerThreadInfo * info = 0,
                    RunDefer runDefer = RD_YES)
    {
        ThreadGcInfoEntry & entry = getEntry(info);

        entry.lockShared(runDefer);

#if GC_LOCK_DEBUG
        using namespace std;
        cerr << "lockShared "
             << this << " index " << index
             << ": now " << entry.print() << " data "
             << data->print() << endl;
#endif
    }

    void unlockShared(GcInfo::PerThreadInfo * info = 0, 
                      RunDefer runDefer = RD_YES)
    {
        ThreadGcInfoEntry & entry = getEntry(info);

        entry.unlockShared(runDefer);

#if GC_LOCK_DEBUG
        using namespace std;
        cerr << "unlockShared "
             << this << " index " << index
             << ": now " << entry.print() << " data "
             << data->print() << endl;
#endif
    }

    void lockSpeculative(GcInfo::PerThreadInfo * info = 0)
    {
        ThreadGcInfoEntry & entry = getEntry(info); 

        entry.lockSpeculative();
    }

    void unlockSpeculative(GcInfo::PerThreadInfo * info = 0)
    {
        ThreadGcInfoEntry & entry = getEntry(info);

        entry.unlockSpeculative();
    }

    void forceUnlock(GcInfo::PerThreadInfo * info = 0) {
        ThreadGcInfoEntry & entry = getEntry(info);

        entry.forceUnlock();
    }
        
    int isLockedShared(GcInfo::PerThreadInfo * info = 0) const
    {
        ThreadGcInfoEntry & entry = getEntry(info);

        return entry.isLockedShared();
    }

    int lockedInEpoch(GcInfo::PerThreadInfo * info = 0) const
    {
        ThreadGcInfoEntry & entry = getEntry(info);

        return entry.inEpoch;
    }

    void lockExclusive(GcInfo::PerThreadInfo * info = 0)
    {
        ThreadGcInfoEntry & entry = getEntry(info);

        entry.lockExclusive();
#if GC_LOCK_DEBUG
        using namespace std;
        cerr << "lockExclusive "
             << this << " index " << index
             << ": now " << entry.print() << " data "
             << data->print() << endl;
#endif
    }

    void unlockExclusive(GcInfo::PerThreadInfo * info = 0)
    {
        ThreadGcInfoEntry & entry = getEntry(info);

        entry.unlockExclusive();

#if GC_LOCK_DEBUG
        using namespace std;
        cerr << "unlockExclusive"
             << this << " index " << index
             << ": now " << entry.print()
             << " data " << data->print() << endl;
#endif
    }

    int isLockedExclusive(GcInfo::PerThreadInfo * info = 0) const
    {
        ThreadGcInfoEntry & entry = getEntry(info);

        return entry.writeLocked;
    }

    enum DoLock {
        DONT_LOCK = 0,
        DO_LOCK = 1
    };

    struct SharedGuard {
        SharedGuard(GcLockBase & lock,
                    RunDefer runDefer = RD_YES,
                    DoLock doLock = DO_LOCK)
            : lock(lock),
              runDefer_(runDefer),
              doLock_(doLock)
        {
            if (doLock_)
                lock.lockShared(0, runDefer_);
        }

        ~SharedGuard()
        {
            if (doLock_)
                lock.unlockShared(0, runDefer_);
        }
        
        GcLockBase & lock;
        const RunDefer runDefer_;  ///< Can this do deferred work?
        const DoLock doLock_;      ///< Do we really lock?
    };

    struct ExclusiveGuard {
        ExclusiveGuard(GcLockBase & lock)
            : lock(lock)
        {
            lock.lockExclusive();
        }

        ~ExclusiveGuard()
        {
            lock.unlockExclusive();
        }

        GcLockBase & lock;
    };

    struct SpeculativeGuard {
        SpeculativeGuard(GcLockBase &lock) :
            lock(lock) 
        {
            lock.lockSpeculative();
        }

        ~SpeculativeGuard() 
        {
            lock.unlockSpeculative();
        }

        GcLockBase & lock;
    };


    /** Wait until everything that's currently visible is no longer
        accessible.
        
        You can't call this if a guard is held, as it would deadlock waiting
        for itself to exit from the critical section.
    */
    void visibleBarrier();

    /** Wait until all defer functions that have been registered have been
        run.<commit_msg>Allowing to control deferred execution on a speculative critical section<commit_after>/* gc_lock.h                                                       -*- C++ -*-
   <NAME>, 19 November 2011
   Copyright (c) 2011 Datacratic.  All rights reserved.

   "Lock" that works by deferring the destruction of objects into a garbage collection
   process which is only run when nothing could be using them.
*/

#ifndef __mmap__gc_lock_h__
#define __mmap__gc_lock_h__

#define GC_LOCK_DEBUG 0

#include "jml/utils/exc_assert.h"
#include "jml/arch/atomic_ops.h"
#include "jml/arch/thread_specific.h"
#include <vector>
#include <iostream>

#if GC_LOCK_DEBUG
#  include <iostream>
#endif

/**
   Deterministic memory reclamation is a fundamental problem in
   lock-free algorithms and data structures. 

   When concurrently updating a lock-free data structure, it is
   not safe to immediatly dispose the old value since a bazillon
   of threads might in the same time hold a reference to the old
   value. We then need a safe memory reclamation mechanism.
   That is why GcLock exists. GcLock works by deferring destruction 
   when it is safe to do (when nobody can hold a reference).

   GcLock works by defining "critical sections". When a thread holds
   a reference to a shared object, it enters a critical section.
   
   Speculative critical section should be used when you only want
   to make progress occasionally. Common use case for speculative
   critical section is inside a loop, when you don't need Gc to be unlocked
   everytime but only occasionally :

   GcLock gc;
   for (condition) {
       gc.enterSpeculative();
       // In critical section

       gc.exitSpeculative();
       // After the call, gc might or might not be unlocked
   } 

   gc.forceUnlock();

   forceUnlock makes sure that after the call, the Gc is "unlocked".
   At the end of the loop, Gc might still be in a speculative 
   section, forceUnlock is a way to enforce the unlock.  
   Under heavy contention, speculative "locking" might help improving
   general throughput.
   
*/

namespace Datacratic {

extern int32_t SpeculativeThreshold;

/*****************************************************************************/
/* GC LOCK BASE                                                              */
/*****************************************************************************/

struct GcLockBase : public boost::noncopyable {

public:

    /** Enum for type safe specification of whether or not we run deferrals on
        entry or exit to a critical sections.  Thoss places that are latency
        sensitive should use RD_NO.
    */
    enum RunDefer {
        RD_NO = 0,      ///< Don't run deferred work on this call
        RD_YES = 1      ///< Potentially run deferred work on this call
    };

    /// A thread's bookkeeping info about each GC area
    struct ThreadGcInfoEntry {
        ThreadGcInfoEntry()
            : inEpoch(-1), readLocked(0), writeLocked(0),
              specLocked(0), specUnlocked(0),
              owner(0)
        {
        }

        ~ThreadGcInfoEntry() {
            using namespace std;
            /* We are not in a speculative critical section, check if
             * Gc has been left locked
             */
            if (!specLocked && !specUnlocked && (readLocked || writeLocked))
               cerr << "Thread died but GcLock is still locked" << endl;

            /* We are in a speculative CS but Gc has not beed unlocked,
             * then forceUnlock 
             */
            else if (!specLocked && specUnlocked) {
                unlockShared(RD_YES);
            }
           
        } 


        int inEpoch;  // 0, 1, -1 = not in 
        int readLocked;
        int writeLocked;

        int specLocked;
        int specUnlocked;

        GcLockBase *owner;

        void init(const GcLockBase * const self) {
            if (!owner) 
                owner = const_cast<GcLockBase *>(self);
        }
                

        void lockShared(RunDefer runDefer) {
            if (!readLocked && !writeLocked)
                owner->enterCS(this, runDefer);

            ++readLocked;
        }

        void unlockShared(RunDefer runDefer) {
            if (readLocked <= 0)
                throw ML::Exception("Bad read lock nesting");

            --readLocked;
            if (!readLocked && !writeLocked) 
                owner->exitCS(this, runDefer);
        }

        bool isLockedShared() {
            return readLocked + writeLocked;
        }

        void lockExclusive() {
            if (!writeLocked)
                owner->enterCSExclusive(this);
            
             ++writeLocked;
        }

        void unlockExclusive() {
            if (writeLocked <= 0)
                throw ML::Exception("Bad write lock nesting");

            --writeLocked;
            if (!writeLocked)
                owner->exitCSExclusive(this);
        }

        void lockSpeculative(RunDefer runDefer) {
            if (!specLocked && !specUnlocked) 
                lockShared(runDefer);

            ++specLocked;
        }

        void unlockSpeculative(RunDefer runDefer) {
            if (!specLocked) 
                throw ML::Exception("Bad speculative lock nesting");

            --specLocked;
            if (!specLocked) {
                if (++specUnlocked == SpeculativeThreshold) {
                    unlockShared(runDefer);
                    specUnlocked = 0;
                }
            }
        }

        void forceUnlock(RunDefer runDefer) {
            ExcCheckEqual(specLocked, 0, "Bad forceUnlock call");

            if (specUnlocked) {
                unlockShared(runDefer);
                specUnlocked = 0;
            }
        }


        std::string print() const;
    };

    typedef ML::ThreadSpecificInstanceInfo<ThreadGcInfoEntry, GcLockBase>
        GcInfo;
    typedef typename GcInfo::PerThreadInfo ThreadGcInfo;

    struct Data {
        Data();
        Data(const Data & other);

        Data & operator = (const Data & other);

        typedef uint64_t q2 __attribute__((__vector_size__(16)));
        
        volatile union {
            struct {
                int32_t epoch;       ///< Current epoch number (could be smaller).
                int16_t in[2];       ///< How many threads in each epoch
                int32_t visibleEpoch;///< Lowest epoch number that's visible
                int32_t exclusive;   ///< Mutex value to lock exclusively
            };
            struct {
                uint64_t bits;
                uint64_t bits2;
            };
            struct {
                q2 q;
            };
        } JML_ALIGNED(16);

        int16_t inCurrent() const { return in[epoch & 1]; }
        int16_t inOld() const { return in[(epoch - 1)&1]; }

        void setIn(int32_t epoch, int val)
        {
            //if (epoch != this->epoch && epoch + 1 != this->epoch)
            //    throw ML::Exception("modifying wrong epoch");
            in[epoch & 1] = val;
        }

        void addIn(int32_t epoch, int val)
        {
            //if (epoch != this->epoch && epoch + 1 != this->epoch)
            //    throw ML::Exception("modifying wrong epoch");
            in[epoch & 1] += val;
        }

        /** Check that the invariants all hold.  Throws an exception if not. */
        void validate() const;

        /** Calculate the appropriate value of visibleEpoch from the rest
            of the fields.  Returns true if waiters should be woken up.
        */
        bool calcVisibleEpoch();
        
        /** Human readable string. */
        std::string print() const;

        bool operator == (const Data & other) const
        {
            return bits == other.bits && bits2 == other.bits2;
        }

        bool operator != (const Data & other) const
        {
            return ! operator == (other);
        }

    } JML_ALIGNED(16);


    void enterCS(ThreadGcInfoEntry * entry = 0, RunDefer runDefer = RD_YES);
    void exitCS(ThreadGcInfoEntry * entry = 0, RunDefer runDefer = RD_YES);
    void enterCSExclusive(ThreadGcInfoEntry * entry = 0);
    void exitCSExclusive(ThreadGcInfoEntry * entry = 0);

    int myEpoch(GcInfo::PerThreadInfo * threadInfo = 0) const
    {
        return getEntry(threadInfo).inEpoch;
    }

    int currentEpoch() const
    {
        return data->epoch;
    }

    JML_ALWAYS_INLINE ThreadGcInfoEntry &
    getEntry(GcInfo::PerThreadInfo * info = 0) const
    {
        ThreadGcInfoEntry *entry = gcInfo.get(info);
        entry->init(this);
        return *entry;

        //return *gcInfo.get(info);
    }

    GcLockBase();

    virtual ~GcLockBase();

    /** Permanently deletes any resources associated with this lock. */
    virtual void unlink() = 0;

    void lockShared(GcInfo::PerThreadInfo * info = 0,
                    RunDefer runDefer = RD_YES)
    {
        ThreadGcInfoEntry & entry = getEntry(info);

        entry.lockShared(runDefer);

#if GC_LOCK_DEBUG
        using namespace std;
        cerr << "lockShared "
             << this << " index " << index
             << ": now " << entry.print() << " data "
             << data->print() << endl;
#endif
    }

    void unlockShared(GcInfo::PerThreadInfo * info = 0, 
                      RunDefer runDefer = RD_YES)
    {
        ThreadGcInfoEntry & entry = getEntry(info);

        entry.unlockShared(runDefer);

#if GC_LOCK_DEBUG
        using namespace std;
        cerr << "unlockShared "
             << this << " index " << index
             << ": now " << entry.print() << " data "
             << data->print() << endl;
#endif
    }

    void lockSpeculative(GcInfo::PerThreadInfo * info = 0,
                         RunDefer runDefer = RD_YES)
    {
        ThreadGcInfoEntry & entry = getEntry(info); 

        entry.lockSpeculative(runDefer);
    }

    void unlockSpeculative(GcInfo::PerThreadInfo * info = 0,
                           RunDefer runDefer = RD_YES)
    {
        ThreadGcInfoEntry & entry = getEntry(info);

        entry.unlockSpeculative(runDefer);
    }

    void forceUnlock(GcInfo::PerThreadInfo * info = 0,
                     RunDefer runDefer = RD_YES) {
        ThreadGcInfoEntry & entry = getEntry(info);

        entry.forceUnlock(runDefer);
    }
        
    int isLockedShared(GcInfo::PerThreadInfo * info = 0) const
    {
        ThreadGcInfoEntry & entry = getEntry(info);

        return entry.isLockedShared();
    }

    int lockedInEpoch(GcInfo::PerThreadInfo * info = 0) const
    {
        ThreadGcInfoEntry & entry = getEntry(info);

        return entry.inEpoch;
    }

    void lockExclusive(GcInfo::PerThreadInfo * info = 0)
    {
        ThreadGcInfoEntry & entry = getEntry(info);

        entry.lockExclusive();
#if GC_LOCK_DEBUG
        using namespace std;
        cerr << "lockExclusive "
             << this << " index " << index
             << ": now " << entry.print() << " data "
             << data->print() << endl;
#endif
    }

    void unlockExclusive(GcInfo::PerThreadInfo * info = 0)
    {
        ThreadGcInfoEntry & entry = getEntry(info);

        entry.unlockExclusive();

#if GC_LOCK_DEBUG
        using namespace std;
        cerr << "unlockExclusive"
             << this << " index " << index
             << ": now " << entry.print()
             << " data " << data->print() << endl;
#endif
    }

    int isLockedExclusive(GcInfo::PerThreadInfo * info = 0) const
    {
        ThreadGcInfoEntry & entry = getEntry(info);

        return entry.writeLocked;
    }

    enum DoLock {
        DONT_LOCK = 0,
        DO_LOCK = 1
    };

    struct SharedGuard {
        SharedGuard(GcLockBase & lock,
                    RunDefer runDefer = RD_YES,
                    DoLock doLock = DO_LOCK)
            : lock(lock),
              runDefer_(runDefer),
              doLock_(doLock)
        {
            if (doLock_)
                lock.lockShared(0, runDefer_);
        }

        ~SharedGuard()
        {
            if (doLock_)
                lock.unlockShared(0, runDefer_);
        }
        
        GcLockBase & lock;
        const RunDefer runDefer_;  ///< Can this do deferred work?
        const DoLock doLock_;      ///< Do we really lock?
    };

    struct ExclusiveGuard {
        ExclusiveGuard(GcLockBase & lock)
            : lock(lock)
        {
            lock.lockExclusive();
        }

        ~ExclusiveGuard()
        {
            lock.unlockExclusive();
        }

        GcLockBase & lock;
    };

    struct SpeculativeGuard {
        SpeculativeGuard(GcLockBase &lock,
                         RunDefer runDefer = RD_YES) :
            lock(lock),
            runDefer_(runDefer) 
        {
            lock.lockSpeculative(0, runDefer_);
        }

        ~SpeculativeGuard() 
        {
            lock.unlockSpeculative(0, runDefer_);
        }

        GcLockBase & lock;
        const RunDefer runDefer_;
    };


    /** Wait until everything that's currently visible is no longer
        accessible.
        
        You can't call this if a guard is held, as it would deadlock waiting
        for itself to exit from the critical section.
    */
    void visibleBarrier();

    /** Wait until all defer functions that have been registered have been
        run.
{'fraction_non_alphanumeric': 0.10379562602933855, 'fraction_numerical': 0.007047378298670956, 'mean_word_length': 1.7585842577918647, 'pattern_counts': {'":': 8, '<': 187, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4685794', 'n_tokens_mistral': 7522, 'n_tokens_neox': 7107, 'n_words': 2055}