<commit_before>// Copyright 2019 The TCMalloc Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#ifndef TCMALLOC_PERCPU_TCMALLOC_H_
#define TCMALLOC_PERCPU_TCMALLOC_H_

#include <bits/wordsize.h>

#include <atomic>
#include <cstring>

#include "absl/base/dynamic_annotations.h"
#include "absl/base/internal/sysinfo.h"
#include "tcmalloc/internal/mincore.h"
#include "tcmalloc/internal/percpu.h"

#if defined(PERCPU_USE_RSEQ) && (!defined(__clang__) || __clang_major__ >= 9)
// asm goto requires the use of Clang 9 or newer:
// https://releases.llvm.org/9.0.0/tools/clang/docs/ReleaseNotes.html#c-language-changes-in-clang
#define PERCPU_USE_RSEQ_ASM_GOTO 1
#else
#define PERCPU_USE_RSEQ_ASM_GOTO 0
#endif

namespace tcmalloc {

struct PerCPUMetadataState {
  size_t virtual_size;
  size_t resident_size;
};

namespace subtle {
namespace percpu {

// Tcmalloc slab for per-cpu caching mode.
// Conceptually it is equivalent to an array of NumClasses PerCpuSlab's,
// and in fallback implementation it is implemented that way. But optimized
// implementation uses more compact layout and provides faster operations.
//
// Methods of this type must only be used in threads where it is known that the
// percpu primitives are available and percpu::IsFast() has previously returned
// 'true'.
//
// The template parameter Shift indicates the number of bits to shift the
// the CPU id in order to get the location of the per-cpu slab. If this
// parameter matches PERCPU_TCMALLOC_FIXED_SLAB_SHIFT as set in
// percpu_intenal.h then the assembly language versions of push/pop batch
// can be used; otherwise batch operations are emulated.
template <size_t Shift, size_t NumClasses>
class TcmallocSlab {
 public:
  TcmallocSlab() {}

  // Init must be called before any other methods.
  // <alloc> is memory allocation callback (e.g. malloc).
  // <capacity> callback returns max capacity for size class <cl>.
  // <lazy> indicates that per-CPU slabs should be populated on demand
  //
  // Initial capacity is 0 for all slabs.
  void Init(void*(alloc)(size_t size), size_t (*capacity)(size_t cl),
            bool lazy);

  // Only may be called if Init(..., lazy = true) was used.
  void InitCPU(int cpu, size_t (*capacity)(size_t cl));

  // For tests.
  void Destroy(void(free)(void*));

  // Number of elements in cpu/cl slab.
  size_t Length(int cpu, size_t cl) const;

  // Number of elements (currently) allowed in cpu/cl slab.
  size_t Capacity(int cpu, size_t cl) const;

  // If running on cpu, increment the cpu/cl slab's capacity to no greater than
  // min(capacity+len, max_cap) and return the increment applied. Otherwise
  // return 0. Note: max_cap must be the same as returned by capacity callback
  // passed to Init.
  size_t Grow(int cpu, size_t cl, size_t len, size_t max_cap);

  // If running on cpu, decrement the cpu/cl slab's capacity to no less than
  // max(capacity-len, 0) and return the actual decrement applied. Otherwise
  // return 0.
  size_t Shrink(int cpu, size_t cl, size_t len);

  // Add an item (which must be non-zero) to the current CPU's slab. Returns
  // true if add succeeds. Otherwise invokes <f> and returns false (assuming
  // that <f> returns negative value).
  bool Push(size_t cl, void* item, OverflowHandler f);

  // Remove an item (LIFO) from the current CPU's slab. If the slab is empty,
  // invokes <f> and returns its result.
  void* Pop(size_t cl, UnderflowHandler f);

  // Add up to <len> items to the current cpu slab from the array located at
  // <batch>. Returns the number of items that were added (possibly 0). All
  // items not added will be returned at the start of <batch>. Items are only
  // not added if there is no space on the current cpu.
  // REQUIRES: len > 0.
  size_t PushBatch(size_t cl, void** batch, size_t len);

  // Pop up to <len> items from the current cpu slab and return them in <batch>.
  // Returns the number of items actually removed.
  // REQUIRES: len > 0.
  size_t PopBatch(size_t cl, void** batch, size_t len);

  // Remove all items (of all classes) from <cpu>'s slab; reset capacity for all
  // classes to zero.  Then, for each sizeclass, invoke
  // DrainHandler(drain_ctx, cl, <items from slab>, <previous slab capacity>);
  //
  // It is invalid to concurrently execute Drain() for the same CPU; calling
  // Push/Pop/Grow/Shrink concurrently (even on the same CPU) is safe.
  typedef void (*DrainHandler)(void* drain_ctx, size_t cl, void** batch,
                               size_t n, size_t cap);
  void Drain(int cpu, void* drain_ctx, DrainHandler f);

  PerCPUMetadataState MetadataMemoryUsage() const;

  // We use a single continuous region of memory for all slabs on all CPUs.
  // This region is split into NumCPUs regions of size kPerCpuMem (256k).
  // First NumClasses words of each CPU region are occupied by slab
  // headers (Header struct). The remaining memory contain slab arrays.
  struct Slabs {
    std::atomic<int64_t> header[NumClasses];
    void* mem[((1ul << Shift) - sizeof(header)) / sizeof(void*)];
  };
  static_assert(sizeof(Slabs) == (1ul << Shift), "Slabs has unexpected size");

 private:
  // Slab header (packed, atomically updated 64-bit).
  struct Header {
    // All values are word offsets from per-CPU region start.
    // The array is [begin, end).
    uint16_t current;
    // Copy of end. Updated by Shrink/Grow, but is not overwritten by Drain.
    uint16_t end_copy;
    // Lock updates only begin and end with a 32-bit write.
    uint16_t begin;
    uint16_t end;

    // Lock is used by Drain to stop concurrent mutations of the Header.
    // Lock sets begin to 0xffff and end to 0, which makes Push and Pop fail
    // regardless of current value.
    bool IsLocked() const;
    void Lock();
  };

  // We cast Header to std::atomic<int64_t>.
  static_assert(sizeof(Header) == sizeof(std::atomic<int64_t>),
                "bad Header size");

  Slabs* slabs_;

  Slabs* CpuMemoryStart(int cpu) const;
  std::atomic<int64_t>* GetHeader(int cpu, size_t cl) const;
  static Header LoadHeader(std::atomic<int64_t>* hdrp);
  static void StoreHeader(std::atomic<int64_t>* hdrp, Header hdr);
  static int CompareAndSwapHeader(int cpu, std::atomic<int64_t>* hdrp,
                                  Header old, Header hdr);
};

template <size_t Shift, size_t NumClasses>
inline size_t TcmallocSlab<Shift, NumClasses>::Length(int cpu,
                                                      size_t cl) const {
  Header hdr = LoadHeader(GetHeader(cpu, cl));
  return hdr.IsLocked() ? 0 : hdr.current - hdr.begin;
}

template <size_t Shift, size_t NumClasses>
inline size_t TcmallocSlab<Shift, NumClasses>::Capacity(int cpu,
                                                        size_t cl) const {
  Header hdr = LoadHeader(GetHeader(cpu, cl));
  return hdr.IsLocked() ? 0 : hdr.end - hdr.begin;
}

template <size_t Shift, size_t NumClasses>
inline size_t TcmallocSlab<Shift, NumClasses>::Grow(int cpu, size_t cl,
                                                    size_t len,
                                                    size_t max_cap) {
  std::atomic<int64_t>* hdrp = GetHeader(cpu, cl);
  for (;;) {
    Header old = LoadHeader(hdrp);
    if (old.IsLocked() || old.end - old.begin == max_cap) {
      return 0;
    }
    uint16_t n = std::min<uint16_t>(len, max_cap - (old.end - old.begin));
    Header hdr = old;
    hdr.end += n;
    hdr.end_copy += n;
    const int ret = CompareAndSwapHeader(cpu, hdrp, old, hdr);
    if (ret == cpu) {
      return n;
    } else if (ret >= 0) {
      return 0;
    }
  }
}

template <size_t Shift, size_t NumClasses>
inline size_t TcmallocSlab<Shift, NumClasses>::Shrink(int cpu, size_t cl,
                                                      size_t len) {
  std::atomic<int64_t>* hdrp = GetHeader(cpu, cl);
  for (;;) {
    Header old = LoadHeader(hdrp);
    if (old.IsLocked() || old.current == old.end) {
      return 0;
    }
    uint16_t n = std::min<uint16_t>(len, old.end - old.current);
    Header hdr = old;
    hdr.end -= n;
    hdr.end_copy -= n;
    const int ret = CompareAndSwapHeader(cpu, hdrp, old, hdr);
    if (ret == cpu) {
      return n;
    } else if (ret >= 0) {
      return 0;
    }
  }
}

#define PERCPU_XSTRINGIFY(s) #s
#define PERCPU_STRINGIFY(s) PERCPU_XSTRINGIFY(s)

#if PERCPU_USE_RSEQ_ASM_GOTO && defined(__x86_64__)
template <size_t Shift, size_t NumClasses>
static inline ABSL_ATTRIBUTE_ALWAYS_INLINE int TcmallocSlab_Push(
    typename TcmallocSlab<Shift, NumClasses>::Slabs* slabs, size_t cl,
    void* item, OverflowHandler f) {
  asm goto(
      // TODO(b/141629158):  __rseq_cs only needs to be writeable to allow for
      // relocations, but could be read-only for non-PIE builds.
      ".pushsection __rseq_cs, \"aw?\"\n"
      ".balign 32\n"
      ".local __rseq_cs_TcmallocSlab_Push_%=\n"
      ".type __rseq_cs_TcmallocSlab_Push_%=,@object\n"
      ".size __rseq_cs_TcmallocSlab_Push_%=,32\n"
      "__rseq_cs_TcmallocSlab_Push_%=:\n"
      ".long 0x0\n"
      ".long 0x0\n"
      ".quad 4f\n"
      ".quad 5f - 4f\n"
      ".quad 2f\n"
      ".popsection\n"
#if !defined(__clang_major__) || __clang_major__ >= 9
      ".reloc 0, R_X86_64_NONE, 1f\n"
#endif
      ".pushsection __rseq_cs_ptr_array, \"aw?\"\n"
      "1:\n"
      ".balign 8;"
      ".quad __rseq_cs_TcmallocSlab_Push_%=\n"
      // Force this section to be retained.  It is for debugging, but is
      // otherwise not referenced.
      ".popsection\n"
      ".pushsection .text.unlikely, \"ax?\"\n"
      ".byte 0x0f, 0x1f, 0x05\n"
      ".long %c[rseq_sig]\n"
      ".local TcmallocSlab_Push_trampoline_%=\n"
      ".type TcmallocSlab_Push_trampoline_%=,@function\n"
      "TcmallocSlab_Push_trampoline_%=:\n"
      "2:\n"
      "jmp 3f\n"
      ".popsection\n"
      // Prepare
      //
      // TODO(b/151503411):  Pending widespread availability of LLVM's asm goto
      // with output contraints
      // (https://github.com/llvm/llvm-project/commit/23c2a5ce33f0), we can
      // return the register allocations to the compiler rather than using
      // explicit clobbers.  Prior to this, blocks which use asm goto cannot
      // also specify outputs.
      //
      // r10: Scratch
      // r11: Current
      "3:\n"
      "lea __rseq_cs_TcmallocSlab_Push_%=(%%rip), %%r10\n"
      "mov %%r10, (%[rseq_cs])\n"
      // Start
      "4:\n"
      // r10 = __rseq_abi.cpu_id;
      "mov (%[rseq_cpu]), %%r10d\n"
      // r10 = slabs + r10
      "shl %[shift], %%r10\n"
      "add %[slabs], %%r10\n"
      // r11 = slabs->current;
      "movzwq (%%r10, %[cl], 8), %%r11\n"
      // if (ABSL_PREDICT_FALSE(r11 >= slabs->end)) { goto overflow; }
      "cmp 6(%%r10, %[cl], 8), %%r11w\n"
      "jae %l[overflow_label]\n"
      "mov %[item], (%%r10, %%r11, 8)\n"
      "inc %%r11\n"
      "mov %%r11w, (%%r10, %[cl], 8)\n"
      // Commit
      "5:\n"
      :
      : [rseq_cs] "r"(&__rseq_abi.rseq_cs), [rseq_cpu] "r"(&__rseq_abi.cpu_id),
        [rseq_sig] "in"(PERCPU_RSEQ_SIGNATURE), [shift] "in"(Shift),
        [slabs] "r"(slabs), [cl] "r"(cl), [item] "r"(item)
      : "cc", "memory", "r10", "r11"
      : overflow_label);
  return 0;
overflow_label:
  // As of 3/2020, LLVM's asm goto (even with output constraints) only provides
  // values for the fallthrough path.  The values on the taken branches are
  // undefined.
  int cpu = __rseq_abi.cpu_id;
  return f(cpu, cl, item);<commit_msg>Revert: Leverage asm goto for delete fast path.<commit_after>// Copyright 2019 The TCMalloc Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#ifndef TCMALLOC_PERCPU_TCMALLOC_H_
#define TCMALLOC_PERCPU_TCMALLOC_H_

#include <bits/wordsize.h>

#include <atomic>
#include <cstring>

#include "absl/base/dynamic_annotations.h"
#include "absl/base/internal/sysinfo.h"
#include "tcmalloc/internal/mincore.h"
#include "tcmalloc/internal/percpu.h"

namespace tcmalloc {

struct PerCPUMetadataState {
  size_t virtual_size;
  size_t resident_size;
};

namespace subtle {
namespace percpu {

// Tcmalloc slab for per-cpu caching mode.
// Conceptually it is equivalent to an array of NumClasses PerCpuSlab's,
// and in fallback implementation it is implemented that way. But optimized
// implementation uses more compact layout and provides faster operations.
//
// Methods of this type must only be used in threads where it is known that the
// percpu primitives are available and percpu::IsFast() has previously returned
// 'true'.
//
// The template parameter Shift indicates the number of bits to shift the
// the CPU id in order to get the location of the per-cpu slab. If this
// parameter matches PERCPU_TCMALLOC_FIXED_SLAB_SHIFT as set in
// percpu_intenal.h then the assembly language versions of push/pop batch
// can be used; otherwise batch operations are emulated.
template <size_t Shift, size_t NumClasses>
class TcmallocSlab {
 public:
  TcmallocSlab() {}

  // Init must be called before any other methods.
  // <alloc> is memory allocation callback (e.g. malloc).
  // <capacity> callback returns max capacity for size class <cl>.
  // <lazy> indicates that per-CPU slabs should be populated on demand
  //
  // Initial capacity is 0 for all slabs.
  void Init(void*(alloc)(size_t size), size_t (*capacity)(size_t cl),
            bool lazy);

  // Only may be called if Init(..., lazy = true) was used.
  void InitCPU(int cpu, size_t (*capacity)(size_t cl));

  // For tests.
  void Destroy(void(free)(void*));

  // Number of elements in cpu/cl slab.
  size_t Length(int cpu, size_t cl) const;

  // Number of elements (currently) allowed in cpu/cl slab.
  size_t Capacity(int cpu, size_t cl) const;

  // If running on cpu, increment the cpu/cl slab's capacity to no greater than
  // min(capacity+len, max_cap) and return the increment applied. Otherwise
  // return 0. Note: max_cap must be the same as returned by capacity callback
  // passed to Init.
  size_t Grow(int cpu, size_t cl, size_t len, size_t max_cap);

  // If running on cpu, decrement the cpu/cl slab's capacity to no less than
  // max(capacity-len, 0) and return the actual decrement applied. Otherwise
  // return 0.
  size_t Shrink(int cpu, size_t cl, size_t len);

  // Add an item (which must be non-zero) to the current CPU's slab. Returns
  // true if add succeeds. Otherwise invokes <f> and returns false (assuming
  // that <f> returns negative value).
  bool Push(size_t cl, void* item, OverflowHandler f);

  // Remove an item (LIFO) from the current CPU's slab. If the slab is empty,
  // invokes <f> and returns its result.
  void* Pop(size_t cl, UnderflowHandler f);

  // Add up to <len> items to the current cpu slab from the array located at
  // <batch>. Returns the number of items that were added (possibly 0). All
  // items not added will be returned at the start of <batch>. Items are only
  // not added if there is no space on the current cpu.
  // REQUIRES: len > 0.
  size_t PushBatch(size_t cl, void** batch, size_t len);

  // Pop up to <len> items from the current cpu slab and return them in <batch>.
  // Returns the number of items actually removed.
  // REQUIRES: len > 0.
  size_t PopBatch(size_t cl, void** batch, size_t len);

  // Remove all items (of all classes) from <cpu>'s slab; reset capacity for all
  // classes to zero.  Then, for each sizeclass, invoke
  // DrainHandler(drain_ctx, cl, <items from slab>, <previous slab capacity>);
  //
  // It is invalid to concurrently execute Drain() for the same CPU; calling
  // Push/Pop/Grow/Shrink concurrently (even on the same CPU) is safe.
  typedef void (*DrainHandler)(void* drain_ctx, size_t cl, void** batch,
                               size_t n, size_t cap);
  void Drain(int cpu, void* drain_ctx, DrainHandler f);

  PerCPUMetadataState MetadataMemoryUsage() const;

  // We use a single continuous region of memory for all slabs on all CPUs.
  // This region is split into NumCPUs regions of size kPerCpuMem (256k).
  // First NumClasses words of each CPU region are occupied by slab
  // headers (Header struct). The remaining memory contain slab arrays.
  struct Slabs {
    std::atomic<int64_t> header[NumClasses];
    void* mem[((1ul << Shift) - sizeof(header)) / sizeof(void*)];
  };
  static_assert(sizeof(Slabs) == (1ul << Shift), "Slabs has unexpected size");

 private:
  // Slab header (packed, atomically updated 64-bit).
  struct Header {
    // All values are word offsets from per-CPU region start.
    // The array is [begin, end).
    uint16_t current;
    // Copy of end. Updated by Shrink/Grow, but is not overwritten by Drain.
    uint16_t end_copy;
    // Lock updates only begin and end with a 32-bit write.
    uint16_t begin;
    uint16_t end;

    // Lock is used by Drain to stop concurrent mutations of the Header.
    // Lock sets begin to 0xffff and end to 0, which makes Push and Pop fail
    // regardless of current value.
    bool IsLocked() const;
    void Lock();
  };

  // We cast Header to std::atomic<int64_t>.
  static_assert(sizeof(Header) == sizeof(std::atomic<int64_t>),
                "bad Header size");

  Slabs* slabs_;

  Slabs* CpuMemoryStart(int cpu) const;
  std::atomic<int64_t>* GetHeader(int cpu, size_t cl) const;
  static Header LoadHeader(std::atomic<int64_t>* hdrp);
  static void StoreHeader(std::atomic<int64_t>* hdrp, Header hdr);
  static int CompareAndSwapHeader(int cpu, std::atomic<int64_t>* hdrp,
                                  Header old, Header hdr);
};

template <size_t Shift, size_t NumClasses>
inline size_t TcmallocSlab<Shift, NumClasses>::Length(int cpu,
                                                      size_t cl) const {
  Header hdr = LoadHeader(GetHeader(cpu, cl));
  return hdr.IsLocked() ? 0 : hdr.current - hdr.begin;
}

template <size_t Shift, size_t NumClasses>
inline size_t TcmallocSlab<Shift, NumClasses>::Capacity(int cpu,
                                                        size_t cl) const {
  Header hdr = LoadHeader(GetHeader(cpu, cl));
  return hdr.IsLocked() ? 0 : hdr.end - hdr.begin;
}

template <size_t Shift, size_t NumClasses>
inline size_t TcmallocSlab<Shift, NumClasses>::Grow(int cpu, size_t cl,
                                                    size_t len,
                                                    size_t max_cap) {
  std::atomic<int64_t>* hdrp = GetHeader(cpu, cl);
  for (;;) {
    Header old = LoadHeader(hdrp);
    if (old.IsLocked() || old.end - old.begin == max_cap) {
      return 0;
    }
    uint16_t n = std::min<uint16_t>(len, max_cap - (old.end - old.begin));
    Header hdr = old;
    hdr.end += n;
    hdr.end_copy += n;
    const int ret = CompareAndSwapHeader(cpu, hdrp, old, hdr);
    if (ret == cpu) {
      return n;
    } else if (ret >= 0) {
      return 0;
    }
  }
}

template <size_t Shift, size_t NumClasses>
inline size_t TcmallocSlab<Shift, NumClasses>::Shrink(int cpu, size_t cl,
                                                      size_t len) {
  std::atomic<int64_t>* hdrp = GetHeader(cpu, cl);
  for (;;) {
    Header old = LoadHeader(hdrp);
    if (old.IsLocked() || old.current == old.end) {
      return 0;
    }
    uint16_t n = std::min<uint16_t>(len, old.end - old.current);
    Header hdr = old;
    hdr.end -= n;
    hdr.end_copy -= n;
    const int ret = CompareAndSwapHeader(cpu, hdrp, old, hdr);
    if (ret == cpu) {
      return n;
    } else if (ret >= 0) {
      return 0;
    }
  }
}

#define PERCPU_XSTRINGIFY(s) #s
#define PERCPU_STRINGIFY(s) PERCPU_XSTRINGIFY(s)

#if defined(__x86_64__)
template <size_t Shift, size_t NumClasses>
static inline ABSL_ATTRIBUTE_ALWAYS_INLINE int TcmallocSlab_Push(
    typename TcmallocSlab<Shift, NumClasses>::Slabs* slabs, size_t cl,
    void* item, OverflowHandler f) {
  // TODO(b/149467541):  Move this to asm goto.
  uint64_t scratch, current;
  bool overflow;
  asm volatile(
      // TODO(b/141629158):  __rseq_cs only needs to be writeable to allow for
      // relocations, but could be read-only for non-PIE builds.
      ".pushsection __rseq_cs, \"aw?\"\n"
      ".balign 32\n"
      ".local __rseq_cs_TcmallocSlab_Push_%=\n"
      ".type __rseq_cs_TcmallocSlab_Push_%=,@object\n"
      ".size __rseq_cs_TcmallocSlab_Push_%=,32\n"
      "__rseq_cs_TcmallocSlab_Push_%=:\n"
      ".long 0x0\n"
      ".long 0x0\n"
      ".quad 4f\n"
      ".quad 5f - 4f\n"
      ".quad 2f\n"
      ".popsection\n"
#if !defined(__clang_major__) || __clang_major__ >= 9
      ".reloc 0, R_X86_64_NONE, 1f\n"
#endif
      ".pushsection __rseq_cs_ptr_array, \"aw?\"\n"
      "1:\n"
      ".balign 8;"
      ".quad __rseq_cs_TcmallocSlab_Push_%=\n"
      // Force this section to be retained.  It is for debugging, but is
      // otherwise not referenced.
      ".popsection\n"
      ".pushsection .text.unlikely, \"ax?\"\n"
      ".byte 0x0f, 0x1f, 0x05\n"
      ".long %c[rseq_sig]\n"
      ".local TcmallocSlab_Push_trampoline_%=\n"
      ".type TcmallocSlab_Push_trampoline_%=,@function\n"
      "TcmallocSlab_Push_trampoline_%=:\n"
      "2:\n"
      "jmp 3f\n"
      ".popsection\n"
      // Prepare
      "3:\n"
      "lea __rseq_cs_TcmallocSlab_Push_%=(%%rip), %[scratch]\n"
      "mov %[scratch], (%[rseq_cs])\n"
      // Start
      "4:\n"
      // scratch = __rseq_abi.cpu_id;
      "mov (%[rseq_cpu]), %k[scratch]\n"
      // scratch = slabs + scratch
      "shl %[shift], %[scratch]\n"
      "add %[slabs], %[scratch]\n"
      // r11 = slabs->current;
      "movzwq (%[scratch], %[cl], 8), %[current]\n"
      // if (ABSL_PREDICT_FALSE(r11 >= slabs->end)) { goto overflow; }
      "cmp 6(%[scratch], %[cl], 8), %w[current]\n"
      "setae %b[overflow]\n"
      "jae 5f\n"
      "mov %[item], (%[scratch], %[current], 8)\n"
      "inc %[current]\n"
      "mov %w[current], (%[scratch], %[cl], 8)\n"
      // Commit
      "5:\n"
      : [ current ] "=&r"(current), [ scratch ] "=&r"(scratch),
        [ overflow ] "=&r"(overflow)
      : [ rseq_cs ] "r"(&__rseq_abi.rseq_cs),
        [ rseq_cpu ] "r"(&__rseq_abi.cpu_id),
        [ rseq_sig ] "in"(PERCPU_RSEQ_SIGNATURE), [ shift ] "in"(Shift),
        [ slabs ] "r"(slabs), [ cl ] "r"(cl), [ item ] "r"(item)
      : "cc", "memory");
  // Undo transformation of cpu_id to the value of scratch.
  int cpu = reinterpret_cast<typename TcmallocSlab<Shift, NumClasses>::Slabs*>(
                scratch) -
            slabs;
  if (ABSL_PREDICT_FALSE(overflow)) {
    return f(cpu, cl, item);
  }
  return cpu;
{'fraction_non_alphanumeric': 0.12080536912751678, 'fraction_numerical': 0.014064036250160304, 'mean_word_length': 2.81320293398533, 'pattern_counts': {'":': 0, '<': 93, '<?xml version=': 0, '>': 102, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3681958', 'n_tokens_mistral': 8186, 'n_tokens_neox': 7755, 'n_words': 2527}