<commit_before>//===--- OperandOwnership.cpp ---------------------------------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//

#include "swift/SIL/ApplySite.h"
#include "swift/SIL/OwnershipUtils.h"
#include "swift/SIL/SILBuiltinVisitor.h"
#include "swift/SIL/SILInstruction.h"
#include "swift/SIL/SILValue.h"
#include "swift/SIL/SILVisitor.h"

using namespace swift;

//===----------------------------------------------------------------------===//
//                      OperandOwnershipKindClassifier
//===----------------------------------------------------------------------===//

namespace {

class OperandOwnershipKindClassifier
    : public SILInstructionVisitor<OperandOwnershipKindClassifier,
                                   OperandOwnershipKindMap> {
public:
  using Map = OperandOwnershipKindMap;

private:
  LLVM_ATTRIBUTE_UNUSED SILModule &mod;

  const Operand &op;
  bool checkingSubObject;

public:
  /// Create a new OperandOwnershipKindClassifier.
  ///
  /// In most cases, one should only pass in \p Op and \p BaseValue will be set
  /// to Op.get(). In cases where one is trying to verify subobjects, Op.get()
  /// should be the subobject and Value should be the parent object. An example
  /// of where one would want to do this is in the case of value projections
  /// like struct_extract.
  OperandOwnershipKindClassifier(
      SILModule &mod, const Operand &op,
      bool checkingSubObject)
      : mod(mod), op(op),
        checkingSubObject(checkingSubObject) {}

  bool isCheckingSubObject() const { return checkingSubObject; }

  SILValue getValue() const { return op.get(); }

  ValueOwnershipKind getOwnershipKind() const {
    assert(getValue().getOwnershipKind() == op.get().getOwnershipKind() &&
           "Expected ownership kind of parent value and operand");
    return getValue().getOwnershipKind();
  }

  unsigned getOperandIndex() const { return op.getOperandNumber(); }

  SILType getType() const { return op.get()->getType(); }

  bool compatibleWithOwnership(ValueOwnershipKind kind) const {
    return getOwnershipKind().isCompatibleWith(kind);
  }

  bool hasExactOwnership(ValueOwnershipKind kind) const {
    return getOwnershipKind() == kind;
  }

  bool isAddressOrTrivialType() const {
    if (getType().isAddress())
      return true;
    return getOwnershipKind() == ValueOwnershipKind::None;
  }

  OperandOwnershipKindMap visitForwardingInst(SILInstruction *i,
                                              ArrayRef<Operand> ops);
  OperandOwnershipKindMap visitForwardingInst(SILInstruction *i) {
    return visitForwardingInst(i, i->getAllOperands());
  }

  OperandOwnershipKindMap
  visitApplyParameter(ValueOwnershipKind requiredConvention,
                      UseLifetimeConstraint requirement);
  OperandOwnershipKindMap visitFullApply(FullApplySite apply);

  OperandOwnershipKindMap visitCallee(CanSILFunctionType substCalleeType);
  OperandOwnershipKindMap
  checkTerminatorArgumentMatchesDestBB(SILBasicBlock *destBB, unsigned opIndex);

// Create declarations for all instructions, so we get a warning at compile
// time if any instructions do not have an implementation.
#define INST(Id, Parent) OperandOwnershipKindMap visit##Id(Id *);
#include "swift/SIL/SILNodes.def"
};

} // end anonymous namespace

/// Implementation for instructions that we should never visit since they are
/// not valid in ossa or do not have operands. Since we should never visit
/// these, we just assert.
#define SHOULD_NEVER_VISIT_INST(INST)                                          \
  OperandOwnershipKindMap OperandOwnershipKindClassifier::visit##INST##Inst(   \
      INST##Inst *i) {                                                         \
    llvm::errs() << "Unhandled inst: " << *i;                                  \
    llvm::report_fatal_error(                                                  \
        "Visited instruction that should never be visited?!");                 \
  }
SHOULD_NEVER_VISIT_INST(AllocBox)
SHOULD_NEVER_VISIT_INST(AllocExistentialBox)
SHOULD_NEVER_VISIT_INST(AllocGlobal)
SHOULD_NEVER_VISIT_INST(AllocStack)
SHOULD_NEVER_VISIT_INST(DifferentiabilityWitnessFunction)
SHOULD_NEVER_VISIT_INST(FloatLiteral)
SHOULD_NEVER_VISIT_INST(FunctionRef)
SHOULD_NEVER_VISIT_INST(DynamicFunctionRef)
SHOULD_NEVER_VISIT_INST(PreviousDynamicFunctionRef)
SHOULD_NEVER_VISIT_INST(GlobalAddr)
SHOULD_NEVER_VISIT_INST(GlobalValue)
SHOULD_NEVER_VISIT_INST(BaseAddrForOffset)
SHOULD_NEVER_VISIT_INST(IntegerLiteral)
SHOULD_NEVER_VISIT_INST(Metatype)
SHOULD_NEVER_VISIT_INST(ObjCProtocol)
SHOULD_NEVER_VISIT_INST(RetainValue)
SHOULD_NEVER_VISIT_INST(RetainValueAddr)
SHOULD_NEVER_VISIT_INST(StringLiteral)
SHOULD_NEVER_VISIT_INST(StrongRetain)
SHOULD_NEVER_VISIT_INST(Unreachable)
SHOULD_NEVER_VISIT_INST(Unwind)
SHOULD_NEVER_VISIT_INST(ReleaseValue)
SHOULD_NEVER_VISIT_INST(ReleaseValueAddr)
SHOULD_NEVER_VISIT_INST(StrongRelease)
SHOULD_NEVER_VISIT_INST(GetAsyncContinuation)

#define ALWAYS_OR_SOMETIMES_LOADABLE_CHECKED_REF_STORAGE(Name, ...)            \
  SHOULD_NEVER_VISIT_INST(StrongRetain##Name)                                  \
  SHOULD_NEVER_VISIT_INST(Name##Retain)
#include "swift/AST/ReferenceStorage.def"
#undef SHOULD_NEVER_VISIT_INST

/// Instructions that are interior pointers into a guaranteed value.
#define INTERIOR_POINTER_PROJECTION(INST)                                      \
  OperandOwnershipKindMap OperandOwnershipKindClassifier::visit##INST##Inst(   \
      INST##Inst *i) {                                                         \
    assert(i->getNumOperands() && "Expected to have non-zero operands");       \
    return Map::compatibilityMap(ValueOwnershipKind::Guaranteed,               \
                                 UseLifetimeConstraint::MustBeLive);           \
  }
INTERIOR_POINTER_PROJECTION(RefElementAddr)
INTERIOR_POINTER_PROJECTION(RefTailAddr)
#undef INTERIOR_POINTER_PROJECTION

/// Instructions whose arguments are always compatible with one convention.
#define CONSTANT_OWNERSHIP_INST(OWNERSHIP, USE_LIFETIME_CONSTRAINT, INST)      \
  OperandOwnershipKindMap OperandOwnershipKindClassifier::visit##INST##Inst(   \
      INST##Inst *i) {                                                         \
    assert(i->getNumOperands() && "Expected to have non-zero operands");       \
    return Map::compatibilityMap(                                              \
        ValueOwnershipKind::OWNERSHIP,                                         \
        UseLifetimeConstraint::USE_LIFETIME_CONSTRAINT);                       \
  }
CONSTANT_OWNERSHIP_INST(Guaranteed, MustBeLive, OpenExistentialValue)
CONSTANT_OWNERSHIP_INST(Guaranteed, MustBeLive, OpenExistentialBoxValue)
CONSTANT_OWNERSHIP_INST(Guaranteed, MustBeLive, OpenExistentialBox)
CONSTANT_OWNERSHIP_INST(Owned, MustBeInvalidated, AutoreleaseValue)
CONSTANT_OWNERSHIP_INST(Owned, MustBeInvalidated, DeallocBox)
CONSTANT_OWNERSHIP_INST(Owned, MustBeInvalidated, DeallocExistentialBox)
CONSTANT_OWNERSHIP_INST(Owned, MustBeInvalidated, DeallocRef)
CONSTANT_OWNERSHIP_INST(Owned, MustBeInvalidated, DestroyValue)
CONSTANT_OWNERSHIP_INST(Owned, MustBeInvalidated, EndLifetime)
CONSTANT_OWNERSHIP_INST(Owned, MustBeInvalidated, BeginCOWMutation)
CONSTANT_OWNERSHIP_INST(Owned, MustBeInvalidated, EndCOWMutation)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, AwaitAsyncContinuation)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, AbortApply)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, AddressToPointer)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, BeginAccess)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, BeginUnpairedAccess)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, BindMemory)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, CheckedCastAddrBranch)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, CondFail)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, CopyAddr)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, DeallocStack)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, DebugValueAddr)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, DeinitExistentialAddr)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, DestroyAddr)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, EndAccess)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, EndApply)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, EndUnpairedAccess)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, GetAsyncContinuationAddr)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, HopToExecutor)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, IndexAddr)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, IndexRawPointer)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, InitBlockStorageHeader)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, InitEnumDataAddr)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, InitExistentialAddr)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, InitExistentialMetatype)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, InjectEnumAddr)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, IsUnique)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, Load)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, LoadBorrow)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, MarkFunctionEscape)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, ObjCExistentialMetatypeToObject)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, ObjCMetatypeToObject)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, ObjCToThickMetatype)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, OpenExistentialAddr)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, OpenExistentialMetatype)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, PointerToAddress)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, PointerToThinFunction)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, ProjectBlockStorage)<commit_msg>[concurrency] SIL: fix operand ownership of hop_to_executor<commit_after>//===--- OperandOwnership.cpp ---------------------------------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//

#include "swift/SIL/ApplySite.h"
#include "swift/SIL/OwnershipUtils.h"
#include "swift/SIL/SILBuiltinVisitor.h"
#include "swift/SIL/SILInstruction.h"
#include "swift/SIL/SILValue.h"
#include "swift/SIL/SILVisitor.h"

using namespace swift;

//===----------------------------------------------------------------------===//
//                      OperandOwnershipKindClassifier
//===----------------------------------------------------------------------===//

namespace {

class OperandOwnershipKindClassifier
    : public SILInstructionVisitor<OperandOwnershipKindClassifier,
                                   OperandOwnershipKindMap> {
public:
  using Map = OperandOwnershipKindMap;

private:
  LLVM_ATTRIBUTE_UNUSED SILModule &mod;

  const Operand &op;
  bool checkingSubObject;

public:
  /// Create a new OperandOwnershipKindClassifier.
  ///
  /// In most cases, one should only pass in \p Op and \p BaseValue will be set
  /// to Op.get(). In cases where one is trying to verify subobjects, Op.get()
  /// should be the subobject and Value should be the parent object. An example
  /// of where one would want to do this is in the case of value projections
  /// like struct_extract.
  OperandOwnershipKindClassifier(
      SILModule &mod, const Operand &op,
      bool checkingSubObject)
      : mod(mod), op(op),
        checkingSubObject(checkingSubObject) {}

  bool isCheckingSubObject() const { return checkingSubObject; }

  SILValue getValue() const { return op.get(); }

  ValueOwnershipKind getOwnershipKind() const {
    assert(getValue().getOwnershipKind() == op.get().getOwnershipKind() &&
           "Expected ownership kind of parent value and operand");
    return getValue().getOwnershipKind();
  }

  unsigned getOperandIndex() const { return op.getOperandNumber(); }

  SILType getType() const { return op.get()->getType(); }

  bool compatibleWithOwnership(ValueOwnershipKind kind) const {
    return getOwnershipKind().isCompatibleWith(kind);
  }

  bool hasExactOwnership(ValueOwnershipKind kind) const {
    return getOwnershipKind() == kind;
  }

  bool isAddressOrTrivialType() const {
    if (getType().isAddress())
      return true;
    return getOwnershipKind() == ValueOwnershipKind::None;
  }

  OperandOwnershipKindMap visitForwardingInst(SILInstruction *i,
                                              ArrayRef<Operand> ops);
  OperandOwnershipKindMap visitForwardingInst(SILInstruction *i) {
    return visitForwardingInst(i, i->getAllOperands());
  }

  OperandOwnershipKindMap
  visitApplyParameter(ValueOwnershipKind requiredConvention,
                      UseLifetimeConstraint requirement);
  OperandOwnershipKindMap visitFullApply(FullApplySite apply);

  OperandOwnershipKindMap visitCallee(CanSILFunctionType substCalleeType);
  OperandOwnershipKindMap
  checkTerminatorArgumentMatchesDestBB(SILBasicBlock *destBB, unsigned opIndex);

// Create declarations for all instructions, so we get a warning at compile
// time if any instructions do not have an implementation.
#define INST(Id, Parent) OperandOwnershipKindMap visit##Id(Id *);
#include "swift/SIL/SILNodes.def"
};

} // end anonymous namespace

/// Implementation for instructions that we should never visit since they are
/// not valid in ossa or do not have operands. Since we should never visit
/// these, we just assert.
#define SHOULD_NEVER_VISIT_INST(INST)                                          \
  OperandOwnershipKindMap OperandOwnershipKindClassifier::visit##INST##Inst(   \
      INST##Inst *i) {                                                         \
    llvm::errs() << "Unhandled inst: " << *i;                                  \
    llvm::report_fatal_error(                                                  \
        "Visited instruction that should never be visited?!");                 \
  }
SHOULD_NEVER_VISIT_INST(AllocBox)
SHOULD_NEVER_VISIT_INST(AllocExistentialBox)
SHOULD_NEVER_VISIT_INST(AllocGlobal)
SHOULD_NEVER_VISIT_INST(AllocStack)
SHOULD_NEVER_VISIT_INST(DifferentiabilityWitnessFunction)
SHOULD_NEVER_VISIT_INST(FloatLiteral)
SHOULD_NEVER_VISIT_INST(FunctionRef)
SHOULD_NEVER_VISIT_INST(DynamicFunctionRef)
SHOULD_NEVER_VISIT_INST(PreviousDynamicFunctionRef)
SHOULD_NEVER_VISIT_INST(GlobalAddr)
SHOULD_NEVER_VISIT_INST(GlobalValue)
SHOULD_NEVER_VISIT_INST(BaseAddrForOffset)
SHOULD_NEVER_VISIT_INST(IntegerLiteral)
SHOULD_NEVER_VISIT_INST(Metatype)
SHOULD_NEVER_VISIT_INST(ObjCProtocol)
SHOULD_NEVER_VISIT_INST(RetainValue)
SHOULD_NEVER_VISIT_INST(RetainValueAddr)
SHOULD_NEVER_VISIT_INST(StringLiteral)
SHOULD_NEVER_VISIT_INST(StrongRetain)
SHOULD_NEVER_VISIT_INST(Unreachable)
SHOULD_NEVER_VISIT_INST(Unwind)
SHOULD_NEVER_VISIT_INST(ReleaseValue)
SHOULD_NEVER_VISIT_INST(ReleaseValueAddr)
SHOULD_NEVER_VISIT_INST(StrongRelease)
SHOULD_NEVER_VISIT_INST(GetAsyncContinuation)

#define ALWAYS_OR_SOMETIMES_LOADABLE_CHECKED_REF_STORAGE(Name, ...)            \
  SHOULD_NEVER_VISIT_INST(StrongRetain##Name)                                  \
  SHOULD_NEVER_VISIT_INST(Name##Retain)
#include "swift/AST/ReferenceStorage.def"
#undef SHOULD_NEVER_VISIT_INST

/// Instructions that are interior pointers into a guaranteed value.
#define INTERIOR_POINTER_PROJECTION(INST)                                      \
  OperandOwnershipKindMap OperandOwnershipKindClassifier::visit##INST##Inst(   \
      INST##Inst *i) {                                                         \
    assert(i->getNumOperands() && "Expected to have non-zero operands");       \
    return Map::compatibilityMap(ValueOwnershipKind::Guaranteed,               \
                                 UseLifetimeConstraint::MustBeLive);           \
  }
INTERIOR_POINTER_PROJECTION(RefElementAddr)
INTERIOR_POINTER_PROJECTION(RefTailAddr)
#undef INTERIOR_POINTER_PROJECTION

/// Instructions whose arguments are always compatible with one convention.
#define CONSTANT_OWNERSHIP_INST(OWNERSHIP, USE_LIFETIME_CONSTRAINT, INST)      \
  OperandOwnershipKindMap OperandOwnershipKindClassifier::visit##INST##Inst(   \
      INST##Inst *i) {                                                         \
    assert(i->getNumOperands() && "Expected to have non-zero operands");       \
    return Map::compatibilityMap(                                              \
        ValueOwnershipKind::OWNERSHIP,                                         \
        UseLifetimeConstraint::USE_LIFETIME_CONSTRAINT);                       \
  }
CONSTANT_OWNERSHIP_INST(Guaranteed, MustBeLive, OpenExistentialValue)
CONSTANT_OWNERSHIP_INST(Guaranteed, MustBeLive, OpenExistentialBoxValue)
CONSTANT_OWNERSHIP_INST(Guaranteed, MustBeLive, OpenExistentialBox)
CONSTANT_OWNERSHIP_INST(Guaranteed, MustBeLive, HopToExecutor)
CONSTANT_OWNERSHIP_INST(Owned, MustBeInvalidated, AutoreleaseValue)
CONSTANT_OWNERSHIP_INST(Owned, MustBeInvalidated, DeallocBox)
CONSTANT_OWNERSHIP_INST(Owned, MustBeInvalidated, DeallocExistentialBox)
CONSTANT_OWNERSHIP_INST(Owned, MustBeInvalidated, DeallocRef)
CONSTANT_OWNERSHIP_INST(Owned, MustBeInvalidated, DestroyValue)
CONSTANT_OWNERSHIP_INST(Owned, MustBeInvalidated, EndLifetime)
CONSTANT_OWNERSHIP_INST(Owned, MustBeInvalidated, BeginCOWMutation)
CONSTANT_OWNERSHIP_INST(Owned, MustBeInvalidated, EndCOWMutation)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, AwaitAsyncContinuation)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, AbortApply)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, AddressToPointer)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, BeginAccess)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, BeginUnpairedAccess)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, BindMemory)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, CheckedCastAddrBranch)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, CondFail)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, CopyAddr)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, DeallocStack)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, DebugValueAddr)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, DeinitExistentialAddr)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, DestroyAddr)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, EndAccess)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, EndApply)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, EndUnpairedAccess)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, GetAsyncContinuationAddr)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, IndexAddr)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, IndexRawPointer)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, InitBlockStorageHeader)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, InitEnumDataAddr)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, InitExistentialAddr)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, InitExistentialMetatype)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, InjectEnumAddr)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, IsUnique)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, Load)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, LoadBorrow)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, MarkFunctionEscape)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, ObjCExistentialMetatypeToObject)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, ObjCMetatypeToObject)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, ObjCToThickMetatype)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, OpenExistentialAddr)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, OpenExistentialMetatype)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, PointerToAddress)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, PointerToThinFunction)
CONSTANT_OWNERSHIP_INST(None, MustBeLive, ProjectBlockStorage)
{'fraction_non_alphanumeric': 0.1041122059119244, 'fraction_numerical': 0.0010054293183189222, 'mean_word_length': 3.9203561711600297, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 36, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6516509', 'n_tokens_mistral': 6401, 'n_tokens_neox': 6067, 'n_words': 1107}