<commit_before>// Copyright 2012 <NAME> <<EMAIL>>
// This program is free software licensed under the MIT license.

#include <stdarg.h>
#include <stdlib.h>
#include <string.h>
#include "8cc.h"
#include "list.h"

static char *REGS[] = {"rdi", "rsi", "rdx", "rcx", "r8", "r9"};
static char *SREGS[] = {"dil", "sil", "dl", "cl", "r8b", "r9b"};
static char *MREGS[] = {"edi", "esi", "edx", "ecx", "r8d", "r9d"};
static int TAB = 8;
static List *functions = &EMPTY_LIST;
static char *lbreak;
static char *lcontinue;
static char *lswitch;
static int stackpos;
static int numgp;
static int numfp;

static void emit_expr(Node *node);
static void emit_decl_init(List *inits, int off);
static void emit_data_int(List *inits, int size, int off, int depth);
static void emit_data(Node *v, int off, int depth);

#define REGAREA_SIZE 304

#define emit(...)        emitf(__LINE__, "\t" __VA_ARGS__)
#define emit_noindent(...)  emitf(__LINE__, __VA_ARGS__)

#ifdef __GNUC__
#define SAVE                                                            \
    int save_hook __attribute__((unused, cleanup(pop_function)));       \
    list_push(functions, (void *)__func__)

static void pop_function(void *ignore) {
    list_pop(functions);
}
#else
#define SAVE
#endif

static char *get_caller_list(void) {
    String *s = make_string();
    for (Iter *i = list_iter(functions); !iter_end(i);) {
        string_appendf(s, "%s", iter_next(i));
        if (!iter_end(i))
            string_appendf(s, " -> ");
    }
    return get_cstring(s);
}

static void emitf(int line, char *fmt, ...) {
    va_list args;
    va_start(args, fmt);
    int col = vprintf(fmt, args);
    va_end(args);

    for (char *p = fmt; *p; p++)
        if (*p == '\t')
            col += TAB - 1;
    int space = (28 - col) > 0 ? (30 - col) : 2;
    printf("%*c %s:%d\n", space, '#', get_caller_list(), line);
}

static char *get_int_reg(Ctype *ctype, char r) {
    assert(r == 'a' || r == 'c');
    switch (ctype->size) {
    case 1: return (r == 'a') ? "al" : "cl";
    case 2: return (r == 'a') ? "ax" : "cx";
    case 4: return (r == 'a') ? "eax" : "ecx";
    case 8: return (r == 'a') ? "rax" : "rcx";
    default:
        error("Unknown data size: %s: %d", c2s(ctype), ctype->size);
    }
}

static char *get_load_inst(Ctype *ctype) {
    switch (ctype->size) {
    case 1: return "movsbq";
    case 2: return "movswq";
    case 4: return "movslq";
    case 8: return "mov";
    default:
        error("Unknown data size: %s: %d", c2s(ctype), ctype->size);
    }
}

static void push_xmm(int reg) {
    SAVE;
    emit("sub $8, %%rsp");
    emit("movsd %%xmm%d, (%%rsp)", reg);
    stackpos += 8;
}

static void pop_xmm(int reg) {
    SAVE;
    emit("movsd (%%rsp), %%xmm%d", reg);
    emit("add $8, %%rsp");
    stackpos -= 8;
    assert(stackpos >= 0);
}

static void push(char *reg) {
    SAVE;
    emit("push %%%s", reg);
    stackpos += 8;
}

static void pop(char *reg) {
    SAVE;
    emit("pop %%%s", reg);
    stackpos -= 8;
    assert(stackpos >= 0);
}

static void maybe_emit_bitshift_load(Ctype *ctype) {
    SAVE;
    if (ctype->bitsize <= 0)
        return;
    emit("shr $%d, %%rax", ctype->bitoff);
    push("rcx");
    emit("mov $0x%lx, %%rcx", (1 << (long)ctype->bitsize) - 1);
    emit("and %%rcx, %%rax");
    pop("rcx");
}

static void maybe_emit_bitshift_save(Ctype *ctype, char *addr) {
    SAVE;
    if (ctype->bitsize <= 0)
        return;
    push("rcx");
    push("rdi");
    emit("mov $0x%lx, %%rdi", (1 << (long)ctype->bitsize) - 1);
    emit("and %%rdi, %%rax");
    emit("shl $%d, %%rax", ctype->bitoff);
    emit("mov %s, %%%s", addr, get_int_reg(ctype, 'c'));
    emit("mov $0x%lx, %%rdi", ~(((1 << (long)ctype->bitsize) - 1) << ctype->bitoff));
    emit("and %%rdi, %%rcx");
    emit("or %%rcx, %%rax");
    pop("rdi");
    pop("rcx");
}

static void emit_gload(Ctype *ctype, char *label, int off) {
    SAVE;
    if (ctype->type == CTYPE_ARRAY) {
        if (off)
            emit("lea %s+%d(%%rip), %%rax", label, off);
        else
            emit("lea %s(%%rip), %%rax", label);
        return;
    }
    char *inst = get_load_inst(ctype);
    emit("%s %s+%d(%%rip), %%rax", inst, label, off);
    maybe_emit_bitshift_load(ctype);
}

static void emit_toint(Ctype *ctype) {
    SAVE;
    if (ctype->type == CTYPE_FLOAT)
        emit("cvttss2si %%xmm0, %%eax");
    else if (ctype->type == CTYPE_FLOAT)
        emit("cvttsd2si %%xmm0, %%eax");
}

static void emit_lload(Ctype *ctype, char *base, int off) {
    SAVE;
    if (ctype->type == CTYPE_ARRAY) {
        emit("lea %d(%%%s), %%rax", off, base);
    } else if (ctype->type == CTYPE_FLOAT) {
        emit("movss %d(%%%s), %%xmm0", off, base);
    } else if (ctype->type == CTYPE_DOUBLE || ctype->type == CTYPE_LDOUBLE) {
        emit("movsd %d(%%%s), %%xmm0", off, base);
    } else {
        char *inst = get_load_inst(ctype);
        emit("%s %d(%%%s), %%rax", inst, off, base);
        maybe_emit_bitshift_load(ctype);
    }
}

static void maybe_convert_bool(Ctype *ctype) {
    if (ctype->type == CTYPE_BOOL) {
        emit("test %%rax, %%rax");
        emit("setne %%al");
    }
}

static void emit_gsave(char *varname, Ctype *ctype, int off) {
    SAVE;
    assert(ctype->type != CTYPE_ARRAY);
    maybe_convert_bool(ctype);
    char *reg = get_int_reg(ctype, 'a');
    char *addr = format("%s+%d(%%rip)", varname, off);
    maybe_emit_bitshift_save(ctype, addr);
    emit("mov %%%s, %s", reg, addr);
}

static void emit_lsave(Ctype *ctype, int off) {
    SAVE;
    if (ctype->type == CTYPE_FLOAT) {
        emit("movss %%xmm0, %d(%%rbp)", off);
    } else if (ctype->type == CTYPE_DOUBLE) {
        emit("movsd %%xmm0, %d(%%rbp)", off);
    } else {
        maybe_convert_bool(ctype);
        char *reg = get_int_reg(ctype, 'a');
        char *addr = format("%d(%%rbp)", off);
        maybe_emit_bitshift_save(ctype, addr);
        emit("mov %%%s, %s", reg, addr);
    }
}

static void emit_assign_deref_int(Ctype *ctype, int off) {
    SAVE;
    emit("mov (%%rsp), %%rcx");
    char *reg = get_int_reg(ctype, 'c');
    if (off)
        emit("mov %%%s, %d(%%rax)", reg, off);
    else
        emit("mov %%%s, (%%rax)", reg);
    pop("rax");
}

static void emit_assign_deref(Node *var) {
    SAVE;
    push("rax");
    emit_expr(var->operand);
    emit_assign_deref_int(var->operand->ctype->ptr, 0);
}

static void emit_pointer_arith(char op, Node *left, Node *right) {
    SAVE;
    emit_expr(left);
    push("rcx");
    push("rax");
    emit_expr(right);
    int size = left->ctype->ptr->size;
    if (size > 1)
        emit("imul $%d, %%rax", size);
    emit("mov %%rax, %%rcx");
    pop("rax");
    emit("add %%rcx, %%rax");
    pop("rcx");
}

static void emit_zero_filler(int start, int end) {
    for (; start <= end - 4; start += 4)
        emit("movl $0, %d(%%rbp)", start);
    for (; start < end; start++)
        emit("movb $0, %d(%%rbp)", start);
}

static void ensure_lvar_init(Node *node) {
    assert(node->type == AST_LVAR);
    if (node->lvarinit) {
        emit_zero_filler(node->loff, node->loff + node->ctype->size);
        emit_decl_init(node->lvarinit, node->loff);
    }
    node->lvarinit = NULL;
}

static void emit_assign_struct_ref(Node *struc, Ctype *field, int off) {
    SAVE;
    switch (struc->type) {
    case AST_LVAR:
        ensure_lvar_init(struc);
        emit_lsave(field, struc->loff + field->offset + off);
        break;
    case AST_GVAR:
        emit_gsave(struc->varname, field, field->offset + off);
        break;
    case AST_STRUCT_REF:
        emit_assign_struct_ref(struc->struc, field, off + struc->ctype->offset);
        break;
    case AST_DEREF:
        push("rax");
        emit_expr(struc->operand);
        emit_assign_deref_int(field, field->offset + off);
        break;
    default:
        error("internal error: %s", a2s(struc));
    }
}

static void emit_load_struct_ref(Node *struc, Ctype *field, int off) {
    SAVE;
    switch (struc->type) {
    case AST_LVAR:
        ensure_lvar_init(struc);
        emit_lload(field, "rbp", struc->loff + field->offset + off);
        break;
    case AST_GVAR:
        emit_gload(field, struc->varname, field->offset + off);
        break;
    case AST_STRUCT_REF:
        emit_load_struct_ref(struc->struc, field, struc->ctype->offset + off);
        break;
    case AST_DEREF:
        emit_expr(struc->operand);
        emit_lload(field, "rax", field->offset + off);
        break;
    default:
        error("internal error: %s", a2s(struc));
    }
}

static void emit_store(Node *var) {
    SAVE;
    switch (var->type) {
    case AST_DEREF: emit_assign_deref(var); break;
    case AST_STRUCT_REF: emit_assign_struct_ref(var->struc, var->ctype, 0); break;
    case AST_LVAR:
        ensure_lvar_init(var);
        emit_lsave(var->ctype, var->loff);
        break;
    case AST_GVAR: emit_gsave(var->varname, var->ctype, 0); break;
    default: error("internal error");
    }
}

static void emit_to_bool(Ctype *ctype) {
    SAVE;
    if (is_flotype(ctype)) {
        push_xmm(1);
        emit("xorpd %%xmm1, %%xmm1");
        emit("%s %%xmm1, %%xmm0", (ctype->type == CTYPE_FLOAT) ? "ucomiss" : "ucomisd");
        emit("setne %%al");
        pop_xmm(1);
    } else {
        emit("cmp $0, %%rax");
        emit("setne %%al");
    }
    emit("movzb %%al, %%eax");
}

static void emit_comp(char *inst, Node *node) {
    SAVE;
    if (is_flotype(node->left->ctype)) {
        emit_expr(node->left);
        push_xmm(0);
        emit_expr(node->right);
        pop_xmm(1);
        if (node->left->ctype->type == CTYPE_FLOAT)
            emit("ucomiss %%xmm0, %%xmm1");
        else
            emit("ucomisd %%xmm0, %%xmm1");
    } else {
        emit_expr(node->left);
        push("rax");
        emit_expr(node->right);
        pop("rcx");
        emit("cmp %%rax, %%rcx");
    }
    emit("%s %%al", inst);
    emit("movzb %%al, %%eax");
}

static void emit_binop_int_arith(Node *node) {
    SAVE;
    char *op = NULL;
    switch (node->type) {
    case '+': op = "add"; break;
    case '-': op = "sub"; break;
    case '*': op = "imul"; break;
    case '^': op = "xor"; break;
    case OP_SAL: op = "sal"; break;
    case OP_SAR: op = "sar"; break;
    case OP_SHR: op = "shr"; break;
    case '/': case '%': break;
    default: error("invalid operator '%d'", node->type);<commit_msg>Compare using 32bit registers for non long long types<commit_after>// Copyright 2012 <NAME> <<EMAIL>>
// This program is free software licensed under the MIT license.

#include <stdarg.h>
#include <stdlib.h>
#include <string.h>
#include "8cc.h"
#include "list.h"

static char *REGS[] = {"rdi", "rsi", "rdx", "rcx", "r8", "r9"};
static char *SREGS[] = {"dil", "sil", "dl", "cl", "r8b", "r9b"};
static char *MREGS[] = {"edi", "esi", "edx", "ecx", "r8d", "r9d"};
static int TAB = 8;
static List *functions = &EMPTY_LIST;
static char *lbreak;
static char *lcontinue;
static char *lswitch;
static int stackpos;
static int numgp;
static int numfp;

static void emit_expr(Node *node);
static void emit_decl_init(List *inits, int off);
static void emit_data_int(List *inits, int size, int off, int depth);
static void emit_data(Node *v, int off, int depth);

#define REGAREA_SIZE 304

#define emit(...)        emitf(__LINE__, "\t" __VA_ARGS__)
#define emit_noindent(...)  emitf(__LINE__, __VA_ARGS__)

#ifdef __GNUC__
#define SAVE                                                            \
    int save_hook __attribute__((unused, cleanup(pop_function)));       \
    list_push(functions, (void *)__func__)

static void pop_function(void *ignore) {
    list_pop(functions);
}
#else
#define SAVE
#endif

static char *get_caller_list(void) {
    String *s = make_string();
    for (Iter *i = list_iter(functions); !iter_end(i);) {
        string_appendf(s, "%s", iter_next(i));
        if (!iter_end(i))
            string_appendf(s, " -> ");
    }
    return get_cstring(s);
}

static void emitf(int line, char *fmt, ...) {
    va_list args;
    va_start(args, fmt);
    int col = vprintf(fmt, args);
    va_end(args);

    for (char *p = fmt; *p; p++)
        if (*p == '\t')
            col += TAB - 1;
    int space = (28 - col) > 0 ? (30 - col) : 2;
    printf("%*c %s:%d\n", space, '#', get_caller_list(), line);
}

static char *get_int_reg(Ctype *ctype, char r) {
    assert(r == 'a' || r == 'c');
    switch (ctype->size) {
    case 1: return (r == 'a') ? "al" : "cl";
    case 2: return (r == 'a') ? "ax" : "cx";
    case 4: return (r == 'a') ? "eax" : "ecx";
    case 8: return (r == 'a') ? "rax" : "rcx";
    default:
        error("Unknown data size: %s: %d", c2s(ctype), ctype->size);
    }
}

static char *get_load_inst(Ctype *ctype) {
    switch (ctype->size) {
    case 1: return "movsbq";
    case 2: return "movswq";
    case 4: return "movslq";
    case 8: return "mov";
    default:
        error("Unknown data size: %s: %d", c2s(ctype), ctype->size);
    }
}

static void push_xmm(int reg) {
    SAVE;
    emit("sub $8, %%rsp");
    emit("movsd %%xmm%d, (%%rsp)", reg);
    stackpos += 8;
}

static void pop_xmm(int reg) {
    SAVE;
    emit("movsd (%%rsp), %%xmm%d", reg);
    emit("add $8, %%rsp");
    stackpos -= 8;
    assert(stackpos >= 0);
}

static void push(char *reg) {
    SAVE;
    emit("push %%%s", reg);
    stackpos += 8;
}

static void pop(char *reg) {
    SAVE;
    emit("pop %%%s", reg);
    stackpos -= 8;
    assert(stackpos >= 0);
}

static void maybe_emit_bitshift_load(Ctype *ctype) {
    SAVE;
    if (ctype->bitsize <= 0)
        return;
    emit("shr $%d, %%rax", ctype->bitoff);
    push("rcx");
    emit("mov $0x%lx, %%rcx", (1 << (long)ctype->bitsize) - 1);
    emit("and %%rcx, %%rax");
    pop("rcx");
}

static void maybe_emit_bitshift_save(Ctype *ctype, char *addr) {
    SAVE;
    if (ctype->bitsize <= 0)
        return;
    push("rcx");
    push("rdi");
    emit("mov $0x%lx, %%rdi", (1 << (long)ctype->bitsize) - 1);
    emit("and %%rdi, %%rax");
    emit("shl $%d, %%rax", ctype->bitoff);
    emit("mov %s, %%%s", addr, get_int_reg(ctype, 'c'));
    emit("mov $0x%lx, %%rdi", ~(((1 << (long)ctype->bitsize) - 1) << ctype->bitoff));
    emit("and %%rdi, %%rcx");
    emit("or %%rcx, %%rax");
    pop("rdi");
    pop("rcx");
}

static void emit_gload(Ctype *ctype, char *label, int off) {
    SAVE;
    if (ctype->type == CTYPE_ARRAY) {
        if (off)
            emit("lea %s+%d(%%rip), %%rax", label, off);
        else
            emit("lea %s(%%rip), %%rax", label);
        return;
    }
    char *inst = get_load_inst(ctype);
    emit("%s %s+%d(%%rip), %%rax", inst, label, off);
    maybe_emit_bitshift_load(ctype);
}

static void emit_toint(Ctype *ctype) {
    SAVE;
    if (ctype->type == CTYPE_FLOAT)
        emit("cvttss2si %%xmm0, %%eax");
    else if (ctype->type == CTYPE_FLOAT)
        emit("cvttsd2si %%xmm0, %%eax");
}

static void emit_lload(Ctype *ctype, char *base, int off) {
    SAVE;
    if (ctype->type == CTYPE_ARRAY) {
        emit("lea %d(%%%s), %%rax", off, base);
    } else if (ctype->type == CTYPE_FLOAT) {
        emit("movss %d(%%%s), %%xmm0", off, base);
    } else if (ctype->type == CTYPE_DOUBLE || ctype->type == CTYPE_LDOUBLE) {
        emit("movsd %d(%%%s), %%xmm0", off, base);
    } else {
        char *inst = get_load_inst(ctype);
        emit("%s %d(%%%s), %%rax", inst, off, base);
        maybe_emit_bitshift_load(ctype);
    }
}

static void maybe_convert_bool(Ctype *ctype) {
    if (ctype->type == CTYPE_BOOL) {
        emit("test %%rax, %%rax");
        emit("setne %%al");
    }
}

static void emit_gsave(char *varname, Ctype *ctype, int off) {
    SAVE;
    assert(ctype->type != CTYPE_ARRAY);
    maybe_convert_bool(ctype);
    char *reg = get_int_reg(ctype, 'a');
    char *addr = format("%s+%d(%%rip)", varname, off);
    maybe_emit_bitshift_save(ctype, addr);
    emit("mov %%%s, %s", reg, addr);
}

static void emit_lsave(Ctype *ctype, int off) {
    SAVE;
    if (ctype->type == CTYPE_FLOAT) {
        emit("movss %%xmm0, %d(%%rbp)", off);
    } else if (ctype->type == CTYPE_DOUBLE) {
        emit("movsd %%xmm0, %d(%%rbp)", off);
    } else {
        maybe_convert_bool(ctype);
        char *reg = get_int_reg(ctype, 'a');
        char *addr = format("%d(%%rbp)", off);
        maybe_emit_bitshift_save(ctype, addr);
        emit("mov %%%s, %s", reg, addr);
    }
}

static void emit_assign_deref_int(Ctype *ctype, int off) {
    SAVE;
    emit("mov (%%rsp), %%rcx");
    char *reg = get_int_reg(ctype, 'c');
    if (off)
        emit("mov %%%s, %d(%%rax)", reg, off);
    else
        emit("mov %%%s, (%%rax)", reg);
    pop("rax");
}

static void emit_assign_deref(Node *var) {
    SAVE;
    push("rax");
    emit_expr(var->operand);
    emit_assign_deref_int(var->operand->ctype->ptr, 0);
}

static void emit_pointer_arith(char op, Node *left, Node *right) {
    SAVE;
    emit_expr(left);
    push("rcx");
    push("rax");
    emit_expr(right);
    int size = left->ctype->ptr->size;
    if (size > 1)
        emit("imul $%d, %%rax", size);
    emit("mov %%rax, %%rcx");
    pop("rax");
    emit("add %%rcx, %%rax");
    pop("rcx");
}

static void emit_zero_filler(int start, int end) {
    for (; start <= end - 4; start += 4)
        emit("movl $0, %d(%%rbp)", start);
    for (; start < end; start++)
        emit("movb $0, %d(%%rbp)", start);
}

static void ensure_lvar_init(Node *node) {
    assert(node->type == AST_LVAR);
    if (node->lvarinit) {
        emit_zero_filler(node->loff, node->loff + node->ctype->size);
        emit_decl_init(node->lvarinit, node->loff);
    }
    node->lvarinit = NULL;
}

static void emit_assign_struct_ref(Node *struc, Ctype *field, int off) {
    SAVE;
    switch (struc->type) {
    case AST_LVAR:
        ensure_lvar_init(struc);
        emit_lsave(field, struc->loff + field->offset + off);
        break;
    case AST_GVAR:
        emit_gsave(struc->varname, field, field->offset + off);
        break;
    case AST_STRUCT_REF:
        emit_assign_struct_ref(struc->struc, field, off + struc->ctype->offset);
        break;
    case AST_DEREF:
        push("rax");
        emit_expr(struc->operand);
        emit_assign_deref_int(field, field->offset + off);
        break;
    default:
        error("internal error: %s", a2s(struc));
    }
}

static void emit_load_struct_ref(Node *struc, Ctype *field, int off) {
    SAVE;
    switch (struc->type) {
    case AST_LVAR:
        ensure_lvar_init(struc);
        emit_lload(field, "rbp", struc->loff + field->offset + off);
        break;
    case AST_GVAR:
        emit_gload(field, struc->varname, field->offset + off);
        break;
    case AST_STRUCT_REF:
        emit_load_struct_ref(struc->struc, field, struc->ctype->offset + off);
        break;
    case AST_DEREF:
        emit_expr(struc->operand);
        emit_lload(field, "rax", field->offset + off);
        break;
    default:
        error("internal error: %s", a2s(struc));
    }
}

static void emit_store(Node *var) {
    SAVE;
    switch (var->type) {
    case AST_DEREF: emit_assign_deref(var); break;
    case AST_STRUCT_REF: emit_assign_struct_ref(var->struc, var->ctype, 0); break;
    case AST_LVAR:
        ensure_lvar_init(var);
        emit_lsave(var->ctype, var->loff);
        break;
    case AST_GVAR: emit_gsave(var->varname, var->ctype, 0); break;
    default: error("internal error");
    }
}

static void emit_to_bool(Ctype *ctype) {
    SAVE;
    if (is_flotype(ctype)) {
        push_xmm(1);
        emit("xorpd %%xmm1, %%xmm1");
        emit("%s %%xmm1, %%xmm0", (ctype->type == CTYPE_FLOAT) ? "ucomiss" : "ucomisd");
        emit("setne %%al");
        pop_xmm(1);
    } else {
        emit("cmp $0, %%rax");
        emit("setne %%al");
    }
    emit("movzb %%al, %%eax");
}

static void emit_comp(char *inst, Node *node) {
    SAVE;
    if (is_flotype(node->left->ctype)) {
        emit_expr(node->left);
        push_xmm(0);
        emit_expr(node->right);
        pop_xmm(1);
        if (node->left->ctype->type == CTYPE_FLOAT)
            emit("ucomiss %%xmm0, %%xmm1");
        else
            emit("ucomisd %%xmm0, %%xmm1");
    } else {
        emit_expr(node->left);
        push("rax");
        emit_expr(node->right);
        pop("rcx");
        if (node->left->ctype->type == CTYPE_LLONG)
          emit("cmp %%rax, %%rcx");
        else
          emit("cmp %%eax, %%ecx");
    }
    emit("%s %%al", inst);
    emit("movzb %%al, %%eax");
}

static void emit_binop_int_arith(Node *node) {
    SAVE;
    char *op = NULL;
    switch (node->type) {
    case '+': op = "add"; break;
    case '-': op = "sub"; break;
    case '*': op = "imul"; break;
    case '^': op = "xor"; break;
    case OP_SAL: op = "sal"; break;
    case OP_SAR: op = "sar"; break;
    case OP_SHR: op = "shr"; break;
    case '/': case '%': break;
    default: error("invalid operator '%d'", node->type);
{'fraction_non_alphanumeric': 0.18998703979263667, 'fraction_numerical': 0.00796812749003984, 'mean_word_length': 2.7907569141193593, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 184, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '24587', 'n_tokens_mistral': 8251, 'n_tokens_neox': 7784, 'n_words': 1864}