<commit_before>#include <stdlib.h>
#include <stdio.h>
#include <inttypes.h>
#include <stdarg.h>
#include <inttypes.h>
#include <ctype.h>
#include <string.h>
#include <assert.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>

#include "parse.h"
#include "mi.h"
#include "asm.h"
#include "../config.h"

/* forward decls */
Loc *selexpr(Isel *s, Node *n);

#define Nfloatregargs 8
#define Nintregargs 6
regid floatargregs[] = {
	Rxmm0d, Rxmm1d, Rxmm2d, Rxmm3d,
	Rxmm4d, Rxmm5d, Rxmm6d, Rxmm7d,
};
regid intargregs[] = {Rrdi, Rrsi, Rrdx, Rrcx, Rr8, Rr9};

/* used to decide which operator is appropriate
 * for implementing various conditional operators */
struct {
	AsmOp test;
	AsmOp jmp;
	AsmOp getflag;
} reloptab[Numops] = {
	[Olnot] = {Itest, Ijz, Isetz}, /* lnot invalid for floats */
	/* signed int */
	[Oeq] = {Icmp, Ijz,  Isetz},
	[One] = {Icmp, Ijnz, Isetnz},
	[Ogt] = {Icmp, Ijg,  Isetg},
	[Oge] = {Icmp, Ijge, Isetge},
	[Olt] = {Icmp, Ijl,  Isetl},
	[Ole] = {Icmp, Ijle, Isetle},
	/* unsigned int */
	[Oueq] = {Icmp, Ijz,  Isetz},
	[Oune] = {Icmp, Ijnz, Isetnz},
	[Ougt] = {Icmp, Ija,  Iseta},
	[Ouge] = {Icmp, Ijae, Isetae},
	[Oult] = {Icmp, Ijb,  Isetb},
	[Oule] = {Icmp, Ijbe, Isetbe},
	/* float */
	[Ofeq] = {Icomis, Ijz,  Isetz},
	[Ofne] = {Icomis, Ijnz, Isetnz},
	[Ofgt] = {Icomis, Ija,  Iseta},
	[Ofge] = {Icomis, Ijae, Isetae},
	[Oflt] = {Icomis, Ijb,  Isetb},
	[Ofle] = {Icomis, Ijbe, Isetbe},
};

static Mode tymode(Type *t)
{
	/* FIXME: What should the mode for, say, structs be when we have no
	 * intention of loading /through/ the pointer? For now, we'll just say it's
	 * the pointer mode, since we expect to address through the pointer */
	t = tybase(t);
	switch (t->type) {
	case Tyflt32:	return ModeF;	break;
	case Tyflt64:	return ModeD;	break;
	default:
		if (isstacktype(t))
			return ModeQ;
		switch (tysize(t)) {
		case 1:	return ModeB;	break;
		case 2:	return ModeW;	break;
		case 4:	return ModeL;	break;
		case 8:	return ModeQ;	break;
		}
		break;
	}
	return ModeNone;
}

static Mode mode(Node *n)
{
	if (n->type == Nexpr)
		return tymode(exprtype(n));
	else if (n->type == Ndecl)
		return tymode(n->decl.type);
	else
		die("invalid node type");
	return ModeNone;
}

static Loc *varloc(Isel *s, Node *n)
{
	ssize_t off;
	Loc *l, *rip;

	/* we need to try getting it from the stack first, in case we
	 * forced it to stack for addressing */
	if (hthas(s->globls, n)) {
		rip = locphysreg(Rrip);
		l = locmeml(htget(s->globls, n), rip, NULL, mode(n));
	} else if (hthas(s->envoff, n)) {
		off = ptoi(htget(s->envoff, n));
		l = locmem(off, s->envp, NULL, mode(n));
	} else if (hthas(s->stkoff, n)) {
		off = ptoi(htget(s->stkoff, n));
		l = locmem(-off, locphysreg(Rrbp), NULL, mode(n));
	}  else {
		l = htget(s->reglocs, n);
		if (!l) {
			l = locreg(mode(n));
			htput(s->reglocs, n, l);
		}
	}
	return l;
}

static Loc *loc(Isel *s, Node *n)
{
	Node *v;
	Loc *l;

	if (n->type == Ndecl) {
		l = varloc(s, n);
	} else {
		switch (exprop(n)) {
		case Ovar:
			l = varloc(s, n); 
			break;
		case Olit:
			v = n->expr.args[0];
			switch (v->lit.littype) {
			case Lvoid:	l = NULL;	break;
			case Lchr:	l = loclit(v->lit.chrval, mode(n));	break;
			case Lbool:	l = loclit(v->lit.boolval, mode(n));	break;
			case Lint:	l = loclit(v->lit.intval, mode(n));	break;
			default:
					die("Literal type %s should be blob", litstr[v->lit.littype]);
			}
			break;
		default:
			die("Node %s not leaf in loc()", opstr[exprop(n)]);
			break;
		}
	}
	return l;
}

static Insn *mkinsnv(AsmOp op, va_list ap)
{
	Loc *l;
	Insn *i;
	int n;
	static size_t insnid;

	n = 0;
	i = malloc(sizeof(Insn));
	i->op = op;
	i->uid = insnid++;
	while ((l = va_arg(ap, Loc*)) != NULL)
		i->args[n++] = l;
	i->nargs = n;
	return i;
}

Insn *mkinsn(AsmOp op, ...)
{
	va_list ap;
	Insn *i;

	va_start(ap, op);
	i = mkinsnv(op, ap);
	va_end(ap);
	return i;
}

static void g(Isel *s, AsmOp op, ...)
{
	va_list ap;
	Insn *i;

	va_start(ap, op);
	i = mkinsnv(op, ap);
	va_end(ap);
	if (debugopt['i']) {
		printf("GEN[uid=%zd] ", i->uid);
		dbgiprintf(stdout, i);
	}
	lappend(&s->curbb->il, &s->curbb->ni, i);
}

static void movz(Isel *s, Loc *src, Loc *dst)
{
	if (src->mode == dst->mode)
		g(s, Imov, src, dst, NULL);
	else
		g(s, Imovzx, src, dst, NULL);
}

static void load(Isel *s, Loc *a, Loc *b)
{
	Loc *l;

	assert(b->type == Locreg);
	if (a->type == Locreg)
		l = locmem(0, b, NULL, a->mode);
	else
		l = a;
	if (isfloatmode(b->mode))
		g(s, Imovs, l, b, NULL);
	else
		g(s, Imov, l, b, NULL);
}

static void stor(Isel *s, Loc *a, Loc *b)
{
	Loc *l;

	assert(a->type == Locreg || a->type == Loclit);
	if (b->type == Locreg)
		l = locmem(0, b, NULL, b->mode);
	else
		l = b;
	if (isfloatmode(b->mode))
		g(s, Imovs, a, l, NULL);
	else
		g(s, Imov, a, l, NULL);
}

/* ensures that a location is within a reg */
static Loc *newr(Isel *s, Loc *a)
{
	Loc *r;

	r = locreg(a->mode);
	if (a->type == Locreg) {
		if (isfloatmode(a->mode))
			g(s, Imovs, a, r, NULL);
		else
			g(s, Imov, a, r, NULL);
	} else {
		load(s, a, r);
	}
	return r;
}

static Loc *inr(Isel *s, Loc *a)
{
	if (a->type == Locreg)
		return a;
	return newr(s, a);
}

/* ensures that a location is within a reg or an imm */
static Loc *inri(Isel *s, Loc *a)
{
	if (a->type == Locreg || a->type == Loclit)
		return a;
	else
		return newr(s, a);
}

/* If we're testing equality, etc, it's a bit silly
 * to generate the test, store it to a bite, expand it
 * to the right width, and then test it again. Try to optimize
 * for these common cases.
 *
 * if we're doing the optimization to avoid
 * multiple tests, we want to eval the children
 * of the first arg, instead of the first arg
 * directly */
static void selcjmp(Isel *s, Node *n, Node **args)
{
	Loc *a, *b;
	Loc *l1, *l2;
	AsmOp cond, jmp;

	cond = reloptab[exprop(args[0])].test;
	jmp = reloptab[exprop(args[0])].jmp;
	/* if we have a cond, we're knocking off the redundant test,
	 * and want to eval the children */
	if (cond) {
		a = selexpr(s, args[0]->expr.args[0]);
		if (args[0]->expr.nargs == 2)
			b = selexpr(s, args[0]->expr.args[1]);
		else
			b = a;
		a = newr(s, a);
	} else {
		cond = Itest;
		jmp = Ijnz;
		b = newr(s, selexpr(s, args[0])); /* cond */
		a = b;
	}

	/* the jump targets will always be evaluated the same way */
	l1 = loclbl(args[1]); /* if true */
	l2 = loclbl(args[2]); /* if false */

	g(s, cond, b, a, NULL);
	g(s, jmp, l1, NULL);
	g(s, Ijmp, l2, NULL);
}

/* Generate variable length jump. There are 3 cases
 * for this:
 *
 * 1) Short list: Simple comparison sequence.
 * 2) Sparse list: binary search
 * 3) Dense list: Jump table
 *
 * If a value is not in the jump table, then the 0th
 * value is used.
 */
static void selvjmp(Isel *s, Node *n, Node **args)
{
}

static Loc *binop(Isel *s, AsmOp op, Node *x, Node *y)
{
	Loc *a, *b;

	a = selexpr(s, x);
	b = selexpr(s, y);
	a = newr(s, a);
	g(s, op, b, a, NULL);
	return a;
}

/* We have a few common cases to optimize here:
 *    Oaddr(expr)
 * or:
 *    Oadd(
 *        reg,
 *        reg||const)
 *
 * or:
 *    Oadd(
 *        reg,
 *        Omul(reg,
 *             2 || 4 || 8)))
 */
static int ismergablemul(Node *n, int *r)
{
	int v;

	if (exprop(n) != Omul)
		return 0;
	if (exprop(n->expr.args[1]) != Olit)
		return 0;
	if (n->expr.args[1]->expr.args[0]->type != Nlit)
		return 0;
	if (n->expr.args[1]->expr.args[0]->lit.littype != Lint)
		return 0;
	v = n->expr.args[1]->expr.args[0]->lit.intval;
	if (v != 2 && v != 4 && v != 8)
		return 0;
	*r = v;
	return 1;
}

static Loc *memloc(Isel *s, Node *e, Mode m)
{
	Node **args;
	Loc *l, *b, *o; /* location, base, offset */
	int scale;

	scale = 1;
	l = NULL;
	args = e->expr.args;
	if (exprop(e) == Oadd) {
		b = selexpr(s, args[0]);
		if (ismergablemul(args[1], &scale))
			o = selexpr(s, args[1]->expr.args[0]);
		else
			o = selexpr(s, args[1]);

		if (b->type != Locreg)
			b = inr(s, b);
		if (o->type == Loclit) {
			l = locmem(scale*o->lit, b, NULL, m);
		} else {
			b = inr(s, b);
			o = inr(s, o);
			l = locmems(0, b, o, scale, m);
		}
	} else {
		l = selexpr(s, e);
		l = inr(s, l);
		l = locmem(0, l, NULL, m);
	}
	assert(l != NULL);
	return l;
}

static const Mode szmodes[] = {
	[8] = ModeQ,
	[4] = ModeL,
	[2] = ModeW,
	[1] = ModeB
};
static void blit(Isel *s, Loc *to, Loc *from, size_t dstoff, size_t srcoff, size_t sz, size_t align)
{
	size_t i, modesz;
	Loc *sp, *dp, *len; /* pointers to src, dst */
	Loc *tmp, *src, *dst; /* source memory, dst memory */

	assert(szmodes[align] != ModeNone);   /* make sure we have a valid alignment */
	if (!sz)
		return;
	sp = inr(s, from);
	dp = inr(s, to);

	i = 0;
	align = 8;
	if (align == 0)
		align = 8;
	if (sz <= 128) { /* arbitrary threshold; should be tuned */
		for (modesz = align; szmodes[modesz] != ModeNone; modesz /= 2) {
			tmp = locreg(szmodes[modesz]);
			while (i + modesz <= sz) {
				src = locmem(i + srcoff, sp, NULL, szmodes[modesz]);
				dst = locmem(i + dstoff, dp, NULL, szmodes[modesz]);
				g(s, Imov, src, tmp, NULL);
				g(s, Imov, tmp, dst, NULL);
				i += modesz;
			}
		}
	} else {
		len = loclit(sz, ModeQ);
		sp = newr(s, from);
		dp = newr(s, to);<commit_msg>Remove leftover debugging cruft.<commit_after>#include <stdlib.h>
#include <stdio.h>
#include <inttypes.h>
#include <stdarg.h>
#include <inttypes.h>
#include <ctype.h>
#include <string.h>
#include <assert.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>

#include "parse.h"
#include "mi.h"
#include "asm.h"
#include "../config.h"

/* forward decls */
Loc *selexpr(Isel *s, Node *n);

#define Nfloatregargs 8
#define Nintregargs 6
regid floatargregs[] = {
	Rxmm0d, Rxmm1d, Rxmm2d, Rxmm3d,
	Rxmm4d, Rxmm5d, Rxmm6d, Rxmm7d,
};
regid intargregs[] = {Rrdi, Rrsi, Rrdx, Rrcx, Rr8, Rr9};

/* used to decide which operator is appropriate
 * for implementing various conditional operators */
struct {
	AsmOp test;
	AsmOp jmp;
	AsmOp getflag;
} reloptab[Numops] = {
	[Olnot] = {Itest, Ijz, Isetz}, /* lnot invalid for floats */
	/* signed int */
	[Oeq] = {Icmp, Ijz,  Isetz},
	[One] = {Icmp, Ijnz, Isetnz},
	[Ogt] = {Icmp, Ijg,  Isetg},
	[Oge] = {Icmp, Ijge, Isetge},
	[Olt] = {Icmp, Ijl,  Isetl},
	[Ole] = {Icmp, Ijle, Isetle},
	/* unsigned int */
	[Oueq] = {Icmp, Ijz,  Isetz},
	[Oune] = {Icmp, Ijnz, Isetnz},
	[Ougt] = {Icmp, Ija,  Iseta},
	[Ouge] = {Icmp, Ijae, Isetae},
	[Oult] = {Icmp, Ijb,  Isetb},
	[Oule] = {Icmp, Ijbe, Isetbe},
	/* float */
	[Ofeq] = {Icomis, Ijz,  Isetz},
	[Ofne] = {Icomis, Ijnz, Isetnz},
	[Ofgt] = {Icomis, Ija,  Iseta},
	[Ofge] = {Icomis, Ijae, Isetae},
	[Oflt] = {Icomis, Ijb,  Isetb},
	[Ofle] = {Icomis, Ijbe, Isetbe},
};

static Mode tymode(Type *t)
{
	/* FIXME: What should the mode for, say, structs be when we have no
	 * intention of loading /through/ the pointer? For now, we'll just say it's
	 * the pointer mode, since we expect to address through the pointer */
	t = tybase(t);
	switch (t->type) {
	case Tyflt32:	return ModeF;	break;
	case Tyflt64:	return ModeD;	break;
	default:
		if (isstacktype(t))
			return ModeQ;
		switch (tysize(t)) {
		case 1:	return ModeB;	break;
		case 2:	return ModeW;	break;
		case 4:	return ModeL;	break;
		case 8:	return ModeQ;	break;
		}
		break;
	}
	return ModeNone;
}

static Mode mode(Node *n)
{
	if (n->type == Nexpr)
		return tymode(exprtype(n));
	else if (n->type == Ndecl)
		return tymode(n->decl.type);
	else
		die("invalid node type");
	return ModeNone;
}

static Loc *varloc(Isel *s, Node *n)
{
	ssize_t off;
	Loc *l, *rip;

	/* we need to try getting it from the stack first, in case we
	 * forced it to stack for addressing */
	if (hthas(s->globls, n)) {
		rip = locphysreg(Rrip);
		l = locmeml(htget(s->globls, n), rip, NULL, mode(n));
	} else if (hthas(s->envoff, n)) {
		off = ptoi(htget(s->envoff, n));
		l = locmem(off, s->envp, NULL, mode(n));
	} else if (hthas(s->stkoff, n)) {
		off = ptoi(htget(s->stkoff, n));
		l = locmem(-off, locphysreg(Rrbp), NULL, mode(n));
	}  else {
		l = htget(s->reglocs, n);
		if (!l) {
			l = locreg(mode(n));
			htput(s->reglocs, n, l);
		}
	}
	return l;
}

static Loc *loc(Isel *s, Node *n)
{
	Node *v;
	Loc *l;

	if (n->type == Ndecl) {
		l = varloc(s, n);
	} else {
		switch (exprop(n)) {
		case Ovar:
			l = varloc(s, n); 
			break;
		case Olit:
			v = n->expr.args[0];
			switch (v->lit.littype) {
			case Lvoid:	l = NULL;	break;
			case Lchr:	l = loclit(v->lit.chrval, mode(n));	break;
			case Lbool:	l = loclit(v->lit.boolval, mode(n));	break;
			case Lint:	l = loclit(v->lit.intval, mode(n));	break;
			default:
					die("Literal type %s should be blob", litstr[v->lit.littype]);
			}
			break;
		default:
			die("Node %s not leaf in loc()", opstr[exprop(n)]);
			break;
		}
	}
	return l;
}

static Insn *mkinsnv(AsmOp op, va_list ap)
{
	Loc *l;
	Insn *i;
	int n;
	static size_t insnid;

	n = 0;
	i = malloc(sizeof(Insn));
	i->op = op;
	i->uid = insnid++;
	while ((l = va_arg(ap, Loc*)) != NULL)
		i->args[n++] = l;
	i->nargs = n;
	return i;
}

Insn *mkinsn(AsmOp op, ...)
{
	va_list ap;
	Insn *i;

	va_start(ap, op);
	i = mkinsnv(op, ap);
	va_end(ap);
	return i;
}

static void g(Isel *s, AsmOp op, ...)
{
	va_list ap;
	Insn *i;

	va_start(ap, op);
	i = mkinsnv(op, ap);
	va_end(ap);
	if (debugopt['i']) {
		printf("GEN[uid=%zd] ", i->uid);
		dbgiprintf(stdout, i);
	}
	lappend(&s->curbb->il, &s->curbb->ni, i);
}

static void movz(Isel *s, Loc *src, Loc *dst)
{
	if (src->mode == dst->mode)
		g(s, Imov, src, dst, NULL);
	else
		g(s, Imovzx, src, dst, NULL);
}

static void load(Isel *s, Loc *a, Loc *b)
{
	Loc *l;

	assert(b->type == Locreg);
	if (a->type == Locreg)
		l = locmem(0, b, NULL, a->mode);
	else
		l = a;
	if (isfloatmode(b->mode))
		g(s, Imovs, l, b, NULL);
	else
		g(s, Imov, l, b, NULL);
}

static void stor(Isel *s, Loc *a, Loc *b)
{
	Loc *l;

	assert(a->type == Locreg || a->type == Loclit);
	if (b->type == Locreg)
		l = locmem(0, b, NULL, b->mode);
	else
		l = b;
	if (isfloatmode(b->mode))
		g(s, Imovs, a, l, NULL);
	else
		g(s, Imov, a, l, NULL);
}

/* ensures that a location is within a reg */
static Loc *newr(Isel *s, Loc *a)
{
	Loc *r;

	r = locreg(a->mode);
	if (a->type == Locreg) {
		if (isfloatmode(a->mode))
			g(s, Imovs, a, r, NULL);
		else
			g(s, Imov, a, r, NULL);
	} else {
		load(s, a, r);
	}
	return r;
}

static Loc *inr(Isel *s, Loc *a)
{
	if (a->type == Locreg)
		return a;
	return newr(s, a);
}

/* ensures that a location is within a reg or an imm */
static Loc *inri(Isel *s, Loc *a)
{
	if (a->type == Locreg || a->type == Loclit)
		return a;
	else
		return newr(s, a);
}

/* If we're testing equality, etc, it's a bit silly
 * to generate the test, store it to a bite, expand it
 * to the right width, and then test it again. Try to optimize
 * for these common cases.
 *
 * if we're doing the optimization to avoid
 * multiple tests, we want to eval the children
 * of the first arg, instead of the first arg
 * directly */
static void selcjmp(Isel *s, Node *n, Node **args)
{
	Loc *a, *b;
	Loc *l1, *l2;
	AsmOp cond, jmp;

	cond = reloptab[exprop(args[0])].test;
	jmp = reloptab[exprop(args[0])].jmp;
	/* if we have a cond, we're knocking off the redundant test,
	 * and want to eval the children */
	if (cond) {
		a = selexpr(s, args[0]->expr.args[0]);
		if (args[0]->expr.nargs == 2)
			b = selexpr(s, args[0]->expr.args[1]);
		else
			b = a;
		a = newr(s, a);
	} else {
		cond = Itest;
		jmp = Ijnz;
		b = newr(s, selexpr(s, args[0])); /* cond */
		a = b;
	}

	/* the jump targets will always be evaluated the same way */
	l1 = loclbl(args[1]); /* if true */
	l2 = loclbl(args[2]); /* if false */

	g(s, cond, b, a, NULL);
	g(s, jmp, l1, NULL);
	g(s, Ijmp, l2, NULL);
}

/* Generate variable length jump. There are 3 cases
 * for this:
 *
 * 1) Short list: Simple comparison sequence.
 * 2) Sparse list: binary search
 * 3) Dense list: Jump table
 *
 * If a value is not in the jump table, then the 0th
 * value is used.
 */
static void selvjmp(Isel *s, Node *n, Node **args)
{
}

static Loc *binop(Isel *s, AsmOp op, Node *x, Node *y)
{
	Loc *a, *b;

	a = selexpr(s, x);
	b = selexpr(s, y);
	a = newr(s, a);
	g(s, op, b, a, NULL);
	return a;
}

/* We have a few common cases to optimize here:
 *    Oaddr(expr)
 * or:
 *    Oadd(
 *        reg,
 *        reg||const)
 *
 * or:
 *    Oadd(
 *        reg,
 *        Omul(reg,
 *             2 || 4 || 8)))
 */
static int ismergablemul(Node *n, int *r)
{
	int v;

	if (exprop(n) != Omul)
		return 0;
	if (exprop(n->expr.args[1]) != Olit)
		return 0;
	if (n->expr.args[1]->expr.args[0]->type != Nlit)
		return 0;
	if (n->expr.args[1]->expr.args[0]->lit.littype != Lint)
		return 0;
	v = n->expr.args[1]->expr.args[0]->lit.intval;
	if (v != 2 && v != 4 && v != 8)
		return 0;
	*r = v;
	return 1;
}

static Loc *memloc(Isel *s, Node *e, Mode m)
{
	Node **args;
	Loc *l, *b, *o; /* location, base, offset */
	int scale;

	scale = 1;
	l = NULL;
	args = e->expr.args;
	if (exprop(e) == Oadd) {
		b = selexpr(s, args[0]);
		if (ismergablemul(args[1], &scale))
			o = selexpr(s, args[1]->expr.args[0]);
		else
			o = selexpr(s, args[1]);

		if (b->type != Locreg)
			b = inr(s, b);
		if (o->type == Loclit) {
			l = locmem(scale*o->lit, b, NULL, m);
		} else {
			b = inr(s, b);
			o = inr(s, o);
			l = locmems(0, b, o, scale, m);
		}
	} else {
		l = selexpr(s, e);
		l = inr(s, l);
		l = locmem(0, l, NULL, m);
	}
	assert(l != NULL);
	return l;
}

static const Mode szmodes[] = {
	[8] = ModeQ,
	[4] = ModeL,
	[2] = ModeW,
	[1] = ModeB
};
static void blit(Isel *s, Loc *to, Loc *from, size_t dstoff, size_t srcoff, size_t sz, size_t align)
{
	size_t i, modesz;
	Loc *sp, *dp, *len; /* pointers to src, dst */
	Loc *tmp, *src, *dst; /* source memory, dst memory */

	assert(szmodes[align] != ModeNone);   /* make sure we have a valid alignment */
	if (!sz)
		return;
	sp = inr(s, from);
	dp = inr(s, to);

	i = 0;
	if (align == 0)
		align = 8;
	if (sz <= 128) { /* arbitrary threshold; should be tuned */
		for (modesz = align; szmodes[modesz] != ModeNone; modesz /= 2) {
			tmp = locreg(szmodes[modesz]);
			while (i + modesz <= sz) {
				src = locmem(i + srcoff, sp, NULL, szmodes[modesz]);
				dst = locmem(i + dstoff, dp, NULL, szmodes[modesz]);
				g(s, Imov, src, tmp, NULL);
				g(s, Imov, tmp, dst, NULL);
				i += modesz;
			}
		}
	} else {
		len = loclit(sz, ModeQ);
		sp = newr(s, from);
		dp = newr(s, to);
{'fraction_non_alphanumeric': 0.1913199934572815, 'fraction_numerical': 0.009214328553513985, 'mean_word_length': 3.0951105157401204, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 155, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1703006', 'n_tokens_mistral': 8792, 'n_tokens_neox': 8263, 'n_words': 2382}