<commit_before>#!/usr/bin/perl

use strict;
use warnings;

use Data::Dumper;
use XML::Simple;

my ($fin, $fout) = (\*STDIN, \*STDOUT);

if (my $ifname = shift) {
	open $fin, '<', $ifname or die $!;
	if (my $ofname = shift) {
		open $fout, '>', $ofname or die $!;
	}
}

my $xml = new XML::Simple;
my $data = $xml->XMLin($fin);

print $fout <<END;
#include "foreign.h"
#include "interp.hh"
#include "util.hh"

#include <string>
#include <vector>
using namespace std;

simple_registry<Interpreter::Function> Interpreter::foreignFunctions({
END

my $IFUNC = "[] (vector <Param> ps, vector <Value> as) -> Value";
for my $catname (keys $data->{'category'}) {
	my $catcname = $data->{'category'}{$catname}{'cname'};
	my $fs = $data->{'category'}{$catname}{'foreign'};
	for my $fname (keys %$fs) {
		my $fcname = $fs->{$fname}{cname};
		print STDERR Dumper($fs->{$fname});
		print "\t{\"$fname\", ($IFUNC {\n";
		print "\t\treturn 0;\n";
		print "\t})},\n";
	}
}

print $fout <<END;
});
END

close $fin;
close $fout;
<commit_msg>Foreign interpreter wrappings seem to work, but need some code added to Value<commit_after>#!/usr/bin/perl

use strict;
use warnings;

use Data::Dumper;
use XML::Simple;

my ($fin, $fout) = (\*STDIN, \*STDOUT);

if (my $ifname = shift) {
	open $fin, '<', $ifname or die $!;
	if (my $ofname = shift) {
		open $fout, '>', $ofname or die $!;
	}
}

my $xml = new XML::Simple;
my $data = $xml->XMLin($fin);

print $fout <<END;
#include "foreign.h"
#include "interp.hh"
#include "util.hh"

#include <string>
#include <vector>
using namespace std;

simple_registry<Interpreter::Function> Interpreter::foreignFunctions({
END

my $IFUNC = "[] (vector <Param> ps, vector <Value> as) -> Value";
for my $catname (keys $data->{'category'}) {
	my $catcname = $data->{'category'}{$catname}{'cname'};
	my $fs = $data->{'category'}{$catname}{'foreign'};
	for my $fname (keys %$fs) {
		my $fcname = $fs->{$fname}{cname};
		my @args;
		my $type = $fs->{$fname}{type};
		my $parameters = $type->{parameter} || [];
		$parameters = [$parameters] if (ref $parameters eq 'HASH');
		my $pn = 0;
		for my $p (@$parameters) {
			die "parameters must be builtins" unless $p->{builtin};
			push @args, "ps[$pn]";
			$pn++;
		}

		my $arguments = $type->{argument} || [];
		$arguments = [$arguments] if (ref $arguments eq 'HASH');
		my $an = 0;
		for my $p (@$arguments) {
			push @args, "as[$an]";
			$an++;
		}

		my $argstr = join ',', @args;
		print "\t{\"$fname\", ($IFUNC {\n";
		print "\t\treturn $fcname($argstr);\n";
		print "\t})},\n";
	}
}

print $fout <<END;
});
END

close $fin;
close $fout;

{'fraction_non_alphanumeric': 0.22765793528505393, 'fraction_numerical': 0.0011556240369799693, 'mean_word_length': 4.225352112676056, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7635770', 'n_tokens_mistral': 1093, 'n_tokens_neox': 1036, 'n_words': 279}