<commit_before>#!/usr/bin/perl

$type = shift;
$nodepartement = shift || 0;

%types = ('LABO' => 2, 'METIER' => 1, 'NATURE AVANTAGE' => 7, 'OBJET CONVENTION' => 10, 'BENEFICIAIRE'=> 4);

open(CSV, "data/all.anonymes.csv");
$premiereligne = <CSV>;
while(<CSV>) {
    @l = split /,/;
    $dep = $l[5];
    if ($nodepartement) {
	$dep = 'DESACTIVE';
    }
    if (!$data{$l[$types{$type}]}{$dep}) {
	$data{$l[$types{$type}]}{$dep}{'AVANTAGE'} = 0;
	$data{$l[$types{$type}]}{$dep}{'CONVENTION'} = 0;
	$data{$l[$types{$type}]}{$dep}{'AVANTAGE_MONTANT'} = 0;
    }
    $data{$l[$types{$type}]}{$dep}{$l[0]}++;
    $data{$l[$types{$type}]}{$dep}{'TOTAL'}++;
    if ($l[0] eq 'AVANTAGE') {
	$data{$l[$types{$type}]}{$dep}{'AVANTAGE_MONTANT'} += $l[8];
    }
    $data{$l[$types{$type}]}{$dep}{'QUALIFICATION'} = $l[3] if ($type eq 'BENEFICIAIRE' && $l[3] && $l[3] ne 'Médecine générale' && $l[3] ne 'Non renseigné');
    $data{$l[$types{$type}]}{$dep}{'QUALIFICATION'} = $l[3] if ($type eq 'BENEFICIAIRE' && !$data{$l[$types{$type}]}{$dep}{'QUALIFICATION'} && $l[3]);
    $data{$l[$types{$type}]}{$dep}{'ORIGIN'} = $l[1] if ($type eq 'BENEFICIAIRE');
}
print $type;
print ",ORIGIN,QUALIFICATION" if ($type eq 'BENEFICIAIRE');
print ",DEPARTEMENT,NB TOTAL CONVENTIONS + AVANTAGES,NB CONVENTIONS,NB AVANTAGES,MONTANT AVANTAGES\n";
foreach $key (keys %data) {
    foreach $dep (keys %{$data{$key}}) {
	print $key;
	print ','.$data{$key}{$dep}{'ORIGIN'} if ($type eq 'BENEFICIAIRE');
	print ','.$data{$key}{$dep}{'QUALIFICATION'} if ($type eq 'BENEFICIAIRE');
	print ",$dep,";
	print $data{$key}{$dep}{'TOTAL'};
	print ",";
	print $data{$key}{$dep}{'CONVENTION'};<commit_msg>Fix ORIGIN in beneficiaire aggregation<commit_after>#!/usr/bin/perl

$type = shift;
$nodepartement = shift || 0;

%types = ('LABO' => 2, 'METIER' => 1, 'NATURE AVANTAGE' => 7, 'OBJET CONVENTION' => 10, 'BENEFICIAIRE'=> 4);

open(CSV, "data/all.anonymes.csv");
$premiereligne = <CSV>;
while(<CSV>) {
    chomp;
    @l = split /,/;
    $dep = $l[5];
    if ($nodepartement) {
	$dep = 'DESACTIVE';
    }
    if (!$data{$l[$types{$type}]}{$dep}) {
	$data{$l[$types{$type}]}{$dep}{'AVANTAGE'} = 0;
	$data{$l[$types{$type}]}{$dep}{'CONVENTION'} = 0;
	$data{$l[$types{$type}]}{$dep}{'AVANTAGE_MONTANT'} = 0;
    }
    $data{$l[$types{$type}]}{$dep}{$l[0]}++;
    $data{$l[$types{$type}]}{$dep}{'TOTAL'}++;
    if ($l[0] eq 'AVANTAGE') {
	$data{$l[$types{$type}]}{$dep}{'AVANTAGE_MONTANT'} += $l[8];
    }
    $data{$l[$types{$type}]}{$dep}{'QUALIFICATION'} = $l[3] if ($type eq 'BENEFICIAIRE' && $l[3] && $l[3] ne 'Médecine générale' && $l[3] ne 'Non renseigné');
    $data{$l[$types{$type}]}{$dep}{'QUALIFICATION'} = $l[3] if ($type eq 'BENEFICIAIRE' && !$data{$l[$types{$type}]}{$dep}{'QUALIFICATION'} && $l[3]);

    if ($l[11] eq 'ETALAB' || !$data{$l[$types{$type}]}{$dep}{'ORIGIN'} ) {
        $data{$l[$types{$type}]}{$dep}{'ORIGIN'} = $l[1] if ($type eq 'BENEFICIAIRE');
    }
}
print $type;
print ",ORIGIN,QUALIFICATION" if ($type eq 'BENEFICIAIRE');
print ",DEPARTEMENT,NB TOTAL CONVENTIONS + AVANTAGES,NB CONVENTIONS,NB AVANTAGES,MONTANT AVANTAGES\n";
foreach $key (keys %data) {
    foreach $dep (keys %{$data{$key}}) {
	print $key;
	print ','.$data{$key}{$dep}{'ORIGIN'} if ($type eq 'BENEFICIAIRE');
	print ','.$data{$key}{$dep}{'QUALIFICATION'} if ($type eq 'BENEFICIAIRE');
	print ",$dep,";
	print $data{$key}{$dep}{'TOTAL'};
	print ",";
	print $data{$key}{$dep}{'CONVENTION'};
{'fraction_non_alphanumeric': 0.2847202087561612, 'fraction_numerical': 0.012757320962597855, 'mean_word_length': 5.73828125, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5506354', 'n_tokens_mistral': 1588, 'n_tokens_neox': 1500, 'n_words': 253}