<commit_before>#!/bin/bash

url=$1
output=$2

if ! test "$output"; then
    output="out/"$(echo $url | base64 -w 0 )".json"
fi

docid=$(echo $url | sed 's|.*/||' | sed 's/.asp//' | awk -F '-' '{print $1}' | sed 's/[^0-9]//g')

find opendata/document/ -name '*'$docid'*' | while read file ; do
    python3 parse_opendata_documents.py $file > "/tmp/documents_$$.json"
    if grep "$url" "/tmp/documents_$$.json"  > "$output" ; then
        break;
    fi
done
rm -f /tmp/documents_$$.json

if ! test -s "$output"; then
    echo "$0: $url not found" 1>&2 ;
    rm -f "$output"
elif ! jq . < "$output" | grep '"contenu": "[^"]' > /dev/null; then
    echo "$0: erreur de contenu avec $url" 1>&2 ;
fi<commit_msg>Format url<commit_after>#!/bin/bash

url=$1
output=$2

if ! test "$output"; then
    output="out/"$(echo $url | base64 -w 0 )".json"
fi

docid=$(echo $url | sed 's|.*/||' | sed 's/.asp//' | awk -F '-' '{print $1}' | sed 's/[^0-9]//g')

find opendata/document/ -name '*'$docid'*' | while read file ; do
    python3 parse_opendata_documents.py $file > "/tmp/documents_$$.json"
    uri=$(echo $url | sed 's|https://www.assemblee-nationale.fr/[^0-9]*/||')
    if grep "$uri" "/tmp/documents_$$.json"  > "$output" ; then
        break;
    fi
done
rm -f /tmp/documents_$$.json

if ! test -s "$output"; then
    echo "$0: $url not found" 1>&2 ;
    rm -f "$output"
elif ! jq . < "$output" | grep '"contenu": "[^"]' > /dev/null; then
    echo "$0: erreur de contenu avec $url" 1>&2 ;
fi
{'fraction_non_alphanumeric': 0.23315363881401618, 'fraction_numerical': 0.0215633423180593, 'mean_word_length': 2.96, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '812430', 'n_tokens_mistral': 618, 'n_tokens_neox': 592, 'n_words': 164}