<commit_before>#!/bin/bash

# Integration testing for the clean.py script

##### Two rows

make spotless > /dev/null
./parse.py -d test/fixtures/xml/ -p. -x ipg120327.two.xml
mkdir -p /tmp/integration/ipg120327.two
python clean.py

for table in inventor inventor_1
do
  sqlite3 inventor.sqlite3 -csv "select * from ${table}"  > /tmp/integration/ipg120327.two/${table}.csv
  `diff test/integration/clean/ipg120327.two/${table}.csv /tmp/integration/ipg120327.two/${table}.csv`
done

for table in assignee assignee_1 grp wrd
do
  sqlite3 assignee.sqlite3 -csv "select * from ${table}"  > /tmp/integration/ipg120327.two/${table}.csv
  `diff test/integration/clean/ipg120327.two/${table}.csv /tmp/integration/ipg120327.two/${table}.csv`
done

for table in patent
do
  sqlite3 patent.sqlite3 -csv "select * from ${table}"  > /tmp/integration/ipg120327.two/${table}.csv
  `diff test/integration/clean/ipg120327.two/${table}.csv /tmp/integration/ipg120327.two/${table}.csv`
done


#### 18 rows

make spotless > /dev/null
./parse.py -d test/fixtures/xml/ -p. -x ipg120327.18.xml
mkdir -p /tmp/integration/ipg120327.18
python clean.py

for table in inventor inventor_1
do
  sqlite3 inventor.sqlite3 -csv "select * from ${table}"  > /tmp/integration/ipg120327.18/${table}.csv
  `diff test/integration/clean/ipg120327.18/${table}.csv /tmp/integration/ipg120327.18/${table}.csv`
done

for table in assignee assignee_1 grp wrd
do
  sqlite3 assignee.sqlite3 -csv "select * from ${table}"  > /tmp/integration/ipg120327.18/${table}.csv
  `diff test/integration/clean/ipg120327.18/${table}.csv /tmp/integration/ipg120327.18/${table}.csv`
done

for table in patent
do
  sqlite3 patent.sqlite3 -csv "select * from ${table}"  > /tmp/integration/ipg120327.18/${table}.csv
  `diff test/integration/clean/ipg120327.18/${table}.csv /tmp/integration/ipg120327.18/${table}.csv`
done

<commit_msg>added some descriptive logging to stdout<commit_after>#!/bin/bash

# Integration testing for the clean.py script

##### Two rows

make spotless > /dev/null
./parse.py -d test/fixtures/xml/ -p. -x ipg120327.two.xml
mkdir -p /tmp/integration/ipg120327.two


echo Starting clean...
python clean.py

echo Starting diffs...
for table in inventor inventor_1
do
  sqlite3 inventor.sqlite3 -csv "select * from ${table}"  > /tmp/integration/ipg120327.two/${table}.csv
  `diff test/integration/clean/ipg120327.two/${table}.csv /tmp/integration/ipg120327.two/${table}.csv`
done

for table in assignee assignee_1 grp wrd
do
  sqlite3 assignee.sqlite3 -csv "select * from ${table}"  > /tmp/integration/ipg120327.two/${table}.csv
  `diff test/integration/clean/ipg120327.two/${table}.csv /tmp/integration/ipg120327.two/${table}.csv`
done

for table in patent
do
  sqlite3 patent.sqlite3 -csv "select * from ${table}"  > /tmp/integration/ipg120327.two/${table}.csv
  `diff test/integration/clean/ipg120327.two/${table}.csv /tmp/integration/ipg120327.two/${table}.csv`
done


#### 18 rows

make spotless > /dev/null
./parse.py -d test/fixtures/xml/ -p. -x ipg120327.18.xml
mkdir -p /tmp/integration/ipg120327.18

echo Starting clean...
python clean.py

echo Starting diffs...
for table in inventor inventor_1
do
  sqlite3 inventor.sqlite3 -csv "select * from ${table}"  > /tmp/integration/ipg120327.18/${table}.csv
  `diff test/integration/clean/ipg120327.18/${table}.csv /tmp/integration/ipg120327.18/${table}.csv`
done

for table in assignee assignee_1 grp wrd
do
  sqlite3 assignee.sqlite3 -csv "select * from ${table}"  > /tmp/integration/ipg120327.18/${table}.csv
  `diff test/integration/clean/ipg120327.18/${table}.csv /tmp/integration/ipg120327.18/${table}.csv`
done

for table in patent
do
  sqlite3 patent.sqlite3 -csv "select * from ${table}"  > /tmp/integration/ipg120327.18/${table}.csv
  `diff test/integration/clean/ipg120327.18/${table}.csv /tmp/integration/ipg120327.18/${table}.csv`
done


{'fraction_non_alphanumeric': 0.15233160621761657, 'fraction_numerical': 0.08911917098445596, 'mean_word_length': 4.047058823529412, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1416431', 'n_tokens_mistral': 1650, 'n_tokens_neox': 1401, 'n_words': 303}