<commit_before>#!/bin/bash

# First crack at building integration test framework for
# ensuring the preprocessing works correctly as we move
# forward fixing bugs, etc.

make spotless > /dev/null
mkdir -p /tmp/integration/ipg120327.one
./parse.py -d test/fixtures/xml/ -p. -x ipg120327.one.xml

for table in assignee citation class inventor lawyer patdesc patent sciref usreldoc
do
  sqlite3 ${table}.sqlite3 -csv "select * from ${table}"  > /tmp/integration/ipg120327.one/${table}.csv
  `diff test/integration/parse/ipg120327.one/${table}.csv /tmp/integration/ipg120327.one/${table}.csv`
done

# TODO: Refactor
make spotless > /dev/null
mkdir -p /tmp/integration/ipg120327.two
./parse.py -d test/fixtures/xml/ -p. -x ipg120327.two.xml

for table in assignee citation class inventor lawyer patdesc patent sciref usreldoc
do
  sqlite3 ${table}.sqlite3 -csv "select * from ${table}"  > /tmp/integration/ipg120327.two/${table}.csv
  `diff test/integration/parse/ipg120327.two/${table}.csv /tmp/integration/ipg120327.two/${table}.csv`
done

<commit_msg>fixed sqlite3 options for parse integration<commit_after>#!/bin/bash

# First crack at building integration test framework for
# ensuring the preprocessing works correctly as we move
# forward fixing bugs, etc.

make spotless > /dev/null
mkdir -p /tmp/integration/ipg120327.one
./parse.py -d test/fixtures/xml/ -p. -x ipg120327.one.xml

for table in assignee citation class inventor lawyer patdesc patent sciref usreldoc
do
  sqlite3 -csv ${table}.sqlite3 "select * from ${table}"  > /tmp/integration/ipg120327.one/${table}.csv
  `diff test/integration/parse/ipg120327.one/${table}.csv /tmp/integration/ipg120327.one/${table}.csv`
done

# TODO: Refactor
make spotless > /dev/null
mkdir -p /tmp/integration/ipg120327.two
./parse.py -d test/fixtures/xml/ -p. -x ipg120327.two.xml

for table in assignee citation class inventor lawyer patdesc patent sciref usreldoc
do
  sqlite3 -csv ${table}.sqlite3 "select * from ${table}"  > /tmp/integration/ipg120327.two/${table}.csv
  `diff test/integration/parse/ipg120327.two/${table}.csv /tmp/integration/ipg120327.two/${table}.csv`
done


{'fraction_non_alphanumeric': 0.12781954887218044, 'fraction_numerical': 0.0606203007518797, 'mean_word_length': 4.192682926829268, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4852205', 'n_tokens_mistral': 826, 'n_tokens_neox': 719, 'n_words': 195}