<commit_before>#!/bin/bash
#
# Script to generate a list of AUs to test in the gln.
#
# cat tdb/prod/*.tdb | ./scripts/tdb/tdbout -MWTN -t publisher,plugin,publisher:info[tester],status,year | ./scripts/tdb/statuses_gln.awk


tpath="/home/$LOGNAME/tmp"
mkdir -p $tpath
  echo "taylor_and_francis.tdb" > $tpath/glnfilter
  echo "sage_publications.tdb" >> $tpath/glnfilter

  ls ../../tdb/prod | grep .tdb > $tpath/glnlist
  ls ../../tdb/clockssingest | grep .tdb > $tpath/clocksslist
  diff $tpath/glnlist $tpath/clocksslist | grep "< " | sed 's/..//' > $tpath/notclockss
  echo "american_medical_association.tdb" >> $tpath/notclockss
  echo "purdue_university_press.tdb" >> $tpath/notclockss
  echo "centro_de_filosofia_da_universidade_de_lisboa.tdb" >> $tpath/notclockss
  diff $tpath/glnlist $tpath/notclockss | grep "< " | sed 's/..//' > $tpath/glnAndclockss

#Report any AU in the gln marked notReady, wanted, or testing.
cat ../../tdb/prod/*.tdb | ./tdbout -NWT -t publisher,plugin,publisher:info[tester],status,year > $tpath/glntest

#Report any AU in the gln marked manifest that does not have a file in clockss.
for file in `cat $tpath/notclockss`
do
  ./tdbout -M -t publisher,plugin,publisher:info[tester],status,year -i ../../tdb/prod/$file >> $tpath/glntest
done

#Report some AUs in the gln marked manifest that do not have an equivalent in clockss.
for file in `cat $tpath/glnAndclockss`
do
  if ! grep $file $tpath/glnfilter > /dev/null
  then
    ./tdbout -M -t publisher,plugin,publisher:info[tester],status,year -Q 'plugin ~ "OJS" or plugin ~ "ProjectMuse"' -i ../../tdb/prod/$file >> $tpath/glntest
    ./tdbout -M -t publisher,plugin,publisher:info[tester],status,year -Q 'plugin !~ "OJS" or plugin !~ "ProjectMuse"' -i ../../tdb/prod/$file | 'sed/^/*/' >> $tpath/glntest
#    ./tdbout -M -t auid,publisher,plugin,publisher:info[tester],status -i ../../tdb/prod/$file > $tpath/glnM
#    cat $tpath/glnM | sed 's/\t.*$//' | sed -e 's/^[^&]*&//' | sort | uniq > $tpath/glnauids
#    ./tdbout -M -t auid | sed -e 's/^[^&]*&//' -i ../../tdb/clockssingest/$file | sort | uniq > $tpath/clockssauids
#    diff $tpath/glnauids $tpath/clockssauids | grep "> " | sed 's/..//' > $tpath/mysteryauids
  fi
done
cat $tpath/glntest | sort | grep -v "needs.plugin"
exit 0




<commit_msg>Add clockss publishers to GLN test list.<commit_after>#!/bin/bash
#
# Script to generate a list of AUs to test in the gln.
#
# cat tdb/prod/*.tdb | ./scripts/tdb/tdbout -MWTN -t publisher,plugin,publisher:info[tester],status,year | ./scripts/tdb/statuses_gln.awk


tpath="/home/$LOGNAME/tmp"
mkdir -p $tpath
  echo "taylor_and_francis.tdb" > $tpath/glnfilter
  echo "sage_publications.tdb" >> $tpath/glnfilter

  ls ../../tdb/prod | grep .tdb > $tpath/glnlist
  ls ../../tdb/clockssingest | grep .tdb > $tpath/clocksslist
  diff $tpath/glnlist $tpath/clocksslist | grep "< " | sed 's/..//' > $tpath/notclockss
  echo "american_medical_association.tdb" >> $tpath/notclockss
  echo "purdue_university_press.tdb" >> $tpath/notclockss
  echo "centro_de_filosofia_da_universidade_de_lisboa.tdb" >> $tpath/notclockss
  diff $tpath/glnlist $tpath/notclockss | grep "< " | sed 's/..//' > $tpath/glnAndclockss

#Report any AU in the gln marked notReady, wanted, or testing.
cat ../../tdb/prod/*.tdb | ./tdbout -NWT -t publisher,plugin,publisher:info[tester],status,year > $tpath/glntest

#Report any AU in the gln marked manifest that does not have a file in clockss.
for file in `cat $tpath/notclockss`
do
  ./tdbout -M -t publisher,plugin,publisher:info[tester],status,year -i ../../tdb/prod/$file >> $tpath/glntest
done

#Report some AUs in the gln marked manifest that do not have an equivalent in clockss.
for file in `cat $tpath/glnAndclockss`
do
  if ! grep $file $tpath/glnfilter > /dev/null
  then
    ./tdbout -M -t publisher,plugin,publisher:info[tester],status,year -Q 'plugin ~ "OJS" or plugin ~ "ProjectMuse"' -i ../../tdb/prod/$file >> $tpath/glntest
    ./tdbout -M -t publisher,plugin,publisher:info[tester],status,year -Q 'plugin !~ "OJS" or plugin !~ "ProjectMuse"' -i ../../tdb/prod/$file | sed 's/^/*/' >> $tpath/glntest
#    ./tdbout -M -t auid,publisher,plugin,publisher:info[tester],status -i ../../tdb/prod/$file > $tpath/glnM
#    cat $tpath/glnM | sed 's/\t.*$//' | sed -e 's/^[^&]*&//' | sort | uniq > $tpath/glnauids
#    ./tdbout -M -t auid | sed -e 's/^[^&]*&//' -i ../../tdb/clockssingest/$file | sort | uniq > $tpath/clockssauids
#    diff $tpath/glnauids $tpath/clockssauids | grep "> " | sed 's/..//' > $tpath/mysteryauids
  fi
done
cat $tpath/glntest | sort | grep -v "needs.plugin"
exit 0





{'fraction_non_alphanumeric': 0.17840884456969433, 'fraction_numerical': 0.00043355733795794494, 'mean_word_length': 3.449373191899711, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3369250', 'n_tokens_mistral': 1905, 'n_tokens_neox': 1852, 'n_words': 431}