<commit_before>#! /bin/bash
t="/home/$LOGNAME/tmp"
year=`date +%Y`

#Highwire
#Ready
echo "###Ready highwire" > $t/tmp_HW  #clear out file.
./scripts/tdb/tdbout -Y -a -Q 'plugin ~ "highwire" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/*.tdb | grep -v oxfordjournals | shuf >> $t/tmp_HW

#Atypon (not T&F), in order by year
#Ready
echo "###Ready Atypon $year" > $t/tmp_Atypon  #clear out file
./scripts/tdb/tdbout -Y -a -Q 'plugin ~ "typon" and year is "'$year'" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/*.tdb | shuf >> $t/tmp_Atypon
echo "###Ready Atypon not $year" >> $t/tmp_Atypon
./scripts/tdb/tdbout -Y -a -Q 'plugin ~ "typon" and year is not "'$year'" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/*.tdb | shuf >> $t/tmp_Atypon
#Manifest
#Don't include this year or last year so that new titles get tested for at least two years.
#echo "###Manifest Sage 2019" >> $t/tmp_Atypon
#./scripts/tdb/tdbout -MT -a -Q 'year ~ "2019$" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/sage_publications.atypon.tdb | shuf >> $t/tmp_Atypon
echo "###Manifest Sage 2018" >> $t/tmp_Atypon
./scripts/tdb/tdbout -MT -a -Q 'year ~ "2018$" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/sage_publications.atypon.tdb | shuf >> $t/tmp_Atypon
echo "###Manifest Sage 2017" >> $t/tmp_Atypon
./scripts/tdb/tdbout -MT -a -Q 'year ~ "2017$" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/sage_publications.atypon.tdb | shuf >> $t/tmp_Atypon
echo "###Manifest Sage 2016" >> $t/tmp_Atypon
./scripts/tdb/tdbout -MT -a -Q 'year ~ "2016$" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/sage_publications.atypon.tdb | shuf >> $t/tmp_Atypon
echo "###Manifest Sage 2015" >> $t/tmp_Atypon
./scripts/tdb/tdbout -MT -a -Q 'year ~ "2015$" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/sage_publications.atypon.tdb | shuf >> $t/tmp_Atypon
echo "###Manifest Sage 2014" >> $t/tmp_Atypon
./scripts/tdb/tdbout -MT -a -Q 'year ~ "2014$" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/sage_publications.atypon.tdb | shuf >> $t/tmp_Atypon
echo "###Manifest Sage 2013" >> $t/tmp_Atypon
./scripts/tdb/tdbout -MT -a -Q 'year ~ "2013$" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/sage_publications.atypon.tdb | shuf >> $t/tmp_Atypon
echo "###Manifest Sage 2012" >> $t/tmp_Atypon
./scripts/tdb/tdbout -MT -a -Q 'year ~ "2012$" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/sage_publications.atypon.tdb | shuf >> $t/tmp_Atypon
echo "###Manifest Sage 2011" >> $t/tmp_Atypon
./scripts/tdb/tdbout -MT -a -Q 'year ~ "2011$" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/sage_publications.atypon.tdb | shuf >> $t/tmp_Atypon
echo "###Manifest Sage 2010" >> $t/tmp_Atypon
./scripts/tdb/tdbout -MT -a -Q 'year ~ "2010$" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/sage_publications.atypon.tdb | shuf >> $t/tmp_Atypon
echo "###Manifest Sage 2009" >> $t/tmp_Atypon
./scripts/tdb/tdbout -MT -a -Q 'year ~ "2009$" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/sage_publications.atypon.tdb | shuf >> $t/tmp_Atypon<commit_msg>Start adding 2019 automatically.<commit_after>#! /bin/bash
t="/home/$LOGNAME/tmp"
year=`date +%Y`

#Highwire
#Ready
echo "###Ready highwire" > $t/tmp_HW  #clear out file.
./scripts/tdb/tdbout -Y -a -Q 'plugin ~ "highwire" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/*.tdb | grep -v oxfordjournals | shuf >> $t/tmp_HW

#Atypon (not T&F), in order by year
#Ready
echo "###Ready Atypon $year" > $t/tmp_Atypon  #clear out file
./scripts/tdb/tdbout -Y -a -Q 'plugin ~ "typon" and year is "'$year'" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/*.tdb | shuf >> $t/tmp_Atypon
echo "###Ready Atypon not $year" >> $t/tmp_Atypon
./scripts/tdb/tdbout -Y -a -Q 'plugin ~ "typon" and year is not "'$year'" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/*.tdb | shuf >> $t/tmp_Atypon
#Manifest
#Don't include this year or last year so that new titles get tested for at least two years.
#echo "###Manifest Sage 2020" >> $t/tmp_Atypon
#./scripts/tdb/tdbout -MT -a -Q 'year ~ "2020$" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/sage_publications.atypon.tdb | shuf >> $t/tmp_Atypon
echo "###Manifest Sage 2019" >> $t/tmp_Atypon
./scripts/tdb/tdbout -MT -a -Q 'year ~ "2019$" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/sage_publications.atypon.tdb | shuf >> $t/tmp_Atypon
echo "###Manifest Sage 2018" >> $t/tmp_Atypon
./scripts/tdb/tdbout -MT -a -Q 'year ~ "2018$" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/sage_publications.atypon.tdb | shuf >> $t/tmp_Atypon
echo "###Manifest Sage 2017" >> $t/tmp_Atypon
./scripts/tdb/tdbout -MT -a -Q 'year ~ "2017$" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/sage_publications.atypon.tdb | shuf >> $t/tmp_Atypon
echo "###Manifest Sage 2016" >> $t/tmp_Atypon
./scripts/tdb/tdbout -MT -a -Q 'year ~ "2016$" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/sage_publications.atypon.tdb | shuf >> $t/tmp_Atypon
echo "###Manifest Sage 2015" >> $t/tmp_Atypon
./scripts/tdb/tdbout -MT -a -Q 'year ~ "2015$" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/sage_publications.atypon.tdb | shuf >> $t/tmp_Atypon
echo "###Manifest Sage 2014" >> $t/tmp_Atypon
./scripts/tdb/tdbout -MT -a -Q 'year ~ "2014$" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/sage_publications.atypon.tdb | shuf >> $t/tmp_Atypon
echo "###Manifest Sage 2013" >> $t/tmp_Atypon
./scripts/tdb/tdbout -MT -a -Q 'year ~ "2013$" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/sage_publications.atypon.tdb | shuf >> $t/tmp_Atypon
echo "###Manifest Sage 2012" >> $t/tmp_Atypon
./scripts/tdb/tdbout -MT -a -Q 'year ~ "2012$" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/sage_publications.atypon.tdb | shuf >> $t/tmp_Atypon
echo "###Manifest Sage 2011" >> $t/tmp_Atypon
./scripts/tdb/tdbout -MT -a -Q 'year ~ "2011$" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/sage_publications.atypon.tdb | shuf >> $t/tmp_Atypon
echo "###Manifest Sage 2010" >> $t/tmp_Atypon
./scripts/tdb/tdbout -MT -a -Q 'year ~ "2010$" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/sage_publications.atypon.tdb | shuf >> $t/tmp_Atypon
echo "###Manifest Sage 2009" >> $t/tmp_Atypon
./scripts/tdb/tdbout -MT -a -Q 'year ~ "2009$" and (au:hidden[proxy] is not set or au:hidden[proxy] is "")' tdb/clockssingest/sage_publications.atypon.tdb | shuf >> $t/tmp_Atypon
{'fraction_non_alphanumeric': 0.1802309208673613, 'fraction_numerical': 0.02647141650239369, 'mean_word_length': 4.519036519036519, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 115, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '642970', 'n_tokens_mistral': 3066, 'n_tokens_neox': 2938, 'n_words': 744}