<commit_before>#! /bin/bash
#
# Script that creates a report on the status of plugins
#

tpath="/home/$LOGNAME/tmp"
mkdir -p $tpath

cd ../../plugins/src

#To get the plugins with AU counts:
grep -rl --include "*.xml" "plugin_identifier" * | sed 's/\(.*\).xml/\1/' | sort -u > $tpath/ab.txt
cat ../../tdb/*/*.tdb | ../../scripts/tdb/tdbout -t plugin | sort -u | sed 's/\./\//g' > $tpath/ac.txt
#plugins that have no AUs.
diff $tpath/ab.txt $tpath/ac.txt | grep "^< " | sed 's/..\(.*\)/\1,0/' > $tpath/foo00.txt
#plugins that have AUs
#shopt -s extglob
find ../../tdb -name clockssingest -prune -o -name \*.tdb -print | xargs cat | ../../scripts/tdb/tdbout -t plugin | sed 's/\./\//g' | awk -F"\t" 'BEGIN {OFS=","} {foo[$0]++} END {for(x in foo) {print x,foo[x]}}' >> $tpath/foo00.txt
#shopt -u extglob

#To get the plugins with and without status:
grep -r -A 1 --include "*.xml" "<string>plugin_status</string>" * | grep "\- " | sed 's/\(.*\).xml-.*<string>\(.*\)<\/string>/\1,\2/' > $tpath/foo01.txt
grep -rL --include "*.xml" "<string>plugin_status</string>" * | sed 's/\(.*\).xml/\1,!/' >> $tpath/foo01.txt

#Create a base list of plugins
cat $tpath/foo01.txt | sed 's/,.*//' | sort -t, -k 1,1 > $tpath/AllPlugins.txt

#To get the plugins with and without substance checkers:
grep -r --include "*.xml" "<string>au_substance_url_pattern</string>" * | sed 's/\(.*\).xml.*<string>.*<\/string>/\1,substance/' > $tpath/foo02.txt
grep -rL --include "*.xml" "<string>au_substance_url_pattern</string>" * | sed 's/\(.*\).xml/\1,!/' >> $tpath/foo02.txt

#To get the plugins with and without article iterators:
grep -r --include "*.xml" "<string>plugin_article_iterator_factory</string>" * | sed 's/\(.*\).xml.*<string>.*<\/string>/\1,artit/' > $tpath/foo03.txt<commit_msg>Remove filter for clockss/clockssingest duplicate AUs.<commit_after>#! /bin/bash
#
# Script that creates a report on the status of plugins
#

tpath="/home/$LOGNAME/tmp"
mkdir -p $tpath

cd ../../plugins/src

#To get the plugins with AU counts:
grep -rl --include "*.xml" "plugin_identifier" * | sed 's/\(.*\).xml/\1/' | sort -u > $tpath/ab.txt
cat ../../tdb/*/*.tdb | ../../scripts/tdb/tdbout -t plugin | sort -u | sed 's/\./\//g' > $tpath/ac.txt
#plugins that have no AUs.
diff $tpath/ab.txt $tpath/ac.txt | grep "^< " | sed 's/..\(.*\)/\1,0/' > $tpath/foo00.txt
#plugins that have AUs
#shopt -s extglob
find ../../tdb -name \*.tdb -print | xargs cat | ../../scripts/tdb/tdbout -t plugin | sed 's/\./\//g' | awk -F"\t" 'BEGIN {OFS=","} {foo[$0]++} END {for(x in foo) {print x,foo[x]}}' >> $tpath/foo00.txt
#shopt -u extglob

#To get the plugins with and without status:
grep -r -A 1 --include "*.xml" "<string>plugin_status</string>" * | grep "\- " | sed 's/\(.*\).xml-.*<string>\(.*\)<\/string>/\1,\2/' > $tpath/foo01.txt
grep -rL --include "*.xml" "<string>plugin_status</string>" * | sed 's/\(.*\).xml/\1,!/' >> $tpath/foo01.txt

#Create a base list of plugins
cat $tpath/foo01.txt | sed 's/,.*//' | sort -t, -k 1,1 > $tpath/AllPlugins.txt

#To get the plugins with and without substance checkers:
grep -r --include "*.xml" "<string>au_substance_url_pattern</string>" * | sed 's/\(.*\).xml.*<string>.*<\/string>/\1,substance/' > $tpath/foo02.txt
grep -rL --include "*.xml" "<string>au_substance_url_pattern</string>" * | sed 's/\(.*\).xml/\1,!/' >> $tpath/foo02.txt

#To get the plugins with and without article iterators:
grep -r --include "*.xml" "<string>plugin_article_iterator_factory</string>" * | sed 's/\(.*\).xml.*<string>.*<\/string>/\1,artit/' > $tpath/foo03.txt
{'fraction_non_alphanumeric': 0.2514817950889077, 'fraction_numerical': 0.016370307648885127, 'mean_word_length': 3.0410490307867732, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 24}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5978723', 'n_tokens_mistral': 1446, 'n_tokens_neox': 1394, 'n_words': 358}