<commit_before>include ../../common.mk
include ../../local.mk


PDB_ROOT:=${COMPBIO_PREFIX}/share/pdb
export PDB_XML_PATH:=${PDB_ROOT}/all.xml.gz


update default all: allxml-todo.load


%.load: %.ids
	(time -p ./load-pdblite < $<) >$@.err 2>&1;
	/bin/mv -f $@.err $@.log; touch $@

allxml.ids: exclude.ids FORCE
	find ${PDB_XML_PATH}/ -name \*.gz | perl -lne 'print $$1 if m%/(\w{4})\.xml\.gz%' >$@.tmp
	sort -u -o $@.tmp $@.tmp
	comm -23 $@.tmp $< >$@.tmp2
	/bin/mv -f $@.tmp2 $@
	/bin/rm $@.tmp
	@wc -l $@

done.ids:
	psql -Atc 'select pdbid from pdb.summary' >$@.tmp
	sort -u -o $@.tmp $@.tmp
	/bin/mv -f $@.tmp $@
	@wc -l $@<commit_msg>replace compbio with research<commit_after>include ../../local.mk
include ../../common.mk
include ../../params.mk



PDB_ROOT:=${RESEARCH_PREFIX}/share/pdb
export PDB_XML_PATH:=${PDB_ROOT}/all.xml.gz


update default all: allxml-todo.load


%.load: %.ids
	(time -p ./load-pdblite < $<) >$@.err 2>&1;
	/bin/mv -f $@.err $@.log; touch $@

allxml.ids: exclude.ids FORCE
	find ${PDB_XML_PATH}/ -name \*.gz | perl -lne 'print $$1 if m%/(\w{4})\.xml\.gz%' >$@.tmp
	sort -u -o $@.tmp $@.tmp
	comm -23 $@.tmp $< >$@.tmp2
	/bin/mv -f $@.tmp2 $@
	/bin/rm $@.tmp
	@wc -l $@

done.ids:
	psql -Atc 'select pdbid from pdb.summary' >$@.tmp
	sort -u -o $@.tmp $@.tmp
	/bin/mv -f $@.tmp $@
	@wc -l $@
{'fraction_non_alphanumeric': 0.25604229607250756, 'fraction_numerical': 0.012084592145015106, 'mean_word_length': 2.660220994475138, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '792514', 'n_tokens_mistral': 651, 'n_tokens_neox': 597, 'n_words': 147}