<commit_before><commit_msg>not tested<commit_after>#!/bin/sh -x
# -*- mode: sh; -*-

# $Id$

# simple script to put all peptides files of the various sources in order.
# It collate the peptide files in case they're not one big file, and 
# makes the geneid appear as the first word of a '>...' line
# (this is only for ensembl, rest already has it like that).

# It's all done in this one script (unlike all-collates.sh, which does something
# similar for the raw gtf's, and which calls other scripts)

### sources="ensembl affymetrix fgenesh"

source=ensembl
cd $source
origfile=$source.pep.orig
pepfile=$source.pep
[ ! -f $origfile ] && echo "$origfile: Not found">&2 && exit 1
[ -f $pepfile ] && echo "found $pepfile; not overwriting">&2 && exit 1
sed '/^>/s/>\(ENSP[0-9]*\) Gene:\(ENSG[0-9]*\)\(.*$\)/>\2 \1\3/' < $origfile \
    >$pepfile  || exit 2

source=affymetrix
cd $source
pepfile=$source.pep
[ -f $pepfile ] && echo "found $pepfile; not overwriting">&2 && exit 1

chroms="1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 X Y UL NA"
echo "make sure that these are the right chromosomes:" | tee /dev/tty >&2
echo $chroms >&2
echo "" > /dev/tty
for chr in "$@"; do
    find . $chr -name '*.affymetrix.aa' -exec cat {} \;
done  > $pepfile

source=fgenesh
cd $source
pepfile=$source.pep

[ -f $pepfile ] && echo "found $pepfile; not overwriting">&2 && exit 1
for i in   chr_pro/*.sgp.pro chr_r_pro/*.sgp.pro; do
    cat $i
done > $pepfile





{'fraction_non_alphanumeric': 0.15742642026009582, 'fraction_numerical': 0.03627652292950034, 'mean_word_length': 3.443768996960486, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4595475', 'n_tokens_mistral': 601, 'n_tokens_neox': 539, 'n_words': 200}