<commit_before>import os
from math import sqrt
import saliweb.backend

class Job(saliweb.backend.Job):
    runnercls = saliweb.backend.SGERunner

    def get_protection_args(self):
        args = ['--blimit_fitting=240', '--elimit_fitting=240',
                 '--blimit_hessian=80', '--elimit_hessian=80',
                 '-v -v -v']
        return ' '.join(args)

    def get_input_args(self):
        args = [l.rstrip() for l in open('input.txt')]
        return ' '.join(args)

    def get_args(self):
        return self.get_input_args() + ' ' + self.get_protection_args()

    def complete(self):
        os.chmod(".", 0775)
    
    def standardize(self, infile, outfile):
        data=[i.split()[:3] for i in open(infile).readlines()]
        fl=open(outfile,'w')
        n=0
        ntot = len(data)
        while n < ntot:
            i=0
            chunk=[]
            while n+i < ntot and data[n][0] == data[n+i][0]:
                chunk.append(data[n+i])
                i += 1
            n += i
            if len(chunk) == 1:
                outline = tuple(chunk[0])
            else:
                qval = chunk[0][0]
                Is = [float(k[1]) for k in chunk]
                ss = [float(k[2]) for k in chunk]
                s2mean = len(ss)/sum([1/k**2 for k in ss])
                Imean = s2mean/len(ss)*sum([k/l**2 for (k,l) in zip(Is,ss)])
                outline = (qval, '%-15.14G' % Imean, '%-15.14G' % (sqrt(s2mean)))
            fl.write(' '.join(outline))
            fl.write('\n')

    def preprocess(self):
        """store all used options for statistical purposes"""
        args = [l.rstrip() for l in open('input.txt') if l.startswith('-')]
        args.sort()
        wd = os.getcwd().split('/')[-1].split('_')
        args = wd + args
        fl=open('/modbase4/home/saxsmerge/saxsmerge_stats.log','a')
        if fl:
            fl.write('\t'.join(args))
            fl.write('\n')

    def postprocess(self):
        if os.path.isfile('data_merged.dat'):
            self.standardize('data_merged.dat','data_merged_3col.dat')
        if os.path.isfile('mean_merged.dat'):
            self.standardize('mean_merged.dat','mean_merged_3col.dat')
        os.system('zip -q -9 saxsmerge.zip data_* mean_* summary.txt saxsmerge.log')

    def run(self):
        args = self.get_args()
        post = self.gen_gnuplots()
        script="""
date
hostname

IMPPY="/netapp/sali/saxsmerge/imp/yannick/build-fast/setup_environment.sh"
SMERGE="/netapp/sali/saxsmerge/imp/yannick/src/applications/saxs_merge/saxs_merge.py"

#. /netapp/sali/yannick/.bashrc
#export PATH="/netapp/sali/yannick/bin:$PATH"
#export LD_LIBRARY_PATH="/netapp/sali/yannick/lib:$LD_LIBRARY_PATH"
#export CPPFLAGS="/netapp/sali/yannick/include:$CPPFLAGS"

$IMPPY $SMERGE %s

cat <<EOF > Cpgnuplot
%s
EOF
/netapp/sali/yannick/bin/gnuplot Cpgnuplot

date
""" % (args,post)
        r = self.runnercls(script)
        r.set_sge_options('-l arch=linux-x64 -j y -o saxsmerge.log')
        return r
    
    def plot_log_scale(self,outfile,subs):
        datafile = "data_merged.dat"
        meanfile = "mean_merged.dat"
        script="reset\n"
        script += 'set terminal canvas solid butt size 400,350 fsize 10 '
        script += 'lw 1.5 fontscale 1 name "%s" jsdir "."\n' % outfile
        script += 'set title "merged data log-scale"\n'
        script += 'set log y\n'<commit_msg>Remove obsolete code.<commit_after>import os
from math import sqrt
import saliweb.backend

class Job(saliweb.backend.Job):
    runnercls = saliweb.backend.SGERunner

    def get_protection_args(self):
        args = ['--blimit_fitting=240', '--elimit_fitting=240',
                 '--blimit_hessian=80', '--elimit_hessian=80',
                 '-v -v -v']
        return ' '.join(args)

    def get_input_args(self):
        args = [l.rstrip() for l in open('input.txt')]
        return ' '.join(args)

    def get_args(self):
        return self.get_input_args() + ' ' + self.get_protection_args()

    def complete(self):
        os.chmod(".", 0775)
    
    def standardize(self, infile, outfile):
        data=[i.split()[:3] for i in open(infile).readlines()]
        fl=open(outfile,'w')
        n=0
        ntot = len(data)
        while n < ntot:
            i=0
            chunk=[]
            while n+i < ntot and data[n][0] == data[n+i][0]:
                chunk.append(data[n+i])
                i += 1
            n += i
            if len(chunk) == 1:
                outline = tuple(chunk[0])
            else:
                qval = chunk[0][0]
                Is = [float(k[1]) for k in chunk]
                ss = [float(k[2]) for k in chunk]
                s2mean = len(ss)/sum([1/k**2 for k in ss])
                Imean = s2mean/len(ss)*sum([k/l**2 for (k,l) in zip(Is,ss)])
                outline = (qval, '%-15.14G' % Imean, '%-15.14G' % (sqrt(s2mean)))
            fl.write(' '.join(outline))
            fl.write('\n')

    def preprocess(self):
        """store all used options for statistical purposes"""
        args = [l.rstrip() for l in open('input.txt') if l.startswith('-')]
        args.sort()
        wd = os.getcwd().split('/')[-1].split('_')
        args = wd + args
        fl=open('/modbase4/home/saxsmerge/saxsmerge_stats.log','a')
        if fl:
            fl.write('\t'.join(args))
            fl.write('\n')

    def postprocess(self):
        if os.path.isfile('data_merged.dat'):
            self.standardize('data_merged.dat','data_merged_3col.dat')
        if os.path.isfile('mean_merged.dat'):
            self.standardize('mean_merged.dat','mean_merged_3col.dat')
        os.system('zip -q -9 saxsmerge.zip data_* mean_* summary.txt saxsmerge.log')

    def run(self):
        args = self.get_args()
        post = self.gen_gnuplots()
        script="""
date
hostname

IMPPY="/netapp/sali/saxsmerge/imp/yannick/build-fast/setup_environment.sh"
SMERGE="/netapp/sali/saxsmerge/imp/yannick/src/applications/saxs_merge/saxs_merge.py"

$IMPPY $SMERGE %s

cat <<EOF > Cpgnuplot
%s
EOF
/netapp/sali/yannick/bin/gnuplot Cpgnuplot

date
""" % (args,post)
        r = self.runnercls(script)
        r.set_sge_options('-l arch=linux-x64 -j y -o saxsmerge.log')
        return r
    
    def plot_log_scale(self,outfile,subs):
        datafile = "data_merged.dat"
        meanfile = "mean_merged.dat"
        script="reset\n"
        script += 'set terminal canvas solid butt size 400,350 fsize 10 '
        script += 'lw 1.5 fontscale 1 name "%s" jsdir "."\n' % outfile
        script += 'set title "merged data log-scale"\n'
        script += 'set log y\n'
{'fraction_non_alphanumeric': 0.15673791117174474, 'fraction_numerical': 0.017583750189480066, 'mean_word_length': 2.113732892873997, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '638490', 'n_tokens_mistral': 2419, 'n_tokens_neox': 2306, 'n_words': 462}