<commit_before>from utility import get_time, rd_print

# Finding files such as `this_(that)` requires `'` be placed on both
# sides of the quote so the `()` are both captured. Files such as
# `du_Parterre_d'Eau` must be converted into
#`'du_Parterre_d'"'"'Eau'
#                ^^^ Required to make sure the `'` is captured.
def shellquote(s):
    return "'" + s.replace("'", "'\"'\"'") + "'"

class RDWork:
    def __init__(self):
        self.failed = False
        self.copy_back_files = ['stdout.txt']
    def parse(self, stdout, stderr):
        self.raw = stdout
        split = None
        try:
            split = self.raw.decode('utf-8').replace(')',' ').split()
            self.pixels = split[1]
            self.size = split[2]
            self.metric = {}
            self.metric['psnr'] = {}
            self.metric["psnr"][0] = split[6]
            self.metric["psnr"][1] = split[8]
            self.metric["psnr"][2] = split[10]
            self.metric['psnrhvs'] = {}
            self.metric["psnrhvs"][0] = split[14]
            self.metric["psnrhvs"][1] = split[16]
            self.metric["psnrhvs"][2] = split[18]
            self.metric['ssim'] = {}
            self.metric["ssim"][0] = split[22]
            self.metric["ssim"][1] = split[24]
            self.metric["ssim"][2] = split[26]
            self.metric['fastssim'] = {}
            self.metric["fastssim"][0] = split[30]
            self.metric["fastssim"][1] = split[32]
            self.metric["fastssim"][2] = split[34]
            self.metric['ciede2000'] = split[36]
            self.metric['apsnr'] = {}
            self.metric['apsnr'][0] = split[40]
            self.metric['apsnr'][1] = split[42]
            self.metric['apsnr'][2] = split[44]
            self.metric['msssim'] = {}
            self.metric['msssim'][0] = split[48]
            self.metric['msssim'][1] = split[50]
            self.metric['msssim'][2] = split[52]
            self.metric['encodetime'] = split[53]
            self.failed = False
        except IndexError:
            rd_print('Decoding result for '+self.filename+' at quality '+str(self.quality)+' failed!')
            rd_print('stdout:')
            rd_print(stdout.decode('utf-8'))
            rd_print('stderr:')
            rd_print(stderr.decode('utf-8'))
            self.failed = True
    def execute(self, slot):
        try:
            slot.setup(self.codec,self.bindir)
            work = self
            input_path = slot.machine.media_path+'/'+work.set+'/'+work.filename
            slot.start_shell(('WORK_ROOT="'+slot.work_root+'" DAALATOOL_ROOT="'+slot.machine.work_root+'/daalatool" x="'+str(work.quality) +
                '" CODEC="'+work.codec+'" EXTRA_OPTIONS="'+work.extra_options +
                '" ' + slot.work_root + '/rd_tool/metrics_gather.sh '+shellquote(input_path)))
            (stdout, stderr) = slot.gather()
            for file in self.copy_back_files:
                slot.get_file(slot.work_root+'/'+work.filename+'-'+str(work.quality)+'-'+file,'../runs/'+work.runid+'/'+work.set+'/')
            self.parse(stdout, stderr)
        except Exception as e:
            rd_print(e)
            self.failed = True
    def get_name(self):
        return self.filename + ' with quality ' + str(self.quality)

class ABWork:
    def __init__(self):
        self.failed = False
    def execute(self, slot):
        work = self
        input_path = slot.machine.media_path +'/' + work.set + '/' + work.filename
<commit_msg>Add - to file list for easier ivf copying later.<commit_after>from utility import get_time, rd_print

# Finding files such as `this_(that)` requires `'` be placed on both
# sides of the quote so the `()` are both captured. Files such as
# `du_Parterre_d'Eau` must be converted into
#`'du_Parterre_d'"'"'Eau'
#                ^^^ Required to make sure the `'` is captured.
def shellquote(s):
    return "'" + s.replace("'", "'\"'\"'") + "'"

class RDWork:
    def __init__(self):
        self.failed = False
        self.copy_back_files = ['-stdout.txt']
    def parse(self, stdout, stderr):
        self.raw = stdout
        split = None
        try:
            split = self.raw.decode('utf-8').replace(')',' ').split()
            self.pixels = split[1]
            self.size = split[2]
            self.metric = {}
            self.metric['psnr'] = {}
            self.metric["psnr"][0] = split[6]
            self.metric["psnr"][1] = split[8]
            self.metric["psnr"][2] = split[10]
            self.metric['psnrhvs'] = {}
            self.metric["psnrhvs"][0] = split[14]
            self.metric["psnrhvs"][1] = split[16]
            self.metric["psnrhvs"][2] = split[18]
            self.metric['ssim'] = {}
            self.metric["ssim"][0] = split[22]
            self.metric["ssim"][1] = split[24]
            self.metric["ssim"][2] = split[26]
            self.metric['fastssim'] = {}
            self.metric["fastssim"][0] = split[30]
            self.metric["fastssim"][1] = split[32]
            self.metric["fastssim"][2] = split[34]
            self.metric['ciede2000'] = split[36]
            self.metric['apsnr'] = {}
            self.metric['apsnr'][0] = split[40]
            self.metric['apsnr'][1] = split[42]
            self.metric['apsnr'][2] = split[44]
            self.metric['msssim'] = {}
            self.metric['msssim'][0] = split[48]
            self.metric['msssim'][1] = split[50]
            self.metric['msssim'][2] = split[52]
            self.metric['encodetime'] = split[53]
            self.failed = False
        except IndexError:
            rd_print('Decoding result for '+self.filename+' at quality '+str(self.quality)+' failed!')
            rd_print('stdout:')
            rd_print(stdout.decode('utf-8'))
            rd_print('stderr:')
            rd_print(stderr.decode('utf-8'))
            self.failed = True
    def execute(self, slot):
        try:
            slot.setup(self.codec,self.bindir)
            work = self
            input_path = slot.machine.media_path+'/'+work.set+'/'+work.filename
            slot.start_shell(('WORK_ROOT="'+slot.work_root+'" DAALATOOL_ROOT="'+slot.machine.work_root+'/daalatool" x="'+str(work.quality) +
                '" CODEC="'+work.codec+'" EXTRA_OPTIONS="'+work.extra_options +
                '" ' + slot.work_root + '/rd_tool/metrics_gather.sh '+shellquote(input_path)))
            (stdout, stderr) = slot.gather()
            for file in self.copy_back_files:
                slot.get_file(slot.work_root+'/'+work.filename+'-'+str(work.quality)+file,'../runs/'+work.runid+'/'+work.set+'/')
            self.parse(stdout, stderr)
        except Exception as e:
            rd_print(e)
            self.failed = True
    def get_name(self):
        return self.filename + ' with quality ' + str(self.quality)

class ABWork:
    def __init__(self):
        self.failed = False
    def execute(self, slot):
        work = self
        input_path = slot.machine.media_path +'/' + work.set + '/' + work.filename

{'fraction_non_alphanumeric': 0.1707142857142857, 'fraction_numerical': 0.018571428571428572, 'mean_word_length': 2.136648745519713, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2378893', 'n_tokens_mistral': 2332, 'n_tokens_neox': 2273, 'n_words': 386}