<commit_before>from snovault import (
    AuditFailure,
    audit_checker,
)
from .conditions import (
    rfa,
)
from .standards_data import pipelines_with_read_depth
from .standards_data import special_assays_with_read_depth


current_statuses = ['released', 'in progress']
not_current_statuses = ['revoked', 'obsolete', 'deleted']
raw_data_formats = [
    'fastq',
    'csfasta',
    'csqual',
    'rcc',
    'idat',
    'CEL',
    ]

paired_end_assays = [
    'RNA-PET',
    'ChIA-PET',
    'DNA-PET',
    ]


@audit_checker('File', frame=['derived_from'])
def audit_file_bam_derived_from(value, system):
    if value['file_format'] != 'bam':
        return
    if value['status'] in ['deleted', 'replaced', 'revoked']:
        return
    if 'derived_from' not in value or \
       'derived_from' in value and len(value['derived_from']) == 0:
        return
    derived_from_files = value.get('derived_from')
    fastq_counter = 0
    for f in derived_from_files:
        if f['status'] not in ['deleted', 'replaced', 'revoked'] and \
           f['file_format'] == 'fastq':
                fastq_counter += 1
                if f['dataset'] != value['dataset']:
                    detail = 'Processed alignments file {} '.format(value['@id']) + \
                             'that belongs to experiment {} '.format(value['dataset']) + \
                             'is derived from file {} '.format(f['@id']) + \
                             'that belongs to different experiment {}.'.format(f['dataset'])
                    yield AuditFailure('mismatched derived_from',
                                       detail, level='DCC_ACTION')
    if fastq_counter == 0:
        detail = 'Processed alignments file {} '.format(value['@id']) + \
                 'that belongs to experiment {} '.format(value['dataset']) + \
                 'does not specify which fastq files it was derived from.'
        yield AuditFailure('missing derived_from',
                           detail, level='DCC_ACTION')


@audit_checker('File', frame=['object'],
               condition=rfa('ENCODE3',
                             'modENCODE',
                             'modERN',
                             'GGR'))
def audit_file_processed_empty_derived_from(value, system):
    if value['output_category'] in ['raw data',
                                    'reference']:
        return
    if value['status'] in ['deleted', 'replaced', 'revoked']:
            return
    if 'derived_from' not in value or \
       'derived_from' in value and len(value['derived_from']) == 0:<commit_msg>fixed missing derived_from audit to accept fasta<commit_after>from snovault import (
    AuditFailure,
    audit_checker,
)
from .conditions import (
    rfa,
)
from .standards_data import pipelines_with_read_depth
from .standards_data import special_assays_with_read_depth


current_statuses = ['released', 'in progress']
not_current_statuses = ['revoked', 'obsolete', 'deleted']
raw_data_formats = [
    'fastq',
    'csfasta',
    'csqual',
    'rcc',
    'idat',
    'CEL',
    ]

paired_end_assays = [
    'RNA-PET',
    'ChIA-PET',
    'DNA-PET',
    ]


@audit_checker('File', frame=['derived_from'])
def audit_file_bam_derived_from(value, system):
    if value['file_format'] != 'bam':
        return
    if value['status'] in ['deleted', 'replaced', 'revoked']:
        return
    if 'derived_from' not in value or \
       'derived_from' in value and len(value['derived_from']) == 0:
        return
    derived_from_files = value.get('derived_from')
    raw_data_counter = 0
    for f in derived_from_files:
        if f['status'] not in ['deleted', 'replaced', 'revoked'] and \
           (f['file_format'] == 'fastq' or (f['file_format'] == 'fasta' and
                                            f['output_type'] == 'reads' and
                                            f['output_category'] == 'raw data')):
                raw_data_counter += 1
                if f['dataset'] != value['dataset']:
                    detail = 'Processed alignments file {} '.format(value['@id']) + \
                             'that belongs to experiment {} '.format(value['dataset']) + \
                             'is derived from file {} '.format(f['@id']) + \
                             'that belongs to different experiment {}.'.format(f['dataset'])
                    yield AuditFailure('mismatched derived_from',
                                       detail, level='DCC_ACTION')
    if raw_data_counter == 0:
        detail = 'Processed alignments file {} '.format(value['@id']) + \
                 'that belongs to experiment {} '.format(value['dataset']) + \
                 'does not specify which raw data files it was derived from.'
        yield AuditFailure('missing derived_from',
                           detail, level='DCC_ACTION')


@audit_checker('File', frame=['object'],
               condition=rfa('ENCODE3',
                             'modENCODE',
                             'modERN',
                             'GGR'))
def audit_file_processed_empty_derived_from(value, system):
    if value['output_category'] in ['raw data',
                                    'reference']:
        return
    if value['status'] in ['deleted', 'replaced', 'revoked']:
            return
    if 'derived_from' not in value or \
       'derived_from' in value and len(value['derived_from']) == 0:
{'fraction_non_alphanumeric': 0.13209990749306197, 'fraction_numerical': 0.0022201665124884367, 'mean_word_length': 2.0456338028169014, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2455933', 'n_tokens_mistral': 1590, 'n_tokens_neox': 1526, 'n_words': 370}