<commit_before>from collections import OrderedDict
from pyramid.compat import bytes_
from pyramid.httpexceptions import (
    HTTPBadRequest,
    HTTPMovedPermanently
)
from base64 import b64decode
from pyramid.view import view_config
from pyramid.response import Response
from snovault import TYPES
from snovault.util import simple_path_ids, debug_log
from itertools import chain

from urllib.parse import (
    parse_qs,
    urlencode,
)
from .search import (
    iter_search_results,
    build_table_columns,
    get_iterable_search_results,
    make_search_subreq
)

import csv
import io
import json
from datetime import datetime

import structlog


log = structlog.getLogger(__name__)


def includeme(config):
    config.add_route('batch_download', '/batch_download/{search_params}')
    config.add_route('metadata', '/metadata/')
    config.add_route('metadata_redirect', '/metadata/{search_params}/{tsv}')
    config.add_route('peak_metadata', '/peak_metadata/{search_params}/{tsv}')
    config.add_route('report_download', '/report.tsv')
    config.scan(__name__)

EXP_SET     = 0
EXP         = 1
FILE        = 2
FILE_ONLY   = 3

# includes concatenated properties
TSV_MAPPING = OrderedDict([
    ('File Download URL',           (FILE,      ['href'])),
    ('Experiment Set Accession',    (EXP_SET,   ['accession'])),
    ('Experiment Accession',        (EXP,       ['accession'])),
    ('Exp. Set Accession',          (FILE_ONLY, ['experiment_sets.accession'])),
    ('Exp. Accession',              (FILE_ONLY, ['experiments.accession'])),
    ('File Accession',              (FILE,      ['accession'])),


    ('Size',                        (FILE,      ['file_size'])),
    ('md5sum',                      (FILE,      ['md5sum'])),
    ('File Type',                   (FILE,      ['file_type'])),
    ('File Format',                 (FILE,      ['file_format.display_title'])),
   #('Experiment Title',            (EXP,       ['display_title'])),
    ('Experiment Type',             (EXP,       ['experiment_type.display_title'])),
    ('Bio Rep No',                  (EXP_SET,   ['replicate_exps.bio_rep_no'])),
    ('Tech Rep No',                 (EXP_SET,   ['replicate_exps.tec_rep_no'])),

    ('Biosource',                   (EXP,       ['biosample.biosource_summary'])),
    ('Biosource Type',              (EXP,       ['biosample.biosource.biosource_type'])),
    ('Organism',                    (EXP,       ['biosample.biosource.individual.organism.name'])),
    ('Assay Details',               (EXP,       ['experiment_categorizer.combined'])),
   #('Digestion Enzyme',            (EXP,       ['digestion_enzyme.name'])),
    ('Related File Relationship',   (FILE,      ['related_files.relationship_type'])),
    ('Related File',                (FILE,      ['related_files.file.accession'])),
    ('Paired end',                  (FILE,      ['paired_end'])),
    ('Lab',                         (EXP_SET,   ['lab.display_title'])),
    ('Project',                     (EXP_SET,   ['award.project'])),
    ('Set Status',                  (EXP_SET,   ['status'])),
    ('File Status',                 (FILE,      ['status'])),
    ('Publication',                 (EXP_SET,   ['produced_in_pub.short_attribution'])),
    ('Experiment Type',             (FILE_ONLY, ['track_and_facet_info.experiment_type'])),
    ('Assay Details',               (FILE_ONLY, ['track_and_facet_info.assay_info'])),
    ('Biosource',                   (FILE_ONLY, ['track_and_facet_info.biosource_name'])),
    ('Condition',                   (FILE_ONLY, ['track_and_facet_info.condition'])),
    ('Dataset',                     (FILE_ONLY, ['track_and_facet_info.dataset'])),
    ('In Experiment As',            (FILE_ONLY, ['track_and_facet_info.experiment_bucket'])),
    ('Generating Lab',                 (FILE_ONLY, ['track_and_facet_info.lab_name'])),
    ('Replicate Info',              (FILE_ONLY, ['track_and_facet_info.replicate_info'])),
    #('UUID',                        (FILE,      ['uuid'])),
    #('Biosample life stage', ['replicates.library.biosample.life_stage']),
    #('Biosample sex', ['replicates.library.biosample.sex']),
    #('Biosample organism', ['replicates.library.biosample.organism.scientific_name']),
    #('Biosample treatments', ['replicates.library.biosample.treatments.treatment_term_name']),
    #('Biosample subcellular fraction term name', ['replicates.library.biosample.subcellular_fraction_term_name']),
    #('Biosample phase', ['replicates.library.biosample.phase']),
    #('Biosample synchronization stage', ['replicates.library.biosample.fly_synchronization_stage',
    #                                     'replicates.library.biosample.worm_synchronization_stage',
    #                                     'replicates.library.biosample.post_synchronization_time',
    #                                     'replicates.library.biosample.post_synchronization_time_units']),
    #('Experiment target', ['target.name']),
    #('Antibody accession', ['replicates.antibody.accession']),
    #('Library made from', ['replicates.library.nucleic_acid_term_name']),
    #('Library depleted in', ['replicates.library.depleted_in_term_name']),
    #('Library extraction method', ['replicates.library.extraction_method']),
    #('Library lysis method', ['replicates.library.lysis_method']),
    #('Library crosslinking method', ['replicates.library.crosslinking_method']),
    #('Date Created', ['experiments_in_set.files.date_created']),
    #('Project', ['award.project']),
    #('RBNS protein concentration', ['files.replicate.rbns_protein_concentration', 'files.replicate.rbns_protein_concentration_units']),
    #('Library fragmentation method', ['files.replicate.library.fragmentation_method']),
    #('Library size range', ['files.replicate.library.size_range']),
    #('Biosample Age', ['files.replicate.library.biosample.age_display']),
    #('Biological replicate(s)', ['files.biological_replicates']),
    #('Technical replicate', ['files.replicate.technical_replicate_number']),
    #('Read length', ['files.read_length']),
    #('Run type', ['files.run_type']),
    #('Paired with', ['experiments_in_set.files.paired_with']),<commit_msg>update field sort<commit_after>from collections import OrderedDict
from pyramid.compat import bytes_
from pyramid.httpexceptions import (
    HTTPBadRequest,
    HTTPMovedPermanently
)
from base64 import b64decode
from pyramid.view import view_config
from pyramid.response import Response
from snovault import TYPES
from snovault.util import simple_path_ids, debug_log
from itertools import chain

from urllib.parse import (
    parse_qs,
    urlencode,
)
from .search import (
    iter_search_results,
    build_table_columns,
    get_iterable_search_results,
    make_search_subreq
)

import csv
import io
import json
from datetime import datetime

import structlog


log = structlog.getLogger(__name__)


def includeme(config):
    config.add_route('batch_download', '/batch_download/{search_params}')
    config.add_route('metadata', '/metadata/')
    config.add_route('metadata_redirect', '/metadata/{search_params}/{tsv}')
    config.add_route('peak_metadata', '/peak_metadata/{search_params}/{tsv}')
    config.add_route('report_download', '/report.tsv')
    config.scan(__name__)

EXP_SET     = 0
EXP         = 1
FILE        = 2
FILE_ONLY   = 3

# includes concatenated properties
TSV_MAPPING = OrderedDict([
    ('File Download URL',           (FILE,      ['href'])),
    ('Experiment Set Accession',    (EXP_SET,   ['accession'])),
    ('Experiment Accession',        (EXP,       ['accession'])),
    ('Exp. Set Accession',          (FILE_ONLY, ['experiment_sets.accession'])),
    ('Exp. Accession',              (FILE_ONLY, ['experiments.accession'])),
    ('File Accession',              (FILE,      ['accession'])),


    ('Size',                        (FILE,      ['file_size'])),
    ('md5sum',                      (FILE,      ['md5sum'])),
    ('File Type',                   (FILE,      ['file_type'])),
    ('File Format',                 (FILE,      ['file_format.display_title'])),
   #('Experiment Title',            (EXP,       ['display_title'])),
    ('Experiment Type',             (EXP,       ['experiment_type.display_title'])),
    ('Bio Rep No',                  (EXP_SET,   ['replicate_exps.bio_rep_no'])),
    ('Tech Rep No',                 (EXP_SET,   ['replicate_exps.tec_rep_no'])),

    ('Biosource',                   (EXP,       ['biosample.biosource_summary'])),
    ('Biosource Type',              (EXP,       ['biosample.biosource.biosource_type'])),
    ('Organism',                    (EXP,       ['biosample.biosource.individual.organism.name'])),
    ('Assay Details',               (EXP,       ['experiment_categorizer.combined'])),
   #('Digestion Enzyme',            (EXP,       ['digestion_enzyme.name'])),
    ('Related File Relationship',   (FILE,      ['related_files.relationship_type'])),
    ('Related File',                (FILE,      ['related_files.file.accession'])),
    ('Paired end',                  (FILE,      ['paired_end'])),
    ('Lab',                         (EXP_SET,   ['lab.display_title'])),
    ('Project',                     (EXP_SET,   ['award.project'])),
    ('Set Status',                  (EXP_SET,   ['status'])),
    ('File Status',                 (FILE,      ['status'])),
    ('Publication',                 (EXP_SET,   ['produced_in_pub.short_attribution'])),
    ('Experiment Type',             (FILE_ONLY, ['track_and_facet_info.experiment_type'])),
    ('Dataset',                     (FILE_ONLY, ['track_and_facet_info.dataset'])),
    ('Condition',                   (FILE_ONLY, ['track_and_facet_info.condition'])),
    ('Assay Details',               (FILE_ONLY, ['track_and_facet_info.assay_info'])),
    ('Biosource',                   (FILE_ONLY, ['track_and_facet_info.biosource_name'])),    
    ('In Experiment As',            (FILE_ONLY, ['track_and_facet_info.experiment_bucket'])),
    ('Replicate Info',              (FILE_ONLY, ['track_and_facet_info.replicate_info'])),
    ('Generating Lab',              (FILE_ONLY, ['track_and_facet_info.lab_name'])),

    #('UUID',                        (FILE,      ['uuid'])),
    #('Biosample life stage', ['replicates.library.biosample.life_stage']),
    #('Biosample sex', ['replicates.library.biosample.sex']),
    #('Biosample organism', ['replicates.library.biosample.organism.scientific_name']),
    #('Biosample treatments', ['replicates.library.biosample.treatments.treatment_term_name']),
    #('Biosample subcellular fraction term name', ['replicates.library.biosample.subcellular_fraction_term_name']),
    #('Biosample phase', ['replicates.library.biosample.phase']),
    #('Biosample synchronization stage', ['replicates.library.biosample.fly_synchronization_stage',
    #                                     'replicates.library.biosample.worm_synchronization_stage',
    #                                     'replicates.library.biosample.post_synchronization_time',
    #                                     'replicates.library.biosample.post_synchronization_time_units']),
    #('Experiment target', ['target.name']),
    #('Antibody accession', ['replicates.antibody.accession']),
    #('Library made from', ['replicates.library.nucleic_acid_term_name']),
    #('Library depleted in', ['replicates.library.depleted_in_term_name']),
    #('Library extraction method', ['replicates.library.extraction_method']),
    #('Library lysis method', ['replicates.library.lysis_method']),
    #('Library crosslinking method', ['replicates.library.crosslinking_method']),
    #('Date Created', ['experiments_in_set.files.date_created']),
    #('Project', ['award.project']),
    #('RBNS protein concentration', ['files.replicate.rbns_protein_concentration', 'files.replicate.rbns_protein_concentration_units']),
    #('Library fragmentation method', ['files.replicate.library.fragmentation_method']),
    #('Library size range', ['files.replicate.library.size_range']),
    #('Biosample Age', ['files.replicate.library.biosample.age_display']),
    #('Biological replicate(s)', ['files.biological_replicates']),
    #('Technical replicate', ['files.replicate.technical_replicate_number']),
    #('Read length', ['files.read_length']),
    #('Run type', ['files.run_type']),
    #('Paired with', ['experiments_in_set.files.paired_with']),
{'fraction_non_alphanumeric': 0.15605815831987077, 'fraction_numerical': 0.0016155088852988692, 'mean_word_length': 2.9467644246094995, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6861889', 'n_tokens_mistral': 3861, 'n_tokens_neox': 3671, 'n_words': 643}