<commit_before>import urllib3
import io
import gzip
import csv
import logging
import pprint
import collections
from pyramid.view import view_config
from sqlalchemy.sql import text
from elasticsearch.exceptions import (
    NotFoundError
)
from snovault import DBSESSION, COLLECTIONS
from snovault.storage import (
    TransactionRecord,
)
from snovault.elasticsearch.indexer import all_uuids
from snovault.elasticsearch.interfaces import (
    ELASTIC_SEARCH,
    SNP_SEARCH_ES,
)
import copy


SEARCH_MAX = 99999  # OutOfMemoryError if too high
log = logging.getLogger(__name__)
#log.setLevel(logging.DEBUG)                                                                                                                                                         
log.setLevel(logging.INFO)

# hashmap of assays and corresponding file types that are being indexed
_INDEXED_DATA = {
    'accessible chromatin': {
        'output_type': ['peaks']
    },
    'binding footprints': {
        'output_type': ['peaks']
    },
    'chromatin state': {
        'file_type': ['bed bed3+']
    }
}

# Species and references being indexed
_ASSEMBLIES = ['hg19', 'mm10', 'mm9', 'GRCh38']


def includeme(config):
    config.add_route('index_file', '/index_file')
    config.scan(__name__)



def tsvreader(file):
    reader = csv.reader(file, delimiter='\t')
    for row in reader:
        yield row

# Mapping should be generated dynamically for each assembly type


def get_mapping(assembly_name='hg19'):
    return {
        assembly_name: {
            '_all': {
                'enabled': False
            },
            '_source': {
                'enabled': True
            },
            'properties': {
                'uuid': {
                    'type': 'string',
                    'index': 'not_analyzed'
                },
                'positions': {
                    'type': 'nested',
                    'properties': {
                        'start': {
                            'type': 'long'
                        },
                        'end': {
                            'type': 'long'
                        },
                        'state': {
                            'type': 'string'
                            },
                        'val': {
                            'type': 'string'
                            }
                        }
                    }
                }
            }
        }


def index_settings():
    return {
        'index': {
            'number_of_shards': 1
        }
    }


def get_annotation_type(accession, request):
    '''
    Input file accession and returns annotation_type of the annotation the file
    belongs to
    '''
    context = request.embed(accession)
    if 'annotation_type' in context:
        return context['annotation_type']<commit_msg>increase number of shards - test indexing-search speed<commit_after>import urllib3
import io
import gzip
import csv
import logging
import pprint
import collections
from pyramid.view import view_config
from sqlalchemy.sql import text
from elasticsearch.exceptions import (
    NotFoundError
)
from snovault import DBSESSION, COLLECTIONS
from snovault.storage import (
    TransactionRecord,
)
from snovault.elasticsearch.indexer import all_uuids
from snovault.elasticsearch.interfaces import (
    ELASTIC_SEARCH,
    SNP_SEARCH_ES,
)
import copy


SEARCH_MAX = 99999  # OutOfMemoryError if too high
log = logging.getLogger(__name__)
#log.setLevel(logging.DEBUG)                                                                                                                                                         
log.setLevel(logging.INFO)

# hashmap of assays and corresponding file types that are being indexed
_INDEXED_DATA = {
    'accessible chromatin': {
        'output_type': ['peaks']
    },
    'binding footprints': {
        'output_type': ['peaks']
    },
    'chromatin state': {
        'file_type': ['bed bed3+']
    }
}

# Species and references being indexed
_ASSEMBLIES = ['hg19', 'mm10', 'mm9', 'GRCh38']


def includeme(config):
    config.add_route('index_file', '/index_file')
    config.scan(__name__)



def tsvreader(file):
    reader = csv.reader(file, delimiter='\t')
    for row in reader:
        yield row

# Mapping should be generated dynamically for each assembly type


def get_mapping(assembly_name='hg19'):
    return {
        assembly_name: {
            '_all': {
                'enabled': False
            },
            '_source': {
                'enabled': True
            },
            'properties': {
                'uuid': {
                    'type': 'string',
                    'index': 'not_analyzed'
                },
                'positions': {
                    'type': 'nested',
                    'properties': {
                        'start': {
                            'type': 'long'
                        },
                        'end': {
                            'type': 'long'
                        },
                        'state': {
                            'type': 'string'
                            },
                        'val': {
                            'type': 'string'
                            }
                        }
                    }
                }
            }
        }


def index_settings():
    return {
        'index': {
            'number_of_shards': 12
        }
    }


def get_annotation_type(accession, request):
    '''
    Input file accession and returns annotation_type of the annotation the file
    belongs to
    '''
    context = request.embed(accession)
    if 'annotation_type' in context:
        return context['annotation_type']
{'fraction_non_alphanumeric': 0.0935754189944134, 'fraction_numerical': 0.006110335195530726, 'mean_word_length': 1.3771784232365145, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '213673', 'n_tokens_mistral': 1533, 'n_tokens_neox': 1427, 'n_words': 361}