<commit_before>import os
from urllib.parse import urljoin
from django.conf import settings
from django.core.urlresolvers import reverse
from django.template.loader import render_to_string
import datasets.freesound as fs


def generate_download_script(dataset):
    access_token_url = urljoin(settings.BASE_URL, reverse('get_access_token'))
    dataset_url = urljoin(settings.BASE_URL, reverse('dataset-sounds',
        kwargs={"short_name": dataset.short_name}))

    tvars = {
        'access_token_url': access_token_url,
        'dataset_url': dataset_url,
        'get_code_url': settings.FS_CLIENT_ID
    }
    return render_to_string('datasets/download_script.py', tvars)


def chunks(l, n):
    """Yield successive n-sized chunks from l."""
    for i in range(0, len(l), n):
        yield l[i:i + n]


def query_freesound_by_id(list_ids):
    """ Query Freesound by chunk of 50 sounds
        Retrieves only id of sounds """
    client = fs.FreesoundClient()
    client.set_token(settings.FS_CLIENT_SECRET)
    results = []
    for sub_list_ids in chunks(list_ids, 50):
        filter_str = 'id:(' + ' OR '.join([str(i) for i in sub_list_ids]) + ')'
        page_result = client.text_search(query="", fields="id", page_size=50, filter=filter_str)
        results += [s for s in page_result]
    return results<commit_msg>Add optional parameters Freesound api query<commit_after>import os
from urllib.parse import urljoin
from django.conf import settings
from django.core.urlresolvers import reverse
from django.template.loader import render_to_string
import datasets.freesound as fs


def generate_download_script(dataset):
    access_token_url = urljoin(settings.BASE_URL, reverse('get_access_token'))
    dataset_url = urljoin(settings.BASE_URL, reverse('dataset-sounds',
        kwargs={"short_name": dataset.short_name}))

    tvars = {
        'access_token_url': access_token_url,
        'dataset_url': dataset_url,
        'get_code_url': settings.FS_CLIENT_ID
    }
    return render_to_string('datasets/download_script.py', tvars)


def chunks(l, n):
    """Yield successive n-sized chunks from l."""
    for i in range(0, len(l), n):
        yield l[i:i + n]


def query_freesound_by_id(list_ids, fields="id,name", descriptors=""):
    """ Query Freesound by chunk of 50 sounds
        Retrieves only id of sounds """
    client = fs.FreesoundClient()
    client.set_token(settings.FS_CLIENT_SECRET)
    results = []
    for sub_list_ids in chunks(list_ids, 50):
        filter_str = 'id:(' + ' OR '.join([str(i) for i in sub_list_ids]) + ')'
        page_result = client.text_search(query="", fields=fields, page_size=50, filter=filter_str,
                                         descriptors=descriptors)
        results += [s for s in page_result]
    return results
{'fraction_non_alphanumeric': 0.10868005738880918, 'fraction_numerical': 0.005021520803443328, 'mean_word_length': 3.6795302013422817, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2437078', 'n_tokens_mistral': 893, 'n_tokens_neox': 877, 'n_words': 215}