<commit_before>"""
 Soundcloud (Music)

 @website     https://soundcloud.com
 @provide-api yes (https://developers.soundcloud.com/)

 @using-api   yes
 @results     JSON
 @stable      yes
 @parse       url, title, content, publishedDate, embedded
"""

from json import loads
from urllib import urlencode, quote_plus
from dateutil import parser

# engine dependent config
categories = ['music']
paging = True

# api-key
guest_client_id = 'b45b1aa10f1ac2941910a7f0d10f8e28'

# search-url
url = 'https://api.soundcloud.com/'
search_url = url + 'search?{query}'\
                         '&facet=model'\
                         '&limit=20'\
                         '&offset={offset}'\
                         '&linked_partitioning=1'\
                         '&client_id={client_id}'   # noqa

embedded_url = '<iframe width="100%" height="166" ' +\
    'scrolling="no" frameborder="no" ' +\
    'data-src="https://w.soundcloud.com/player/?url={uri}"></iframe>'


# do search-request
def request(query, params):
    offset = (params['pageno'] - 1) * 20<commit_msg>[fix]soundcloud.com guest client_id fetches dynamically<commit_after>"""
 Soundcloud (Music)

 @website     https://soundcloud.com
 @provide-api yes (https://developers.soundcloud.com/)

 @using-api   yes
 @results     JSON
 @stable      yes
 @parse       url, title, content, publishedDate, embedded
"""

import re
from StringIO import StringIO
from json import loads
from lxml import etree
from urllib import urlencode, quote_plus
from dateutil import parser
from searx import logger
from searx.poolrequests import get as http_get

# engine dependent config
categories = ['music']
paging = True

# search-url
url = 'https://api.soundcloud.com/'
search_url = url + 'search?{query}'\
                         '&facet=model'\
                         '&limit=20'\
                         '&offset={offset}'\
                         '&linked_partitioning=1'\
                         '&client_id={client_id}'   # noqa

embedded_url = '<iframe width="100%" height="166" ' +\
    'scrolling="no" frameborder="no" ' +\
    'data-src="https://w.soundcloud.com/player/?url={uri}"></iframe>'


def get_client_id():
    response = http_get("https://soundcloud.com")
    rx_namespace = {"re": "http://exslt.org/regular-expressions"}

    if response.ok:
        tree = etree.parse(StringIO(response.content), etree.HTMLParser())
        script_tags = tree.xpath("//script[re:match(@src, '(.*app.*js)')]", namespaces=rx_namespace)
        app_js_urls = [script_tag.get('src') for script_tag in script_tags if script_tag is not None]

        # extracts valid app_js urls from soundcloud.com content
        for app_js_url in app_js_urls:
            # gets app_js and searches for the clientid
            response = http_get(app_js_url)
            if response.ok:
                cids = re.search(r'client_id:"([^"]*)"', response.content, re.M | re.I)
                if cids is not None and len(cids.groups()):
                    return cids.groups()[0]
    logger.warning("Unable to fetch guest client_id from SoundCloud, check parser!")
    return ""

# api-key
guest_client_id = get_client_id()


# do search-request
def request(query, params):
    offset = (params['pageno'] - 1) * 20
{'fraction_non_alphanumeric': 0.1346272335181762, 'fraction_numerical': 0.013863216266173753, 'mean_word_length': 2.648314606741573, 'pattern_counts': {'":': 1, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 9, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4010488', 'n_tokens_mistral': 1050, 'n_tokens_neox': 985, 'n_words': 238}