<commit_before># python modules
from urllib.parse import urlparse

# third party
import requests
from bs4 import BeautifulSoup

# own modules
from scrapper_helpers.utils import caching

BASE_URL = 'https://www.olx.pl'
DEBUG = True

WHITELISTED_DOMAINS = [
    'olx.pl',
    'www.olx.pl',
]


def flatten(container):
    for i in container:
        if isinstance(i, (list, tuple)):
            for j in flatten(i):
                yield j
        else:
            yield i


def get_url(main_category, subcategory, detail_category, region, page, **filters):
    return "/".join([BASE_URL, main_category, subcategory, detail_category, region, page or ""])


@caching
def get_content_for_url(url):
    response = requests.get(url, allow_redirects=False)
    try:
        response.raise_for_status()
    except requests.HTTPError as e:
        print('Request for {} failed.'.format(url))
        return None
    return response

def parse_offer(offer_markup):
    html_parser = BeautifulSoup(offer_markup, "html.parser")
    url = html_parser.find(class_="linkWithHash").attrs['href']
    title = html_parser.find(class_="linkWithHash").child
    if not url:
        # detail url is not present
        return []

    if urlparse(url).hostname not in WHITELISTED_DOMAINS:
        # domain is not supported by this backend
        return []
    # print(urlparse(url).hostname, url)
    return {
        'detail_url': url
    }


def parse_content(markup):
    html_parser = BeautifulSoup(markup, "html.parser")
    offers = html_parser.find_all(class_='offer')
    parsed_offers = [parse_offer(str(offer)) for offer in offers if offer]
    # delete featured offers (always first 3)
    for i in range(3):
        del parsed_offers[0]
    return parsed_offers


def get_category(main_category, subcategory, detail_category, region, **filters):<commit_msg>code styling<commit_after># python modules
from urllib.parse import urlparse

# third party
import requests
from bs4 import BeautifulSoup

# own modules
from scrapper_helpers.utils import caching

BASE_URL = 'https://www.olx.pl'
DEBUG = True

WHITELISTED_DOMAINS = [
    'olx.pl',
    'www.olx.pl',
]


def flatten(container):
    for i in container:
        if isinstance(i, (list, tuple)):
            for j in flatten(i):
                yield j
        else:
            yield i


def get_url(main_category, subcategory, detail_category, region, page, **filters):
    return "/".join([BASE_URL, main_category, subcategory, detail_category, region, page or ""])


@caching
def get_content_for_url(url):
    response = requests.get(url, allow_redirects=False)
    try:
        response.raise_for_status()
    except requests.HTTPError as e:
        print('Request for {} failed.'.format(url))
        return None
    return response


def parse_offer(offer_markup):
    html_parser = BeautifulSoup(offer_markup, "html.parser")
    url = html_parser.find(class_="linkWithHash").attrs['href']
    title = html_parser.find(class_="linkWithHash").child
    if not url:
        # detail url is not present
        return []

    if urlparse(url).hostname not in WHITELISTED_DOMAINS:
        # domain is not supported by this backend
        return []
    # print(urlparse(url).hostname, url)
    return {
        'detail_url': url
    }


def parse_content(markup):
    html_parser = BeautifulSoup(markup, "html.parser")
    offers = html_parser.find_all(class_='offer')
    parsed_offers = [parse_offer(str(offer)) for offer in offers if offer]
    # delete featured offers (always first 3)
    for i in range(3):
        del parsed_offers[0]
    return parsed_offers


def get_category(main_category, subcategory, detail_category, region, **filters):
{'fraction_non_alphanumeric': 0.10221741481882099, 'fraction_numerical': 0.002163331530557058, 'mean_word_length': 3.2664359861591694, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1278952', 'n_tokens_mistral': 1172, 'n_tokens_neox': 1153, 'n_words': 304}