<commit_before>import logging

from cwharaj.items import Ad, CacheItem, WebsiteTypes, City, Member
from cwharaj.parser.base_parser import BaseParser
from cwharaj.parser.utils.harajs_comments import HarajsComments
from cwharaj.parser.utils.harajs_section import HarajsSection
from cwharaj.utils.phone_number_set import PhoneNumberItem


class OpensooqParse(BaseParser):
    def __init__(self):
        super(OpensooqParse, self).__init__()

    # Here,we store items from newest to oldest.
    # then fetch the first item from the databse become the oldest.
    def parse_paginate(self, url, hxs, cache_db, history_db):
        links = hxs.xpath('//*[@id="gridPostListing"]/li')
        logging.debug("Get rows count from the opensooq: {}.".format(len(links)))

        count = 1
        for link in links:
            Li_selector = '//*[@id="gridPostListing"]/li[' + str(count) + ']'

            count += 1

            href = self.get_value_from_response_with_urljoin(hxs,
                                                             Li_selector + '/div/div[@class="rectLiDetails"]/h3/a/@href',
                                                             url)

            from cwharaj.utils.crawl_utils import CrawlUtils
            _ID = CrawlUtils.url_parse_id_from_page_url(href, 3)

            # If the link already exist on the history database,ignore it.
            if history_db.check_exist_by_id(_ID):
                logging.debug("  item exist {} on the history database".format(_ID))
                continue

            item = CacheItem(
                id=_ID,
                url_from=WebsiteTypes.opensooq.value,
            )

            cache_db.save_cache(href, item, count)
            # here, must sleep a second.
            # time.sleep(1)

    def parse(self, url, hxs, item_db, phoneNumberSet=None):
        from cwharaj.utils.crawl_utils import CrawlUtils
        _ID = CrawlUtils.url_parse_id_from_page_url(url, 3)

        # ADs User
        _memberName = self.get_value_response(hxs,
                                              '//*[@class="userDet tableCell vTop"]/strong/a/text()')  # len(list) = 2
        _member_timeregister = self.get_value_response(hxs, '//span[@class="joinDate"]/text()')
        _ads_city = self.get_value_response(hxs,
                                            '//*[@class="sellerAddress"]/span[@class="sellerAddressText"]/a/text()')

        # ADs
        _ads_title = self.get_value_response(hxs, '//*[@class="postTitleCont"]/div/h1/text()')
        _image_link = self.get_pictures(hxs, '//*[@class="galleryLeftList fLeft"]/ul/li/a/img/@src')
        _time_added = self.get_value_response(hxs, '//*[@class="postDate fRight"]/text()')
        _ads_body = self.get_all_value_response(hxs, '//*[@class="postDesc"]/p/text()')

        _sections = self.get_section(self.get_value_response(hxs, '//*[@class="breadcrumbs"]'))

        if (_ads_title == '') and (len(_sections) == 0):
            logging.debug("  Empty page on the opensooq")
            return PhoneNumberItem(url, _ID)

        _section_item = HarajsSection(_sections, item_db).get_section_item_for_opensooq()

        # Replace "\n","\r"
        _ads_city = _ads_city.strip()
        _time_added = _time_added.replace("\n", "").replace("\r", "").strip()
        _ads_title = _ads_title.replace("\n", "").replace("\r", "").strip()
        _memberName = _memberName.strip()

        # ====
        # Save to relative database<commit_msg>charset="utf8mb4",<commit_after>import logging

from cwharaj.items import Ad, CacheItem, WebsiteTypes, City, Member
from cwharaj.parser.base_parser import BaseParser
from cwharaj.parser.utils.harajs_comments import HarajsComments
from cwharaj.parser.utils.harajs_section import HarajsSection
from cwharaj.utils.phone_number_set import PhoneNumberItem


class OpensooqParse(BaseParser):
    def __init__(self):
        super(OpensooqParse, self).__init__()

    # Here,we store items from newest to oldest.
    # then fetch the first item from the databse become the oldest.
    def parse_paginate(self, url, hxs, cache_db, history_db):
        links = hxs.xpath('//*[@id="gridPostListing"]/li')
        logging.debug("Get rows count from the opensooq: {}.".format(len(links)))

        count = 1
        for link in links:
            Li_selector = '//*[@id="gridPostListing"]/li[' + str(count) + ']'

            count += 1

            href = self.get_value_from_response_with_urljoin(hxs,
                                                             Li_selector + '/div/div[@class="rectLiDetails"]/h3/a/@href',
                                                             url)

            from cwharaj.utils.crawl_utils import CrawlUtils
            _ID = CrawlUtils.url_parse_id_from_page_url(href, 3)

            # If the link already exist on the history database,ignore it.
            if history_db.check_exist_by_id(_ID):
                logging.debug("  item exist {} on the history database".format(_ID))
                continue

            item = CacheItem(
                id=_ID,
                url_from=WebsiteTypes.opensooq.value,
            )

            cache_db.save_cache(href, item, count)
            # here, must sleep a second.
            # time.sleep(1)

    def parse(self, url, hxs, item_db, phoneNumberSet=None):
        from cwharaj.utils.crawl_utils import CrawlUtils
        _ID = CrawlUtils.url_parse_id_from_page_url(url, 3)

        # ADs User
        # memberName len(list) = 2
        _memberName = self.get_value_response(hxs, '//*[@class="userDet tableCell vTop"]/strong/a/text()')
        _member_timeregister = self.get_value_response(hxs, '//span[@class="joinDate"]/text()')
        _ads_city = self.get_value_response(hxs,
                                            '//*[@class="sellerAddress"]/span[@class="sellerAddressText"]/a/text()')

        # ADs
        _ads_title = self.get_value_response(hxs, '//*[@class="postTitleCont"]/div/h1/text()')
        _image_link = self.get_pictures(hxs, '//*[@class="galleryLeftList fLeft"]/ul/li/a/img/@src')
        _time_added = self.get_value_response(hxs, '//*[@class="postDate fRight"]/text()')
        _ads_body = self.get_all_value_response(hxs, '//*[@class="postDesc"]/p/text()')

        _sections = self.get_section(self.get_value_response(hxs, '//*[@class="breadcrumbs"]'))

        if (_ads_title == '') and (len(_sections) == 0):
            logging.debug("  Empty page on the opensooq")
            return PhoneNumberItem(url, _ID)

        _section_item = HarajsSection(_sections, item_db).get_section_item_for_opensooq()

        # Replace "\n","\r"
        _ads_city = _ads_city.strip()
        _time_added = _time_added.replace("\n", "").replace("\r", "").strip()
        _ads_title = _ads_title.replace("\n", "").replace("\r", "").strip()
        _memberName = _memberName.strip()

        # ====
        # Save to relative database
{'fraction_non_alphanumeric': 0.12939643942683457, 'fraction_numerical': 0.0028947749312490953, 'mean_word_length': 2.3756717147044455, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5438409', 'n_tokens_mistral': 2194, 'n_tokens_neox': 2118, 'n_words': 404}