<commit_before>"""
bwdata contains the BWData class.
"""
import datetime
import filters
import logging

logger = logging.getLogger("bwapi")


class BWData:
    """
    This class is a superclass for brandwatch BWQueries and BWGroups.  It was built to handle resources that access data (e.g. mentions, topics, charts, etc).
    """
    def get_mentions(self, name=None, startDate=None, max_pages=None, **kwargs):
        """
        Retrieves a list of mentions.
        Note: Clients do not have access to full Twitter mentions through the API because of our data agreement with Twitter.

        Args:
            name:       You must pass in a query / group name (string).
            startDate:  You must pass in a start date (string).
            max_pages:  Maximum number of pages to retrieve, where each page is 5000 mentions by default - Optional.  If you don't pass max_pages, it will retrieve all mentions that match your request.
            kwargs:     All other filters are optional and can be found in filters.py.

        Raises:
            KeyError:   If the mentions call fails.

        Returns:
            A list of mentions.
        """
        params = self._fill_params(name, startDate, kwargs)
        params["pageSize"] = kwargs["pageSize"] if "pageSize" in kwargs else 5000
        next_page = True
        max_id = 0
        page_idx = 0

        all_mentions = []

        while next_page:
            params['sinceId'] = max_id
            if max_pages and page_idx >= max_pages:
                break
            else:
                page_idx += 1
            new_max_id, next_mentions = self._get_mentions_page(params)
            if len(next_mentions) > 0 and new_max_id > max_id:
                max_id = new_max_id
                all_mentions += next_mentions
                logger.info("Mentions since id {} of {} {} retrieved".format(params["sinceId"],
                                                                             self.resource_type, name))
            else:
                break
            params["page"] += 1

        logger.info("{} mentions downloaded".format(len(all_mentions)))
        return all_mentions
<commit_msg>Removed unnecessary page increment<commit_after>"""
bwdata contains the BWData class.
"""
import datetime
import filters
import logging

logger = logging.getLogger("bwapi")


class BWData:
    """
    This class is a superclass for brandwatch BWQueries and BWGroups.  It was built to handle resources that access data (e.g. mentions, topics, charts, etc).
    """
    def get_mentions(self, name=None, startDate=None, max_pages=None, **kwargs):
        """
        Retrieves a list of mentions.
        Note: Clients do not have access to full Twitter mentions through the API because of our data agreement with Twitter.

        Args:
            name:       You must pass in a query / group name (string).
            startDate:  You must pass in a start date (string).
            max_pages:  Maximum number of pages to retrieve, where each page is 5000 mentions by default - Optional.  If you don't pass max_pages, it will retrieve all mentions that match your request.
            kwargs:     All other filters are optional and can be found in filters.py.

        Raises:
            KeyError:   If the mentions call fails.

        Returns:
            A list of mentions.
        """
        params = self._fill_params(name, startDate, kwargs)
        params["pageSize"] = kwargs["pageSize"] if "pageSize" in kwargs else 5000
        next_page = True
        max_id = 0
        page_idx = 0

        all_mentions = []

        while next_page:
            params['sinceId'] = max_id
            if max_pages and page_idx >= max_pages:
                break
            else:
                page_idx += 1
            new_max_id, next_mentions = self._get_mentions_page(params)
            if len(next_mentions) > 0 and new_max_id > max_id:
                max_id = new_max_id
                all_mentions += next_mentions
                logger.info("Mentions since id {} of {} {} retrieved".format(params["sinceId"],
                                                                             self.resource_type, name))
            else:
                break

        logger.info("{} mentions downloaded".format(len(all_mentions)))
        return all_mentions

{'fraction_non_alphanumeric': 0.07489087985297496, 'fraction_numerical': 0.005743165632896853, 'mean_word_length': 1.9007328447701533, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '586859', 'n_tokens_mistral': 1182, 'n_tokens_neox': 1104, 'n_words': 413}