<commit_before>from .base import Module, extract_value


class Vuln(Module):
    _name = 'vuln'

    def init(self):
        return self._request('init')

    @extract_value('results')
    def query(self, tool, source='cumulative', size=None, offset=None,
              sort=None, direction=None, scan=None, view=None,
              filters=None, **filter_by):
        # source cumulative, patched, individual
        # directory YYYY-MM-DD from scan finish time
        #     required but ignored by server
        # view all, patched, or new

        if scan is not None:
            source = 'individual'

            if view is None:
                view = 'all'

        if filters is None:
            filters = []

        for key, value in filter_by.iteritems():
            filters.append({
                'filterName': key,
                'operator': '=',
                'value': value
            })

        if size is not None:
            start = offset or 0
            end = start + size
        else:
            start, end = None, None

        input = {
            'tool': tool, 'sourceType': source,
            'startOffset': start, 'endOffset': end,
            'sortField': sort, 'sortDir': direction and direction.upper(),
            'filters': filters,
            'view': view, 'dateDirectory': 'null', 'scanID': scan
        }

        if tool in ('vulndetails', 'listvuln') and size is None:
            start, end = 0, 1000
            input['startOffset'] = start
            input['endOffset'] = end
            out = self._request('query', input)
            results = out['results']
            total = int(out['totalRecords'])
            while end < total:
                start, end = end, end + 1000
                input['startOffset'] = start
                input['endOffset'] = end
                results.extend(self._request('query', input)['results'])
            return out

        return self._request('query', input)

    def download(self):
        #TODO vuln::download
        raise NotImplementedError

    def validate_params(self, user_id, org_id, start_offset, end_offset, 
        tool=None, source_type=None, view=None, date_directory=None, scan_id=None):
        """Validate the parameters.

        :param user_id: the user's id
        :param org_id: the org's id
        :param start_offset: the numeric value for starting record
        :param end_offset: the numeric value for ending record
        :param tool: the tool to be used for the query
        :param source_type: the source type
        :param view: a view for the individual data source
        :param date_directory: a date directory for data source
        :param scan_id: scan id for data source

        :return: return values
        """

        if source_type is not None:
            self.view = view
            self.date_directory = date_directory
            self.scan_id = scan_id

        return self._request('validateParams', {
            'userId': user_id,
            'orgID': org_id,
            'tool': tool,
            'startOffset': start_offset,
            'endOffset': end_offset,
            'sourceType': source_type,
            'view': view,
            'dateDirectory': date_directory,
            'scanID': scan_id
        })

    def scrub_params(self, filters):
        """Perform input scrubbing by removing whitespace and new lines.

        :param filters: identify what will be scrubbed

        :return: return array scrubbed params
        """

        return self._request('scrubParams', {
            'filters': filters
        })

    @extract_value('records')
    def get_ip(self, ip, repos=None):
        if repos is not None:
            repos = [{'id': r_id} for r_id in repos]

        return self._request('getIP', {
            'ip': ip,
            'repositories': repos
        })

    def get_asset_data(self, id):
        """Gets the details of a given asset.

        :param id: id must be set and a valid asset id

        :return: returns full details for a given asset
        """

        return self._request('getAssetData', {
            'id': id
        })

    def get_asset_intersections(self, ip, asset_id, dns_name, repos):
        #TODO vuln::getAssetIntersections
        raise NotImplementedError<commit_msg>correct ordering<commit_after>from .base import Module, extract_value


class Vuln(Module):
    _name = 'vuln'

    def init(self):
        return self._request('init')

    def validate_params(self, user_id, org_id, start_offset, end_offset,
        tool=None, source_type=None, view=None, date_directory=None, scan_id=None):
        """Validate the parameters.

        :param user_id: the user's id
        :param org_id: the org's id
        :param start_offset: the numeric value for starting record
        :param end_offset: the numeric value for ending record
        :param tool: the tool to be used for the query
        :param source_type: the source type
        :param view: a view for the individual data source
        :param date_directory: a date directory for data source
        :param scan_id: scan id for data source

        :return: return values
        """

        if source_type is not None:
            self.view = view
            self.date_directory = date_directory
            self.scan_id = scan_id

        return self._request('validateParams', {
            'userId': user_id,
            'orgID': org_id,
            'tool': tool,
            'startOffset': start_offset,
            'endOffset': end_offset,
            'sourceType': source_type,
            'view': view,
            'dateDirectory': date_directory,
            'scanID': scan_id
        })

    def scrub_params(self, filters):
        """Perform input scrubbing by removing whitespace and new lines.

        :param filters: identify what will be scrubbed

        :return: return array scrubbed params
        """

        return self._request('scrubParams', {
            'filters': filters
        })

    @extract_value('results')
    def query(self, tool, source='cumulative', size=None, offset=None,
              sort=None, direction=None, scan=None, view=None,
              filters=None, **filter_by):
        # source cumulative, patched, individual
        # directory YYYY-MM-DD from scan finish time
        #     required but ignored by server
        # view all, patched, or new

        if scan is not None:
            source = 'individual'

            if view is None:
                view = 'all'

        if filters is None:
            filters = []

        for key, value in filter_by.iteritems():
            filters.append({
                'filterName': key,
                'operator': '=',
                'value': value
            })

        if size is not None:
            start = offset or 0
            end = start + size
        else:
            start, end = None, None

        input = {
            'tool': tool, 'sourceType': source,
            'startOffset': start, 'endOffset': end,
            'sortField': sort, 'sortDir': direction and direction.upper(),
            'filters': filters,
            'view': view, 'dateDirectory': 'null', 'scanID': scan
        }

        if tool in ('vulndetails', 'listvuln') and size is None:
            start, end = 0, 1000
            input['startOffset'] = start
            input['endOffset'] = end
            out = self._request('query', input)
            results = out['results']
            total = int(out['totalRecords'])
            while end < total:
                start, end = end, end + 1000
                input['startOffset'] = start
                input['endOffset'] = end
                results.extend(self._request('query', input)['results'])
            return out

        return self._request('query', input)

    @extract_value('records')
    def get_ip(self, ip, repos=None):
        if repos is not None:
            repos = [{'id': r_id} for r_id in repos]

        return self._request('getIP', {
            'ip': ip,
            'repositories': repos
        })

    def get_asset_data(self, id):
        """Gets the details of a given asset.

        :param id: id must be set and a valid asset id

        :return: returns full details for a given asset
        """

        return self._request('getAssetData', {
            'id': id
        })

    def download(self):
        #TODO vuln::download
        raise NotImplementedError

    def get_asset_intersections(self, ip, asset_id, dns_name, repos):
        #TODO vuln::getAssetIntersections
        raise NotImplementedError
{'fraction_non_alphanumeric': 0.0984443928488507, 'fraction_numerical': 0.0023218017181332712, 'mean_word_length': 1.8385502471169688, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2343374', 'n_tokens_mistral': 2371, 'n_tokens_neox': 2253, 'n_words': 742}