<commit_before>from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
import unittest
import sys
import os
from base_test_class import BaseTestCase, on_exception_html_source_logger
from product_test import ProductTest
import time
import logging


logger = logging.getLogger(__name__)

dir_path = os.path.dirname(os.path.realpath(__file__))


class DedupeTest(BaseTestCase):
    # --------------------------------------------------------------------------------------------------------
    # Initialization
    # --------------------------------------------------------------------------------------------------------
    def setUp(self):
        super().setUp()
        self.relative_path = dir_path = os.path.dirname(os.path.realpath(__file__))

    def check_nb_duplicates(self, expected_number_of_duplicates):
        logger.debug("checking duplicates...")
        driver = self.driver
        retries = 0
        for i in range(0, 18):
            time.sleep(5)  # wait bit for celery dedupe task which can be slow on travis
            self.goto_all_findings_list(driver)
            dupe_count = 0
            # iterate over the rows of the findings table and concatenates all columns into td.text
            trs = driver.find_elements_by_xpath('//*[@id="open_findings"]/tbody/tr')
            for row in trs:
                concatRow = ' '.join([td.text for td in row.find_elements_by_xpath(".//td")])
                # print(concatRow)
                if '(DUPE)' and 'Duplicate' in concatRow:
                    dupe_count += 1

            if (dupe_count != expected_number_of_duplicates):
                logger.debug("duplicate count mismatch, let's wait a bit for the celery dedupe task to finish and try again (5s)")
            else:
                break

        if (dupe_count != expected_number_of_duplicates):
            findings_table = driver.find_element_by_id('open_findings')
            print(findings_table.get_attribute('innerHTML'))

        self.assertEqual(dupe_count, expected_number_of_duplicates)

    @on_exception_html_source_logger
    def test_enable_deduplication(self):
        logger.debug("enabling deduplication...")
        driver = self.driver
        driver.get(self.base_url + 'system_settings')
        if not driver.find_element_by_id('id_enable_deduplication').is_selected():
            driver.find_element_by_xpath('//*[@id="id_enable_deduplication"]').click()
            # save settings
            driver.find_element_by_css_selector("input.btn.btn-primary").click()
            # check if it's enabled after reload
            driver.get(self.base_url + 'system_settings')
            self.assertTrue(driver.find_element_by_id('id_enable_deduplication').is_selected())

    @on_exception_html_source_logger
    def test_delete_findings(self):
        logger.debug("removing previous findings...")
        driver = self.driver
        driver.get(self.base_url + "finding?page=1")

        if self.element_exists_by_id("no_findings"):
            text = driver.find_element_by_id("no_findings").text
            if 'No findings found.' in text:
                return

        driver.find_element_by_id("select_all").click()
        driver.find_element_by_css_selector("i.fa.fa-trash").click()
        try:
            WebDriverWait(driver, 1).until(EC.alert_is_present(),
                                        'Timed out waiting for PA creation ' +
                                        'confirmation popup to appear.')
            driver.switch_to.alert.accept()
        except TimeoutException:
            self.fail('Confirmation dialogue not shown, cannot delete previous findings')

        text = None
        if self.element_exists_by_id("no_findings"):
            text = driver.find_element_by_id("no_findings").text

        self.assertTrue('No findings found.' in text)
        # check that user was redirect back to url where it came from based on return_url
        self.assertTrue(driver.current_url.endswith('page=1'))


# --------------------------------------------------------------------------------------------------------
# Same scanner deduplication - Deduplication on engagement
#   Test deduplication for Bandit SAST scanner
# --------------------------------------------------------------------------------------------------------
    @on_exception_html_source_logger  # noqa: E301
    def test_add_path_test_suite(self):
        logger.debug("Same scanner deduplication - Deduplication on engagement - static. Creating tests...")
        # Create engagement
        driver = self.driver
        self.goto_product_overview(driver)
        driver.find_element_by_class_name("pull-left").click()
        driver.find_element_by_link_text("Add New Engagement").click()
        driver.find_element_by_id("id_name").send_keys("Dedu<PASSWORD>")
        driver.find_element_by_xpath('//*[@id="id_deduplication_on_engagement"]').click()
        driver.find_element_by_name("_Add Tests").click()<commit_msg>integration tests: wait for findings datatable (#3704)<commit_after>from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
import unittest
import sys
import os
from base_test_class import BaseTestCase, on_exception_html_source_logger
from product_test import ProductTest
import time
import logging


logger = logging.getLogger(__name__)

dir_path = os.path.dirname(os.path.realpath(__file__))


class DedupeTest(BaseTestCase):
    # --------------------------------------------------------------------------------------------------------
    # Initialization
    # --------------------------------------------------------------------------------------------------------
    def setUp(self):
        super().setUp()
        self.relative_path = dir_path = os.path.dirname(os.path.realpath(__file__))

    def check_nb_duplicates(self, expected_number_of_duplicates):
        logger.debug("checking duplicates...")
        driver = self.driver
        retries = 0
        for i in range(0, 18):
            time.sleep(5)  # wait bit for celery dedupe task which can be slow on travis
            self.goto_all_findings_list(driver)
            dupe_count = 0
            # iterate over the rows of the findings table and concatenates all columns into td.text
            trs = driver.find_elements_by_xpath('//*[@id="open_findings"]/tbody/tr')
            for row in trs:
                concatRow = ' '.join([td.text for td in row.find_elements_by_xpath(".//td")])
                # print(concatRow)
                if '(DUPE)' and 'Duplicate' in concatRow:
                    dupe_count += 1

            if (dupe_count != expected_number_of_duplicates):
                logger.debug("duplicate count mismatch, let's wait a bit for the celery dedupe task to finish and try again (5s)")
            else:
                break

        if (dupe_count != expected_number_of_duplicates):
            findings_table = driver.find_element_by_id('open_findings')
            print(findings_table.get_attribute('innerHTML'))

        self.assertEqual(dupe_count, expected_number_of_duplicates)

    @on_exception_html_source_logger
    def test_enable_deduplication(self):
        logger.debug("enabling deduplication...")
        driver = self.driver
        driver.get(self.base_url + 'system_settings')
        if not driver.find_element_by_id('id_enable_deduplication').is_selected():
            driver.find_element_by_xpath('//*[@id="id_enable_deduplication"]').click()
            # save settings
            driver.find_element_by_css_selector("input.btn.btn-primary").click()
            # check if it's enabled after reload
            driver.get(self.base_url + 'system_settings')
            self.assertTrue(driver.find_element_by_id('id_enable_deduplication').is_selected())

    @on_exception_html_source_logger
    def test_delete_findings(self):
        logger.debug("removing previous findings...")
        driver = self.driver
        driver.get(self.base_url + "finding?page=1")

        if self.element_exists_by_id("no_findings"):
            text = driver.find_element_by_id("no_findings").text
            if 'No findings found.' in text:
                return

        driver.find_element_by_id("select_all").click()
        driver.find_element_by_css_selector("i.fa.fa-trash").click()
        try:
            WebDriverWait(driver, 1).until(EC.alert_is_present(),
                                        'Timed out waiting for finding delete ' +
                                        'confirmation popup to appear.')
            driver.switch_to.alert.accept()
        except TimeoutException:
            self.fail('Confirmation dialogue not shown, cannot delete previous findings')

        # not sure if this is needed. the datatables js logic only kicks in
        # if the dev with id "findings" is present
        # so if it isn't, the no_findings div is available straight after
        # the page load. but we see some errors here about the div not being there
        # but when we log the page source, the no_findings div is present
        self.wait_for_datatable_if_content("no_findings", "findings_wrapper")

        text = None
        if self.element_exists_by_id("no_findings"):
            text = driver.find_element_by_id("no_findings").text

        self.assertIsNotNone(text)
        self.assertTrue('No findings found.' in text)
        # check that user was redirect back to url where it came from based on return_url
        self.assertTrue(driver.current_url.endswith('page=1'))


# --------------------------------------------------------------------------------------------------------
# Same scanner deduplication - Deduplication on engagement
#   Test deduplication for Bandit SAST scanner
# --------------------------------------------------------------------------------------------------------
    @on_exception_html_source_logger  # noqa: E301
    def test_add_path_test_suite(self):
        logger.debug("Same scanner deduplication - Deduplication on engagement - static. Creating tests...")
        # Create engagement
        driver = self.driver
        self.goto_product_overview(driver)
        driver.find_element_by_class_name("pull-left").click()
        driver.find_element_by_link_text("Add New Engagement").click()
        driver.find_element_by_id("id_name").send_keys("<NAME>")
        driver.find_element_by_xpath('//*[@id="id_deduplication_on_engagement"]').click()
        driver.find_element_by_name("_Add Tests").click()
{'fraction_non_alphanumeric': 0.16164510166358595, 'fraction_numerical': 0.0029574861367837337, 'mean_word_length': 2.12114219786559, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4699269', 'n_tokens_mistral': 2916, 'n_tokens_neox': 2797, 'n_words': 637}