<commit_before>from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import WebDriverException
import csv
import json
import datetime
import os
import re
import random
import time
import signal
from .settings import Settings
from .time_util import sleep
from .time_util import sleep_actual
import errno
from util.exceptions import PageNotFound404
from util.instalogger import InstaLogger
import requests


def web_adress_navigator(browser, link):
    """Checks and compares current URL of web page and the URL to be navigated and if it is different, it does navigate"""

    try:
        current_url = browser.current_url
    except WebDriverException:
        try:
            current_url = browser.execute_script("return window.location.href")
        except WebDriverException:
            current_url = None

    if current_url is None or current_url != link:
        response = browser.get(link)

        if check_page_title_notfound(browser):
            InstaLogger.logger().error("Failed to get page " + link)
            raise PageNotFound404("Failed to get page " + link)
        #if response.status_code == 404:
        #    InstaLogger.logger().error("Failed to get page " + link)
        #   raise PageNotFound404()
        # update server calls
        sleep(2)


def check_page_title_notfound(browser):
    """ little bit hacky but selenium doesn't shown if 404 is send"""
    """ more infos https://github.com/seleniumhq/selenium-google-code-issue-archive/issues/141 """

    title = browser.title
    if title.lower().startswith('page not found'):
        return True<commit_msg>waiting for each page is now efficient<commit_after>from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import WebDriverException
import csv
import json
import datetime
import os
import re
import random
import time
import signal
from .settings import Settings
from .time_util import sleep
from .time_util import sleep_actual
import errno
from util.exceptions import PageNotFound404
from util.instalogger import InstaLogger
import requests

from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait # available since 2.4.0
from selenium.webdriver.support import expected_conditions as EC

def web_adress_navigator(browser, link):
    """Checks and compares current URL of web page and the URL to be navigated and if it is different, it does navigate"""

    try:
        current_url = browser.current_url
    except WebDriverException:
        try:
            current_url = browser.execute_script("return window.location.href")
        except WebDriverException:
            current_url = None

    if current_url is None or current_url != link:
        response = browser.get(link)

        if check_page_title_notfound(browser):
            InstaLogger.logger().error("Failed to get page " + link)
            raise PageNotFound404("Failed to get page " + link)
        #if response.status_code == 404:
        #    InstaLogger.logger().error("Failed to get page " + link)
        #   raise PageNotFound404()
        # update server calls

        WebDriverWait(browser, 10).until(EC.presence_of_element_located((By.ID, "viewport")))

        # sleep(2)


def check_page_title_notfound(browser):
    """ little bit hacky but selenium doesn't shown if 404 is send"""
    """ more infos https://github.com/seleniumhq/selenium-google-code-issue-archive/issues/141 """

    title = browser.title
    if title.lower().startswith('page not found'):
        return True
{'fraction_non_alphanumeric': 0.07561383928571429, 'fraction_numerical': 0.011997767857142858, 'mean_word_length': 3.4368811881188117, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3893586', 'n_tokens_mistral': 1015, 'n_tokens_neox': 939, 'n_words': 317}