<commit_before># -*- encoding: utf-8 -*-

from selenium import webdriver
from selenium.common.exceptions import WebDriverException
import os
import re
import time
import urllib

CHROME_DRIVER_PATH = os.path.join(os.path.dirname('__file__'), 'chromedriver')
TOP_PAGE = 'https://www.facebook.com/NirvanamTokyo/'
SAVE_DIR = 'download'

def list_menu_anchor(anchors):
    lst = []
    for a in anchors:
        href = a.get_attribute('href')
        if href is None: continue
        if href.find('NirvanamTokyo/photos/') <= -1: continue
        imgs = a.find_elements_by_tag_name('img')
        if len(imgs) == 0: continue
        img = imgs[0]
        w = img.get_attribute('width')
        if w is None: continue
        if int(w) < 200: continue
        lst.append(a)
    return lst

def list_src(imgs):
    return list(filter(lambda p: p is not None, map(to_src, imgs)))

def to_src(e):
    return e.get_attribute('src')

def is_jpg(p):
    return p.find('.jpg') > -1 or p.find('.jpeg') > -1

def mkdir_p(d):
    if not os.path.isdir(d):
        os.mkdir(d)
        if not os.path.isdir(d):
            raise Error("failed to make directory")

def click_later(browser):
    time.sleep(2.0)
    later = browser.find_elements_by_link_text('後で')
    if len(later) != 0:
        later[0].click()

def fetch_menu(dir, path):
    print("Opening web browser...")
    browser = webdriver.Chrome(CHROME_DRIVER_PATH)
    browser.get(TOP_PAGE)
    print("Opening web browser...done")
    
    click_later(browser)

    print("Collecting menu anchors...")
    anchors = browser.find_elements_by_tag_name('a')
    anchors = list_menu_anchor(anchors)
    print("Collecting menu anchors...done")

    for anchor in anchors[1:]:
        print("next anchor")
        
        try:
            anchor.click()
            time.sleep(3.0)
        except WebDriverException:
            print('failed to click menu anchor, retry')
            script = 'window.scrollTo({0}, {1})'.format(0, anchor.location['y'])
            browser.execute_script(script)
            click_later(browser)
    
            anchor.click()
            time.sleep(3.0)
    
        spotlight = browser.find_elements_by_class_name('spotlight')
    
        if len(spotlight) >= 0:
            src = spotlight[0].get_attribute('src')
            print("spotlight found, src=" + src)
            
            download_as(src, dir, path)
            browser.find_element_by_tag_name('body').click()
            yield os.path.join(dir, path)
    
    browser.close()

def fetch_menu_image(dir, path):
    for image in fetch_menu(dir, path):
        yield image

def download_to(url, d):
    mkdir_p(d)
    fetch_url(url, d + "/" + os.path.basename(url))

def download_as(url, d, f):<commit_msg>improved performance of searching image<commit_after># -*- encoding: utf-8 -*-

from selenium import webdriver
from selenium.common.exceptions import WebDriverException
import os
import re
import time
import urllib

CHROME_DRIVER_PATH = os.path.join(os.path.dirname('__file__'), 'chromedriver')
TOP_PAGE = 'https://www.facebook.com/NirvanamTokyo/'
SAVE_DIR = 'download'

def list_menu_anchor(anchors):
    lst = []
    for a in anchors:
        href = a.get_attribute('href')
        if href is None: continue
        if href.find('NirvanamTokyo/photos/') <= -1: continue
        imgs = a.find_elements_by_tag_name('img')
        if len(imgs) == 0: continue
        img = imgs[0]
        w = img.get_attribute('width')
        if w is None: continue
        if int(w) < 200: continue
        lst.append(a)
    return lst

def menu_anchors(browser):
    for a in browser.find_elements_by_tag_name('a'):
        href = a.get_attribute('href')
        if href is None: continue
        if href.find('NirvanamTokyo/photos/') <= -1: continue
        imgs = a.find_elements_by_tag_name('img')
        if len(imgs) == 0: continue
        img = imgs[0]
        w = img.get_attribute('width')
        if w is None: continue
        if int(w) < 200: continue
        yield a

def list_src(imgs):
    return list(filter(lambda p: p is not None, map(to_src, imgs)))

def to_src(e):
    return e.get_attribute('src')

def is_jpg(p):
    return p.find('.jpg') > -1 or p.find('.jpeg') > -1

def mkdir_p(d):
    if not os.path.isdir(d):
        os.mkdir(d)
        if not os.path.isdir(d):
            raise Error("failed to make directory")

def click_later(browser):
    time.sleep(2.0)
    later = browser.find_elements_by_link_text('後で')
    if len(later) != 0:
        later[0].click()

def scroll_to(browser, element):
    script = 'window.scrollTo({0}, {1})'.format(0, element.location['y'])
    browser.execute_script(script)

def fetch_menu(dir, path):
    print("Opening web browser...")
    browser = webdriver.Chrome(CHROME_DRIVER_PATH)
    browser.get(TOP_PAGE)
    print("Opening web browser...done")
    
    click_later(browser)

    print("Searching menu anchors...")
    for anchor in menu_anchors(browser):
        print("next anchor")
        
        try:
            anchor.click()
            time.sleep(3.0)
        except WebDriverException:
            print('failed to click menu anchor, retry')
            scroll_to(browser, anchor)
            click_later(browser)
    
            anchor.click()
            time.sleep(3.0)
    
        spotlight = browser.find_elements_by_class_name('spotlight')
    
        if len(spotlight) >= 0:
            src = spotlight[0].get_attribute('src')
            print("spotlight found, src=" + src)
            
            download_as(src, dir, path)
            browser.find_element_by_tag_name('body').click()
            yield os.path.join(dir, path)
    
    browser.close()

def fetch_menu_image(dir, path):
    for image in fetch_menu(dir, path):
        yield image

def download_to(url, d):
    mkdir_p(d)
    fetch_url(url, d + "/" + os.path.basename(url))

def download_as(url, d, f):
{'fraction_non_alphanumeric': 0.1282442748091603, 'fraction_numerical': 0.008651399491094147, 'mean_word_length': 2.5074360499702557, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1592252', 'n_tokens_mistral': 1990, 'n_tokens_neox': 1930, 'n_words': 439}