<commit_before># -*- coding: utf-8 -*-
import urllib2, time
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
import os
import stat


class PyInsta_DL(object):
    def __init__(self):

        self.userAgent = 'Mozilla/5.0 (Linux; U; Android 2.2.1; en-ca; LG-P505R Build/FRG83) \
        AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1'
        self.phantomjs ='/usr/local/phantomjs/phantomjs'
        #os.chmod(self.phantomjs, stat.S_IEXEC)
        DesiredCapabilities.PHANTOMJS[
            'phantomjs.page.customHeaders.User-Agent'] = self.userAgent
        self.driver = webdriver.PhantomJS(executable_path=self.phantomjs, \
                                          service_args=['--ignore-ssl-errors=true'])

    def get(self, url):
        self.url = url
        try:
            soup = BeautifulSoup(urllib2.urlopen(self.url).read(), "html.parser")
        except:
            return NotImplementedError

        for meta in soup.findAll('meta'):
            if meta.get("property") == "og:image" and meta.get("content") != '':
                return meta.get("content")
            elif meta.get("property") == "og:video" and meta.get("content") != '':
                return meta.get("content")
        return None

    def get_all(self, url):
        self.url = url
        self.driver.get(self.url)
        time.sleep(5)
        for i in range(self.count_scrolls()):
            self.scroll()
        return self._get_all_urls

    def count_scrolls(self):
        _default_number = 12
        publicatios = self.driver.find_element_by_class_name('_bkw5z')
        number = int(publicatios.text.replace('.', ''))
        if number > 12:
            more = self.driver.find_element_by_class_name('_1ooyk')
            more.click()
            return (number / _default_number) + 1

        else:
            return 1

    def scroll(self):
        self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(3)

    @property
    def _get_all_urls(self):
        images = self.driver.find_elements_by_class_name('_icyx7')
        links = []
        for i in images:
            if i != None:
                links.append(i.get_attribute('src'))
        return links


DEFAULT_DOWNLOAD = PyInsta_DL()


def get(url):
    return DEFAULT_DOWNLOAD.get(url)


def get_all(url_profile):
    return DEFAULT_DOWNLOAD.get_all(url_profile)<commit_msg>performance enhancements<commit_after># -*- coding: utf-8 -*-
import urllib2, time
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
import os
import stat


class PyInsta_DL(object):
    def __init__(self):

        self.userAgent = 'Mozilla/5.0 (Linux; U; Android 2.2.1; en-ca; LG-P505R Build/FRG83) \
        AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1'
        self.phantomjs = '/usr/local/phantomjs/phantomjs'
        DesiredCapabilities.PHANTOMJS[
            'phantomjs.page.customHeaders.User-Agent'] = self.userAgent
        self.driver = webdriver.PhantomJS(executable_path=self.phantomjs, \
                                          service_args=['--ignore-ssl-errors=true'])

    def get(self, url):
        self.url = url
        try:
            soup = BeautifulSoup(urllib2.urlopen(self.url).read(), "html.parser")
        except ValueError:
            pass

        for meta in soup.findAll('meta'):
            if meta.get("property") == "og:image" and meta.get("content") != '':
                return meta.get("content")
            elif meta.get("property") == "og:video" and meta.get("content") != '':
                return meta.get("content")
        return None

    def get_all(self, url):
        self.url = url
        self.driver.get(self.url)
        time.sleep(5)
        [self.scroll() for i in range(self.count_scrolls()) ]
        return self._get_all_urls

    def count_scrolls(self):
        _default_number = 12
        publicatios = self.driver.find_element_by_class_name('_bkw5z')
        number = int(publicatios.text.replace('.', ''))
        if number > 12:
            more = self.driver.find_element_by_class_name('_1ooyk')
            more.click()
            return (number / _default_number) + 1

        else:
            return 1

    def scroll(self):
        self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(3)

    @property
    def _get_all_urls(self):
        images = self.driver.find_elements_by_class_name('_icyx7')
        links = []
        for i in images:
            links.append(i.get_attribute('src'))
        return links


DEFAULT_DOWNLOAD = PyInsta_DL()


def get(url):
    return DEFAULT_DOWNLOAD.get(url)


def get_all(url_profile):
    return DEFAULT_DOWNLOAD.get_all(url_profile)
{'fraction_non_alphanumeric': 0.1190233977619532, 'fraction_numerical': 0.014649033570701932, 'mean_word_length': 2.351056578050443, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2189055', 'n_tokens_mistral': 1598, 'n_tokens_neox': 1512, 'n_words': 295}