<jupyter_start><jupyter_text>Scraping info on stock splits using fake_useragent and ThreadPoolExecutor<jupyter_code>import requests
from bs4 import BeautifulSoup
import re
import numpy as np
import pandas as pd
from fake_useragent import UserAgent
session = requests.session()
session.proxies = {}

r = session.get("http://httpbin.org/ip")
print(r.text)
# turned out that fake useragent is enough, and tor with changing IPs is not necessary
# !service tor start

# session = requests.session()
# session.proxies = {}

# session.proxies['http'] = 'socks4://localhost:9051'
# session.proxies['https'] = 'socks4://localhost:9051'

# r = session.get("http://httpbin.org/ip")
# print(r.text)
ticker_df = pd.read_csv("tickers.csv", names=['ticker'])
tickers = ticker_df['ticker'].tolist()
print(len(tickers))<jupyter_output>13502<jupyter_text>13502 tickers is a lot. Executing all this in a row takes a couple of hours<jupyter_code>def get_splits_data(t: str) -> pd.DataFrame:
    """Get data on single ticker"""
    
    ticker = t.lower()
    regex_split_info = re.compile(r"(\d{2}/\d{2}/\d{4})\s*([\d.]+?)(\s*for\s*)(\s*[\d.]+)")  # x for x part - split ratio
    

    r = session.get("https://www.stocksplithistory.com/{}/".format(ticker), timeout = 10,
                    headers={'User-Agent': UserAgent().chrome})

    soup = BeautifulSoup(r.text,'html.parser')

    all_text = ' '.join(soup.findAll(text=True)).replace('\n\n', '')
    
    try:
        tab = re.findall(r"{}\s*split\s*history\s*.*?stock\s*splits".format(ticker), all_text.lower(), flags=re.DOTALL)[0]
    except IndexError:
        return  # no info on this ticker at all
    if 'for' in tab:
        splits = regex_split_info.findall(tab)
        if splits:
            return pd.DataFrame([(ticker.upper(), x[0], float(x[1]), float(x[3])) for x in splits], columns=['ticker', 'date', 'now', 'was'])

    return 
get_splits_data("AAPL")
from concurrent.futures import ThreadPoolExecutor
from time import sleep
%%time
data = None
while 1:
    with ThreadPoolExecutor(max_workers=70) as pool:  # arbitrary value - too much causes problems as well
        future = [pool.submit(get_splits_data, ticker) for ticker in tickers]
    
    try:
        data_temp = pd.concat([x.result() for x in future if x.done()
                        and not x.exception()
                        and x.result() is not None], ignore_index=True)  # concatenation of data for tickers
    except ValueError:
        data_temp = None
    
    if data_temp is not None:
        if data is None:
            data = data_temp
        else:
            data = pd.concat([data, data_temp], ignore_index=True)
        
    number_of_runs_with_errors = len([x.exception() for x in future if x.exception()])  # timeout errors, connection refuses
    print(number_of_runs_with_errors)
    if number_of_runs_with_errors == 0:
        break
        
    errored = list(filter(lambda x: x[0] is not None, zip([x.exception() for x in future], tickers)))
    tickers = [x[1] for x in errored]  # new tickers list - those that with errors. Scraping them once again
    sleep(10)
data.shape
data['ticker'].unique().shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13670647391577623, 'fraction_numerical': 0.011942174732872407, 'mean_word_length': 3.3247282608695654, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '84133', 'license': 'MIT', 'n_tokens_mistral': 1091, 'n_tokens_neox': 1055, 'n_words': 274, 'path': 'scraping/fake_useragent_scraping.ipynb', 'repo_name': 'Levich125/scraping_tests', 'size': '3182'}