<jupyter_start><jupyter_text>Imports<jupyter_code>import urllib.request as req
from bs4 import BeautifulSoup as beausp
import os, re, logging, sys, time
from IPython.display import Image
sps = set(sys.path)
sps.add('..')
sys.path = list(sps)
from core.util import strings

logging.basicConfig(stream=sys.stderr, level=logging.INFO)
log = logging.getLogger("main")


tweettextattrs={'class': 'tweet-text'}<jupyter_output><empty_output><jupyter_text>Local test Get sample HTML containing tweets with outer divs<jupyter_code># Get the real path to the file for the URL

test_file=os.path.realpath('../data/smaller.html')
test_file<jupyter_output><empty_output><jupyter_text>Open test file, parse into `tsoup` object<jupyter_code>testpage='file:///' + test_file
tpage = req.urlopen(url=testpage)
tsoup = beausp(tpage, 'html.parser')
print (f"Type of soup object is: {type(tsoup)}")<jupyter_output><empty_output><jupyter_text>Find `` elements in html<jupyter_code>twts=tsoup.findAll(name='p', attrs=tweettextattrs)
# help(tsoup.find)

twtext=twts[0].text
print(f"Matches found: {len(twts)}")
print(twtext)<jupyter_output><empty_output><jupyter_text>Read a page from the web and parse it<jupyter_code>mypage='https://twitter.com/jon_bois'
page = req.urlopen(url=mypage)
soup = beausp(page, 'html.parser')<jupyter_output><empty_output><jupyter_text>Find `` elements<jupyter_code>log.setLevel(logging.INFO)

outertwts=soup.findAll(name='div', attrs={'class':'tweet',
                                          'data-screen-name':'jon_bois'})
print(f"Tweets found: {len(outertwts)}")

twt1=outertwts[0]
twttext=twt1.find(name='p', attrs=tweettextattrs)

print(f"First tweet text: {twttext.text}")

nourls=strings.strip_url(twttext.text)
print(f"Stripped urls: {nourls}")

nononwords=strings.clean(inp=twttext.text,
                         strip_non_word=True, strip_urls=True)
print(f"Stripped non words: {nononwords}")<jupyter_output><empty_output><jupyter_text>Find the time of posting within the element<jupyter_code>timeel=twt1.find(name='small', attrs={'class':'time'})
int(timeel.
    findChild(name='a').
    findChild(name='span').
    attrs['data-time'])<jupyter_output><empty_output><jupyter_text>Find the user icon<jupyter_code>img=soup.find(name='img', attrs={'class': 'ProfileAvatar-image'})

Image(url=img.attrs['src'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1508071367884452, 'fraction_numerical': 0.002548853016142736, 'mean_word_length': 6.983050847457627, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '524841', 'license': 'Apache-2.0', 'n_tokens_mistral': 892, 'n_tokens_neox': 893, 'n_words': 141, 'path': 'notebooks/httptest.ipynb', 'repo_name': 'levinsamuel/boisbot', 'size': '2354'}