<jupyter_start><jupyter_text>Web requests<jupyter_code>import requests

response = requests.get('https://www.eltiempo.com/')

#response?
#response??
print(dir(response))
print(response.status_code)
print(response.headers)
print(response.headers['date'])
print(response.text)<jupyter_output><!DOCTYPE html>
<html>
    <head>
                                    <script type='text/javascript'>var _sf_startpt=(new Date()).getTime()</script>
            <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
                
                
                
        <meta name="format-detection" content="telephone=no"/>

        <meta name="author" content="<NAME>"/>

        <meta name="genre" content="News"/>

        <meta name="geo.placename" content="Colombia"/>

        <meta name="geo.position" content="4.570868;-74.297333"/>

        <meta name="geo.region" content="CO"/>

        <meta name="google" content="notranslate"/>

        <meta name="ICBM" content="4.570868, -74.297333"/>

        <meta name="language" content="spanish"/>

    

                <meta name="twitter:card" content="summary"/>

            <meta name="twitter:site" content="@ELTIEMPO"/>

            <meta name="twitter:creator" content="@ELTIEMPO"/>

            <[...]<jupyter_text>Extracting information from newspaper page<jupyter_code>import bs4

soup = bs4.BeautifulSoup(response.text, 'html.parser')

print(soup.title.text)
print(soup.select('meta[name=description]'))
print(soup.select('meta[name=description]')[0]['content'])
soup = bs4.BeautifulSoup(response.text, 'html.parser')

news_links = soup.select('.title-container a')
# print(news_links)
news = [item['href'] for item in news_links]

for item in news:
    print(item)<jupyter_output>/economia/sectores/pib-colombia-2019-crecio-al-3-3-como-esperaba-carrasquilla-462188
/economia/sectores/el-semaforo-de-la-economia-a-pocos-dias-de-conocerse-el-pib-del-2019-460838
/politica/gobierno/colombianos-en-wuhan-seran-evacuados-en-22-de-febrero-cancilleria-462214
/unidad-investigativa/video-de-rescate-de-maximo-lider-de-la-mafia-centroamericana-462166
/unidad-investigativa/video-de-rescate-de-maximo-lider-de-la-mafia-centroamericana-462166
/justicia/conflicto-y-narcotrafico/masacre-en-la-frontera-colombo-peruana-deja-seis-personas-muertas-462122
/justicia/delitos/capturados-en-finca-de-embajador-no-aceptaron-cargos-462136
/unidad-investigativa/media-tonelada-de-coca-colombiana-enreda-a-exjefe-de-adianas-de-bulgaria-461720
/politica/partidos-politicos/entrevista-con-maria-fernanda-cabal-sobre-ivan-duque-y-alvaro-uribe-462034
/bogota/bogota-se-blinda-ante-posible-amenaza-terrorista-462042
/bogota/delitos-del-detenido-que-hizo-cambiazo-en-hospital-de-meissen-y-se-fugo-461398
/just[...]
{'fraction_non_alphanumeric': 0.1620253164556962, 'fraction_numerical': 0.040867992766726945, 'mean_word_length': 3.835664335664336, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 26, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '517540', 'license': 'Apache-2.0', 'n_tokens_mistral': 1124, 'n_tokens_neox': 1030, 'n_words': 101, 'path': 'Web-requests.ipynb', 'repo_name': 'cristian-rincon/data-eng-1', 'size': '2783'}