<jupyter_start><jupyter_text>Team crawler notebook  Imports<jupyter_code>import pandas as pd
import numpy as np
import requests
from bs4 import BeautifulSoup
import bs4
import re<jupyter_output><empty_output><jupyter_text>Crawling variables<jupyter_code>headers = {
   'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
}
base_url = 'https://www.transfermarkt.co.uk/'<jupyter_output><empty_output><jupyter_text>Data<jupyter_code>teams = pd.read_csv('../src/teams.csv', index_col=0)
teams_url = teams.url.values
print(base_url + teams_url[0][1:])
s = 'https://www.transfermarkt.co.uk/gelios-kharkiv/startseite/verein/16242/saison_id/2018'
requests.get(base_url + teams_url[0][1:], headers=headers)
url = s
print(url)
result = requests.get(url, headers=headers)
soup = BeautifulSoup(result.content, 'html5lib')
table = soup.find(name='table', attrs={'class': 'items'})
table_body = table.find(name='tbody')
rows = table_body.findAll('tr', attrs={'class': ['odd', 'even']})
t = []
for r in rows:
    t.append({
        'name': get_name(r),
        'number': get_number(r),
        'position': get_position(r),
        'date_of_birth_and_age': get_date_of_birth_and_age(r),
        'nationality': get_nationalities(r),
        'market_value': get_market_value(r)
    })
pd.DataFrame(t)
rows[0].find('td', {'class': 'rechts hauptlink'}).contents[0]
def check_if_squad_is_empty(soup):
    return soup.find('span', {'class': 'empty'}).contents[0] == 'No squad'

def get_number(row):
    return row.find('div', {'class': 'rn_nummer'}).contents[0]


def get_name(row):
    return row.find('a', {'class': 'spielprofil_tooltip'}).contents[0]


def get_position(row):
    return row.find('table').findAll('td')[2].contents[0]


def get_date_of_birth_and_age(row):
    return row.findAll('td', {'class': 'zentriert'})[1].contents[0]


def get_nationalities(row):
    countries_img = row.findAll('img', {'class': 'flaggenrahmen'})
    nationalities = [x.get('title') for x in countries_img]
    if len(nationalities) == 1:
        return nationalities[0]
    return nationalities


def get_market_value(row):
    return row.find('td', {'class': 'rechts hauptlink'}).contents[0]
leagues = pd.read_csv('../src/leagues.csv', index_col=0)
teams = pd.read_csv('../src/teams.csv', index_col=0)
leagues.head()
teams.head()
teams.shape
teams.drop_duplicates()
teams.merge(leagues, on='league', how='outer')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1736, 'fraction_numerical': 0.0228, 'mean_word_length': 5.530026109660574, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '246897', 'license': 'MIT', 'n_tokens_mistral': 979, 'n_tokens_neox': 950, 'n_words': 161, 'path': 'data-pipelines/crawler/notebooks/players_explorer.ipynb', 'repo_name': 'otaviocv/footballers_inequality', 'size': '2500'}