<jupyter_start><jupyter_text>Multiple Products<jupyter_code>headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5).AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
# empty dataframe
df_details = pd.DataFrame()
# unique columns for all products
aux = []
cols = ['Art. No.', 'Composition', 'Fit', 'Product safety', 'Size']
df_pattern = pd.DataFrame( columns=cols )

for i in range( len( data ) ):
    # API Requests
    url = 'https://www2.hm.com/en_us/productpage.' + data.loc[i, 'product_id']+ '.html'
           
    page = requests.get( url, headers=headers )
    
    # Beautiful Soup object
    soup = BeautifulSoup( page.text, 'html.parser' )

    # ==================== color name =================================
    product_list = soup.find_all( 'a', class_='filter-option miniature' )
    color_name = [p.get( 'data-color' ) for p in product_list]

    # product id
    product_id = [p.get( 'data-articlecode' ) for p in product_list]
    df_color = pd.DataFrame( [product_id, color_name] ).T
    df_color.columns = ['product_id', 'color_name']
    
    # generate style id + color id
    df_color['style_id'] = df_color['product_id'].apply( lambda x: x[:-3] )
    df_color['color_id'] = df_color['product_id'].apply( lambda x: x[-3:] )

    # ==================== composition =================================
    product_composition_list = soup.find_all( 'div', class_='pdp-description-list-item' )
    product_composition = [list( filter( None, p.get_text().split( '\n' ) ) ) for p in product_composition_list]

    # reaname dataframe
    df_composition = pd.DataFrame( product_composition ).T
    df_composition.columns = df_composition.iloc[0]

    # delete first row
    df_composition = df_composition.iloc[1:].fillna( method='ffill' )

    # garantee the same number of columns
    df_composition = pd.concat( [df_pattern, df_composition], axis=0 )

    # generate style id + color id
    df_composition['style_id'] = df_composition['Art. No.'].apply( lambda x: x[:-3] )
    df_composition['color_id'] = df_composition['Art. No.'].apply( lambda x: x[-3:] )
    aux = aux + df_composition.columns.tolist()

    # merge data color + decomposition
    data_sku = pd.merge( df_color, df_composition[['style_id', 'Fit','Composition', 'Size', 'Product safety']], how='left', on='style_id' )          
    
    # all details products
    df_details = pd.concat( [df_details, data_sku], axis=0 )

# # Join Showroom data + details
# data['style_id'] = data['product_id'].apply( lambda x: x[:-3] )
# data['color_id'] = data['product_id'].apply( lambda x: x[-3:] )
# data_raw = pd.merge( data, df_details[['style_id', 'color_name', 'Fit','Composition', 'Size', 'Product safety']],how='left', on='style_id' )
data_raw.shape
data_raw.head()
# Product Name data set
data_raw['product_name'] = data_raw['product_name'].apply( lambda x: x.replace( ' ','_' ).lower() )

# Product Fit data set
data_raw['Fit'] = data_raw['Fit'].apply( lambda x: x.replace( ' ','_' ).lower() if pd.notnull( x ) else x )

# Product Price data set
# data_raw['product_price'] = data_raw['product_price'].apply( lambda x: x.replace( '$ ','')).astype( float )

# Product ID
data_raw['product_id'] = data_raw['product_id'].astype( int )

# Product Color
data_raw['color_id'] = data_raw['color_id'].astype( int )

# Style
data_raw['style_id'] = data_raw['style_id'].astype( int )

# Date
data_raw['scrapy_datetime'] = pd.to_datetime( data_raw['scrapy_datetime'], format='%Y-%m-%d %H:%M:%S' )

# Select the size
data_raw['size_number'] = data_raw['Size'].apply( lambda x: re.search('\d{3}cm', x).group(0) if pd.notnull( x ) else x )
data_raw['size_number'] = data_raw['size_number'].apply( lambda x: re.search('\d+', x).group(0) if pd.notnull( x ) else x)
data_raw['size_model'] = data_raw['Size'].str.extract('(\d+/\\d+)') 
data_raw = data_raw.drop( columns = ['Size','Product safety'])

# Cleanning unkown composition
data_raw = data_raw[~data_raw['Composition'].str.contains('Pocket lining', na = False )]
data_raw = data_raw[~data_raw['Composition'].str.contains('Shell', na = False )]
data_raw = data_raw[~data_raw['Composition'].str.contains('Lining', na = False )]
data_raw = data_raw[~data_raw['Composition'].str.contains('Pocket', na = False )]
data_raw.shape
# Break composition by comma
df1 = data_raw['Composition'].str.split( ',', expand=True )

# cotton | Spandex | elasterell
df_ref = pd.DataFrame(columns=['cotton','spandex', 'elasterell'])

# cotton
df_cotton = df1[0]
df_cotton.name = 'cotton'
df_ref = pd.concat( [df_ref, df_cotton ], axis=1 )
df_ref = df_ref.iloc[:, ~df_ref.columns.duplicated( keep='last') ]
df_ref['cotton'] = df_ref['cotton'].apply( lambda x: int( re.search('\d+', x ).group(0) ) / 100 if pd.notnull( x ) else x )

# Spandex
df_spandex = df1.loc[df1[1].str.contains( 'Spandex', na=True ), 1]
df_spandex.name = 'spandex'
df_ref = pd.concat( [df_ref, df_spandex], axis=1 )
df_ref = df_ref.iloc[:, ~df_ref.columns.duplicated( keep='last') ]
df_ref['spandex'] = df_ref['spandex'].apply( lambda x: int( re.search('\d+', x ).group(0) ) / 100 if pd.notnull( x ) else x )

# elasterell
df_elasterell = df1.loc[df1[1].str.contains( 'Elasterell', na=True ), 1]
df_elasterell.name = 'elasterell'
df_ref = pd.concat( [df_ref, df_elasterell], axis=1 )
df_ref = df_ref.iloc[:, ~df_ref.columns.duplicated( keep='last') ]
df_ref['elasterell'] = df_ref['elasterell'].apply( lambda x: int( re.search('\d+', x ).group(0) ) / 100 if pd.notnull( x ) else x )

# final join
data_raw = pd.concat( [data_raw, df_ref], axis=1 )
data_raw<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19382473674817063, 'fraction_numerical': 0.012136355523826522, 'mean_word_length': 4.789256198347108, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '39057', 'license': 'MIT', 'n_tokens_mistral': 2091, 'n_tokens_neox': 2018, 'n_words': 434, 'path': 'Modulo_4/webscrapping_beautifulsoup.ipynb', 'repo_name': 'RPerottoni/HM_WebScraping', 'size': '5603'}