<jupyter_start><jupyter_text>Population Distribution Comparison<jupyter_code>ad_brazil_cities = pd.DataFrame(brazil_impressions['city'].value_counts(normalize=True)).reset_index()
ad_brazil_cities.rename(columns={"index": "name", "city": "ad_frequency"}, inplace=True)
brazil_city_pops = pd.read_csv('brazil_pop_by_city.csv')
brazil_city_pops = brazil_city_pops.drop(columns=['location'])
total_pop = brazil_city_pops['pop'].sum()
brazil_city_pops['proportion'] = brazil_city_pops['pop']/total_pop
brazil_city_pops = brazil_city_pops.drop(columns=['pop'])

ad_brazil_cities.set_index('name', inplace=True)
brazil_city_pops.set_index('name', inplace=True)
ad_brazil_cities

joined = brazil_city_pops.join(ad_brazil_cities)
joined.sort_values(by='ad_frequency', ascending=False)[:100]<jupyter_output><empty_output><jupyter_text>Device Comparisoni.e. samsung, motorola, apple, etc.<jupyter_code>brazil_impressions["device_make"].value_counts()
ad_device_data_brazil = pd.DataFrame(brazil_impressions["device_make"].value_counts(normalize=True)).reset_index()
ad_device_data_brazil.rename(columns={"index": "device_make", "device_make": "ad_frequency"}, inplace=True)
ad_device_data_brazil = ad_device_data_brazil.set_index("device_make")
len(ad_device_data_brazil)

#https://gs.statcounter.com/vendor-market-share/mobile-tablet-console/brazil/#monthly-201909-202009
#make sure its all devices, not just mobile

brazil_device_dist = pd.read_csv("brazil_all_device.csv")

distribution_2020 = brazil_device_dist[brazil_device_dist["Date"]=='2020-09'].iloc[0, 1:]/100
brazil_mobile_devices = pd.DataFrame(distribution_2020).rename(columns={12:'true frequency'})
brazil_mobile_devices.index = brazil_mobile_devices.index.rename("device_make")
len(brazil_mobile_devices)

joined_devices = brazil_mobile_devices.join(ad_device_data_brazil)
joined_devices
brazil_impressions['device_type'].value_counts()<jupyter_output><empty_output><jupyter_text>Browsers i.e. Chrome, Safari, Firefox  Browsers: All Devices<jupyter_code>#https://gs.statcounter.com/browser-market-share/all/brazil
#distribution of all browsers 

brazil_browser = pd.read_csv('brazil_browser.csv')
brazil_browser_2020 = brazil_browser[brazil_browser["Date"]=='2020-09'].iloc[0, 1:]/100
brazil_browser_2020 = pd.DataFrame(brazil_browser_2020).rename(columns={12:'actual dist'})
brazil_browser_2020.index.name = 'browser_name'

ad_brazil_browser = pd.DataFrame(brazil_impressions['browser'].value_counts(normalize=True)).reset_index()
ad_brazil_browser.rename(columns={"index": "browser_name", "browser": "browser_frequency"}, inplace=True)
browser_map = {1: 'other', 7: 'Safari', 8: 'Opera', 6: 'Chrome', 15:'other' }
ad_brazil_browser['browser_name'] = ad_brazil_browser['browser_name'].map(browser_map)
ad_brazil_browser = ad_brazil_browser.set_index('browser_name')

ad_brazil_browser.join(brazil_browser_2020)<jupyter_output><empty_output><jupyter_text>Browser: Mobile<jupyter_code>#https://gs.statcounter.com/browser-market-share/mobile/brazil/#monthly-201909-202009
#browser on mobile devices 

browser_map = {1: 'other', 7: 'Safari', 8: 'Opera', 6: 'Chrome', 15:'other' }

brazil_mobile_browser = pd.read_csv('brazil_mobile_browser.csv')
brazil_mobile_browser = brazil_mobile_browser[brazil_mobile_browser["Date"]=='2020-09'].iloc[0, 1:]/100
brazil_mobile_browser = pd.DataFrame(brazil_mobile_browser).rename(columns={12:'actual dist'})
brazil_mobile_browser.index.name = 'browser_name'

mobile = brazil_impressions[brazil_impressions['device_type']==4]

ad_brazil_mobile_browser = pd.DataFrame(mobile['browser'].value_counts(normalize=True)).reset_index()
ad_brazil_mobile_browser.rename(columns={"index": "browser_name", "browser": "browser_frequency"}, inplace=True)
ad_brazil_mobile_browser['browser_name'] = ad_brazil_mobile_browser['browser_name'].map(browser_map)
ad_brazil_mobile_browser = ad_brazil_mobile_browser.set_index('browser_name')

ad_brazil_mobile_browser.join(brazil_mobile_browser)<jupyter_output><empty_output><jupyter_text>Browser: Tablet<jupyter_code># https://gs.statcounter.com/browser-market-share/tablet/brazil/#monthly-201909-202009
# browser on tablet distrbution comparison

browser_map = {1: 'other', 7: 'Safari', 8: 'Opera', 6: 'Chrome', 15:'other' }

brazil_tablet_browser = pd.read_csv('brazil_tablet_browser.csv')
brazil_tablet_browser = brazil_tablet_browser[brazil_tablet_browser["Date"]=='2020-09'].iloc[0, 1:]/100
brazil_tablet_browser = pd.DataFrame(brazil_tablet_browser).rename(columns={12:'actual dist'})
brazil_tablet_browser.index.name = 'browser_name'

tablet = brazil_impressions[brazil_impressions['device_type']==3]

ad_brazil_tablet_browser = pd.DataFrame(tablet['browser'].value_counts(normalize=True)).reset_index()
ad_brazil_tablet_browser.rename(columns={"index": "browser_name", "browser": "browser_frequency"}, inplace=True)
ad_brazil_tablet_browser['browser_name'] = ad_brazil_tablet_browser['browser_name'].map(browser_map)
ad_brazil_tablet_browser = ad_brazil_tablet_browser.set_index('browser_name')

ad_brazil_tablet_browser.join(brazil_tablet_browser)<jupyter_output><empty_output><jupyter_text>Device Type: General i.e. Desktop vs mobile vs Tablet Distribution<jupyter_code>#https://gs.statcounter.com/platform-market-share/desktop-mobile-tablet/brazil
#distribution of device types 

device_type_map = {1: 'Other', 2: 'PC', 3: 'Tablet', 4:'Mobile', 5:'Roku', 6:'ConnectedTV'}
brazil_impressions['device_type'].map(device_type_map)
ad_device_type = brazil_impressions['device_type'].map(device_type_map).value_counts(normalize=True)

device_type = pd.read_csv('brazil_desktop_v_mobile.csv')
device_type_distribution_2020 = device_type[device_type["Date"]=='2020-09'].iloc[0, 1:]/100
device_type_distribution_2020 = pd.DataFrame(device_type_distribution_2020)
device_type_distribution_2020 = device_type_distribution_2020.rename(columns={12: 'actual_device_dist'})

device_type_distribution_2020.join(ad_device_type)<jupyter_output><empty_output><jupyter_text>Operating Systems  Operating System: All<jupyter_code>#https://gs.statcounter.com/os-market-share/all/brazil/#monthly-201909-202009
#all operating system comparisons

os_map = {1: 'other', 2:'Windows', 3:'OS X', 4: 'Linux', 5:'iOS', 6:'Android', 7:'Windows Phone'}

ad_os = pd.DataFrame(brazil_impressions['os_family'].map(os_map).value_counts(normalize=True))
ad_os.index.name = 'name'
ad_os = ad_os.rename(columns={'os_family':'ad_mobile_os'})

brazil_os = pd.read_csv('brazil_os_general.csv')
brazil_os = brazil_os[brazil_os["Date"]=='2020-09'].iloc[0, 1:]/100
brazil_os = pd.DataFrame(brazil_os).rename(columns={12:'actual dist'})
brazil_os.index.name = 'name'
brazil_os.join(ad_os)<jupyter_output><empty_output><jupyter_text>Operating System: Mobile Device<jupyter_code>#https://gs.statcounter.com/os-market-share/mobile/brazil/#monthly-201909-202009
#operating system broken down by mobile device

os_map = {1: 'other', 2:'Windows', 3:'OS X', 4: 'Linux', 5:'iOS', 6:'Android', 7:'Windows Phone'}

mobile = brazil_impressions[brazil_impressions['device_type']==4]

ad_mobile_os = pd.DataFrame(mobile['os_family'].map(os_map).value_counts(normalize=True))
ad_mobile_os.index.name = 'name'
ad_mobile_os = ad_mobile_os.rename(columns={'os_family':'ad_mobile_os'})

brazil_os_mobile = pd.read_csv('brazil_os_for_mobile.csv')
brazil_os_mobile = brazil_os_mobile[brazil_os_mobile["Date"]=='2020-09'].iloc[0, 1:]/100
brazil_os_mobile = pd.DataFrame(brazil_os_mobile).rename(columns={12:'actual dist'})
brazil_os_mobile.index.name = 'name'
brazil_os_mobile.join(ad_mobile_os)<jupyter_output><empty_output><jupyter_text>Operating System: Tablet<jupyter_code>#https://gs.statcounter.com/os-market-share/tablet/brazil/#monthly-201909-202009
#operating system broken down by tablet

os_map = {1: 'other', 2:'Windows', 3:'OS X', 4: 'Linux', 5:'iOS', 6:'Android', 7:'Windows Phone'}

tablet = brazil_impressions[brazil_impressions['device_type']==3]

ad_mobile_os = pd.DataFrame(tablet['os_family'].map(os_map).value_counts(normalize=True))
ad_mobile_os.index.name = 'name'
ad_mobile_os = ad_mobile_os.rename(columns={'os_family':'ad_mobile_os'})

brazil_os_mobile = pd.read_csv('brazil_tablet_os.csv')
brazil_os_mobile = brazil_os_mobile[brazil_os_mobile["Date"]=='2020-09'].iloc[0, 1:]/100
brazil_os_mobile = pd.DataFrame(brazil_os_mobile).rename(columns={12:'actual dist'})
brazil_os_mobile.index.name = 'name'
brazil_os_mobile.join(ad_mobile_os)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1465405085748078, 'fraction_numerical': 0.03311649911295092, 'mean_word_length': 11.114613180515759, 'pattern_counts': {'":': 10, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '511181', 'license': 'Apache-2.0', 'n_tokens_mistral': 3477, 'n_tokens_neox': 3235, 'n_words': 344, 'path': 'student-projects/fall-2020/Kinesso-AdShift-Diversifies-Marketing-Audiences/eda/[DEPRECATED] international_eda/brazil/brazil_eda.ipynb', 'repo_name': 'UCBerkeley-SCET/DataX-Berkeley', 'size': '8455'}