<jupyter_start><jupyter_text>1.Page Link Scraping<jupyter_code>import bs4
import requests
from time import sleep
datastorage=[]
import csv

for w in range(1000,2015):
    url='https://www.pakwheels.com/used-cars/search/-/?page='+str(w)
    mainpage=requests.get(url)
    so0=bs4.BeautifulSoup(mainpage.text,'lxml')
    sleep(2)
    car_details_link=so0.find_all('a',class_="car-name ad-detail-path")
    carlink0='https://www.pakwheels.com/'
    for q in range(0,len(car_details_link)):
        datastr=['-']*2
        carlnk1=car_details_link[q].get('href')
        carlink=carlink0+carlnk1
        datastr[0]=carlink
        datastr[1]=w
#         datastorage.append(datastr)
        spamWriter = csv.writer(open('output_all_cars_links.csv', 'a'))
        spamWriter.writerow(datastr)
        
    if w%10==0:
        print (w)<jupyter_output><empty_output><jupyter_text>change the csv to excel remove duplicates  2.Loading the links from excel<jupyter_code>from openpyxl import load_workbook
wb = load_workbook(filename = 'output_all_cars_links.xlsx')
sheet_ranges = wb['output_all_cars_links']


linksto_profiles=[]
for row in sheet_ranges:
    linksto_profiles.append(row[0].value)

len(linksto_profiles)<jupyter_output><empty_output><jupyter_text>3.Image downloading<jupyter_code>import requests
import shutil
from os.path import basename
import os
from urlparse import urlsplit
import urllib2
#dir is not keyword
def makemydir(whatever):
    try:
        os.makedirs(whatever)
    except OSError:
        pass
    # let exception propagate if we just can't
    # cd into the specified directory
    os.chdir(whatever)
    
for lnk in range(0,len(linksto_profiles)):
    
    sleep(2)
    mainpage=requests.get(linksto_profiles[lnk])
    so0=bs4.BeautifulSoup(mainpage.text,'lxml')

    print linksto_profiles[lnk],lnk

    breadcrumb=so0.find_all('ul',class_="breadcrumb")[0]
    breadcrumb

    Brand=breadcrumb.find_all('span')[2].text
    Brand

    Model=breadcrumb.find_all('span')[3].text
    Model

    year=so0.find_all('p',itemprop="vehicleModelDate")[0].text.replace('\n','').replace(' ','')
    year

    color=so0.find_all('li',itemprop="color")[0].text
    color

    featured_clearfix=so0.find_all('ul',class_="list-unstyled ul-featured clearfix")[0]
    ada_data=featured_clearfix.find_all('li')
    for ad in range(len(ada_data)):
        if ada_data[ad].text=='Ad Ref #':
            Ref=ada_data[ad+1].text
    Ref



    mb40=so0.find_all('div',class_="mb40 pos-rel")[0]

    makemydir(Brand+'/'+Model+'/'+year+'/'+color)

    images=mb40.find_all('img')
    for image_number in range(len(images)):
        url_image=images[image_number].get('data-original')
        if url_image!=None:    
            image_name=Brand+'_'+Model+'_'+year+'_'+color+'_'+Ref+'_'+str(image_number)+'.'+url_image.split('.')[-1]

            r = requests.get(url_image, stream=True, headers={'User-agent': 'Mozilla/5.0'})
            if r.status_code == 200:
                with open(image_name, 'wb') as f:
                    r.raw.decode_content = True
                    shutil.copyfileobj(r.raw, f)

    os.chdir('..')
    os.chdir('..')
    os.chdir('..')
    os.chdir('..')<jupyter_output><empty_output><jupyter_text>if error occured or programmed stoppedcheck the directory first directory must be the main where code is running.<jupyter_code>os.getcwd() # current directory<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14302427610412402, 'fraction_numerical': 0.016964024568587307, 'mean_word_length': 3.776536312849162, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '316935', 'license': 'MIT', 'n_tokens_mistral': 1290, 'n_tokens_neox': 1232, 'n_words': 192, 'path': 'pakwheels.com/pictures_Saving_pakwheels.ipynb', 'repo_name': 'afaq-ahmad/Web-Scraping-Practices', 'size': '3419'}