<jupyter_start><jupyter_text>A Machine Learning Program to predict price of used cars for sale  Importing Libraries<jupyter_code>import requests
from bs4 import BeautifulSoup
import re
import os
from datetime import datetime
import numpy as np
import pandas as pd
import glob<jupyter_output><empty_output><jupyter_text>Creating a function to get and parse a url<jupyter_code>def getAndParseURL(url):
    result = requests.get(url)
    soup = BeautifulSoup(result.text, 'html.parser')
    return(soup)
carUrls = []
for page in range(0,629):
    main_url = "https://www.cheki.com.ng/vehicles?page="+str(page)
    
    soup = getAndParseURL(main_url)
    carlink = [x.a.get('href') for x in soup.findAll("div", class_ = "card-main")]
    #car = soup.find_all("div", class_ = "card-main")
    for i in carlink:
        carurl = "https://www.cheki.com.ng" + i
        carUrls.append(carurl)
dat = pd.DataFrame(data={"url": carUrls})
dat.to_csv("carurl.csv", sep=',',index=False)
current_dir = os.path.abspath(os.curdir)
os.chdir("../..")
main_dir = os.path.abspath(os.curdir)
main_dir
# File Paths
raw_data_path = os.path.join(main_dir,'data','raw')
#raw_data_path
car_data_path = os.path.join(raw_data_path, 'used_car_data.csv')
car_data_path
carUrls[6]<jupyter_output><empty_output><jupyter_text>What we need to getConsider getting the name, brand, price, mileage, condition, body type, color, drive type, fuel, drive setup, transmision, interior type, door count and year.<jupyter_code>df = pd.DataFrame(columns = ['year',
            'brand', 'model', 'mileage', 'condition',
            'body_type', 'color', 'drive_type', 'fuel',
            'drive_setup',
            'transmission', 'interior_type', 'door_count',
            'star', 'review_no', 'date_posted', 'price'])

for car in dat.url[11000:12049]:
    # Fetch the car brand and model
    soup = getAndParseURL(car)
    try:
      soup.find('div', {'class': 'bread-crumbs'}).findAll('li')
    except AttributeError:
      brand = 'NA'
      model = 'NA'
    else:
      if soup.find('div', {'class': 'bread-crumbs'}).findAll('li') == None:
        brand = 'NA'
        model = 'NA'
      else:
        brand_model = soup.find('div', {'class': 'bread-crumbs'}).findAll('li')
        try:
          brand_model[2].text
        except IndexError:
          brand = 'NA'
        else:
          if brand_model[2].text == None:
            brand = 'NA'
          else:
            brand = brand_model[2].text
        try:
          brand_model[3].text
        except IndexError:
          model = 'NA'
        else:
          if brand_model[3].text == None:
            model = 'NA'
          else:
            model = brand_model[3].text
            #print(brand, model)
    
    # Fetch the car price
    try:
      soup.find('div', class_ = "listing-detail__price").span.text[2:]
    except AttributeError:
      price = np.NaN
    else:
      if soup.find('div', class_ = "listing-detail__price").span.text[2:] == None:
        price = np.NaN
      else:
        price = soup.find('div', class_ = "listing-detail__price").span.text[2:]
        price = price.strip()
        price = re.sub('[,]', '', price)
        #type(price)
        price = int(price)
        #print(type(price))
    
    # Fetch other car feature
    info = soup.find('div', {'class':'listing-detail__attributes'})
    # Mileage
    try:
      info.find("dt", text="Mileage (km)")
    except AttributeError:
      mileage = np.NaN
    else:
      if info.find("dt", text="Mileage (km)") == None:
        mileage = np.NaN
      else:
        mileage = info.find("dt",text="Mileage (km)").findNext("dd").string
        #print(mileage)
    
    # Fetch car condition
    try:
      info.find("dt",text="Condition")
    except AttributeError:
      cond = 'NA'
    else:
      if info.find("dt",text="Condition") == None:
        cond = 'NA'
      else:
        cond = info.find("dt",text="Condition").findNext("dd").string
    #print(cond)
    # Fetch body type
    try:
      info.find("dt",text="Body Type")
    except AttributeError:
      b_type = 'NA'
    else:
      if info.find("dt",text="Body Type") == None:
        b_type = 'NA'
      else:
        b_type = info.find("dt",text="Body Type").findNext("dd").string
    #print(b_type)
    # Fetch color
    try:
      info.find("dt",text="Colour")
    except AttributeError:
      color = 'NA'
    else:
      if info.find("dt",text="Colour") == None:
        color = 'NA'
      else:
        color = info.find("dt",text="Colour").findNext("dd").string
    #print(color)
    # Fetch drive type
    try:
      info.find("dt",text="Drive Type")
    except AttributeError:
      d_type = 'NA'
    else:
      if info.find("dt",text="Drive Type") == None:
        d_type = 'NA'
      else:
        d_type = info.find("dt",text="Drive Type").findNext("dd").string
        #print(d_type)
    # Fetch Fuel type
    try:
      info.find("dt",text="Fuel")
    except AttributeError:
      fuel = 'NA'
    else:
      if info.find("dt",text="Fuel") == None:
        fuel = 'NA'
      else:
        fuel = info.find("dt",text="Fuel").findNext("dd").string
    #print(fuel)
    # Fetch drive setup
    try:
      info.find("dt",text="Drive Setup")
    except AttributeError:
      d_setup = 'NA'
    else:
      if info.find("dt",text="Drive Setup") == None:
        d_setup = 'NA'
      else:
        d_setup = info.find("dt",text="Drive Setup").findNext("dd").string
    #print(d_setup)
    # Fetch transmission
    try:
      info.find("dt",text="Transmission")
    except AttributeError:
      trans = 'NA'
    else:
      if info.find("dt",text="Transmission") == None:
        trans = 'NA'
      else:
        trans = info.find("dt",text="Transmission").findNext("dd").string
    #print(trans)
    # Fetch interior type
    try:
      info.find("dt",text="Interior Type")
    except AttributeError:
      i_type = 'NA'
    else:
      if info.find("dt",text="Interior Type") == None:
        i_type = 'NA'
      else:
        i_type = info.find("dt",text="Interior Type").findNext("dd").string
    #print(i_type)
    # Fetch door count
    try:
      info.find("dt",text="Door Count")
    except AttributeError:
      d_count = 'NA'
    else:
      if info.find("dt",text="Door Count") == None:
        d_count = 'NA'
      else:
        d_count = info.find("dt",text="Door Count").findNext("dd").string
    # Fetch production year
    try:
      info.find("dt",text="Year").findNext("dd")
    except AttributeError:
      year = np.NaN
    else:
      year = info.find("dt",text="Year").findNext("dd").string
      year = int(year)
    #print(year)
    #print(type(year))
    try:
      soup.find("meta", attrs={'name':'sailthru.date'})
    except TypeError:
      date = 'NA'
    else:
      if soup.find("meta", attrs={'name':'sailthru.date'}) == None:
        date = 'NA'
      else:
        date = soup.find("meta", attrs={'name':'sailthru.date'}) #name = "sailthru.date"
        date = date["content"]
        date = datetime.strptime(date, '%Y-%m-%d').date()
    
    # Check for star ratings and review
    try:
      soup.find('div', {'class':'review-stars-block'})
    except AttributeError:
      star_no = 0
      review_no = 0
    else:
      if soup.find('div', {'class':'review-stars-block'}) == None:
        star_no = 0
        review_no = 0
      else:
        review_list = soup.find('div', {'class':'review-stars-block'}).span.text
        review = review_list.split()
        star_no = review[0]
        star_no = int(star_no)
        review_no = re.sub('[()]', '', review[1])
        review_no = int(review_no)
        
    # Entering crawled data into a data frame
    data = {
            'year' : year, 
            'brand': brand.lower(),
            'model': model.lower(),
            'mileage' : mileage,
            'condition' : cond.lower(),
            'body_type' : b_type.lower(),
            'color' : color.lower(),
            'drive_type' : d_type.lower(),
            'fuel' : fuel.lower(),
            'drive_setup' : d_setup.lower(),
            'transmission' : trans.lower(),
            'interior_type' : i_type.lower(),
            'door_count' : d_count,
            'star': star_no,
            'review_no' : review_no,
            'date_posted' : date,
            'price': price}
    df.loc[len(df)] = data
    df.to_csv('cars.csv', index=False)
all_files = glob.glob(raw_data_path + "/*.csv")

li = []

for filename in all_files:

    df = pd.read_csv(filename, index_col=None, header=0)

    li.append(df)

car_data = pd.concat(li, axis=0, ignore_index=True)
car_data.to_csv(raw_data_path+'/Car_price/all_car_data.csv', index=False)<jupyter_output><empty_output><jupyter_text>Importing Data<jupyter_code># File Paths
current_dir = os.path.abspath(os.curdir)
os.chdir("../..")
main_dir = os.path.abspath(os.curdir)
raw_data_path = os.path.join(main_dir,'data','raw')
raw_data_path
car_data = pd.read_csv(raw_data_path+'/Car_price/all_car_data.csv')
type(car_data)<jupyter_output><empty_output><jupyter_text>Basic Structure<jupyter_code>car_data.info()
car_data.head(10)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15861540146784972, 'fraction_numerical': 0.0038339358089604556, 'mean_word_length': 2.5817967830521775, 'pattern_counts': {'":': 1, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '30077', 'license': 'MIT', 'n_tokens_mistral': 3141, 'n_tokens_neox': 3103, 'n_words': 622, 'path': 'src/notebooks/.ipynb_checkpoints/car-query-checkpoint.ipynb', 'repo_name': 'Danselem/car-price-prediction', 'size': '9129'}