<jupyter_start><jupyter_text>Advanced Lane Finding ProjectThe goals / steps of this project are the following:* Compute the camera calibration matrix and distortion coefficients given a set of chessboard images.* Apply a distortion correction to raw images.* Use color transforms, gradients, etc., to create a thresholded binary image.* Apply a perspective transform to rectify binary image ("birds-eye view").* Detect lane pixels and fit to find the lane boundary.* Determine the curvature of the lane and vehicle position with respect to center.* Warp the detected lane boundaries back onto the original image.* Output visual display of the lane boundaries and numerical estimation of lane curvature and vehicle position.--- Import libraries<jupyter_code>import helpers

from helpers import np
from helpers import glob
from helpers import cv2
from helpers import plt
from helpers import mpimg
from helpers import pickle
from helpers import os

%matplotlib inline<jupyter_output><empty_output><jupyter_text>Function to calibrate the camera<jupyter_code>def calibrateCamera(imgpath, nx, ny, calibrationfile):
    # prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
    objp = np.zeros((ny*nx,3), np.float32)
    objp[:,:2] = np.mgrid[0:nx,0:ny].T.reshape(-1,2)

    # Arrays to store object points and image points from all the images.
    objpoints = [] # 3d points in real world space
    imgpoints = [] # 2d points in image plane.

    # Make a list of calibration images
    images = glob.glob(imgpath)

    # Step through the list and search for chessboard corners
    for fname in images:
        img = cv2.imread(fname)
        gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)

        # Find the chessboard corners
        ret, corners = cv2.findChessboardCorners(gray, (nx,ny),None)

        # If found, add object points, image points
        if ret == True:
            objpoints.append(objp)
            imgpoints.append(corners)
            # Save the camera calibration result for later use (we won't worry about rvecs / tvecs)
            dist_pickle = {}
            dist_pickle["objpoints"] = objpoints
            dist_pickle["imgpoints"] = imgpoints
            pickle.dump(dist_pickle, open(calibrationfile, "wb"))

    cv2.destroyAllWindows()
# Example code
calibrateCamera('camera_cal/calibration*.jpg', 9, 6, "wide_dist_pickle.p")<jupyter_output><empty_output><jupyter_text>Undistort image function<jupyter_code>def undistortImage(img, calibrationfile):
    if(os.path.isfile(calibrationfile)):
        # Read in the saved objpoints and imgpoints
        dist_pickle = pickle.load(open(calibrationfile, "rb"))
        objpoints = dist_pickle["objpoints"]
        imgpoints = dist_pickle["imgpoints"]

        # Do camera calibration given object points and image points
        img_size = (img.shape[1], img.shape[0])
        ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, img_size, None, None)

        # Undistort the image
        undist = cv2.undistort(img, mtx, dist, None, mtx)
    else:
        undist = img

    return undist
# Example code
img = plt.imread('test_images/test1.jpg')
undist = undistortImage(img, "wide_dist_pickle.p")
plt.imshow(undist)
plt.imsave('doc_images/undist.jpg', undist)<jupyter_output><empty_output><jupyter_text>Thresholding functions to be combined<jupyter_code>def sobelThreshold(img, orient = 'x', channel = 'R', sobel_kernel = 3, thresh = (0, 255)):
    if channel == 'R':
        img_channel = img[:,:,0]
    elif channel == 'G':
        img_channel = img[:,:,1]
    elif channel == 'H':    
        hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)    
        img_channel = hls[:,:,0]    
    elif channel == 'S':    
        hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)    
        img_channel = hls[:,:,2]
    else:
        img_channel = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)

    # Apply x or y gradient with the OpenCV Sobel() function
    # and take the absolute value
    if orient == 'x':
        abs_sobel = np.absolute(cv2.Sobel(img_channel, cv2.CV_64F, 1, 0, ksize = sobel_kernel))
    if orient == 'y':
        abs_sobel = np.absolute(cv2.Sobel(img_channel, cv2.CV_64F, 0, 1, ksize = sobel_kernel))

    # Rescale back to 8 bit integer
    scaled_sobel = np.uint8(255 * abs_sobel / np.max(abs_sobel))

    # Create a copy and apply the threshold
    binary_output = np.zeros_like(scaled_sobel)

    # Here I'm using inclusive (>=, <=) thresholds, but exclusive is ok too
    binary_output[(scaled_sobel >= thresh[0]) & (scaled_sobel <= thresh[1])] = 1

    # Return the result
    return binary_output

def magThreshold(img, channel = 'R', sobel_kernel = 3, thresh = (0, 255)):
    if channel == 'R':
        img_channel = img[:,:,0]
    elif channel == 'G':
        img_channel = img[:,:,1]
    elif channel == 'H':    
        hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)    
        img_channel = hls[:,:,0]    
    elif channel == 'S':    
        hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)    
        img_channel = hls[:,:,2]
    else:
        img_channel = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    
    # Take both Sobel x and y gradients
    sobelx = cv2.Sobel(img_channel, cv2.CV_64F, 1, 0, ksize = sobel_kernel)
    sobely = cv2.Sobel(img_channel, cv2.CV_64F, 0, 1, ksize = sobel_kernel)

    # Calculate the gradient magnitude
    gradmag = np.sqrt(sobelx**2 + sobely**2)

    # Rescale to 8 bit
    scale_factor = np.max(gradmag)/255
    gradmag = (gradmag/scale_factor).astype(np.uint8)

    # Create a binary image of ones where threshold is met, zeros otherwise
    binary_output = np.zeros_like(gradmag)
    binary_output[(gradmag >= thresh[0]) & (gradmag <= thresh[1])] = 1

    # Return the binary image
    return binary_output

def dirThreshold(img, channel = 'R', sobel_kernel=3, thresh = (0, np.pi/2)):
    if channel == 'R':
        img_channel = img[:,:,0]
    elif channel == 'G':
        img_channel = img[:,:,1]
    elif channel == 'H':    
        hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)    
        img_channel = hls[:,:,0]    
    elif channel == 'S':    
        hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)    
        img_channel = hls[:,:,2]
    else:
        img_channel = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)

    # Calculate the x and y gradients
    sobelx = cv2.Sobel(img_channel, cv2.CV_64F, 1, 0, ksize = sobel_kernel)
    sobely = cv2.Sobel(img_channel, cv2.CV_64F, 0, 1, ksize = sobel_kernel)

    # Take the absolute value of the gradient direction,
    # apply a threshold, and create a binary image result
    absgraddir = np.arctan2(np.absolute(sobely), np.absolute(sobelx))
    binary_output =  np.zeros_like(absgraddir)
    binary_output[(absgraddir >= thresh[0]) & (absgraddir <= thresh[1])] = 1

    # Return the binary image
    return binary_output
# Example code
sx = sobelThreshold(undist, 'x', 'R', 9, thresh = (20, 100))
sy = sobelThreshold(undist, 'y', 'R', 9, thresh = (20, 100))
magthr = magThreshold(undist, 'R', 9, thresh = (30, 100))
dirthr = dirThreshold(undist, 'R', 15, (0.7, 1.3))

gr_binary = np.zeros_like(sx)
gr_binary[((sx == 1) & (sy == 1)) | ((magthr == 1) & (dirthr == 1))] = 1

plt.imshow(gr_binary, cmap = 'gray')
plt.imsave('doc_images/gr_binary.jpg', gr_binary, cmap = 'gray')<jupyter_output><empty_output><jupyter_text>Color transform function<jupyter_code>def color_select(img):
    LOW_WHITE = np.array([0, 0, 223], dtype=np.uint8)
    HIGH_WHITE = np.array([90,63,255], dtype=np.uint8)
    LOW_YELLOW = np.array([15,127,223], dtype=np.uint8)
    HIGH_YELLOW = np.array([30,255,255], dtype=np.uint8)

    hsv = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)

    wmask = cv2.inRange(hsv, LOW_WHITE, HIGH_WHITE)//255
    ymask = cv2.inRange(hsv, LOW_YELLOW, HIGH_YELLOW)//255
    binary_output = cv2.bitwise_or(ymask, wmask)
    
    return binary_output
# Example code
color_binary = color_select(undist)
plt.imshow(color_binary, cmap = 'gray')
plt.imsave('doc_images/color_binary.jpg', color_binary, cmap = 'gray')<jupyter_output><empty_output><jupyter_text>Combine the gradient and the color transformed images<jupyter_code>combined_binary = np.zeros_like(color_binary)
combined_binary[(color_binary == 1) | (gr_binary == 1)] = 1

plt.imshow(combined_binary, cmap = 'gray')
plt.imsave('doc_images/combined_binary.jpg', combined_binary, cmap = 'gray')<jupyter_output><empty_output><jupyter_text>Function to get the perspective change matrixes<jupyter_code>def getPerspectiveMatrix():
    # Define source and destination points
    # Considered line is about 20m length and 3.7m width
    src = np.float32([ [707, 462], [1043, 672], [275, 677], [578, 462] ])
    # 720 px length = 20m, 800 px width = 3.7m 
    dst = np.float32([ [1040, 0], [1040, 719], [240, 719], [240, 0] ])

    M = cv2.getPerspectiveTransform(src, dst)
    Minv = cv2.getPerspectiveTransform(dst, src)
    
    # Get and return M
    return M, Minv
# Example code
M, Minv = getPerspectiveMatrix()<jupyter_output><empty_output><jupyter_text>Warp image<jupyter_code># Example code
binary_warped = cv2.warpPerspective(combined_binary, M, (combined_binary.shape[1], combined_binary.shape[0]))
plt.imshow(binary_warped, cmap = 'gray')
plt.imsave('doc_images/binary_warped.jpg', binary_warped, cmap = 'gray')<jupyter_output><empty_output><jupyter_text>Lane finder and polynomial fit functions<jupyter_code># Function to find the line and fit the polynomial of the first considered frame
def findLines(binary_warped, nwindows = 9, margin = 100, minpix = 50, visualization = False):
    # Take a histogram of the bottom half of the image
    histogram = np.sum(binary_warped[binary_warped.shape[0]//2:,:], axis=0)
    # Create an output image to draw on and visualize the result
    out_img = np.dstack((binary_warped, binary_warped, binary_warped))
    # Find the peak of the left and right halves of the histogram
    # These will be the starting point for the left and right lines
    midpoint = np.int(histogram.shape[0]//2)
    leftx_base = np.argmax(histogram[:midpoint])
    rightx_base = np.argmax(histogram[midpoint:]) + midpoint
    
    # Compute distance from the center of the lane
    midlane = np.int(np.mean([leftx_base, rightx_base]))

    # Set height of windows - based on nwindows above and image shape
    window_height = np.int(binary_warped.shape[0]//nwindows)
    # Identify the x and y positions of all nonzero pixels in the image
    nonzero = binary_warped.nonzero()
    nonzeroy = np.array(nonzero[0])
    nonzerox = np.array(nonzero[1])
    # Current positions to be updated later for each window in nwindows
    leftx_current = leftx_base
    rightx_current = rightx_base

    # Create empty lists to receive left and right lane pixel indices
    left_lane_inds = []
    right_lane_inds = []

    # Step through the windows one by one
    for window in range(nwindows):
        # Identify window boundaries in x and y (and right and left)
        win_y_low = binary_warped.shape[0] - (window+1)*window_height
        win_y_high = binary_warped.shape[0] - window*window_height
        win_xleft_low = leftx_current - margin
        win_xleft_high = leftx_current + margin
        win_xright_low = rightx_current - margin
        win_xright_high = rightx_current + margin
        
        # Identify the nonzero pixels in x and y within the window #
        good_left_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & 
        (nonzerox >= win_xleft_low) &  (nonzerox < win_xleft_high)).nonzero()[0]
        good_right_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & 
        (nonzerox >= win_xright_low) &  (nonzerox < win_xright_high)).nonzero()[0]
        
        # Append these indices to the lists
        left_lane_inds.append(good_left_inds)
        right_lane_inds.append(good_right_inds)
        
        # If you found > minpix pixels, recenter next window on their mean position
        if len(good_left_inds) > minpix:
            leftx_current = np.int(np.mean(nonzerox[good_left_inds]))
        if len(good_right_inds) > minpix:        
            rightx_current = np.int(np.mean(nonzerox[good_right_inds]))

    # Concatenate the arrays of indices (previously was a list of lists of pixels)
    left_lane_inds = np.concatenate(left_lane_inds)
    right_lane_inds = np.concatenate(right_lane_inds)

    # Extract left and right line pixel positions
    leftx = nonzerox[left_lane_inds]
    lefty = nonzeroy[left_lane_inds] 
    rightx = nonzerox[right_lane_inds]
    righty = nonzeroy[right_lane_inds]

    left_fit, left_fitx, ploty = helpers.fit_poly(binary_warped.shape, leftx, lefty)
    right_fit, right_fitx, ploty = helpers.fit_poly(binary_warped.shape, rightx, righty)
    
    left_curv = helpers.compute_curvature(binary_warped, leftx, lefty)
    right_curv = helpers.compute_curvature(binary_warped, rightx, righty)
    
    xm_per_pix = 3.7 / 800
    distance = (midpoint - midlane) * xm_per_pix
    
    # Visualization
    if(visualization == True):
        helpers.line_visualization(binary_warped, nonzerox, nonzeroy, left_lane_inds, right_lane_inds, left_fitx, right_fitx, ploty, margin)
    
    return left_fit, right_fit, left_fitx, right_fitx, ploty, left_curv, right_curv, distance
# Example code
left_fit, right_fit, left_fitx, right_fitx, ploty, left_curv, right_curv, distance = findLines(binary_warped, visualization = True)<jupyter_output><empty_output><jupyter_text>Drawing<jupyter_code># Example code
curv = np.mean([left_curv, right_curv])

result = helpers.drawLane(undist, binary_warped, ploty, left_fitx, right_fitx, curv, distance, Minv)
plt.imshow(result)
plt.imsave('doc_images/result.jpg', result)<jupyter_output><empty_output><jupyter_text>Image pipeline<jupyter_code>left_fit = [] 
right_fit = [] 
left_fitx = []
right_fitx = []
ploty = []
left_curv = 0.0
right_curv = 0.0
distance = 0.0

def processImage(img):
    # Undistort the image
    undist = undistortImage(img, "wide_dist_pickle.p")

    # Set result
    result = undist

    # Apply gradient thresholding
    sx = sobelThreshold(undist, 'x', 'R', 9, thresh = (20, 100))
    sy = sobelThreshold(undist, 'y', 'R', 9, thresh = (20, 100))
    magthr = magThreshold(undist, 'R', 9, thresh = (30, 100))
    dirthr = dirThreshold(undist, 'R', 15, (0.7, 1.3))

    gr_binary = np.zeros_like(sx)
    gr_binary[((sx == 1) & (sy == 1)) | ((magthr == 1) & (dirthr == 1))] = 1

    # Apply color thresholding on S channel
    color_binary = color_select(undist)

    # Combine gradient and color thresholding
    combined_binary = np.zeros_like(color_binary)
    combined_binary[(color_binary == 1) | (gr_binary == 1)] = 1

    # Warp image
    binary_warped = cv2.warpPerspective(combined_binary, M, (combined_binary.shape[1], combined_binary.shape[0]))

    try:
        # Find lines
        left_fit, right_fit, left_fitx, right_fitx, ploty, left_curv, right_curv, distance = \
        findLines(binary_warped)

        # Draw lane
        curv = np.mean([left_curv, right_curv])
        result = helpers.drawLane(undist, binary_warped, ploty, left_fitx, right_fitx, curv, distance, Minv)
    except:
        print('Undetected line')

    return result<jupyter_output><empty_output><jupyter_text>Apply pipeline on images<jupyter_code>input_folder = 'test_images/'
output_folder = 'output_images/'

for fname in os.listdir(input_folder):
    # Read the image
    img = plt.imread(input_folder + fname)
    # Apply the pipeline
    result = processImage(img)
    # Save the image
    plt.imsave(output_folder + fname, result)<jupyter_output><empty_output><jupyter_text>Video pipeline<jupyter_code>class LaneFinder():
    def __init__(self):
        # Polynome coefficients
        self.__left_fit = []
        self.__right_fit = []
        
        # Average of x points
        self.__left_fitx_avg = []
        self.__right_fitx_avg = []
        
        # x and y points
        self.__left_fitx = []
        self.__right_fitx = []
        self.__ploty = []
        
        # Curvature and distance
        self.__curv_avg = 0.0
        self.__left_curv = []
        self.__right_curv = []
        
        self.__distance_avg = 0.0
        self.__distance = []
        
    def __setAttributes(self, left_fit, right_fit, left_fitx, right_fitx, ploty, left_curv, right_curv, distance):
        # Current polynome coefficients
        self.__left_fit = left_fit
        self.__right_fit = right_fit
        
        # Append current x points
        self.__left_fitx.append(left_fitx)
        self.__right_fitx.append(right_fitx)
        # Append curvature and distance
        self.__left_curv.append(left_curv)
        self.__right_curv.append(right_curv)
        self.__distance.append(distance)

        # y points
        self.__ploty = ploty
        
        # Averages
        if(len(self.__left_fitx) > 5):
            self.__left_fitx = self.__left_fitx[1:]
            self.__right_fitx = self.__right_fitx[1:]
            self.__left_curv = self.__left_curv[1:]
            self.__right_curv = self.__right_curv[1:]
            self.__distance = self.__distance[1:]
        self.__left_fitx_avg = np.mean(self.__left_fitx, axis = 0)
        self.__right_fitx_avg = np.mean(self.__right_fitx, axis = 0)
        self.__curv_avg = (np.mean(self.__left_curv, axis = 0) + np.mean(self.__right_curv, axis = 0))/2
        self.__distance_avg = np.mean(self.__distance, axis = 0)
        
    def __sanityCheck(self, left_fitx, right_fitx, left_curv, right_curv):
        # Is curvature right? Is the distance between the two lines right?
        right_curvature = (left_curv > 100) and (right_curv > 100)
        distance = np.absolute(np.mean(left_fitx) - np.mean(right_fitx))
        right_distance = distance > 700 and distance < 900
        
        return right_curvature and right_distance

    def processImage(self, img):
        # Undistort the image
        undist = undistortImage(img, "wide_dist_pickle.p")
        
        # Set result
        result = undist

        # Apply gradient thresholding
        sx = sobelThreshold(undist, 'x', 'R', 9, thresh = (20, 100))
        sy = sobelThreshold(undist, 'y', 'R', 9, thresh = (20, 100))
        magthr = magThreshold(undist, 'R', 9, thresh = (30, 100))
        dirthr = dirThreshold(undist, 'R', 15, (0.7, 1.3))

        gr_binary = np.zeros_like(sx)
        gr_binary[((sx == 1) & (sy == 1)) | ((magthr == 1) & (dirthr == 1))] = 1

        # Apply color thresholding
        color_binary = color_select(undist)

        # Combine gradient and color thresholding
        combined_binary = np.zeros_like(color_binary)
        combined_binary[(color_binary == 1) | (gr_binary == 1)] = 1

        # Warp image
        binary_warped = cv2.warpPerspective(combined_binary, M, (combined_binary.shape[1], combined_binary.shape[0]))
        
        try:
            # Find lines
            left_fit, right_fit, left_fitx, right_fitx, ploty, left_curv, right_curv, distance = \
            findLines(binary_warped)
            
            # Sanity check
            if(self.__sanityCheck(left_fitx, right_fitx, left_curv, right_curv)):
                self.__setAttributes(left_fit, right_fit, left_fitx, right_fitx, ploty, 
                                     left_curv, right_curv, distance)
            
            # Draw lane
            if(len(self.__ploty) != 0):
                result = helpers.drawLane(undist, binary_warped, self.__ploty, self.__left_fitx_avg, 
                                          self.__right_fitx_avg, self.__curv_avg, self.__distance_avg, Minv)
        except:
            print('Undetected line')

        return result<jupyter_output><empty_output><jupyter_text>Include libraries for videos<jupyter_code># Import everything needed to edit/save/watch video clips
from moviepy.editor import VideoFileClip
from IPython.display import HTML<jupyter_output><empty_output><jupyter_text>Apply the pipeline on project video<jupyter_code>lf = LaneFinder()

white_output = 'output_project_video.mp4'
## To speed up the testing process you may want to try your pipeline on a shorter subclip of the video
## To do so add .subclip(start_second,end_second) to the end of the line below
## Where start_second and end_second are integer values representing the start and end of the subclip
## You may also uncomment the following line for a subclip of the first 5 seconds
##clip1 = VideoFileClip("test_videos/solidWhiteRight.mp4").subclip(0,5)
clip1 = VideoFileClip("project_video.mp4")
white_clip = clip1.fl_image(lf.processImage) #NOTE: this function expects color images!!
%time white_clip.write_videofile(white_output, audio=False)
HTML("""
<video width="960" height="540" controls>
  <source src="{0}">
</video>
""".format(white_output))<jupyter_output><empty_output><jupyter_text>Apply the pipeline on challeng video<jupyter_code>lf = LaneFinder()

white_output = 'output_challenge_video.mp4'
## To speed up the testing process you may want to try your pipeline on a shorter subclip of the video
## To do so add .subclip(start_second,end_second) to the end of the line below
## Where start_second and end_second are integer values representing the start and end of the subclip
## You may also uncomment the following line for a subclip of the first 5 seconds
##clip1 = VideoFileClip("test_videos/solidWhiteRight.mp4").subclip(0,5)
clip1 = VideoFileClip("challenge_video.mp4")
white_clip = clip1.fl_image(lf.processImage) #NOTE: this function expects color images!!
%time white_clip.write_videofile(white_output, audio=False)
HTML("""
<video width="960" height="540" controls>
  <source src="{0}">
</video>
""".format(white_output))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10919168591224018, 'fraction_numerical': 0.02092378752886836, 'mean_word_length': 3.403294691885296, 'pattern_counts': {'":': 0, '<': 80, '<?xml version=': 0, '>': 86, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '132153', 'license': 'MIT', 'n_tokens_mistral': 7676, 'n_tokens_neox': 7326, 'n_words': 1807, 'path': 'P2.ipynb', 'repo_name': 'edoardomazzella/CarND-Advanced-Lane-Lines-P2', 'size': '21650'}