<jupyter_start><jupyter_text>Advanced Lane Finding for Video Import libraries and define pipeline functions<jupyter_code>import pickle
import cv2
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import os

## Define Line class to store current line information (left and right) and average across previous information
class Line():
    
    def __init__(self):
        # was the line detected in the last iteration?
        self.detected = False  
        # x values of the last n fits of the line
        self.recent_xfitted = np.array([])
        #average x values of the fitted line over the last n iterations
        self.bestx = None     
        #polynomial coefficients of the best (averaged) fit
        self.best_fit = None  
        #polynomial coefficients for the most recent fit
        self.current_fit = [np.array([False])]  
        #radius of curvature of the line in meters
        self.rad = None 
        #distance in meters of vehicle center from the line
        self.line_base_pos = None 
        #difference in fit coefficients between last and new fits
        self.diffs = np.array([0,0,0], dtype='float') 
        self.mag = None #norm of this difference vector
        self.diffs_magnitudes = np.array([]) #previous difference magnitudes for troubleshooting
        #x values for detected line pixels
        self.allx = None  
        #y values for detected line pixels
        self.ally = None 
        
    ## populate leftlane and rightlane objects with up-to-date lane information
    def populate_line(self, fitpoly, xfitted, ploty, pixelx, pixely):
       
        # maximum allowable differences between each fit coefficient (tuned)
        max_diff_mag = [10,30,47] # for straight
#         max_diff_mag = [10,30,50] # for wobbly
        n = 10 # number of iterations to keep
        
        # For tuning
        #difference in fit coefficients between last and new fits, store n iterations
        if len(self.recent_xfitted) > 0:
            self.diffs = self.current_fit - fitpoly
        self.mag = abs(self.diffs) #np.linalg.norm(self.diffs)
        if (len(self.diffs_magnitudes) < n) :
            self.diffs_magnitudes = np.hstack([self.diffs_magnitudes,self.mag])
        else: 
            self.diffs_magnitudes = np.hstack([self.diffs_magnitudes[1:],self.mag])
            
        # was the line detected?
        if fitpoly[2].all()==0 or fitpoly.size ==0 or self.mag[0] > max_diff_mag[0] or self.mag[1] > max_diff_mag[1] or self.mag[2] > max_diff_mag[2] :
            self.detected = False #if cooefficients not found, or difference in prev and current polynomial is too great,
                                #do nothing to keep last line's info, and mark flag false
        else: 
            self.detected = True #if coefficients found, update with current line info, and mark flag true

            # x values of the last n fits of the line
            if len(self.recent_xfitted) == 0: #if no fitted x values are stored, this is the first update
                self.recent_xfitted = xfitted
            elif (len(self.recent_xfitted.shape)==1): #if only one iteration has been stored, stack newest iteration
                self.recent_xfitted = np.vstack([self.recent_xfitted,xfitted])
            elif (self.recent_xfitted.shape[0] < n) : #as long as max iterations haven't been reached, stack newest
                self.recent_xfitted = np.vstack([self.recent_xfitted,xfitted])
            else: #remove first iteration in storage and stack newest (like a FIFO)
                self.recent_xfitted = np.vstack([self.recent_xfitted[1:],xfitted])

            #average x values of the fitted line over the last n iterations
            if len(self.recent_xfitted.shape)==2: #if the storage variable is an array, take the average
                self.bestx = self.recent_xfitted.sum(axis = 0)/np.shape(self.recent_xfitted)[0]
            else: #if it is a vector, directly copy
                self.bestx = self.recent_xfitted
                
            # calculate best polynomial coefficients using the average x-values of the previous n iterations
            self.best_fit = np.polyfit(ploty, self.bestx, 2) 
            self.current_fit = fitpoly
            self.rad = measure_curvature_real(self.bestx,ploty) # use best x to find radius of curvature
            self.line_base_pos = np.sum(self.bestx)/np.shape(self.bestx)[0] #calculate base position of line by taking average of best x-values of that line
            self.allx = pixelx
            self.ally = pixely    
            
def find_vehicle_pos(img_xsize):
    ## Finds the x-position of the vehicle with respect to the lane's center
    xm_per_pix = 3.7/700
    lane_center = (leftlane.line_base_pos + rightlane.line_base_pos)/2
    vehicle_center = img_xsize/2
    vehicle_pos = (vehicle_center-lane_center)*xm_per_pix
    if vehicle_pos < 0:
        direc = 'left of'
    elif vehicle_pos > 0:
        direc = 'right of'
    else:
        direc = 'on'
    return vehicle_pos, direc

def load_calibration_values():
    ## camera matrix and distortion coefficients have previously been calculated, this function provides those values
    ## for this particular camera
    mtx = np.array([[  1.15692360e+03,   0.00000000e+00,   6.65960994e+02],
       [  0.00000000e+00,   1.15212248e+03,   3.88738309e+02],
       [  0.00000000e+00,   0.00000000e+00,   1.00000000e+00]])
    dist = np.array([[ -2.37879129e-01,  -8.39288524e-02,  -7.83525418e-04,
         -1.15779462e-04,   1.03385417e-01]])
    return mtx, dist

def thresholds(img, s_thresh=(170, 255), sx_thresh=(20, 100)):
    ## performs saturation and gradient thresholds to provide binary image of possible lane line pixels
    
    hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)
    l_channel = hls[:,:,1]
    s_channel = hls[:,:,2]
    gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
    # Sobel x
    sobelx = cv2.Sobel(l_channel, cv2.CV_64F, 1, 0) # Take the derivative in x
    abs_sobelx = np.absolute(sobelx) # Absolute x derivative to accentuate lines away from horizontal
    scaled_sobel = np.uint8(255*abs_sobelx/np.max(abs_sobelx))
    
    # Threshold x gradient
    sxbinary = np.zeros_like(scaled_sobel)
    sxbinary[(scaled_sobel >= sx_thresh[0]) & (scaled_sobel <= sx_thresh[1])] = 1
    
    # Threshold color channel
    s_binary = np.zeros_like(s_channel)
    s_binary[(s_channel >= s_thresh[0]) & (s_channel <= s_thresh[1])] = 1
    # combine both binary images
    combined_binary = np.zeros_like(sxbinary)
    combined_binary[(s_binary == 1) | (sxbinary == 1)] = 1
    return combined_binary

def find_lane_pixels(binary_warped, nwindows = 9, margin = 75, minpix = 100):
    ## uses the sliding window technique to find lane line pixels from a binary image
    
    # Take a histogram of the bottom half of the image
    histogram = np.sum(binary_warped[binary_warped.shape[0]//2:,:], axis=0)
    # Create an output image to draw on and visualize the result
    out_img = np.dstack((binary_warped, binary_warped, binary_warped))
    # Find the peak of the left and right halves of the histogram
    # These will be the starting point for the left and right lines
    midpoint = np.int(histogram.shape[0]//2)
    leftx_base = np.argmax(histogram[:midpoint])
    rightx_base = np.argmax(histogram[midpoint:]) + midpoint

    # Set height of windows - based on nwindows above and image shape
    window_height = np.int(binary_warped.shape[0]//nwindows)
    # Identify the x and y positions of all nonzero pixels in the image
    nonzero = binary_warped.nonzero()
    nonzeroy = np.array(nonzero[0])
    nonzerox = np.array(nonzero[1])
    # Current positions to be updated later for each window in nwindows
    leftx_current = leftx_base
    rightx_current = rightx_base

    # Create empty lists to receive left and right lane pixel indices
    left_lane_inds = []
    right_lane_inds = []

    # Step through the windows one by one
    for window in range(nwindows):
        # Identify window boundaries in x and y (and right and left)
        win_y_low = binary_warped.shape[0] - (window+1)*window_height
        win_y_high = binary_warped.shape[0] - window*window_height
        win_xleft_low = leftx_current - margin  # Update this
        win_xleft_high = leftx_current + margin  # Update this
        win_xright_low = rightx_current - margin  # Update this
        win_xright_high = rightx_current + margin  # Update this
        
        # Draw the windows on the visualization image
        cv2.rectangle(out_img,(win_xleft_low,win_y_low),
        (win_xleft_high,win_y_high),(0,255,0), 2) 
        cv2.rectangle(out_img,(win_xright_low,win_y_low),
        (win_xright_high,win_y_high),(0,255,0), 2) 
        
        # Identify the nonzero pixels in x and y within the window
        good_left_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & 
        (nonzerox >= win_xleft_low) &  (nonzerox < win_xleft_high)).nonzero()[0]
        good_right_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & (nonzerox >= win_xright_low) & (nonzerox < win_xright_high)).nonzero()[0]
        
        # Append these indices to the lists
        left_lane_inds.append(good_left_inds)
        right_lane_inds.append(good_right_inds)
        
        # If you found > minpix pixels, recenter next window #
        # (`right` or `leftx_current`) on their mean position #
        if len(good_left_inds) > minpix:
            meanposleft = np.int(np.mean(nonzerox[good_left_inds]))
            leftx_current = meanposleft
        if len(good_right_inds) > minpix:
            meanposright = np.int(np.mean(nonzerox[good_right_inds]))
            rightx_current = meanposright

    # Concatenate the arrays of indices
    try:
        left_lane_inds = np.concatenate(left_lane_inds)
        right_lane_inds = np.concatenate(right_lane_inds)
    except ValueError:
        # Avoids an error if the above is not implemented fully
        pass

    # Extract left and right line pixel positions
    leftx = nonzerox[left_lane_inds]
    lefty = nonzeroy[left_lane_inds] 
    rightx = nonzerox[right_lane_inds]
    righty = nonzeroy[right_lane_inds]

    return leftx, lefty, rightx, righty, out_img


def fit_polynomial(undist, binary_warped, Minv):
    ## Fit a polynomial to pixels detected by the sliding window approach
    # Find our lane pixels first
    leftx, lefty, rightx, righty, out_img = find_lane_pixels(binary_warped)

    # Fit a second order polynomial to each
    left_fit = np.polyfit(lefty,leftx,2)
    right_fit = np.polyfit(righty,rightx,2)

    # Generate x and y values for plotting
    ploty = np.linspace(0, binary_warped.shape[0]-1, binary_warped.shape[0] )
    try:
        left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
        right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
    except TypeError:
        # Avoids an error if `left` and `right_fit` are still none or incorrect
        print('The function failed to fit a line!')
        left_fitx = 1*ploty**2 + 1*ploty
        right_fitx = 1*ploty**2 + 1*ploty
    
    # Populate information about each line
    leftlane.populate_line(left_fit, left_fitx, ploty, leftx, lefty)
    rightlane.populate_line(right_fit, right_fitx, ploty, rightx, righty)
    
    # Draw the lane on the original image
    result = draw_lane(undist, binary_warped, Minv, leftlane.bestx, rightlane.bestx, ploty) # return to original vehicle view and draw lane
    
    return result

def fit_poly(img_shape, leftx, lefty, rightx, righty):
    ## Fit a polynomial to pixels detected by the search-from-prior approach
    #Fit a second order polynomial to each line
    left_fit = np.polyfit(lefty, leftx,2)
    right_fit = np.polyfit(righty, rightx,2)
    # Generate x and y values for plotting
    ploty = np.linspace(0, img_shape[0]-1, img_shape[0])
    # Calc the x values resulting from polynomials
    left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2] 
    right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
    
    return left_fitx, right_fitx, ploty

def measure_curvature_real(fitx, ploty):
    ## Find radius of curvature for a lane line
    ym_per_pix = 30/720 # meters per pixel in y dimension
    xm_per_pix = 3.7/700 # meters per pixel in x dimension
    
    # calculate polyynomial coefficients from fiting in real space
    fit_cr = np.polyfit(ploty*ym_per_pix, fitx*xm_per_pix, 2)
    
    # Define y-value where we want radius of curvature
    y_eval = np.max(ploty)
    
    # Calculation of radius of curvature
    curverad = ((1 + (2*fit_cr[0]*y_eval*ym_per_pix + fit_cr[1])**2)**1.5) / np.absolute(2*fit_cr[0])
    
    return curverad

def search_around_poly(undist, binary_warped, Minv, margin = 100):
    ## Search for line pixels around the previous lines' polynomials
    
    # Grab activated pixels
    nonzero = binary_warped.nonzero()
    nonzeroy = np.array(nonzero[0])
    nonzerox = np.array(nonzero[1])
    
    # choose previous polynomial, based on whether the line was detected previously
    if leftlane.detected:
        left_fit = leftlane.current_fit
    else: left_fit = leftlane.best_fit # if not detected, choose the average of the past n iterations
    if rightlane.detected:
        right_fit = rightlane.current_fit
    else: right_fit = rightlane.best_fit
    
    #Set the area of search based on activated x-values within the +/- margin of our polynomial function
    left_lane_inds = ((nonzerox > (left_fit[0]*(nonzeroy**2) + left_fit[1]*nonzeroy + 
                    left_fit[2] - margin)) & (nonzerox < (left_fit[0]*(nonzeroy**2) + 
                    left_fit[1]*nonzeroy + left_fit[2] + margin)))
    right_lane_inds = ((nonzerox > (right_fit[0]*(nonzeroy**2) + right_fit[1]*nonzeroy + 
                    right_fit[2] - margin)) & (nonzerox < (right_fit[0]*(nonzeroy**2) + 
                    right_fit[1]*nonzeroy + right_fit[2] + margin)))
    
    # Again, extract left and right line pixel positions
    leftx = nonzerox[left_lane_inds]
    lefty = nonzeroy[left_lane_inds] 
    rightx = nonzerox[right_lane_inds]
    righty = nonzeroy[right_lane_inds]

    # Fit new polynomials
    left_fitx, right_fitx, ploty = fit_poly(binary_warped.shape, leftx, lefty, rightx, righty)
    left_fit = np.polyfit(lefty, leftx,2)
    right_fit = np.polyfit(righty, rightx,2)
    
    # populate updated line information
    leftlane.populate_line(left_fit, left_fitx, ploty, leftx, lefty)
    rightlane.populate_line(right_fit, right_fitx, ploty, rightx, righty)
    
    # draw detected lane on original image
    result = draw_lane(undist, binary_warped, Minv, leftlane.bestx, rightlane.bestx, ploty) # return to original vehicle view and draw lane
    
    return result

def draw_lane(undist, warped, Minv, left_fitx, right_fitx, ploty):
    ## Draw the detected lane onto the original image
    # Create an image to draw the lines on
    warp_zero = np.zeros_like(warped).astype(np.uint8)
    color_warp = np.dstack((warp_zero, warp_zero, warp_zero))

    # Recast the x and y points into usable format for cv2.fillPoly()
    pts_left = np.array([np.transpose(np.vstack([left_fitx, ploty]))])
    pts_right = np.array([np.flipud(np.transpose(np.vstack([right_fitx, ploty])))])
    pts = np.hstack((pts_left, pts_right))

    # Draw the lane onto the warped blank image
    cv2.fillPoly(color_warp, np.int_([pts]), (0,255, 0))

    # Warp the blank back to original image space using inverse perspective matrix (Minv)
    newwarp = cv2.warpPerspective(color_warp, Minv, (undist.shape[1], undist.shape[0])) 
    # Combine the result with the original image
    result = cv2.addWeighted(undist, 1, newwarp, 0.3, 0)
    
    # Add information text to image
    vehicle_pos, direction = find_vehicle_pos(warped.shape[1]) #calculate vehicle position
    string1 = 'Vehicle is {}m {} center'.format(round(abs(vehicle_pos),2),direction)
    string2 = 'Radius of curvature: {}m'.format(round((leftlane.rad+rightlane.rad)/2))
    font = cv2.FONT_HERSHEY_SIMPLEX
    text_result = cv2.putText(np.copy(result),string1,(0,50),font,2,(255,255,255),3,cv2.LINE_AA)
    text_result = cv2.putText(np.copy(text_result),string2,(0,105),font,2,(255,255,255),3,cv2.LINE_AA)
    
    return text_result<jupyter_output><empty_output><jupyter_text>The Pipeline<jupyter_code>def lane_find_pipeline(img):
    ## To process each image in the video
    # undistort, thresholds
    mtx, dist = load_calibration_values() #load camera matrix and distortion coefficients, assuming camera calibration is complete
    undist = np.copy(img)
    undist = cv2.undistort(undist, mtx, dist, None, mtx) #undistort image
    threshold_binary = thresholds(undist) #apply saturation and x gradient thresholds to produce binary image indicating possible lane pixels

    src = np.float32([[200,720],[580, 460], [705, 460], [1110,720]]) #choose 4 corners of lane's region of interest
    dst = np.float32([[310,720],[310,0],[950,0],[950,720]]) #choose 4 corners of lane's region of interest in overhead view
    M = cv2.getPerspectiveTransform(src, dst) #find the perspective transform matrix to transform from vehicle view into overhead view to better identify lane line polynomials
    Minv = cv2.getPerspectiveTransform(dst,src) #find the inverse transform to revert from overhead to vehicle view
    warped = cv2.warpPerspective(threshold_binary, M, (img.shape[1],img.shape[0])) #produce the resulting overhead view of the image
    
    # for first image only
    if ((leftlane.recent_xfitted.size == 0) and (rightlane.recent_xfitted.size == 0)):
        final_image = fit_polynomial(undist, warped, Minv) #using the sliding window approach, find the appropriate lane pixels and the corresponding polynomial to fit the lane lines
    # for subsequent images
    else:
        final_image = search_around_poly(undist, warped, Minv) # use the search-from-prior approach
        
        
    return final_image<jupyter_output><empty_output><jupyter_text>Implement lane finding on project video<jupyter_code>#define Line objects for both the left and right line to store their information
leftlane = Line() 
rightlane = Line()
#import video libraries
from moviepy.editor import VideoFileClip
from IPython.display import HTML
project_output = 'output_videos/project_video_output.mp4'
# project_output = 'output_videos/project_video_output_wobbly.mp4'

# apply pipeline to video
clip1 = VideoFileClip("project_video.mp4")
project_clip = clip1.fl_image(lane_find_pipeline)
%time project_clip.write_videofile(project_output, audio=False)

# play video output
HTML("""
<video width="960" height="540" controls>
  <source src="{0}">
</video>
""".format(project_output))<jupyter_output>[MoviePy] >>>> Building video output_videos/project_video_output.mp4
[MoviePy] Writing video output_videos/project_video_output.mp4<jupyter_text>Challenge Video<jupyter_code>#define Line objects for both the left and right line to store their information
leftlane = Line() 
rightlane = Line()
challenge_output = 'output_videos/challenge_video_output.mp4'

# apply pipeline to video
clip1 = VideoFileClip("challenge_video.mp4")
project_clip = clip1.fl_image(lane_find_pipeline)
%time project_clip.write_videofile(challenge_output, audio=False)

# play video output
HTML("""
<video width="960" height="540" controls>
  <source src="{0}">
</video>
""".format(challenge_output))<jupyter_output><empty_output><jupyter_text>Harder Challenge Video<jupyter_code>#define Line objects for both the left and right line to store their information
leftlane = Line() 
rightlane = Line()
challenge_output = 'output_videos/harder_challenge_video_output.mp4'

# apply pipeline to video
clip1 = VideoFileClip("harder_challenge_video.mp4")
project_clip = clip1.fl_image(lane_find_pipeline)
%time project_clip.write_videofile(challenge_output, audio=False)

# play video output
HTML("""
<video width="960" height="540" controls>
  <source src="{0}">
</video>
""".format(challenge_output))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.0944646504907578, 'fraction_numerical': 0.024961387075880623, 'mean_word_length': 3.3892411983380715, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 5, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '236256', 'license': 'MIT', 'n_tokens_mistral': 6854, 'n_tokens_neox': 6461, 'n_words': 1791, 'path': 'Video.ipynb', 'repo_name': 'sayeayed/Udacity-Project2', 'size': '20071'}