<jupyter_start><jupyter_text>Advanced Lane Finding ProjectThe goals / steps of this project are the following:* Compute the camera calibration matrix and distortion coefficients given a set of chessboard images.* Apply a distortion correction to raw images.* Use color transforms, gradients, etc., to create a thresholded binary image.* Apply a perspective transform to rectify binary image ("birds-eye view").* Detect lane pixels and fit to find the lane boundary.* Determine the curvature of the lane and vehicle position with respect to center.* Warp the detected lane boundaries back onto the original image.* Output visual display of the lane boundaries and numerical estimation of lane curvature and vehicle position.<jupyter_code>%matplotlib qt

from helper import *

#Calibrating camera
ret, mtx, dist, rvecs, tvecs = calibration()
# Tuning Warp Images for src and dst using straght_line image
straight = cv2.imread('test_images/straight_lines2.jpg')
xsize, ysize = (straight.shape[1], straight.shape[0])
# Pts for polylines use int32 type
src = np.int32([[xsize//2 - 50, 450 ],
                  [xsize//2 + 53, 450],
                  [xsize//2 + 465,ysize-10],
                  [xsize//2 - 430,ysize-10]])
cv2.polylines(straight, [src], True, (0,0,255), 3)
cv2.imshow("Straight",straight)
# Assume length of road cover is 30 meters and lanes are 3.7 meters apart
src = src.astype(np.float32)
dst = np.float32([[300, 0],
                     [1000, 0],
                     [1000, 720],
                     [300, 720]])
def filter_image(original_img):
    #Undistort Image
    undist = cv2.undistort(original_img, mtx, dist, None, mtx)
    #Create Threshold
    hls_img = hls_select(undist, thresh=(100,255))
    mag_img = mag_thresh(undist, mag_thresh=(50, 255))
    dir_img = dir_threshold(undist, thresh=(np.pi/4,2*np.pi/4))
    sobelx_img = abs_sobel_thresh(undist, orient='x', thresh_min=50, thresh_max=100)
    sobely_img = abs_sobel_thresh(undist, orient='y', thresh_min=50, thresh_max=100)

    #Combine Threshold
    combined = np.zeros_like(dir_img)
    combined[((sobelx_img == 255) & (sobely_img == 255)) | ((mag_img == 255) & (dir_img == 255)) | (hls_img == 255)] = 255
    return combined

def find_lane_pixels(binary_warped):
    # Take a histogram of the bottom half of the image
    histogram = np.sum(binary_warped[binary_warped.shape[0]//2:,:], axis=0)
    # Create an output image to draw on and visualize the result
    out_img = np.dstack((binary_warped, binary_warped, binary_warped))
    # Find the peak of the left and right halves of the histogram
    # These will be the starting point for the left and right lines
    midpoint = np.int(histogram.shape[0]//2)
    leftx_base = np.argmax(histogram[:midpoint])
    rightx_base = np.argmax(histogram[midpoint:]) + midpoint

    # HYPERPARAMETERS
    # Choose the number of sliding windows
    nwindows = 9
    # Set the width of the windows +/- margin
    margin = 100
    # Set minimum number of pixels found to recenter window
    minpix = 50

    # Set height of windows - based on nwindows above and image shape
    window_height = np.int(binary_warped.shape[0]//nwindows)
    # Identify the x and y positions of all nonzero pixels in the image
    nonzero = binary_warped.nonzero()
    nonzeroy = np.array(nonzero[0])
    nonzerox = np.array(nonzero[1])
    # Current positions to be updated later for each window in nwindows
    leftx_current = leftx_base
    rightx_current = rightx_base
    left_line.line_base_pos = leftx_base
    right_line.line_base_pos = rightx_base

    # Create empty lists to receive left and right lane pixel indices
    left_lane_inds = []
    right_lane_inds = []

    # Step through the windows one by one
    for window in range(nwindows):
        # Identify window boundaries in x and y (and right and left)
        win_y_low = binary_warped.shape[0] - (window+1)*window_height
        win_y_high = binary_warped.shape[0] - window*window_height
        ### TO-DO: Find the four below boundaries of the window ###
        win_xleft_low = leftx_current - margin
        win_xleft_high = leftx_current + margin
        win_xright_low = rightx_current - margin
        win_xright_high = rightx_current + margin
        
        # Draw the windows on the visualization image
        cv2.rectangle(out_img,(win_xleft_low,win_y_low),
        (win_xleft_high,win_y_high),(0,255,0), 2) 
        cv2.rectangle(out_img,(win_xright_low,win_y_low),
        (win_xright_high,win_y_high),(0,255,0), 2) 
        
        ### Identify the nonzero pixels in x and y within the window ###
        good_left_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & 
        (nonzerox >= win_xleft_low) &  (nonzerox < win_xleft_high)).nonzero()[0]
        good_right_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & 
        (nonzerox >= win_xright_low) &  (nonzerox < win_xright_high)).nonzero()[0]
        
        # Append these indices to the lists
        left_lane_inds.append(good_left_inds)
        right_lane_inds.append(good_right_inds)
        
        ### If you found > minpix pixels, recenter next window ###
        ### (`right` or `leftx_current`) on their mean position ###
        if len(good_left_inds) > minpix:
            leftx_current = np.int(np.mean(nonzerox[good_left_inds]))
        if len(good_right_inds) > minpix:        
            rightx_current = np.int(np.mean(nonzerox[good_right_inds])) # Remove this when you add your function

    # Concatenate the arrays of indices (previously was a list of lists of pixels)
    try:
        left_lane_inds = np.concatenate(left_lane_inds)
        right_lane_inds = np.concatenate(right_lane_inds)
    except ValueError:
        # Avoids an error if the above is not implemented fully
        pass

    # Extract left and right line pixel positions
    leftx = nonzerox[left_lane_inds]
    lefty = nonzeroy[left_lane_inds] 
    rightx = nonzerox[right_lane_inds]
    righty = nonzeroy[right_lane_inds]
    left_line.allx , left_line.ally = leftx, lefty
    right_line.allx, right_line.ally = rightx, righty
    

    return leftx, lefty, rightx, righty, out_img

def fit_polynomial(binary_warped):
    # Find our lane pixels first
    leftx, lefty, rightx, righty, out_img = find_lane_pixels(binary_warped)

    ### Fit a second order polynomial to each using `np.polyfit` ###
    # For Pixel
    left_fit= np.polyfit(lefty, leftx, 2)
    right_fit= np.polyfit(righty, rightx, 2)
    left_line.pix_current_fit = left_fit
    right_line.pix_current_fit = right_fit
    # For Actual Road, scale it with ym and xm per pixel
    leftnewfit = np.polyfit(lefty*ym_per_pix, leftx*xm_per_pix, 2)
    rightnewfit = np.polyfit(righty*ym_per_pix, rightx*xm_per_pix, 2)
    left_line.diffs = left_line.current_fit - leftnewfit
    right_line.diffs = right_line.current_fit - rightnewfit
    left_line.current_fit = leftnewfit
    right_line.current_fit = rightnewfit
    # Generate x and y values for plotting
    Line.ploty = np.linspace(0, binary_warped.shape[0]-1, binary_warped.shape[0] )
    try:
#         left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
#         right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
        left_line.pix_fitx = left_fit[0]*Line.ploty**2 + left_fit[1]*Line.ploty + left_fit[2]
        right_line.pix_fitx = right_fit[0]*Line.ploty**2 + right_fit[1]*Line.ploty + right_fit[2]
        left_line.detected = True
        right_line.detected = True
    except TypeError:
        # Avoids an error if `left` and `right_fit` are still none or incorrect
        print('The function failed to fit a line!')
        left_line.pix_fitx = 1*Line.ploty**2 + 1*Line.ploty
        right_line.pix_fitx = 1*Line.ploty**2 + 1*Line.ploty
        left_line.detected = False
        right_line.detected = False

    ## Visualization ##
    # Colors in the left and right lane regions
    out_img[lefty, leftx] = [255, 0, 0]
    out_img[righty, rightx] = [0, 0, 255]

    # Plots the left and right polynomials on the lane lines for pixel plotting
#     plt.imshow(out_img)
#     plt.plot(left_line.pix_fitx, Line.ploty, color='yellow')
#     plt.plot(right_line.pix_fitx, Line.ploty, color='yellow')
#     plt.xlim(0, 1280)
#     plt.ylim(720, 0)

    return out_img

def fit_poly(img_shape, leftx, lefty, rightx, righty):
    ### Fit a second order polynomial to each with np.polyfit() ###
    # For Pixel
    left_fit = np.polyfit(lefty, leftx, 2)
    right_fit = np.polyfit(righty, rightx, 2)
    # For Actual Road
    leftnewfit = np.polyfit(lefty*ym_per_pix, leftx*xm_per_pix, 2)
    rightnewfit = np.polyfit(righty*ym_per_pix, rightx*xm_per_pix, 2)
    left_line.diffs = left_line.current_fit - leftnewfit
    right_line.diffs = right_line.current_fit - rightnewfit
    left_line.current_fit = leftnewfit
    right_line.current_fit = rightnewfit
    # Generate x and y values for plotting
    ploty = np.linspace(0, img_shape[0]-1, img_shape[0])
    ### Calc both polynomials using ploty, left_fit and right_fit ###
    left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
    right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
    
    return left_fitx, right_fitx, ploty

def search_around_poly(binary_warped):
    # HYPERPARAMETER
    # Choose the width of the margin around the previous polynomial to search
    # The quiz grader expects 100 here, but feel free to tune on your own!
    margin = 30

    # Grab activated pixels
    nonzero = binary_warped.nonzero()
    nonzeroy = np.array(nonzero[0])
    nonzerox = np.array(nonzero[1])
    
    left_fit = left_line.pix_current_fit
    right_fit = right_line.pix_current_fit
    
    ### Set the area of search based on activated x-values ###
    ### within the +/- margin of our polynomial function ###
    ### Hint: consider the window areas for the similarly named variables ###
    ### in the previous quiz, but change the windows to our new search area ###
    left_lane_inds = ((nonzerox > (left_fit[0]*(nonzeroy**2) + left_fit[1]*nonzeroy + 
                    left_fit[2] - margin)) & (nonzerox < (left_fit[0]*(nonzeroy**2) + 
                    left_fit[1]*nonzeroy + left_fit[2] + margin)))
    right_lane_inds = ((nonzerox > (right_fit[0]*(nonzeroy**2) + right_fit[1]*nonzeroy + 
                    right_fit[2] - margin)) & (nonzerox < (right_fit[0]*(nonzeroy**2) + 
                    right_fit[1]*nonzeroy + right_fit[2] + margin)))
    # Again, extract left and right line pixel positions
    leftx = nonzerox[left_lane_inds]
    lefty = nonzeroy[left_lane_inds] 
    rightx = nonzerox[right_lane_inds]
    righty = nonzeroy[right_lane_inds]
    left_line.allx, left_line.ally = leftx, lefty
    right_line.allx, right_line.ally = rightx, righty

    # Fit new polynomials
    left_line.pix_fitx, right_line.pix_fitx, Line.ploty = fit_poly(binary_warped.shape, leftx, lefty, rightx, righty)
    left_line.line_base_pose = left_line.pix_fitx[719]
    right_line.line_base_pos = right_line.pix_fitx[719]
    
    ## Visualization ##
    # Create an image to draw on and an image to show the selection window
    out_img = np.dstack((binary_warped, binary_warped, binary_warped))*255
    window_img = np.zeros_like(out_img)
    # Color in left and right line pixels
    out_img[nonzeroy[left_lane_inds], nonzerox[left_lane_inds]] = [255, 0, 0]
    out_img[nonzeroy[right_lane_inds], nonzerox[right_lane_inds]] = [0, 0, 255]

    # Generate a polygon to illustrate the search window area
    # And recast the x and y points into usable format for cv2.fillPoly()
    left_line_window1 = np.array([np.transpose(np.vstack([left_line.pix_fitx-margin, Line.ploty]))])
    left_line_window2 = np.array([np.flipud(np.transpose(np.vstack([left_line.pix_fitx+margin, 
                              Line.ploty])))])
    left_line_pts = np.hstack((left_line_window1, left_line_window2))
    right_line_window1 = np.array([np.transpose(np.vstack([right_line.pix_fitx-margin, Line.ploty]))])
    right_line_window2 = np.array([np.flipud(np.transpose(np.vstack([right_line.pix_fitx+margin, 
                              Line.ploty])))])
    right_line_pts = np.hstack((right_line_window1, right_line_window2))

    # Draw the lane onto the warped blank image
    cv2.fillPoly(window_img, np.int_([left_line_pts]), (0,255, 0))
    cv2.fillPoly(window_img, np.int_([right_line_pts]), (0,255, 0))
    result = cv2.addWeighted(out_img, 1, window_img, 0.3, 0)
    
    # Plot the polynomial lines onto the image
#     plt.imshow(out_img)
#     plt.plot(left_line.pix_fitx, Line.ploty, color='yellow')
#     plt.plot(right_line.pix_fitx, Line.ploty, color='yellow')
#     plt.xlim(0, 1280)
#     plt.ylim(720, 0)
    ## End visualization steps ##
    
    return result

# Create an image to draw the lines on
def project_lane(original, warped):
    warp_zero = np.zeros_like(warped).astype(np.uint8)
    color_warp = np.dstack((warp_zero, warp_zero, warp_zero))

    # Recast the x and y points into usable format for cv2.fillPoly()
    pts_left = np.array([np.transpose(np.vstack([left_line.pix_fitx, Line.ploty]))])
    pts_right = np.array([np.flipud(np.transpose(np.vstack([right_line.pix_fitx, Line.ploty])))])
    pts = np.hstack((pts_left, pts_right))

    # Draw the lane onto the warped blank image
    cv2.fillPoly(color_warp, np.int_([pts]), (0,255, 0))

    # Warp the blank back to original image space using inverse perspective matrix (Minv)
    # newwarp = cv2.warpPerspective(color_warp, Minv, (image.shape[1], image.shape[0])) 
    newwarp = warper(color_warp, dst, src)
    # Combine the result with the original image
    result = cv2.addWeighted(original, 1, newwarp, 0.3, 0)
#     cv2.imshow("project lane", result)
    return result

def measure_curvature(left_fit, right_fit):
    '''
    Calculates the curvature of polynomial functions in pixels.
    '''
    # Define y-value where we want radius of curvature
    # We'll choose the maximum y-value, corresponding to the bottom of the image
    y_eval = 720
    
    # Implement the calculation of R_curve (radius of curvature) #####
    left_radius = ((1 + (2*left_fit[0]*y_eval*ym_per_pix + left_fit[1])**2)**1.5) / np.absolute(2*left_fit[0])
    right_radius = ((1 + (2*right_fit[0]*y_eval*ym_per_pix + right_fit[1])**2)**1.5) / np.absolute(2*right_fit[0])
    
    return left_radius, right_radius
#Pipeline for Lane Detection
def process_image(img):
    combined = filter_image(img)
    warped = warper(combined, src, dst)
    if left_line.detected is False or right_line.detected is False:
        outimg = fit_polynomial(warped)
        #left_pix_current_fit, right_pix_current_fit_y = fit_polynomial(warped)
    else:
        outimg = search_around_poly(warped)
        #left_pix_current_fit, right_pix_current_fit_y = search_around_poly(warped, left_current_fit, right_current_fit)
    
    left_line.radius_of_curvature, right_line.radius_of_curvature = measure_curvature(left_line.current_fit, right_line.current_fit)
    if np.any(abs(right_line.diffs) > 0.05) or np.any(abs(left_line.diffs) > 0.05):
        left_line.detected = False
        right_line.detected = False
    #project_lane()
    projected = project_lane(img, warped)
    
    cv2.putText(projected, "Radius of Curvature: " + 
           str(np.round((left_line.radius_of_curvature + right_line.radius_of_curvature)/2, 3)) + " m",
           (20,50), cv2.FONT_HERSHEY_SIMPLEX, 1.5, (255,0,0), 2, cv2.LINE_AA)
    centerdiff = (left_line.line_base_pos + right_line.line_base_pos)/2 - img.shape[1]/2
    closerside = ""
    if centerdiff > 0:
        closerside = "left"
    else:
        closerside = "right"
    cv2.putText(projected, "Vehicle is " + 
            str(abs(np.round(((left_line.line_base_pos + right_line.line_base_pos)/2 - warped.shape[1]/2) * xm_per_pix, 3))) + "m " +
            closerside + " of center",
           (20,120), cv2.FONT_HERSHEY_SIMPLEX, 1.5, (255,0,0), 2, cv2.LINE_AA)
    return projected
#Testing Single Image
file = 'test_images/test5.jpg'
original = cv2.imread(file)
ym_per_pix = 30/720 # meters per pixel in y dimension
xm_per_pix = 3.7/700 # meters per pixel in x dimension
left_line = Line()
right_line = Line()

out = process_image(original)
# Import everything needed to edit/save/watch video clips
from moviepy.editor import VideoFileClip
from IPython.display import HTML
ym_per_pix = 30/720 # meters per pixel in y dimension
xm_per_pix = 3.7/700 # meters per pixel in x dimension
left_line = Line()
right_line = Line()
white_output = 'output_images/harder_challenge_video_output.mp4'
## To speed up the testing process you may want to try your pipeline on a shorter subclip of the video
## To do so add .subclip(start_second,end_second) to the end of the line below
## Where start_second and end_second are integer values representing the start and end of the subclip
## You may also uncomment the following line for a subclip of the first 5 seconds
##clip1 = VideoFileClip("test_videos/solidWhiteRight.mp4").subclip(0,5)
clip1 = VideoFileClip("harder_challenge_video.mp4")
white_clip = clip1.fl_image(process_image) #NOTE: this function expects color images!!
%time white_clip.write_videofile(white_output, audio=False)<jupyter_output>t:   0%|          | 0/1199 [00:00<?, ?it/s, now=None]
{'fraction_non_alphanumeric': 0.10626010626010626, 'fraction_numerical': 0.02390852390852391, 'mean_word_length': 3.5261369576581285, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '486637', 'license': 'MIT', 'n_tokens_mistral': 6251, 'n_tokens_neox': 5975, 'n_words': 1478, 'path': 'Advanced_Lane_Detection.ipynb', 'repo_name': 'NeoDJoKer/Advanced-Lanelines-Detection', 'size': '17316'}