<jupyter_start><jupyter_text>Video Pipeline<jupyter_code># Declare required libraries
import numpy as np
import cv2
import glob
import pickle
from moviepy.editor import VideoFileClip
from IPython.display import HTML
# Function to open pickled calibration data

def get_calibration():
    # Read in the saved camera matrix and distortion coefficients - read bytes
    ret_calib = pickle.load(open('./camera_calibration.p', 'rb'))
    # assign unpickled data to variables
    mtx = ret_calib['mtx']
    dist = ret_calib['dist']
    # return unpickled data - matix & distortion parameters
    return mtx, dist

def dir_threshold(img, sobel_kernel=3, thresh=(0, np.pi/2)):
    # Calculate gradient direction
    
    #convert image to grayscale ready for sobel
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    
    # Apply cv2.Sobel() - get x gradient
    sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
    # Apply cv2.Sobel() - get x gradient
    sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
    
    #Use np.arctan2(abs_sobely, abs_sobelx) to calculate the direction of the gradient 
    grad_dir = np.arctan2(np.absolute(sobely), np.absolute(sobelx))
    grad_dir = np.absolute(grad_dir)
    # Apply threshold
    # Create a binary mask where direction thresholds are met
    dir_binary = np.zeros_like(grad_dir)
    dir_binary[(grad_dir >= thresh[0]) & (grad_dir <= thresh[1])] = 1
    
    return dir_binary
# Image conversion pipeline

def image_conversion_pipeline(img, s_thresh=(90, 255), sx_thresh=(20, 100), dir_thresh=(0.7, 1.3)):

    img_height, img_width = img.shape[0], img.shape[1]
    
    # Convert to HLS color space and separate the V channel
    hls = cv2.cvtColor(img, cv2.COLOR_BGR2HLS)
    h_channel = hls[:,:,0]
    l_channel = hls[:,:,1]
    s_channel = hls[:,:,2]
    # Sobel x
    sobelx = cv2.Sobel(l_channel, cv2.CV_64F, 1, 0) # Take the derivative in x
    abs_sobelx = np.absolute(sobelx) # Absolute x derivative to accentuate lines away from horizontal
    scaled_sobel = np.uint8(255*abs_sobelx/np.max(abs_sobelx))
    
    # Threshold x gradient
    sxbinary = np.zeros_like(scaled_sobel)
    sxbinary[(scaled_sobel >= sx_thresh[0]) & (scaled_sobel <= sx_thresh[1])] = 1
    
    # apply gradient direction threshold so that only edges closer to vertical are detected.
    dir_binary = dir_threshold(img, 9 , dir_thresh)
    
    grad_combined = ((sxbinary == 1) & (dir_binary == 1))
    
    # Threshold color channel
    s_binary = np.zeros_like(s_channel)
    s_binary[(s_channel >= s_thresh[0]) & (s_channel <= s_thresh[1])] = 1
    
    # Stack each channel
    color_binary = np.dstack(( np.zeros_like(s_binary), sxbinary, s_binary )) * 255
    
    #combined_binary = np.zeros_like(grad_combined)
    combined_binary = np.zeros_like(sxbinary)
    combined_binary[((s_binary == 1) | (grad_combined == 1)) ] = 1
    
    # apply the region of interest mask
    # create blank
    mask = np.zeros_like(s_channel)
    # specify polygon shape for region of interest
    roi = np.float32([(img_width/2-85,420),(img_width/2+85,420),(img_width-20,img_height),(20,img_height)])
    # convert roi to numpy array
    roi = np.array(roi, dtype=np.int32)
    
    # fill the region of interest polygon with  colour 1
    cv2.fillPoly(mask, [roi], 1)
    
    # bitwise mask and image to filter image to show only interesting area inside mask
    mask_binary = cv2.bitwise_and(combined_binary, mask)
    
    return mask_binary
# Function to perform perspective transform - returns a warped image
def perspective_transform(src, dst, corrected_image, inv=False):
    
    # if inverse transform is requested
    #if (inv):
        # get the inverse transform matrix M to trnaform image back before a warp
    M_inv = cv2.getPerspectiveTransform(dst, src)
    #else:
        # get the transform matrix M
    M = cv2.getPerspectiveTransform(src, dst)
    
    img_size = (corrected_image.shape[1], corrected_image.shape[0])
    
    # warp image to a top-down view
    warped_image = cv2.warpPerspective(corrected_image, M, img_size, flags=cv2.INTER_LINEAR)
    
    return warped_image, M, M_inv
def image_correct(img):
    
    # retrieve the camera calibration from a pickled file
    mtx, dist = get_calibration()
    # undistort image with paramters received from calibration
    corrected_image = cv2.undistort(img, mtx, dist, None, mtx)
    # return corrected image
    return corrected_image
def birds_eye_view(img):
    
    img_width, img_height = img.shape[1], img.shape[0]
   
    # set area to distort
    # top left, top-right, bottom-right, bottom-left
    src = np.float32([(img_width/2-65,460),(img_width/2+65,460),(1060,670),(245,670)])
    
    # set destination of warped image
    # top left, top-right, bottom-right, bottom-left
    dst = np.float32([(450,0),(img_width-450,0),(img_width-450,img_height),(450,img_height)])

    # use image_correct function to get a correct image using camera stored calibration
    corrected_image = image_correct(img)
    
    # get a warped image from corrected image
    warped, M, M_inv = perspective_transform(src, dst, corrected_image)
    
    # return a warped image
    return warped, M, M_inv
# function to create histogram which detects white lines in a warped image
# split histogram into two sides
def histogram_split(img):
    
    # Grab only the bottom half of the image
    # Lane lines are likely to be mostly vertical nearest to the car
    # sum across image pixels vertically
    # the highest areas of vertical lines should be larger values
    histogram = np.sum(img[img.shape[0]//2:,:], axis=0)
    
    # find peak in left half of histogram
    midpoint = np.int(histogram.shape[0]/2)
    leftx_base = np.argmax(histogram[:midpoint])
    # find peak in right half of histogram
    rightx_base = np.argmax(histogram[midpoint:]) + midpoint
    
    # return the histogram and the left and right detected positions
    return leftx_base, rightx_base
def sliding_window_search(img):
    
    # detect initial lane positions
    leftx_base_point, rightx_base_point = histogram_split(img)
    
    # set up the sliding window parameters
    # Choose the number of sliding windows
    nwindows = 9
    # Set the width of the windows +/- margin
    margin = 60
    # Set minimum number of pixels found to recenter window
    minpix = 50

    # Set height of windows
    window_height = np.int(img.shape[0]//nwindows)

    # create arrays to hold the positions of all non-zero pixels in the image
    nonzero = img.nonzero()
    # get the y positions of each non-zero pixel in image
    nonzeroy = np.array(nonzero[0])
    # get the x positions of each non-zero pixel in image
    nonzerox = np.array(nonzero[1])
    
    
    # Current X postion pixel of left lane and right lane from histogram
    ## to be updated later for each window in nwindows
    leftx_current = leftx_base_point
    rightx_current = rightx_base_point

    # Create empty lists to receive left and right lane pixel indices
    left_lane_inds = []
    right_lane_inds = []

    # loop through number of windows defined
    for window in range(nwindows):
        # Identify left and right window corners in x and y
        win_y_low = img.shape[0] - (window+1)*window_height
        win_y_high = img.shape[0] - window*window_height
        win_xleft_low = leftx_current - margin
        win_xleft_high = leftx_current + margin
        win_xright_low = rightx_current - margin
        win_xright_high = rightx_current + margin

        # Identify the nonzero pixels in x and y within the window
        good_left_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & 
        (nonzerox >= win_xleft_low) &  (nonzerox < win_xleft_high)).nonzero()[0]
        good_right_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & 
        (nonzerox >= win_xright_low) &  (nonzerox < win_xright_high)).nonzero()[0]

        # Append these indices to the lists
        left_lane_inds.append(good_left_inds)
        right_lane_inds.append(good_right_inds)

        # re-centre current position to best position at top of window based on mean of pixels
        # If you found > minpix pixels, recenter next window on their mean position
        if len(good_left_inds) > minpix:
            leftx_current = np.int(np.mean(nonzerox[good_left_inds]))
        if len(good_right_inds) > minpix:        
            rightx_current = np.int(np.mean(nonzerox[good_right_inds]))

    # Concatenate the arrays of indices (previously was a list of lists of pixels)
    try:
        left_lane_inds_cat = np.concatenate(left_lane_inds)
        right_lane_inds_cat = np.concatenate(right_lane_inds)
    except ValueError:
        # Avoids an error if the above is not implemented fully
        pass

    # Extract left and right line pixel positions
    leftx = nonzerox[left_lane_inds_cat]
    lefty = nonzeroy[left_lane_inds_cat] 
    rightx = nonzerox[right_lane_inds_cat]
    righty = nonzeroy[right_lane_inds_cat]
    
    # Fit a second order polynomial to each using `np.polyfit`
    left_fit = np.polyfit(lefty, leftx, 2)
    right_fit = np.polyfit(righty, rightx, 2)
    
    return left_fit, right_fit
    
    
def continue_previous_fit(img, prev_left_fit, prev_right_fit):
    
    # search around previous polyline for conintuation of line

    # margin width - area to search for polyline
    search_margin = 50

    # Grab activated pixels
    nonzero = img.nonzero()
    nonzeroy = np.array(nonzero[0])
    nonzerox = np.array(nonzero[1])


    left_lane_inds = ((nonzerox > (prev_left_fit[0]*(nonzeroy**2) + prev_left_fit[1]*nonzeroy + 
                    prev_left_fit[2] - search_margin)) & (nonzerox < (prev_left_fit[0]*(nonzeroy**2) + 
                    prev_left_fit[1]*nonzeroy + prev_left_fit[2] + search_margin)))
    right_lane_inds = ((nonzerox > (prev_right_fit[0]*(nonzeroy**2) + prev_right_fit[1]*nonzeroy + 
                    prev_right_fit[2] - search_margin)) & (nonzerox < (prev_right_fit[0]*(nonzeroy**2) + 
                    prev_right_fit[1]*nonzeroy + prev_right_fit[2] + search_margin)))


    # Again, extract left and right line pixel positions
    leftx = nonzerox[left_lane_inds]
    lefty = nonzeroy[left_lane_inds] 
    rightx = nonzerox[right_lane_inds]
    righty = nonzeroy[right_lane_inds]
    
    # Fit a second order polynomial to each using `np.polyfit`
    left_fit = np.polyfit(lefty, leftx, 2)
    right_fit = np.polyfit(righty, rightx, 2)
    
    return left_fit, right_fit
    
def draw_lane_lines(img, binary_img, left_fit, right_fit, M_inv):
    
    img_copy = np.copy(img)
    
    # Create an image to draw the lines on
    # create a blank array same size as binary image
    zero_blank = np.zeros_like(binary_img).astype(np.uint8)
    # Stack blank into 3 channels
    coloured_blank = np.dstack((zero_blank, zero_blank, zero_blank))
    
    # set variables from image size
    image_height, image_width = binary_img.shape
    
    # Return evenly spaced numbers over specified interval - height of image
    ploty = np.linspace(0, binary_img.shape[0]-1, binary_img.shape[0] )
    
    # create line of points from polynominal
    left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
    right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]

    # covert to array for cv.poly functions
    left_line_window = np.array([np.transpose(np.vstack([left_fitx, ploty]))])
    right_line_window = np.array([np.flipud(np.transpose(np.vstack([right_fitx, ploty])))])
    inner_lane = np.hstack((left_line_window, right_line_window))
    
    # Draw the lane onto the warped blank image
    cv2.fillPoly(coloured_blank, np.int_([inner_lane]), (0,255, 0))
    cv2.polylines(coloured_blank, np.int32([left_line_window]), isClosed=False, color=(255,0,0), thickness=20)
    cv2.polylines(coloured_blank, np.int32([right_line_window]), isClosed=False, color=(0,0,255), thickness=20)

    # Warp the blank with annotated lines back to original image space using inverse perspective matrix (Minv)
    undistort = cv2.warpPerspective(coloured_blank, M_inv, (image_width, image_height))
    # Combine the result with the original image
    output_img = cv2.addWeighted(img_copy, 1, undistort, 1.0, 0)
    
    # return a full colour image with the hightlighted image painted onto image
    return output_img
def calulate_curvature(img_size, left_fit, right_fit):
    
    # Return evenly spaced numbers over specified interval - height of image
    ploty = np.linspace(0, img_size[0]-1, img_size[0])
    
    # genreate a poly line of points for each line
    left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
    right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
    
    # get lane pixel point positions at bottom of image
    lane_width = right_fitx[0] - left_fitx[0]
    # Define conversions in x and y from pixels space to meters
    ym_per_pix = 30/720 # meters per pixel in y dimension
    xm_per_pix = 3.7/lane_width # meters per pixel in x dimension, lane width is 12 ft = 3.7 meters
    
    # Calculate offset of car
    camera_centre = (left_fitx[-1] + right_fitx[-1]) / 2
    centre_diff = (camera_centre - img_size[1] / 2) * xm_per_pix
    side_pos = 'left'
    if centre_diff <= 0:
        side_pos = 'right'
    
    # create polyline in metres
    left_fit_cr = np.polyfit(ploty*ym_per_pix, left_fitx*xm_per_pix, 2)
    right_fit_cr = np.polyfit(ploty*ym_per_pix, right_fitx*xm_per_pix, 2)
    
    #y_eval = np.max(ploty)
    y_eval = img_size[0]
    
    # calculate curves radius in metre
    left_curverad_real = ((1 + (2*left_fit_cr[0]*y_eval*ym_per_pix + left_fit_cr[1])**2)**1.5) / np.absolute(2*left_fit_cr[0])
    right_curverad_real = ((1 + (2*right_fit_cr[0]*y_eval*ym_per_pix + right_fit_cr[1])**2)**1.5) / np.absolute(2*right_fit_cr[0])
  
    # return radius of left and right curves
    return left_curverad_real, right_curverad_real, centre_diff, side_pos
    
    
    
def processImage(img):
    
    # create copy of image passed in
    img_copy = np.copy(img)

    # get a binary verion of image in area of interest
    binary = image_conversion_pipeline(img_copy)
    
    # warp the binary into a top down view
    warped, M, M_inv = birds_eye_view(binary)
    
    # set image size from binary image 
    # image_height[0], image_width[1]
    img_size = binary.shape
    
    
    # select whether sliding window or previous fit as method to use for lane detection
    if (left_lane_line.detected is False) and (right_lane_line.detected is False):
        # no lane lines exist; apply sliding window method
        left_lane_line.current_fit, right_lane_line.current_fit = sliding_window_search(warped)
        left_lane_line.detected = True
        right_lane_line.detected = True
        
    else:
        # use previous polyfit as start point for search of next lane lines
        left_lane_line.current_fit, right_lane_line.current_fit = continue_previous_fit(warped, left_lane_line.current_fit, right_lane_line.current_fit)
        

    # draw the lane line area onto image
    output = draw_lane_lines(img_copy, warped, left_lane_line.current_fit, right_lane_line.current_fit, M_inv )
    
    
    left_lane_line.radius_of_curvature, right_lane_line.radius_of_curvature, centre_diff, side_pos = calulate_curvature(img_size, left_lane_line.current_fit, right_lane_line.current_fit)
    
    
    # add text to image to display curvature and vehicle offset information
    cv2.putText(output, 'Left Lane Curvature Radius: {:.2f}m'.format(left_lane_line.radius_of_curvature), 
                (int(img.shape[0]/6), 100), cv2.FONT_HERSHEY_SIMPLEX, 1.5, (255,255,255), 3)
    cv2.putText(output, 'Vehicle is: {0:.2f}m {1} of centre'.format(centre_diff, side_pos), 
                (int(img.shape[0]/6), 150), cv2.FONT_HERSHEY_SIMPLEX, 1.5, (255,255,255), 3)

    return output
    
    
    
# Define a class to receive the characteristics of each line detection
class Line():
    def __init__(self):
        # was the line detected in the last iteration?
        self.detected = False  
        # x values of the last n fits of the line
        self.recent_xfitted = [] 
        #average x values of the fitted line over the last n iterations
        self.bestx = None     
        #polynomial coefficients averaged over the last n iterations
        self.best_fit = None  
        #polynomial coefficients for the most recent fit
        self.current_fit = [np.array([False])]  
        #radius of curvature of the line in some units
        self.radius_of_curvature = None 
        #distance in meters of vehicle center from the line
        self.line_base_pos = None 
        #difference in fit coefficients between last and new fits
        self.diffs = np.array([0,0,0], dtype='float') 
        #x values for detected line pixels
        self.allx = None  
        #y values for detected line pixels
        self.ally = None 
output_Video = "output_project_video.mp4"
input_Video = "project_video.mp4"
# create 2 instances of Line to keep track of left and right lane lines
left_lane_line = Line()
right_lane_line = Line()

# specify the video to use
clip1 = VideoFileClip(input_Video)
# feed a colour processed image into video_clip
video_clip = clip1.fl_image(processImage)
# append video image to video file and print CPU time and wall time
%time video_clip.write_videofile(output_Video, audio=False)
from IPython.display import HTML
HTML("""
<video width="640" height="360" controls>
  <source src="{0}">
</video>
""".format('output_project_video.mp4'))
output2_Video = "output_challenge_video.mp4"
input2_Video = "challenge_video.mp4"
# create 2 instances of Line to keep track of left and right lane lines
left_lane_line = Line()
right_lane_line = Line()

# specify the video to use
clip2 = VideoFileClip(input2_Video)
# feed a colour processed image into video_clip
video_clip2 = clip2.fl_image(processImage)
# append video image to video file and print CPU time and wall time
%time video_clip2.write_videofile(output2_Video, audio=False)
from IPython.display import HTML
HTML("""
<video width="640" height="360" controls>
  <source src="{0}">
</video>
""".format('output_challenge_video.mp4'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09106810976276375, 'fraction_numerical': 0.020115884989614082, 'mean_word_length': 3.3904487640988723, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '41002', 'license': 'MIT', 'n_tokens_mistral': 6307, 'n_tokens_neox': 6044, 'n_words': 1669, 'path': 'VideoPipeline.ipynb', 'repo_name': 'nutmas/CarND-Advanced-Lane-Lines', 'size': '18294'}