<jupyter_start><jupyter_text>Advanced Lane Finding ProjectThe goals / steps of this project are the following:* Compute the camera calibration matrix and distortion coefficients given a set of chessboard images.* Apply a distortion correction to raw images.* Use color transforms, gradients, etc., to create a thresholded binary image.* Apply a perspective transform to rectify binary image ("birds-eye view").* Detect lane pixels and fit to find the lane boundary.* Determine the curvature of the lane and vehicle position with respect to center.* Warp the detected lane boundaries back onto the original image.* Output visual display of the lane boundaries and numerical estimation of lane curvature and vehicle position.--- First, I'll compute the camera calibration using chessboard images<jupyter_code>import numpy as np
import cv2
import glob
import pickle
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
# Prepare object points like (0,0,0), (1,0,0), (2,0,0), ...,(6,5,0)
objpts = np.zeros((6*9, 3), np.float32)
# Get the meshgrid points one by one and replace them in the x and y 
# coordinates of the object points
objpts[:,:2] = np.mgrid[0:9, 0:6].T.reshape(-1,2)

objpoints = []
imgpoints = []

def get_cam_params():
    '''
    Gets undistortion parameters for the camera, based on a set of calibration images
    
    Takes all the images in the `camera_cal` folder and finds the undistortion coefficients based
    on the images.
    The calibration matrix and dist parameters are the only things we will use later on.
    
    :return: Camera parameters, namely `mtx` and `dist`
    
    '''
    images = glob.glob('camera_cal/*.jpg')

    for idx, frame_name in enumerate(images):
        image = cv2.imread(frame_name)
        gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

        ret, corners = cv2.findChessboardCorners(gray, (9,6), None)

        if ret == True:
            objpoints.append(objpts)
            imgpoints.append(corners)

            # Draw and display the corners
            cv2.drawChessboardCorners(image, (9,6), corners, ret)

            cv2.imshow('image', image)
            cv2.waitKey(1)
            
    test_img = cv2.imread('camera_cal/calibration4.jpg')
    imshape = test_img.shape

    ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, (imshape[1], imshape[0]), None, None)

    result = cv2.undistort(test_img, mtx, dist, None, mtx)
    cv2.imwrite('camera_cal/test_undist.jpg', result)
    
    return mtx, dist
    
    
def store_cam_params(mtx, dist):
    '''
    Stores the two camera parameters as a binary file for later use
    
    :param mtx: Transformation matrix to store in the pickle file
    :param dist: Parameter containing distances, to store in the pickle file
    
    '''
    pkl = {}
    pkl["mtx"] = mtx
    pkl["dist"] = dist
    pickle.dump( pkl, open( "camera_cal/camera_param_pickle.p", "wb" ) )
    
def load_pickle():
    '''
    Loads the camera parameters from the pickle file previously stored.
    
    :return: Dictionary containg the data in pickle
    
    '''
    infile = open("camera_cal/camera_param_pickle.p",'rb')
    new_dict = pickle.load(infile)
    infile.close()
    
    return new_dict

def lab_select(image, thresh=(0, 255), channel='l'):
    '''
    Masks the given channel of the CIE Lab image using the thresholds given.
    
    :param image: Image to be converted in CIE Lab and then thresholded
    :param thresh: Tuple containing upper and lower threshold values
    :param channel: Character indicating the channel to work on
    :return: The thresholded binary image
    
    '''
    # Convert the image to the CIE Lab colorspace
    cie_lab = cv2.cvtColor(image, cv2.COLOR_RGB2LAB)
#     cie_lab = cv2.cvtColor(image, cv2.COLOR_BGR2LAB)

    # Select the appropriate channel for processing
    if channel == 'l':
        ch = 0
        channel = "CIE L"
    elif channel == 'a':
        ch = 1
        channel = "CIE A"
    elif channel == 'b':
        ch = 2
        channel = "CIE B"

    layer = cie_lab[:, :, ch]
    
    # The L channel has an automatic thresholding mode, regardles of the input thresholds
    if channel == 'CIE L':
        
        a = 39.20193
        b= 5.121302
        c = 42.48095
        d = 88.16211

        avg = np.average(layer[420:,:])
        
        thresh0 = d + (a - d) / (1 + (avg / c) ** b)
        thresh0 = thresh0
        thresh1 = 100

    else:
        thresh0 = thresh[0]
        thresh1 = thresh[1]
        
    # Return a binary image of threshold result
    mask = np.zeros_like(layer)
    
    mask[(layer >= thresh0) & (layer <= thresh1)] = 1
    
    
    return mask

def combined_filters(image):
    '''
    Combine different filters in order to get the lanes from the image.
    
    :param image: Image to process
    :return: Binary mask of the image with everything but the lane lines removed
    
    '''
    # Parameters for the bilateral filter
    diam = 9
    sig_col = 13
    sig_sp = 31

    
    blur_filter = cv2.bilateralFilter(image, d=diam, sigmaColor=sig_col, sigmaSpace=sig_sp)

    # Take two masked images, applying different thresholds on each of them
    # The cie_l image automatically calculates it's thresholds
    cie_l = lab_select(blur_filter)
    cie_b = lab_select(blur_filter, (18, 100), "b")
    
    # Combine the two images in a single one
    combined = np.zeros_like(cie_l)
    combined[(cie_l == 1) | (cie_b == 1)] = 1
    
#     return np.dstack((combined, combined, combined))
    return combined
def filtered_image(image):
    '''
    Convert the image to another format, apply the filtering function and convert it back to the original format
    
    :param image: Image to process
    :return: Processed image of type uint8
    
    '''
    # Convert image from float32 to uint8
    image = image.astype(np.float32)
    image = image/255

        
    # Send the image for processing to combined_image()
    comb = combined_filters(image)

    # Convert the image back to uint8
#     weighted = cv2.addWeighted(image, 0.5, comb, 1, 0)
    
    weighted = comb
    uint_img = cv2.convertScaleAbs(weighted * 255)
    
    return uint_img

def undistort(image, mtx, dist):
    ''' Undistorts given image based on the parameters `mtx` adn `dist`'''
    
    result = cv2.undistort(image, mtx, dist, None, mtx)
    return result
def warp_perspective(image, mode="direct"):
    '''
    Transform the image in bird's eye view or from bird's eye view back to normal
    
    :param image: Image to process
    :mode: String indicating if the transformation is a direct or inverse one
    :return: Perspective transformed image
    
    '''
    height = image.shape[0]
    width  = image.shape[1]
    
    # Set the horizontal boundaries of the roi to be warped
    y1 = 450
    y2 = 690

    coef = 300
    up = 33

    pts1 = np.float32([[587 - up, y1], [692 + up,  y1], [1068 + coef,      y2], [230 - coef,   y2]])
    pts2 = np.float32([[0,    0], [width, 0], [width, height], [0, height]])

    if mode == "direct":
        M = cv2.getPerspectiveTransform(pts1, pts2)
    else:
        M = cv2.getPerspectiveTransform(pts2, pts1)

    # Transform the image
    dst = cv2.warpPerspective(image, M, (width, height))

    return dst

def find_lane_pixels(binary_warped):

    # Apply this operations on a warped binary image
    histogram = np.sum(binary_warped[binary_warped.shape[0]//2:, :], axis=0)
    # Create an output image, to visualize the result
    out_img = np.dstack((binary_warped, binary_warped, binary_warped))
    # Find the peak of the left and right halves of the histogram
    midpoint = np.int(histogram.shape[0]//2)
    leftx_base = np.argmax(histogram[:midpoint])
    rightx_base = np.argmax(histogram[midpoint:]) + midpoint

    # print(leftx_base, " ", rightx_base)

    # HYPERPARAMETERS
    # Number of sliding windows
    nwindows = 9
    # Set width of windows +/- margin
    margin = 100
    # Minimum number of pixels to recenter image
    minpix = 50

    # Set height of windows - based on the parameters above
    window_height = np.int(binary_warped.shape[0]//nwindows)
    # Identify the x and y positions of all nonzero pixels in the image
    # NOTE: np.nonzero() returns the indices of the nonzero points in the image
    nonzero = binary_warped.nonzero()
    
    # Split the array of nonzero points in two arrays that contain only the x and y 
    # coordinates respectively
    nonzeroy = np.array(nonzero[0])
    nonzerox = np.array(nonzero[1])

    # Set the current position to be the base of the first window. The position will be 
    # updated for each window in `nwindows`
    leftx_current = leftx_base
    rightx_current = rightx_base

    # Create empty lists to receive left and right lane pixel indices
    left_lane_inds = []
    right_lane_inds = []

    # Step through the windows one by one
    for window in range(nwindows):
        # Identify window boundaries in x and y (and right and left)
        win_y_low = binary_warped.shape[0] - (window + 1) * window_height
        win_y_high = binary_warped.shape[0] - window * window_height

        win_xleft_low   = leftx_current - margin
        win_xleft_high  = leftx_current + margin
        win_xright_low  = rightx_current - margin
        win_xright_high = rightx_current + margin

        cv2.rectangle(out_img, (win_xleft_low, win_y_low),
                      (win_xleft_high, win_y_high), (0,255,0), 2)
        cv2.rectangle(out_img, (win_xright_low, win_y_low),
                      (win_xright_high, win_y_high), (0,255,0), 2)

        # TODO: Identify the nonzero pixels in x and y within the window
        good_left_inds = np.array(np.nonzero((win_xleft_low <= nonzerox) & (nonzerox < win_xleft_high) &
                                             (win_y_low <= nonzeroy) & (nonzeroy < win_y_high)))
        # good_left_inds = nonzerox[(win_xleft_low < nonzerox) & (nonzerox < win_xleft_high) &
        #                           (win_y_low < nonzeroy) & (nonzeroy < win_y_high)]
        good_right_inds = np.array(np.nonzero((win_xright_low <= nonzerox) & (nonzerox < win_xright_high) &
                                              (win_y_low <= nonzeroy) & (nonzeroy < win_y_high)))

        # Append these indices to the lists
        left_lane_inds.append(good_left_inds[0])
        right_lane_inds.append(good_right_inds[0])

        # TODO: If you found > minpix pixels, recenter window
        # (`right` or `leftx_current`) on their mean position

        if (np.size(good_right_inds) > minpix):
            rightx_current = np.int(np.average(nonzerox[good_right_inds], axis=1))

        if (np.size(good_left_inds) > minpix):
            leftx_current = np.int(np.average(nonzerox[good_left_inds], axis=1))
            # print(leftx_current)

    # Concatenate the arrays of indices (previously was a list of lists of pixels)
    left_lane_inds = np.concatenate(left_lane_inds)
    right_lane_inds = np.concatenate(right_lane_inds)

    # Extract left and right line pixel positions
    leftx = nonzerox[left_lane_inds]
    lefty = nonzeroy[left_lane_inds]
    rightx = nonzerox[right_lane_inds]
    righty = nonzeroy[right_lane_inds]

    return leftx, lefty, rightx, righty, out_img

def fit_polynomial(binary_warped):
    
    ym_per_pix = 30 / 720  # meters per pixel in y dimension
    xm_per_pix = 3.7 / 700  # meters per pixel in x dimension
    
    # Find our lane pixels first
    leftx, lefty, rightx, righty, out_img = find_lane_pixels(binary_warped)

    ### TODO: Fit a second order polynomial to each using `np.polyfit`
    left_fit = np.polyfit(lefty, leftx, 2)
    right_fit = np.polyfit(righty, rightx, 2)
    
#     left_fit = np.polyfit(lefty * ym_per_pix, leftx * xm_per_pix, 2)
#     right_fit = np.polyfit(righty * ym_per_pix, rightx * xm_per_pix, 2)
    
#     right_fit_cr = np.polyfit(ploty * ym_per_pix, rightx * xm_per_pix , 2)
    

    # Generate x and y values for plotting
    # np.linspace() is similar to range(first, last, values), with the difference that the last parameters specifies the
    # number of values to be generated, not the step
    ploty = np.linspace(0, binary_warped.shape[0] - 1, binary_warped.shape[0])

    # Generate the x coordinates for each of the y coordinates of the fitted function
    try:
        left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
        right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]

    except:
        print("The function failed to fit a line!")

    ### Visualization
    # Colors in the left an right lane regions
    out_img[lefty, leftx] = [255, 0, 0]
    out_img[righty, rightx] = [0, 0, 255]

    # Plots the left and right polynomials on the lane lines
    plt.plot(left_fitx, ploty, color='yellow')
    plt.plot(right_fitx, ploty, color='yellow')

    plt.imshow(out_img)

    return out_img, left_fit, right_fit
def measure_curvature_real(left_fit_cr, right_fit_cr):
    '''
    Calculates the curvature of polynomial functions in meters.
    '''
    # Define conversions in x and y from pixels space to meters
    ym_per_pix = 30 / 720  # meters per pixel in y dimension
    xm_per_pix = 3.7 / 700  # meters per pixel in x dimension

    # Define y-value where we want radius of curvature
    # We'll choose the maximum y-value, corresponding to the bottom of the image
    y_eval = 719
    
    left_fit_cr[0] = left_fit_cr[0] * (xm_per_pix/(ym_per_pix ** 2))
    left_fit_cr[1] = left_fit_cr[1] * (xm_per_pix/ym_per_pix)
    
    right_fit_cr[0] = right_fit_cr[0] * (xm_per_pix/(ym_per_pix ** 2))
    right_fit_cr[1] = right_fit_cr[1] * (xm_per_pix/ym_per_pix)

    ##### TO-DO: Implement the calculation of R_curve (radius of curvature) #####
    left_curverad =  ((1 + (2 * left_fit_cr[0] * y_eval * ym_per_pix + left_fit_cr[1]) ** 2) ** (3/2)) / np.abs(2 * left_fit_cr[0])
    right_curverad = ((1 + (2 * right_fit_cr[0] * y_eval * ym_per_pix + right_fit_cr[1]) ** 2) ** (3/2)) / np.abs(2 * right_fit_cr[0])

    return left_curverad, right_curverad

image = cv2.imread('test_images/7.jpg')
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
''' Warping works'''

params_dict = load_pickle()

image = cv2.imread('test_images/2.jpg')
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

undist = undistort(image, params_dict["mtx"], params_dict["dist"])
warped = warp_perspective(undist)

# res = np.vstack((image, undist))

# res = cv2.resize(res, (res.shape[1]//2,res.shape[0]//2))
cv2.imshow('res', warped)
cv2.waitKey(0)
# cv2.imwrite('output_images/camera_cal_before.png', image)
# cv2.imwrite('output_images/camera_cal_after.png', undist)


cv2.destroyAllWindows()
image = cv2.imread('test_images/7.jpg')
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)


''' Camera undistortion works '''

''' IMportant code lines below'''
# mtx, dist = get_cam_params()
# store_cam_params(mtx, dist)
params_dict = load_pickle()

undist = undistort(image, params_dict["mtx"], params_dict["dist"])
binary = filtered_image(undist)
warped = warp_perspective(binary)
fit, left_fit, right_fit    = fit_polynomial(warped)

radius = measure_curvature_real(left_fit, right_fit)

print(radius)

cv2.imshow('res', fit)
# cv2.waitKey(0)
cv2.destroyAllWindows()
unwarped = warp_perspective(fit, "inverse")

cv2.imshow('res', unwarped)
cv2.waitKey(0)
cv2.destroyAllWindows()
''' Facui lucrurile cerute mai jos '''
"""
Ramane pe maine de implementat clasa Lane, cu functionalitatea ei.
Am vrut sa pun conversia din pixeli in metri mai sus, in polyfit, dar n-am putut, din cauza ca
np.polyfit e utilizata si pentru desenarea benzii pe imagine, iar daca o convertesc in metri, e bai, 
liniile de fit ies din imagine.
Am pus conversia doar la calcularea razei, dar banuiesc ca rezultatele nu-s bune. :)


"""
''' Coming up....

    - bird's eye view
    - indentify lanes (customize the aalgorithm a little)
    - gather lane points (start with box, continue from prior)
                          check for boxes touching edge
    - fit polynomial on lanes
    - calculate the radius
    
    
    - implement a Lane class 
        (with history, averaging over n previous values, outlier rejection - based on 
        polynomial parameters variation or radius variation)
        + reset method for deeply flawed measurements
        
    - apply all of it on the video


'''
# Import everything needed to edit/save/watch video clips
from moviepy.editor import VideoFileClip
from IPython.display import HTML
white_output = 'test_videos_output/project_video.mp4'
## To speed up the testing process you may want to try your pipeline on a shorter subclip of the video
## To do so add .subclip(start_second,end_second) to the end of the line below
## Where start_second and end_second are integer values representing the start and end of the subclip
## You may also uncomment the following line for a subclip of the first 5 seconds
##clip1 = VideoFileClip("test_videos/solidWhiteRight.mp4").subclip(0,5)
# clip1 = VideoFileClip("project_video.mp4").subclip(20,25)
clip1 = VideoFileClip("challenge_video.mp4")
white_clip = clip1.fl_image(process_images) #NOTE: this function expects color images!!
%time white_clip.write_videofile(white_output, audio=False)
HTML("""
<video width="960" height="540" controls>
  <source src="{0}">
</video>
""".format(white_output))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09778672032193159, 'fraction_numerical': 0.020465651049152055, 'mean_word_length': 3.1086443079829946, 'pattern_counts': {'":': 1, '<': 21, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '385481', 'license': 'MIT', 'n_tokens_mistral': 5949, 'n_tokens_neox': 5681, 'n_words': 1747, 'path': '.ipynb_checkpoints/Advanced-Lane-Lines-checkpoint.ipynb', 'repo_name': 'ZeicBeniamin/CarND-Advanced-Lane-Lines', 'size': '17395'}