Having trouble with Camera calibration for stereovision in open CV

[Refrigerator]

I'm revisiting my old openCV project, where i need to get some 3d-ish imaging from two stereo cameras but i'm having trouble with calibration.
The code i'm using is just example code from the internet (Nicolai Nielsen i believe), because what i need is the just end result - a stereo camera setup that produces a depth map.

Code: [Select]


# import libs
import numpy as np
import cv2 as cv
import glob
import sys

################ FIND CHESSBOARD CORNERS - OBJECT POINTS AND IMAGE POINTS #############################
# define chessboard
chessboardSize = (7,5)

# define image frame size
frameSize = (640,480)


# termination criteria
criteria = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 30, 0.001)


# prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
objp = np.zeros((chessboardSize[0] * chessboardSize[1], 3), np.float32)
objp[:,:2] = np.mgrid[0:chessboardSize[0],0:chessboardSize[1]].T.reshape(-1,2)

size_of_chessboard_squares_mm = 31
objp = objp * size_of_chessboard_squares_mm

# Arrays to store object points and image points from all the images.
objpoints = [] # 3d point in real world space
imgpointsL = [] # 2d points in image plane.
imgpointsR = [] # 2d points in image plane.

num = 0
corners_detected = 0

imagesLeft = sorted(glob.glob('images/stereoLeft/*.png'))
imagesRight = sorted(glob.glob('images/stereoRight/*.png'))

kernel = np.array([[0, -1, 0],
                   [-1, 5,-1],
                   [0, -1, 0]])


for imgLeft, imgRight in zip(imagesLeft, imagesRight):
    num += 1
    imgL = cv.imread(imgLeft)
    imgR = cv.imread(imgRight)
    grayL = cv.cvtColor(imgL, cv.COLOR_BGR2GRAY)
    grayR = cv.cvtColor(imgR, cv.COLOR_BGR2GRAY)

    #grayL = cv.filter2D(grayL, ddepth=-1, kernel=kernel)
    #grayR = cv.filter2D(grayR, ddepth=-1, kernel=kernel)

    # Find the chess board corners
    retL, cornersL = cv.findChessboardCorners(grayL, chessboardSize, None)
    retR, cornersR = cv.findChessboardCorners(grayR, chessboardSize, None)

    # If found, add object points, image points (after refining them)
    if retL and retR == True:
        corners_detected += 1
        objpoints.append(objp)

        cornersL = cv.cornerSubPix(grayL, cornersL, (11,11), (-1,-1), criteria)
        imgpointsL.append(cornersL)

        cornersR = cv.cornerSubPix(grayR, cornersR, (11,11), (-1,-1), criteria)
        imgpointsR.append(cornersR)

        # Draw and display the corners
        cv.drawChessboardCorners(imgL, chessboardSize, cornersL, retL)
        cv.imshow('img left', imgL)
        cv.drawChessboardCorners(imgR, chessboardSize, cornersR, retR)
        cv.imshow('img right', imgR)
        cv.waitKey(10)
    else:
        print("chessboard corners not found in image " + str(num))


cv.destroyAllWindows()

# terminate program is no corners are detected
if corners_detected == 0:
    sys.exit("No corners detected")


############## CALIBRATION #######################################################

retL, cameraMatrixL, distL, rvecsL, tvecsL = cv.calibrateCamera(objpoints, imgpointsL, frameSize, None, None)
heightL, widthL, channelsL = imgL.shape
newCameraMatrixL, roi_L = cv.getOptimalNewCameraMatrix(cameraMatrixL, distL, (widthL, heightL), 1, (widthL, heightL))

retR, cameraMatrixR, distR, rvecsR, tvecsR = cv.calibrateCamera(objpoints, imgpointsR, frameSize, None, None)
heightR, widthR, channelsR = imgR.shape
newCameraMatrixR, roi_R = cv.getOptimalNewCameraMatrix(cameraMatrixR, distR, (widthR, heightR), 1, (widthR, heightR))

# assign ports to cams
camR = cv.VideoCapture(0, cv.CAP_DSHOW)
camL = cv.VideoCapture(1, cv.CAP_DSHOW)
num = 0
avg_img = 0

# morphology settings
kernel = np.ones((10,10),np.uint8)

# begin loop
while camL.isOpened():
    #frame_counter += 1
   
    # capture image frame
    resultL, imgL = camL.read()
    resultR, imgR = camR.read()

    Right_grey = cv.cvtColor(imgR, cv.COLOR_BGR2GRAY)
    Left_grey = cv.cvtColor(imgL, cv.COLOR_BGR2GRAY)

    # undistort image
    # undistort
    Right_grey = cv.undistort(Right_grey, cameraMatrixR, distR, None, newCameraMatrixR)
    Left_grey = cv.undistort(Left_grey, cameraMatrixL, distL, None, newCameraMatrixL)

    # resize images
    xR, yR, wR, hR = roi_R
    xL, yL, wL, hL = roi_L

    # Right_grey = Right_grey[yR:yR + hR, xR:xR + wR]
    # Left_grey = Left_grey[yL:yL + hL, xL:xL + wL]
    num_disp = 16*8
    size_block = 15
    test_no = 3

    #stereo = cv.StereoSGBM_create(numDisparities=num_disp, blockSize=size_block)
    stereo = cv.StereoBM_create(numDisparities=num_disp, blockSize=size_block)

    disparity = stereo.compute(Left_grey,Right_grey)
    #plt.imshow(disparity,'gray')
    #plt.show()
    # apply morphological transformation
    #disparity = cv.morphologyEx(disparity, cv.MORPH_OPEN, kernel)

    norm_image = cv.normalize(disparity, None, alpha = 0, beta = 1, norm_type=cv.NORM_MINMAX, dtype=cv.CV_32F)
   
    #norm_image = cv.medianBlur(norm_image,3)
 
    # wait for key press
    k = cv.waitKey(5)
    file_ident = "_nD" + str(num_disp) + "_sB" + str(size_block) + "_test" + str(test_no) + "_Run"
    if k == 27:
        break
    elif k == ord('s'): # wait for 's' key to save and exit
        cv.imwrite('disparity test without cal/test recal2/Uncorrected_' + file_ident + str(num) + '.png', imgR)
        cv.imwrite('disparity test without cal/test recal2/corrected_' + file_ident + str(num) + '.png', cv.undistort(imgR, cameraMatrixR, distR, None, newCameraMatrixR))
        cv.imwrite('disparity test without cal/test recal2/disparity_' + file_ident + str(num) + '.png', disparity)
        print("images saved")
        num += 1

    avg_img += norm_image
    avg_img = avg_img / 2
    cv.imshow('filtered image disparity', avg_img)
    cv.imshow('normalized image disparity', norm_image)
    cv.imshow('Test Window Left',imgL)
    cv.imshow('Test Window Right',imgR)
    cv.imshow('undistorted right', cv.undistort(imgR, cameraMatrixR, distR, None, newCameraMatrixR))
    # show regular disparity
    #cv.imshow('regular disparity', imgR - imgL)

    cv.imshow('regular disparity mono', Right_grey - Left_grey)

# Release and destroy all windows before termination
camL.release()
camR.release()

cv.destroyAllWindows()

Unfortunately i'm having trouble calibrating the distortion and alignment of my two cameras.
My problem is that the end result is never perfectly aligned or calibrated and i can't find the reason why, because i followed tutorials to a T and sometimes my calibrated image comes out worse than before. 
I'm trying to troubleshoot my system because i have the classic deadline-coming-up crunch session and not enough time.
The cameras i'm using are 1080p native web cameras, that stream 480p images, because that's what my code is set to (to ease the processing a bit). These cameras seem to have pretty good sensors.

I've included some images to show what i'm working with. First is the calibration image with checked board, which seems to detect just fine. Second is the absolute best i could after multiple days of non stop recalibrating and in the end having to tweak the cameras by hand to align the images. 

The images may look blurry, because they're just screenshots but in reality they're quite high fidelity and all checkerboard corners identify just fine, which is why i don't understand how my final image becomes so distorted.

Perhaps there's not enough light? Maybe there's a problem with reading 480p images from a 1080p webcam? Are the lenses just too wide-angle for this code to work? My cameras were about 20cm apart, mounted on a 20x40mm aluminum extrusion with 3D printed holders, for solid mounting.
I mean there could be some simple detail i'm missing out on, like the autofocus in my cameras messing something up. 

Any help would be greatly appreciated.