Difference between revisions of "Example Stereo Disparity 3D"

From BoofCV
Jump to navigationJump to search
m
m
(4 intermediate revisions by the same user not shown)
Line 1: Line 1:
<center>
<center>
<gallery widths=320px heights=240px>
{| class="wikitable" width="400pt"
file:Example_stereo_disparity3d_pointcloud.jpg | 3D point cloud generated from disparity image.
| [[file:Example_stereo_disparity3d_pointcloud.jpg|400px]] || {{#ev:youtube|https://www.youtube.com/watch?v=8pn9Ebw90uk|400|center|||start=672}}
</gallery>
|-
| 3D point cloud generated from disparity image. || Stereo Update Video
|}
</center>
</center>


Line 8: Line 10:


Example Code:
Example Code:
* [https://github.com/lessthanoptimal/BoofCV/blob/v0.25/examples/src/boofcv/examples/stereo/ExampleStereoDisparity3D.java ExampleStereoDisparity3D.java]
* [https://github.com/lessthanoptimal/BoofCV/blob/v0.35/examples/src/main/java/boofcv/examples/stereo/ExampleStereoDisparity3D.java ExampleStereoDisparity3D.java]


Concepts:
Concepts:
Line 14: Line 16:
* Point clouds
* Point clouds


Relevant Applets:
Related Videos
* [[Applet_Stereo_Disparity| Stereo Disparity]]
* [https://www.youtube.com/watch?v=8pn9Ebw90uk&t=672s Stereo Update 2020]


Related Examples:
Related Examples:
Line 33: Line 35:
public class ExampleStereoDisparity3D {
public class ExampleStereoDisparity3D {


// Specifies what size input images are scaled to
// Specifies what disparity values are considered
public static final double scale = 0.5;
public static final int minDisparity = 10;
 
public static final int rangeDisparity = 60;
// Specifies what range of disparity is considered
public static final int minDisparity = 0;
public static final int maxDisparity = 40;
public static final int rangeDisparity = maxDisparity-minDisparity;


public static void main( String args[] ) {
public static void main( String args[] ) {
Line 56: Line 54:
GrayU8 distLeft = ConvertBufferedImage.convertFrom(origLeft, (GrayU8) null);
GrayU8 distLeft = ConvertBufferedImage.convertFrom(origLeft, (GrayU8) null);
GrayU8 distRight = ConvertBufferedImage.convertFrom(origRight,(GrayU8)null);
GrayU8 distRight = ConvertBufferedImage.convertFrom(origRight,(GrayU8)null);
// re-scale input images
GrayU8 scaledLeft = new GrayU8((int)(distLeft.width*scale),(int)(distLeft.height*scale));
GrayU8 scaledRight = new GrayU8((int)(distRight.width*scale),(int)(distRight.height*scale));
new FDistort(distLeft,scaledLeft).scaleExt().apply();
new FDistort(distRight,scaledRight).scaleExt().apply();
// Don't forget to adjust camera parameters for the change in scale!
PerspectiveOps.scaleIntrinsic(param.left, scale);
PerspectiveOps.scaleIntrinsic(param.right,scale);


// rectify images and compute disparity
// rectify images and compute disparity
GrayU8 rectLeft = new GrayU8(scaledLeft.width,scaledLeft.height);
GrayU8 rectLeft = distLeft.createSameShape();
GrayU8 rectRight = new GrayU8(scaledRight.width,scaledRight.height);
GrayU8 rectRight = distRight.createSameShape();


RectifyCalibrated rectAlg = ExampleStereoDisparity.rectify(scaledLeft,scaledRight,param,rectLeft,rectRight);
RectifyCalibrated rectAlg = ExampleStereoDisparity.rectify(distLeft,distRight,param,rectLeft,rectRight);


// GrayU8 disparity = ExampleStereoDisparity.denseDisparity(rectLeft, rectRight, 3,minDisparity, maxDisparity);
// GrayU8 disparity = ExampleStereoDisparity.denseDisparity(rectLeft, rectRight, 3,minDisparity, rangeDisparity);
GrayF32 disparity = ExampleStereoDisparity.denseDisparitySubpixel(rectLeft, rectRight, 3, minDisparity, maxDisparity);
GrayF32 disparity = ExampleStereoDisparity.denseDisparitySubpixel(
rectLeft, rectRight, 5, minDisparity, rangeDisparity);


// ------------- Convert disparity image into a 3D point cloud
// ------------- Convert disparity image into a 3D point cloud


// The point cloud will be in the left cameras reference frame
// The point cloud will be in the left cameras reference frame
DenseMatrix64F rectK = rectAlg.getCalibrationMatrix();
DMatrixRMaj rectK = rectAlg.getCalibrationMatrix();
DenseMatrix64F rectR = rectAlg.getRectifiedRotation();
DMatrixRMaj rectR = rectAlg.getRectifiedRotation();
 
// used to display the point cloud
PointCloudViewer viewer = new PointCloudViewer(rectK, 10);
viewer.setPreferredSize(new Dimension(rectLeft.width,rectLeft.height));


// extract intrinsic parameters from rectified camera
// extract intrinsic parameters from rectified camera
double baseline = param.getBaseline();
double baseline = param.getBaseline()*0.1;
double fx = rectK.get(0,0);
double fx = rectK.get(0,0);
double fy = rectK.get(1,1);
double fy = rectK.get(1,1);
double cx = rectK.get(0,2);
double cx = rectK.get(0,2);
double cy = rectK.get(1,2);
double cy = rectK.get(1,2);
double maxZ = baseline*100;


// Iterate through each pixel in disparity image and compute its 3D coordinate
// Iterate through each pixel in disparity image and compute its 3D coordinate
PointCloudViewer pcv = VisualizeData.createPointCloudViewer();
pcv.setTranslationStep(1.5);
Point3D_F64 pointRect = new Point3D_F64();
Point3D_F64 pointRect = new Point3D_F64();
Point3D_F64 pointLeft = new Point3D_F64();
Point3D_F64 pointLeft = new Point3D_F64();
Line 102: Line 91:


// skip over pixels were no correspondence was found
// skip over pixels were no correspondence was found
if( d >= rangeDisparity )
if( d >= rangeDisparity || d <= 0 )
continue;
continue;


Line 109: Line 98:
pointRect.x = pointRect.z*(x - cx)/fx;
pointRect.x = pointRect.z*(x - cx)/fx;
pointRect.y = pointRect.z*(y - cy)/fy;
pointRect.y = pointRect.z*(y - cy)/fy;
// prune points which are likely to be noise
if( pointRect.z >= maxZ )
continue;


// rotate into the original left camera frame
// rotate into the original left camera frame
Line 115: Line 108:
// add pixel to the view for display purposes and sets its gray scale value
// add pixel to the view for display purposes and sets its gray scale value
int v = rectLeft.unsafe_get(x, y);
int v = rectLeft.unsafe_get(x, y);
viewer.addPoint(pointLeft.x, pointLeft.y, pointLeft.z, v << 16 | v << 8 | v);
pcv.addPoint(pointLeft.x,pointLeft.y,pointLeft.z,v << 16 | v << 8 | v);
// temp.add( pointLeft.copy() );
}
}
}
}
// move it back a bit to make the 3D structure more apparent
Se3_F64 cameraToWorld = new Se3_F64();
cameraToWorld.T.z = -baseline*5;
cameraToWorld.T.x = baseline*12;
ConvertRotation3D_F64.eulerToMatrix(EulerType.XYZ,0.1,-0.4,0,cameraToWorld.R);
// Configure the display
// pcv.setFog(true);
// pcv.setClipDistance(baseline*45);
// PeriodicColorizer colorizer = new TwoAxisRgbPlane.Z_XY(4.0);
// colorizer.setPeriod(baseline*5);
// pcv.setColorizer(colorizer); // sometimes pseudo color can be easier to view
pcv.setDotSize(1);
pcv.setCameraHFov(PerspectiveOps.computeHFov(param.left));
pcv.setCameraToWorld(cameraToWorld);
JComponent viewer = pcv.getComponent();
viewer.setPreferredSize(new Dimension(600,600*param.left.height/param.left.width));


// display the results.  Click and drag to change point cloud camera
// display the results.  Click and drag to change point cloud camera
BufferedImage visualized = VisualizeImageData.disparity(disparity, null,minDisparity, maxDisparity,0);
BufferedImage visualized = VisualizeImageData.disparity(disparity, null,rangeDisparity,0);
ShowImages.showWindow(visualized,"Disparity");
ShowImages.showWindow(visualized,"Disparity", true);
ShowImages.showWindow(viewer,"Point Cloud");
ShowImages.showWindow(viewer,"Point Cloud", true);
}
}
}
}
</syntaxhighlight>
</syntaxhighlight>

Revision as of 21:38, 21 June 2020

Example stereo disparity3d pointcloud.jpg
3D point cloud generated from disparity image. Stereo Update Video

An additional example for stereo disparity. This one shows you how to properly resize input images and convert the disparity image into a 3D point cloud.

Example Code:

Concepts:

  • Stereo disparity
  • Point clouds

Related Videos

Related Examples:

Example Code

/**
 * Expanding upon ExampleStereoDisparity, this example demonstrates how to rescale an image for stereo processing and
 * then compute its 3D point cloud.  Images are often rescaled to improve speed and some times quality.  Creating
 * 3D point clouds from disparity images is easy and well documented in the literature, but there are some nuances
 * to it.
 *
 * @author Peter Abeles
 */
public class ExampleStereoDisparity3D {

	// Specifies what disparity values are considered
	public static final int minDisparity = 10;
	public static final int rangeDisparity = 60;

	public static void main( String args[] ) {
		// ------------- Compute Stereo Correspondence

		// Load camera images and stereo camera parameters
		String calibDir = UtilIO.pathExample("calibration/stereo/Bumblebee2_Chess/");
		String imageDir = UtilIO.pathExample("stereo/");

		StereoParameters param = CalibrationIO.load(new File(calibDir , "stereo.yaml"));

		// load and convert images into a BoofCV format
		BufferedImage origLeft = UtilImageIO.loadImage(imageDir , "chair01_left.jpg");
		BufferedImage origRight = UtilImageIO.loadImage(imageDir , "chair01_right.jpg");

		GrayU8 distLeft = ConvertBufferedImage.convertFrom(origLeft, (GrayU8) null);
		GrayU8 distRight = ConvertBufferedImage.convertFrom(origRight,(GrayU8)null);

		// rectify images and compute disparity
		GrayU8 rectLeft = distLeft.createSameShape();
		GrayU8 rectRight = distRight.createSameShape();

		RectifyCalibrated rectAlg = ExampleStereoDisparity.rectify(distLeft,distRight,param,rectLeft,rectRight);

//		GrayU8 disparity = ExampleStereoDisparity.denseDisparity(rectLeft, rectRight, 3,minDisparity, rangeDisparity);
		GrayF32 disparity = ExampleStereoDisparity.denseDisparitySubpixel(
				rectLeft, rectRight, 5, minDisparity, rangeDisparity);

		// ------------- Convert disparity image into a 3D point cloud

		// The point cloud will be in the left cameras reference frame
		DMatrixRMaj rectK = rectAlg.getCalibrationMatrix();
		DMatrixRMaj rectR = rectAlg.getRectifiedRotation();

		// extract intrinsic parameters from rectified camera
		double baseline = param.getBaseline()*0.1;
		double fx = rectK.get(0,0);
		double fy = rectK.get(1,1);
		double cx = rectK.get(0,2);
		double cy = rectK.get(1,2);

		double maxZ = baseline*100;

		// Iterate through each pixel in disparity image and compute its 3D coordinate
		PointCloudViewer pcv = VisualizeData.createPointCloudViewer();
		pcv.setTranslationStep(1.5);

		Point3D_F64 pointRect = new Point3D_F64();
		Point3D_F64 pointLeft = new Point3D_F64();
		for( int y = 0; y < disparity.height; y++ ) {
			for( int x = 0; x < disparity.width; x++ ) {
				double d = disparity.unsafe_get(x,y) + minDisparity;

				// skip over pixels were no correspondence was found
				if( d >= rangeDisparity || d <= 0 )
					continue;

				// Coordinate in rectified camera frame
				pointRect.z = baseline*fx/d;
				pointRect.x = pointRect.z*(x - cx)/fx;
				pointRect.y = pointRect.z*(y - cy)/fy;

				// prune points which are likely to be noise
				if( pointRect.z >= maxZ )
					continue;

				// rotate into the original left camera frame
				GeometryMath_F64.multTran(rectR, pointRect, pointLeft);

				// add pixel to the view for display purposes and sets its gray scale value
				int v = rectLeft.unsafe_get(x, y);
				pcv.addPoint(pointLeft.x,pointLeft.y,pointLeft.z,v << 16 | v << 8 | v);
//				temp.add( pointLeft.copy() );
			}
		}

		// move it back a bit to make the 3D structure more apparent
		Se3_F64 cameraToWorld = new Se3_F64();
		cameraToWorld.T.z = -baseline*5;
		cameraToWorld.T.x = baseline*12;
		ConvertRotation3D_F64.eulerToMatrix(EulerType.XYZ,0.1,-0.4,0,cameraToWorld.R);

		// Configure the display
//		pcv.setFog(true);
//		pcv.setClipDistance(baseline*45);
//		PeriodicColorizer colorizer = new TwoAxisRgbPlane.Z_XY(4.0);
//		colorizer.setPeriod(baseline*5);
//		pcv.setColorizer(colorizer); // sometimes pseudo color can be easier to view
		pcv.setDotSize(1);
		pcv.setCameraHFov(PerspectiveOps.computeHFov(param.left));
		pcv.setCameraToWorld(cameraToWorld);
		JComponent viewer = pcv.getComponent();
		viewer.setPreferredSize(new Dimension(600,600*param.left.height/param.left.width));

		// display the results.  Click and drag to change point cloud camera
		BufferedImage visualized = VisualizeImageData.disparity(disparity, null,rangeDisparity,0);
		ShowImages.showWindow(visualized,"Disparity", true);
		ShowImages.showWindow(viewer,"Point Cloud", true);
	}
}