Example Multiview Scene Reconstruction - Revision history

Peter at 05:17, 18 May 2020

2020-05-18T05:17:46Z

← Older revision		Revision as of 05:17, 18 May 2020
Line 8:		Line 8:

	Example Code:		Example Code:
	* [https://github.com/lessthanoptimal/BoofCV/blob/v0.34/examples/src/main/java/boofcv/examples/sfm/ExampleMultiviewSceneReconstruction.java ExampleMultiviewSceneReconstruction.java]		* [https://github.com/lessthanoptimal/BoofCV/blob/v0.36/examples/src/main/java/boofcv/examples/sfm/ExampleMultiviewSceneReconstruction.java ExampleMultiviewSceneReconstruction.java]

	Concepts:		Concepts:
Line 38:		Line 38:
	ScoreAssociation scorer = FactoryAssociation.defaultScore(detDesc.getDescriptionType());		ScoreAssociation scorer = FactoryAssociation.defaultScore(detDesc.getDescriptionType());
	AssociateDescription<TupleDesc> associate =		AssociateDescription<TupleDesc> associate =
	FactoryAssociation.greedy(scorer~~, Double.MAX_VALUE, true~~);		FactoryAssociation.greedy(new ConfigAssociateGreedy(true),scorer);
	PairwiseImageMatching<GrayF32> imageMatching = new PairwiseImageMatching<>(detDesc,associate);		PairwiseImageMatching<GrayF32> imageMatching = new PairwiseImageMatching<>(detDesc,associate);
	imageMatching.setVerbose(System.out,0);		imageMatching.setVerbose(System.out,0);

Peter at 18:31, 7 July 2019

2019-07-07T18:31:49Z

← Older revision		Revision as of 18:31, 7 July 2019
Line 8:		Line 8:

	Example Code:		Example Code:
	* [https://github.com/lessthanoptimal/BoofCV/blob/v0.33/examples/src/main/java/boofcv/examples/sfm/ExampleMultiviewSceneReconstruction.java ExampleMultiviewSceneReconstruction.java]		* [https://github.com/lessthanoptimal/BoofCV/blob/v0.34/examples/src/main/java/boofcv/examples/sfm/ExampleMultiviewSceneReconstruction.java ExampleMultiviewSceneReconstruction.java]

	Concepts:		Concepts:
Line 90:		Line 90:
	int pruneCycles=5;		int pruneCycles=5;
	for (int i = 0; i < pruneCycles; i++) {		for (int i = 0; i < pruneCycles; i++) {
	System.out.println("BA + Prune iteration = "+i+" points="+structure.points.~~length~~+" obs="+observations.getObservationCount());		System.out.println("BA + Prune iteration = "+i+" points="+structure.points.size+" obs="+observations.getObservationCount());
	bundleScale.applyScale(structure,observations);		bundleScale.applyScale(structure,observations);
	sba.setParameters(structure,observations);		sba.setParameters(structure,observations);
Line 125:		Line 125:
	Point3D_F64 camera = new Point3D_F64();		Point3D_F64 camera = new Point3D_F64();
	Point2D_F64 pixel = new Point2D_F64();		Point2D_F64 pixel = new Point2D_F64();
	for( int i = 0; i < structure.points.~~length~~; i++ ) {		for( int i = 0; i < structure.points.size; i++ ) {
	// Get 3D location		// Get 3D location
	SceneStructureMetric.Point p = structure.points[i];		SceneStructureMetric.Point p = structure.points.get(i);
	p.get(world);		p.get(world);

Line 133:		Line 133:
	for (int j = 0; j < p.views.size; j++) {		for (int j = 0; j < p.views.size; j++) {
	int viewIdx = p.views.get(j);		int viewIdx = p.views.get(j);
	SePointOps_F64.transform(structure.views[viewIdx].worldToView,world,camera);		SePointOps_F64.transform(structure.views.data[viewIdx].worldToView,world,camera);
	int cameraIdx = structure.views[viewIdx].camera;		int cameraIdx = structure.views.data[viewIdx].camera;
	structure.cameras[cameraIdx].model.project(camera.x,camera.y,camera.z,pixel);		structure.cameras.get(cameraIdx).model.project(camera.x,camera.y,camera.z,pixel);

	// Get the points color		// Get the points color

Peter at 14:59, 15 March 2019

2019-03-15T14:59:51Z

← Older revision		Revision as of 14:59, 15 March 2019
Line 8:		Line 8:

	Example Code:		Example Code:
	* [https://github.com/lessthanoptimal/BoofCV/blob/v0.32/examples/src/main/java/boofcv/examples/sfm/ExampleMultiviewSceneReconstruction.java ExampleMultiviewSceneReconstruction.java]		* [https://github.com/lessthanoptimal/BoofCV/blob/v0.33/examples/src/main/java/boofcv/examples/sfm/ExampleMultiviewSceneReconstruction.java ExampleMultiviewSceneReconstruction.java]

	Concepts:		Concepts:
Line 33:		Line 33:
	* images.		* images.
	*/		*/
	public void process(~~CameraPinholeRadial~~ intrinsic , List<BufferedImage> colorImages ) {		public void process(CameraPinholeBrown intrinsic , List<BufferedImage> colorImages ) {

	DetectDescribePoint detDesc = FactoryDetectDescribe.surfStable(null, null, null, GrayF32.class);		DetectDescribePoint detDesc = FactoryDetectDescribe.surfStable(null, null, null, GrayF32.class);
Line 74:		Line 74:
	configSBA.configOptimizer = configLM;		configSBA.configOptimizer = configLM;

	BundleAdjustment<SceneStructureMetric> sba = FactoryMultiView.~~bundleAdjustmentMetric~~(configSBA);		BundleAdjustment<SceneStructureMetric> sba = FactoryMultiView.bundleSparseMetric(configSBA);
	sba.configure(1e-10,1e-10,100);		sba.configure(1e-10,1e-10,100);
	sba.setVerbose(System.out,0);		sba.setVerbose(System.out,0);
Line 168:		Line 168:
	String directory = UtilIO.pathExample("sfm/chair");		String directory = UtilIO.pathExample("sfm/chair");

	~~CameraPinholeRadial~~ intrinsic = CalibrationIO.load(		CameraPinholeBrown intrinsic = CalibrationIO.load(
	new File(directory,"/intrinsic_DSC-HX5_3648x2736_to_640x480.yaml"));		new File(directory,"/intrinsic_DSC-HX5_3648x2736_to_640x480.yaml"));

Peter at 04:01, 27 December 2018

2018-12-27T04:01:51Z

← Older revision		Revision as of 04:01, 27 December 2018
Line 8:		Line 8:

	Example Code:		Example Code:
	* [https://github.com/lessthanoptimal/BoofCV/blob/v0.31/examples/src/main/java/boofcv/examples/sfm/ExampleMultiviewSceneReconstruction.java ExampleMultiviewSceneReconstruction.java]		* [https://github.com/lessthanoptimal/BoofCV/blob/v0.32/examples/src/main/java/boofcv/examples/sfm/ExampleMultiviewSceneReconstruction.java ExampleMultiviewSceneReconstruction.java]

	Concepts:		Concepts:
Line 25:		Line 25:
	* structure from motion to understand. In other words, this is not for beginners and requires good clean set of		* structure from motion to understand. In other words, this is not for beginners and requires good clean set of
	* images to work.		* images to work.
	*
	* TODO Update comment
	* One key element it is missing is bundle adjustment to improve the estimated camera location and 3D points. The
	* current bundle adjustment in BoofCV is too inefficient. Better noise removal and numerous other improvements
	* are needed before it can compete with commercial equivalents.
	*		*
	* @author Peter Abeles		* @author Peter Abeles
Line 48:		Line 43:

	String cameraName = "camera";		String cameraName = "camera";
	imageMatching.addCamera(cameraName,~~LensDistortionOps~~.narrow(intrinsic).undistort_F64(true,false),intrinsic);		imageMatching.addCamera(cameraName, LensDistortionFactory.narrow(intrinsic).undistort_F64(true,false),intrinsic);

	for (int i = 0; i < colorImages.size(); i++) {		for (int i = 0; i < colorImages.size(); i++) {
Line 59:		Line 54:

	if( !imageMatching.process() ) {		if( !imageMatching.process() ) {
	throw new RuntimeException("Failed to match images!");		throw new RuntimeException("Failed to match images! total="+colorImages.size());
	}		}

Line 86:		Line 81:
	ScaleSceneStructure bundleScale = new ScaleSceneStructure();		ScaleSceneStructure bundleScale = new ScaleSceneStructure();

	~~PruneStructureFromScene~~ pruner = new ~~PruneStructureFromScene~~(structure,observations);		PruneStructureFromSceneMetric pruner = new PruneStructureFromSceneMetric(structure,observations);

	// Requiring 3 views per point reduces the number of outliers by a lot but also removes		// Requiring 3 views per point reduces the number of outliers by a lot but also removes

Peter at 03:27, 17 October 2018

2018-10-17T03:27:18Z

← Older revision		Revision as of 03:27, 17 October 2018
Line 14:		Line 14:
	* Multiple Views		* Multiple Views
	* Single camera		* Single camera
	* ~~calibrated~~ camera		* Calibrated camera
			* [[Example_Sparse_Bundle_Adjustment\|Sparse Bundle Adjustment]]

	= Example Code =		= Example Code =

Peter at 03:01, 17 October 2018

2018-10-17T03:01:52Z

Show changes

Peter at 16:07, 17 August 2017

2017-08-17T16:07:04Z

← Older revision		Revision as of 16:07, 17 August 2017
Line 8:		Line 8:

	Example Code:		Example Code:
	* [https://github.com/lessthanoptimal/BoofCV/blob/v0.25/examples/src/boofcv/examples/sfm/ExampleMultiviewSceneReconstruction.java ExampleMultiviewSceneReconstruction.java]		* [https://github.com/lessthanoptimal/BoofCV/blob/v0.27/examples/src/boofcv/examples/sfm/ExampleMultiviewSceneReconstruction.java ExampleMultiviewSceneReconstruction.java]

	Concepts:		Concepts:
Line 82:		Line 82:
	public void process(CameraPinholeRadial intrinsic , List<BufferedImage> colorImages ) {		public void process(CameraPinholeRadial intrinsic , List<BufferedImage> colorImages ) {

	pixelToNorm = LensDistortionOps.~~transformPoint~~(intrinsic).undistort_F64(true,false);		pixelToNorm = LensDistortionOps.narrow(intrinsic).undistort_F64(true,false);

	estimateEssential = FactoryMultiViewRobust.essentialRansac(		estimateEssential = FactoryMultiViewRobust.essentialRansac(

Peter at 00:53, 2 December 2016

2016-12-02T00:53:12Z

Peter at 12:36, 28 March 2016

2016-03-28T12:36:06Z

← Older revision		Revision as of 12:36, 28 March 2016
Line 8:		Line 8:

	Example Code:		Example Code:
	* [https://github.com/lessthanoptimal/BoofCV/blob/v0.21/examples/src/boofcv/examples/sfm/ExampleMultiviewSceneReconstruction.java ExampleMultiviewSceneReconstruction.java]		* [https://github.com/lessthanoptimal/BoofCV/blob/v0.23/examples/src/boofcv/examples/sfm/ExampleMultiviewSceneReconstruction.java ExampleMultiviewSceneReconstruction.java]

	Concepts:		Concepts:
Line 46:		Line 46:

	// Detects and describes image interest points		// Detects and describes image interest points
	DetectDescribePoint<~~ImageFloat32~~, BrightFeature> detDesc = FactoryDetectDescribe.surfStable(null, null, null, ~~ImageFloat32~~.class);		DetectDescribePoint<GrayF32, BrightFeature> detDesc = FactoryDetectDescribe.surfStable(null, null, null, GrayF32.class);
	// score ans association algorithm		// score ans association algorithm
	ScoreAssociation<BrightFeature> scorer = FactoryAssociation.scoreEuclidean(BrightFeature.class, true);		ScoreAssociation<BrightFeature> scorer = FactoryAssociation.scoreEuclidean(BrightFeature.class, true);
Line 236:		Line 236:
	GrowQueue_I32 colors ) {		GrowQueue_I32 colors ) {

	~~ImageFloat32~~ image = ConvertBufferedImage.convertFrom(colorImage, (~~ImageFloat32~~) null);		GrayF32 image = ConvertBufferedImage.convertFrom(colorImage, (GrayF32) null);

	features.reset();		features.reset();

Peter at 05:17, 24 January 2016

2016-01-24T05:17:14Z

← Older revision		Revision as of 05:17, 24 January 2016
Line 8:		Line 8:

	Example Code:		Example Code:
	* [https://github.com/lessthanoptimal/BoofCV/blob/v0.20/examples/src/boofcv/examples/sfm/ExampleMultiviewSceneReconstruction.java ExampleMultiviewSceneReconstruction.java]		* [https://github.com/lessthanoptimal/BoofCV/blob/v0.21/examples/src/boofcv/examples/sfm/ExampleMultiviewSceneReconstruction.java ExampleMultiviewSceneReconstruction.java]

	Concepts:		Concepts:
Line 46:		Line 46:

	// Detects and describes image interest points		// Detects and describes image interest points
	DetectDescribePoint<ImageFloat32, ~~SurfFeature~~> detDesc = FactoryDetectDescribe.surfStable(null, null, null, ImageFloat32.class);		DetectDescribePoint<ImageFloat32, BrightFeature> detDesc = FactoryDetectDescribe.surfStable(null, null, null, ImageFloat32.class);
	// score ans association algorithm		// score ans association algorithm
	ScoreAssociation<~~SurfFeature~~> scorer = FactoryAssociation.scoreEuclidean(~~SurfFeature~~.class, true);		ScoreAssociation<BrightFeature> scorer = FactoryAssociation.scoreEuclidean(BrightFeature.class, true);
	AssociateDescription<~~SurfFeature~~> associate = FactoryAssociation.greedy(scorer, 1, true);		AssociateDescription<BrightFeature> associate = FactoryAssociation.greedy(scorer, 1, true);

	// Triangulates the 3D coordinate of a point from two observations		// Triangulates the 3D coordinate of a point from two observations
Line 55:		Line 55:

	// List of visual features (e.g. SURF) descriptions in each image		// List of visual features (e.g. SURF) descriptions in each image
	List<FastQueue<~~SurfFeature~~>> imageVisualFeatures = new ArrayList<FastQueue<~~SurfFeature~~>>();		List<FastQueue<BrightFeature>> imageVisualFeatures = new ArrayList<FastQueue<BrightFeature>>();
	// List of visual feature locations as normalized image coordinates in each image		// List of visual feature locations as normalized image coordinates in each image
	List<FastQueue<Point2D_F64>> imagePixels = new ArrayList<FastQueue<Point2D_F64>>();		List<FastQueue<Point2D_F64>> imagePixels = new ArrayList<FastQueue<Point2D_F64>>();
Line 180:		Line 180:
	System.out.print("*");		System.out.print("*");
	BufferedImage colorImage = colorImages.get(i);		BufferedImage colorImage = colorImages.get(i);
	FastQueue<~~SurfFeature~~> features = new SurfFeatureQueue(64);		FastQueue<BrightFeature> features = new SurfFeatureQueue(64);
	FastQueue<Point2D_F64> pixels = new FastQueue<Point2D_F64>(Point2D_F64.class, true);		FastQueue<Point2D_F64> pixels = new FastQueue<Point2D_F64>(Point2D_F64.class, true);
	GrowQueue_I32 colors = new GrowQueue_I32();		GrowQueue_I32 colors = new GrowQueue_I32();
Line 233:		Line 233:
	*/		*/
	private void detectFeatures(BufferedImage colorImage,		private void detectFeatures(BufferedImage colorImage,
	FastQueue<~~SurfFeature~~> features, FastQueue<Point2D_F64> pixels,		FastQueue<BrightFeature> features, FastQueue<Point2D_F64> pixels,
	GrowQueue_I32 colors ) {		GrowQueue_I32 colors ) {

← Older revision		Revision as of 00:53, 2 December 2016
Line 5:		Line 5:
	</center>		</center>

	Example showing how multiple views from a calibrated camera can be combined together to estimate the camera location and the 3D location of interest points. As discussed in the source code this is still a work in progress and the output is very noisy still.		Example showing how multiple views from a calibrated camera can be combined together to estimate the camera location and the 3D location of interest points. '''As discussed in the source code this is still a work in progress and the output is very noisy still.'''

	Example Code:		Example Code:
	* [https://github.com/lessthanoptimal/BoofCV/blob/v0.23/examples/src/boofcv/examples/sfm/ExampleMultiviewSceneReconstruction.java ExampleMultiviewSceneReconstruction.java]		* [https://github.com/lessthanoptimal/BoofCV/blob/v0.25/examples/src/boofcv/examples/sfm/ExampleMultiviewSceneReconstruction.java ExampleMultiviewSceneReconstruction.java]

	Concepts:		Concepts:
Line 15:		Line 15:
	* Single camera		* Single camera
	* calibrated camera		* calibrated camera

	~~Relevant Applets:~~
	* [[Example_Stereo_Single_Camera\| Stereo Single Camera]]

	= Example Code =		= Example Code =
Line 37:		Line 34:

	// Converts a point from pixel to normalized image coordinates		// Converts a point from pixel to normalized image coordinates
	~~PointTransform_F64~~ pixelToNorm;		Point2Transform2_F64 pixelToNorm;

	// ratio of matching features to unmatched features for two images to be considered connected		// ratio of matching features to unmatched features for two images to be considered connected
Line 55:		Line 52:

	// List of visual features (e.g. SURF) descriptions in each image		// List of visual features (e.g. SURF) descriptions in each image
	List<FastQueue<BrightFeature>> imageVisualFeatures = new ArrayList<~~FastQueue<BrightFeature>~~>();		List<FastQueue<BrightFeature>> imageVisualFeatures = new ArrayList<>();
	// List of visual feature locations as normalized image coordinates in each image		// List of visual feature locations as normalized image coordinates in each image
	List<FastQueue<Point2D_F64>> imagePixels = new ArrayList<~~FastQueue<Point2D_F64>~~>();		List<FastQueue<Point2D_F64>> imagePixels = new ArrayList<>();
	// Color of the pixel at each feature location		// Color of the pixel at each feature location
	List<GrowQueue_I32> imageColors = new ArrayList<~~GrowQueue_I32~~>();		List<GrowQueue_I32> imageColors = new ArrayList<>();
	// List of 3D features in each image		// List of 3D features in each image
	List<List<Feature3D>> imageFeature3D = new ArrayList<~~List<Feature3D>~~>();		List<List<Feature3D>> imageFeature3D = new ArrayList<>();

	// Transform from world to each camera image		// Transform from world to each camera image
Line 72:		Line 69:

	// List of all 3D features		// List of all 3D features
	List<Feature3D> featuresAll = new ArrayList<~~Feature3D~~>();		List<Feature3D> featuresAll = new ArrayList<>();

	// used to provide initial estimate of the 3D scene		// used to provide initial estimate of the 3D scene
Line 83:		Line 80:
	* images.		* images.
	*/		*/
	public void process(~~IntrinsicParameters~~ intrinsic , List<BufferedImage> colorImages ) {		public void process(CameraPinholeRadial intrinsic , List<BufferedImage> colorImages ) {

	pixelToNorm = LensDistortionOps.transformPoint(intrinsic).undistort_F64(true,false);		pixelToNorm = LensDistortionOps.transformPoint(intrinsic).undistort_F64(true,false);
Line 108:		Line 105:

	// Process rest of the images and compute 3D coordinates		// Process rest of the images and compute 3D coordinates
	List<Integer> seed = new ArrayList<~~Integer~~>();		List<Integer> seed = new ArrayList<>();
	seed.add(bestImage);		seed.add(bestImage);
	performReconstruction(seed, -1, matrix);		performReconstruction(seed, -1, matrix);
Line 181:		Line 178:
	BufferedImage colorImage = colorImages.get(i);		BufferedImage colorImage = colorImages.get(i);
	FastQueue<BrightFeature> features = new SurfFeatureQueue(64);		FastQueue<BrightFeature> features = new SurfFeatureQueue(64);
	FastQueue<Point2D_F64> pixels = new FastQueue<~~Point2D_F64~~>(Point2D_F64.class, true);		FastQueue<Point2D_F64> pixels = new FastQueue<>(Point2D_F64.class, true);
	GrowQueue_I32 colors = new GrowQueue_I32();		GrowQueue_I32 colors = new GrowQueue_I32();
	detectFeatures(colorImage, features, pixels, colors);		detectFeatures(colorImage, features, pixels, colors);
Line 281:		Line 278:
	// Compute the 3D pose and find valid image features		// Compute the 3D pose and find valid image features
	Se3_F64 motionAtoB = new Se3_F64();		Se3_F64 motionAtoB = new Se3_F64();
	List<AssociatedIndex> inliers = new ArrayList<~~AssociatedIndex~~>();		List<AssociatedIndex> inliers = new ArrayList<>();

	if( !estimateStereoPose(imageA, imageB, motionAtoB, inliers))		if( !estimateStereoPose(imageA, imageB, motionAtoB, inliers))
Line 332:		Line 329:
	System.out.println("--------- Total Parents "+parents.size());		System.out.println("--------- Total Parents "+parents.size());

	List<Integer> children = new ArrayList<~~Integer~~>();		List<Integer> children = new ArrayList<>();

	if( childAdd != -1 ) {		if( childAdd != -1 ) {
Line 364:		Line 361:
	// initially prune features using essential matrix		// initially prune features using essential matrix
	Se3_F64 dummy = new Se3_F64();		Se3_F64 dummy = new Se3_F64();
	List<AssociatedIndex> inliers = new ArrayList<~~AssociatedIndex~~>();		List<AssociatedIndex> inliers = new ArrayList<>();

	if( !estimateStereoPose(imageA, imageB, dummy, inliers))		if( !estimateStereoPose(imageA, imageB, dummy, inliers))
Line 375:		Line 372:

	// create the associated pair for motion estimation		// create the associated pair for motion estimation
	List<Point2D3D> features = new ArrayList<~~Point2D3D~~>();		List<Point2D3D> features = new ArrayList<>();
	List<AssociatedIndex> inputRansac = new ArrayList<~~AssociatedIndex~~>();		List<AssociatedIndex> inputRansac = new ArrayList<>();
	List<AssociatedIndex> unmatched = new ArrayList<~~AssociatedIndex~~>();		List<AssociatedIndex> unmatched = new ArrayList<>();
	for (int i = 0; i < inliers.size(); i++) {		for (int i = 0; i < inliers.size(); i++) {
	AssociatedIndex a = inliers.get(i);		AssociatedIndex a = inliers.get(i);
Line 528:		Line 525:
	FastQueue<Point2D_F64> pixelsA = imagePixels.get(imageA);		FastQueue<Point2D_F64> pixelsA = imagePixels.get(imageA);
	FastQueue<Point2D_F64> pixelsB = imagePixels.get(imageB);		FastQueue<Point2D_F64> pixelsB = imagePixels.get(imageB);
	List<AssociatedPair> pairs = new ArrayList<~~AssociatedPair~~>();		List<AssociatedPair> pairs = new ArrayList<>();
	for (int i = 0; i < matches.size(); i++) {		for (int i = 0; i < matches.size(); i++) {
	AssociatedIndex a = matches.get(i);		AssociatedIndex a = matches.get(i);
Line 574:		Line 571:
	Point3D_F64 worldPt = new Point3D_F64();		Point3D_F64 worldPt = new Point3D_F64();
	// observations in each frame that it's visible		// observations in each frame that it's visible
	FastQueue<Point2D_F64> obs = new FastQueue<~~Point2D_F64~~>(Point2D_F64.class, true);		FastQueue<Point2D_F64> obs = new FastQueue<>(Point2D_F64.class, true);
	// index of each frame its visible in		// index of each frame its visible in
	GrowQueue_I32 frame = new GrowQueue_I32();		GrowQueue_I32 frame = new GrowQueue_I32();
Line 583:		Line 580:
	String directory = UtilIO.pathExample("sfm/chair");		String directory = UtilIO.pathExample("sfm/chair");

	~~IntrinsicParameters~~ intrinsic = ~~UtilIO~~.~~loadXML~~(directory,"/intrinsic_DSC-HX5_3648x2736_to_640x480.~~xml~~");		CameraPinholeRadial intrinsic = CalibrationIO.load(
			new File(directory,"/intrinsic_DSC-HX5_3648x2736_to_640x480.yaml"));

	List<BufferedImage> images = UtilImageIO.loadImages(directory,".*jpg");		List<BufferedImage> images = UtilImageIO.loadImages(directory,".*jpg");