Example Associate Three View

Three view feature association from using this example.

Associating features across three views is useful for 3D reconstruction as it produces a more stable solution than 2-views.

Example Code:

• ExampleAssociateThreeView.java

Concepts:

• Describe point features
• Associate descriptions

Example Code

/**
 * Common matches between sets of three views are important in SFM as they filter out even more false positives
 * and three view geometry, unlike two view-geometry, has a unique projection in each image. This makes it even
 * easier to remove false matches using geometric constraints. In BoofCV's reconstruction pipeline three views
 * are always used over two views whenever possible due to the added robustness.
 *
 * In this example, association is first done pairwise between each image pair. Then the matches are traversed
 * to find features which form a "ring", that is that when traversed from image 1 -> 2 - > 3 -> 1 you wind up
 * back at the same location.
 */
public class ExampleAssociateThreeView {

	// Stores image pixel coordinate
	public final DogArray<Point2D_F64> locations01 = new DogArray<>(Point2D_F64::new);
	public final DogArray<Point2D_F64> locations02 = new DogArray<>(Point2D_F64::new);
	public final DogArray<Point2D_F64> locations03 = new DogArray<>(Point2D_F64::new);

	// Stores the descriptor for each feature
	public DogArray<TupleDesc_F64> features01, features02, features03;

	// Indicates which "set" a feature belongs in. SURF can be white or black. Using sets simplifies
	// feature association since only features in the same set can be matched
	public final DogArray_I32 featureSet01 = new DogArray_I32();
	public final DogArray_I32 featureSet02 = new DogArray_I32();
	public final DogArray_I32 featureSet03 = new DogArray_I32();

	// Reference to the feature detector/descriptor
	DetectDescribePoint<GrayU8, TupleDesc_F64> detDesc;

	// Create lists when accessing by index makes more sense
	List<DogArray<Point2D_F64>> listLocations = BoofMiscOps.asList(locations01, locations02, locations03);
	List<DogArray_I32> listFeatureSets = BoofMiscOps.asList(featureSet01, featureSet02, featureSet03);
	List<DogArray<TupleDesc_F64>> listFeatures;

	/**
	 * Initializes data structures to use the feature descriptor
	 */
	public<T extends ImageBase<T>> void initialize( DetectDescribePoint<GrayU8, TupleDesc_F64> detDesc ) {
		this.detDesc = detDesc;
		features01 = UtilFeature.createArray(detDesc, 100);
		features02 = UtilFeature.createArray(detDesc, 100);
		features03 = UtilFeature.createArray(detDesc, 100);
		listFeatures = BoofMiscOps.asList(features01, features02, features03);
	}

	/**
	 * Detects and saves features in the specified image
	 */
	public void detectFeatures( GrayU8 gray, int which ) {
		DogArray<Point2D_F64> locations = listLocations.get(which);
		DogArray<TupleDesc_F64> features = listFeatures.get(which);
		DogArray_I32 featureSet = listFeatureSets.get(which);

		detDesc.detect(gray);
		for (int i = 0; i < detDesc.getNumberOfFeatures(); i++) {
			Point2D_F64 pixel = detDesc.getLocation(i);
			locations.grow().setTo(pixel.x, pixel.y);
			features.grow().setTo(detDesc.getDescription(i));
			featureSet.add(detDesc.getSet(i));
		}
	}

	/**
	 * BoofCV comes with a class which does all the three view matching for you. Which association and scoring
	 * function are used is all configurable.
	 */
	public DogArray<AssociatedTripleIndex> threeViewPairwiseAssociate() {
		ScoreAssociation<TupleDesc_F64> scorer =
				FactoryAssociation.scoreEuclidean(TupleDesc_F64.class, true);
		AssociateDescription<TupleDesc_F64> associate =
				FactoryAssociation.greedy(new ConfigAssociateGreedy(true, 0.1), scorer);

		var associateThree = new AssociateThreeByPairs<>(associate);

		associateThree.initialize(detDesc.getNumberOfSets());
		associateThree.setFeaturesA(features01, featureSet01);
		associateThree.setFeaturesB(features02, featureSet02);
		associateThree.setFeaturesC(features03, featureSet03);

		associateThree.associate();

		return associateThree.getMatches();
	}

	public static void main( String[] args ) {
		String name = "rock_leaves_";
		GrayU8 gray01 = UtilImageIO.loadImage(UtilIO.pathExample("triple/" + name + "01.jpg"), GrayU8.class);
		GrayU8 gray02 = UtilImageIO.loadImage(UtilIO.pathExample("triple/" + name + "02.jpg"), GrayU8.class);
		GrayU8 gray03 = UtilImageIO.loadImage(UtilIO.pathExample("triple/" + name + "03.jpg"), GrayU8.class);

		// Using SURF features. Robust and fairly fast to compute
		DetectDescribePoint<GrayU8, TupleDesc_F64> detDesc = FactoryDetectDescribe.surfStable(
				new ConfigFastHessian(0, 4, 1000, 1, 9, 4, 2), null, null, GrayU8.class);

		ExampleAssociateThreeView example = new ExampleAssociateThreeView();
		example.initialize(detDesc);

		// Compute and describe features inside the image
		example.detectFeatures(gray01, 0);
		example.detectFeatures(gray02, 1);
		example.detectFeatures(gray03, 2);

		System.out.println("features01.size = " + example.features01.size);
		System.out.println("features02.size = " + example.features02.size);
		System.out.println("features03.size = " + example.features03.size);

		// Find features for an association ring across all the views. This removes most false positives.
		DogArray<AssociatedTripleIndex> associatedIdx = example.threeViewPairwiseAssociate();

		// Convert the matched indexes into AssociatedTriple which contain the actual pixel coordinates
		var associated = new DogArray<>(AssociatedTriple::new);
		associatedIdx.forEach(p->associated.grow().setTo(
				example.locations01.get(p.a), example.locations02.get(p.b), example.locations03.get(p.c)));

		System.out.println("Total Matched Triples = " + associated.size);

		// Show remaining associations from RANSAC
		var triplePanel = new AssociatedTriplePanel();
		triplePanel.setImages(
				UtilImageIO.loadImage(UtilIO.pathExample("triple/" + name + "01.jpg")),
				UtilImageIO.loadImage(UtilIO.pathExample("triple/" + name + "02.jpg")),
				UtilImageIO.loadImage(UtilIO.pathExample("triple/" + name + "03.jpg")));
		triplePanel.setAssociation(associated.toList());
		ShowImages.showWindow(triplePanel, "Associations", true);
	}
}