Difference between revisions of "Example Video Mosaic"

From BoofCV
Jump to navigationJump to search
m
m
 
(10 intermediate revisions by the same user not shown)
Line 5: Line 5:
</center>
</center>


An image mosaic is created by combining numerious images together to create one large one.  In this example, it is demonstrated how to create a mosaic from a video image in real-time using a high level interface which hides much
An image mosaic is created by combining numerous images together to create one large one.  In this example, it is demonstrated how to create a mosaic from a video image in real-time using a high level interface which hides much
of the complexity.  The mosaic is created by tracking image features and fitting a motion model (affine or homography) to the feature's location in each frame.
of the complexity.  The mosaic is created by tracking image features and fitting a motion model (affine or homography) to the feature's location in each frame.


Example File: [https://github.com/lessthanoptimal/BoofCV/blob/v0.15/examples/src/boofcv/examples/ExampleVideoMosaic.java ExampleVideoMosaic.java]
Example File: [https://github.com/lessthanoptimal/BoofCV/blob/v0.38/examples/src/main/java/boofcv/examples/geometry/ExampleVideoMosaic.java ExampleVideoMosaic.java]


Concepts:
Concepts:
Line 15: Line 15:
* 2D image motion
* 2D image motion


Relevant Applets:
Relevant Videos:
* [[Applet_Video_Mosaic| Video Mosaic]] ([http://www.youtube.com/watch?v=59RJeLlDAxQ YouTube Video])
* [http://www.youtube.com/watch?v=59RJeLlDAxQ YouTube Video of Demonstration]


Related Examples:
Related Examples:
* [[Example_Video_Stabilization| Video Stabilization]]
* [[Example_Image_Stitching| Image Stitching]]
* [[Example_Image_Stitching| Image Stitching]]


Line 24: Line 25:
<syntaxhighlight lang="java">
<syntaxhighlight lang="java">
/**
/**
  * Example of how to create a mosaic from a video sequence using StitchingFromMotion2D. Mosaics work best
  * Example of how to create a mosaic from a video sequence using StitchingFromMotion2D. Mosaics work best
  * when the scene being observed is far away or a flat surface. The camera motion should typically be rotational only,
  * when the scene being observed is far away or a flat surface. The camera motion should typically be rotational only,
  * but translation can work depending on the scene's geometry. Motion blur and cheap cameras in general will degrade
  * but translation can work depending on the scene's geometry. Motion blur and cheap cameras in general will degrade
  * performance significantly with the current algorithm. This example just demonstrates a gray scale image, but
  * performance significantly with the current algorithm. This example just demonstrates a gray scale image, but
  * with additional work color images can also be processed.
  * with additional work color images can also be processed.
  *
  *
Line 33: Line 34:
  */
  */
public class ExampleVideoMosaic {
public class ExampleVideoMosaic {
public static void main( String args[] ) {
public static void main( String[] args ) {
 
// Configure the feature detector
// Configure the feature detector
ConfigGeneralDetector confDetector = new ConfigGeneralDetector();
ConfigPointDetector configDetector = new ConfigPointDetector();
confDetector.threshold = 10;
configDetector.type = PointDetectorTypes.SHI_TOMASI;
confDetector.maxFeatures = 300;
configDetector.general.maxFeatures = 300;
confDetector.radius = 2;
configDetector.general.radius = 3;
configDetector.general.threshold = 1;


// Use a KLT tracker
// Use a KLT tracker
PointTracker<ImageFloat32> tracker = FactoryPointTracker.klt(new int[]{1,2,4,8},confDetector,3,
PointTracker<GrayF32> tracker = FactoryPointTracker.klt(4, configDetector, 3, GrayF32.class, GrayF32.class);
ImageFloat32.class,ImageFloat32.class);


// This estimates the 2D image motion
// This estimates the 2D image motion
// An Affine2D_F64 model also works quite well.
// An Affine2D_F64 model also works quite well.
ImageMotion2D<ImageFloat32,Homography2D_F64> motion2D =
ImageMotion2D<GrayF32, Homography2D_F64> motion2D =
FactoryMotion2D.createMotion2D(200,3,2,30,0.6,0.5,false,tracker,new Homography2D_F64());
FactoryMotion2D.createMotion2D(220, 3, 2, 30, 0.6, 0.5, false, tracker, new Homography2D_F64());


// wrap it so it output color images while estimating motion from gray
// wrap it so it output color images while estimating motion from gray
ImageMotion2D<MultiSpectral<ImageFloat32>,Homography2D_F64> motion2DColor =
ImageMotion2D<Planar<GrayF32>, Homography2D_F64> motion2DColor =
new MsToGrayMotion2D<ImageFloat32,Homography2D_F64>(motion2D,ImageFloat32.class);
new PlToGrayMotion2D<>(motion2D, GrayF32.class);


// This fuses the images together
// This fuses the images together
StitchingFromMotion2D<MultiSpectral<ImageFloat32>,Homography2D_F64>
StitchingFromMotion2D<Planar<GrayF32>, Homography2D_F64>
stitch = FactoryMotion2D.createVideoStitchMS(0.5, motion2DColor, ImageFloat32.class);
stitch = FactoryMotion2D.createVideoStitch(0.5, motion2DColor, ImageType.pl(3, GrayF32.class));


// Load an image sequence
// Load an image sequence
MediaManager media = DefaultMediaManager.INSTANCE;
MediaManager media = DefaultMediaManager.INSTANCE;
String fileName = "../data/applet/mosaic/airplane01.mjpeg";
String fileName = UtilIO.pathExample("mosaic/airplane01.mjpeg");
SimpleImageSequence<MultiSpectral<ImageFloat32>> video =
SimpleImageSequence<Planar<GrayF32>> video =
media.openVideo(fileName, ImageDataType.ms(3,ImageFloat32.class));
media.openVideo(fileName, ImageType.pl(3, GrayF32.class));


MultiSpectral<ImageFloat32> frame = video.next();
Planar<GrayF32> frame = video.next();


// shrink the input image and center it
// shrink the input image and center it
Homography2D_F64 shrink = new Homography2D_F64(0.5,0,frame.width/3,0,0.5,frame.height/2,0,0,1);
Homography2D_F64 shrink = new Homography2D_F64(0.5, 0, frame.width/4, 0, 0.5, frame.height/4, 0, 0, 1);
shrink = shrink.invert(null);
shrink = shrink.invert(null);


// Make the mosaic image the same size as the input image
// The mosaic will be larger in terms of pixels but the image will be scaled down.
stitch.configure(frame.width*2,3*frame.height/2,shrink);
// To change this into stabilization just make it the same size as the input with no shrink.
stitch.configure(frame.width, frame.height, shrink);
// process the first frame
// process the first frame
stitch.process(frame);
stitch.process(frame);


// Create the GUI for displaying the results + input image
// Create the GUI for displaying the results + input image
ImageGridPanel gui = new ImageGridPanel(1,2);
ImageGridPanel gui = new ImageGridPanel(1, 2);
gui.setImage(0,0,new BufferedImage(frame.width,frame.height,BufferedImage.TYPE_INT_RGB));
gui.setImage(0, 0, new BufferedImage(frame.width, frame.height, BufferedImage.TYPE_INT_RGB));
gui.setImage(0,1,new BufferedImage(frame.width*2,3*frame.height/2,BufferedImage.TYPE_INT_RGB));
gui.setImage(0, 1, new BufferedImage(frame.width, frame.height, BufferedImage.TYPE_INT_RGB));
gui.autoSetPreferredSize();
gui.setPreferredSize(new Dimension(3*frame.width, frame.height*2));


ShowImages.showWindow(gui,"Example Mosaic");
ShowImages.showWindow(gui, "Example Mosaic", true);
 
boolean enlarged = false;


// process the video sequence one frame at a time
// process the video sequence one frame at a time
while( video.hasNext() ) {
while (video.hasNext()) {
frame = video.next();
frame = video.next();
stitch.process(frame);
if (!stitch.process(frame))
throw new RuntimeException("You should handle failures");


// if the current image is close to the image border recenter the mosaic
// if the current image is close to the image border recenter the mosaic
StitchingFromMotion2D.Corners corners = stitch.getImageCorners(frame.width,frame.height,null);
Quadrilateral_F64 corners = stitch.getImageCorners(frame.width, frame.height, null);
if( nearBorder(corners.p0,stitch) || nearBorder(corners.p1,stitch) ||
if (nearBorder(corners.a, stitch) || nearBorder(corners.b, stitch) ||
nearBorder(corners.p2,stitch) || nearBorder(corners.p3,stitch) ) {
nearBorder(corners.c, stitch) || nearBorder(corners.d, stitch)) {
stitch.setOriginToCurrent();
stitch.setOriginToCurrent();
// only enlarge the image once
if (!enlarged) {
enlarged = true;
// double the image size and shift it over to keep it centered
int widthOld = stitch.getStitchedImage().width;
int heightOld = stitch.getStitchedImage().height;
int widthNew = widthOld*2;
int heightNew = heightOld*2;
int tranX = (widthNew - widthOld)/2;
int tranY = (heightNew - heightOld)/2;
Homography2D_F64 newToOldStitch = new Homography2D_F64(1, 0, -tranX, 0, 1, -tranY, 0, 0, 1);
stitch.resizeStitchImage(widthNew, heightNew, newToOldStitch);
gui.setImage(0, 1, new BufferedImage(widthNew, heightNew, BufferedImage.TYPE_INT_RGB));
}
corners = stitch.getImageCorners(frame.width, frame.height, null);
}
}
// display the mosaic
ConvertBufferedImage.convertTo(frame, gui.getImage(0, 0), true);
ConvertBufferedImage.convertTo(stitch.getStitchedImage(), gui.getImage(0, 1), true);


// display the mosaic
// draw a red quadrilateral around the current frame in the mosaic
ConvertBufferedImage.convertTo(frame,gui.getImage(0, 0));
Graphics2D g2 = gui.getImage(0, 1).createGraphics();
ConvertBufferedImage.convertTo(stitch.getStitchedImage(), gui.getImage(0, 1));
g2.setColor(Color.RED);
g2.drawLine((int)corners.a.x, (int)corners.a.y, (int)corners.b.x, (int)corners.b.y);
g2.drawLine((int)corners.b.x, (int)corners.b.y, (int)corners.c.x, (int)corners.c.y);
g2.drawLine((int)corners.c.x, (int)corners.c.y, (int)corners.d.x, (int)corners.d.y);
g2.drawLine((int)corners.d.x, (int)corners.d.y, (int)corners.a.x, (int)corners.a.y);


gui.repaint();
gui.repaint();
Line 109: Line 140:
* Checks to see if the point is near the image border
* Checks to see if the point is near the image border
*/
*/
private static boolean nearBorder( Point2D_F64 p , StitchingFromMotion2D<?,?> stitch ) {
private static boolean nearBorder( Point2D_F64 p, StitchingFromMotion2D<?, ?> stitch ) {
int r = 10;
int r = 10;
if( p.x < r || p.y < r )
if (p.x < r || p.y < r)
return true;
return true;
if( p.x >= stitch.getStitchedImage().width-r )
if (p.x >= stitch.getStitchedImage().width - r)
return true;
return true;
if( p.y >= stitch.getStitchedImage().height-r )
if (p.y >= stitch.getStitchedImage().height - r)
return true;
return true;



Latest revision as of 11:17, 12 July 2021

An image mosaic is created by combining numerous images together to create one large one. In this example, it is demonstrated how to create a mosaic from a video image in real-time using a high level interface which hides much of the complexity. The mosaic is created by tracking image features and fitting a motion model (affine or homography) to the feature's location in each frame.

Example File: ExampleVideoMosaic.java

Concepts:

  • Image Stitching
  • Image Mosaic
  • 2D image motion

Relevant Videos:

Related Examples:

Example Code

/**
 * Example of how to create a mosaic from a video sequence using StitchingFromMotion2D. Mosaics work best
 * when the scene being observed is far away or a flat surface. The camera motion should typically be rotational only,
 * but translation can work depending on the scene's geometry. Motion blur and cheap cameras in general will degrade
 * performance significantly with the current algorithm. This example just demonstrates a gray scale image, but
 * with additional work color images can also be processed.
 *
 * @author Peter Abeles
 */
public class ExampleVideoMosaic {
	public static void main( String[] args ) {
		// Configure the feature detector
		ConfigPointDetector configDetector = new ConfigPointDetector();
		configDetector.type = PointDetectorTypes.SHI_TOMASI;
		configDetector.general.maxFeatures = 300;
		configDetector.general.radius = 3;
		configDetector.general.threshold = 1;

		// Use a KLT tracker
		PointTracker<GrayF32> tracker = FactoryPointTracker.klt(4, configDetector, 3, GrayF32.class, GrayF32.class);

		// This estimates the 2D image motion
		// An Affine2D_F64 model also works quite well.
		ImageMotion2D<GrayF32, Homography2D_F64> motion2D =
				FactoryMotion2D.createMotion2D(220, 3, 2, 30, 0.6, 0.5, false, tracker, new Homography2D_F64());

		// wrap it so it output color images while estimating motion from gray
		ImageMotion2D<Planar<GrayF32>, Homography2D_F64> motion2DColor =
				new PlToGrayMotion2D<>(motion2D, GrayF32.class);

		// This fuses the images together
		StitchingFromMotion2D<Planar<GrayF32>, Homography2D_F64>
				stitch = FactoryMotion2D.createVideoStitch(0.5, motion2DColor, ImageType.pl(3, GrayF32.class));

		// Load an image sequence
		MediaManager media = DefaultMediaManager.INSTANCE;
		String fileName = UtilIO.pathExample("mosaic/airplane01.mjpeg");
		SimpleImageSequence<Planar<GrayF32>> video =
				media.openVideo(fileName, ImageType.pl(3, GrayF32.class));

		Planar<GrayF32> frame = video.next();

		// shrink the input image and center it
		Homography2D_F64 shrink = new Homography2D_F64(0.5, 0, frame.width/4, 0, 0.5, frame.height/4, 0, 0, 1);
		shrink = shrink.invert(null);

		// The mosaic will be larger in terms of pixels but the image will be scaled down.
		// To change this into stabilization just make it the same size as the input with no shrink.
		stitch.configure(frame.width, frame.height, shrink);
		// process the first frame
		stitch.process(frame);

		// Create the GUI for displaying the results + input image
		ImageGridPanel gui = new ImageGridPanel(1, 2);
		gui.setImage(0, 0, new BufferedImage(frame.width, frame.height, BufferedImage.TYPE_INT_RGB));
		gui.setImage(0, 1, new BufferedImage(frame.width, frame.height, BufferedImage.TYPE_INT_RGB));
		gui.setPreferredSize(new Dimension(3*frame.width, frame.height*2));

		ShowImages.showWindow(gui, "Example Mosaic", true);

		boolean enlarged = false;

		// process the video sequence one frame at a time
		while (video.hasNext()) {
			frame = video.next();
			if (!stitch.process(frame))
				throw new RuntimeException("You should handle failures");

			// if the current image is close to the image border recenter the mosaic
			Quadrilateral_F64 corners = stitch.getImageCorners(frame.width, frame.height, null);
			if (nearBorder(corners.a, stitch) || nearBorder(corners.b, stitch) ||
					nearBorder(corners.c, stitch) || nearBorder(corners.d, stitch)) {
				stitch.setOriginToCurrent();

				// only enlarge the image once
				if (!enlarged) {
					enlarged = true;
					// double the image size and shift it over to keep it centered
					int widthOld = stitch.getStitchedImage().width;
					int heightOld = stitch.getStitchedImage().height;

					int widthNew = widthOld*2;
					int heightNew = heightOld*2;

					int tranX = (widthNew - widthOld)/2;
					int tranY = (heightNew - heightOld)/2;

					Homography2D_F64 newToOldStitch = new Homography2D_F64(1, 0, -tranX, 0, 1, -tranY, 0, 0, 1);

					stitch.resizeStitchImage(widthNew, heightNew, newToOldStitch);
					gui.setImage(0, 1, new BufferedImage(widthNew, heightNew, BufferedImage.TYPE_INT_RGB));
				}
				corners = stitch.getImageCorners(frame.width, frame.height, null);
			}
			// display the mosaic
			ConvertBufferedImage.convertTo(frame, gui.getImage(0, 0), true);
			ConvertBufferedImage.convertTo(stitch.getStitchedImage(), gui.getImage(0, 1), true);

			// draw a red quadrilateral around the current frame in the mosaic
			Graphics2D g2 = gui.getImage(0, 1).createGraphics();
			g2.setColor(Color.RED);
			g2.drawLine((int)corners.a.x, (int)corners.a.y, (int)corners.b.x, (int)corners.b.y);
			g2.drawLine((int)corners.b.x, (int)corners.b.y, (int)corners.c.x, (int)corners.c.y);
			g2.drawLine((int)corners.c.x, (int)corners.c.y, (int)corners.d.x, (int)corners.d.y);
			g2.drawLine((int)corners.d.x, (int)corners.d.y, (int)corners.a.x, (int)corners.a.y);

			gui.repaint();

			// throttle the speed just in case it's on a fast computer
			BoofMiscOps.pause(50);
		}
	}

	/**
	 * Checks to see if the point is near the image border
	 */
	private static boolean nearBorder( Point2D_F64 p, StitchingFromMotion2D<?, ?> stitch ) {
		int r = 10;
		if (p.x < r || p.y < r)
			return true;
		if (p.x >= stitch.getStitchedImage().width - r)
			return true;
		if (p.y >= stitch.getStitchedImage().height - r)
			return true;

		return false;
	}
}