From dd28e1cc00373c02adf88dff878dbbe5d8be9e59 Mon Sep 17 00:00:00 2001 From: mbansal Date: Wed, 21 Sep 2011 14:19:17 -0400 Subject: Updates to handle textureless scenes during capture. 1) Starts stitching only when the camera sees a textured scene at the beginning. 2) If a texturess scene is encountered in the middle of a capture, the stitching continues with the intermediate frames translated using the pan velocity estimate. 3) Added more error codes and percolated them up to the java layer. 4) Fix a build error in Mosaic::addFrame() and added comments. 5) Update the javadoc in Mosaic.java to reflect the new returning codes. Change-Id: I7727ace615ece22adefe313a19ac2cbe8c8d21a8 --- jni/feature_mos/src/mosaic/AlignFeatures.cpp | 119 +++++++++++++++++++++++---- jni/feature_mos/src/mosaic/AlignFeatures.h | 16 +++- jni/feature_mos/src/mosaic/Mosaic.cpp | 42 ++++++---- jni/feature_mos/src/mosaic/Mosaic.h | 2 + jni/feature_mos_jni.cpp | 23 ++++-- jni/feature_stab/src/dbreg/dbreg.cpp | 2 +- jni/feature_stab/src/dbreg/dbreg.h | 5 ++ 7 files changed, 159 insertions(+), 50 deletions(-) (limited to 'jni') diff --git a/jni/feature_mos/src/mosaic/AlignFeatures.cpp b/jni/feature_mos/src/mosaic/AlignFeatures.cpp index a181dd8..aeabf8f 100644 --- a/jni/feature_mos/src/mosaic/AlignFeatures.cpp +++ b/jni/feature_mos/src/mosaic/AlignFeatures.cpp @@ -26,12 +26,18 @@ #include "trsMatrix.h" #include "MatrixUtils.h" #include "AlignFeatures.h" +#include "Log.h" + +#define LOG_TAG "AlignFeatures" Align::Align() { width = height = 0; frame_number = 0; + num_frames_captured = 0; + reference_frame_index = 0; db_Identity3x3(Hcurr); + db_Identity3x3(Hprev); } Align::~Align() @@ -54,8 +60,8 @@ int Align::initialize(int width, int height, bool _quarter_res, float _thresh_st int nrsamples = DB_DEFAULT_NR_SAMPLES; double scale = DB_POINT_STANDARDDEV; int chunk_size = DB_DEFAULT_CHUNK_SIZE; - int nrhorz = 20; // 1280/32 = 40 - int nrvert = 12; // 720/30 = 24 + int nrhorz = width/48; // Empirically determined number of horizontal + int nrvert = height/60; // and vertical buckets for harris corner detection. bool linear_polish = false; unsigned int reference_update_period = DEFAULT_REFERENCE_UPDATE_PERIOD; @@ -66,10 +72,17 @@ int Align::initialize(int width, int height, bool _quarter_res, float _thresh_st thresh_still = _thresh_still; frame_number = 0; + num_frames_captured = 0; + reference_frame_index = 0; db_Identity3x3(Hcurr); + db_Identity3x3(Hprev); + if (!reg.Initialized()) { - reg.Init(width,height,motion_model_type,20,linear_polish,quarter_res,scale,reference_update_period, false, 0, nrsamples,chunk_size,nr_corners,max_disparity,use_smaller_matching_window, nrhorz, nrvert); + reg.Init(width, height, motion_model_type, 20, linear_polish, quarter_res, + scale, reference_update_period, false, 0, nrsamples, chunk_size, + nr_corners, max_disparity, use_smaller_matching_window, + nrhorz, nrvert); } this->width = width; this->height = height; @@ -90,47 +103,90 @@ int Align::addFrameRGB(ImageType imageRGB) int Align::addFrame(ImageType imageGray_) { - // compute the homography: - double Hinv[9]; - double Hinv33[3][3]; - double Hprev33[3][3]; - double Hcurr33[3][3]; + int ret_code = ALIGN_RET_OK; // Obtain a vector of pointers to rows in image and pass in to dbreg ImageType *m_rows = ImageUtils::imageTypeToRowPointers(imageGray_, width, height); - reg.AddFrame(m_rows, Hcurr); + if (frame_number == 0) + { + reg.AddFrame(m_rows, Hcurr, true); // Force this to be a reference frame + int num_corner_ref = reg.GetNrRefCorners(); + + if (num_corner_ref < MIN_NR_REF_CORNERS) + { + return ALIGN_RET_LOW_TEXTURE; + } + } + else + { + reg.AddFrame(m_rows, Hcurr, false); + } + + // Average translation per frame = + // [Translation from Frame0 to Frame(n-1)] / [(n-1)] + average_tx_per_frame = (num_frames_captured < 2) ? 0.0 : + Hprev[2] / (num_frames_captured - 1); + + // Increment the captured frame counter if we already have a reference frame + num_frames_captured++; if (frame_number != 0) { + int num_inliers = reg.GetNrInliers(); + + if(num_inliers < MIN_NR_INLIERS) + { + ret_code = ALIGN_RET_FEW_INLIERS; + + Hcurr[0] = 1.0; + Hcurr[1] = 0.0; + // Set this as the average per frame translation taking into acccount + // the separation of the current frame from the reference frame... + Hcurr[2] = -average_tx_per_frame * + (num_frames_captured - reference_frame_index); + Hcurr[3] = 0.0; + Hcurr[4] = 1.0; + Hcurr[5] = 0.0; + Hcurr[6] = 0.0; + Hcurr[7] = 0.0; + Hcurr[8] = 1.0; + } if(fabs(Hcurr[2])image = imageYVU; - int align_flag = Align::ALIGN_RET_OK; - // Add frame to aligner + int ret = MOSAIC_RET_ERROR; if (aligner != NULL) { // Note aligner takes in RGB images - printf("Adding frame to aligner...\n"); + int align_flag = Align::ALIGN_RET_OK; align_flag = aligner->addFrame(frame->image); aligner->getLastTRS(frame->trs); - printf("Frame width %d,%d\n", frame->width, frame->height); if (frames_size >= max_frames) { - fprintf(stderr, "WARNING: More frames than preallocated, ignoring. Increase maximum number of frames (-f ) to avoid this\n"); + LOGV("WARNING: More frames than preallocated, ignoring." + "Increase maximum number of frames (-f ) to avoid this"); return MOSAIC_RET_ERROR; } - else if(align_flag == Align::ALIGN_RET_OK) - { - frames_size++; - return MOSAIC_RET_OK; - } - else + + switch (align_flag) { - return MOSAIC_RET_ERROR; + case Align::ALIGN_RET_OK: + frames_size++; + ret = MOSAIC_RET_OK; + break; + case Align::ALIGN_RET_FEW_INLIERS: + frames_size++; + ret = MOSAIC_RET_FEW_INLIERS; + break; + case Align::ALIGN_RET_LOW_TEXTURE: + ret = MOSAIC_RET_LOW_TEXTURE; + break; + case Align::ALIGN_RET_ERROR: + ret = MOSAIC_RET_ERROR; + break; + default: + break; } } - else - { - return MOSAIC_RET_ERROR; - } + + return ret; } int Mosaic::createMosaic(float &progress, bool &cancelComputation) { - printf("Creating mosaic\n"); - if (frames_size <= 0) { // Haven't accepted any frame in aligner. No need to do blending. diff --git a/jni/feature_mos/src/mosaic/Mosaic.h b/jni/feature_mos/src/mosaic/Mosaic.h index 36eafe7..fc6ecd9 100644 --- a/jni/feature_mos/src/mosaic/Mosaic.h +++ b/jni/feature_mos/src/mosaic/Mosaic.h @@ -146,6 +146,8 @@ public: static const int MOSAIC_RET_OK = 1; static const int MOSAIC_RET_ERROR = -1; static const int MOSAIC_RET_CANCELLED = -2; + static const int MOSAIC_RET_LOW_TEXTURE = -3; + static const int MOSAIC_RET_FEW_INLIERS = 2; protected: diff --git a/jni/feature_mos_jni.cpp b/jni/feature_mos_jni.cpp index 5d02793..db9fd7d 100644 --- a/jni/feature_mos_jni.cpp +++ b/jni/feature_mos_jni.cpp @@ -50,7 +50,7 @@ ImageType tImage[NR][MAX_FRAMES_LR];// = {{ImageUtils::IMAGE_TYPE_NOIMAGE}}; // Mosaic *mosaic[NR] = {NULL,NULL}; ImageType resultYVU = ImageUtils::IMAGE_TYPE_NOIMAGE; ImageType resultBGR = ImageUtils::IMAGE_TYPE_NOIMAGE; -float gTRS[10]; +float gTRS[11]; // 9 elements of the transformation, 1 for frame-number, 1 for alignment error code. // Variables to keep track of the mosaic computation progress for both LR & HR. float gProgress[NR]; // Variables to be able to cancel the mosaic computation when the GUI says so. @@ -373,6 +373,7 @@ JNIEXPORT jfloatArray JNICALL Java_com_android_camera_panorama_Mosaic_setSourceI { double t0, t1, time_c; t0 = now_ms(); + int ret_code; if(frame_number_HRNewFloatArray(10); + jfloatArray bytes = env->NewFloatArray(11); if(bytes != 0) { - env->SetFloatArrayRegion(bytes, 0, 10, (jfloat*) gTRS); + env->SetFloatArrayRegion(bytes, 0, 11, (jfloat*) gTRS); } return bytes; } @@ -424,6 +426,8 @@ JNIEXPORT jfloatArray JNICALL Java_com_android_camera_panorama_Mosaic_setSourceI double t0, t1, time_c; t0 = now_ms(); + int ret_code; + if(frame_number_HRGetByteArrayElements(photo_data, 0); @@ -445,9 +449,9 @@ JNIEXPORT jfloatArray JNICALL Java_com_android_camera_panorama_Mosaic_setSourceI gPreviewImageWidth[LR], gPreviewImageHeight[LR]); sem_post(&gPreviewImage_semaphore); - int ret_code = AddFrame(LR, frame_number_LR, gTRS); + ret_code = AddFrame(LR, frame_number_LR, gTRS); - if(ret_code == Mosaic::MOSAIC_RET_OK) + if(ret_code == Mosaic::MOSAIC_RET_OK || ret_code == Mosaic::MOSAIC_RET_FEW_INLIERS) { frame_number_LR++; frame_number_HR++; @@ -463,11 +467,12 @@ JNIEXPORT jfloatArray JNICALL Java_com_android_camera_panorama_Mosaic_setSourceI UpdateWarpTransformation(gTRS); gTRS[9] = frame_number_HR; + gTRS[10] = ret_code; - jfloatArray bytes = env->NewFloatArray(10); + jfloatArray bytes = env->NewFloatArray(11); if(bytes != 0) { - env->SetFloatArrayRegion(bytes, 0, 10, (jfloat*) gTRS); + env->SetFloatArrayRegion(bytes, 0, 11, (jfloat*) gTRS); } return bytes; } diff --git a/jni/feature_stab/src/dbreg/dbreg.cpp b/jni/feature_stab/src/dbreg/dbreg.cpp index fb42838..dc7d58f 100644 --- a/jni/feature_stab/src/dbreg/dbreg.cpp +++ b/jni/feature_stab/src/dbreg/dbreg.cpp @@ -344,7 +344,7 @@ bool db_FrameToReferenceRegistration::NeedReferenceUpdate() int db_FrameToReferenceRegistration::AddFrame(const unsigned char * const * im, double H[9],bool force_reference,bool prewarp) { m_current_is_reference = false; - if(!m_reference_set) + if(!m_reference_set || force_reference) { db_Identity3x3(m_H_ref_to_ins); db_Copy9(H,m_H_ref_to_ins); diff --git a/jni/feature_stab/src/dbreg/dbreg.h b/jni/feature_stab/src/dbreg/dbreg.h index 92cd0e3..4eb2444 100644 --- a/jni/feature_stab/src/dbreg/dbreg.h +++ b/jni/feature_stab/src/dbreg/dbreg.h @@ -222,6 +222,11 @@ public: int GetNrMatches() { return m_nr_matches; } /*! + * Returns the number of corners detected in the current reference image. + */ + int GetNrRefCorners() { return m_nr_corners_ref; } + + /*! * Returns the pointer to an array of indices that were found to be RANSAC inliers from the matched corner lists. */ int* GetInliers() { return m_inlier_indices; } -- cgit v1.1