Ubuntu Question posted in
The official Ubuntu documentation can be found here.

Ubuntu – Hole-Filling Filter in OpenCV C++

I have a basic implementation of hole-filling filter as shown below:

#include <iostream>
#include <opencv2/opencv.hpp>

int main(int argc, char** argv)
{
    // please note that the depthImg is (720 x 576) 8UC1
    // let's make a smaller one for testing
    uchar flatten[6 * 8] = { 140, 185,  48, 235, 201, 192, 131,  57,
                              55,  87,  82,   0,   6, 201,   0,  38,
                               6, 239,  82, 142,  46,  33, 172,  72,
                             133,   0, 232, 226,  66,  59,  10, 204,
                             214, 123, 202, 100,   0,  32,   6, 147,
                             105, 191,  50,  21,  87, 117, 118, 244};

    cv::Mat depthImg = cv::Mat(6, 8, CV_8UC1, flatten);

    // please ignore the border pixels in this case
    for (int i = 1; i < depthImg.cols - 1; i++) {
        for (int j = 1; j < depthImg.rows - 1; j++) {
            unsigned short sumNonZeroAdjs = 0;
            uchar countNonZeroAdjs = 0;
            if (depthImg.at<uchar>(j, i) == 0) {
                uchar iMinus1 = depthImg.at<uchar>(j, i - 1);
                uchar  iPlus1 = depthImg.at<uchar>(j, i + 1);
                uchar jMinus1 = depthImg.at<uchar>(j - 1, i);
                uchar  jPlus1 = depthImg.at<uchar>(j + 1, i);
                if (iMinus1 != 0) {
                    sumNonZeroAdjs += iMinus1;
                    countNonZeroAdjs++;
                }
                if (iPlus1 != 0) {
                    sumNonZeroAdjs += iPlus1;
                    countNonZeroAdjs++;
                }
                if (jMinus1 != 0) {
                    sumNonZeroAdjs += jMinus1;
                    countNonZeroAdjs++;
                }
                if (jPlus1 != 0) {
                    sumNonZeroAdjs += jPlus1;
                    countNonZeroAdjs++;
                }
                depthImg.at<uchar>(j, i) = sumNonZeroAdjs / countNonZeroAdjs;
            }
        }
    }

    std::cout << depthImg << std::endl;
    return 0;
}
// prints the following:
[140, 185, 48, 235, 201, 192, 131, 57;
  55, 87, 82, 116, 6, 201, 135, 38;
  6, 239, 82, 142, 46, 33, 172, 72;
  133, 181, 232, 226, 66, 59, 10, 204;
  214, 123, 202, 100, 71, 32, 6, 147;
  105, 191, 50, 21, 87, 117, 118, 244]

The above filter computes an average of adjacent pixels to fill the 0 pixels. The output from this implementation is satisfactory. However, as we can see, the above prototype is not elegant and painfully slow.

I am looking for a similar logic (using adjacent pixels to fill 0 pixels) but faster (execution time) hole-filling filter inbuilt in OpenCV

PS: I am using OpenCV v4.2.0 on Ubuntu 20.04 LTS.

Update 1

Based on the suggestions, I designed pointer style access. Complete code is shown below:

#include <iostream>
#include <opencv2/opencv.hpp>

void inPlaceHoleFillingExceptBorderPtrStyle(cv::Mat& img) {
  typedef uchar T;
  T* ptr = img.data;
  size_t elemStep = img.step / sizeof(T);

  for (int i = 1; i < img.rows - 1; i++) {
    for (int j = 1; j < img.cols - 1; j++) {
      T& curr = ptr[i * elemStep + j];
      if (curr != 0) {
        continue;
      }

      ushort sumNonZeroAdjs = 0;
      uchar countNonZeroAdjs = 0;
      T iM1 = ptr[(i - 1) * elemStep + j];
      T iP1 = ptr[(i + 1) * elemStep + j];
      T jM1 = ptr[i * elemStep + (j - 1)];
      T jP1 = ptr[i * elemStep + (j + 1)];

      if (iM1 != 0) {
        sumNonZeroAdjs += iM1;
        countNonZeroAdjs++;
      }
      if (iP1 != 0) {
        sumNonZeroAdjs += iP1;
        countNonZeroAdjs++;
      }
      if (jM1 != 0) {
        sumNonZeroAdjs += jM1;
        countNonZeroAdjs++;
      }
      if (jP1 != 0) {
        sumNonZeroAdjs += jP1;
        countNonZeroAdjs++;
      }
      if (countNonZeroAdjs > 0) {
        curr = sumNonZeroAdjs / countNonZeroAdjs;
      }
    }
  }
}

void inPlaceHoleFillingExceptBorder(cv::Mat& img) {
  typedef uchar T;

  for (int i = 1; i < img.cols - 1; i++) {
    for (int j = 1; j < img.rows - 1; j++) {
      ushort sumNonZeroAdjs = 0;
      uchar countNonZeroAdjs = 0;
      if (img.at<T>(j, i) != 0) {
        continue;
      }

      T iM1 = img.at<T>(j, i - 1);
      T iP1 = img.at<T>(j, i + 1);
      T jM1 = img.at<T>(j - 1, i);
      T jP1 = img.at<T>(j + 1, i);

      if (iM1 != 0) {
        sumNonZeroAdjs += iM1;
        countNonZeroAdjs++;
      }
      if (iP1 != 0) {
        sumNonZeroAdjs += iP1;
        countNonZeroAdjs++;
      }
      if (jM1 != 0) {
        sumNonZeroAdjs += jM1;
        countNonZeroAdjs++;
      }
      if (jP1 != 0) {
        sumNonZeroAdjs += jP1;
        countNonZeroAdjs++;
      }
      if (countNonZeroAdjs > 0) {
        img.at<T>(j, i) = sumNonZeroAdjs / countNonZeroAdjs;
      }
    }
  }
}

int main(int argc, char** argv) {
  // please note that the img is (720 x 576) 8UC1
  // let's make a smaller one for testing
  // clang-format off
  uchar flatten[6 * 8] = { 140, 185,  48, 235, 201, 192, 131,  57,
                            55,  87,  82,   0,   6, 201,   0,  38,
                             6, 239,  82, 142,  46,  33, 172,  72,
                           133,   0, 232, 226,  66,  59,  10, 204,
                           214, 123, 202, 100,   0,  32,   6, 147,
                           105, 191,  50,  21,  87, 117, 118, 244};
  // clang-format on

  cv::Mat img = cv::Mat(6, 8, CV_8UC1, flatten);
  cv::Mat img1 = img.clone();
  cv::Mat img2 = img.clone();

  inPlaceHoleFillingExceptBorderPtrStyle(img1);
  inPlaceHoleFillingExceptBorder(img2);

  return 0;
}

/*** expected output
[140, 185,  48, 235, 201, 192, 131, 57;
  55,  87,  82, 116,  6,  201, 135, 38;
   6, 239,  82, 142, 46,   33, 172, 72;
 133, 181, 232, 226, 66,   59,  10, 204;
 214, 123, 202, 100, 71,   32,   6, 147;
 105, 191,  50,  21, 87,  117, 118, 244]
***/

Update 2

Based on the suggestion, the point style code is further improved as shown below:


void inPlaceHoleFillingExceptBorderImpv(cv::Mat& img) {
  typedef uchar T;
  size_t elemStep = img.step1();
  const size_t margin = 1;

  for (size_t i = margin; i < img.rows - margin; ++i) {
    T* ptr = img.data + i * elemStep;
    for (size_t j = margin; j < img.cols - margin; ++j, ++ptr) {
      T& curr = ptr[margin];
      if (curr != 0) {
        continue;
      }

      T& north = ptr[margin - elemStep];
      T& south = ptr[margin + elemStep];
      T&  east = ptr[margin + 1];
      T&  west = ptr[margin - 1];

      ushort  sumNonZeroAdjs = 0;
      uchar countNonZeroAdjs = 0;
      if (north != 0) {
        sumNonZeroAdjs += north;
        countNonZeroAdjs++;
      }
      if (south != 0) {
        sumNonZeroAdjs += south;
        countNonZeroAdjs++;
      }
      if (east != 0) {
        sumNonZeroAdjs += east;
        countNonZeroAdjs++;
      }
      if (west != 0) {
        sumNonZeroAdjs += west;
        countNonZeroAdjs++;
      }
      if (countNonZeroAdjs > 0) {
        curr = sumNonZeroAdjs / countNonZeroAdjs;
      }
    }
  }
}
Tags:

2

Answers


  1. Chosen as BEST ANSWER
    0

    @QuangHoang advised a fantastic kernel-based approach. Unfortunately, the result is not matching with the expected output.

    Therefore, based on various comments from @CrisLuengo, I managed to design time efficient version of the filter, as shown below:

    void inPlaceHoleFillingExceptBorderFast(cv::Mat& img) {
  typedef uchar T;
  const size_t elemStep = img.step1();
  const size_t margin = 1;

  for (size_t i = margin; i < img.rows - margin; ++i) {
    T* ptr = img.data + i * elemStep;
    for (size_t j = margin; j < img.cols - margin; ++j, ++ptr) {
      T& curr = ptr[margin];
      if (curr != 0) {
        continue;
      }

      T& east  = ptr[margin + 1];
      T& west  = ptr[margin - 1];
      T& north = ptr[margin - elemStep];
      T& south = ptr[margin + elemStep];

      // convert to 0/1 and sum them up
      uchar count = static_cast<bool>(north) + static_cast<bool>(south) +
                    static_cast<bool>(east)  + static_cast<bool>(west);

      // we do not want to divide by 0
      if (count > 0) {
        curr = (north + south + east + west) / count;
      }
    }
  }
}

  2. 0

    There are three parts: 1) find the zeros, 2) find the mean, and 3) fill the found zeros with mean. So:

    /****
 * in-place fill zeros with the mean of the surrounding neighborhoods
 ***/
void fillHoles(Mat gray){    
    // find the zeros
    Mat mask = (gray == 0);
    
    // find the mean with filter2d
    Mat kernel = (Mat_<double>(3,3) << 
    1/8, 1/8, 1/8
    1/8, 0  , 1/8
    1/8, 1/8, 1/8
    );
    Mat avg;
    cv::filter2d(gray, avg, CV_8U, kernel)
    
    // then fill the zeros, only where indicated by `mask`
    cv::bitwise_or(gray, avg, gray, mask);

}

    Note I just realize that this is plainly taking the average, not the non-zero average. For that operation, you might want to do two filters, one for the sum, one for the non-zero counts, then divide the two:

    // find the neighborhood sum with filter2d
Mat kernel = (Mat_<double>(3,3) << 
1, 1, 1
1, 0, 1
1, 1, 1
);
Mat sums;
cv::filter2d(gray, sums, CV_64F, kernel);

// find the neighborhood count with filter2d
Mat counts;
cv::filter2d(gray!=0, counts, CV_64F, kernel);    
counts /= 255; // because gray!=0 returns 255 where true

// force counts to 1 if 0, so we can divide later
cv::max(counts, 1, counts);

Mat out;
cv::divide(sums, counts, out);

out.convertTo(gray, CV_8U);
    Login or Signup to reply.
Please signup or login to give your own answer.

Back To Top
Search