Medical Imaging Interaction Toolkit  2018.4.99-3e3f1a6e
Medical Imaging Interaction Toolkit
mitkGrabCutOpenCVImageFilter.cpp
Go to the documentation of this file.
1 /*============================================================================
2 
3 The Medical Imaging Interaction Toolkit (MITK)
4 
5 Copyright (c) German Cancer Research Center (DKFZ)
6 All rights reserved.
7 
8 Use of this source code is governed by a 3-clause BSD license that can be
9 found in the LICENSE file.
10 
11 ============================================================================*/
12 
13 // mitk headers
15 #include "mitkPointSet.h"
16 
17 // itk headers
18 #include "itkMultiThreader.h"
19 #include "itkFastMutexLock.h"
20 #include "itkConditionVariable.h"
21 
22 #include <opencv2/imgproc.hpp>
23 
24 // This is a magic number defined in "grabcut.cpp" of OpenCV.
25 // GrabCut function crashes if less than this number of model
26 // points are given. There must be at least as much model points
27 // as components of the Gaussian Mixture Model.
28 #define GMM_COMPONENTS_COUNT 5
29 
31  : m_ModelPointsDilationSize(0),
32  m_UseOnlyRegionAroundModelPoints(false),
33  m_CurrentProcessImageNum(0),
34  m_InputImageId(AbstractOpenCVImageFilter::INVALID_IMAGE_ID),
35  m_ResultImageId(AbstractOpenCVImageFilter::INVALID_IMAGE_ID),
36  m_ThreadId(-1), m_StopThread(false),
37  m_MultiThreader(itk::MultiThreader::New()),
38  m_WorkerBarrier(itk::ConditionVariable::New()),
39  m_ImageMutex(itk::FastMutexLock::New()),
40  m_ResultMutex(itk::FastMutexLock::New()),
41  m_PointSetsMutex(itk::FastMutexLock::New())
42 {
43  m_ThreadId = m_MultiThreader->SpawnThread(this->SegmentationWorker, this);
44 }
45 
47 {
48  // terminate worker thread on destruction
49  m_StopThread = true;
50  m_WorkerBarrier->Broadcast();
51  if ( m_ThreadId >= 0) { m_MultiThreader->TerminateThread(m_ThreadId); }
52 }
53 
55 {
56  if ( image.empty() )
57  {
58  MITK_WARN << "Filtering empty image?";
59  return false;
60  }
61 
62  // make sure that the image is an rgb image as needed
63  // by the GrabCut algorithm
64  if (image.type() != CV_8UC3)
65  {
66  cv::Mat tmp = image.clone();
67  cv::cvtColor(tmp, image, CV_GRAY2RGB);
68  }
69 
70  // set image as the current input image, guarded by
71  // a mutex as the worker thread reads this imagr
72  m_ImageMutex->Lock();
73  m_InputImage = image.clone();
75  m_ImageMutex->Unlock();
76 
77  // wake up the worker thread if there was an image set
78  // and foreground model points are available
79  if ( ! m_ForegroundPoints.empty()) { m_WorkerBarrier->Broadcast(); }
80 
81  return true;
82 }
83 
85 {
86  m_PointSetsMutex->Lock();
87  m_ForegroundPoints = foregroundPoints;
88  m_PointSetsMutex->Unlock();
89 }
90 
92 {
93  m_PointSetsMutex->Lock();
94  m_BackgroundPoints = backgroundPoints;
95  m_ForegroundPoints = foregroundPoints;
96  m_PointSetsMutex->Unlock();
97 }
98 
100 {
101  m_PointSetsMutex->Lock();
102  m_ForegroundPoints = this->ConvertMaskToModelPointsList(foregroundMask);
103  m_PointSetsMutex->Unlock();
104 }
105 
106 void mitk::GrabCutOpenCVImageFilter::SetModelPoints(cv::Mat foregroundMask, cv::Mat backgroundMask)
107 {
108  m_PointSetsMutex->Lock();
109  m_ForegroundPoints = this->ConvertMaskToModelPointsList(foregroundMask);
110  m_BackgroundPoints = this->ConvertMaskToModelPointsList(backgroundMask);
111  m_PointSetsMutex->Unlock();
112 }
113 
115 {
116  if ( modelPointsDilationSize < 0 )
117  {
118  MITK_ERROR("AbstractOpenCVImageFilter")("GrabCutOpenCVImageFilter")
119  << "Model points dilation size must not be smaller then zero.";
120  mitkThrow() << "Model points dilation size must not be smaller then zero.";
121  }
122 
123  m_ModelPointsDilationSize = modelPointsDilationSize;
124 }
125 
127 {
129  m_AdditionalWidth = additionalWidth;
130 }
131 
133 {
135 }
136 
138 {
139  return m_BoundingBox;
140 }
141 
143 {
144  return m_ResultImageId;
145 }
146 
148 {
149  cv::Mat result;
150 
151  m_ResultMutex->Lock();
152  result = m_ResultMask.clone();
153  m_ResultMutex->Unlock();
154 
155  return result;
156 }
157 
158 std::vector<mitk::GrabCutOpenCVImageFilter::ModelPointsList> mitk::GrabCutOpenCVImageFilter::GetResultContours()
159 {
160  std::vector<std::vector<cv::Point> > cvContours;
161  std::vector<cv::Vec4i> hierarchy;
162  std::vector<mitk::GrabCutOpenCVImageFilter::ModelPointsList> contourPoints;
163 
164  cv::Mat resultMask = this->GetResultMask();
165  if (resultMask.empty()) { return contourPoints; }
166 
167  cv::findContours(resultMask, cvContours, hierarchy, cv::RETR_LIST, cv::CHAIN_APPROX_NONE);
168 
169  // convert cvContours to vector of ModelPointsLists
170  for ( unsigned int i = 0; i < cvContours.size(); ++i )
171  {
173 
174  for ( auto it = cvContours[i].begin();
175  it != cvContours[i].end(); ++it)
176  {
177  itk::Index<2> index;
178  index.SetElement(0, it->x);
179  index.SetElement(1, it->y);
180  curContourPoints.push_back(index);
181  }
182 
183  contourPoints.push_back(curContourPoints);
184  }
185 
186  return contourPoints;
187 }
188 
190 {
191  cv::Mat mask = this->GetResultMask();
192  if (mask.empty()) { return mitk::GrabCutOpenCVImageFilter::ModelPointsList(); }
193 
194  // return empty model point list if given pixel is outside the image borders
195  if (pixelIndex.GetElement(0) < 0 || pixelIndex.GetElement(0) >= mask.size().height
196  || pixelIndex.GetElement(1) < 0 || pixelIndex.GetElement(1) >= mask.size().width)
197  {
198  MITK_WARN("AbstractOpenCVImageFilter")("GrabCutOpenCVImageFilter")
199  << "Given pixel index ("<< pixelIndex.GetElement(0) << ", " << pixelIndex.GetElement(1)
200  << ") is outside the image (" << mask.size().height << ", " << mask.size().width << ").";
201 
203  }
204 
205  // create a mask where the segmentation around the given pixel index is
206  // set (done by flood filling the result mask using the pixel as seed)
207  cv::floodFill(mask, cv::Point(pixelIndex.GetElement(0), pixelIndex.GetElement(1)), 5);
208 
209  cv::Mat foregroundMask;
210  cv::compare(mask, 5, foregroundMask, cv::CMP_EQ);
211 
212  // find the contour on the flood filled image (there can be only one now)
213  std::vector<std::vector<cv::Point> > cvContours;
214  std::vector<cv::Vec4i> hierarchy;
215  cv::findContours(foregroundMask, cvContours, hierarchy, cv::RETR_LIST, cv::CHAIN_APPROX_NONE);
216 
217  ModelPointsList contourPoints;
218 
219  // convert cvContours to ModelPointsList
220  for ( auto it = cvContours[0].begin();
221  it != cvContours[0].end(); ++it)
222  {
223  itk::Index<2> index;
224  index.SetElement(0, it->x);
225  index.SetElement(1, it->y);
226  contourPoints.push_back(index);
227  }
228 
229  return contourPoints;
230 }
231 
233 {
234  // initialize mask with values of propably background
235  cv::Mat mask(m_InputImage.size().height, m_InputImage.size().width, CV_8UC1, cv::GC_PR_BGD);
236 
237  // get foreground and background points (guarded by mutex)
238  m_PointSetsMutex->Lock();
240  m_PointSetsMutex->Unlock();
241 
242  // define values for foreground and background pixels
243  unsigned int pixelValues[2] = {cv::GC_FGD, cv::GC_BGD};
244 
245  for (unsigned int n = 0; n < 2; ++n)
246  {
247  for (auto it = pointsLists[n].begin();
248  it != pointsLists[n].end(); ++it)
249  {
250  // set pixels around current pixel to the same value (size of this
251  // area is specified by ModelPointsDilationSize)
252  for ( int i = -m_ModelPointsDilationSize; i <= m_ModelPointsDilationSize; ++i )
253  {
254  for ( int j = -m_ModelPointsDilationSize; j <= m_ModelPointsDilationSize; ++j)
255  {
256  int x = it->GetElement(1) + i; int y = it->GetElement(0) + j;
257  if ( x >= 0 && y >= 0 && x < mask.cols && y < mask.rows)
258  {
259  mask.at<unsigned char>(x, y) = pixelValues[n];
260  }
261  }
262  }
263  }
264  }
265 
266  return mask;
267 }
268 
270 {
271  cv::Mat nonPropablyBackgroundMask, modelPoints;
272  cv::compare(mask, cv::GC_PR_BGD, nonPropablyBackgroundMask, cv::CMP_NE);
273  cv::findNonZero(nonPropablyBackgroundMask, modelPoints);
274 
275  if (modelPoints.empty())
276  {
277  MITK_WARN("AbstractOpenCVImageFilter")("GrabCutOpenCVImageFilter")
278  << "Cannot find any foreground points. Returning full image size as bounding rectangle.";
279  return cv::Rect(0, 0, mask.rows, mask.cols);
280  }
281 
282  // calculate bounding rect around the model points
283  cv::Rect boundingRect = cv::boundingRect(modelPoints);
284 
285  // substract additional width to x and y value (and make sure that they aren't outside the image then)
286  boundingRect.x = static_cast<unsigned int>(boundingRect.x) > m_AdditionalWidth ? boundingRect.x - m_AdditionalWidth : 0;
287  boundingRect.y = static_cast<unsigned int>(boundingRect.y) > m_AdditionalWidth ? boundingRect.y - m_AdditionalWidth : 0;
288 
289  // add additional width to width of bounding rect (twice as x value was moved before)
290  // and make sure that the bounding rect will stay inside the image borders)
291  if ( static_cast<unsigned int>(boundingRect.x + boundingRect.width)
292  + 2 * m_AdditionalWidth < static_cast<unsigned int>(mask.size().width) )
293  {
294  boundingRect.width += 2 * m_AdditionalWidth;
295  }
296  else
297  {
298  boundingRect.width = mask.size().width - boundingRect.x - 1;
299  }
300 
301  // add additional width to height of bounding rect (twice as y value was moved before)
302  // and make sure that the bounding rect will stay inside the image borders)
303  if ( static_cast<unsigned int>(boundingRect.y + boundingRect.height)
304  + 2 * m_AdditionalWidth < static_cast<unsigned int>(mask.size().height) )
305  {
306  boundingRect.height += 2 * m_AdditionalWidth;
307  }
308  else
309  {
310  boundingRect.height = mask.size().height - boundingRect.y - 1;
311  }
312 
313  assert(boundingRect.x + boundingRect.width < mask.size().width);
314  assert(boundingRect.y + boundingRect.height < mask.size().height);
315 
316  return boundingRect;
317 }
318 
320 {
321  // test if foreground and background models are large enough for GrabCut
322  cv::Mat compareFgResult, compareBgResult;
323  cv::compare(mask, cv::GC_FGD, compareFgResult, cv::CMP_EQ);
324  cv::compare(mask, cv::GC_PR_BGD, compareBgResult, cv::CMP_EQ);
325  if ( cv::countNonZero(compareFgResult) < GMM_COMPONENTS_COUNT
326  || cv::countNonZero(compareBgResult) < GMM_COMPONENTS_COUNT)
327  {
328  // return result mask with no pixels set to foreground
329  return cv::Mat::zeros(mask.size(), mask.type());
330  }
331 
332  // do the actual grab cut segmentation (initialized with the mask)
333  cv::Mat bgdModel, fgdModel;
334  cv::grabCut(input, mask, cv::Rect(), bgdModel, fgdModel, 1, cv::GC_INIT_WITH_MASK);
335 
336  // set propably foreground pixels to white on result mask
337  cv::Mat result;
338  cv::compare(mask, cv::GC_PR_FGD, result, cv::CMP_EQ);
339 
340  // set foreground pixels to white on result mask
341  cv::Mat foregroundMat;
342  cv::compare(mask, cv::GC_FGD, foregroundMat, cv::CMP_EQ);
343  foregroundMat.copyTo(result, foregroundMat);
344 
345  return result; // now the result mask can be returned
346 }
347 
349 {
350  cv::Mat points;
351  cv::findNonZero(mask, points);
352 
353  // push extracted points into a vector of itk indices
354  ModelPointsList pointsVector;
355  for ( size_t n = 0; n < points.total(); ++n)
356  {
357  itk::Index<2> index;
358  index.SetElement(0, points.at<cv::Point>(n).x);
359  index.SetElement(1, points.at<cv::Point>(n).y);
360  pointsVector.push_back(index);
361  }
362 
363  return pointsVector;
364 }
365 
366 ITK_THREAD_RETURN_TYPE mitk::GrabCutOpenCVImageFilter::SegmentationWorker(void* pInfoStruct)
367 {
368  // extract this pointer from thread info structure
369  struct itk::MultiThreader::ThreadInfoStruct * pInfo = (struct itk::MultiThreader::ThreadInfoStruct*)pInfoStruct;
370  mitk::GrabCutOpenCVImageFilter* thisObject = static_cast<mitk::GrabCutOpenCVImageFilter*>(pInfo->UserData);
371 
372  itk::SimpleMutexLock mutex;
373  mutex.Lock();
374 
375  while (true)
376  {
377  if (thisObject->m_StopThread) { break; }
378 
379  thisObject->m_WorkerBarrier->Wait(&mutex);
380 
381  if (thisObject->m_StopThread) { break; }
382 
383  thisObject->m_ImageMutex->Lock();
384  cv::Mat image = thisObject->m_InputImage.clone();
385  int inputImageId = thisObject->m_InputImageId;
386  thisObject->m_ImageMutex->Unlock();
387 
388  cv::Mat mask = thisObject->GetMaskFromPointSets();
389 
390  cv::Mat result;
391  if (thisObject->m_UseOnlyRegionAroundModelPoints)
392  {
393  result = cv::Mat(mask.rows, mask.cols, mask.type(), 0.0);
394  thisObject->m_BoundingBox = thisObject->GetBoundingRectFromMask(mask);
395  thisObject->RunSegmentation(image(thisObject->m_BoundingBox), mask(thisObject->m_BoundingBox)).copyTo(result(thisObject->m_BoundingBox));
396  }
397  else
398  {
399  result = thisObject->RunSegmentation(image, mask);
400  }
401 
402  // save result to member attribute
403  thisObject->m_ResultMutex->Lock();
404  thisObject->m_ResultMask = result;
405  thisObject->m_ResultImageId = inputImageId;
406  thisObject->m_ResultMutex->Unlock();
407  }
408 
409  mutex.Unlock();
410 
411  return ITK_THREAD_RETURN_VALUE;
412 }
std::vector< itk::Index< 2 > > ModelPointsList
List holding image indices of the model points.
cv::Mat RunSegmentation(cv::Mat input, cv::Mat mask)
Performs a GrabCut segmentation of the given input image.
ModelPointsList ConvertMaskToModelPointsList(cv::Mat mask)
Creates a list of points from every non-zero pixel of the given mask.
int m_ResultImageId
id of the image which segmentation result is currently present in m_ResultMask
std::vector< ModelPointsList > GetResultContours()
Getter for the contours of the current segmentation.
cv::Rect GetBoundingRectFromMask(cv::Mat mask)
Creates a bounding box around all pixels which aren&#39;t propably background. The bounding box is widene...
#define MITK_ERROR
Definition: mitkLogMacros.h:20
bool OnFilterImage(cv::Mat &image) override
Implementation of the virtual image filtering method. The input image is copied to a member attribute...
ModelPointsList GetResultContourWithPixel(itk::Index< 2 > pixelIndex)
Getter for one specific contour of the current segmentation.
void SetUseFullImage()
The full image is used as input for the segmentation. This method sets the behaviour back to the defa...
cv::Mat GetMaskFromPointSets()
Creates an image mask for GrabCut algorithm by using the foreground and background point sets...
void SetModelPoints(ModelPointsList foregroundPoints)
Sets a list of image indices as foreground model points.
#define MITK_WARN
Definition: mitkLogMacros.h:19
Makes the OpenCV GrabCut filter available as OpenCVImageFilter.
bool compare(std::pair< double, int > i, std::pair< double, int > j)
#define GMM_COMPONENTS_COUNT
#define mitkThrow()
cv::Rect GetRegionAroundModelPoints()
Getter for the rectangle used for the area of segmentation. See mitk::GrabCutOpenCVImageFilter::SetUs...
void SetModelPointsDilationSize(int modelPointsDilationSize)
Set a size of which each model point is dilated before image filtering. The more color information of...
mitk::Image::Pointer image
Interface for image filters on OpenCV images.
void SetUseOnlyRegionAroundModelPoints(unsigned int additionalBorder)
Use only the region around the foreground model points for the segmentation.
cv::Mat GetResultMask()
Getter for the result mask of the current segmentation. The result of this method is not necessarily ...
mitk::Image::Pointer mask
int m_InputImageId
id of the image currently set as m_InputImage
int GetResultImageId()
Getter for an ascending id of the current result image. The id will be increased for every segmentati...