Medical Imaging Interaction Toolkit  2016.11.0
Medical Imaging Interaction Toolkit
mitkGrabCutOpenCVImageFilter.cpp
Go to the documentation of this file.
1 /*===================================================================
2 
3 The Medical Imaging Interaction Toolkit (MITK)
4 
5 Copyright (c) German Cancer Research Center,
6 Division of Medical and Biological Informatics.
7 All rights reserved.
8 
9 This software is distributed WITHOUT ANY WARRANTY; without
10 even the implied warranty of MERCHANTABILITY or FITNESS FOR
11 A PARTICULAR PURPOSE.
12 
13 See LICENSE.txt or http://www.mitk.org for details.
14 
15 ===================================================================*/
16 
17 // mitk headers
19 #include "mitkPointSet.h"
20 
21 // itk headers
22 #include "itkMultiThreader.h"
23 #include "itkFastMutexLock.h"
24 #include "itkConditionVariable.h"
25 
26 // This is a magic number defined in "grabcut.cpp" of OpenCV.
27 // GrabCut function crashes if less than this number of model
28 // points are given. There must be at least as much model points
29 // as components of the Gaussian Mixture Model.
30 #define GMM_COMPONENTS_COUNT 5
31 
33  : m_ModelPointsDilationSize(0),
34  m_UseOnlyRegionAroundModelPoints(false),
35  m_CurrentProcessImageNum(0),
36  m_InputImageId(AbstractOpenCVImageFilter::INVALID_IMAGE_ID),
37  m_ResultImageId(AbstractOpenCVImageFilter::INVALID_IMAGE_ID),
38  m_ThreadId(-1), m_StopThread(false),
39  m_MultiThreader(itk::MultiThreader::New()),
40  m_WorkerBarrier(itk::ConditionVariable::New()),
41  m_ImageMutex(itk::FastMutexLock::New()),
42  m_ResultMutex(itk::FastMutexLock::New()),
43  m_PointSetsMutex(itk::FastMutexLock::New())
44 {
45  m_ThreadId = m_MultiThreader->SpawnThread(this->SegmentationWorker, this);
46 }
47 
49 {
50  // terminate worker thread on destruction
51  m_StopThread = true;
52  m_WorkerBarrier->Broadcast();
53  if ( m_ThreadId >= 0) { m_MultiThreader->TerminateThread(m_ThreadId); }
54 }
55 
57 {
58  if ( image.empty() )
59  {
60  MITK_WARN << "Filtering empty image?";
61  return false;
62  }
63 
64  // make sure that the image is an rgb image as needed
65  // by the GrabCut algorithm
66  if (image.type() != CV_8UC3)
67  {
68  cv::Mat tmp = image.clone();
69  cv::cvtColor(tmp, image, CV_GRAY2RGB);
70  }
71 
72  // set image as the current input image, guarded by
73  // a mutex as the worker thread reads this imagr
74  m_ImageMutex->Lock();
75  m_InputImage = image.clone();
76  m_InputImageId = this->GetCurrentImageId();
77  m_ImageMutex->Unlock();
78 
79  // wake up the worker thread if there was an image set
80  // and foreground model points are available
81  if ( ! m_ForegroundPoints.empty()) { m_WorkerBarrier->Broadcast(); }
82 
83  return true;
84 }
85 
87 {
88  m_PointSetsMutex->Lock();
89  m_ForegroundPoints = foregroundPoints;
90  m_PointSetsMutex->Unlock();
91 }
92 
94 {
95  m_PointSetsMutex->Lock();
96  m_BackgroundPoints = backgroundPoints;
97  m_ForegroundPoints = foregroundPoints;
98  m_PointSetsMutex->Unlock();
99 }
100 
102 {
103  m_PointSetsMutex->Lock();
104  m_ForegroundPoints = this->ConvertMaskToModelPointsList(foregroundMask);
105  m_PointSetsMutex->Unlock();
106 }
107 
108 void mitk::GrabCutOpenCVImageFilter::SetModelPoints(cv::Mat foregroundMask, cv::Mat backgroundMask)
109 {
110  m_PointSetsMutex->Lock();
111  m_ForegroundPoints = this->ConvertMaskToModelPointsList(foregroundMask);
112  m_BackgroundPoints = this->ConvertMaskToModelPointsList(backgroundMask);
113  m_PointSetsMutex->Unlock();
114 }
115 
117 {
118  if ( modelPointsDilationSize < 0 )
119  {
120  MITK_ERROR("AbstractOpenCVImageFilter")("GrabCutOpenCVImageFilter")
121  << "Model points dilation size must not be smaller then zero.";
122  mitkThrow() << "Model points dilation size must not be smaller then zero.";
123  }
124 
125  m_ModelPointsDilationSize = modelPointsDilationSize;
126 }
127 
129 {
130  m_UseOnlyRegionAroundModelPoints = true;
131  m_AdditionalWidth = additionalWidth;
132 }
133 
135 {
136  m_UseOnlyRegionAroundModelPoints = false;
137 }
138 
140 {
141  return m_BoundingBox;
142 }
143 
145 {
146  return m_ResultImageId;
147 }
148 
150 {
151  cv::Mat result;
152 
153  m_ResultMutex->Lock();
154  result = m_ResultMask.clone();
155  m_ResultMutex->Unlock();
156 
157  return result;
158 }
159 
160 std::vector<mitk::GrabCutOpenCVImageFilter::ModelPointsList> mitk::GrabCutOpenCVImageFilter::GetResultContours()
161 {
162  std::vector<std::vector<cv::Point> > cvContours;
163  std::vector<cv::Vec4i> hierarchy;
164  std::vector<mitk::GrabCutOpenCVImageFilter::ModelPointsList> contourPoints;
165 
166  cv::Mat resultMask = this->GetResultMask();
167  if (resultMask.empty()) { return contourPoints; }
168 
169  cv::findContours(resultMask, cvContours, hierarchy, cv::RETR_LIST, cv::CHAIN_APPROX_NONE);
170 
171  // convert cvContours to vector of ModelPointsLists
172  for ( unsigned int i = 0; i < cvContours.size(); ++i )
173  {
175 
176  for ( auto it = cvContours[i].begin();
177  it != cvContours[i].end(); ++it)
178  {
179  itk::Index<2> index;
180  index.SetElement(0, it->x);
181  index.SetElement(1, it->y);
182  curContourPoints.push_back(index);
183  }
184 
185  contourPoints.push_back(curContourPoints);
186  }
187 
188  return contourPoints;
189 }
190 
192 {
193  cv::Mat mask = this->GetResultMask();
194  if (mask.empty()) { return mitk::GrabCutOpenCVImageFilter::ModelPointsList(); }
195 
196  // return empty model point list if given pixel is outside the image borders
197  if (pixelIndex.GetElement(0) < 0 || pixelIndex.GetElement(0) >= mask.size().height
198  || pixelIndex.GetElement(1) < 0 || pixelIndex.GetElement(1) >= mask.size().width)
199  {
200  MITK_WARN("AbstractOpenCVImageFilter")("GrabCutOpenCVImageFilter")
201  << "Given pixel index ("<< pixelIndex.GetElement(0) << ", " << pixelIndex.GetElement(1)
202  << ") is outside the image (" << mask.size().height << ", " << mask.size().width << ").";
203 
205  }
206 
207  // create a mask where the segmentation around the given pixel index is
208  // set (done by flood filling the result mask using the pixel as seed)
209  cv::floodFill(mask, cv::Point(pixelIndex.GetElement(0), pixelIndex.GetElement(1)), 5);
210 
211  cv::Mat foregroundMask;
212  cv::compare(mask, 5, foregroundMask, cv::CMP_EQ);
213 
214  // find the contour on the flood filled image (there can be only one now)
215  std::vector<std::vector<cv::Point> > cvContours;
216  std::vector<cv::Vec4i> hierarchy;
217  cv::findContours(foregroundMask, cvContours, hierarchy, cv::RETR_LIST, cv::CHAIN_APPROX_NONE);
218 
219  ModelPointsList contourPoints;
220 
221  // convert cvContours to ModelPointsList
222  for ( auto it = cvContours[0].begin();
223  it != cvContours[0].end(); ++it)
224  {
225  itk::Index<2> index;
226  index.SetElement(0, it->x);
227  index.SetElement(1, it->y);
228  contourPoints.push_back(index);
229  }
230 
231  return contourPoints;
232 }
233 
235 {
236  // initialize mask with values of propably background
237  cv::Mat mask(m_InputImage.size().height, m_InputImage.size().width, CV_8UC1, cv::GC_PR_BGD);
238 
239  // get foreground and background points (guarded by mutex)
240  m_PointSetsMutex->Lock();
241  ModelPointsList pointsLists[2] = {ModelPointsList(m_ForegroundPoints), ModelPointsList(m_BackgroundPoints)};
242  m_PointSetsMutex->Unlock();
243 
244  // define values for foreground and background pixels
245  unsigned int pixelValues[2] = {cv::GC_FGD, cv::GC_BGD};
246 
247  for (unsigned int n = 0; n < 2; ++n)
248  {
249  for (auto it = pointsLists[n].begin();
250  it != pointsLists[n].end(); ++it)
251  {
252  // set pixels around current pixel to the same value (size of this
253  // area is specified by ModelPointsDilationSize)
254  for ( int i = -m_ModelPointsDilationSize; i <= m_ModelPointsDilationSize; ++i )
255  {
256  for ( int j = -m_ModelPointsDilationSize; j <= m_ModelPointsDilationSize; ++j)
257  {
258  int x = it->GetElement(1) + i; int y = it->GetElement(0) + j;
259  if ( x >= 0 && y >= 0 && x < mask.cols && y < mask.rows)
260  {
261  mask.at<unsigned char>(x, y) = pixelValues[n];
262  }
263  }
264  }
265  }
266  }
267 
268  return mask;
269 }
270 
272 {
273  cv::Mat nonPropablyBackgroundMask, modelPoints;
274  cv::compare(mask, cv::GC_PR_BGD, nonPropablyBackgroundMask, cv::CMP_NE);
275  cv::findNonZero(nonPropablyBackgroundMask, modelPoints);
276 
277  if (modelPoints.empty())
278  {
279  MITK_WARN("AbstractOpenCVImageFilter")("GrabCutOpenCVImageFilter")
280  << "Cannot find any foreground points. Returning full image size as bounding rectangle.";
281  return cv::Rect(0, 0, mask.rows, mask.cols);
282  }
283 
284  // calculate bounding rect around the model points
285  cv::Rect boundingRect = cv::boundingRect(modelPoints);
286 
287  // substract additional width to x and y value (and make sure that they aren't outside the image then)
288  boundingRect.x = static_cast<unsigned int>(boundingRect.x) > m_AdditionalWidth ? boundingRect.x - m_AdditionalWidth : 0;
289  boundingRect.y = static_cast<unsigned int>(boundingRect.y) > m_AdditionalWidth ? boundingRect.y - m_AdditionalWidth : 0;
290 
291  // add additional width to width of bounding rect (twice as x value was moved before)
292  // and make sure that the bounding rect will stay inside the image borders)
293  if ( static_cast<unsigned int>(boundingRect.x + boundingRect.width)
294  + 2 * m_AdditionalWidth < static_cast<unsigned int>(mask.size().width) )
295  {
296  boundingRect.width += 2 * m_AdditionalWidth;
297  }
298  else
299  {
300  boundingRect.width = mask.size().width - boundingRect.x - 1;
301  }
302 
303  // add additional width to height of bounding rect (twice as y value was moved before)
304  // and make sure that the bounding rect will stay inside the image borders)
305  if ( static_cast<unsigned int>(boundingRect.y + boundingRect.height)
306  + 2 * m_AdditionalWidth < static_cast<unsigned int>(mask.size().height) )
307  {
308  boundingRect.height += 2 * m_AdditionalWidth;
309  }
310  else
311  {
312  boundingRect.height = mask.size().height - boundingRect.y - 1;
313  }
314 
315  assert(boundingRect.x + boundingRect.width < mask.size().width);
316  assert(boundingRect.y + boundingRect.height < mask.size().height);
317 
318  return boundingRect;
319 }
320 
321 cv::Mat mitk::GrabCutOpenCVImageFilter::RunSegmentation(cv::Mat input, cv::Mat mask)
322 {
323  // test if foreground and background models are large enough for GrabCut
324  cv::Mat compareFgResult, compareBgResult;
325  cv::compare(mask, cv::GC_FGD, compareFgResult, cv::CMP_EQ);
326  cv::compare(mask, cv::GC_PR_BGD, compareBgResult, cv::CMP_EQ);
327  if ( cv::countNonZero(compareFgResult) < GMM_COMPONENTS_COUNT
328  || cv::countNonZero(compareBgResult) < GMM_COMPONENTS_COUNT)
329  {
330  // return result mask with no pixels set to foreground
331  return cv::Mat::zeros(mask.size(), mask.type());
332  }
333 
334  // do the actual grab cut segmentation (initialized with the mask)
335  cv::Mat bgdModel, fgdModel;
336  cv::grabCut(input, mask, cv::Rect(), bgdModel, fgdModel, 1, cv::GC_INIT_WITH_MASK);
337 
338  // set propably foreground pixels to white on result mask
339  cv::Mat result;
340  cv::compare(mask, cv::GC_PR_FGD, result, cv::CMP_EQ);
341 
342  // set foreground pixels to white on result mask
343  cv::Mat foregroundMat;
344  cv::compare(mask, cv::GC_FGD, foregroundMat, cv::CMP_EQ);
345  foregroundMat.copyTo(result, foregroundMat);
346 
347  return result; // now the result mask can be returned
348 }
349 
351 {
352  cv::Mat points;
353  cv::findNonZero(mask, points);
354 
355  // push extracted points into a vector of itk indices
356  ModelPointsList pointsVector;
357  for ( size_t n = 0; n < points.total(); ++n)
358  {
359  itk::Index<2> index;
360  index.SetElement(0, points.at<cv::Point>(n).x);
361  index.SetElement(1, points.at<cv::Point>(n).y);
362  pointsVector.push_back(index);
363  }
364 
365  return pointsVector;
366 }
367 
368 ITK_THREAD_RETURN_TYPE mitk::GrabCutOpenCVImageFilter::SegmentationWorker(void* pInfoStruct)
369 {
370  // extract this pointer from thread info structure
371  struct itk::MultiThreader::ThreadInfoStruct * pInfo = (struct itk::MultiThreader::ThreadInfoStruct*)pInfoStruct;
372  mitk::GrabCutOpenCVImageFilter* thisObject = static_cast<mitk::GrabCutOpenCVImageFilter*>(pInfo->UserData);
373 
374  itk::SimpleMutexLock mutex;
375  mutex.Lock();
376 
377  while (true)
378  {
379  if (thisObject->m_StopThread) { break; }
380 
381  thisObject->m_WorkerBarrier->Wait(&mutex);
382 
383  if (thisObject->m_StopThread) { break; }
384 
385  thisObject->m_ImageMutex->Lock();
386  cv::Mat image = thisObject->m_InputImage.clone();
387  int inputImageId = thisObject->m_InputImageId;
388  thisObject->m_ImageMutex->Unlock();
389 
390  cv::Mat mask = thisObject->GetMaskFromPointSets();
391 
392  cv::Mat result;
393  if (thisObject->m_UseOnlyRegionAroundModelPoints)
394  {
395  result = cv::Mat(mask.rows, mask.cols, mask.type(), 0.0);
396  thisObject->m_BoundingBox = thisObject->GetBoundingRectFromMask(mask);
397  thisObject->RunSegmentation(image(thisObject->m_BoundingBox), mask(thisObject->m_BoundingBox)).copyTo(result(thisObject->m_BoundingBox));
398  }
399  else
400  {
401  result = thisObject->RunSegmentation(image, mask);
402  }
403 
404  // save result to member attribute
405  thisObject->m_ResultMutex->Lock();
406  thisObject->m_ResultMask = result;
407  thisObject->m_ResultImageId = inputImageId;
408  thisObject->m_ResultMutex->Unlock();
409  }
410 
411  mutex.Unlock();
412 
413  return ITK_THREAD_RETURN_VALUE;
414 }
std::vector< itk::Index< 2 > > ModelPointsList
List holding image indices of the model points.
cv::Mat RunSegmentation(cv::Mat input, cv::Mat mask)
Performs a GrabCut segmentation of the given input image.
ModelPointsList ConvertMaskToModelPointsList(cv::Mat mask)
Creates a list of points from every non-zero pixel of the given mask.
int m_ResultImageId
id of the image which segmentation result is currently present in m_ResultMask
std::vector< ModelPointsList > GetResultContours()
Getter for the contours of the current segmentation.
cv::Rect GetBoundingRectFromMask(cv::Mat mask)
Creates a bounding box around all pixels which aren't propably background. The bounding box is widene...
#define MITK_ERROR
Definition: mitkLogMacros.h:24
bool OnFilterImage(cv::Mat &image) override
Implementation of the virtual image filtering method. The input image is copied to a member attribute...
ModelPointsList GetResultContourWithPixel(itk::Index< 2 > pixelIndex)
Getter for one specific contour of the current segmentation.
void SetUseFullImage()
The full image is used as input for the segmentation. This method sets the behaviour back to the defa...
cv::Mat GetMaskFromPointSets()
Creates an image mask for GrabCut algorithm by using the foreground and background point sets...
void SetModelPoints(ModelPointsList foregroundPoints)
Sets a list of image indices as foreground model points.
#define MITK_WARN
Definition: mitkLogMacros.h:23
Makes the OpenCV GrabCut filter available as OpenCVImageFilter.
bool compare(std::pair< double, int > i, std::pair< double, int > j)
#define GMM_COMPONENTS_COUNT
#define mitkThrow()
cv::Rect GetRegionAroundModelPoints()
Getter for the rectangle used for the area of segmentation. See mitk::GrabCutOpenCVImageFilter::SetUs...
void SetModelPointsDilationSize(int modelPointsDilationSize)
Set a size of which each model point is dilated before image filtering. The more color information of...
Interface for image filters on OpenCV images.
void SetUseOnlyRegionAroundModelPoints(unsigned int additionalBorder)
Use only the region around the foreground model points for the segmentation.
cv::Mat GetResultMask()
Getter for the result mask of the current segmentation. The result of this method is not necessarily ...
int m_InputImageId
id of the image currently set as m_InputImage
int GetResultImageId()
Getter for an ascending id of the current result image. The id will be increased for every segmentati...
static itkEventMacro(BoundingShapeInteractionEvent, itk::AnyEvent) class MITKBOUNDINGSHAPE_EXPORT BoundingShapeInteractor Pointer New()
Basic interaction methods for mitk::GeometryData.