Mean Shift Segmentation – Revisited

Posted: September 3, 2013 at 11:52 am

In the last post I talked about a different segmentation approach rather than trying to figure out why the FloodFill() operation was using more and more time (Eventually getting to over 200 seconds per frame). A quick look at the Creativity and Cognition (C&C) version did not show any functional differences compared to the unitTest code. Of course the C&C version was crashing after about 24 hours, which likely was not enough to exhibit the problem. I took a few days to rewrite the backgroundSegmentation() function from scratch. In doing so I noticed a nice new function: meanShiftSegmentation(). The code is now much cleaner, but unfortunately it is not any faster than the old flood-fill version. This is partially because the segmentation is happening at the full 1080p resolution (not 1/4 the pixels as in the floodFill version). I should be able to get it under 1s with some optimization. The good news is that this segmenter works much better, there are no more “blind spots” as the segmenter breaks up the whole image and joins small regions into larger ones rather than leaving unsegmented areas. Here is an example of the segmentation results (in improperly mapped HSV colourspace):

segments

Following is the new segmentBackground() function:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
// Process one frame
void segmentation::segmentBackground(Mat &image, list<percepUnit> &percepUnitsBackground) {

    Mat segments, labels;
    gpu::GpuMat gpuImageRGB, gpuImageHSV, gpuClose, gpuOpen;
    double t1, t2;

    t1 = getTime();

    // Do hard image processing on the GPU: (only works on micro!)
    gpuImageRGB.upload(image);
    gpu::cvtColor(gpuImageRGB, gpuImageHSV, CV_RGB2HSV, 4); // 1 or 4 channels for gpu operation

    // Morphology
    // TODO could use less variables to save texture memory.
    Mat element = getStructuringElement(MORPH_ELLIPSE, Size(5,5), Point(2,2) ); // was 5x5 at 2,2
    gpu::morphologyEx(gpuImageHSV, gpuClose, MORPH_CLOSE, element);
    gpu::morphologyEx(gpuClose, gpuOpen, MORPH_OPEN, element);

    // Mean shift
    TermCriteria iterations = TermCriteria(CV_TERMCRIT_ITER, 2, 0);
    gpu::meanShiftSegmentation(gpuOpen, segments, 10, 20, 300, iterations);

    // convert to greyscale
    vector<Mat> channels;
    split(segments, channels);

    // get labels from histogram of image.
    int size = 256;
    labels = Mat(256, 1, CV_32SC1);
    calcHist(&channels.at(2), 1, 0, Mat(), labels, 1, &size, 0);

    // Loop through hist
    int numROIs=0;
    for (int i=0; i<256; i++) {
        // If this bin matches a label.
        if (labels.at<float>(i) > 0) {
            // find areas of the image that match this label and findConours on the result.
            //stringstream ss;
            Mat label = Mat(channels.at(2).rows, channels.at(2).cols, CV_8UC1, Scalar::all(i)); // image filled with label colour.
            Mat boolImage = (channels.at(2) == label); // which pixels in labeled image are identical to this label?
            vector<vector<Point>> labelContours;
            findContours(boolImage, labelContours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE);
            // Loop through contours.
            for (int idx = 0; idx < labelContours.size(); idx++) {
                // get bounds for this contour.
                Rect bounds = boundingRect(labelContours[idx]);
                float area = contourArea(labelContours[idx]);

                // filter strange results (new segmentation may not cause any!)
                if (bounds.width < 1920 and bounds.height < 1080 and bounds.width > 10 and bounds.height > 10) {

                    // add padding to bounding box.
                    // TODO this count be a function! (extend scaleRect?)
                    if (bounds.x-imagePadding >= 0)
                        bounds.x -= imagePadding;
                    else
                        bounds.x = 0;
                    if (bounds.y-imagePadding >= 0)
                        bounds.y -= imagePadding;
                    else
                        bounds.y = 0;

                    int scalePadding = imagePadding*2;
                    int leftEdge = bounds.width+bounds.x;
                    int bottomEdge = bounds.height+bounds.y;
                    if (leftEdge+scalePadding <= 1920)
                        bounds.width += scalePadding;
                    else
                        bounds.width += 1920-leftEdge;
                    if (bottomEdge+scalePadding <= 1080)
                        bounds.height += scalePadding;
                    else
                        bounds.height += 1080-bottomEdge;

                    // create percepUnit
                    Mat patchROI = image(bounds);
                    Mat maskROI = boolImage(bounds);

                    percepUnit thisUnit = percepUnit(patchROI, maskROI, bounds.x, bounds.y, bounds.width, bounds.height, area);
                    thisUnit.calcColour(); // only do this on segmentation!
                    thisUnit.FGBG = "BG";
                    percepUnitsBackground.push_back(thisUnit); // Append percepUnit to background percept list.

                    numROIs++;      
                }
            }
        }
    }


    t2 = getTime();

    // TODO add a debug mode.
    cout << "Module Processing Time: backgroundSegmentation: " << t2-t1 << endl;
    cout << "Total Regions: " << numROIs << endl;

    // release GpuMats
    gpuImageRGB.release();
    gpuImageHSV.release();
    gpuClose.release();
    gpuOpen.release();
}

This seems a little unwieldy to me, but its works.