Update to 2.0.0 tree from current Fremantle build
[opencv] / src / cvaux / vs / blobtrackingmsfg.cpp
1 /*M///////////////////////////////////////////////////////////////////////////////////////
2 //
3 //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
4 //
5 //  By downloading, copying, installing or using the software you agree to this license.
6 //  If you do not agree to this license, do not download, install,
7 //  copy or use the software.
8 //
9 //
10 //                        Intel License Agreement
11 //
12 // Copyright (C) 2000, Intel Corporation, all rights reserved.
13 // Third party copyrights are property of their respective owners.
14 //
15 // Redistribution and use in source and binary forms, with or without modification,
16 // are permitted provided that the following conditions are met:
17 //
18 //   * Redistribution's of source code must retain the above copyright notice,
19 //     this list of conditions and the following disclaimer.
20 //
21 //   * Redistribution's in binary form must reproduce the above copyright notice,
22 //     this list of conditions and the following disclaimer in the documentation
23 //     and/or other materials provided with the distribution.
24 //
25 //   * The name of Intel Corporation may not be used to endorse or promote products
26 //     derived from this software without specific prior written permission.
27 //
28 // This software is provided by the copyright holders and contributors "as is" and
29 // any express or implied warranties, including, but not limited to, the implied
30 // warranties of merchantability and fitness for a particular purpose are disclaimed.
31 // In no event shall the Intel Corporation or contributors be liable for any direct,
32 // indirect, incidental, special, exemplary, or consequential damages
33 // (including, but not limited to, procurement of substitute goods or services;
34 // loss of use, data, or profits; or business interruption) however caused
35 // and on any theory of liability, whether in contract, strict liability,
36 // or tort (including negligence or otherwise) arising in any way out of
37 // the use of this software, even if advised of the possibility of such damage.
38 //
39 //M*/
40
41 #include "_cvaux.h"
42
43 // Uncomment to trade flexibility for speed
44 //#define CONST_HIST_SIZE
45
46 // Uncomment to get some performance stats in stderr
47 //#define REPORT_TICKS
48
49 #ifdef CONST_HIST_SIZE
50 #define m_BinBit 5
51 #define m_ByteShift 3
52 #endif
53
54 typedef float DefHistType;
55 #define DefHistTypeMat CV_32F
56 #define HIST_INDEX(_pData) (((_pData)[0]>>m_ByteShift) + (((_pData)[1]>>(m_ByteShift))<<m_BinBit)+((pImgData[2]>>m_ByteShift)<<(m_BinBit*2)))
57
58 class DefHist
59 {
60 public:
61     CvMat*          m_pHist;
62     DefHistType     m_HistVolume;
63     DefHist(int BinNum=0)
64     {
65         m_pHist = NULL;
66         m_HistVolume = 0;
67         Resize(BinNum);
68     }
69
70     ~DefHist()
71     {
72         if(m_pHist)cvReleaseMat(&m_pHist);
73     }
74
75     void Resize(int BinNum)
76     {
77         if(m_pHist)cvReleaseMat(&m_pHist);
78         if(BinNum>0)
79         {
80             m_pHist = cvCreateMat(1, BinNum, DefHistTypeMat);
81             cvZero(m_pHist);
82         }
83         m_HistVolume = 0;
84     }
85
86     void Update(DefHist* pH, float W)
87     {   /* Update histogram: */
88         double  Vol, WM, WC;
89         Vol = 0.5*(m_HistVolume + pH->m_HistVolume);
90         WM = Vol*(1-W)/m_HistVolume;
91         WC = Vol*(W)/pH->m_HistVolume;
92         cvAddWeighted(m_pHist, WM, pH->m_pHist,WC,0,m_pHist);
93         m_HistVolume = (float)cvSum(m_pHist).val[0];
94     }   /* Update histogram: */
95 };
96
97 class CvBlobTrackerOneMSFG:public CvBlobTrackerOne
98 {
99 protected:
100     int             m_BinNumTotal; /* protected only for parralel MSPF */
101     CvSize          m_ObjSize;
102
103     void ReAllocKernel(int  w, int h)
104     {
105         int     x,y;
106         float   x0 = 0.5f*(w-1);
107         float   y0 = 0.5f*(h-1);
108         assert(w>0);
109         assert(h>0);
110         m_ObjSize = cvSize(w,h);
111
112         if(m_KernelHist) cvReleaseMat(&m_KernelHist);
113         if(m_KernelMeanShift) cvReleaseMat(&m_KernelMeanShift);
114         m_KernelHist = cvCreateMat(h, w, DefHistTypeMat);
115         m_KernelMeanShift = cvCreateMat(h, w, DefHistTypeMat);
116
117         for(y=0; y<h; ++y) for(x=0; x<w; ++x)
118         {
119             double r2 = ((x-x0)*(x-x0)/(x0*x0)+(y-y0)*(y-y0)/(y0*y0));
120 //            double r2 = ((x-x0)*(x-x0)+(y-y0)*(y-y0))/((y0*y0)+(x0*x0));
121             CV_MAT_ELEM(m_KernelHist[0],DefHistType, y, x) = (DefHistType)GetKernelHist(r2);
122             CV_MAT_ELEM(m_KernelMeanShift[0],DefHistType, y, x) = (DefHistType)GetKernelMeanShift(r2);
123         }
124     }
125
126 private:
127     /* Parameters: */
128     int             m_IterNum;
129     float           m_FGWeight;
130     float           m_Alpha;
131     CvMat*          m_KernelHist;
132     CvMat*          m_KernelMeanShift;
133 #ifndef CONST_HIST_SIZE
134     int             m_BinBit;
135     int             m_ByteShift;
136 #endif
137     int             m_BinNum;
138     int             m_Dim;
139     /*
140     CvMat*          m_HistModel;
141     float           m_HistModelVolume;
142     CvMat*          m_HistCandidate;
143     float           m_HistCandidateVolume;
144     CvMat*          m_HistTemp;
145     */
146     DefHist         m_HistModel;
147     DefHist         m_HistCandidate;
148     DefHist         m_HistTemp;
149
150     CvBlob          m_Blob;
151     int             m_Collision;
152
153     void ReAllocHist(int Dim, int BinBit)
154     {
155 #ifndef CONST_HIST_SIZE
156         m_BinBit = BinBit;
157         m_ByteShift = 8-BinBit;
158 #endif
159         m_Dim = Dim;
160         m_BinNum = (1<<BinBit);
161         m_BinNumTotal = cvRound(pow((double)m_BinNum,(double)m_Dim));
162         /*
163         if(m_HistModel) cvReleaseMat(&m_HistModel);
164         if(m_HistCandidate) cvReleaseMat(&m_HistCandidate);
165         if(m_HistTemp) cvReleaseMat(&m_HistTemp);
166         m_HistCandidate = cvCreateMat(1, m_BinNumTotal, DefHistTypeMat);
167         m_HistModel = cvCreateMat(1, m_BinNumTotal, DefHistTypeMat);
168         m_HistTemp = cvCreateMat(1, m_BinNumTotal, DefHistTypeMat);
169         cvZero(m_HistCandidate);
170         cvZero(m_HistModel);
171         m_HistModelVolume = 0.0f;
172         m_HistCandidateVolume = 0.0f;
173         */
174         m_HistCandidate.Resize(m_BinNumTotal);
175         m_HistModel.Resize(m_BinNumTotal);
176         m_HistTemp.Resize(m_BinNumTotal);
177     }
178
179     double GetKernelHist(double r2)
180     {
181         return (r2 < 1) ? 1 -  r2 : 0;
182     }
183
184     double GetKernelMeanShift(double r2)
185     {
186         return (r2<1) ? 1 : 0;
187     }
188
189 //    void CollectHist(IplImage* pImg, IplImage* pMask, CvPoint Center, CvMat* pHist, DefHistType* pHistVolume)
190 //    void CollectHist(IplImage* pImg, IplImage* pMask, CvPoint Center, DefHist* pHist)
191
192     void CollectHist(IplImage* pImg, IplImage* pMask, CvBlob* pBlob, DefHist* pHist)
193     {
194         int UsePrecalculatedKernel = 0;
195         int BW = cvRound(pBlob->w);
196         int BH = cvRound(pBlob->h);
197         DefHistType Volume = 0;
198         int         x0 = cvRound(pBlob->x - BW*0.5);
199         int         y0 = cvRound(pBlob->y - BH*0.5);
200         int         x,y;
201
202         UsePrecalculatedKernel = (BW == m_ObjSize.width && BH == m_ObjSize.height ) ;
203
204         //cvZero(pHist);
205         cvSet(pHist->m_pHist,cvScalar(1.0/m_BinNumTotal)); /* no zero bins, all bins have very small value*/
206         Volume = 1;
207
208         assert(BW < pImg->width);
209         assert(BH < pImg->height);
210         if((x0+BW)>=pImg->width) BW=pImg->width-x0-1;
211         if((y0+BH)>=pImg->height) BH=pImg->height-y0-1;
212         if(x0<0){ x0=0;}
213         if(y0<0){ y0=0;}
214
215         if(m_Dim == 3)
216         {
217             for(y=0; y<BH; ++y)
218             {
219                 unsigned char* pImgData = &CV_IMAGE_ELEM(pImg,unsigned char,y+y0,x0*3);
220                 unsigned char* pMaskData = pMask?(&CV_IMAGE_ELEM(pMask,unsigned char,y+y0,x0)):NULL;
221                 DefHistType* pKernelData = NULL;
222
223                 if(UsePrecalculatedKernel)
224                 {
225                     pKernelData = ((DefHistType*)CV_MAT_ELEM_PTR_FAST(m_KernelHist[0],y,0,sizeof(DefHistType)));
226                 }
227
228                 for(x=0; x<BW; ++x, pImgData+=3)
229                 {
230                     DefHistType K;
231                     int index = HIST_INDEX(pImgData);
232                     assert(index >= 0 && index < pHist->m_pHist->cols);
233
234                     if(UsePrecalculatedKernel)
235                     {
236                         K = pKernelData[x];
237                     }
238                     else
239                     {
240                         float dx = (x+x0-pBlob->x)/(pBlob->w*0.5f);
241                         float dy = (y+y0-pBlob->y)/(pBlob->h*0.5f);
242                         double r2 = dx*dx+dy*dy;
243                         K = (float)GetKernelHist(r2);
244                     }
245
246                     if(pMaskData)
247                     {
248                         K *= pMaskData[x]*0.003921568627450980392156862745098f;
249                     }
250                     Volume += K;
251                     ((DefHistType*)(pHist->m_pHist->data.ptr))[index] += K;
252
253                 }   /* Next column. */
254             }   /*  Next row. */
255         }   /* if m_Dim == 3. */
256
257         pHist->m_HistVolume = Volume;
258
259     };  /* CollectHist */
260
261     double calcBhattacharyya(DefHist* pHM = NULL, DefHist* pHC = NULL, DefHist* pHT = NULL)
262     {
263         if(pHM==NULL) pHM = &m_HistModel;
264         if(pHC==NULL) pHC = &m_HistCandidate;
265         if(pHT==NULL) pHT = &m_HistTemp;
266         if(pHC->m_HistVolume*pHM->m_HistVolume > 0)
267         {
268 #if 0
269             // Use CV functions:
270             cvMul(pHM->m_pHist,pHC->m_pHist,pHT->m_pHist);
271             cvPow(pHT->m_pHist,pHT->m_pHist,0.5);
272             return cvSum(pHT->m_pHist).val[0] / sqrt(pHC->m_HistVolume*pHM->m_HistVolume);
273 #else
274             // Do computations manually and let autovectorizer do the job:
275             DefHistType *hm, *hc, *ht;
276             double sum;
277             int     size;
278             hm=(DefHistType *)(pHM->m_pHist->data.ptr);
279             hc=(DefHistType *)(pHC->m_pHist->data.ptr);
280             ht=(DefHistType *)(pHT->m_pHist->data.ptr);
281             size = pHM->m_pHist->width*pHM->m_pHist->height;
282             sum = 0.;
283             for(int i = 0; i < size; i++ )
284             {
285                 sum += sqrt(hm[i]*hc[i]);
286             }
287             return sum / sqrt(pHC->m_HistVolume*pHM->m_HistVolume);
288 #endif
289         }
290         return 0;
291     }   /* calcBhattacharyyaCoefficient. */
292
293 protected:
294     // double GetBhattacharyya(IplImage* pImg, IplImage* pImgFG, float x, float y, DefHist* pHist=NULL)
295     double GetBhattacharyya(IplImage* pImg, IplImage* pImgFG, CvBlob* pBlob, DefHist* pHist=NULL, int /*thread_number*/ = 0)
296     {
297         if(pHist==NULL)pHist = &m_HistTemp;
298         CollectHist(pImg, pImgFG, pBlob, pHist);
299         return calcBhattacharyya(&m_HistModel, pHist, pHist);
300     }
301
302     void UpdateModelHist(IplImage* pImg, IplImage* pImgFG, CvBlob* pBlob)
303     {
304         if(m_Alpha>0 && !m_Collision)
305         {   /* Update histogram: */
306             CollectHist(pImg, pImgFG, pBlob, &m_HistCandidate);
307             m_HistModel.Update(&m_HistCandidate, m_Alpha);
308         }   /* Update histogram. */
309
310     }   /* UpdateModelHist */
311
312 public:
313     CvBlobTrackerOneMSFG()
314     {
315
316         /* Add several parameters for external use: */
317         m_FGWeight = 2;
318         AddParam("FGWeight", &m_FGWeight);
319         CommentParam("FGWeight","Weight of FG mask using (0 - mask will not be used for tracking)");
320
321         m_Alpha = 0.01f;
322         AddParam("Alpha", &m_Alpha);
323         CommentParam("Alpha","Coefficient for model histogram updating (0 - hist is not upated)");
324
325         m_IterNum = 10;
326         AddParam("IterNum", &m_IterNum);
327         CommentParam("IterNum","Maximal number of iteration in meanshift operation");
328
329         /* Initialize internal data: */
330         m_Collision = 0;
331 //        m_BinBit=0;
332         m_Dim = 0;
333         /*
334         m_HistModel = NULL;
335         m_HistCandidate = NULL;
336         m_HistTemp = NULL;
337         */
338         m_KernelHist = NULL;
339         m_KernelMeanShift = NULL;
340         ReAllocHist(3,5);   /* 3D hist, each dim has 2^5 bins*/
341
342         SetModuleName("MSFG");
343     }
344
345     ~CvBlobTrackerOneMSFG()
346     {
347         /*
348         if(m_HistModel) cvReleaseMat(&m_HistModel);
349         if(m_HistCandidate) cvReleaseMat(&m_HistCandidate);
350         if(m_HistTemp) cvReleaseMat(&m_HistTemp);
351         */
352         if(m_KernelHist) cvReleaseMat(&m_KernelHist);
353         if(m_KernelMeanShift) cvReleaseMat(&m_KernelMeanShift);
354     }
355
356     /* Interface: */
357     virtual void Init(CvBlob* pBlobInit, IplImage* pImg, IplImage* pImgFG = NULL)
358     {
359         int w = cvRound(CV_BLOB_WX(pBlobInit));
360         int h = cvRound(CV_BLOB_WY(pBlobInit));
361         if(w<CV_BLOB_MINW)w=CV_BLOB_MINW;
362         if(h<CV_BLOB_MINH)h=CV_BLOB_MINH;
363         if(pImg)
364         {
365             if(w>pImg->width)w=pImg->width;
366             if(h>pImg->height)h=pImg->height;
367         }
368         ReAllocKernel(w,h);
369         if(pImg)
370             CollectHist(pImg, pImgFG, pBlobInit, &m_HistModel);
371         m_Blob = pBlobInit[0];
372     };
373
374     virtual CvBlob* Process(CvBlob* pBlobPrev, IplImage* pImg, IplImage* pImgFG = NULL)
375     {
376         int     iter;
377
378         if(pBlobPrev)
379         {
380             m_Blob = pBlobPrev[0];
381         }
382
383         {   /* Check blob size and realloc kernels if it is necessary: */
384             int w = cvRound(m_Blob.w);
385             int h = cvRound(m_Blob.h);
386             if( w != m_ObjSize.width || h!=m_ObjSize.height)
387             {
388                 ReAllocKernel(w,h);
389                 /* after this ( w != m_ObjSize.width || h!=m_ObjSize.height) shoiuld be false */
390             }
391         }   /* Check blob size and realloc kernels if it is necessary: */
392
393
394         for(iter=0; iter<m_IterNum; ++iter)
395         {
396             float   newx=0,newy=0,sum=0;
397             //int     x,y;
398             double  B0;
399
400             //CvPoint Center = cvPoint(cvRound(m_Blob.x),cvRound(m_Blob.y));
401             CollectHist(pImg, NULL, &m_Blob, &m_HistCandidate);
402             B0 = calcBhattacharyya();
403
404             if(m_Wnd)if(CV_BLOB_ID(pBlobPrev)==0 && iter == 0)
405             {   /* Debug: */
406                 IplImage*   pW = cvCloneImage(pImgFG);
407                 IplImage*   pWFG = cvCloneImage(pImgFG);
408                 int         x,y;
409
410                 cvZero(pW);
411                 cvZero(pWFG);
412
413                 assert(m_ObjSize.width < pImg->width);
414                 assert(m_ObjSize.height < pImg->height);
415
416                 /* Calculate shift vector: */
417                 for(y=0; y<pImg->height; ++y)
418                 {
419                     unsigned char* pImgData = &CV_IMAGE_ELEM(pImg,unsigned char,y,0);
420                     unsigned char* pMaskData = pImgFG?(&CV_IMAGE_ELEM(pImgFG,unsigned char,y,0)):NULL;
421
422                     for(x=0; x<pImg->width; ++x, pImgData+=3)
423                     {
424                         int         xk = cvRound(x-(m_Blob.x-m_Blob.w*0.5));
425                         int         yk = cvRound(y-(m_Blob.y-m_Blob.h*0.5));
426                         double      HM = 0;
427                         double      HC = 0;
428                         double      K;
429                         int index = HIST_INDEX(pImgData);
430                         assert(index >= 0 && index < m_BinNumTotal);
431
432                         if(fabs(x-m_Blob.x)>m_Blob.w*0.6) continue;
433                         if(fabs(y-m_Blob.y)>m_Blob.h*0.6) continue;
434
435                         if(xk < 0 || xk >= m_KernelMeanShift->cols) continue;
436                         if(yk < 0 || yk >= m_KernelMeanShift->rows) continue;
437
438                         if(m_HistModel.m_HistVolume>0)
439                             HM = ((DefHistType*)m_HistModel.m_pHist->data.ptr)[index]/m_HistModel.m_HistVolume;
440
441                         if(m_HistCandidate.m_HistVolume>0)
442                             HC = ((DefHistType*)m_HistCandidate.m_pHist->data.ptr)[index]/m_HistCandidate.m_HistVolume;
443
444                         K = *(DefHistType*)CV_MAT_ELEM_PTR_FAST(m_KernelMeanShift[0],yk,xk,sizeof(DefHistType));
445
446                         if(HC>0)
447                         {
448                             double  V = sqrt(HM / HC);
449                             int     Vi = cvRound(V * 64);
450                             if(Vi < 0) Vi = 0;
451                             if(Vi > 255) Vi = 255;
452                             CV_IMAGE_ELEM(pW,uchar,y,x) = (uchar)Vi;
453
454                             V += m_FGWeight*(pMaskData?(pMaskData[x]/255.0f):0);
455                             V*=K;
456                             Vi = cvRound(V * 64);
457                             if(Vi < 0) Vi = 0;
458                             if(Vi > 255) Vi = 255;
459                             CV_IMAGE_ELEM(pWFG,uchar,y,x) = (uchar)Vi;
460                         }
461
462                     }   /* Next column. */
463                 }   /*  Next row. */
464
465                 //cvNamedWindow("MSFG_W",0);
466                 //cvShowImage("MSFG_W",pW);
467                 //cvNamedWindow("MSFG_WFG",0);
468                 //cvShowImage("MSFG_WFG",pWFG);
469                 //cvNamedWindow("MSFG_FG",0);
470                 //cvShowImage("MSFG_FG",pImgFG);
471
472                 //cvSaveImage("MSFG_W.bmp",pW);
473                 //cvSaveImage("MSFG_WFG.bmp",pWFG);
474                 //cvSaveImage("MSFG_FG.bmp",pImgFG);
475
476             }   /* Debug. */
477
478
479             /* Calculate new position by meanshift: */
480
481             /* Calculate new position: */
482             if(m_Dim == 3)
483             {
484                 int         x0 = cvRound(m_Blob.x - m_ObjSize.width*0.5);
485                 int         y0 = cvRound(m_Blob.y - m_ObjSize.height*0.5);
486                 int         x,y;
487
488                 assert(m_ObjSize.width < pImg->width);
489                 assert(m_ObjSize.height < pImg->height);
490
491                 /* Crop blob bounds: */
492                 if((x0+m_ObjSize.width)>=pImg->width) x0=pImg->width-m_ObjSize.width-1;
493                 if((y0+m_ObjSize.height)>=pImg->height) y0=pImg->height-m_ObjSize.height-1;
494                 if(x0<0){ x0=0;}
495                 if(y0<0){ y0=0;}
496
497                 /* Calculate shift vector: */
498                 for(y=0; y<m_ObjSize.height; ++y)
499                 {
500                     unsigned char* pImgData = &CV_IMAGE_ELEM(pImg,unsigned char,y+y0,x0*3);
501                     unsigned char* pMaskData = pImgFG?(&CV_IMAGE_ELEM(pImgFG,unsigned char,y+y0,x0)):NULL;
502                     DefHistType* pKernelData = (DefHistType*)CV_MAT_ELEM_PTR_FAST(m_KernelMeanShift[0],y,0,sizeof(DefHistType));
503
504                     for(x=0; x<m_ObjSize.width; ++x, pImgData+=3)
505                     {
506                         DefHistType K = pKernelData[x];
507                         double      HM = 0;
508                         double      HC = 0;
509                         int index = HIST_INDEX(pImgData);
510                         assert(index >= 0 && index < m_BinNumTotal);
511
512                         if(m_HistModel.m_HistVolume>0)
513                             HM = ((DefHistType*)m_HistModel.m_pHist->data.ptr)[index]/m_HistModel.m_HistVolume;
514
515                         if(m_HistCandidate.m_HistVolume>0)
516                             HC = ((DefHistType*)m_HistCandidate.m_pHist->data.ptr)[index]/m_HistCandidate.m_HistVolume;
517
518                         if(HC>0)
519                         {
520                             double V = sqrt(HM / HC);
521                             if(!m_Collision && m_FGWeight>0 && pMaskData)
522                             {
523                                 V += m_FGWeight*(pMaskData[x]/255.0f);
524                             }
525                             K *= (float)MIN(V,100000.);
526                         }
527
528                         sum += K;
529                         newx += K*x;
530                         newy += K*y;
531                     }   /* Next column. */
532                 }   /*  Next row. */
533
534                 if(sum > 0)
535                 {
536                     newx /= sum;
537                     newy /= sum;
538                 }
539                 newx += x0;
540                 newy += y0;
541
542             }   /* if m_Dim == 3. */
543
544             /* Calculate new position by meanshift: */
545
546             for(;;)
547             {   /* Iterate using bahattcharrya coefficient: */
548                 double  B1;
549                 CvBlob  B = m_Blob;
550 //                CvPoint NewCenter = cvPoint(cvRound(newx),cvRound(newy));
551                 B.x = newx;
552                 B.y = newy;
553                 CollectHist(pImg, NULL, &B, &m_HistCandidate);
554                 B1 = calcBhattacharyya();
555                 if(B1 > B0) break;
556                 newx = 0.5f*(newx+m_Blob.x);
557                 newy = 0.5f*(newy+m_Blob.y);
558                 if(fabs(newx-m_Blob.x)<0.1 && fabs(newy-m_Blob.y)<0.1) break;
559             }   /* Iterate using bahattcharrya coefficient. */
560
561             if(fabs(newx-m_Blob.x)<0.5 && fabs(newy-m_Blob.y)<0.5) break;
562             m_Blob.x = newx;
563             m_Blob.y = newy;
564         }   /* Next iteration. */
565
566         while(!m_Collision && m_FGWeight>0)
567         {   /* Update size if no collision. */
568             float       Alpha = 0.04f;
569             CvBlob      NewBlob;
570             double      M00,X,Y,XX,YY;
571             CvMoments   m;
572             CvRect      r;
573             CvMat       mat;
574
575             r.width = cvRound(m_Blob.w*1.5+0.5);
576             r.height = cvRound(m_Blob.h*1.5+0.5);
577             r.x = cvRound(m_Blob.x - 0.5*r.width);
578             r.y = cvRound(m_Blob.y - 0.5*r.height);
579             if(r.x < 0) break;
580             if(r.y < 0) break;
581             if(r.x+r.width >= pImgFG->width) break;
582             if(r.y+r.height >= pImgFG->height) break;
583             if(r.height < 5 || r.width < 5) break;
584
585             cvMoments( cvGetSubRect(pImgFG,&mat,r), &m, 0 );
586             M00 = cvGetSpatialMoment( &m, 0, 0 );
587             if(M00 <= 0 ) break;
588             X = cvGetSpatialMoment( &m, 1, 0 )/M00;
589             Y = cvGetSpatialMoment( &m, 0, 1 )/M00;
590             XX = (cvGetSpatialMoment( &m, 2, 0 )/M00) - X*X;
591             YY = (cvGetSpatialMoment( &m, 0, 2 )/M00) - Y*Y;
592             NewBlob = cvBlob(r.x+(float)X,r.y+(float)Y,(float)(4*sqrt(XX)),(float)(4*sqrt(YY)));
593
594             NewBlob.w = Alpha*NewBlob.w+m_Blob.w*(1-Alpha);
595             NewBlob.h = Alpha*NewBlob.h+m_Blob.h*(1-Alpha);
596
597             m_Blob.w = MAX(NewBlob.w,5);
598             m_Blob.h = MAX(NewBlob.h,5);
599             break;
600
601         }   /* Update size if no collision. */
602
603         return &m_Blob;
604
605     };  /* CvBlobTrackerOneMSFG::Process */
606
607     virtual double GetConfidence(CvBlob* pBlob, IplImage* pImg, IplImage* /*pImgFG*/ = NULL, IplImage* pImgUnusedReg = NULL)
608     {
609         double  S = 0.2;
610         double  B = GetBhattacharyya(pImg, pImgUnusedReg, pBlob, &m_HistTemp);
611         return exp((B-1)/(2*S));
612
613     };  /*CvBlobTrackerOneMSFG::*/
614
615     virtual void Update(CvBlob* pBlob, IplImage* pImg, IplImage* pImgFG = NULL)
616     {   /* Update histogram: */
617         UpdateModelHist(pImg, pImgFG, pBlob?pBlob:&m_Blob);
618     }   /*CvBlobTrackerOneMSFG::*/
619
620     virtual void Release(){delete this;};
621     virtual void SetCollision(int CollisionFlag)
622     {
623         m_Collision = CollisionFlag;
624     }
625     virtual void SaveState(CvFileStorage* fs)
626     {
627         cvWriteStruct(fs, "Blob", &m_Blob, "ffffi");
628         cvWriteInt(fs,"Collision", m_Collision);
629         cvWriteInt(fs,"HistVolume", cvRound(m_HistModel.m_HistVolume));
630         cvWrite(fs,"Hist", m_HistModel.m_pHist);
631     };
632     virtual void LoadState(CvFileStorage* fs, CvFileNode* node)
633     {
634         CvMat* pM;
635         cvReadStructByName(fs, node, "Blob",&m_Blob, "ffffi");
636         m_Collision = cvReadIntByName(fs,node,"Collision",m_Collision);
637         pM = (CvMat*)cvRead(fs,cvGetFileNodeByName(fs,node,"Hist"));
638         if(pM)
639         {
640             m_HistModel.m_pHist = pM;
641             m_HistModel.m_HistVolume = (float)cvSum(pM).val[0];
642         }
643     };
644
645 };  /*CvBlobTrackerOneMSFG*/
646
647 #if 0
648 void CvBlobTrackerOneMSFG::CollectHist(IplImage* pImg, IplImage* pMask, CvBlob* pBlob, DefHist* pHist)
649 {
650     int UsePrecalculatedKernel = 0;
651     int BW = cvRound(pBlob->w);
652     int BH = cvRound(pBlob->h);
653     DefHistType Volume = 0;
654     int         x0 = cvRound(pBlob->x - BW*0.5);
655     int         y0 = cvRound(pBlob->y - BH*0.5);
656     int         x,y;
657
658     UsePrecalculatedKernel = (BW == m_ObjSize.width && BH == m_ObjSize.height ) ;
659
660     //cvZero(pHist);
661     cvSet(pHist->m_pHist,cvScalar(1.0/m_BinNumTotal)); /* no zero bins, all bins have very small value*/
662     Volume = 1;
663
664     assert(BW < pImg->width);
665     assert(BH < pImg->height);
666     if((x0+BW)>=pImg->width) BW=pImg->width-x0-1;
667     if((y0+BH)>=pImg->height) BH=pImg->height-y0-1;
668     if(x0<0){ x0=0;}
669     if(y0<0){ y0=0;}
670
671     if(m_Dim == 3)
672     {
673         for(y=0; y<BH; ++y)
674         {
675             unsigned char* pImgData = &CV_IMAGE_ELEM(pImg,unsigned char,y+y0,x0*3);
676             unsigned char* pMaskData = pMask?(&CV_IMAGE_ELEM(pMask,unsigned char,y+y0,x0)):NULL;
677             DefHistType* pKernelData = NULL;
678
679             if(UsePrecalculatedKernel)
680             {
681                 pKernelData = ((DefHistType*)CV_MAT_ELEM_PTR_FAST(m_KernelHist[0],y,0,sizeof(DefHistType)));
682             }
683
684             for(x=0; x<BW; ++x, pImgData+=3)
685             {
686                 DefHistType K;
687                 int index = HIST_INDEX(pImgData);
688                 assert(index >= 0 && index < pHist->m_pHist->cols);
689
690                 if(UsePrecalculatedKernel)
691                 {
692                     K = pKernelData[x];
693                 }
694                 else
695                 {
696                     float dx = (x+x0-pBlob->x)/(pBlob->w*0.5);
697                     float dy = (y+y0-pBlob->y)/(pBlob->h*0.5);
698                     double r2 = dx*dx+dy*dy;
699                     K = GetKernelHist(r2);
700                 }
701
702                 if(pMaskData)
703                 {
704                     K *= pMaskData[x]*0.003921568627450980392156862745098;
705                 }
706                 Volume += K;
707                 ((DefHistType*)(pHist->m_pHist->data.ptr))[index] += K;
708
709             }   /* Next column. */
710         }   /*  Next row. */
711     }   /*  if m_Dim == 3. */
712
713     pHist->m_HistVolume = Volume;
714
715 };  /* CollectHist */
716 #endif
717
718 CvBlobTrackerOne* cvCreateBlobTrackerOneMSFG()
719 {
720     return (CvBlobTrackerOne*) new CvBlobTrackerOneMSFG;
721 }
722
723 CvBlobTracker* cvCreateBlobTrackerMSFG()
724 {
725     return cvCreateBlobTrackerList(cvCreateBlobTrackerOneMSFG);
726 }
727
728 /* Create specific tracker without FG
729  * usin - just simple mean-shift tracker: */
730 class CvBlobTrackerOneMS:public CvBlobTrackerOneMSFG
731 {
732 public:
733     CvBlobTrackerOneMS()
734     {
735         SetParam("FGWeight",0);
736         DelParam("FGWeight");
737         SetModuleName("MS");
738     };
739 };
740
741 CvBlobTrackerOne* cvCreateBlobTrackerOneMS()
742 {
743     return (CvBlobTrackerOne*) new CvBlobTrackerOneMS;
744 }
745
746 CvBlobTracker* cvCreateBlobTrackerMS()
747 {
748     return cvCreateBlobTrackerList(cvCreateBlobTrackerOneMS);
749 }
750
751 typedef struct DefParticle
752 {
753     CvBlob  blob;
754     float   Vx,Vy;
755     double  W;
756 } DefParticle;
757
758 class CvBlobTrackerOneMSPF:public CvBlobTrackerOneMS
759 {
760 private:
761     /* parameters */
762     int             m_ParticleNum;
763     float           m_UseVel;
764     float           m_SizeVar;
765     float           m_PosVar;
766
767     CvSize          m_ImgSize;
768     CvBlob          m_Blob;
769     DefParticle*    m_pParticlesPredicted;
770     DefParticle*    m_pParticlesResampled;
771     CvRNG           m_RNG;
772 #ifdef _OPENMP
773     int             m_ThreadNum;
774     DefHist*        m_HistForParalel;
775 #endif
776
777 public:
778     virtual void SaveState(CvFileStorage* fs)
779     {
780         CvBlobTrackerOneMS::SaveState(fs);
781         cvWriteInt(fs,"ParticleNum",m_ParticleNum);
782         cvWriteStruct(fs,"ParticlesPredicted",m_pParticlesPredicted,"ffffiffd",m_ParticleNum);
783         cvWriteStruct(fs,"ParticlesResampled",m_pParticlesResampled,"ffffiffd",m_ParticleNum);
784     };
785
786     virtual void LoadState(CvFileStorage* fs, CvFileNode* node)
787     {
788         //CvMat* pM;
789         CvBlobTrackerOneMS::LoadState(fs,node);
790         m_ParticleNum = cvReadIntByName(fs,node,"ParticleNum",m_ParticleNum);
791         if(m_ParticleNum>0)
792         {
793             Realloc();
794             printf("sizeof(DefParticle) is %d\n", (int)sizeof(DefParticle));
795             cvReadStructByName(fs,node,"ParticlesPredicted",m_pParticlesPredicted,"ffffiffd");
796             cvReadStructByName(fs,node,"ParticlesResampled",m_pParticlesResampled,"ffffiffd");
797         }
798     };
799     CvBlobTrackerOneMSPF()
800     {
801         m_pParticlesPredicted = NULL;
802         m_pParticlesResampled = NULL;
803         m_ParticleNum = 200;
804
805         AddParam("ParticleNum",&m_ParticleNum);
806         CommentParam("ParticleNum","Number of particles");
807         Realloc();
808
809         m_UseVel = 0;
810         AddParam("UseVel",&m_UseVel);
811         CommentParam("UseVel","Percent of particles which use velocity feature");
812
813         m_SizeVar = 0.05f;
814         AddParam("SizeVar",&m_SizeVar);
815         CommentParam("SizeVar","Size variation (in object size)");
816
817         m_PosVar = 0.2f;
818         AddParam("PosVar",&m_PosVar);
819         CommentParam("PosVar","Position variation (in object size)");
820
821         m_RNG = cvRNG(0);
822
823         SetModuleName("MSPF");
824
825 #ifdef _OPENMP
826         {
827             m_ThreadNum = omp_get_num_procs();
828             m_HistForParalel = new DefHist[m_ThreadNum];
829         }
830 #endif
831     }
832
833     ~CvBlobTrackerOneMSPF()
834     {
835         if(m_pParticlesResampled)cvFree(&m_pParticlesResampled);
836         if(m_pParticlesPredicted)cvFree(&m_pParticlesPredicted);
837 #ifdef _OPENMP
838         if(m_HistForParalel) delete[] m_HistForParalel;
839 #endif
840     }
841
842 private:
843     void Realloc()
844     {
845         if(m_pParticlesResampled)cvFree(&m_pParticlesResampled);
846         if(m_pParticlesPredicted)cvFree(&m_pParticlesPredicted);
847         m_pParticlesPredicted = (DefParticle*)cvAlloc(sizeof(DefParticle)*m_ParticleNum);
848         m_pParticlesResampled = (DefParticle*)cvAlloc(sizeof(DefParticle)*m_ParticleNum);
849     };  /* Realloc*/
850
851     void DrawDebug(IplImage* pImg, IplImage* /*pImgFG*/)
852     {
853         int k;
854         for(k=0; k<2; ++k)
855         {
856             DefParticle*    pBP = k?m_pParticlesResampled:m_pParticlesPredicted;
857             //const char*   name = k?"MSPF resampled particle":"MSPF Predicted particle";
858             IplImage*       pI = cvCloneImage(pImg);
859             int             h,hN = m_ParticleNum;
860             CvBlob          C = cvBlob(0,0,0,0);
861             double          WS = 0;
862             for(h=0; h<hN; ++h)
863             {
864                 CvBlob  B = pBP[h].blob;
865                 int     CW = cvRound(255*pBP[h].W);
866                 CvBlob* pB = &B;
867                 int x = cvRound(CV_BLOB_RX(pB)), y = cvRound(CV_BLOB_RY(pB));
868                 CvSize  s = cvSize(MAX(1,x), MAX(1,y));
869                 double  W = pBP[h].W;
870                 C.x += pB->x;
871                 C.y += pB->y;
872                 C.w += pB->w;
873                 C.h += pB->h;
874                 WS+=W;
875
876                 s = cvSize(1,1);
877                 cvEllipse( pI,
878                     cvPointFrom32f(CV_BLOB_CENTER(pB)),
879                     s,
880                     0, 0, 360,
881                     CV_RGB(CW,0,0), 1 );
882
883             }   /* Next hypothesis. */
884
885             C.x /= hN;
886             C.y /= hN;
887             C.w /= hN;
888             C.h /= hN;
889
890             cvEllipse( pI,
891                 cvPointFrom32f(CV_BLOB_CENTER(&C)),
892                 cvSize(cvRound(C.w*0.5),cvRound(C.h*0.5)),
893                 0, 0, 360,
894                 CV_RGB(0,0,255), 1 );
895
896             cvEllipse( pI,
897                 cvPointFrom32f(CV_BLOB_CENTER(&m_Blob)),
898                 cvSize(cvRound(m_Blob.w*0.5),cvRound(m_Blob.h*0.5)),
899                 0, 0, 360,
900                 CV_RGB(0,255,0), 1 );
901
902             //cvNamedWindow(name,0);
903             //cvShowImage(name,pI);
904             cvReleaseImage(&pI);
905         } /*  */
906
907         //printf("Blob %d, point (%.1f,%.1f) size (%.1f,%.1f)\n",m_Blob.ID,m_Blob.x,m_Blob.y,m_Blob.w,m_Blob.h);
908     } /* ::DrawDebug */
909
910 private:
911     void Prediction()
912     {
913         int p;
914         for(p=0; p<m_ParticleNum; ++p)
915         {   /* "Prediction" of particle: */
916             //double  t;
917             float   r[5];
918             CvMat   rm = cvMat(1,5,CV_32F,r);
919             cvRandArr(&m_RNG,&rm,CV_RAND_NORMAL,cvScalar(0),cvScalar(1));
920
921             m_pParticlesPredicted[p] = m_pParticlesResampled[p];
922
923             if(cvRandReal(&m_RNG)<0.5)
924             {   /* Half of particles will predict based on external blob: */
925                 m_pParticlesPredicted[p].blob = m_Blob;
926             }
927
928             if(cvRandReal(&m_RNG)<m_UseVel)
929             {   /* Predict moving particle by usual way by using speed: */
930                 m_pParticlesPredicted[p].blob.x += m_pParticlesPredicted[p].Vx;
931                 m_pParticlesPredicted[p].blob.y += m_pParticlesPredicted[p].Vy;
932             }
933             else
934             {   /* Stop several particles: */
935                 m_pParticlesPredicted[p].Vx = 0;
936                 m_pParticlesPredicted[p].Vy = 0;
937             }
938
939             {   /* Update position: */
940                 float S = (m_Blob.w + m_Blob.h)*0.5f;
941                 m_pParticlesPredicted[p].blob.x += m_PosVar*S*r[0];
942                 m_pParticlesPredicted[p].blob.y += m_PosVar*S*r[1];
943
944                 /* Update velocity: */
945                 m_pParticlesPredicted[p].Vx += (float)(m_PosVar*S*0.1*r[3]);
946                 m_pParticlesPredicted[p].Vy += (float)(m_PosVar*S*0.1*r[4]);
947             }
948
949             /* Update size: */
950             m_pParticlesPredicted[p].blob.w *= (1+m_SizeVar*r[2]);
951             m_pParticlesPredicted[p].blob.h *= (1+m_SizeVar*r[2]);
952
953             /* Truncate size of particle: */
954             if(m_pParticlesPredicted[p].blob.w > m_ImgSize.width*0.5f)
955             {
956                 m_pParticlesPredicted[p].blob.w = m_ImgSize.width*0.5f;
957             }
958
959             if(m_pParticlesPredicted[p].blob.h > m_ImgSize.height*0.5f)
960             {
961                 m_pParticlesPredicted[p].blob.h = m_ImgSize.height*0.5f;
962             }
963
964             if(m_pParticlesPredicted[p].blob.w < 1 )
965             {
966                 m_pParticlesPredicted[p].blob.w = 1;
967             }
968
969             if(m_pParticlesPredicted[p].blob.h < 1)
970             {
971                 m_pParticlesPredicted[p].blob.h = 1;
972             }
973         }   /* "Prediction" of particle. */
974     }   /* Prediction */
975
976     void UpdateWeightsMS(IplImage* pImg, IplImage* /*pImgFG*/)
977     {
978         int p;
979 #ifdef _OPENMP
980         if( m_HistForParalel[0].m_pHist==NULL || m_HistForParalel[0].m_pHist->cols != m_BinNumTotal)
981         {
982             int t;
983             for(t=0; t<m_ThreadNum; ++t)
984                 m_HistForParalel[t].Resize(m_BinNumTotal);
985         }
986 #endif
987
988 #ifdef _OPENMP
989 #pragma omp parallel for num_threads(m_ThreadNum) schedule(runtime)
990 #endif
991         for(p=0;p<m_ParticleNum;++p)
992         {   /* Calculate weights for particles: */
993             double  S = 0.2;
994             double  B = 0;
995 #ifdef _OPENMP
996             assert(omp_get_thread_num()<m_ThreadNum);
997 #endif
998
999             B = GetBhattacharyya(
1000                 pImg, NULL,
1001                 &(m_pParticlesPredicted[p].blob)
1002 #ifdef _OPENMP
1003                 ,&(m_HistForParalel[omp_get_thread_num()])
1004 #endif
1005                 );
1006             m_pParticlesPredicted[p].W *= exp((B-1)/(2*S));
1007
1008         }   /* Calculate weights for particles. */
1009     }
1010
1011     void UpdateWeightsCC(IplImage* /*pImg*/, IplImage* /*pImgFG*/)
1012     {
1013         int p;
1014 #ifdef _OPENMP
1015 #pragma omp parallel for
1016 #endif
1017         for(p=0; p<m_ParticleNum; ++p)
1018         {   /* Calculate weights for particles: */
1019             double W = 1;
1020             m_pParticlesPredicted[p].W *= W;
1021         }   /* Calculate weights for particles. */
1022     }
1023
1024     void Resample()
1025     {   /* Resample particle: */
1026         int         p;
1027         double      Sum = 0;
1028
1029         for(p=0; p<m_ParticleNum; ++p)
1030         {
1031             Sum += m_pParticlesPredicted[p].W;
1032         }
1033
1034         for(p=0; p<m_ParticleNum; ++p)
1035         {
1036             double  T = Sum * cvRandReal(&m_RNG);   /* Set current random threshold for cululative weight. */
1037             int     p2;
1038             double  Sum2 = 0;
1039
1040             for(p2=0; p2<m_ParticleNum; ++p2)
1041             {
1042                 Sum2 += m_pParticlesPredicted[p2].W;
1043                 if(Sum2 >= T)break;
1044             }
1045
1046             if(p2>=m_ParticleNum)p2=m_ParticleNum-1;
1047             m_pParticlesResampled[p] = m_pParticlesPredicted[p2];
1048             m_pParticlesResampled[p].W = 1;
1049
1050         }   /* Find next particle. */
1051     }   /*  Resample particle. */
1052
1053
1054 public:
1055     virtual void Init(CvBlob* pBlobInit, IplImage* pImg, IplImage* pImgFG = NULL)
1056     {
1057         int i;
1058         CvBlobTrackerOneMSFG::Init(pBlobInit, pImg, pImgFG);
1059         DefParticle PP;
1060         PP.W = 1;
1061         PP.Vx = 0;
1062         PP.Vy = 0;
1063         PP.blob = pBlobInit[0];
1064         for(i=0;i<m_ParticleNum;++i)
1065         {
1066             m_pParticlesPredicted[i] = PP;
1067             m_pParticlesResampled[i] = PP;
1068         }
1069         m_Blob = pBlobInit[0];
1070
1071     }   /* CvBlobTrackerOneMSPF::Init*/
1072
1073     virtual CvBlob* Process(CvBlob* pBlobPrev, IplImage* pImg, IplImage* pImgFG = NULL)
1074     {
1075         int p;
1076
1077         m_ImgSize.width = pImg->width;
1078         m_ImgSize.height = pImg->height;
1079
1080
1081         m_Blob = pBlobPrev[0];
1082
1083         {   /* Check blob size and realloc kernels if it is necessary: */
1084             int w = cvRound(m_Blob.w);
1085             int h = cvRound(m_Blob.h);
1086             if( w != m_ObjSize.width || h!=m_ObjSize.height)
1087             {
1088                 ReAllocKernel(w,h);
1089                 /* After this ( w != m_ObjSize.width || h!=m_ObjSize.height) should be false. */
1090             }
1091         }   /* Check blob size and realloc kernels if it is necessary. */
1092
1093         Prediction();
1094
1095 #ifdef REPORT_TICKS
1096         int64 ticks = cvGetTickCount();
1097 #endif
1098
1099         UpdateWeightsMS(pImg, pImgFG);
1100
1101 #ifdef REPORT_TICKS
1102         ticks = cvGetTickCount() - ticks;
1103         fprintf(stderr, "PF UpdateWeights, %d ticks\n",  (int)ticks);
1104         ticks = cvGetTickCount();
1105 #endif
1106
1107         Resample();
1108
1109 #ifdef REPORT_TICKS
1110         ticks = cvGetTickCount() - ticks;
1111         fprintf(stderr, "PF Resampling, %d ticks\n",  (int)ticks);
1112 #endif
1113
1114         {   /* Find average result: */
1115             float   x = 0;
1116             float   y = 0;
1117             float   w = 0;
1118             float   h = 0;
1119             float   Sum = 0;
1120
1121             DefParticle* pP = m_pParticlesResampled;
1122
1123             for(p=0; p<m_ParticleNum; ++p)
1124             {
1125                 float W = (float)pP[p].W;
1126                 x += W*pP[p].blob.x;
1127                 y += W*pP[p].blob.y;
1128                 w += W*pP[p].blob.w;
1129                 h += W*pP[p].blob.h;
1130                 Sum += W;
1131             }
1132
1133             if(Sum>0)
1134             {
1135                 m_Blob.x = x / Sum;
1136                 m_Blob.y = y / Sum;
1137                 m_Blob.w = w / Sum;
1138                 m_Blob.h = h / Sum;
1139             }
1140         }   /* Find average result. */
1141
1142         if(m_Wnd)
1143         {
1144             DrawDebug(pImg, pImgFG);
1145         }
1146
1147         return &m_Blob;
1148
1149     }   /* CvBlobTrackerOneMSPF::Process */
1150
1151     virtual void SkipProcess(CvBlob* pBlob, IplImage* /*pImg*/, IplImage* /*pImgFG*/ = NULL)
1152     {
1153         int p;
1154         for(p=0; p<m_ParticleNum; ++p)
1155         {
1156             m_pParticlesResampled[p].blob = pBlob[0];
1157             m_pParticlesResampled[p].Vx = 0;
1158             m_pParticlesResampled[p].Vy = 0;
1159             m_pParticlesResampled[p].W = 1;
1160         }
1161     }
1162
1163     virtual void Release(){delete this;};
1164     virtual void ParamUpdate()
1165     {
1166         Realloc();
1167     }
1168
1169 };  /* CvBlobTrackerOneMSPF */
1170
1171 CvBlobTrackerOne* cvCreateBlobTrackerOneMSPF()
1172 {
1173     return (CvBlobTrackerOne*) new CvBlobTrackerOneMSPF;
1174 }
1175
1176 CvBlobTracker* cvCreateBlobTrackerMSPF()
1177 {
1178     return cvCreateBlobTrackerList(cvCreateBlobTrackerOneMSPF);
1179 }
1180