1 /*M///////////////////////////////////////////////////////////////////////////////////////
3 // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
5 // By downloading, copying, installing or using the software you agree to this license.
6 // If you do not agree to this license, do not download, install,
7 // copy or use the software.
10 // Intel License Agreement
11 // For Open Source Computer Vision Library
13 // Copyright (C) 2000, Intel Corporation, all rights reserved.
14 // Third party copyrights are property of their respective owners.
16 // Redistribution and use in source and binary forms, with or without modification,
17 // are permitted provided that the following conditions are met:
19 // * Redistribution's of source code must retain the above copyright notice,
20 // this list of conditions and the following disclaimer.
22 // * Redistribution's in binary form must reproduce the above copyright notice,
23 // this list of conditions and the following disclaimer in the documentation
24 // and/or other materials provided with the distribution.
26 // * The name of Intel Corporation may not be used to endorse or promote products
27 // derived from this software without specific prior written permission.
29 // This software is provided by the copyright holders and contributors "as is" and
30 // any express or implied warranties, including, but not limited to, the implied
31 // warranties of merchantability and fitness for a particular purpose are disclaimed.
32 // In no event shall the Intel Corporation or contributors be liable for any direct,
33 // indirect, incidental, special, exemplary, or consequential damages
34 // (including, but not limited to, procurement of substitute goods or services;
35 // loss of use, data, or profits; or business interruption) however caused
36 // and on any theory of liability, whether in contract, strict liability,
37 // or tort (including negligence or otherwise) arising in any way out of
38 // the use of this software, even if advised of the possibility of such damage.
56 #pragma warning( disable: 4710 )
59 #define CV_TS_VERSION "CxTest 0.1"
61 // Helper class for growing vector (to avoid dependency from STL)
62 template < typename T > class CvTestVec
65 CvTestVec() { _max_size = _size = 0; _buf = 0; }
66 ~CvTestVec() { delete[] _buf; }
67 T& operator []( int i ) { assert( (unsigned)i < (unsigned)_size ); return _buf[i]; }
68 T at( int i ) { assert( (unsigned)i < (unsigned)_size ); return _buf[i]; }
69 T pop() { assert( _size > 0 ); return _buf[--_size]; }
70 void push( const T& elem )
72 if( _size >= _max_size )
74 int i, _new_size = _max_size < 16 ? 16 : _max_size*3/2;
75 T* temp = new T[_new_size];
76 for( i = 0; i < _size; i++ )
79 _max_size = _new_size;
85 int size() { return _size; }
86 T* data() { return _buf; }
87 void clear() { _size = 0; }
94 /*****************************************************************************************\
95 * Base class for tests *
96 \*****************************************************************************************/
101 class CV_EXPORTS CvTest
104 // constructor(s) and destructor
105 CvTest( const char* test_name, const char* test_funcs, const char* test_descr = "" );
108 virtual int init( CvTS* system );
110 // writes default parameters to file storage
111 virtual int write_defaults(CvTS* ts);
113 // the main procedure of the test
114 virtual void run( int start_from );
116 // the wrapper for run that cares of exceptions
117 virtual void safe_run( int start_from );
119 const char* get_name() const { return name; }
120 const char* get_func_list() const { return tested_functions; }
121 const char* get_description() const { return description; }
122 const char* get_group_name( char* buffer ) const;
123 CvTest* get_next() { return next; }
124 static CvTest* get_first_test();
125 static const char* get_parent_name( const char* name, char* buffer );
127 // returns true if and only if the different test cases do not depend on each other
128 // (so that test system could get right to a problematic test case)
129 virtual bool can_do_fast_forward();
131 // deallocates all the memory.
132 // called by init() (before initialization) and by the destructor
133 virtual void clear();
135 // returns the testing modes supported by the particular test
136 int get_support_testing_modes();
138 enum { TIMING_EXTRA_PARAMS=5 };
141 static CvTest* first;
143 static int test_count;
146 const char** default_timing_param_names; // the names of timing parameters to write
147 const CvFileNode* timing_param_names; // and the read param names
148 const CvFileNode** timing_param_current; // the current tuple of timing parameters
149 const CvFileNode** timing_param_seqs; // the array of parameter sequences
150 int* timing_param_idxs; // the array of indices
151 int timing_param_count; // the number of parameters in the tuple
152 int support_testing_modes;
154 int test_case_count; // the total number of test cases
156 // called from write_defaults
157 virtual int write_default_params(CvFileStorage* fs);
160 virtual int read_params( CvFileStorage* fs );
162 // returns the number of tests or -1 if it is unknown a-priori
163 virtual int get_test_case_count();
165 // prepares data for the next test case. rng seed is updated by the function
166 virtual int prepare_test_case( int test_case_idx );
168 // checks if the test output is valid and accurate
169 virtual int validate_test_results( int test_case_idx );
171 // calls the tested function. the method is called from run_test_case()
172 virtual void run_func(); // runs tested func(s)
174 // prints results of timing test
175 virtual void print_time( int test_case_idx, double time_usecs );
177 // updates progress bar
178 virtual int update_progress( int progress, int test_case_idx, int count, double dt );
180 // finds test parameter
181 const CvFileNode* find_param( CvFileStorage* fs, const char* param_name );
184 void write_param( CvFileStorage* fs, const char* paramname, int val );
185 void write_param( CvFileStorage* fs, const char* paramname, double val );
186 void write_param( CvFileStorage* fs, const char* paramname, const char* val );
187 void write_string_list( CvFileStorage* fs, const char* paramname, const char** val, int count=-1 );
188 void write_int_list( CvFileStorage* fs, const char* paramname, const int* val,
189 int count, int stop_value=INT_MIN );
190 void write_real_list( CvFileStorage* fs, const char* paramname, const double* val,
191 int count, double stop_value=DBL_MIN );
192 void start_write_param( CvFileStorage* fs );
194 // returns the specified parameter from the current parameter tuple
195 const CvFileNode* find_timing_param( const char* paramname );
197 // gets the next tuple of timing parameters
198 int get_next_timing_param_tuple();
200 // name of the test (it is possible to locate a test by its name)
203 // comma-separated list of functions that are invoked
204 // (and, thus, tested explicitly or implicitly) by the test
205 // methods of classes can be grouped using {}.
207 // "cvCanny, cvAdd, cvSub, cvMul"
208 // "CvImage::{Create, CopyOf}, cvMatMulAdd, CvCalibFilter::{PushFrame, SetCameraCount}"
209 const char* tested_functions;
211 // description of the test
212 const char* description;
214 // pointer to the system that includes the test
221 /*****************************************************************************************\
222 * Information about a failed test *
223 \*****************************************************************************************/
225 typedef struct CvTestInfo
227 // pointer to the test
230 // failure code (CV_FAIL*)
233 // seed value right before the data for the failed test case is prepared.
236 // index of test case, can be then passed to CvTest::proceed_to_test_case()
239 // index of the corrupted or leaked block
242 // index of the first block in the group
243 // (used to adjust alloc_index when some test/test cases are skipped).
244 int base_alloc_index;
248 /*****************************************************************************************\
249 * Base Class for test system *
250 \*****************************************************************************************/
252 class CvTestMemoryManager;
254 typedef CvTestVec<int> CvTestIntVec;
255 typedef CvTestVec<void*> CvTestPtrVec;
256 typedef CvTestVec<CvTestInfo> CvTestInfoVec;
258 class CV_EXPORTS CvTS
262 // constructor(s) and destructor
270 SUMMARY=1 << SUMMARY_IDX,
276 CONSOLE=1 << CONSOLE_IDX,
280 // low-level printing functions that are used by individual tests and by the system itself
281 virtual void printf( int streams, const char* fmt, ... );
282 virtual void vprintf( int streams, const char* fmt, va_list arglist );
284 // runs the tests (the whole set or some selected tests)
285 virtual int run( int argc, char** argv );
287 // updates the context: current test, test case, rng state
288 virtual void update_context( CvTest* test, int test_case_idx, bool update_ts_context );
290 const CvTestInfo* get_current_test_info() { return ¤t_test_info; }
292 // sets information about a failed test
293 virtual void set_failed_test_info( int fail_code, int alloc_index = -1 );
298 CORRECTNESS_CHECK_MODE = 1,
302 // the modes of timing tests:
303 enum { AVG_TIME = 1, MIN_TIME = 2 };
311 // generic error: stub value to be used
312 // temporarily if the error's cause is unknown
315 // the test is missing some essential data to proceed further
316 FAIL_MISSING_TEST_DATA=-2,
318 // the tested function raised an error via cxcore error handler
319 FAIL_ERROR_IN_CALLED_FUNC=-3,
321 // an exception has been raised;
322 // for memory and arithmetic exception
323 // there are two specialized codes (see below...)
326 // a memory exception
327 // (access violation, access to missed page, stack overflow etc.)
328 FAIL_MEMORY_EXCEPTION=-5,
330 // arithmetic exception (overflow, division by zero etc.)
331 FAIL_ARITHM_EXCEPTION=-6,
333 // the tested function corrupted memory (no exception have been raised)
334 FAIL_MEMORY_CORRUPTION_BEGIN=-7,
335 FAIL_MEMORY_CORRUPTION_END=-8,
337 // the tested function (or test ifself) do not deallocate some memory
340 // the tested function returned invalid object, e.g. matrix, containing NaNs,
341 // structure with NULL or out-of-range fields (while it should not)
342 FAIL_INVALID_OUTPUT=-10,
344 // the tested function returned valid object, but it does not match to
345 // the original (or produced by the test) object
348 // the tested function returned valid object (a single number or numerical array),
349 // but it differs too much from the original (or produced by the test) object
350 FAIL_BAD_ACCURACY=-12,
352 // the tested function hung. Sometimes, can be determined by unexpectedly long
353 // processing time (in this case there should be possibility to interrupt such a function
356 // unexpected responce on passing bad arguments to the tested function
357 // (the function crashed, proceed succesfully (while it should not), or returned
358 // error code that is different from what is expected)
359 FAIL_BAD_ARG_CHECK=-14,
361 // the test data (in whole or for the particular test case) is invalid
362 FAIL_INVALID_TEST_DATA=-15,
364 // the test has been skipped because it is not in the selected subset of the tests to run,
365 // because it has been run already within the same run with the same parameters, or because
366 // of some other reason and this is not considered as an error.
367 // Normally CvTS::run() (or overrided method in the derived class) takes care of what
368 // needs to be run, so this code should not occur.
373 CvFileStorage* get_file_storage() { return fs; }
375 // get RNG to generate random input data for a test
376 CvRNG* get_rng() { return &rng; }
378 // returns the current error code
379 int get_err_code() { return current_test_info.code; }
381 // retrieves the first registered test
382 CvTest* get_first_test() { return CvTest::get_first_test(); }
384 // retrieves one of global options of the test system
385 bool is_debug_mode() { return params.debug_mode; }
387 // returns the current testing mode
388 int get_testing_mode() { return params.test_mode; }
390 // returns the current timing mode
391 int get_timing_mode() { return params.timing_mode; }
393 // returns the test extensivity scale
394 double get_test_case_count_scale() { return params.test_case_count_scale; }
396 int find_written_param( CvTest* test, const char* paramname,
397 int valtype, const void* val );
399 const char* get_data_path() { return params.data_path ? params.data_path : ""; }
402 // deallocates memory buffers and closes all the streams;
403 // called by init() and from destructor. It does not remove any tests!!!
404 virtual void clear();
406 // retrieves information about the test libraries (names, versions, build dates etc.)
407 virtual const char* get_libs_info( const char** loaded_ipp_modules );
409 // returns textual description of failure code
410 virtual const char* str_from_code( int code );
412 // prints header of summary of test suite run.
413 // It goes before the results of individual tests and contains information about tested libraries
414 // (as reported by get_libs_info()), information about test environment (CPU, test machine name),
415 // date and time etc.
416 virtual void print_summary_header( int streams );
418 // prints tailer of summary of test suite run.
419 // it goes after the results of individual tests and contains the number of
420 // failed tests, total running time, exit code (whether the system has been crashed,
421 // interrupted by the user etc.), names of files with additional information etc.
422 virtual void print_summary_tailer( int streams );
424 // reads common parameters of the test system; called from init()
425 virtual int read_params( CvFileStorage* fs );
427 // checks, whether the test needs to be run (1) or not (0); called from run()
428 virtual int filter( CvTest* test );
430 // makes base name of output files
431 virtual void make_output_stream_base_name( const char* config_name );
433 // forms default test configuration file that can be
434 // customized further
435 virtual void write_default_params( CvFileStorage* fs );
437 // enables/disables the specific output stream[s]
438 virtual void enable_output_streams( int streams, int flag );
440 // sets memory and exception handlers
441 virtual void set_handlers( bool on );
443 // changes the path to test data files
444 virtual void set_data_path( const char* data_path );
446 // prints the information about command-line parameters
447 virtual void print_help();
449 // changes the text color in console
450 virtual void set_color(int color);
452 // a sequence of tests to run
453 CvTestPtrVec* selected_tests;
455 // a sequence of written test params
456 CvTestPtrVec* written_params;
458 // a sequence of failed tests
459 CvTestInfoVec* failed_tests;
461 // base name for output streams
462 char* ostrm_base_name;
463 const char* ostrm_suffixes[MAX_IDX];
465 // parameters that can be read from file storage
468 enum { CHOOSE_TESTS = 0, CHOOSE_FUNCTIONS = 1 };
470 // common parameters:
473 // if non-zero, the tests are run in unprotected mode to debug possible crashes,
474 // otherwise the system tries to catch the exceptions and continue with other tests
477 // if non-zero, the header is not print
480 // if non-zero, the system includes only failed tests into summary
481 bool print_only_failed;
483 // rerun failed tests in debug mode
486 // if non-zero, the failed tests are rerun immediately
487 bool rerun_immediately;
489 // choose_tests or choose_functions;
490 int test_filter_mode;
492 // correctness or performance [or bad-arg, stress etc.]
498 // pattern for choosing tests
499 const char* test_filter_pattern;
501 // RNG seed, passed to and updated by every test executed.
504 // relative or absolute path of directory containing subfolders with test data
505 const char* resource_path;
507 // whether to use IPP, MKL etc. or not
510 // extensivity of the tests, scale factor for test_case_count
511 double test_case_count_scale;
513 // the path to data files used by tests
516 // whether the output to console should be colored
521 // these are allocated within a test to try keep them valid in case of stack corruption
524 // test system start time
527 // test system version (=CV_TS_VERSION by default)
530 // name of config file
531 const char* config_name;
533 // information about the current test
534 CvTestInfo current_test_info;
536 // memory manager used to detect memory corruptions and leaks
537 CvTestMemoryManager* memory_manager;
543 //const char* filename;
544 int default_handle; // for stderr
548 StreamInfo output_streams[MAX_IDX];
549 int ostream_testname_mask;
553 /*****************************************************************************************\
554 * Subclass of CvTest for testing functions that process dense arrays *
555 \*****************************************************************************************/
557 class CV_EXPORTS CvArrTest : public CvTest
560 // constructor(s) and destructor
561 CvArrTest( const char* test_name, const char* test_funcs, const char* test_descr = "" );
562 virtual ~CvArrTest();
564 virtual int write_default_params( CvFileStorage* fs );
565 virtual void clear();
569 virtual int read_params( CvFileStorage* fs );
570 virtual int prepare_test_case( int test_case_idx );
571 virtual int validate_test_results( int test_case_idx );
573 virtual void prepare_to_validation( int test_case_idx );
574 virtual void get_test_array_types_and_sizes( int test_case_idx, CvSize** sizes, int** types );
575 virtual void get_timing_test_array_types_and_sizes( int test_case_idx, CvSize** sizes, int** types,
576 CvSize** whole_sizes, bool *are_images );
577 virtual void fill_array( int test_case_idx, int i, int j, CvMat* arr );
578 virtual void get_minmax_bounds( int i, int j, int type, CvScalar* low, CvScalar* high );
579 virtual double get_success_error_level( int test_case_idx, int i, int j );
580 virtual void print_time( int test_case_idx, double time_usecs );
581 virtual void print_timing_params( int test_case_idx, char* ptr, int params_left=TIMING_EXTRA_PARAMS );
584 bool iplimage_allowed;
586 bool element_wise_relative_error;
588 int min_log_array_size;
589 int max_log_array_size;
591 int max_arr; // = MAX_ARR by default, the number of different types of arrays
592 int max_hdr; // size of header buffer
593 enum { INPUT, INPUT_OUTPUT, OUTPUT, REF_INPUT_OUTPUT, REF_OUTPUT, TEMP, MASK, MAX_ARR };
595 const CvSize* size_list;
596 const CvSize* whole_size_list;
597 const int* depth_list;
600 CvTestPtrVec* test_array;
601 CvMat* test_mat[MAX_ARR];
607 /****************************************************************************************\
608 * Utility Functions *
609 \****************************************************************************************/
611 CV_EXPORTS const char* cvTsGetTypeName( int type );
612 CV_EXPORTS int cvTsTypeByName( const char* type_name );
614 inline int cvTsClipInt( int val, int min_val, int max_val )
623 // return min & max values for given type, e.g. for CV_8S ~ -128 and 127, respectively.
624 CV_EXPORTS double cvTsMinVal( int type );
625 CV_EXPORTS double cvTsMaxVal( int type );
627 // returns c-norm of the array
628 CV_EXPORTS double cvTsMaxVal( const CvMat* arr );
630 inline CvMat* cvTsGetMat( const CvMat* arr, CvMat* stub, int* coi=0 )
632 return cvGetMat( arr, stub, coi );
635 // fills array with random numbers
636 CV_EXPORTS void cvTsRandUni( CvRNG* rng, CvMat* a, CvScalar param1, CvScalar param2 );
638 inline unsigned cvTsRandInt( CvRNG* rng )
641 temp = (uint64)(unsigned)temp*1554115554 + (temp >> 32);
643 return (unsigned)temp;
646 inline double cvTsRandReal( CvRNG* rng )
648 return cvTsRandInt( rng ) * 2.3283064365386962890625e-10 /* 2^-32 */;
651 // fills c with zeros
652 CV_EXPORTS void cvTsZero( CvMat* c, const CvMat* mask=0 );
654 // initializes scaled identity matrix
655 CV_EXPORTS void cvTsSetIdentity( CvMat* c, CvScalar diag_value );
657 // copies a to b (whole matrix or only the selected region)
658 CV_EXPORTS void cvTsCopy( const CvMat* a, CvMat* b, const CvMat* mask=0 );
660 // converts one array to another
661 CV_EXPORTS void cvTsConvert( const CvMat* src, CvMat* dst );
663 // working with multi-channel arrays
664 CV_EXPORTS void cvTsExtract( const CvMat* a, CvMat* plane, int coi );
665 CV_EXPORTS void cvTsInsert( const CvMat* plane, CvMat* a, int coi );
667 // c = alpha*a + beta*b + gamma
668 CV_EXPORTS void cvTsAdd( const CvMat* a, CvScalar alpha, const CvMat* b, CvScalar beta,
669 CvScalar gamma, CvMat* c, int calc_abs );
672 CV_EXPORTS void cvTsMul( const CvMat* _a, const CvMat* _b, CvScalar alpha, CvMat* _c );
675 CV_EXPORTS void cvTsDiv( const CvMat* _a, const CvMat* _b, CvScalar alpha, CvMat* _c );
677 enum { CV_TS_MIN = 0, CV_TS_MAX = 1 };
680 CV_EXPORTS void cvTsMinMax( const CvMat* _a, const CvMat* _b, CvMat* _c, int op_type );
681 CV_EXPORTS void cvTsMinMaxS( const CvMat* _a, double scalar, CvMat* _c, int op_type );
683 // checks that the array does not have NaNs and/or Infs and all the elements are
684 // within [min_val,max_val). idx is the index of the first "bad" element.
685 CV_EXPORTS int cvTsCheck( const CvMat* data, double min_val, double max_val, CvPoint* idx );
687 // compares two arrays. max_diff is the maximum actual difference,
688 // success_err_level is maximum allowed difference, idx is the index of the first
689 // element for which difference is >success_err_level
690 // (or index of element with the maximum difference)
691 CV_EXPORTS int cvTsCmpEps( const CvMat* data, const CvMat* etalon, double* max_diff,
692 double success_err_level, CvPoint* idx,
693 bool element_wise_relative_error );
695 // a wrapper for the previous function. in case of error prints the message to log file.
696 CV_EXPORTS int cvTsCmpEps2( CvTS* ts, const CvArr* _a, const CvArr* _b, double success_err_level,
697 bool element_wise_relative_error, const char* desc );
699 CV_EXPORTS int cvTsCmpEps2_64f( CvTS* ts, const double* val, const double* ref_val, int len,
700 double eps, const char* param_name );
702 // compares two arrays. the result is 8s image that takes values -1, 0, 1
703 CV_EXPORTS void cvTsCmp( const CvMat* a, const CvMat* b, CvMat* result, int cmp_op );
705 // compares array and a scalar.
706 CV_EXPORTS void cvTsCmpS( const CvMat* a, double fval, CvMat* result, int cmp_op );
708 // retrieves C, L1 or L2 norm of array or its region
709 CV_EXPORTS double cvTsNorm( const CvMat* _arr, const CvMat* _mask, int norm_type, int coi );
711 // retrieves mean, standard deviation and the number of nonzero mask pixels
712 CV_EXPORTS int cvTsMeanStdDevNonZero( const CvMat* _arr, const CvMat* _mask,
713 CvScalar* _mean, CvScalar* _stddev, int coi );
715 // retrieves global extremums and their positions
716 CV_EXPORTS void cvTsMinMaxLoc( const CvMat* _arr, const CvMat* _mask,
717 double* _minval, double* _maxval,
718 CvPoint* _minidx, CvPoint* _maxidx, int coi );
720 enum { CV_TS_LOGIC_AND = 0, CV_TS_LOGIC_OR = 1, CV_TS_LOGIC_XOR = 2, CV_TS_LOGIC_NOT = 3 };
722 CV_EXPORTS void cvTsLogic( const CvMat* a, const CvMat* b, CvMat* c, int logic_op );
723 CV_EXPORTS void cvTsLogicS( const CvMat* a, CvScalar s, CvMat* c, int logic_op );
725 enum { CV_TS_GEMM_A_T = 1, CV_TS_GEMM_B_T = 2, CV_TS_GEMM_C_T = 4 };
727 CV_EXPORTS void cvTsGEMM( const CvMat* a, const CvMat* b, double alpha,
728 const CvMat* c, double beta, CvMat* d, int flags );
730 CV_EXPORTS void cvTsConvolve2D( const CvMat* a, CvMat* b, const CvMat* kernel, CvPoint anchor );
731 // op_type == CV_TS_MIN/CV_TS_MAX
732 CV_EXPORTS void cvTsMinMaxFilter( const CvMat* a, CvMat* b,
733 const IplConvKernel* element, int op_type );
735 enum { CV_TS_BORDER_REPLICATE=0, CV_TS_BORDER_REFLECT=1, CV_TS_BORDER_FILL=2 };
737 CV_EXPORTS void cvTsPrepareToFilter( const CvMat* a, CvMat* b, CvPoint ofs,
738 int border_mode = CV_TS_BORDER_REPLICATE,
739 CvScalar fill_val=cvScalarAll(0));
741 CV_EXPORTS double cvTsCrossCorr( const CvMat* a, const CvMat* b );
743 CV_EXPORTS CvMat* cvTsSelect( const CvMat* a, CvMat* header, CvRect rect );
745 CV_EXPORTS CvMat* cvTsTranspose( const CvMat* a, CvMat* b );
746 CV_EXPORTS void cvTsFlip( const CvMat* a, CvMat* b, int flip_type );
748 CV_EXPORTS void cvTsTransform( const CvMat* a, CvMat* b, const CvMat* transmat, const CvMat* shift );
750 // modifies values that are close to zero
751 CV_EXPORTS void cvTsPatchZeros( CvMat* mat, double level );