[Monetdb-developers] How to create Tests

15 Jul 2004

Hi,

I haven't had any time, yet, to prepare a real HowTo on this topic,
but for those of you who are interested, here's a trace of what I did
to make Arjan's sql/src/test/bugs/bug-sf-990096.sql script into a "real"
test.

Stefan

========

14:51:06 manegold@draco:~/Monet/4.3/sql $ echo 'bug-sf-990096' >> src/test/bugs/Tests/All

14:51:13 manegold@draco:~/Monet/4.3/sql $ echo '$RELSRCDIR/../bug-sf-990096.sql' > src/test/bugs/Tests/bug-sf-990096.sql.src

14:51:21 manegold@draco:~/Monet/4.3/sql $ Mtest_sql src/test/bugs bug-sf-990096
IGNORE = ^#
TIMEOUT = 60
RECURSIVE = 0
CONTEXT = -C1
ACCURACY = 1
MONET_SOURCE = /ufs/manegold/Monet/4.3/MonetDB
MONET_BUILD = /var/tmp/_BUILD_MONET_GNU_32_d__
MONET_PREFIX = /var/tmp/_PREFIX__GNU_32_d__
TSTSRCBASE = /ufs/manegold/Monet/4.3/sql
TSTBLDBASE = /var/tmp/_BUILD_SQL_GNU_32_d__
TSTTRGBASE = /var/tmp/_PREFIX__GNU_32_d__
SQL_PREFIX = /var/tmp/_PREFIX__GNU_32_d__
SQLCLIENT = MapiClient --language=sql
MAPICLIENT = MapiClient
X100CLIENT = x100run_mapi
MAPIPORT = 40891
SQLPORT = 52395
GDK_DEBUG = 10
MONET_MOD_PATH = /var/tmp/_PREFIX__GNU_32_d__/lib/MonetDB
SQL_LOGDIR = /var/tmp/_PREFIX__GNU_32_d__/var/MonetDB/log
GDK_DBFARM = /var/tmp/_PREFIX__GNU_32_d__/var/MonetDB/dbfarm
setMONET_MOD_PATH = --set "monet_mod_path=/var/tmp/_PREFIX__GNU_32_d__/lib/MonetDB"
MAPI_MIL = /var/tmp/_BUILD_MONET_GNU_32_d__/src/mapi/mapi.mil
SQL_SERVER = sql_server
SQL_SERVER_MIL = /var/tmp/_BUILD_SQL_GNU_32_d__/src/backends/monet4/sql_server.mil
MDIFF = /var/tmp/_PREFIX__GNU_32_d__/bin/Mdiff : Mdiff
MSERVER = /var/tmp/_PREFIX__GNU_32_d__/bin/Mserver : Mserver "--config=/var/tmp/_PREFIX__GNU_32_d__/etc/monet.conf" --debug=10 --set "monet_mod_path=/var/tmp/_PREFIX__GNU_32_d__/lib/MonetDB" --set "gdk_dbfarm=/var/tmp/_PREFIX__GNU_32_d__/var/MonetDB/dbfarm" --set "sql_logdir=/var/tmp/_PREFIX__GNU_32_d__/var/MonetDB/log" --set mapi_port=40891 --set sql_port=52395 --set monet_prompt= --trace
MTIMEOUT = /var/tmp/_PREFIX__GNU_32_d__/bin/Mtimeout : Mtimeout -timeout 60 
MSHUTDOWN = /var/tmp/_PREFIX__GNU_32_d__/bin/Mshutdown.py : Mshutdown.py "--config=/var/tmp/_PREFIX__GNU_32_d__/etc/monet.conf" --host=draco --port=40891
PYTHON = /usr/bin/python : python
SQL_CLIENT = /var/tmp/_PREFIX__GNU_32_d__/bin/MapiClient : MapiClient --language=sql -u monetdb -P monetdb --host=draco --port=52395 
X100_CLIENT =  : x100run_mapi -u monetdb -P monetdb "--config=/var/tmp/_PREFIX__GNU_32_d__/etc/monet.conf" --host=draco --port=40891
MAPI_CLIENT = /var/tmp/_PREFIX__GNU_32_d__/bin/MapiClient : MapiClient -u monetdb -P monetdb "--config=/var/tmp/_PREFIX__GNU_32_d__/etc/monet.conf" --host=draco --port=40891
Bits:  32bit
Modules:  ('main', 'kernel', '_kernel', 'arith', '_arith', 'str', '_str', 'uchr', '_uchr', 'bat', '_bat', 'algebra', '_algebra', 'sys', '_sys', 'trans', '_trans', 'streams', 'xtables', 'mkey', 'profiler', 'aggr', 'aggrX3', 'alarm', 'ascii_io', 'bat_arith', 'blob', 'decimal', 'enum', 'lock', 'mmath', 'monettime', 'radix', 'stats', 'tcpip', 'unix', 'url', 'counters', 'pcl', 'bitset', 'bitvector', 'cluster', 'iterator', 'malalgebra', 'mel', 'mprof', 'oo7', 'qt', 'salgebra', 'tpcd', 'vector', 'wisc', 'ddbench', 'bat_mmath', 'bat_cast', 'array', 'color', 'image', 'txtsim', 'calib', 'mapi', 'sql_server', 'pf_support')

Running tests ['bug-sf-990096'] in directory /ufs/manegold/Monet/4.3/sql/src/test/bugs.

14:51:29> src/test/bugs/bug-sf-990096.sql  (<=240s) ...   1.167s 
bug-sf-990096.stable.out.FILTERED and bug-sf-990096.test.out.FILTERED differ SIGNIFICANTLY!
bug-sf-990096.stable.err.FILTERED and bug-sf-990096.test.err.FILTERED differ SIGNIFICANTLY!

 !ERROR:  Testing FAILED SIGNIFICANTLY ! 

    1 out of   1 tests produced SIGNIFICANTLY different output

 First, check the testing results in  /var/tmp/_PREFIX__GNU_32_d__/mTests/index.html  ! 

 Then, fix the problems by: 
  - fixing sources and test scripts 
  - fixing stable output by hand 
  - approving test output by Mapprove.py (cf. Mapprove.py -?) 

 After that, re-run Mtest. 


14:51:32 manegold@draco:~/Monet/4.3/sql $ Mapprove_sql src/test/bugs bug-sf-990096
FORCE = 0
RECURSIVE = 0
EXTENSION = ['out', 'err']
SYSTEM = 
MONET_SOURCE = /ufs/manegold/Monet/4.3/MonetDB
MONET_BUILD = /var/tmp/_BUILD_MONET_GNU_32_d__
MONET_PREFIX = /var/tmp/_PREFIX__GNU_32_d__
TSTSRCBASE = /ufs/manegold/Monet/4.3/sql
TSTBLDBASE = /var/tmp/_BUILD_SQL_GNU_32_d__
TSTTRGBASE = /var/tmp/_PREFIX__GNU_32_d__
SQL_PREFIX = /var/tmp/_PREFIX__GNU_32_d__

Running tests ['bug-sf-990096'] in directory /ufs/manegold/Monet/4.3/sql/src/test/bugs.

Approving  src/test/bugs/bug-sf-990096.test.out  ->  stable.out   (creating new file)
patching file /ufs/manegold/Monet/4.3/sql/src/test/bugs/Tests/bug-sf-990096.stable.out
Approving  src/test/bugs/bug-sf-990096.test.err  ->  stable.err   (creating new file)

Mapprove.py  Warning:  Skipping new (error) message: "+ERROR = !Column: a.i unknown"

patching file /ufs/manegold/Monet/4.3/sql/src/test/bugs/Tests/bug-sf-990096.stable.err

 First, run 'cvs -q diff to check what you have changed.

 Then, re-run Mtest.py.

 In case (some of) the skipped error messages are correct/expected,
 re-run Mapprove.py with -f to force their approval.


14:51:51 manegold@draco:~/Monet/4.3/sql $ joe src/test/bugs/Tests/bug-sf-990096.stable.???
Processing '/ufs/manegold/.joerc'...done
Processing '/ufs/manegold/.joerc'...done

File src/test/bugs/Tests/bug-sf-990096.stable.err not changed so no update needed.

14:52:45 manegold@draco:~/Monet/4.3/sql $ diff -u src/test/bugs/Tests/bug-sf-990096.stable.out{~,}

--- src/test/bugs/Tests/bug-sf-990096.stable.out~       2004-07-13 14:52:43.000000000 +0200
+++ src/test/bugs/Tests/bug-sf-990096.stable.out        2004-07-13 14:52:43.000000000 +0200
@@ -27,6 +27,12 @@
 # 0,    # id
 [ 1,   1       ]
 
+# i,   i # name
+# mediumint,   mediumint # type
+# 1,    # tuplecount
+# 0,    # id
+[ 1,   1       ]
+
 # 14:51:31 >  
 # 14:51:31 >  Done.
 # 14:51:31 >  

14:52:54 manegold@draco:~/Monet/4.3/sql $ diff -u src/test/bugs/Tests/bug-sf-990096.stable.err{~,}
--- src/test/bugs/Tests/bug-sf-990096.stable.err~       2004-07-13 14:52:36.000000000 +0200
+++ src/test/bugs/Tests/bug-sf-990096.stable.err        2004-07-13 14:52:36.000000000 +0200
@@ -56,10 +56,6 @@
 # 14:51:30 >  Mtimeout -timeout 60 MapiClient --language=sql -u monetdb -P monetdb --host=draco --port=52395  < bug-sf-990096.sql
 # 14:51:30 >  
 
-MAPI  = monetdb@draco:52395
-ACTION= read_into_cache
-QUERY = SELECT a.i FROM
-
 
 # 14:51:31 >  
 # 14:51:31 >  Done.

14:52:58 manegold@draco:~/Monet/4.3/sql $ Mtest_sql src/test/bugs bug-sf-990096
IGNORE = ^#
TIMEOUT = 60
RECURSIVE = 0
CONTEXT = -C1
ACCURACY = 1
MONET_SOURCE = /ufs/manegold/Monet/4.3/MonetDB
MONET_BUILD = /var/tmp/_BUILD_MONET_GNU_32_d__
MONET_PREFIX = /var/tmp/_PREFIX__GNU_32_d__
TSTSRCBASE = /ufs/manegold/Monet/4.3/sql
TSTBLDBASE = /var/tmp/_BUILD_SQL_GNU_32_d__
TSTTRGBASE = /var/tmp/_PREFIX__GNU_32_d__
SQL_PREFIX = /var/tmp/_PREFIX__GNU_32_d__
SQLCLIENT = MapiClient --language=sql
MAPICLIENT = MapiClient
X100CLIENT = x100run_mapi
MAPIPORT = 46225
SQLPORT = 53058
GDK_DEBUG = 10
MONET_MOD_PATH = /var/tmp/_PREFIX__GNU_32_d__/lib/MonetDB
SQL_LOGDIR = /var/tmp/_PREFIX__GNU_32_d__/var/MonetDB/log
GDK_DBFARM = /var/tmp/_PREFIX__GNU_32_d__/var/MonetDB/dbfarm
setMONET_MOD_PATH = --set "monet_mod_path=/var/tmp/_PREFIX__GNU_32_d__/lib/MonetDB"
MAPI_MIL = /var/tmp/_BUILD_MONET_GNU_32_d__/src/mapi/mapi.mil
SQL_SERVER = sql_server
SQL_SERVER_MIL = /var/tmp/_BUILD_SQL_GNU_32_d__/src/backends/monet4/sql_server.mil
MDIFF = /var/tmp/_PREFIX__GNU_32_d__/bin/Mdiff : Mdiff
MSERVER = /var/tmp/_PREFIX__GNU_32_d__/bin/Mserver : Mserver "--config=/var/tmp/_PREFIX__GNU_32_d__/etc/monet.conf" --debug=10 --set "monet_mod_path=/var/tmp/_PREFIX__GNU_32_d__/lib/MonetDB" --set "gdk_dbfarm=/var/tmp/_PREFIX__GNU_32_d__/var/MonetDB/dbfarm" --set "sql_logdir=/var/tmp/_PREFIX__GNU_32_d__/var/MonetDB/log" --set mapi_port=46225 --set sql_port=53058 --set monet_prompt= --trace
MTIMEOUT = /var/tmp/_PREFIX__GNU_32_d__/bin/Mtimeout : Mtimeout -timeout 60 
MSHUTDOWN = /var/tmp/_PREFIX__GNU_32_d__/bin/Mshutdown.py : Mshutdown.py "--config=/var/tmp/_PREFIX__GNU_32_d__/etc/monet.conf" --host=draco --port=46225
PYTHON = /usr/bin/python : python
SQL_CLIENT = /var/tmp/_PREFIX__GNU_32_d__/bin/MapiClient : MapiClient --language=sql -u monetdb -P monetdb --host=draco --port=53058 
X100_CLIENT =  : x100run_mapi -u monetdb -P monetdb "--config=/var/tmp/_PREFIX__GNU_32_d__/etc/monet.conf" --host=draco --port=46225
MAPI_CLIENT = /var/tmp/_PREFIX__GNU_32_d__/bin/MapiClient : MapiClient -u monetdb -P monetdb "--config=/var/tmp/_PREFIX__GNU_32_d__/etc/monet.conf" --host=draco --port=46225
Bits:  32bit
Modules:  ('main', 'kernel', '_kernel', 'arith', '_arith', 'str', '_str', 'uchr', '_uchr', 'bat', '_bat', 'algebra', '_algebra', 'sys', '_sys', 'trans', '_trans', 'streams', 'xtables', 'mkey', 'profiler', 'aggr', 'aggrX3', 'alarm', 'ascii_io', 'bat_arith', 'blob', 'decimal', 'enum', 'lock', 'mmath', 'monettime', 'radix', 'stats', 'tcpip', 'unix', 'url', 'counters', 'pcl', 'bitset', 'bitvector', 'cluster', 'iterator', 'malalgebra', 'mel', 'mprof', 'oo7', 'qt', 'salgebra', 'tpcd', 'vector', 'wisc', 'ddbench', 'bat_mmath', 'bat_cast', 'array', 'color', 'image', 'txtsim', 'calib', 'mapi', 'sql_server', 'pf_support')

Running tests ['bug-sf-990096'] in directory /ufs/manegold/Monet/4.3/sql/src/test/bugs.


14:53:17> src/test/bugs/bug-sf-990096.sql  (<=240s) ...   1.065s 
bug-sf-990096.stable.out.FILTERED and bug-sf-990096.test.out.FILTERED differ SIGNIFICANTLY!
bug-sf-990096.stable.err.FILTERED and bug-sf-990096.test.err.FILTERED differ SIGNIFICANTLY!

 !ERROR:  Testing FAILED SIGNIFICANTLY ! 

    1 out of   1 tests produced SIGNIFICANTLY different output

 First, check the testing results in  /var/tmp/_PREFIX__GNU_32_d__/mTests/index.html  ! 

 Then, fix the problems by: 
  - fixing sources and test scripts 
  - fixing stable output by hand 
  - approving test output by Mapprove.py (cf. Mapprove.py -?) 

 After that, re-run Mtest. 


14:53:19 manegold@draco:~/Monet/4.3/sql $ rm src/test/bugs/Tests/bug-sf-990096.stable.???~
-rw-r-----    1 manegold ins          2807 Jul 13 14:52 src/test/bugs/Tests/bug-sf-990096.stable.err~
-rw-r-----    1 manegold ins          1140 Jul 13 14:52 src/test/bugs/Tests/bug-sf-990096.stable.out~
Delete these 2 files/directories? (y/N) \c
y
2 files deleted.

14:53:56 manegold@draco:~/Monet/4.3/sql $ cvs add src/test/bugs/Tests/bug-sf-990096.*
cvs add: warning: unrecognized response '' from cvs server
cvs add: use 'cvs commit' to add these files permanently

14:54:16 manegold@draco:~/Monet/4.3/sql $ cvs up src/test/bugs/Tests
M src/test/bugs/Tests/All
M src/test/bugs/Tests/bug-00001.stable.out
A src/test/bugs/Tests/bug-sf-990096.sql.src
A src/test/bugs/Tests/bug-sf-990096.stable.err
A src/test/bugs/Tests/bug-sf-990096.stable.out

14:54:34 manegold@draco:~/Monet/4.3/sql $ cvs diff src/test/bugs/Tests
Index: src/test/bugs/Tests/All
===================================================================
RCS file: /cvsroot/monetdb/sql/src/test/bugs/Tests/All,v
retrieving revision 1.17
diff -u -r1.17 All
--- src/test/bugs/Tests/All     24 Jun 2004 15:26:59 -0000      1.17
+++ src/test/bugs/Tests/All     13 Jul 2004 12:54:42 -0000
@@ -25,3 +25,4 @@
 bug-sf-975308
 bug-sf-978043
 bug-sf-978045
+bug-sf-990096
Index: src/test/bugs/Tests/bug-00001.stable.out
===================================================================
RCS file: /cvsroot/monetdb/sql/src/test/bugs/Tests/bug-00001.stable.out,v
retrieving revision 1.3
diff -u -r1.3 bug-00001.stable.out
--- src/test/bugs/Tests/bug-00001.stable.out    20 Jun 2004 21:46:22 -0000      1.3
+++ src/test/bugs/Tests/bug-00001.stable.out    13 Jul 2004 12:54:42 -0000
@@ -60,8 +60,8 @@
 [ 1005711,     "privileges",   1005548,        "",     1       ]
 [ 1005740,     "history",      1005548,        "",     1       ]
 [ 1005778,     "env",  1005548,        "",     1       ]
-[ 1007047,     "foo",  1005548,        "",     0       ]
-[ 1007212,     "ff",   1005548,        "",     0       ]
+[ 1007035,     "foo",  1005548,        "",     0       ]
+[ 1007200,     "ff",   1005548,        "",     0       ]
 # i # name
 # int # type
 # 2 # tuplecount
@@ -91,9 +91,9 @@
 [ 1005711,     "privileges",   1005548,        "",     1       ]
 [ 1005740,     "history",      1005548,        "",     1       ]
 [ 1005778,     "env",  1005548,        "",     1       ]
-[ 1007047,     "foo",  1005548,        "",     0       ]
-[ 1007212,     "ff",   1005548,        "",     0       ]
-[ 1007468,     "s4",   1005548,        "",     0       ]
+[ 1007035,     "foo",  1005548,        "",     0       ]
+[ 1007200,     "ff",   1005548,        "",     0       ]
+[ 1007456,     "s4",   1005548,        "",     0       ]
 # i # name
 # int # type
 # 1 # tuplecount
cvs diff: src/test/bugs/Tests/bug-sf-990096.sql.src is a new entry, no comparison available
cvs diff: src/test/bugs/Tests/bug-sf-990096.stable.err is a new entry, no comparison available
cvs diff: src/test/bugs/Tests/bug-sf-990096.stable.out is a new entry, no comparison available

14:54:43 manegold@draco:~/Monet/4.3/sql $ 
========

-- 
| Dr. Stefan Manegold | mailto:Stefan.Manegold@cwi.nl |
| CWI,  P.O.Box 94079 | http://www.cwi.nl/~manegold/  |
| 1090 GB Amsterdam   | Tel.: +31 (20) 592-4212       |
| The Netherlands     | Fax : +31 (20) 592-4312       |

    

Stefan.Manegold＠cwi.nl

tags

participants (1)