monotone

monotone Mtn Source Tree

Root/testlib.lua

1-- misc global values
2-- where the main testsuite file is
3srcdir = get_source_dir()
4-- where the individual test dirs are
5-- most paths will be testdir.."/something"
6-- normally reset by the main testsuite file
7testdir = srcdir
8-- was the -d switch given?
9debugging = false
10
11-- combined logfile; tester.cc will reset this to a filename, which is
12-- then opened in run_tests
13logfile = nil
14
15-- This is for redirected output from local implementations
16-- of shellutils type stuff (ie, grep).
17-- Reason: {set,clear}_redirect don't seem to (always?) work
18-- for this (at least on Windows).
19files = {stdout = nil, stdin = nil, stderr = nil}
20
21-- for convenience, this is the first word of what get_ostype() returns.
22ostype = string.sub(get_ostype(), 1, string.find(get_ostype(), " ")-1)
23
24-- table of per-test values
25test = {}
26-- misc per-test values
27test.root = nil
28test.name = nil
29test.wanted_fail = false
30test.partial_skip = false -- set this to true if you skip part of the test
31
32--probably should put these in the error that gets thrown...
33test.errfile = ""
34test.errline = -1
35
36-- for tracking background processes
37test.bgid = 0
38test.bglist = {}
39
40test.log = nil -- logfile for this test
41
42-- hook to be overridden by the main testsuite file, if necessary;
43-- called after determining the set of tests to run.
44-- P may be used to write messages to the user's tty.
45function prepare_to_run_tests(P)
46 return 0
47end
48
49-- hook to be overridden by the main testsuite file, if necessary;
50-- called after opening the master logfile, but _before_ parsing
51-- arguments or determining the set of tests to run.
52-- P may be used to write messages to the user's tty.
53function prepare_to_enumerate_tests(P)
54 return 0
55end
56
57function L(...)
58 test.log:write(unpack(arg))
59 test.log:flush()
60end
61
62function getsrcline()
63 local info
64 local depth = 1
65 repeat
66 depth = depth + 1
67 info = debug.getinfo(depth)
68 until info == nil
69 while src == nil and depth > 1 do
70 depth = depth - 1
71 info = debug.getinfo(depth)
72 if string.find(info.source, "^@.*__driver__%.lua") then
73 -- return info.source, info.currentline
74 return test.name, info.currentline
75 end
76 end
77end
78
79function locheader()
80 local _,line = getsrcline()
81 if line == nil then line = -1 end
82 if test.name == nil then
83 return "\n<unknown>:" .. line .. ": "
84 else
85 return "\n" .. test.name .. ":" .. line .. ": "
86 end
87end
88
89function err(what, level)
90 if level == nil then level = 2 end
91 test.errfile, test.errline = getsrcline()
92 local e
93 if type(what) == "table" then
94 e = what
95 if e.bt == nil then e.bt = {} end
96 table.insert(e.bt, debug.traceback())
97 else
98 e = {e = what, bt = {debug.traceback()}}
99 end
100 error(e, level)
101end
102
103do -- replace some builtings with logged versions
104 unlogged_mtime = mtime
105 mtime = function(name)
106 local x = unlogged_mtime(name)
107 L(locheader(), "mtime(", name, ") = ", tostring(x), "\n")
108 return x
109 end
110
111 unlogged_mkdir = mkdir
112 mkdir = function(name)
113 L(locheader(), "mkdir ", name, "\n")
114 unlogged_mkdir(name)
115 end
116
117 unlogged_existsonpath = existsonpath
118 existsonpath = function(name)
119 local r = (unlogged_existsonpath(name) == 0)
120 local what
121 if r then
122 what = "exists"
123 else
124 what = "does not exist"
125 end
126 L(locheader(), name, " ", what, " on the path\n")
127 return r
128 end
129end
130
131function numlines(filename)
132 local n = 0
133 for _ in io.lines(filename) do n = n + 1 end
134 L(locheader(), "numlines(", filename, ") = ", n, "\n")
135 return n
136end
137
138function open_or_err(filename, mode, depth)
139 local file, e = io.open(filename, mode)
140 if file == nil then
141 err("Cannot open file " .. filename .. ": " .. e, depth)
142 end
143 return file
144end
145
146function fsize(filename)
147 local file = open_or_err(filename, "r", 3)
148 local size = file:seek("end")
149 file:close()
150 return size
151end
152
153function readfile_q(filename)
154 local file = open_or_err(filename, "rb", 3)
155 local dat = file:read("*a")
156 file:close()
157 return dat
158end
159
160function readfile(filename)
161 L(locheader(), "readfile ", filename, "\n")
162 return readfile_q(filename)
163end
164
165function readstdfile(filename)
166 return readfile(testdir.."/"..filename)
167end
168
169-- Return all but the first N lines of FILENAME.
170-- Note that (unlike readfile()) the result will
171-- end with a \n whether or not the file did.
172function tailfile(filename, n)
173 L(locheader(), "tailfile ", filename, ", ", n, "\n")
174 local i = 1
175 local t = {}
176 for l in io.lines(filename) do
177 if i > n then
178 table.insert(t, l)
179 end
180 i = i + 1
181 end
182 table.insert(t, "")
183 return table.concat(t, "\n")
184end
185
186function writefile_q(filename, dat)
187 local file,e
188 if dat == nil then
189 file,e = open_or_err(filename, "a+b", 3)
190 else
191 file,e = open_or_err(filename, "wb", 3)
192 end
193 if dat ~= nil then
194 file:write(dat)
195 end
196 file:close()
197 return true
198end
199
200function writefile(filename, dat)
201 L(locheader(), "writefile ", filename, "\n")
202 return writefile_q(filename, dat)
203end
204
205function append(filename, dat)
206 L(locheader(), "append to file ", filename, "\n")
207 local file,e = open_or_err(filename, "a+", 3)
208 file:write(dat)
209 file:close()
210 return true
211end
212
213do
214 unlogged_copy = copy_recursive
215 copy_recursive = nil
216 function copy(from, to)
217 L(locheader(), "copy ", from, " -> ", to, "\n")
218 local ok, res = unlogged_copy(from, to)
219 if not ok then
220 L(res, "\n")
221 return false
222 else
223 return true
224 end
225 end
226end
227
228do
229 local os_rename = os.rename
230 os.rename = nil
231 os.remove = nil
232 function rename(from, to)
233 L(locheader(), "rename ", from, " ", to, "\n")
234 if exists(to) and not isdir(to) then
235 L("Destination ", to, " exists; removing...\n")
236 local ok, res = unlogged_remove(to)
237 if not ok then
238 L("Could not remove ", to, ": ", res, "\n")
239 return false
240 end
241 end
242 local ok,res = os_rename(from, to)
243 if not ok then
244 L(res, "\n")
245 return false
246 else
247 return true
248 end
249 end
250 function unlogged_rename(from, to)
251 if exists(to) and not isdir(to) then
252 unlogged_remove(to)
253 end
254 os_rename(from, to)
255 end
256 unlogged_remove = remove_recursive
257 remove_recursive = nil
258 function remove(file)
259 L(locheader(), "remove ", file, "\n")
260 local ok,res = unlogged_remove(file)
261 if not ok then
262 L(res, "\n")
263 return false
264 else
265 return true
266 end
267 end
268end
269
270
271function getstd(name, as)
272 if as == nil then as = name end
273 local ret = copy(testdir .. "/" .. name, as)
274 make_tree_accessible(as)
275 return ret
276end
277
278function get(name, as)
279 if as == nil then as = name end
280 return getstd(test.name .. "/" .. name, as)
281end
282
283-- include from the main tests directory; there's no reason
284-- to want to include from the dir for the current test,
285-- since in that case it could just go in the driver file.
286function include(name)
287 local func, e = loadfile(testdir.."/"..name)
288 if func == nil then err(e, 2) end
289 setfenv(func, getfenv(2))
290 func()
291end
292
293function trim(str)
294 return string.gsub(str, "^%s*(.-)%s*$", "%1")
295end
296
297function getpathof(exe, ext)
298 local function gotit(now)
299 if test.log == nil then
300 logfile:write(exe, " found at ", now, "\n")
301 else
302 test.log:write(exe, " found at ", now, "\n")
303 end
304 return now
305 end
306 local path = os.getenv("PATH")
307 local char
308 if ostype == "Windows" then
309 char = ';'
310 else
311 char = ':'
312 end
313 if ostype == "Windows" then
314 if ext == nil then ext = ".exe" end
315 else
316 if ext == nil then ext = "" end
317 end
318 local now = initial_dir.."/"..exe..ext
319 if exists(now) then return gotit(now) end
320 for x in string.gmatch(path, "[^"..char.."]*"..char) do
321 local dir = string.sub(x, 0, -2)
322 if string.find(dir, "[\\/]$") then
323 dir = string.sub(dir, 0, -2)
324 end
325 local now = dir.."/"..exe..ext
326 if exists(now) then return gotit(now) end
327 end
328 if test.log == nil then
329 logfile:write("Cannot find ", exe, "\n")
330 else
331 test.log:write("Cannot find ", exe, "\n")
332 end
333 return nil
334end
335
336function prepare_redirect(fin, fout, ferr)
337 local cwd = chdir(".").."/"
338 redir = {fin = cwd..fin, fout = cwd..fout, ferr = cwd..ferr}
339end
340do
341 oldspawn = spawn
342 function spawn(...)
343 if redir == nil then
344 return oldspawn(unpack(arg))
345 else
346 return spawn_redirected(redir.fin, redir.fout, redir.ferr, unpack(arg))
347 end
348 end
349end
350function execute(path, ...)
351 local pid
352 local ret = -1
353 pid = spawn(path, unpack(arg))
354 redir = nil
355 if (pid ~= -1) then ret, pid = wait(pid) end
356 return ret
357end
358
359function cmd_as_str(cmd_table)
360 local str = ""
361 for i,x in ipairs(cmd_table) do
362 if str ~= "" then str = str .. " " end
363 if type(x) == "function" then
364 str = str .. "<function>"
365 else
366 local s = tostring(x)
367 if string.find(s, " ") then
368 str = str .. '"'..s..'"'
369 else
370 str = str .. s
371 end
372 end
373 end
374 return str
375end
376
377function runcmd(cmd, prefix, bgnd)
378 if prefix == nil then prefix = "ts-" end
379 if type(cmd) ~= "table" then err("runcmd called with bad argument") end
380 local local_redir = cmd.local_redirect
381 if cmd.local_redirect == nil then
382 if type(cmd[1]) == "function" then
383 local_redir = true
384 else
385 local_redir = false
386 end
387 end
388 if bgnd == true and type(cmd[1]) == "string" then local_redir = false end
389 L("\nruncmd: ", tostring(cmd[1]), ", local_redir = ", tostring(local_redir), ", requested = ", tostring(cmd.local_redirect))
390 local redir
391 if local_redir then
392 files.stdin = open_or_err(prefix.."stdin", nil, 2)
393 files.stdout = open_or_err(prefix.."stdout", "w", 2)
394 files.stderr = open_or_err(prefix.."stderr", "w", 2)
395 else
396 prepare_redirect(prefix.."stdin", prefix.."stdout", prefix.."stderr")
397 end
398
399 local result
400 if cmd.logline ~= nil then
401 L(locheader(), cmd.logline, "\n")
402 else
403 L(locheader(), cmd_as_str(cmd), "\n")
404 end
405
406 local oldexec = execute
407 if bgnd then
408 execute = spawn
409 end
410 if type(cmd[1]) == "function" then
411 result = {pcall(unpack(cmd))}
412 elseif type(cmd[1]) == "string" then
413 result = {pcall(execute, unpack(cmd))}
414 else
415 execute = oldexec
416 err("runcmd called with bad command table " ..
417"(first entry is a " .. type(cmd[1]) ..")")
418 end
419 execute = oldexec
420
421 if local_redir then
422 files.stdin:close()
423 files.stdout:close()
424 files.stderr:close()
425 end
426 return unpack(result)
427end
428
429function samefile(left, right)
430 if left == "-" or right == "-" then
431 err("tests may not rely on standard input")
432 end
433 if fsize(left) ~= fsize(right) then
434 return false
435 else
436 local ldat = readfile(left)
437 local rdat = readfile(right)
438 return ldat == rdat
439 end
440end
441
442function samelines(f, t)
443 local fl = {}
444 for l in io.lines(f) do table.insert(fl, l) end
445 if not table.getn(fl) == table.getn(t) then
446 L(locheader(), string.format("file has %s lines; table has %s\n",
447 table.getn(fl), table.getn(t)))
448 return false
449 end
450 for i=1,table.getn(t) do
451 if fl[i] ~= t[i] then
452 if fl[i] then
453 L(locheader(), string.format("file[i] = '%s'; table[i] = '%s'\n",
454 fl[i], t[i]))
455 else
456 L(locheader(), string.format("file[i] = ''; table[i] = '%s'\n",
457 t[i]))
458 end
459 return false
460 end
461 end
462 return true
463end
464
465function greplines(f, t)
466 local fl = {}
467 for l in io.lines(f) do table.insert(fl, l) end
468 if not table.getn(fl) == table.getn(t) then
469 L(locheader(), string.format("file has %s lines; table has %s\n",
470 table.getn(fl), table.getn(t)))
471 return false
472 end
473 for i=1,table.getn(t) do
474 if not regex.search(t[i], fl[i]) then
475 L(locheader(), string.format("file[i] = '%s'; table[i] = '%s'\n",
476 fl[i], t[i]))
477 return false
478 end
479 end
480 return true
481end
482
483function grep(...)
484 local flags, what, where = unpack(arg)
485 local dogrep = function ()
486 if where == nil and string.sub(flags, 1, 1) ~= "-" then
487 where = what
488 what = flags
489 flags = ""
490 end
491 local quiet = string.find(flags, "q") ~= nil
492 local reverse = string.find(flags, "v") ~= nil
493 if not quiet and files.stdout == nil then err("non-quiet grep not redirected") end
494 local out = 1
495 local infile = files.stdin
496 if where ~= nil then infile = open_or_err(where) end
497 for line in infile:lines() do
498 local matched = regex.search(what, line)
499 if reverse then matched = not matched end
500 if matched then
501 if not quiet then files.stdout:write(line, "\n") end
502 out = 0
503 end
504 end
505 if where ~= nil then infile:close() end
506 return out
507 end
508 return {dogrep, logline = "grep "..cmd_as_str(arg)}
509end
510
511function cat(...)
512 local arguments = arg
513 local function docat()
514 local bsize = 8*1024
515 for _,x in ipairs(arguments) do
516 local infile
517 if x == "-" then
518 infile = files.stdin
519 else
520 infile = open_or_err(x, "rb", 3)
521 end
522 local block = infile:read(bsize)
523 while block do
524 files.stdout:write(block)
525 block = infile:read(bsize)
526 end
527 if x ~= "-" then
528 infile:close()
529 end
530 end
531 return 0
532 end
533 return {docat, logline = "cat "..cmd_as_str(arg)}
534end
535
536function tail(...)
537 local file, num = unpack(arg)
538 local function dotail()
539 if num == nil then num = 10 end
540 local mylines = {}
541 for l in io.lines(file) do
542 table.insert(mylines, l)
543 if table.getn(mylines) > num then
544 table.remove(mylines, 1)
545 end
546 end
547 for _,x in ipairs(mylines) do
548 files.stdout:write(x, "\n")
549 end
550 return 0
551 end
552 return {dotail, logline = "tail "..cmd_as_str(arg)}
553end
554
555function sort(file)
556 local function dosort(file)
557 local infile
558 if file == nil then
559 infile = files.stdin
560 else
561 infile = open_or_err(file)
562 end
563 local lines = {}
564 for l in infile:lines() do
565 table.insert(lines, l)
566 end
567 if file ~= nil then infile:close() end
568 table.sort(lines)
569 for _,l in ipairs(lines) do
570 files.stdout:write(l, "\n")
571 end
572 return 0
573 end
574 return {dosort, file, logline = "sort "..file}
575end
576
577function log_file_contents(filename)
578 L(readfile_q(filename), "\n")
579end
580
581function pre_cmd(stdin, ident)
582 if ident == nil then ident = "ts-" end
583 if stdin == true then
584 unlogged_copy("stdin", ident .. "stdin")
585 elseif type(stdin) == "table" then
586 unlogged_copy(stdin[1], ident .. "stdin")
587 else
588 local infile = open_or_err(ident .. "stdin", "w", 3)
589 if stdin ~= nil and stdin ~= false then
590 infile:write(stdin)
591 end
592 infile:close()
593 end
594 L("stdin:\n")
595 log_file_contents(ident .. "stdin")
596end
597
598function post_cmd(result, ret, stdout, stderr, ident)
599 if ret == nil then ret = 0 end
600 if ident == nil then ident = "ts-" end
601 L("stdout:\n")
602 log_file_contents(ident .. "stdout")
603 L("stderr:\n")
604 log_file_contents(ident .. "stderr")
605 if result ~= ret and ret ~= false then
606 err("Check failed (return value): wanted " .. ret .. " got " .. result, 3)
607 end
608
609 if stdout == nil then
610 if fsize(ident .. "stdout") ~= 0 then
611 err("Check failed (stdout): not empty", 3)
612 end
613 elseif type(stdout) == "string" then
614 local realout = open_or_err(ident .. "stdout", nil, 3)
615 local contents = realout:read("*a")
616 realout:close()
617 if contents ~= stdout then
618 err("Check failed (stdout): doesn't match", 3)
619 end
620 elseif type(stdout) == "table" then
621 if not samefile(ident .. "stdout", stdout[1]) then
622 err("Check failed (stdout): doesn't match", 3)
623 end
624 elseif stdout == true then
625 unlogged_remove("stdout")
626 unlogged_rename(ident .. "stdout", "stdout")
627 end
628
629 if stderr == nil then
630 if fsize(ident .. "stderr") ~= 0 then
631 err("Check failed (stderr): not empty", 3)
632 end
633 elseif type(stderr) == "string" then
634 local realerr = open_or_err(ident .. "stderr", nil, 3)
635 local contents = realerr:read("*a")
636 realerr:close()
637 if contents ~= stderr then
638 err("Check failed (stderr): doesn't match", 3)
639 end
640 elseif type(stderr) == "table" then
641 if not samefile(ident .. "stderr", stderr[1]) then
642 err("Check failed (stderr): doesn't match", 3)
643 end
644 elseif stderr == true then
645 unlogged_remove("stderr")
646 unlogged_rename(ident .. "stderr", "stderr")
647 end
648end
649
650-- std{out,err} can be:
651-- * false: ignore
652-- * true: ignore, copy to stdout
653-- * string: check that it matches the contents
654-- * nil: must be empty
655-- * {string}: check that it matches the named file
656-- stdin can be:
657-- * true: use existing "stdin" file
658-- * nil, false: empty input
659-- * string: contents of string
660-- * {string}: contents of the named file
661
662function bg(torun, ret, stdout, stderr, stdin)
663 test.bgid = test.bgid + 1
664 local out = {}
665 out.prefix = "ts-" .. test.bgid .. "-"
666 pre_cmd(stdin, out.prefix)
667 L("Starting background command...")
668 local ok,pid = runcmd(torun, out.prefix, true)
669 if not ok then err(pid, 2) end
670 if pid == -1 then err("Failed to start background process\n", 2) end
671 out.pid = pid
672 test.bglist[test.bgid] = out
673 out.id = test.bgid
674 out.retval = nil
675 out.locstr = locheader()
676 out.cmd = torun
677 out.expret = ret
678 out.expout = stdout
679 out.experr = stderr
680 local mt = {}
681 mt.__index = mt
682 mt.finish = function(obj, timeout)
683 if obj.retval ~= nil then return end
684
685 if timeout == nil then timeout = 0 end
686 if type(timeout) ~= "number" then
687 err("Bad timeout of type "..type(timeout))
688 end
689 local res
690 obj.retval, res = timed_wait(obj.pid, timeout)
691 if (res == -1) then
692 if (obj.retval ~= 0) then
693 L(locheader(), "error in timed_wait ", obj.retval, "\n")
694 end
695 kill(obj.pid, 15) -- TERM
696 obj.retval, res = timed_wait(obj.pid, 2)
697 if (res == -1) then
698 kill(obj.pid, 9) -- KILL
699 obj.retval, res = timed_wait(obj.pid, 2)
700 end
701 end
702
703 test.bglist[obj.id] = nil
704 L(locheader(), "checking background command from ", out.locstr,
705 cmd_as_str(out.cmd), "\n")
706 post_cmd(obj.retval, out.expret, out.expout, out.experr, obj.prefix)
707 return true
708 end
709 mt.wait = function(obj, timeout)
710 if obj.retval ~= nil then return end
711 if timeout == nil then
712 obj.retval = wait(obj.pid)
713 else
714 local res
715 obj.retval, res = timed_wait(obj.pid, timeout)
716 if res == -1 then
717 obj.retval = nil
718 return false
719 end
720 end
721 test.bglist[obj.id] = nil
722 L(locheader(), "checking background command from ", out.locstr,
723 table.concat(out.cmd, " "), "\n")
724 post_cmd(obj.retval, out.expret, out.expout, out.experr, obj.prefix)
725 return true
726 end
727 return setmetatable(out, mt)
728end
729
730function runcheck(cmd, ret, stdout, stderr, stdin)
731 if ret == nil then ret = 0 end
732 pre_cmd(stdin)
733 local ok, result = runcmd(cmd)
734 if ok == false then
735 err(result, 2)
736 end
737 post_cmd(result, ret, stdout, stderr)
738 return result
739end
740
741function indir(dir, what)
742 if type(what) ~= "table" then
743 err("bad argument of type "..type(what).." to indir()")
744 end
745 local function do_indir()
746 local savedir = chdir(dir)
747 if savedir == nil then
748 err("Cannot chdir to "..dir)
749 end
750 local ok, res
751 if type(what[1]) == "function" then
752 ok, res = pcall(unpack(what))
753 elseif type(what[1]) == "string" then
754 ok, res = pcall(execute, unpack(what))
755 else
756 err("bad argument to indir(): cannot execute a "..type(what[1]))
757 end
758 chdir(savedir)
759 if not ok then err(res) end
760 return res
761 end
762 local want_local
763 if type(what[1]) == "function" then
764 if type(what.local_redirect) == "nil" then
765 want_local = true
766 else
767 want_local = what.local_redirect
768 end
769 else
770 want_local = false
771 end
772 local ll = "In directory "..dir..": "
773 if what.logline ~= nil then ll = ll .. tostring(what.logline)
774 else
775 ll = ll .. cmd_as_str(what)
776 end
777 return {do_indir, local_redirect = want_local, logline = ll}
778end
779
780function check(first, ...)
781 if type(first) == "table" then
782 return runcheck(first, unpack(arg))
783 elseif type(first) == "boolean" then
784 if not first then err("Check failed: false", 2) end
785 elseif type(first) == "number" then
786 if first ~= 0 then
787 err("Check failed: " .. first .. " ~= 0", 2)
788 end
789 else
790 err("Bad argument to check() (" .. type(first) .. ")", 2)
791 end
792 return first
793end
794
795function skip_if(chk)
796 if chk then
797 err(true, 2)
798 end
799end
800
801function xfail_if(chk, ...)
802 local ok,res = pcall(check, unpack(arg))
803 if ok == false then
804 if chk then err(false, 2) else err(err, 2) end
805 else
806 if chk then
807 test.wanted_fail = true
808 L("UNEXPECTED SUCCESS\n")
809 end
810 end
811end
812
813function xfail(...)
814 xfail_if(true, unpack(arg))
815end
816
817function log_error(e)
818 if type(e) == "table" then
819 L("\n", tostring(e.e), "\n")
820 for i,bt in ipairs(e.bt) do
821 if i ~= 1 then L("Rethrown from:") end
822 L(bt)
823 end
824 else
825 L("\n", tostring(e), "\n")
826 end
827end
828
829function run_tests(debugging, list_only, run_dir, logname, args, progress)
830 local torun = {}
831 local run_all = true
832
833 local function P(...)
834 progress(unpack(arg))
835 logfile:write(unpack(arg))
836 end
837
838 -- NLS nuisances.
839 for _,name in pairs({ "LANG",
840 "LANGUAGE",
841 "LC_ADDRESS",
842 "LC_ALL",
843 "LC_COLLATE",
844 "LC_CTYPE",
845 "LC_IDENTIFICATION",
846 "LC_MEASUREMENT",
847 "LC_MESSAGES",
848 "LC_MONETARY",
849 "LC_NAME",
850 "LC_NUMERIC",
851 "LC_PAPER",
852 "LC_TELEPHONE",
853 "LC_TIME" }) do
854 set_env(name,"C")
855 end
856
857 -- no test suite should touch the user's ssh agent
858 unset_env("SSH_AUTH_SOCK")
859
860 logfile = io.open(logname, "w")
861 chdir(run_dir);
862
863 do
864 local s = prepare_to_enumerate_tests(P)
865 if s ~= 0 then
866P("Enumeration of tests failed.\n")
867return s
868 end
869 end
870
871 -- testdir is set by the testsuite definition
872 -- any directory in testdir with a __driver__.lua inside is a test case
873 local tests = {}
874 for _,candidate in ipairs(read_directory(testdir)) do
875 -- n.b. it is not necessary to throw out directories before doing
876 -- this check, because exists(nondirectory/__driver__.lua) will
877 -- never be true.
878 if exists(testdir .. "/" .. candidate .. "/__driver__.lua") then
879table.insert(tests, candidate)
880 end
881 end
882 table.sort(tests)
883
884 for i,a in pairs(args) do
885 local _1,_2,l,r = string.find(a, "^(-?%d+)%.%.(-?%d+)$")
886 if _1 then
887 l = l + 0
888 r = r + 0
889 if l < 1 then l = table.getn(tests) + l + 1 end
890 if r < 1 then r = table.getn(tests) + r + 1 end
891 if l > r then l,r = r,l end
892 for j = l,r do
893 torun[j] = tests[j]
894 end
895 run_all = false
896 elseif string.find(a, "^-?%d+$") then
897 r = a + 0
898 if r < 1 then r = table.getn(tests) + r + 1 end
899 torun[r] = tests[r]
900 run_all = false
901 else
902 -- pattern
903 run_all = false
904 local matched = false
905 for i,t in pairs(tests) do
906 if regex.search(a, t) then
907 torun[i] = t
908 matched = true
909 end
910 end
911 if not matched then
912 print(string.format("Warning: pattern '%s' does not match any tests.", a))
913 end
914 end
915 end
916
917 if run_all then torun = tests end
918
919 if list_only then
920 for i,t in pairs(torun) do
921 if i < 10 then P(" ") end
922 if i < 100 then P(" ") end
923 P(i .. " " .. t .. "\n")
924 end
925 logfile:close()
926 return 0
927 end
928
929 logfile:write("Running on ", get_ostype(), "\n\n")
930 local s = prepare_to_run_tests(P)
931 if s ~= 0 then
932 P("Test suite preparation failed.\n")
933 return s
934 end
935 P("Running tests...\n")
936
937 local counts = {}
938 counts.success = 0
939 counts.skip = 0
940 counts.xfail = 0
941 counts.noxfail = 0
942 counts.fail = 0
943 counts.total = 0
944 counts.of_interest = 0
945 local of_interest = {}
946 local failed_testlogs = {}
947
948 -- exit codes which indicate failure at a point in the process-spawning
949 -- code where it is impossible to give more detailed diagnostics
950 local magic_exit_codes = {
951 [121] = "error creating test directory",
952 [122] = "error spawning test process",
953 [123] = "error entering test directory",
954 [124] = "unhandled exception in child process"
955 }
956
957 -- callback closure passed to run_tests_in_children
958 local function report_one_test(tno, tname, status)
959 local tdir = run_dir .. "/" .. tname
960 local test_header = string.format("%3d %-45s ", tno, tname)
961 local what
962 local can_delete
963 -- the child should always exit successfully, just to avoid
964 -- headaches. if we get any other code we report it as a failure.
965 if status ~= 0 then
966if status < 0 then
967 what = string.format("FAIL (signal %d)", -status)
968elseif magic_exit_codes[status] ~= nil then
969 what = string.format("FAIL (%s)", magic_exit_codes[status])
970else
971 what = string.format("FAIL (exit %d)", status)
972end
973 else
974local wfile, err = io.open(tdir .. "/STATUS", "r")
975if wfile ~= nil then
976 what = string.gsub(wfile:read("*a"), "\n$", "")
977 wfile:close()
978else
979 what = string.format("FAIL (status file: %s)", err)
980end
981 end
982 if what == "unexpected success" then
983counts.noxfail = counts.noxfail + 1
984counts.of_interest = counts.of_interest + 1
985table.insert(of_interest, test_header .. "unexpected success")
986can_delete = false
987 elseif what == "partial skip" or what == "ok" then
988counts.success = counts.success + 1
989can_delete = true
990 elseif string.find(what, "skipped ") == 1 then
991counts.skip = counts.skip + 1
992can_delete = true
993 elseif string.find(what, "expected failure ") == 1 then
994counts.xfail = counts.xfail + 1
995can_delete = false
996 elseif string.find(what, "FAIL ") == 1 then
997counts.fail = counts.fail + 1
998table.insert(of_interest, test_header .. what)
999table.insert(failed_testlogs, tdir .. "/tester.log")
1000can_delete = false
1001 else
1002counts.fail = counts.fail + 1
1003what = "FAIL (gobbledygook: " .. what .. ")"
1004table.insert(of_interest, test_header .. what)
1005table.insert(failed_testlogs, tdir .. "/tester.log")
1006can_delete = false
1007 end
1008
1009 counts.total = counts.total + 1
1010 P(string.format("%s%s\n", test_header, what))
1011 return (can_delete and not debugging)
1012 end
1013
1014 run_tests_in_children(torun, report_one_test)
1015
1016 if counts.of_interest ~= 0 and (counts.total / counts.of_interest) > 4 then
1017 P("\nInteresting tests:\n")
1018 for i,x in ipairs(of_interest) do
1019 P(x, "\n")
1020 end
1021 end
1022 P("\n")
1023
1024 for i,log in pairs(failed_testlogs) do
1025 local tlog = io.open(log, "r")
1026 if tlog ~= nil then
1027 local dat = tlog:read("*a")
1028 tlog:close()
1029 logfile:write("\n", string.rep("*", 50), "\n")
1030 logfile:write(dat)
1031 end
1032 end
1033
1034 -- Write out this summary in one go so that it does not get interrupted
1035 -- by concurrent test suites' summaries.
1036 P(string.format("Of %i tests run:\n"..
1037 "\t%i succeeded\n"..
1038 "\t%i failed\n"..
1039 "\t%i had expected failures\n"..
1040 "\t%i succeeded unexpectedly\n"..
1041 "\t%i were skipped\n",
1042 counts.total, counts.success, counts.fail,
1043 counts.xfail, counts.noxfail, counts.skip))
1044
1045 logfile:close()
1046 if counts.success + counts.skip + counts.xfail == counts.total then
1047 return 0
1048 else
1049 return 1
1050 end
1051end
1052
1053function run_one_test(tname)
1054 test.bgid = 0
1055 test.name = tname
1056 test.wanted_fail = false
1057 test.partial_skip = false
1058 test.root = chdir(".")
1059 test.errfile = ""
1060 test.errline = -1
1061 test.bglist = {}
1062 test.log = io.open("tester.log", "w")
1063
1064 L("Test ", test.name, "\n")
1065
1066 local driverfile = testdir .. "/" .. test.name .. "/__driver__.lua"
1067 local driver, e = loadfile(driverfile)
1068 local r
1069 if driver == nil then
1070 r = false
1071 e = "Could not load driver file " .. driverfile .. ".\n" .. e
1072 else
1073 local oldmask = posix_umask(0)
1074 posix_umask(oldmask)
1075 r,e = xpcall(driver, debug.traceback)
1076 local errline = test.errline
1077 for i,b in pairs(test.bglist) do
1078 local a,x = pcall(function () b:finish(0) end)
1079 if r and not a then
1080 r = a
1081 e = x
1082 elseif not a then
1083 L("Error cleaning up background processes: ",
1084 tostring(b.locstr), "\n")
1085 end
1086 end
1087 if type(cleanup) == "function" then
1088 local a,b = pcall(cleanup)
1089 if r and not a then
1090 r = a
1091 e = b
1092 end
1093 end
1094 test.errline = errline
1095 posix_umask(oldmask)
1096 end
1097
1098 if not r then
1099 if test.errline == nil then test.errline = -1 end
1100 if type(e) ~= "table" then
1101 local tbl = {e = e, bt = {"no backtrace; type(err) = "..type(e)}}
1102 e = tbl
1103 end
1104 if type(e.e) ~= "boolean" then
1105 log_error(e)
1106 end
1107 end
1108 test.log:close()
1109
1110 -- record the short status where report_one_test can find it
1111 local s = io.open(test.root .. "/STATUS", "w")
1112 if r then
1113 if test.wanted_fail then
1114 s:write("unexpected success\n")
1115 else
1116 if test.partial_skip then
1117 s:write("partial skip\n")
1118 else
1119 s:write("ok\n")
1120 end
1121 end
1122 else
1123 if e.e == true then
1124 s:write(string.format("skipped (line %i)\n", test.errline))
1125 elseif e.e == false then
1126 s:write(string.format("expected failure (line %i)\n",
1127 test.errline))
1128 else
1129 s:write(string.format("FAIL (line %i)\n", test.errline))
1130 end
1131 end
1132 s:close()
1133 return 0
1134end

Archive Download this file

Branches

Tags

Quick Links:     www.monotone.ca    -     Downloads    -     Documentation    -     Wiki    -     Code Forge    -     Build Status