monotone

monotone Mtn Source Tree

Root/testlib.lua

1-- misc global values
2-- where the main testsuite file is
3srcdir = get_source_dir()
4-- where the individual test dirs are
5-- most paths will be testdir.."/something"
6-- normally reset by the main testsuite file
7testdir = srcdir
8-- was the -d switch given?
9debugging = false
10
11-- combined logfile; tester.cc will reset this to a filename, which is
12-- then opened in run_tests
13logfile = nil
14
15-- This is for redirected output from local implementations
16-- of shellutils type stuff (ie, grep).
17-- Reason: {set,clear}_redirect don't seem to (always?) work
18-- for this (at least on Windows).
19files = {stdout = nil, stdin = nil, stderr = nil}
20
21-- for convenience, this is the first word of what get_ostype() returns.
22ostype = string.sub(get_ostype(), 1, string.find(get_ostype(), " ")-1)
23
24-- table of per-test values
25test = {}
26-- misc per-test values
27test.root = nil
28test.name = nil
29test.wanted_fail = false
30test.partial_skip = false -- set this to true if you skip part of the test
31
32--probably should put these in the error that gets thrown...
33test.errfile = ""
34test.errline = -1
35
36-- for tracking background processes
37test.bgid = 0
38test.bglist = {}
39
40test.log = nil -- logfile for this test
41
42-- hook to be overridden by the main testsuite file, if necessary;
43-- called after determining the set of tests to run.
44-- P may be used to write messages to the user's tty.
45function prepare_to_run_tests(P)
46 return 0
47end
48
49-- hook to be overridden by the main testsuite file, if necessary;
50-- called after opening the master logfile, but _before_ parsing
51-- arguments or determining the set of tests to run.
52-- P may be used to write messages to the user's tty.
53function prepare_to_enumerate_tests(P)
54 return 0
55end
56
57function L(...)
58 test.log:write(unpack(arg))
59 test.log:flush()
60end
61
62function getsrcline()
63 local info
64 local depth = 1
65 repeat
66 depth = depth + 1
67 info = debug.getinfo(depth)
68 until info == nil
69 while src == nil and depth > 1 do
70 depth = depth - 1
71 info = debug.getinfo(depth)
72 if string.find(info.source, "^@.*__driver__%.lua") then
73 -- return info.source, info.currentline
74 return test.name, info.currentline
75 end
76 end
77end
78
79function locheader()
80 local _,line = getsrcline()
81 if line == nil then line = -1 end
82 if test.name == nil then
83 return "\n<unknown>:" .. line .. ": "
84 else
85 return "\n" .. test.name .. ":" .. line .. ": "
86 end
87end
88
89function err(what, level)
90 if level == nil then level = 2 end
91 test.errfile, test.errline = getsrcline()
92 local e
93 if type(what) == "table" then
94 e = what
95 if e.bt == nil then e.bt = {} end
96 table.insert(e.bt, debug.traceback())
97 else
98 e = {e = what, bt = {debug.traceback()}}
99 end
100 error(e, level)
101end
102
103do -- replace some builtings with logged versions
104 unlogged_mtime = mtime
105 mtime = function(name)
106 local x = unlogged_mtime(name)
107 L(locheader(), "mtime(", name, ") = ", tostring(x), "\n")
108 return x
109 end
110
111 unlogged_mkdir = mkdir
112 mkdir = function(name)
113 L(locheader(), "mkdir ", name, "\n")
114 unlogged_mkdir(name)
115 end
116
117 unlogged_existsonpath = existsonpath
118 existsonpath = function(name)
119 local r = (unlogged_existsonpath(name) == 0)
120 local what
121 if r then
122 what = "exists"
123 else
124 what = "does not exist"
125 end
126 L(locheader(), name, " ", what, " on the path\n")
127 return r
128 end
129end
130
131function numlines(filename)
132 local n = 0
133 for _ in io.lines(filename) do n = n + 1 end
134 L(locheader(), "numlines(", filename, ") = ", n, "\n")
135 return n
136end
137
138function open_or_err(filename, mode, depth)
139 local file, e = io.open(filename, mode)
140 if file == nil then
141 err("Cannot open file " .. filename .. ": " .. e, depth)
142 end
143 return file
144end
145
146function fsize(filename)
147 local file = open_or_err(filename, "r", 3)
148 local size = file:seek("end")
149 file:close()
150 return size
151end
152
153function readfile_q(filename)
154 local file = open_or_err(filename, "rb", 3)
155 local dat = file:read("*a")
156 file:close()
157 return dat
158end
159
160function readfile(filename)
161 L(locheader(), "readfile ", filename, "\n")
162 return readfile_q(filename)
163end
164
165function readstdfile(filename)
166 return readfile(testdir.."/"..filename)
167end
168
169-- Return all but the first N lines of FILENAME.
170-- Note that (unlike readfile()) the result will
171-- end with a \n whether or not the file did.
172function tailfile(filename, n)
173 L(locheader(), "tailfile ", filename, ", ", n, "\n")
174 local i = 1
175 local t = {}
176 for l in io.lines(filename) do
177 if i > n then
178 table.insert(t, l)
179 end
180 i = i + 1
181 end
182 table.insert(t, "")
183 return table.concat(t, "\n")
184end
185
186function writefile_q(filename, dat)
187 local file,e
188 if dat == nil then
189 file,e = open_or_err(filename, "a+b", 3)
190 else
191 file,e = open_or_err(filename, "wb", 3)
192 end
193 if dat ~= nil then
194 file:write(dat)
195 end
196 file:close()
197 return true
198end
199
200function writefile(filename, dat)
201 L(locheader(), "writefile ", filename, "\n")
202 return writefile_q(filename, dat)
203end
204
205function append(filename, dat)
206 L(locheader(), "append to file ", filename, "\n")
207 local file,e = open_or_err(filename, "a+", 3)
208 file:write(dat)
209 file:close()
210 return true
211end
212
213do
214 unlogged_copy = copy_recursive
215 copy_recursive = nil
216 function copy(from, to)
217 L(locheader(), "copy ", from, " -> ", to, "\n")
218 local ok, res = unlogged_copy(from, to)
219 if not ok then
220 L(res, "\n")
221 return false
222 else
223 return true
224 end
225 end
226end
227
228do
229 local os_rename = os.rename
230 os.rename = nil
231 os.remove = nil
232 function rename(from, to)
233 L(locheader(), "rename ", from, " ", to, "\n")
234 if exists(to) and not isdir(to) then
235 L("Destination ", to, " exists; removing...\n")
236 local ok, res = unlogged_remove(to)
237 if not ok then
238 L("Could not remove ", to, ": ", res, "\n")
239 return false
240 end
241 end
242 local ok,res = os_rename(from, to)
243 if not ok then
244 L(res, "\n")
245 return false
246 else
247 return true
248 end
249 end
250 function unlogged_rename(from, to)
251 if exists(to) and not isdir(to) then
252 unlogged_remove(to)
253 end
254 os_rename(from, to)
255 end
256 unlogged_remove = remove_recursive
257 remove_recursive = nil
258 function remove(file)
259 L(locheader(), "remove ", file, "\n")
260 local ok,res = unlogged_remove(file)
261 if not ok then
262 L(res, "\n")
263 return false
264 else
265 return true
266 end
267 end
268end
269
270
271function getstd(name, as)
272 if as == nil then as = name end
273 local ret = copy(testdir .. "/" .. name, as)
274 make_tree_accessible(as)
275 return ret
276end
277
278function get(name, as)
279 if as == nil then as = name end
280 return getstd(test.name .. "/" .. name, as)
281end
282
283-- include from the main tests directory; there's no reason
284-- to want to include from the dir for the current test,
285-- since in that case it could just go in the driver file.
286function include(name)
287 local func, e = loadfile(testdir.."/"..name)
288 if func == nil then err(e, 2) end
289 setfenv(func, getfenv(2))
290 func()
291end
292
293function trim(str)
294 return string.gsub(str, "^%s*(.-)%s*$", "%1")
295end
296
297function getpathof(exe, ext)
298 local function gotit(now)
299 if test.log == nil then
300 logfile:write(exe, " found at ", now, "\n")
301 else
302 test.log:write(exe, " found at ", now, "\n")
303 end
304 return now
305 end
306 local path = os.getenv("PATH")
307 local char
308 if ostype == "Windows" then
309 char = ';'
310 else
311 char = ':'
312 end
313 if ostype == "Windows" then
314 if ext == nil then ext = ".exe" end
315 else
316 if ext == nil then ext = "" end
317 end
318 local now = initial_dir.."/"..exe..ext
319 if exists(now) then return gotit(now) end
320 for x in string.gmatch(path, "[^"..char.."]*"..char) do
321 local dir = string.sub(x, 0, -2)
322 if string.find(dir, "[\\/]$") then
323 dir = string.sub(dir, 0, -2)
324 end
325 local now = dir.."/"..exe..ext
326 if exists(now) then return gotit(now) end
327 end
328 if test.log == nil then
329 logfile:write("Cannot find ", exe, "\n")
330 else
331 test.log:write("Cannot find ", exe, "\n")
332 end
333 return nil
334end
335
336function prepare_redirect(fin, fout, ferr)
337 local cwd = chdir(".").."/"
338 redir = {fin = cwd..fin, fout = cwd..fout, ferr = cwd..ferr}
339end
340do
341 oldspawn = spawn
342 function spawn(...)
343 if redir == nil then
344 return oldspawn(unpack(arg))
345 else
346 return spawn_redirected(redir.fin, redir.fout, redir.ferr, unpack(arg))
347 end
348 end
349end
350function execute(path, ...)
351 local pid
352 local ret = -1
353 pid = spawn(path, unpack(arg))
354 redir = nil
355 if (pid ~= -1) then ret, pid = wait(pid) end
356 return ret
357end
358
359function cmd_as_str(cmd_table)
360 local str = ""
361 for i,x in ipairs(cmd_table) do
362 if str ~= "" then str = str .. " " end
363 if type(x) == "function" then
364 str = str .. "<function>"
365 else
366 local s = tostring(x)
367 if string.find(s, " ") then
368 str = str .. '"'..s..'"'
369 else
370 str = str .. s
371 end
372 end
373 end
374 return str
375end
376
377function runcmd(cmd, prefix, bgnd)
378 if prefix == nil then prefix = "ts-" end
379 if type(cmd) ~= "table" then err("runcmd called with bad argument") end
380 local local_redir = cmd.local_redirect
381 if cmd.local_redirect == nil then
382 if type(cmd[1]) == "function" then
383 local_redir = true
384 else
385 local_redir = false
386 end
387 end
388 if bgnd == true and type(cmd[1]) == "string" then local_redir = false end
389 L("\nruncmd: ", tostring(cmd[1]), ", local_redir = ", tostring(local_redir), ", requested = ", tostring(cmd.local_redirect))
390 local redir
391 if local_redir then
392 files.stdin = open_or_err(prefix.."stdin", nil, 2)
393 files.stdout = open_or_err(prefix.."stdout", "w", 2)
394 files.stderr = open_or_err(prefix.."stderr", "w", 2)
395 else
396 prepare_redirect(prefix.."stdin", prefix.."stdout", prefix.."stderr")
397 end
398
399 local result
400 if cmd.logline ~= nil then
401 L(locheader(), cmd.logline, "\n")
402 else
403 L(locheader(), cmd_as_str(cmd), "\n")
404 end
405
406 local oldexec = execute
407 if bgnd then
408 execute = spawn
409 end
410 if type(cmd[1]) == "function" then
411 result = {pcall(unpack(cmd))}
412 elseif type(cmd[1]) == "string" then
413 result = {pcall(execute, unpack(cmd))}
414 else
415 execute = oldexec
416 err("runcmd called with bad command table " ..
417"(first entry is a " .. type(cmd[1]) ..")")
418 end
419 execute = oldexec
420
421 if local_redir then
422 files.stdin:close()
423 files.stdout:close()
424 files.stderr:close()
425 end
426 return unpack(result)
427end
428
429function samefile(left, right)
430 if left == "-" or right == "-" then
431 err("tests may not rely on standard input")
432 end
433 if fsize(left) ~= fsize(right) then
434 return false
435 else
436 local ldat = readfile(left)
437 local rdat = readfile(right)
438 return ldat == rdat
439 end
440end
441
442function samelines(f, t)
443 local fl = {}
444 for l in io.lines(f) do table.insert(fl, l) end
445 if not table.getn(fl) == table.getn(t) then
446 L(locheader(), string.format("file has %s lines; table has %s\n",
447 table.getn(fl), table.getn(t)))
448 return false
449 end
450 for i=1,table.getn(t) do
451 if fl[i] ~= t[i] then
452 if fl[i] then
453 L(locheader(), string.format("file[i] = '%s'; table[i] = '%s'\n",
454 fl[i], t[i]))
455 else
456 L(locheader(), string.format("file[i] = ''; table[i] = '%s'\n",
457 t[i]))
458 end
459 return false
460 end
461 end
462 return true
463end
464
465function greplines(f, t)
466 local fl = {}
467 for l in io.lines(f) do table.insert(fl, l) end
468 if not table.getn(fl) == table.getn(t) then
469 L(locheader(), string.format("file has %s lines; table has %s\n",
470 table.getn(fl), table.getn(t)))
471 return false
472 end
473 for i=1,table.getn(t) do
474 if not regex.search(t[i], fl[i]) then
475 L(locheader(), string.format("file[i] = '%s'; table[i] = '%s'\n",
476 fl[i], t[i]))
477 return false
478 end
479 end
480 return true
481end
482
483function grep(...)
484 local flags, what, where = unpack(arg)
485 local dogrep = function ()
486 if where == nil and string.sub(flags, 1, 1) ~= "-" then
487 where = what
488 what = flags
489 flags = ""
490 end
491 local quiet = string.find(flags, "q") ~= nil
492 local reverse = string.find(flags, "v") ~= nil
493 if not quiet and files.stdout == nil then err("non-quiet grep not redirected") end
494 local out = 1
495 local infile = files.stdin
496 if where ~= nil then infile = open_or_err(where) end
497 for line in infile:lines() do
498 local matched = regex.search(what, line)
499 if reverse then matched = not matched end
500 if matched then
501 if not quiet then files.stdout:write(line, "\n") end
502 out = 0
503 end
504 end
505 if where ~= nil then infile:close() end
506 return out
507 end
508 return {dogrep, logline = "grep "..cmd_as_str(arg)}
509end
510
511function cat(...)
512 local arguments = arg
513 local function docat()
514 local bsize = 8*1024
515 for _,x in ipairs(arguments) do
516 local infile
517 if x == "-" then
518 infile = files.stdin
519 else
520 infile = open_or_err(x, "rb", 3)
521 end
522 local block = infile:read(bsize)
523 while block do
524 files.stdout:write(block)
525 block = infile:read(bsize)
526 end
527 if x ~= "-" then
528 infile:close()
529 end
530 end
531 return 0
532 end
533 return {docat, logline = "cat "..cmd_as_str(arg)}
534end
535
536function tail(...)
537 local file, num = unpack(arg)
538 local function dotail()
539 if num == nil then num = 10 end
540 local mylines = {}
541 for l in io.lines(file) do
542 table.insert(mylines, l)
543 if table.getn(mylines) > num then
544 table.remove(mylines, 1)
545 end
546 end
547 for _,x in ipairs(mylines) do
548 files.stdout:write(x, "\n")
549 end
550 return 0
551 end
552 return {dotail, logline = "tail "..cmd_as_str(arg)}
553end
554
555function sort(file)
556 local function dosort(file)
557 local infile
558 if file == nil then
559 infile = files.stdin
560 else
561 infile = open_or_err(file)
562 end
563 local lines = {}
564 for l in infile:lines() do
565 table.insert(lines, l)
566 end
567 if file ~= nil then infile:close() end
568 table.sort(lines)
569 for _,l in ipairs(lines) do
570 files.stdout:write(l, "\n")
571 end
572 return 0
573 end
574 return {dosort, file, logline = "sort "..file}
575end
576
577function log_file_contents(filename)
578 L(readfile_q(filename), "\n")
579end
580
581function pre_cmd(stdin, ident)
582 if ident == nil then ident = "ts-" end
583 if stdin == true then
584 unlogged_copy("stdin", ident .. "stdin")
585 elseif type(stdin) == "table" then
586 unlogged_copy(stdin[1], ident .. "stdin")
587 else
588 local infile = open_or_err(ident .. "stdin", "w", 3)
589 if stdin ~= nil and stdin ~= false then
590 infile:write(stdin)
591 end
592 infile:close()
593 end
594 L("stdin:\n")
595 log_file_contents(ident .. "stdin")
596end
597
598function post_cmd(result, ret, stdout, stderr, ident)
599 if ret == nil then ret = 0 end
600 if ident == nil then ident = "ts-" end
601 L("stdout:\n")
602 log_file_contents(ident .. "stdout")
603 L("stderr:\n")
604 log_file_contents(ident .. "stderr")
605 L("exit code: " .. result .. "\n")
606 if result ~= ret and ret ~= false then
607 err("Check failed (return value): wanted " .. ret .. " got " .. result, 3)
608 end
609
610 if stdout == nil then
611 if fsize(ident .. "stdout") ~= 0 then
612 err("Check failed (stdout): not empty", 3)
613 end
614 elseif type(stdout) == "string" then
615 local realout = open_or_err(ident .. "stdout", nil, 3)
616 local contents = realout:read("*a")
617 realout:close()
618 if contents ~= stdout then
619 err("Check failed (stdout): doesn't match", 3)
620 end
621 elseif type(stdout) == "table" then
622 if not samefile(ident .. "stdout", stdout[1]) then
623 err("Check failed (stdout): doesn't match", 3)
624 end
625 elseif stdout == true then
626 unlogged_remove("stdout")
627 unlogged_rename(ident .. "stdout", "stdout")
628 end
629
630 if stderr == nil then
631 if fsize(ident .. "stderr") ~= 0 then
632 err("Check failed (stderr): not empty", 3)
633 end
634 elseif type(stderr) == "string" then
635 local realerr = open_or_err(ident .. "stderr", nil, 3)
636 local contents = realerr:read("*a")
637 realerr:close()
638 if contents ~= stderr then
639 err("Check failed (stderr): doesn't match", 3)
640 end
641 elseif type(stderr) == "table" then
642 if not samefile(ident .. "stderr", stderr[1]) then
643 err("Check failed (stderr): doesn't match", 3)
644 end
645 elseif stderr == true then
646 unlogged_remove("stderr")
647 unlogged_rename(ident .. "stderr", "stderr")
648 end
649end
650
651-- std{out,err} can be:
652-- * false: ignore
653-- * true: ignore, copy to stdout
654-- * string: check that it matches the contents
655-- * nil: must be empty
656-- * {string}: check that it matches the named file
657-- stdin can be:
658-- * true: use existing "stdin" file
659-- * nil, false: empty input
660-- * string: contents of string
661-- * {string}: contents of the named file
662
663function bg(torun, ret, stdout, stderr, stdin)
664 test.bgid = test.bgid + 1
665 local out = {}
666 out.prefix = "ts-" .. test.bgid .. "-"
667 pre_cmd(stdin, out.prefix)
668 L("Starting background command...")
669 local ok,pid = runcmd(torun, out.prefix, true)
670 if not ok then err(pid, 2) end
671 if pid == -1 then err("Failed to start background process\n", 2) end
672 out.pid = pid
673 test.bglist[test.bgid] = out
674 out.id = test.bgid
675 out.retval = nil
676 out.locstr = locheader()
677 out.cmd = torun
678 out.expret = ret
679 out.expout = stdout
680 out.experr = stderr
681 local mt = {}
682 mt.__index = mt
683 mt.finish = function(obj, timeout)
684 if obj.retval ~= nil then return end
685
686 if timeout == nil then timeout = 0 end
687 if type(timeout) ~= "number" then
688 err("Bad timeout of type "..type(timeout))
689 end
690 local res
691 obj.retval, res = timed_wait(obj.pid, timeout)
692 if (res == -1) then
693 if (obj.retval ~= 0) then
694 L(locheader(), "error in timed_wait ", obj.retval, "\n")
695 end
696 kill(obj.pid, 15) -- TERM
697 obj.retval, res = timed_wait(obj.pid, 2)
698 if (res == -1) then
699 kill(obj.pid, 9) -- KILL
700 obj.retval, res = timed_wait(obj.pid, 2)
701 end
702 end
703
704 test.bglist[obj.id] = nil
705 L(locheader(), "checking background command from ", out.locstr,
706 cmd_as_str(out.cmd), "\n")
707 post_cmd(obj.retval, out.expret, out.expout, out.experr, obj.prefix)
708 return true
709 end
710 mt.wait = function(obj, timeout)
711 if obj.retval ~= nil then return end
712 if timeout == nil then
713 obj.retval = wait(obj.pid)
714 else
715 local res
716 obj.retval, res = timed_wait(obj.pid, timeout)
717 if res == -1 then
718 obj.retval = nil
719 return false
720 end
721 end
722 test.bglist[obj.id] = nil
723 L(locheader(), "checking background command from ", out.locstr,
724 table.concat(out.cmd, " "), "\n")
725 post_cmd(obj.retval, out.expret, out.expout, out.experr, obj.prefix)
726 return true
727 end
728 return setmetatable(out, mt)
729end
730
731function runcheck(cmd, ret, stdout, stderr, stdin)
732 if ret == nil then ret = 0 end
733 pre_cmd(stdin)
734 local ok, result = runcmd(cmd)
735 if ok == false then
736 err(result, 2)
737 end
738 post_cmd(result, ret, stdout, stderr)
739 return result
740end
741
742function indir(dir, what)
743 if type(what) ~= "table" then
744 err("bad argument of type "..type(what).." to indir()")
745 end
746 local function do_indir()
747 local savedir = chdir(dir)
748 if savedir == nil then
749 err("Cannot chdir to "..dir)
750 end
751 local ok, res
752 if type(what[1]) == "function" then
753 ok, res = pcall(unpack(what))
754 elseif type(what[1]) == "string" then
755 ok, res = pcall(execute, unpack(what))
756 else
757 err("bad argument to indir(): cannot execute a "..type(what[1]))
758 end
759 chdir(savedir)
760 if not ok then err(res) end
761 return res
762 end
763 local want_local
764 if type(what[1]) == "function" then
765 if type(what.local_redirect) == "nil" then
766 want_local = true
767 else
768 want_local = what.local_redirect
769 end
770 else
771 want_local = false
772 end
773 local ll = "In directory "..dir..": "
774 if what.logline ~= nil then ll = ll .. tostring(what.logline)
775 else
776 ll = ll .. cmd_as_str(what)
777 end
778 return {do_indir, local_redirect = want_local, logline = ll}
779end
780
781function check(first, ...)
782 if type(first) == "table" then
783 return runcheck(first, unpack(arg))
784 elseif type(first) == "boolean" then
785 if not first then err("Check failed: false", 2) end
786 elseif type(first) == "number" then
787 if first ~= 0 then
788 err("Check failed: " .. first .. " ~= 0", 2)
789 end
790 else
791 err("Bad argument to check() (" .. type(first) .. ")", 2)
792 end
793 return first
794end
795
796function skip_if(chk)
797 if chk then
798 err(true, 2)
799 end
800end
801
802function xfail_if(chk, ...)
803 local ok,res = pcall(check, unpack(arg))
804 if ok == false then
805 if chk then err(false, 2) else err(err, 2) end
806 else
807 if chk then
808 test.wanted_fail = true
809 L("UNEXPECTED SUCCESS\n")
810 end
811 end
812end
813
814function xfail(...)
815 xfail_if(true, unpack(arg))
816end
817
818function log_error(e)
819 if type(e) == "table" then
820 L("\n", tostring(e.e), "\n")
821 for i,bt in ipairs(e.bt) do
822 if i ~= 1 then L("Rethrown from:") end
823 L(bt)
824 end
825 else
826 L("\n", tostring(e), "\n")
827 end
828end
829
830function run_tests(debugging, list_only, run_dir, logname, args, progress)
831 local torun = {}
832 local run_all = true
833
834 local function P(...)
835 progress(unpack(arg))
836 logfile:write(unpack(arg))
837 end
838
839 -- NLS nuisances.
840 for _,name in pairs({ "LANG",
841 "LANGUAGE",
842 "LC_ADDRESS",
843 "LC_ALL",
844 "LC_COLLATE",
845 "LC_CTYPE",
846 "LC_IDENTIFICATION",
847 "LC_MEASUREMENT",
848 "LC_MESSAGES",
849 "LC_MONETARY",
850 "LC_NAME",
851 "LC_NUMERIC",
852 "LC_PAPER",
853 "LC_TELEPHONE",
854 "LC_TIME" }) do
855 set_env(name,"C")
856 end
857
858 -- no test suite should touch the user's ssh agent
859 unset_env("SSH_AUTH_SOCK")
860
861 logfile = io.open(logname, "w")
862 chdir(run_dir);
863
864 do
865 local s = prepare_to_enumerate_tests(P)
866 if s ~= 0 then
867P("Enumeration of tests failed.\n")
868return s
869 end
870 end
871
872 -- testdir is set by the testsuite definition
873 -- any directory in testdir with a __driver__.lua inside is a test case
874 local tests = {}
875 for _,candidate in ipairs(read_directory(testdir)) do
876 -- n.b. it is not necessary to throw out directories before doing
877 -- this check, because exists(nondirectory/__driver__.lua) will
878 -- never be true.
879 if exists(testdir .. "/" .. candidate .. "/__driver__.lua") then
880table.insert(tests, candidate)
881 end
882 end
883 table.sort(tests)
884
885 for i,a in pairs(args) do
886 local _1,_2,l,r = string.find(a, "^(-?%d+)%.%.(-?%d+)$")
887 if _1 then
888 l = l + 0
889 r = r + 0
890 if l < 1 then l = table.getn(tests) + l + 1 end
891 if r < 1 then r = table.getn(tests) + r + 1 end
892 if l > r then l,r = r,l end
893 for j = l,r do
894 torun[j] = tests[j]
895 end
896 run_all = false
897 elseif string.find(a, "^-?%d+$") then
898 r = a + 0
899 if r < 1 then r = table.getn(tests) + r + 1 end
900 torun[r] = tests[r]
901 run_all = false
902 else
903 -- pattern
904 run_all = false
905 local matched = false
906 for i,t in pairs(tests) do
907 if regex.search(a, t) then
908 torun[i] = t
909 matched = true
910 end
911 end
912 if not matched then
913 print(string.format("Warning: pattern '%s' does not match any tests.", a))
914 end
915 end
916 end
917
918 if run_all then torun = tests end
919
920 if list_only then
921 for i,t in pairs(torun) do
922 if i < 10 then P(" ") end
923 if i < 100 then P(" ") end
924 P(i .. " " .. t .. "\n")
925 end
926 logfile:close()
927 return 0
928 end
929
930 logfile:write("Running on ", get_ostype(), "\n\n")
931 local s = prepare_to_run_tests(P)
932 if s ~= 0 then
933 P("Test suite preparation failed.\n")
934 return s
935 end
936 P("Running tests...\n")
937
938 local counts = {}
939 counts.success = 0
940 counts.skip = 0
941 counts.xfail = 0
942 counts.noxfail = 0
943 counts.fail = 0
944 counts.total = 0
945 counts.of_interest = 0
946 local of_interest = {}
947 local failed_testlogs = {}
948
949 -- exit codes which indicate failure at a point in the process-spawning
950 -- code where it is impossible to give more detailed diagnostics
951 local magic_exit_codes = {
952 [121] = "error creating test directory",
953 [122] = "error spawning test process",
954 [123] = "error entering test directory",
955 [124] = "unhandled exception in child process"
956 }
957
958 -- callback closure passed to run_tests_in_children
959 local function report_one_test(tno, tname, status)
960 local tdir = run_dir .. "/" .. tname
961 local test_header = string.format("%3d %-45s ", tno, tname)
962 local what
963 local can_delete
964 -- the child should always exit successfully, just to avoid
965 -- headaches. if we get any other code we report it as a failure.
966 if status ~= 0 then
967if status < 0 then
968 what = string.format("FAIL (signal %d)", -status)
969elseif magic_exit_codes[status] ~= nil then
970 what = string.format("FAIL (%s)", magic_exit_codes[status])
971else
972 what = string.format("FAIL (exit %d)", status)
973end
974 else
975local wfile, err = io.open(tdir .. "/STATUS", "r")
976if wfile ~= nil then
977 what = string.gsub(wfile:read("*a"), "\n$", "")
978 wfile:close()
979else
980 what = string.format("FAIL (status file: %s)", err)
981end
982 end
983 if what == "unexpected success" then
984counts.noxfail = counts.noxfail + 1
985counts.of_interest = counts.of_interest + 1
986table.insert(of_interest, test_header .. "unexpected success")
987can_delete = false
988 elseif what == "partial skip" or what == "ok" then
989counts.success = counts.success + 1
990can_delete = true
991 elseif string.find(what, "skipped ") == 1 then
992counts.skip = counts.skip + 1
993can_delete = true
994 elseif string.find(what, "expected failure ") == 1 then
995counts.xfail = counts.xfail + 1
996can_delete = false
997 elseif string.find(what, "FAIL ") == 1 then
998counts.fail = counts.fail + 1
999table.insert(of_interest, test_header .. what)
1000table.insert(failed_testlogs, tdir .. "/tester.log")
1001can_delete = false
1002 else
1003counts.fail = counts.fail + 1
1004what = "FAIL (gobbledygook: " .. what .. ")"
1005table.insert(of_interest, test_header .. what)
1006table.insert(failed_testlogs, tdir .. "/tester.log")
1007can_delete = false
1008 end
1009
1010 counts.total = counts.total + 1
1011 P(string.format("%s%s\n", test_header, what))
1012 return (can_delete and not debugging)
1013 end
1014
1015 run_tests_in_children(torun, report_one_test)
1016
1017 if counts.of_interest ~= 0 and (counts.total / counts.of_interest) > 4 then
1018 P("\nInteresting tests:\n")
1019 for i,x in ipairs(of_interest) do
1020 P(x, "\n")
1021 end
1022 end
1023 P("\n")
1024
1025 for i,log in pairs(failed_testlogs) do
1026 local tlog = io.open(log, "r")
1027 if tlog ~= nil then
1028 local dat = tlog:read("*a")
1029 tlog:close()
1030 logfile:write("\n", string.rep("*", 50), "\n")
1031 logfile:write(dat)
1032 end
1033 end
1034
1035 -- Write out this summary in one go so that it does not get interrupted
1036 -- by concurrent test suites' summaries.
1037 P(string.format("Of %i tests run:\n"..
1038 "\t%i succeeded\n"..
1039 "\t%i failed\n"..
1040 "\t%i had expected failures\n"..
1041 "\t%i succeeded unexpectedly\n"..
1042 "\t%i were skipped\n",
1043 counts.total, counts.success, counts.fail,
1044 counts.xfail, counts.noxfail, counts.skip))
1045
1046 logfile:close()
1047 if counts.success + counts.skip + counts.xfail == counts.total then
1048 return 0
1049 else
1050 return 1
1051 end
1052end
1053
1054function run_one_test(tname)
1055 test.bgid = 0
1056 test.name = tname
1057 test.wanted_fail = false
1058 test.partial_skip = false
1059 test.root = chdir(".")
1060 test.errfile = ""
1061 test.errline = -1
1062 test.bglist = {}
1063 test.log = io.open("tester.log", "w")
1064
1065 -- Sanitize $HOME. This is done here so that each test gets its
1066 -- very own empty directory (in case some test writes stuff inside).
1067 unlogged_mkdir("emptyhomedir")
1068 test.home = test.root .. "/emptyhomedir"
1069 if ostype == "Windows" then
1070 set_env("APPDATA", test.home)
1071 else
1072 set_env("HOME", test.home)
1073 end
1074
1075 L("Test ", test.name, "\n")
1076
1077 local driverfile = testdir .. "/" .. test.name .. "/__driver__.lua"
1078 local driver, e = loadfile(driverfile)
1079 local r
1080 if driver == nil then
1081 r = false
1082 e = "Could not load driver file " .. driverfile .. ".\n" .. e
1083 else
1084 local oldmask = posix_umask(0)
1085 posix_umask(oldmask)
1086 r,e = xpcall(driver, debug.traceback)
1087 local errline = test.errline
1088 for i,b in pairs(test.bglist) do
1089 local a,x = pcall(function () b:finish(0) end)
1090 if r and not a then
1091 r = a
1092 e = x
1093 elseif not a then
1094 L("Error cleaning up background processes: ",
1095 tostring(b.locstr), "\n")
1096 end
1097 end
1098 if type(cleanup) == "function" then
1099 local a,b = pcall(cleanup)
1100 if r and not a then
1101 r = a
1102 e = b
1103 end
1104 end
1105 test.errline = errline
1106 posix_umask(oldmask)
1107 end
1108
1109 if not r then
1110 if test.errline == nil then test.errline = -1 end
1111 if type(e) ~= "table" then
1112 local tbl = {e = e, bt = {"no backtrace; type(err) = "..type(e)}}
1113 e = tbl
1114 end
1115 if type(e.e) ~= "boolean" then
1116 log_error(e)
1117 end
1118 end
1119 test.log:close()
1120
1121 -- record the short status where report_one_test can find it
1122 local s = io.open(test.root .. "/STATUS", "w")
1123 if r then
1124 if test.wanted_fail then
1125 s:write("unexpected success\n")
1126 else
1127 if test.partial_skip then
1128 s:write("partial skip\n")
1129 else
1130 s:write("ok\n")
1131 end
1132 end
1133 else
1134 if e.e == true then
1135 s:write(string.format("skipped (line %i)\n", test.errline))
1136 elseif e.e == false then
1137 s:write(string.format("expected failure (line %i)\n",
1138 test.errline))
1139 else
1140 s:write(string.format("FAIL (line %i)\n", test.errline))
1141 end
1142 end
1143 s:close()
1144 return 0
1145end

Archive Download this file

Branches

Tags

Quick Links:     www.monotone.ca    -     Downloads    -     Documentation    -     Wiki    -     Code Forge    -     Build Status