convert pactest to TAP output

Each test produces a single TAP result with the rules run in a sub-test.
This reduces output when run under automake and makes it possible to
continue setting expectfailure at the test level rather than per-rule.

Signed-off-by: Andrew Gregory <andrew.gregory.8@gmail.com>
Signed-off-by: Allan McRae <allan@archlinux.org>

7 files changed, 121 insertions, 59 deletions

diff --git a/test/pacman/pactest.py b/test/pacman/pactest.py
index 2b1dee63..fe04c2b3 100755
--- a/test/pacman/pactest.py
+++ b/test/pacman/pactest.py
@@ -26,6 +26,7 @@ import sys
 import tempfile
 
 import pmenv
+import tap
 import util
 
 __author__ = "Aurelien FORET"
@@ -110,7 +111,7 @@ if __name__ == "__main__":
     env.pacman["ldconfig"] = opts.ldconfig
 
     if opts.testcases is None or len(opts.testcases) == 0:
-        print "no tests defined, nothing to do"
+        tap.bail("no tests defined, nothing to do")
         os.rmdir(root_path)
         sys.exit(2)
 
@@ -124,7 +125,7 @@ if __name__ == "__main__":
     if not opts.keeproot:
         shutil.rmtree(root_path)
     else:
-        print "pacman testing root saved: %s" % root_path
+        tap.diag("pacman testing root saved: %s" % root_path)
 
     if env.failed > 0:
         sys.exit(1)
diff --git a/test/pacman/pmdb.py b/test/pacman/pmdb.py
index 3e9d305e..b7b3522a 100644
--- a/test/pacman/pmdb.py
+++ b/test/pacman/pmdb.py
@@ -23,6 +23,7 @@ from StringIO import StringIO
 import tarfile
 
 import pmpkg
+import tap
 import util
 
 def _getsection(fd):
@@ -105,7 +106,7 @@ class pmdb(object):
         # desc
         filename = os.path.join(path, "desc")
         if not os.path.isfile(filename):
-            print "invalid db entry found (desc missing) for pkg", pkgname
+            tap.bail("invalid db entry found (desc missing) for pkg " + pkgname)
             return None
         fd = open(filename, "r")
         while 1:
@@ -160,7 +161,7 @@ class pmdb(object):
         # files
         filename = os.path.join(path, "files")
         if not os.path.isfile(filename):
-            print "invalid db entry found (files missing) for pkg", pkgname
+            tap.bail("invalid db entry found (files missing) for pkg " + pkgname)
             return None
         fd = open(filename, "r")
         while 1:
diff --git a/test/pacman/pmenv.py b/test/pacman/pmenv.py
index 5eaa473c..a3a8f54f 100644
--- a/test/pacman/pmenv.py
+++ b/test/pacman/pmenv.py
@@ -20,6 +20,7 @@
 import os
 
 import pmtest
+import tap
 
 
 class pmenv(object):
@@ -58,26 +59,18 @@ class pmenv(object):
     def run(self):
         """
         """
-
+        tap.plan(len(self.testcases))
         for t in self.testcases:
-            print "=========="*8
-            print "Running '%s'" % t.testname
+            tap.diag("==========" * 8)
+            tap.diag("Running '%s'" % t.testname)
 
             t.load()
-            print t.description
-            print "----------"*8
-
             t.generate(self.pacman)
-
             t.run(self.pacman)
 
-            t.check()
-            print "==> Test result"
-            if t.result["fail"] == 0:
-                print "\tPASS"
-            else:
-                print "\tFAIL"
-            print
+            tap.diag("==> Checking rules")
+            tap.todo = t.expectfailure
+            tap.subtest(lambda: t.check(), t.description)
 
     def results(self):
         """
@@ -109,40 +102,42 @@ class pmenv(object):
                 result = "[PASS]"
             else:
                 result = "[FAIL]"
-            print result,
-            print "%s Rules: OK = %2u  FAIL = %2u" \
-                    % (t.testname.ljust(34), success, fail)
+            tap.diag("%s %s Rules: OK = %2u  FAIL = %2u" \
+                    % (result, t.testname.ljust(34), success, fail))
             if fail != 0:
                 # print test description if test failed
-                print "      ", t.description
+                tap.diag("       " + t.description)
 
-        print "=========="*8
-        print "Results"
-        print "----------"*8
-        print " Passed:"
+        tap.diag("==========" * 8)
+        tap.diag("Results")
+        tap.diag("----------" * 8)
+        tap.diag(" Passed:")
         for test in tpassed:
             _printtest(test)
-        print "----------"*8
-        print " Expected Failures:"
+        tap.diag("----------" * 8)
+        tap.diag(" Expected Failures:")
         for test in texpectedfail:
             _printtest(test)
-        print "----------"*8
-        print " Unexpected Passes:"
+        tap.diag("----------" * 8)
+        tap.diag(" Unexpected Passes:")
         for test in tunexpectedpass:
             _printtest(test)
-        print "----------"*8
-        print " Failed:"
+        tap.diag("----------" * 8)
+        tap.diag(" Failed:")
         for test in tfailed:
             _printtest(test)
-        print "----------"*8
+        tap.diag("----------" * 8)
 
         total = len(self.testcases)
-        print "Total            = %3u" % total
+        tap.diag("Total            = %3u" % total)
         if total:
-            print "Pass             = %3u (%6.2f%%)" % (self.passed, float(self.passed) * 100 / total)
-            print "Expected Fail    = %3u (%6.2f%%)" % (self.expectedfail, float(self.expectedfail) * 100 / total)
-            print "Unexpected Pass  = %3u (%6.2f%%)" % (self.unexpectedpass, float(self.unexpectedpass) * 100 / total)
-            print "Fail             = %3u (%6.2f%%)" % (self.failed, float(self.failed) * 100 / total)
-        print ""
+            tap.diag("Pass             = %3u (%6.2f%%)" % (self.passed,
+                float(self.passed) * 100 / total))
+            tap.diag("Expected Fail    = %3u (%6.2f%%)" % (self.expectedfail,
+                float(self.expectedfail) * 100 / total))
+            tap.diag("Unexpected Pass  = %3u (%6.2f%%)" % (self.unexpectedpass,
+                float(self.unexpectedpass) * 100 / total))
+            tap.diag("Fail             = %3u (%6.2f%%)" % (self.failed,
+                float(self.failed) * 100 / total))
 
 # vim: set ts=4 sw=4 et:
diff --git a/test/pacman/pmrule.py b/test/pacman/pmrule.py
index 3d38b852..c97a1584 100644
--- a/test/pacman/pmrule.py
+++ b/test/pacman/pmrule.py
@@ -19,6 +19,7 @@
 import os
 import stat
 
+import tap
 import util
 
 class pmrule(object):
@@ -57,12 +58,12 @@ class pmrule(object):
             elif case == "OUTPUT":
                 logfile = os.path.join(test.root, util.LOGFILE)
                 if not os.access(logfile, os.F_OK):
-                    print "LOGFILE not found, cannot validate 'OUTPUT' rule"
+                    tap.diag("LOGFILE not found, cannot validate 'OUTPUT' rule")
                     success = 0
                 elif not util.grep(logfile, key):
                     success = 0
             else:
-                print "PACMAN rule '%s' not found" % case
+                tap.diag("PACMAN rule '%s' not found" % case)
                 success = -1
         elif kind == "PKG":
             localdb = test.db["local"]
@@ -108,7 +109,7 @@ class pmrule(object):
                     if not found:
                         success = 0
                 else:
-                    print "PKG rule '%s' not found" % case
+                    tap.diag("PKG rule '%s' not found" % case)
                     success = -1
         elif kind == "FILE":
             filename = os.path.join(test.root, key)
@@ -148,7 +149,7 @@ class pmrule(object):
                 if not os.path.isfile("%s.pacsave" % filename):
                     success = 0
             else:
-                print "FILE rule '%s' not found" % case
+                tap.diag("FILE rule '%s' not found" % case)
                 success = -1
         elif kind == "DIR":
             filename = os.path.join(test.root, key)
@@ -156,7 +157,7 @@ class pmrule(object):
                 if not os.path.isdir(filename):
                     success = 0
             else:
-                print "DIR rule '%s' not found" % case
+                tap.diag("DIR rule '%s' not found" % case)
                 success = -1
         elif kind == "LINK":
             filename = os.path.join(test.root, key)
@@ -164,7 +165,7 @@ class pmrule(object):
                 if not os.path.islink(filename):
                     success = 0
             else:
-                print "LINK rule '%s' not found" % case
+                tap.diag("LINK rule '%s' not found" % case)
                 success = -1
         elif kind == "CACHE":
             cachedir = os.path.join(test.root, util.PM_CACHEDIR)
@@ -174,7 +175,7 @@ class pmrule(object):
                         os.path.join(cachedir, pkg.filename())):
                     success = 0
         else:
-            print "Rule kind '%s' not found" % kind
+            tap.diag("Rule kind '%s' not found" % kind)
             success = -1
 
         if self.false and success != -1:
diff --git a/test/pacman/pmtest.py b/test/pacman/pmtest.py
index cea584da..b343d55f 100644
--- a/test/pacman/pmtest.py
+++ b/test/pacman/pmtest.py
@@ -27,6 +27,7 @@ import time
 import pmrule
 import pmdb
 import pmfile
+import tap
 import util
 from util import vprint
 
@@ -104,7 +105,7 @@ class pmtest(object):
             raise IOError("file %s does not exist!" % self.name)
 
     def generate(self, pacman):
-        print "==> Generating test environment"
+        tap.diag("==> Generating test environment")
 
         # Cleanup leftover files from a previous test session
         if os.path.isdir(self.root):
@@ -192,23 +193,23 @@ class pmtest(object):
 
     def run(self, pacman):
         if os.path.isfile(util.PM_LOCK):
-            print "\tERROR: another pacman session is on-going -- skipping"
+            tap.bail("\tERROR: another pacman session is on-going -- skipping")
             return
 
-        print "==> Running test"
+        tap.diag("==> Running test")
         vprint("\tpacman %s" % self.args)
 
         cmd = []
         if os.geteuid() != 0:
             fakeroot = util.which("fakeroot")
             if not fakeroot:
-                print "WARNING: fakeroot not found!"
+                tap.diag("WARNING: fakeroot not found!")
             else:
                 cmd.append("fakeroot")
 
             fakechroot = util.which("fakechroot")
             if not fakechroot:
-                print "WARNING: fakechroot not found!"
+                tap.diag("WARNING: fakechroot not found!")
             else:
                 cmd.append("fakechroot")
 
@@ -252,23 +253,20 @@ class pmtest(object):
 
         # Check if the lock is still there
         if os.path.isfile(util.PM_LOCK):
-            print "\tERROR: %s not removed" % util.PM_LOCK
+            tap.diag("\tERROR: %s not removed" % util.PM_LOCK)
             os.unlink(util.PM_LOCK)
         # Look for a core file
         if os.path.isfile(os.path.join(self.root, util.TMPDIR, "core")):
-            print "\tERROR: pacman dumped a core file"
+            tap.diag("\tERROR: pacman dumped a core file")
 
     def check(self):
-        print "==> Checking rules"
-
+        tap.plan(len(self.rules))
         for i in self.rules:
             success = i.check(self)
             if success == 1:
-                msg = " OK "
                 self.result["success"] += 1
             else:
-                msg = "FAIL"
                 self.result["fail"] += 1
-            print "\t[%s] %s" % (msg, i)
+            tap.ok(success, i)
 
 # vim: set ts=4 sw=4 et:
diff --git a/test/pacman/tap.py b/test/pacman/tap.py
new file mode 100644
index 00000000..c70a5350
--- /dev/null
+++ b/test/pacman/tap.py
@@ -0,0 +1,64 @@
+#  Copyright (c) 2013 Pacman Development Team <pacman-dev@archlinux.org>
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+todo = None
+count = 0
+level = 0
+failed = 0
+
+def _output(msg):
+    print("%s%s" % ("    "*level, msg))
+
+def ok(ok, description=""):
+    global count, failed
+    count += 1
+    if not ok:
+        failed += 1
+    directive = " # TODO" if todo else ""
+    _output("%s %d - %s%s" % ("ok" if ok else "not ok", count,
+        description, directive))
+
+def plan(count):
+    _output("1..%d" % (count))
+
+def diag(msg):
+    _output("# %s" % (msg))
+
+def bail(reason=""):
+    _output("Bail out! %s" % (reason))
+
+def subtest(func, description=""):
+    global todo, count, level, failed
+
+    save_todo = todo
+    save_count = count
+    save_level = level
+    save_failed = failed
+
+    todo = None
+    count = 0
+    level += 1
+    failed = 0
+
+    func()
+
+    subtest_ok = not failed
+
+    todo = save_todo
+    count = save_count
+    level = save_level
+    failed = save_failed
+
+    ok(subtest_ok, description)
diff --git a/test/pacman/util.py b/test/pacman/util.py
index 65540ed3..14035d76 100644
--- a/test/pacman/util.py
+++ b/test/pacman/util.py
@@ -21,6 +21,8 @@ import os
 import re
 import hashlib
 
+import tap
+
 SELFPATH    = os.path.abspath(os.path.dirname(__file__))
 
 # ALPM
@@ -43,7 +45,7 @@ verbose = 0
 
 def vprint(msg):
     if verbose:
-        print msg
+        tap.diag(msg)
 
 #
 # Methods to generate files