diff --git a/runtests.py b/runtests.py
index b1c559a8ff8e55e5fa94a7c45d8860d876572bb7_cnVudGVzdHMucHk=..4d81d404fbba3a9f5a4a1c08b40a732e448ae7a0_cnVudGVzdHMucHk= 100755
--- a/runtests.py
+++ b/runtests.py
@@ -1747,6 +1747,8 @@
                 out.append(_out)
                 err.append(_err)
             res = p.returncode
+            if res == 0 and b'REFNANNY: ' in _out:
+                res = -1
             if res != 0:
                 for c, o, e in zip(cmd, out, err):
                     sys.stderr.write("%s\n%s\n%s\n\n" % (