diff --git a/tools/http_benchmark.py b/tools/http_benchmark.py
index 64abbf8baa..d5067d86b6 100755
--- a/tools/http_benchmark.py
+++ b/tools/http_benchmark.py
@@ -186,6 +186,10 @@ def run(server_cmd, port, merge_env=None, origin_cmd=None):
         print output
         return stats
     finally:
+        server_retcode = server.poll()
+        if server_retcode is not None and server_retcode != 0:
+            print "server ended with error"
+            sys.exit(1)
         server.kill()
         if origin is not None:
             origin.kill()
diff --git a/tools/http_server.py b/tools/http_server.py
index 0ed61465c2..e2e9f2d98e 100755
--- a/tools/http_server.py
+++ b/tools/http_server.py
@@ -352,6 +352,9 @@ def spawn():
         yield servers
     finally:
         for s in servers:
+            # Make sure all servers still running,
+            # if not assume there was an error
+            assert s.thread.is_alive()
             s.server.shutdown()