diff --git a/spring-boot-cli/src/main/java/org/springframework/boot/cli/command/CommandRunner.java b/spring-boot-cli/src/main/java/org/springframework/boot/cli/command/CommandRunner.java
index f2bd5165348..d59b6970c01 100644
--- a/spring-boot-cli/src/main/java/org/springframework/boot/cli/command/CommandRunner.java
+++ b/spring-boot-cli/src/main/java/org/springframework/boot/cli/command/CommandRunner.java
@@ -25,6 +25,7 @@ import java.util.List;
 import java.util.Set;
 
 import org.springframework.boot.cli.command.status.ExitStatus;
+import org.springframework.boot.cli.command.test.TestFailedException;
 import org.springframework.boot.cli.util.Log;
 import org.springframework.util.Assert;
 import org.springframework.util.StringUtils;
@@ -178,6 +179,9 @@ public class CommandRunner implements Iterable<Command> {
 			showUsage();
 			return 1;
 		}
+		catch(TestFailedException e) {
+			return 1;
+		}
 		catch (Exception ex) {
 			return handleError(debug, ex);
 		}
diff --git a/spring-boot-cli/src/main/java/org/springframework/boot/cli/command/test/TestFailedException.java b/spring-boot-cli/src/main/java/org/springframework/boot/cli/command/test/TestFailedException.java
new file mode 100644
index 00000000000..47d2265d774
--- /dev/null
+++ b/spring-boot-cli/src/main/java/org/springframework/boot/cli/command/test/TestFailedException.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2014 original authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.boot.cli.command.test;
+
+/**
+ * Thrown when tests fail to execute
+ *
+ * @author Graeme Rocher
+ * @since 1.2
+ */
+public class TestFailedException extends RuntimeException {
+    public TestFailedException(Throwable cause) {
+        super(cause);
+    }
+}
diff --git a/spring-boot-cli/src/main/java/org/springframework/boot/cli/command/test/TestRunner.java b/spring-boot-cli/src/main/java/org/springframework/boot/cli/command/test/TestRunner.java
index b97af7bd6da..758deeb72ca 100644
--- a/spring-boot-cli/src/main/java/org/springframework/boot/cli/command/test/TestRunner.java
+++ b/spring-boot-cli/src/main/java/org/springframework/boot/cli/command/test/TestRunner.java
@@ -17,6 +17,7 @@
 package org.springframework.boot.cli.command.test;
 
 import java.lang.annotation.Annotation;
+import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.List;
@@ -37,6 +38,8 @@ public class TestRunner {
 
 	private final GroovyCompiler compiler;
 
+	private volatile Throwable threadException;
+
 	/**
 	 * Create a new {@link TestRunner} instance.
 	 * @param configuration
@@ -58,7 +61,20 @@ public class TestRunner {
 		// Run in new thread to ensure that the context classloader is setup
 		RunThread runThread = new RunThread(sources);
 		runThread.start();
+		runThread.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
+			@Override
+			public void uncaughtException(Thread t, Throwable e) {
+				// rethrow exception
+				threadException = e;
+			}
+		});
+
 		runThread.join();
+		if(threadException != null) {
+			Throwable t = threadException;
+			threadException = null;
+			throw new TestFailedException(t);
+		}
 	}
 
 	/**
@@ -131,15 +147,33 @@ public class TestRunner {
 					System.out.println("No tests found");
 				}
 				else {
-					Class<?> delegateClass = Thread.currentThread()
-							.getContextClassLoader()
-							.loadClass(DelegateTestRunner.class.getName());
-					Method runMethod = delegateClass.getMethod("run", Class[].class);
-					runMethod.invoke(null, new Object[] { this.testClasses });
+					ClassLoader contextClassLoader = Thread.currentThread()
+							.getContextClassLoader();
+					Class<?> delegateClass = contextClassLoader
+												.loadClass(DelegateTestRunner.class.getName());
+					Class resultClass = contextClassLoader.loadClass("org.junit.runner.Result");
+					Method runMethod = delegateClass.getMethod("run", Class[].class, resultClass);
+					Object result = resultClass.newInstance();
+					runMethod.invoke(null, this.testClasses, result);
+					Boolean wasSuccessful = (Boolean)resultClass.getMethod("wasSuccessful").invoke(result);
+					if(!wasSuccessful) {
+						try {
+							throw new RuntimeException("Tests Failed.");
+						} catch (Throwable throwable) {
+							throw new RuntimeException(throwable);
+						}
+					}
 				}
-			}
-			catch (Exception ex) {
-				ex.printStackTrace();
+			} catch (ClassNotFoundException e) {
+				throw new RuntimeException("Exception occurred running tests: " + e.getMessage(), e);
+			} catch (NoSuchMethodException e) {
+				throw new RuntimeException("Exception occurred running tests: " + e.getMessage(), e);
+			} catch (InstantiationException e) {
+				throw new RuntimeException("Exception occurred running tests: " + e.getMessage(), e);
+			} catch (IllegalAccessException e) {
+				throw new RuntimeException("Exception occurred running tests: " + e.getMessage(), e);
+			} catch (InvocationTargetException e) {
+				throw new RuntimeException("Exception occurred running tests: " + e.getMessage(), e);
 			}
 		}
 	}
diff --git a/spring-boot-cli/src/main/java/org/springframework/boot/groovy/DelegateTestRunner.java b/spring-boot-cli/src/main/java/org/springframework/boot/groovy/DelegateTestRunner.java
index 3ec9aa784a4..2177e517b62 100644
--- a/spring-boot-cli/src/main/java/org/springframework/boot/groovy/DelegateTestRunner.java
+++ b/spring-boot-cli/src/main/java/org/springframework/boot/groovy/DelegateTestRunner.java
@@ -18,6 +18,7 @@ package org.springframework.boot.groovy;
 
 import org.junit.internal.TextListener;
 import org.junit.runner.JUnitCore;
+import org.junit.runner.Result;
 import org.springframework.boot.cli.command.test.TestRunner;
 
 /**
@@ -28,9 +29,10 @@ import org.springframework.boot.cli.command.test.TestRunner;
  */
 public class DelegateTestRunner {
 
-	public static void run(Class<?>[] testClasses) {
+	public static void run(Class<?>[] testClasses, Result result) {
 		JUnitCore jUnitCore = new JUnitCore();
 		jUnitCore.addListener(new TextListener(System.out));
+		jUnitCore.addListener(result.createListener());
 		jUnitCore.run(testClasses);
 	}
 
diff --git a/spring-boot-cli/src/test/java/org/springframework/boot/cli/CliTester.java b/spring-boot-cli/src/test/java/org/springframework/boot/cli/CliTester.java
index b934eeb8141..046689d71b5 100644
--- a/spring-boot-cli/src/test/java/org/springframework/boot/cli/CliTester.java
+++ b/spring-boot-cli/src/test/java/org/springframework/boot/cli/CliTester.java
@@ -78,8 +78,12 @@ public class CliTester implements TestRule {
 
 	public String test(String... args) throws Exception {
 		Future<TestCommand> future = submitCommand(new TestCommand(), args);
-		this.commands.add(future.get(this.timeout, TimeUnit.MILLISECONDS));
-		return getOutput();
+		try {
+			this.commands.add(future.get(this.timeout, TimeUnit.MILLISECONDS));
+			return getOutput();
+		} catch (Exception e) {
+			return getOutput();
+		}
 	}
 
 	public String grab(String... args) throws Exception {