Verify statements in TestNG

If you are using TestNG framework for your Java tests then sometimes you might have faced a situation where you want your test execution to continue even if there are any assertion failures and throw all errors at the end. For this you need verify statements (also called soft assertions). Currently TestNG does not support verify statements. We have to implement them by our own.

There are several ways you can mimic verify statements in your test. One of the most widely accepted way is using StringBuffer where assertion statement will be placed inside try catch block and whenever assertion fails, the thrown error will be caught and stored in StringBuffer allowing your test method execution to continue. At the end of the test, buffered errors will be checked and logged if found, marking your test as failed. I personally don’t recommend using try catch block in your tests since it involves too much code writing and maintenance. Also, if you miss to check StringBuffer at the end of the test then the errors that you caught will not be logged making your test faulty.

We need to implement verify statements such that they look more cleaner and easy to use. For that we can leverage TestNG’s listener capability, so let us start implementing them,

Download complete source code here https://github.com/Gadigeppa-J/testng.git.

TestMethodErrorBuffer.java – Buffer class to hold your verification errors.

import java.util.List;

/* package access only */
class TestMethodErrorBuffer {

	// thread safe while running tests in parallel
	private static ThreadLocal<List<Throwable>> testErrorBuffer = new ThreadLocal<>();
	
	static List<Throwable> get(){
		return testErrorBuffer.get();
	}
	
	static void set(List<Throwable> errorBuffer){
		testErrorBuffer.set(errorBuffer);
	}
	
	static void remove(){
		testErrorBuffer.remove();
	}
	
}

VerificationError.java – Thrown to indicate that verification has failed. Error objects of this class will be buffered in TestMethodErrorBuffer.

public class VerificationError extends Error{

	private static final long serialVersionUID = 8247563849457669512L;

	public VerificationError(String message){
		super(message);
	}
	
}

Verify.java – Verify statements. Wrapper class around TestNG’s Assertion tool.

import org.testng.Assert;

public class Verify {

	protected Verify() {
		// hide constructor
	}

	static public void verifyTrue(boolean condition, String message) {		  
		try{
			Assert.assertTrue(condition, message);
		}catch(AssertionError e){
			addToErrorBuffer(e);
		}		  
	}

	static public void verifyFalse(boolean condition, String message) {
		try{
			Assert.assertFalse(condition, message);
		}catch(AssertionError e){
			addToErrorBuffer(e);
		}
	}

	static public void verifyEquals(Object actual, Object expected, String message) {
		try{
			Assert.assertEquals(actual, expected, message);
		}catch(AssertionError e){
			addToErrorBuffer(e);
		}
	}

	public static void verifyNotEquals(Object actual1, Object actual2, String message) {
		try{
			Assert.assertNotEquals(actual1,actual2,message);
		}catch(AssertionError e){
			addToErrorBuffer(e);
		}
	}

	/*
	 * Add other wrapper methods
	 */
	// ......
	// ......
	// ......
	// ......
	// ......

	private static void addToErrorBuffer(AssertionError e){	  

		try{				

			VerificationError verificationError = new VerificationError(e.getMessage());

			verificationError.setStackTrace(e.getStackTrace());

			TestMethodErrorBuffer.get().add(verificationError);

		}catch(NullPointerException ex){

			throw new RuntimeException("Please let TestNG know about " + TestMethodListener.class.getName() + " listener for verify statements to work. For more information go to http://testng.org/doc/documentation-main.html#testng-listeners");
		}

	}
}

TestMethodListener.java– Implementation of TestNG’s IInvokedMethodListener interface. Logic for this implementation is adapted from Dave’s blog (http://seleniumexamples.com/blog/guide/using-soft-assertions-in-testng). For verify statements to work you need to let TestNG know about this listener. For more information go to http://testng.org/doc/documentation-main.html#testng-listeners.

import java.util.ArrayList;
import java.util.List;

import org.testng.IInvokedMethod;
import org.testng.IInvokedMethodListener;
import org.testng.ITestResult;
import org.testng.internal.Utils;

public class TestMethodListener implements IInvokedMethodListener{

	@Override
	public void beforeInvocation(IInvokedMethod method, ITestResult testResult) {

		if(method.isTestMethod()){	
			
			if(TestMethodErrorBuffer.get()!=null){
				throw new RuntimeException("Stale error buffer detected!");
			}
			
			TestMethodErrorBuffer.set(new ArrayList<Throwable>()); // each test method will have its own error buffer
		}

	}

	@Override
	public void afterInvocation(IInvokedMethod method, ITestResult testResult) {

		if(method.isTestMethod()){

			List<Throwable> lThrowable = TestMethodErrorBuffer.get();

			/* if there are verification failures */
			if(lThrowable.size() > 0){

				/* set test result to failure */
				testResult.setStatus(ITestResult.FAILURE);

				/* if there is assertion error/exception then add it to throwable list */
				if(testResult.getThrowable() != null){
					lThrowable.add(testResult.getThrowable());
				}

				int size = lThrowable.size();

				/* if there is only one throwable then set it directly to test result */
				if(size == 1){
					testResult.setThrowable(lThrowable.get(0));
				}else{

					StringBuffer failureMessage = new StringBuffer("Multiple failures (").append(size).append(")\n");
					StringBuffer fullStack = new StringBuffer();

					for(int i =0 ; i < size-1; i++){	
						failureMessage.append("(").append(i+1).append(")").append(lThrowable.get(i).getClass().getName()).append(":").append(lThrowable.get(i).getMessage()).append("\n");						
						fullStack.append("Failure ").append(i+1).append(" of ").append(size).append("\n");	
						fullStack.append(Utils.stackTrace(lThrowable.get(i),false)[1]).append("\n");
					}

					fullStack.append("Failure ").append(size).append(" of ").append(size).append("\n");
					Throwable last = lThrowable.get(size-1);					
					failureMessage.append("(").append(size).append(")").append(last.getClass().getName()).append(":").append(last.getMessage()).append("\n\n");
					
					fullStack.append(last.toString());

					testResult.setThrowable(new Throwable(failureMessage.toString() + fullStack.toString()));
					testResult.getThrowable().setStackTrace(last.getStackTrace());
				}

			}
			
			TestMethodErrorBuffer.remove(); // remove stale
			
		}
	}

}

Now let us test out our new verification tool,

import org.testng.Assert;
import org.testng.annotations.Listeners;
import org.testng.annotations.Test;

/*
*  Using Listeners annotation (@Listeners) this is one of the way
*  to let TestNG know about your listeners
*/ 
@Listeners(TestMethodListener.class) 
public class VerifyTests {

	@Test
	public void verifyEqualsTest(){
		
		// verify pass
		Verify.verifyEquals("test1", "test1", "test1 doesn't match"); 
		
		// verify fail (continue)
		Verify.verifyEquals("test!", "test2", "test2 doesn't match"); 
		
		// verify pass
		Verify.verifyEquals("test3", "test3", "test3 doesn't match");
		
		// verify fail (continue)
		Verify.verifyEquals("test#", "test4", "test4 doesn't match");
		
		// verify pass
		Verify.verifyEquals("test5", "test5", "test5 doesn't match");
		
		// verify pass
		Verify.verifyEquals("test6", "test6", "test6 doesn't match");
		
		// assert fail (exit)
		Assert.assertEquals("test$", "test7", "test7 doesn't match");
		
		// assert not run!
		Assert.assertEquals("test8", "test8", "test8 doesn't match");
		
		// verify not run!
		Verify.verifyEquals("test9", "test9", "test9 doesn't match");
	}
	
}

// Output

/*		FAILED: test01
		java.lang.Throwable: Multiple failures (3)
		(1)org.muthu.VerificationError:test2 doesn't matches expected [test2] but found [test!]
		(2)org.muthu.VerificationError:test4 doesn't matches expected [test4] but found [test#]
		(3)java.lang.AssertionError:test7 doesn't matches expected [test7] but found [test$]

		Failure 1 of 3
		org.muthu.VerificationError: test2 doesn't matches expected [test2] but found [test!]
			at org.testng.Assert.fail(Assert.java:94)
			at org.testng.Assert.failNotEquals(Assert.java:494)
			at org.testng.Assert.assertEquals(Assert.java:123)
			at org.testng.Assert.assertEquals(Assert.java:176)
			at org.muthu.Verify.verifyEquals(Verify.java:90)
			at org.muthu.TestVerify.test01(TestVerify.java:15)
			at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
			at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
			at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
			at java.lang.reflect.Method.invoke(Unknown Source)
			at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:84)
			at org.testng.internal.Invoker.invokeMethod(Invoker.java:714)
			at org.testng.internal.Invoker.invokeTestMethod(Invoker.java:901)
			at org.testng.internal.Invoker.invokeTestMethods(Invoker.java:1231)
			at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:127)
			at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:111)
			at org.testng.TestRunner.privateRun(TestRunner.java:767)
			at org.testng.TestRunner.run(TestRunner.java:617)
			at org.testng.SuiteRunner.runTest(SuiteRunner.java:334)
			at org.testng.SuiteRunner.runSequentially(SuiteRunner.java:329)
			at org.testng.SuiteRunner.privateRun(SuiteRunner.java:291)
			at org.testng.SuiteRunner.run(SuiteRunner.java:240)
			at org.testng.SuiteRunnerWorker.runSuite(SuiteRunnerWorker.java:52)
			at org.testng.SuiteRunnerWorker.run(SuiteRunnerWorker.java:86)
			at org.testng.TestNG.runSuitesSequentially(TestNG.java:1224)
			at org.testng.TestNG.runSuitesLocally(TestNG.java:1149)
			at org.testng.TestNG.run(TestNG.java:1057)
			at org.testng.remote.RemoteTestNG.run(RemoteTestNG.java:111)
			at org.testng.remote.RemoteTestNG.initAndRun(RemoteTestNG.java:204)
			at org.testng.remote.RemoteTestNG.main(RemoteTestNG.java:175)

		Failure 2 of 3
		org.muthu.VerificationError: test4 doesn't matches expected [test4] but found [test#]
			at org.testng.Assert.fail(Assert.java:94)
			at org.testng.Assert.failNotEquals(Assert.java:494)
			at org.testng.Assert.assertEquals(Assert.java:123)
			at org.testng.Assert.assertEquals(Assert.java:176)
			at org.muthu.Verify.verifyEquals(Verify.java:90)
			at org.muthu.TestVerify.test01(TestVerify.java:17)
			at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
			at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
			at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
			at java.lang.reflect.Method.invoke(Unknown Source)
			at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:84)
			at org.testng.internal.Invoker.invokeMethod(Invoker.java:714)
			at org.testng.internal.Invoker.invokeTestMethod(Invoker.java:901)
			at org.testng.internal.Invoker.invokeTestMethods(Invoker.java:1231)
			at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:127)
			at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:111)
			at org.testng.TestRunner.privateRun(TestRunner.java:767)
			at org.testng.TestRunner.run(TestRunner.java:617)
			at org.testng.SuiteRunner.runTest(SuiteRunner.java:334)
			at org.testng.SuiteRunner.runSequentially(SuiteRunner.java:329)
			at org.testng.SuiteRunner.privateRun(SuiteRunner.java:291)
			at org.testng.SuiteRunner.run(SuiteRunner.java:240)
			at org.testng.SuiteRunnerWorker.runSuite(SuiteRunnerWorker.java:52)
			at org.testng.SuiteRunnerWorker.run(SuiteRunnerWorker.java:86)
			at org.testng.TestNG.runSuitesSequentially(TestNG.java:1224)
			at org.testng.TestNG.runSuitesLocally(TestNG.java:1149)
			at org.testng.TestNG.run(TestNG.java:1057)
			at org.testng.remote.RemoteTestNG.run(RemoteTestNG.java:111)
			at org.testng.remote.RemoteTestNG.initAndRun(RemoteTestNG.java:204)
			at org.testng.remote.RemoteTestNG.main(RemoteTestNG.java:175)

		Failure 3 of 3
		java.lang.AssertionError: test7 doesn't matches expected [test7] but found [test$]
			at org.testng.Assert.fail(Assert.java:94)
			at org.testng.Assert.failNotEquals(Assert.java:494)
			at org.testng.Assert.assertEquals(Assert.java:123)
			at org.testng.Assert.assertEquals(Assert.java:176)
			at org.muthu.TestVerify.test01(TestVerify.java:20)
			at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
			at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
			at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
			at java.lang.reflect.Method.invoke(Unknown Source)
			at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:84)
			at org.testng.internal.Invoker.invokeMethod(Invoker.java:714)
			at org.testng.internal.Invoker.invokeTestMethod(Invoker.java:901)
			at org.testng.internal.Invoker.invokeTestMethods(Invoker.java:1231)
			at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:127)
			at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:111)
			at org.testng.TestRunner.privateRun(TestRunner.java:767)
			at org.testng.TestRunner.run(TestRunner.java:617)
			at org.testng.SuiteRunner.runTest(SuiteRunner.java:334)
			at org.testng.SuiteRunner.runSequentially(SuiteRunner.java:329)
			at org.testng.SuiteRunner.privateRun(SuiteRunner.java:291)
			at org.testng.SuiteRunner.run(SuiteRunner.java:240)
			at org.testng.SuiteRunnerWorker.runSuite(SuiteRunnerWorker.java:52)
			at org.testng.SuiteRunnerWorker.run(SuiteRunnerWorker.java:86)
			at org.testng.TestNG.runSuitesSequentially(TestNG.java:1224)
			at org.testng.TestNG.runSuitesLocally(TestNG.java:1149)
			at org.testng.TestNG.run(TestNG.java:1057)
			at org.testng.remote.RemoteTestNG.run(RemoteTestNG.java:111)
			at org.testng.remote.RemoteTestNG.initAndRun(RemoteTestNG.java:204)
			at org.testng.remote.RemoteTestNG.main(RemoteTestNG.java:175)


		===============================================
		    Default test
		    Tests run: 1, Failures: 1, Skips: 0
		===============================================*/