diff --git a/ci/templates/maven-test-project/README.md b/ci/templates/maven-test-project/README.md
index 8de7396e72..d632853e53 100644
--- a/ci/templates/maven-test-project/README.md
+++ b/ci/templates/maven-test-project/README.md
@@ -1,5 +1,5 @@
 The purpose of this test project is to check if Compose Multiplatform is resolvable via pom files, which are used by JPS, which is used by IntelliJ
 
 ```
-mvn install exec:java -Dexec.mainClass="MainKt" -Dkotlin.version=2.1.0 -Dcompose.version=1.8.0-alpha02
+./check.sh -Dkotlin.version=2.1.0 -Dcompose.version=1.8.0-alpha02
 ```
diff --git a/ci/templates/maven-test-project/check.sh b/ci/templates/maven-test-project/check.sh
new file mode 100644
index 0000000000..e69227abfa
--- /dev/null
+++ b/ci/templates/maven-test-project/check.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -o pipefail
+
+tempfile=$(mktemp)
+
+# Pipe output to the file and terminal
+if ! mvn clean install exec:java -Dexec.mainClass="MainKt" "$@" | tee "$tempfile"; then
+    exit 1
+fi
+
+# For failing on warnings like
+#   [WARNING] The POM for org.jetbrains.kotlin:kotlin-stdlib:jar:unspecified is missing, no dependency information available
+#
+# There is no a flag in Maven to fail on warnings in dependency resolution
+
+if grep -q "\[WARNING\]" "$tempfile"; then
+    echo "[ERROR] Warnings found"
+    exit 1
+fi