diff --git a/test/integration/targets/callback_log_plays/aliases b/test/integration/targets/callback_log_plays/aliases
new file mode 100644
index 0000000000..b59832142f
--- /dev/null
+++ b/test/integration/targets/callback_log_plays/aliases
@@ -0,0 +1 @@
+shippable/posix/group3
diff --git a/test/integration/targets/callback_log_plays/ping_log.yml b/test/integration/targets/callback_log_plays/ping_log.yml
new file mode 100644
index 0000000000..8015726ebb
--- /dev/null
+++ b/test/integration/targets/callback_log_plays/ping_log.yml
@@ -0,0 +1,4 @@
+- hosts: localhost
+  gather_facts: false
+  tasks:
+    - ping:
diff --git a/test/integration/targets/callback_log_plays/runme.sh b/test/integration/targets/callback_log_plays/runme.sh
new file mode 100755
index 0000000000..059f8976ef
--- /dev/null
+++ b/test/integration/targets/callback_log_plays/runme.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -eux
+
+export ANSIBLE_CALLBACK_WHITELIST="log_plays,${ANSIBLE_CALLBACK_WHITELIST:-}"
+
+# run play, should create log and dir if needed
+export ANSIBLE_LOG_FOLDER="logit"
+ansible-playbook ping_log.yml -v "$@"
+[[ -f "${ANSIBLE_LOG_FOLDER}/localhost" ]]
+
+# now force it to fail
+export ANSIBLE_LOG_FOLDER="logit.file"
+touch "${ANSIBLE_LOG_FOLDER}"
+ansible-playbook ping_log.yml -v "$@" 2>&1| grep 'Failure using method (v2_runner_on_ok) in callback plugin'
+[[ ! -f "${ANSIBLE_LOG_FOLDER}/localhost" ]]