diff --git a/roles/letsencrypt/tasks/main.yml b/roles/letsencrypt/tasks/main.yml
index 9d7879cc53..8b1f212165 100644
--- a/roles/letsencrypt/tasks/main.yml
+++ b/roles/letsencrypt/tasks/main.yml
@@ -3,7 +3,7 @@
   command: certbot certonly --keep -n --webroot --webroot-path /var/www/html/ -d {{','.join([site_name] + server_aliases)}}
   run_once: true
   register: certbot_output
-  changed_when: "not ('no action taken' in certbot_output)"
+  changed_when: "not ('no action taken' in certbot_output.stderr)"
   tags:
   - letsencrypt
 
@@ -12,7 +12,7 @@
   delegate_to: certgetter01.phx2.fedoraproject.org
   command: cat /etc/letsencrypt/live/{{site_name}}/cert.pem
   register: certbot_certificate
-  when: "not ('no action taken' in certbot_output)"
+  when: "not ('no action taken' in certbot_output.stderr)"
   tags:
   - letsencrypt
 
@@ -20,7 +20,7 @@
   delegate_to: certgetter01.phx2.fedoraproject.org
   command: cat /etc/letsencrypt/live/{{site_name}}/chain.pem
   register: certbot_chain
-  when: "not ('no action taken' in certbot_output)"
+  when: "not ('no action taken' in certbot_output.stderr)"
   tags:
   - letsencrypt
 
@@ -28,7 +28,7 @@
   delegate_to: certgetter01.phx2.fedoraproject.org
   command: cat /etc/letsencrypt/live/{{site_name}}/privkey.pem
   register: certbot_key
-  when: "not ('no action taken' in certbot_output)"
+  when: "not ('no action taken' in certbot_output.stderr)"
   tags:
   - letsencrypt
 
@@ -39,7 +39,7 @@
     owner=root
     group=root
     mode=0644
-  when: "not ('no action taken' in certbot_output)"
+  when: "not ('no action taken' in certbot_output.stderr)"
   notify:
   - reload proxyhttpd
   tags:
@@ -52,7 +52,7 @@
     owner=root
     group=root
     mode=0644
-  when: "not ('no action taken' in certbot_output)"
+  when: "not ('no action taken' in certbot_output.stderr)"
   notify:
   - reload proxyhttpd
   tags:
@@ -65,7 +65,7 @@
     owner=root
     group=root
     mode=0600
-  when: "not ('no action taken' in certbot_output)"
+  when: "not ('no action taken' in certbot_output.stderr)"
   notify:
   - reload proxyhttpd
   tags: