diff --git a/roles/check-compose/tasks/main.yml b/roles/check-compose/tasks/main.yml
index 289ac2cf69..c456b7996a 100644
--- a/roles/check-compose/tasks/main.yml
+++ b/roles/check-compose/tasks/main.yml
@@ -110,7 +110,8 @@
 
 - name: Install required packages
   package:
-    name: ['fedora-messaging', 'python3-fedfind', 'python3-openqa_client', 'python3-setuptools']
+    name: ['fedora-messaging', 'python3-fedfind', 'python3-openqa_client', 'python3-setuptools',
+           'python3-pip']
     state: present
   tags:
   - packages
@@ -121,11 +122,18 @@
     dest: /root/check-compose
   register: gitcc
 
+- name: Check if check-compose is installed for current Python
+  command: "pip show check_compose"
+  register: instcc
+  changed_when: "1 != 1"
+  failed_when: "1 != 1"
+  check_mode: no
+
 - name: Install check-compose
   command: "python3 setup.py install --nodeps"
   args:
     chdir: /root/check-compose
-  when: "gitcc is changed"
+  when: "gitcc is changed or instcc.rc != 0"
   notify:
   - restart check-compose
 
diff --git a/roles/openqa/dispatcher/tasks/main.yml b/roles/openqa/dispatcher/tasks/main.yml
index 56e23845ba..38ac5691fe 100644
--- a/roles/openqa/dispatcher/tasks/main.yml
+++ b/roles/openqa/dispatcher/tasks/main.yml
@@ -169,7 +169,7 @@
   package:
     name: ['fedora-messaging', 'python3-fedfind', 'python3-wikitcms', 'python3-openqa_client',
            'python3-requests', 'python3-resultsdb_api', 'python3-resultsdb_conventions-fedora',
-           'python3-setuptools', 'python3-six']
+           'python3-setuptools', 'python3-six', 'python3-pip']
     state: present
   tags:
   - packages
@@ -204,8 +204,8 @@
   register: gittools
   when: "(toolsbranch.stderr.find('Not a git repository') != -1) or (toolsbranch.stdout.find('On branch master') != -1)"
 
-- name: Check if fedora_openqa has ever been installed
-  stat: path=/usr/local/bin/fedora-openqa
+- name: Check if fedora_openqa is installed for current Python
+  command: "pip show fedora_openqa"
   register: insttools
   changed_when: "1 != 1"
   failed_when: "1 != 1"
@@ -215,7 +215,7 @@
   command: "python3 setup.py install"
   args:
     chdir: /root/fedora_openqa
-  when: "gittools is changed or not insttools.stat.exists"
+  when: "gittools is changed or insttools.rc != 0"
   notify:
   - restart openqa consumers
 
diff --git a/roles/relvalconsumer/tasks/main.yml b/roles/relvalconsumer/tasks/main.yml
index 19d8695089..9f47301e98 100644
--- a/roles/relvalconsumer/tasks/main.yml
+++ b/roles/relvalconsumer/tasks/main.yml
@@ -107,7 +107,7 @@
     # 'relval' itself is needed as we call it directly for size
     # checking
     name: ['python3-fedfind', 'python3-wikitcms', 'fedora-messaging',
-           'python3-mwclient', 'python3-setuptools', 'relval']
+           'python3-mwclient', 'python3-pip', 'python3-setuptools', 'relval']
     state: present
   tags:
   - packages
@@ -140,11 +140,18 @@
     dest: /root/relvalconsumer
   register: gitrvc
 
+- name: Check if relvalconsumer is installed for current Python
+  command: "pip show relvalconsumer"
+  register: instrvc
+  changed_when: "1 != 1"
+  failed_when: "1 != 1"
+  check_mode: no
+
 - name: Install relvalconsumer
   command: "python3 setup.py install --nodeps"
   args:
     chdir: /root/relvalconsumer
-  when: "gitrvc is changed"
+  when: "gitrvc is changed or instrvc.rc != 0"
   notify:
   - restart relvalconsumer