If we fail to call bugzilla, retry a few times before raising the exception

This allows for temporary network glitches to be ignored.

Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>

distgit_bugzilla_sync/script.py

@@ -35,6 +35,7 @@ import datetime
 from email.message import EmailMessage
 import itertools
 import json
+import math
 from operator import itemgetter
 import os
 import re
@@ -157,6 +158,20 @@ class BugzillaProxy:
     def build_product_cache(self, pagure_projects):
         """ Cache the bugzilla info about each package in each product.
         """
+        def _query_component(query, num_attempts=5):
+            for i in range(num_attempts):
+                try:
+                    raw_data = self.server._proxy.Component.get({'names': query})
+                    break
+                except Exception as e:
+                    if i >= num_attempts - 1:
+                        raise
+                    if self.config['verbose']:
+                        print(f"    ERROR {e}")
+                        print("    - Query failed, going to try again in 20 seconds")
+                    # Wait 20 seconds and try again
+                    time.sleep(20)
+            return raw_data
 
         if self.config['bugzilla']['compat_api'] == 'getcomponentsdetails':
             # Old API -- in python-bugzilla.  But with current server, this
@@ -167,6 +182,7 @@ class BugzillaProxy:
         elif self.config['bugzilla']['compat_api'] == 'component.get':
             # Way that's undocumented in the partner-bugzilla api but works
             # currently
+            chunk = self.config['bugzilla']['req_segment']
             for collection, product in self.config["products"].items():
                 bz_product_name = product.get('bz_product_name', collection)
                 # restrict the list of info returned to only the packages of
@@ -177,7 +193,8 @@ class BugzillaProxy:
                     if bz_product_name in project["products"]
                 ]
                 product_info_by_pkg = {}
-                for pkg_segment in segment(pkglist, self.config['bugzilla']['req_segment']):
+                estimated_size = math.ceil(len(pkglist) / chunk)
+                for cnt, pkg_segment in enumerate(segment(pkglist, chunk), start=1):
                     # Format that bugzilla will understand.  Strip None's that
                     # segment() pads out the final data segment() with
                     query = [
@@ -185,7 +202,11 @@ class BugzillaProxy:
                         for p in pkg_segment
                         if p is not None
                     ]
-                    raw_data = self.server._proxy.Component.get({'names': query})
+                    if self.config['verbose']:
+                        print(
+                            f"  - Querying product `{bz_product_name}`, "
+                            f"query {cnt} of {estimated_size}")
+                    raw_data = _query_component(query)
                     for package in raw_data['components']:
                         # Reformat data to be the same as what's returned from
                         # getcomponentsdetails