Some fixes for the bot and the backoff.

2 files changed, 19 insertions, 13 deletions

diff --git a/bot/sx-bot.el b/bot/sx-bot.el
index 9448a26..1a8bceb 100644
--- a/bot/sx-bot.el
+++ b/bot/sx-bot.el
@@ -60,20 +60,27 @@ File is savedd in `sx-bot-out-dir'."
   "Get a list of all tags of all sites and save to disk."
   (make-directory sx-bot-out-dir t)
   (let* ((url-show-status nil)
-         (site-tokens (sx-site-get-api-tokens))
+         (site-tokens (mapcar (lambda (site)  ;Fix bad sites
+                                (pcase site
+                                  ("metamunitybuilding" "meta.communitybuilding")
+                                  ((or "gamification" "meta.gamification") nil)
+                                  (_ site)))
+                              (sx-site-get-api-tokens)))
          (number-of-sites (length site-tokens))
          (current-site-number 0)
          (sx-request-remaining-api-requests-message-threshold 4000)
-         (sx-request-all-items-delay 1))
+         (sx-request-all-items-delay 0))
     (mapcar
      (lambda (site)
-       (message "[%d/%d] Working on %S"
-                (cl-incf current-site-number)
-                number-of-sites
-                site)
-       (sx-bot-write-to-file
-        (cons (concat site ".el")
-              (sx-tag--get-all site))))
+       (when site
+         (message "[%d/%d] Working on %S"
+                  (cl-incf current-site-number)
+                  number-of-sites
+                  site)
+         (with-demoted-errors (concat "Failed to get " site ": %S")
+           (sx-bot-write-to-file
+            (cons (concat site ".el")
+                  (sx-tag--get-all site))))))
      site-tokens)))
 
 
diff --git a/sx-request.el b/sx-request.el
index 7a7bbf4..10bec4a 100644
--- a/sx-request.el
+++ b/sx-request.el
@@ -92,8 +92,7 @@ number of requests left every time it finishes a call."
   :group 'sx
   :type 'integer)
 
-(defvar sx-request-all-items-delay
-  1
+(defvar sx-request-all-items-delay 0
   "Delay in seconds with each `sx-request-all-items' iteration.
 It is good to use a reasonable delay to avoid rate-limiting.")
 
@@ -110,9 +109,9 @@ It is good to use a reasonable delay to avoid rate-limiting.")
           ;; we've likely just looped around the "least significant"
           ;; bits of `current-time'.
           (setq sx--backoff-time time)
-        (message "Backoff detected, waiting %s seconds" (< time sx--backoff-time))
         (when (< time sx--backoff-time)
-          (sleep-for (+ 2 (- sx--backoff-time time))))))))
+          (message "Backoff detected, waiting %s seconds" (- sx--backoff-time time))
+          (sleep-for (+ 0.3 (- sx--backoff-time time))))))))
 
 (defun sx-request-all-items (method &optional args request-method
                                     stop-when)