Merge pull request #187 from vermiculus/get-all

Get all items

3 files changed, 89 insertions, 6 deletions

diff --git a/sx-method.el b/sx-method.el
index 1078014..8fa30c9 100644
--- a/sx-method.el
+++ b/sx-method.el
@@ -38,6 +38,9 @@
                                       (filter '(()))
                                       auth
                                       (url-method "GET")
+                                      get-all
+                                      (process-function
+                                       #'sx-request-response-get-items)
                                       site)
   "Call METHOD with additional keys.
 
@@ -50,6 +53,8 @@ user.
 authentication.
 :URL-METHOD is either \"POST\" or \"GET\"
 :SITE is the api parameter specifying the site.
+:GET-ALL is nil or non-nil
+:PROCESS-FUNCTION is a response-processing function
 
 When AUTH is nil, it is assumed that no auth-requiring filters or
 methods will be used.  If they are an error will be signaled.  This is
@@ -66,6 +71,18 @@ for interactive commands that absolutely require authentication
 \(submitting questions/answers, reading inbox, etc).  Filters will
 treat 'warn as equivalent to t.
 
+If GET-ALL is nil, this method will only return the first (or
+specified) page available from this method call.  If t, all pages
+will be retrieved (`sx-request-all-stop-when-no-more') .
+Otherwise, it is a function STOP-WHEN for `sx-request-all-items'.
+
+If PROCESS-FUNCTION is nil, only the items of the response will
+be returned (`sx-request-response-get-items').  Otherwise, it is
+a function that processes the entire response (as returned by
+`json-read').
+
+See `sx-request-make' and `sx-request-all-items'.
+
 Return the entire response as a complex alist."
   (declare (indent 1))
   (let ((access-token (sx-cache-get 'auth))
@@ -82,7 +99,12 @@ Return the entire response as a complex alist."
                                (prog1
                                    (format "?site=%s" site)
                                  (setq site nil)))))
-        (call #'sx-request-make)
+        (call (if get-all #'sx-request-all-items #'sx-request-make))
+        (get-all-stop-when
+         (cond
+          ((eq get-all t) #'sx-request-all-stop-when-no-more)
+          (get-all get-all)
+          (t nil)))
         parameters)
     (lwarn "sx-call-method" :debug "A: %S T: %S. M: %S,%s. F: %S" (equal 'warn auth)
            access-token method-auth full-method filter-auth)
@@ -106,11 +128,15 @@ Return the entire response as a complex alist."
           (cons (cons 'filter (sx-filter-get-var filter))
                 keywords))
     (when site
+      ;; @TODO: Maybe use `push' instead?
       (setq parameters (cons (cons 'site site) parameters)))
     (funcall call
              full-method
              parameters
-             url-method)))
+             url-method
+             (if get-all
+                 get-all-stop-when
+               process-function))))
 
 (provide 'sx-method)
 ;;; sx-method.el ends here
diff --git a/sx-request.el b/sx-request.el
index bc34f9c..da0a1fb 100644
--- a/sx-request.el
+++ b/sx-request.el
@@ -92,16 +92,52 @@ number of requests left every time it finishes a call."
   :group 'sx
   :type 'integer)
 
+(defvar sx-request-all-items-delay
+  1
+  "Delay in seconds with each `sx-request-all-items' iteration.
+It is good to use a reasonable delay to avoid rate-limiting.")
+
 
 ;;; Making Requests
+(defun sx-request-all-items (method &optional args request-method
+                                    stop-when)
+  "Call METHOD with ARGS until there are no more items.
+STOP-WHEN is a function that takes the entire response and
+returns non-nil if the process should stop.
+
+All other arguments are identical to `sx-request-make', but
+PROCESS-FUNCTION is given the default value of `identity' (rather
+than `sx-request-response-get-items') to allow STOP-WHEN to
+access the response wrapper."
+  ;; @TODO: Refactor.  This is the product of a late-night jam
+  ;; session...  it is not intended to be model code.
+  (declare (indent 1))
+  (let* ((return-value [])
+         (current-page 1)
+         (stop-when (or stop-when #'sx-request-all-stop-when-no-more))
+         (process-function #'identity)
+         (response
+          (sx-request-make method `((page . ,current-page) ,@args)
+                           request-method process-function)))
+    (while (not (funcall stop-when response))
+      (setq current-page (1+ current-page)
+            return-value
+            (vconcat return-value
+                     (cdr (assoc 'items response))))
+      (sleep-for sx-request-all-items-delay)
+      (setq response
+            (sx-request-make method `((page . ,current-page) ,@args)
+                             request-method process-function)))
+    (vconcat return-value
+             (cdr (assoc 'items response)))))
 
-(defun sx-request-make (method &optional args request-method)
+(defun sx-request-make (method &optional args request-method process-function)
   "Make a request to the API, executing METHOD with ARGS.
 You should almost certainly be using `sx-method-call' instead of
 this function. REQUEST-METHOD is one of `GET' (default) or `POST'.
 
-Returns cleaned response content.
-See (`sx-encoding-clean-content-deep').
+Returns the entire response as processed by PROCESS-FUNCTION.
+This defaults to `sx-request-response-get-items'.
 
 The full set of arguments is built with
 `sx-request--build-keyword-arguments', prepending
@@ -117,6 +153,7 @@ then read with `json-read-from-string'.
 
 `sx-request-remaining-api-requests' is updated appropriately and
 the main content of the response is returned."
+  (declare (indent 1))
   (let* ((url-automatic-caching t)
          (url-inhibit-uncompression t)
          (url-request-data (sx-request--build-keyword-arguments args nil))
@@ -164,7 +201,8 @@ the main content of the response is returned."
                      sx-request-remaining-api-requests-message-threshold)
               (sx-message "%d API requests remaining"
                           sx-request-remaining-api-requests))
-            (sx-encoding-clean-content-deep .items)))))))
+            (funcall (or process-function #'sx-request-response-get-items)
+                       response)))))))
 
 (defun sx-request-fallback (_method &optional _args _request-method)
   "Fallback method when authentication is not available.
@@ -205,6 +243,16 @@ false, use the symbol `false'.  Each element is processed with
                 alist))
      "&")))
 
+
+;;; Response Processors
+(defun sx-request-response-get-items (response)
+  "Returns the items from RESPONSE."
+  (sx-assoc-let response
+    (sx-encoding-clean-content-deep .items)))
+
+(defun sx-request-all-stop-when-no-more (response)
+  (or (not response)
+      (equal :json-false (cdr (assoc 'has_more response)))))
 
 (provide 'sx-request)
 ;;; sx-request.el ends here
diff --git a/test/test-api.el b/test/test-api.el
index ca775ff..91a8adb 100644
--- a/test/test-api.el
+++ b/test/test-api.el
@@ -11,3 +11,12 @@
   (should-error
    (sx-request-make "questions" '(()))))
 
+(ert-deftest test-request-all ()
+  "Test request all items"
+  (should
+   (< 250
+      (length (sx-request-all-items "sites")))))
+
+(ert-deftest test-method-get-all ()
+  "Tests sx-method interface to `sx-request-all-items'"
+  (should (< 250 (length (sx-method-call 'sites :get-all t)))))