diff options
| -rw-r--r-- | sx-method.el | 30 | ||||
| -rw-r--r-- | sx-request.el | 56 | ||||
| -rw-r--r-- | test/test-api.el | 9 |
3 files changed, 89 insertions, 6 deletions
diff --git a/sx-method.el b/sx-method.el index 1078014..8fa30c9 100644 --- a/sx-method.el +++ b/sx-method.el @@ -38,6 +38,9 @@ (filter '(())) auth (url-method "GET") + get-all + (process-function + #'sx-request-response-get-items) site) "Call METHOD with additional keys. @@ -50,6 +53,8 @@ user. authentication. :URL-METHOD is either \"POST\" or \"GET\" :SITE is the api parameter specifying the site. +:GET-ALL is nil or non-nil +:PROCESS-FUNCTION is a response-processing function When AUTH is nil, it is assumed that no auth-requiring filters or methods will be used. If they are an error will be signaled. This is @@ -66,6 +71,18 @@ for interactive commands that absolutely require authentication \(submitting questions/answers, reading inbox, etc). Filters will treat 'warn as equivalent to t. +If GET-ALL is nil, this method will only return the first (or +specified) page available from this method call. If t, all pages +will be retrieved (`sx-request-all-stop-when-no-more') . +Otherwise, it is a function STOP-WHEN for `sx-request-all-items'. + +If PROCESS-FUNCTION is nil, only the items of the response will +be returned (`sx-request-response-get-items'). Otherwise, it is +a function that processes the entire response (as returned by +`json-read'). + +See `sx-request-make' and `sx-request-all-items'. + Return the entire response as a complex alist." (declare (indent 1)) (let ((access-token (sx-cache-get 'auth)) @@ -82,7 +99,12 @@ Return the entire response as a complex alist." (prog1 (format "?site=%s" site) (setq site nil))))) - (call #'sx-request-make) + (call (if get-all #'sx-request-all-items #'sx-request-make)) + (get-all-stop-when + (cond + ((eq get-all t) #'sx-request-all-stop-when-no-more) + (get-all get-all) + (t nil))) parameters) (lwarn "sx-call-method" :debug "A: %S T: %S. M: %S,%s. F: %S" (equal 'warn auth) access-token method-auth full-method filter-auth) @@ -106,11 +128,15 @@ Return the entire response as a complex alist." (cons (cons 'filter (sx-filter-get-var filter)) keywords)) (when site + ;; @TODO: Maybe use `push' instead? (setq parameters (cons (cons 'site site) parameters))) (funcall call full-method parameters - url-method))) + url-method + (if get-all + get-all-stop-when + process-function)))) (provide 'sx-method) ;;; sx-method.el ends here diff --git a/sx-request.el b/sx-request.el index bc34f9c..da0a1fb 100644 --- a/sx-request.el +++ b/sx-request.el @@ -92,16 +92,52 @@ number of requests left every time it finishes a call." :group 'sx :type 'integer) +(defvar sx-request-all-items-delay + 1 + "Delay in seconds with each `sx-request-all-items' iteration. +It is good to use a reasonable delay to avoid rate-limiting.") + ;;; Making Requests +(defun sx-request-all-items (method &optional args request-method + stop-when) + "Call METHOD with ARGS until there are no more items. +STOP-WHEN is a function that takes the entire response and +returns non-nil if the process should stop. + +All other arguments are identical to `sx-request-make', but +PROCESS-FUNCTION is given the default value of `identity' (rather +than `sx-request-response-get-items') to allow STOP-WHEN to +access the response wrapper." + ;; @TODO: Refactor. This is the product of a late-night jam + ;; session... it is not intended to be model code. + (declare (indent 1)) + (let* ((return-value []) + (current-page 1) + (stop-when (or stop-when #'sx-request-all-stop-when-no-more)) + (process-function #'identity) + (response + (sx-request-make method `((page . ,current-page) ,@args) + request-method process-function))) + (while (not (funcall stop-when response)) + (setq current-page (1+ current-page) + return-value + (vconcat return-value + (cdr (assoc 'items response)))) + (sleep-for sx-request-all-items-delay) + (setq response + (sx-request-make method `((page . ,current-page) ,@args) + request-method process-function))) + (vconcat return-value + (cdr (assoc 'items response))))) -(defun sx-request-make (method &optional args request-method) +(defun sx-request-make (method &optional args request-method process-function) "Make a request to the API, executing METHOD with ARGS. You should almost certainly be using `sx-method-call' instead of this function. REQUEST-METHOD is one of `GET' (default) or `POST'. -Returns cleaned response content. -See (`sx-encoding-clean-content-deep'). +Returns the entire response as processed by PROCESS-FUNCTION. +This defaults to `sx-request-response-get-items'. The full set of arguments is built with `sx-request--build-keyword-arguments', prepending @@ -117,6 +153,7 @@ then read with `json-read-from-string'. `sx-request-remaining-api-requests' is updated appropriately and the main content of the response is returned." + (declare (indent 1)) (let* ((url-automatic-caching t) (url-inhibit-uncompression t) (url-request-data (sx-request--build-keyword-arguments args nil)) @@ -164,7 +201,8 @@ the main content of the response is returned." sx-request-remaining-api-requests-message-threshold) (sx-message "%d API requests remaining" sx-request-remaining-api-requests)) - (sx-encoding-clean-content-deep .items))))))) + (funcall (or process-function #'sx-request-response-get-items) + response))))))) (defun sx-request-fallback (_method &optional _args _request-method) "Fallback method when authentication is not available. @@ -205,6 +243,16 @@ false, use the symbol `false'. Each element is processed with alist)) "&"))) + +;;; Response Processors +(defun sx-request-response-get-items (response) + "Returns the items from RESPONSE." + (sx-assoc-let response + (sx-encoding-clean-content-deep .items))) + +(defun sx-request-all-stop-when-no-more (response) + (or (not response) + (equal :json-false (cdr (assoc 'has_more response))))) (provide 'sx-request) ;;; sx-request.el ends here diff --git a/test/test-api.el b/test/test-api.el index ca775ff..91a8adb 100644 --- a/test/test-api.el +++ b/test/test-api.el @@ -11,3 +11,12 @@ (should-error (sx-request-make "questions" '(())))) +(ert-deftest test-request-all () + "Test request all items" + (should + (< 250 + (length (sx-request-all-items "sites"))))) + +(ert-deftest test-method-get-all () + "Tests sx-method interface to `sx-request-all-items'" + (should (< 250 (length (sx-method-call 'sites :get-all t))))) |