From 5babd59dfd51b5dd33cfd411fc2c2754adf63381 Mon Sep 17 00:00:00 2001 From: Sean Allred Date: Fri, 2 Jan 2015 00:15:09 -0500 Subject: Add process-function to sx-request-make --- sx-request.el | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'sx-request.el') diff --git a/sx-request.el b/sx-request.el index bc34f9c..f1d20af 100644 --- a/sx-request.el +++ b/sx-request.el @@ -95,13 +95,13 @@ number of requests left every time it finishes a call." ;;; Making Requests -(defun sx-request-make (method &optional args request-method) +(defun sx-request-make (method &optional args request-method process-function) "Make a request to the API, executing METHOD with ARGS. You should almost certainly be using `sx-method-call' instead of this function. REQUEST-METHOD is one of `GET' (default) or `POST'. -Returns cleaned response content. -See (`sx-encoding-clean-content-deep'). +Returns the entire response as processed by PROCESS-FUNCTION. +This defaults to `sx-request-response-get-items'. The full set of arguments is built with `sx-request--build-keyword-arguments', prepending @@ -164,7 +164,8 @@ the main content of the response is returned." sx-request-remaining-api-requests-message-threshold) (sx-message "%d API requests remaining" sx-request-remaining-api-requests)) - (sx-encoding-clean-content-deep .items))))))) + (funcall (or process-function #'sx-request-response-get-items) + response))))))) (defun sx-request-fallback (_method &optional _args _request-method) "Fallback method when authentication is not available. @@ -205,6 +206,13 @@ false, use the symbol `false'. Each element is processed with alist)) "&"))) + +;;; Response Processors +(defun sx-request-response-get-items (response) + "Returns the items from RESPONSE." + (sx-assoc-let response + (sx-encoding-clean-content-deep .items))) + (provide 'sx-request) ;;; sx-request.el ends here -- cgit v1.2.3 From 27eb38cfc4bba9013e8454bbe81ce497bf224474 Mon Sep 17 00:00:00 2001 From: Sean Allred Date: Fri, 2 Jan 2015 00:59:59 -0500 Subject: Introduce `sx-request-all-items' This function repeatedly makes API requests until a condition is satisfied (such as 'no more items'). First and foremost, this will allow us to retrieve all tags for a site. --- sx-request.el | 32 ++++++++++++++++++++++++++++++++ test/test-api.el | 5 +++++ 2 files changed, 37 insertions(+) (limited to 'sx-request.el') diff --git a/sx-request.el b/sx-request.el index f1d20af..387bde5 100644 --- a/sx-request.el +++ b/sx-request.el @@ -94,6 +94,35 @@ number of requests left every time it finishes a call." ;;; Making Requests +(defun sx-request-all-items (method &optional args request-method + stop-when process-function) + "Call METHOD with ARGS until there are no more items. +STOP-WHEN is a function that takes the entire response and +returns non-nil if the process should stop. + +All other arguments are identical to `sx-request-make', but +PROCESS-FUNCTION is given the default value of `identity' (rather +than `sx-request-response-get-items') to allow STOP-WHEN to +access the response wrapper." + ;; @TODO: Refactor. This is the product of a late-night jam + ;; session... it is not intended to be model code. + (let* ((return-value []) + (current-page 1) + (stop-when (or stop-when #'sx-request-all-stop-when-no-more)) + (process-function (or process-function #'identity)) + (response + (sx-request-make method `((page . ,current-page) ,@args) + request-method process-function))) + (while (not (funcall stop-when response)) + (setq return-value + (vconcat return-value + (cdr (assoc 'items response)))) + (setq current-page (1+ current-page) + response + (sx-request-make method `((page . ,current-page) ,@args) + request-method process-function))) + (vconcat return-value + (cdr (assoc 'items response))))) (defun sx-request-make (method &optional args request-method process-function) "Make a request to the API, executing METHOD with ARGS. @@ -213,6 +242,9 @@ false, use the symbol `false'. Each element is processed with (sx-assoc-let response (sx-encoding-clean-content-deep .items))) +(defun sx-request-all-stop-when-no-more (response) + (or (not response) + (equal :json-false (cdr (assoc 'has_more response))))) (provide 'sx-request) ;;; sx-request.el ends here diff --git a/test/test-api.el b/test/test-api.el index ca775ff..b99ec7a 100644 --- a/test/test-api.el +++ b/test/test-api.el @@ -11,3 +11,8 @@ (should-error (sx-request-make "questions" '(())))) +(ert-deftest test-request-all () + "Test request all items" + (should + (< 250 + (length (sx-request-all-items "sites"))))) -- cgit v1.2.3 From f3bc9b692da7305acec568122992ee62dca45496 Mon Sep 17 00:00:00 2001 From: Sean Allred Date: Fri, 2 Jan 2015 01:06:04 -0500 Subject: Consolidate state changes --- sx-request.el | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'sx-request.el') diff --git a/sx-request.el b/sx-request.el index 387bde5..f9b93ea 100644 --- a/sx-request.el +++ b/sx-request.el @@ -114,11 +114,11 @@ access the response wrapper." (sx-request-make method `((page . ,current-page) ,@args) request-method process-function))) (while (not (funcall stop-when response)) - (setq return-value + (setq current-page (1+ current-page) + return-value (vconcat return-value (cdr (assoc 'items response)))) - (setq current-page (1+ current-page) - response + (setq response (sx-request-make method `((page . ,current-page) ,@args) request-method process-function))) (vconcat return-value -- cgit v1.2.3 From caad878334e548de2e5157f692746c21dee3ff0d Mon Sep 17 00:00:00 2001 From: Sean Allred Date: Fri, 2 Jan 2015 01:06:23 -0500 Subject: Introduce anti-throttling delay --- sx-request.el | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'sx-request.el') diff --git a/sx-request.el b/sx-request.el index f9b93ea..ba5f6f7 100644 --- a/sx-request.el +++ b/sx-request.el @@ -95,10 +95,12 @@ number of requests left every time it finishes a call." ;;; Making Requests (defun sx-request-all-items (method &optional args request-method - stop-when process-function) + process-function stop-when delay) "Call METHOD with ARGS until there are no more items. STOP-WHEN is a function that takes the entire response and -returns non-nil if the process should stop. +returns non-nil if the process should stop. DELAY is the number +of seconds (possibly a float value) to wait after each request is +made (to avoid throttling). The default value is 0.25. All other arguments are identical to `sx-request-make', but PROCESS-FUNCTION is given the default value of `identity' (rather @@ -118,6 +120,7 @@ access the response wrapper." return-value (vconcat return-value (cdr (assoc 'items response)))) + (sleep-for (or delay 0.25)) (setq response (sx-request-make method `((page . ,current-page) ,@args) request-method process-function))) -- cgit v1.2.3 From 466fab1790145808e00a1b16d180c0b8cfd8ee99 Mon Sep 17 00:00:00 2001 From: Sean Allred Date: Fri, 2 Jan 2015 01:28:52 -0500 Subject: Declare indentation patterns for request functions --- sx-request.el | 2 ++ 1 file changed, 2 insertions(+) (limited to 'sx-request.el') diff --git a/sx-request.el b/sx-request.el index ba5f6f7..8f5056f 100644 --- a/sx-request.el +++ b/sx-request.el @@ -108,6 +108,7 @@ than `sx-request-response-get-items') to allow STOP-WHEN to access the response wrapper." ;; @TODO: Refactor. This is the product of a late-night jam ;; session... it is not intended to be model code. + (declare (indent 1)) (let* ((return-value []) (current-page 1) (stop-when (or stop-when #'sx-request-all-stop-when-no-more)) @@ -149,6 +150,7 @@ then read with `json-read-from-string'. `sx-request-remaining-api-requests' is updated appropriately and the main content of the response is returned." + (declare (indent 1)) (let* ((url-automatic-caching t) (url-inhibit-uncompression t) (url-request-data (sx-request--build-keyword-arguments args nil)) -- cgit v1.2.3 From dd9c2017cb3f0b145b39150562a0c4e59c244df1 Mon Sep 17 00:00:00 2001 From: Sean Allred Date: Fri, 2 Jan 2015 01:35:12 -0500 Subject: Use variable instead of default for request-delay --- sx-request.el | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'sx-request.el') diff --git a/sx-request.el b/sx-request.el index 8f5056f..00b90be 100644 --- a/sx-request.el +++ b/sx-request.el @@ -92,15 +92,18 @@ number of requests left every time it finishes a call." :group 'sx :type 'integer) +(defvar sx-request-all-items-delay + 1 + "Delay in seconds with each `sx-request-all-items' iteration. +It is good to use a reasonable delay to avoid rate-limiting.") + ;;; Making Requests (defun sx-request-all-items (method &optional args request-method - process-function stop-when delay) + process-function stop-when) "Call METHOD with ARGS until there are no more items. STOP-WHEN is a function that takes the entire response and -returns non-nil if the process should stop. DELAY is the number -of seconds (possibly a float value) to wait after each request is -made (to avoid throttling). The default value is 0.25. +returns non-nil if the process should stop. All other arguments are identical to `sx-request-make', but PROCESS-FUNCTION is given the default value of `identity' (rather @@ -121,7 +124,7 @@ access the response wrapper." return-value (vconcat return-value (cdr (assoc 'items response)))) - (sleep-for (or delay 0.25)) + (sleep-for sx-request-all-items-delay) (setq response (sx-request-make method `((page . ,current-page) ,@args) request-method process-function))) -- cgit v1.2.3 From 07b9d0e64d488670cdb6672103e1d044c065d5ba Mon Sep 17 00:00:00 2001 From: Sean Allred Date: Fri, 2 Jan 2015 19:28:25 -0500 Subject: Remove process-function from request-all-items It doesn't really make sense. --- sx-request.el | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'sx-request.el') diff --git a/sx-request.el b/sx-request.el index 00b90be..da0a1fb 100644 --- a/sx-request.el +++ b/sx-request.el @@ -100,7 +100,7 @@ It is good to use a reasonable delay to avoid rate-limiting.") ;;; Making Requests (defun sx-request-all-items (method &optional args request-method - process-function stop-when) + stop-when) "Call METHOD with ARGS until there are no more items. STOP-WHEN is a function that takes the entire response and returns non-nil if the process should stop. @@ -115,7 +115,7 @@ access the response wrapper." (let* ((return-value []) (current-page 1) (stop-when (or stop-when #'sx-request-all-stop-when-no-more)) - (process-function (or process-function #'identity)) + (process-function #'identity) (response (sx-request-make method `((page . ,current-page) ,@args) request-method process-function))) -- cgit v1.2.3