From d06afce72182573c7ec1834c866fc39213c151cc Mon Sep 17 00:00:00 2001 From: Artur Malabarba Date: Sat, 6 Dec 2014 18:04:56 +0000 Subject: Improve some header comments. --- sx-request.el | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'sx-request.el') diff --git a/sx-request.el b/sx-request.el index 0994fbd..a17a982 100644 --- a/sx-request.el +++ b/sx-request.el @@ -1,4 +1,4 @@ -;;; sx-request.el --- requests and url manipulation -*- lexical-binding: t; -*- +;;; sx-request.el --- Requests and url manipulation. -*- lexical-binding: t; -*- ;; Copyright (C) 2014 Sean Allred -- cgit v1.2.3 From ceaee4e85d44caad2db8adacdff2e2772bf254c0 Mon Sep 17 00:00:00 2001 From: Artur Malabarba Date: Mon, 15 Dec 2014 18:18:08 -0200 Subject: _ indicates a variable is ignored. --- sx-request.el | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'sx-request.el') diff --git a/sx-request.el b/sx-request.el index a17a982..2d894f0 100644 --- a/sx-request.el +++ b/sx-request.el @@ -156,7 +156,7 @@ the main content of the response is returned." sx-request-remaining-api-requests)) (sx-encoding-clean-content-deep .items))))))) -(defun sx-request-fallback (method &optional args request-method) +(defun sx-request-fallback (_method &optional _args _request-method) "Fallback method when authentication is not available. This is for UI generation when the associated API call would require authentication. -- cgit v1.2.3 From 5ee595c3740c2b24274c9405e4bed4a5ae81953f Mon Sep 17 00:00:00 2001 From: Artur Malabarba Date: Sat, 20 Dec 2014 15:39:58 -0200 Subject: Indentation was off by one --- sx-request.el | 68 +++++++++++++++++++++++++++++------------------------------ 1 file changed, 34 insertions(+), 34 deletions(-) (limited to 'sx-request.el') diff --git a/sx-request.el b/sx-request.el index 2d894f0..389a471 100644 --- a/sx-request.el +++ b/sx-request.el @@ -121,40 +121,40 @@ the main content of the response is returned." (url-request-extra-headers '(("Content-Type" . "application/x-www-form-urlencoded"))) (response-buffer (url-retrieve-synchronously request-url))) - (if (not response-buffer) - (error "Something went wrong in `url-retrieve-synchronously'") - (with-current-buffer response-buffer - (let* ((data (progn - ;; @TODO use url-http-end-of-headers - (goto-char (point-min)) - (if (not (search-forward "\n\n" nil t)) - (error "Headers missing; response corrupt") - (delete-region (point-min) (point)) - (buffer-string)))) - (response-zipped-p (sx-encoding-gzipped-p data)) - (data (if (not response-zipped-p) data - (shell-command-on-region - (point-min) (point-max) - sx-request-unzip-program - nil t) - (buffer-string))) - ;; @TODO should use `condition-case' here -- set - ;; RESPONSE to 'corrupt or something - (response (with-demoted-errors "`json' error: %S" - (json-read-from-string data)))) - (when (and (not response) (string-equal data "{}")) - (sx-message "Unable to parse response: %S" response) - (error "Response could not be read by `json-read-from-string'")) - ;; If we get here, the response is a valid data structure - (sx-assoc-let response - (when .error_id - (error "Request failed: (%s) [%i %s] %S" - .method .error_id .error_name .error_message)) - (when (< (setq sx-request-remaining-api-requests .quota_remaining) - sx-request-remaining-api-requests-message-threshold) - (sx-message "%d API requests reamining" - sx-request-remaining-api-requests)) - (sx-encoding-clean-content-deep .items))))))) + (if (not response-buffer) + (error "Something went wrong in `url-retrieve-synchronously'") + (with-current-buffer response-buffer + (let* ((data (progn + ;; @TODO use url-http-end-of-headers + (goto-char (point-min)) + (if (not (search-forward "\n\n" nil t)) + (error "Headers missing; response corrupt") + (delete-region (point-min) (point)) + (buffer-string)))) + (response-zipped-p (sx-encoding-gzipped-p data)) + (data (if (not response-zipped-p) data + (shell-command-on-region + (point-min) (point-max) + sx-request-unzip-program + nil t) + (buffer-string))) + ;; @TODO should use `condition-case' here -- set + ;; RESPONSE to 'corrupt or something + (response (with-demoted-errors "`json' error: %S" + (json-read-from-string data)))) + (when (and (not response) (string-equal data "{}")) + (sx-message "Unable to parse response: %S" response) + (error "Response could not be read by `json-read-from-string'")) + ;; If we get here, the response is a valid data structure + (sx-assoc-let response + (when .error_id + (error "Request failed: (%s) [%i %s] %S" + .method .error_id .error_name .error_message)) + (when (< (setq sx-request-remaining-api-requests .quota_remaining) + sx-request-remaining-api-requests-message-threshold) + (sx-message "%d API requests reamining" + sx-request-remaining-api-requests)) + (sx-encoding-clean-content-deep .items))))))) (defun sx-request-fallback (_method &optional _args _request-method) "Fallback method when authentication is not available. -- cgit v1.2.3 From 0c380f42cf56e64d3b4446b6168fa8028760d8c5 Mon Sep 17 00:00:00 2001 From: Artur Malabarba Date: Sat, 20 Dec 2014 15:47:56 -0200 Subject: Prefer zlib-decompress-region when available. Fix #157 --- sx-request.el | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'sx-request.el') diff --git a/sx-request.el b/sx-request.el index 389a471..9f2ecdb 100644 --- a/sx-request.el +++ b/sx-request.el @@ -70,7 +70,11 @@ (defcustom sx-request-unzip-program "gunzip" "Program used to unzip the response if it is compressed. -This program must accept compressed data on standard input." +This program must accept compressed data on standard input. + +This is only used (and necessary) if the function +`zlib-decompress-region' is not defined, which is the case for +Emacs versions < 24.4." :group 'sx :type 'string) @@ -133,10 +137,11 @@ the main content of the response is returned." (buffer-string)))) (response-zipped-p (sx-encoding-gzipped-p data)) (data (if (not response-zipped-p) data - (shell-command-on-region - (point-min) (point-max) - sx-request-unzip-program - nil t) + (if (fboundp 'zlib-decompress-region) + (zlib-decompress-region (point-min) (point-max)) + (shell-command-on-region + (point-min) (point-max) + sx-request-unzip-program nil t)) (buffer-string))) ;; @TODO should use `condition-case' here -- set ;; RESPONSE to 'corrupt or something -- cgit v1.2.3 From a56f77ba8d990e0bf0333b5b93b9cdd1bf55b33f Mon Sep 17 00:00:00 2001 From: Artur Malabarba Date: Sat, 20 Dec 2014 16:10:01 -0200 Subject: Handle utf-8 encoding the right way. --- sx-question-print.el | 3 --- sx-request.el | 19 ++++++++++++------- 2 files changed, 12 insertions(+), 10 deletions(-) (limited to 'sx-request.el') diff --git a/sx-question-print.el b/sx-question-print.el index 2f07132..b58a84a 100644 --- a/sx-question-print.el +++ b/sx-question-print.el @@ -182,9 +182,6 @@ QUESTION must be a data structure returned by `json-read'." (mapc #'sx-question-mode--print-section .answers)) (insert "\n\n ") (insert-text-button "Write an Answer" :type 'sx-button-answer) - ;; Display weird chars correctly - (set-buffer-multibyte nil) - (set-buffer-multibyte t) ;; Go up (goto-char (point-min)) (sx-question-mode-next-section)) diff --git a/sx-request.el b/sx-request.el index 9f2ecdb..1031ea7 100644 --- a/sx-request.el +++ b/sx-request.el @@ -136,13 +136,18 @@ the main content of the response is returned." (delete-region (point-min) (point)) (buffer-string)))) (response-zipped-p (sx-encoding-gzipped-p data)) - (data (if (not response-zipped-p) data - (if (fboundp 'zlib-decompress-region) - (zlib-decompress-region (point-min) (point-max)) - (shell-command-on-region - (point-min) (point-max) - sx-request-unzip-program nil t)) - (buffer-string))) + (data + ;; Turn string of bytes into string of characters. See + ;; http://emacs.stackexchange.com/q/4100/50 + (decode-coding-string + (if (not response-zipped-p) data + (if (fboundp 'zlib-decompress-region) + (zlib-decompress-region (point-min) (point-max)) + (shell-command-on-region + (point-min) (point-max) + sx-request-unzip-program nil t)) + (buffer-string)) + 'utf-8 'nocopy)) ;; @TODO should use `condition-case' here -- set ;; RESPONSE to 'corrupt or something (response (with-demoted-errors "`json' error: %S" -- cgit v1.2.3 From b8eb3d978109c1d5bf18be8cc1e1678afb6c017a Mon Sep 17 00:00:00 2001 From: Sean Allred Date: Tue, 30 Dec 2014 23:10:01 -0500 Subject: Fix typo --- sx-request.el | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'sx-request.el') diff --git a/sx-request.el b/sx-request.el index 1031ea7..bc34f9c 100644 --- a/sx-request.el +++ b/sx-request.el @@ -162,7 +162,7 @@ the main content of the response is returned." .method .error_id .error_name .error_message)) (when (< (setq sx-request-remaining-api-requests .quota_remaining) sx-request-remaining-api-requests-message-threshold) - (sx-message "%d API requests reamining" + (sx-message "%d API requests remaining" sx-request-remaining-api-requests)) (sx-encoding-clean-content-deep .items))))))) -- cgit v1.2.3 From 5babd59dfd51b5dd33cfd411fc2c2754adf63381 Mon Sep 17 00:00:00 2001 From: Sean Allred Date: Fri, 2 Jan 2015 00:15:09 -0500 Subject: Add process-function to sx-request-make --- sx-request.el | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'sx-request.el') diff --git a/sx-request.el b/sx-request.el index bc34f9c..f1d20af 100644 --- a/sx-request.el +++ b/sx-request.el @@ -95,13 +95,13 @@ number of requests left every time it finishes a call." ;;; Making Requests -(defun sx-request-make (method &optional args request-method) +(defun sx-request-make (method &optional args request-method process-function) "Make a request to the API, executing METHOD with ARGS. You should almost certainly be using `sx-method-call' instead of this function. REQUEST-METHOD is one of `GET' (default) or `POST'. -Returns cleaned response content. -See (`sx-encoding-clean-content-deep'). +Returns the entire response as processed by PROCESS-FUNCTION. +This defaults to `sx-request-response-get-items'. The full set of arguments is built with `sx-request--build-keyword-arguments', prepending @@ -164,7 +164,8 @@ the main content of the response is returned." sx-request-remaining-api-requests-message-threshold) (sx-message "%d API requests remaining" sx-request-remaining-api-requests)) - (sx-encoding-clean-content-deep .items))))))) + (funcall (or process-function #'sx-request-response-get-items) + response))))))) (defun sx-request-fallback (_method &optional _args _request-method) "Fallback method when authentication is not available. @@ -205,6 +206,13 @@ false, use the symbol `false'. Each element is processed with alist)) "&"))) + +;;; Response Processors +(defun sx-request-response-get-items (response) + "Returns the items from RESPONSE." + (sx-assoc-let response + (sx-encoding-clean-content-deep .items))) + (provide 'sx-request) ;;; sx-request.el ends here -- cgit v1.2.3 From 27eb38cfc4bba9013e8454bbe81ce497bf224474 Mon Sep 17 00:00:00 2001 From: Sean Allred Date: Fri, 2 Jan 2015 00:59:59 -0500 Subject: Introduce `sx-request-all-items' This function repeatedly makes API requests until a condition is satisfied (such as 'no more items'). First and foremost, this will allow us to retrieve all tags for a site. --- sx-request.el | 32 ++++++++++++++++++++++++++++++++ test/test-api.el | 5 +++++ 2 files changed, 37 insertions(+) (limited to 'sx-request.el') diff --git a/sx-request.el b/sx-request.el index f1d20af..387bde5 100644 --- a/sx-request.el +++ b/sx-request.el @@ -94,6 +94,35 @@ number of requests left every time it finishes a call." ;;; Making Requests +(defun sx-request-all-items (method &optional args request-method + stop-when process-function) + "Call METHOD with ARGS until there are no more items. +STOP-WHEN is a function that takes the entire response and +returns non-nil if the process should stop. + +All other arguments are identical to `sx-request-make', but +PROCESS-FUNCTION is given the default value of `identity' (rather +than `sx-request-response-get-items') to allow STOP-WHEN to +access the response wrapper." + ;; @TODO: Refactor. This is the product of a late-night jam + ;; session... it is not intended to be model code. + (let* ((return-value []) + (current-page 1) + (stop-when (or stop-when #'sx-request-all-stop-when-no-more)) + (process-function (or process-function #'identity)) + (response + (sx-request-make method `((page . ,current-page) ,@args) + request-method process-function))) + (while (not (funcall stop-when response)) + (setq return-value + (vconcat return-value + (cdr (assoc 'items response)))) + (setq current-page (1+ current-page) + response + (sx-request-make method `((page . ,current-page) ,@args) + request-method process-function))) + (vconcat return-value + (cdr (assoc 'items response))))) (defun sx-request-make (method &optional args request-method process-function) "Make a request to the API, executing METHOD with ARGS. @@ -213,6 +242,9 @@ false, use the symbol `false'. Each element is processed with (sx-assoc-let response (sx-encoding-clean-content-deep .items))) +(defun sx-request-all-stop-when-no-more (response) + (or (not response) + (equal :json-false (cdr (assoc 'has_more response))))) (provide 'sx-request) ;;; sx-request.el ends here diff --git a/test/test-api.el b/test/test-api.el index ca775ff..b99ec7a 100644 --- a/test/test-api.el +++ b/test/test-api.el @@ -11,3 +11,8 @@ (should-error (sx-request-make "questions" '(())))) +(ert-deftest test-request-all () + "Test request all items" + (should + (< 250 + (length (sx-request-all-items "sites"))))) -- cgit v1.2.3 From f3bc9b692da7305acec568122992ee62dca45496 Mon Sep 17 00:00:00 2001 From: Sean Allred Date: Fri, 2 Jan 2015 01:06:04 -0500 Subject: Consolidate state changes --- sx-request.el | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'sx-request.el') diff --git a/sx-request.el b/sx-request.el index 387bde5..f9b93ea 100644 --- a/sx-request.el +++ b/sx-request.el @@ -114,11 +114,11 @@ access the response wrapper." (sx-request-make method `((page . ,current-page) ,@args) request-method process-function))) (while (not (funcall stop-when response)) - (setq return-value + (setq current-page (1+ current-page) + return-value (vconcat return-value (cdr (assoc 'items response)))) - (setq current-page (1+ current-page) - response + (setq response (sx-request-make method `((page . ,current-page) ,@args) request-method process-function))) (vconcat return-value -- cgit v1.2.3 From caad878334e548de2e5157f692746c21dee3ff0d Mon Sep 17 00:00:00 2001 From: Sean Allred Date: Fri, 2 Jan 2015 01:06:23 -0500 Subject: Introduce anti-throttling delay --- sx-request.el | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'sx-request.el') diff --git a/sx-request.el b/sx-request.el index f9b93ea..ba5f6f7 100644 --- a/sx-request.el +++ b/sx-request.el @@ -95,10 +95,12 @@ number of requests left every time it finishes a call." ;;; Making Requests (defun sx-request-all-items (method &optional args request-method - stop-when process-function) + process-function stop-when delay) "Call METHOD with ARGS until there are no more items. STOP-WHEN is a function that takes the entire response and -returns non-nil if the process should stop. +returns non-nil if the process should stop. DELAY is the number +of seconds (possibly a float value) to wait after each request is +made (to avoid throttling). The default value is 0.25. All other arguments are identical to `sx-request-make', but PROCESS-FUNCTION is given the default value of `identity' (rather @@ -118,6 +120,7 @@ access the response wrapper." return-value (vconcat return-value (cdr (assoc 'items response)))) + (sleep-for (or delay 0.25)) (setq response (sx-request-make method `((page . ,current-page) ,@args) request-method process-function))) -- cgit v1.2.3 From 466fab1790145808e00a1b16d180c0b8cfd8ee99 Mon Sep 17 00:00:00 2001 From: Sean Allred Date: Fri, 2 Jan 2015 01:28:52 -0500 Subject: Declare indentation patterns for request functions --- sx-request.el | 2 ++ 1 file changed, 2 insertions(+) (limited to 'sx-request.el') diff --git a/sx-request.el b/sx-request.el index ba5f6f7..8f5056f 100644 --- a/sx-request.el +++ b/sx-request.el @@ -108,6 +108,7 @@ than `sx-request-response-get-items') to allow STOP-WHEN to access the response wrapper." ;; @TODO: Refactor. This is the product of a late-night jam ;; session... it is not intended to be model code. + (declare (indent 1)) (let* ((return-value []) (current-page 1) (stop-when (or stop-when #'sx-request-all-stop-when-no-more)) @@ -149,6 +150,7 @@ then read with `json-read-from-string'. `sx-request-remaining-api-requests' is updated appropriately and the main content of the response is returned." + (declare (indent 1)) (let* ((url-automatic-caching t) (url-inhibit-uncompression t) (url-request-data (sx-request--build-keyword-arguments args nil)) -- cgit v1.2.3 From dd9c2017cb3f0b145b39150562a0c4e59c244df1 Mon Sep 17 00:00:00 2001 From: Sean Allred Date: Fri, 2 Jan 2015 01:35:12 -0500 Subject: Use variable instead of default for request-delay --- sx-request.el | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'sx-request.el') diff --git a/sx-request.el b/sx-request.el index 8f5056f..00b90be 100644 --- a/sx-request.el +++ b/sx-request.el @@ -92,15 +92,18 @@ number of requests left every time it finishes a call." :group 'sx :type 'integer) +(defvar sx-request-all-items-delay + 1 + "Delay in seconds with each `sx-request-all-items' iteration. +It is good to use a reasonable delay to avoid rate-limiting.") + ;;; Making Requests (defun sx-request-all-items (method &optional args request-method - process-function stop-when delay) + process-function stop-when) "Call METHOD with ARGS until there are no more items. STOP-WHEN is a function that takes the entire response and -returns non-nil if the process should stop. DELAY is the number -of seconds (possibly a float value) to wait after each request is -made (to avoid throttling). The default value is 0.25. +returns non-nil if the process should stop. All other arguments are identical to `sx-request-make', but PROCESS-FUNCTION is given the default value of `identity' (rather @@ -121,7 +124,7 @@ access the response wrapper." return-value (vconcat return-value (cdr (assoc 'items response)))) - (sleep-for (or delay 0.25)) + (sleep-for sx-request-all-items-delay) (setq response (sx-request-make method `((page . ,current-page) ,@args) request-method process-function))) -- cgit v1.2.3