1 files changed, 26 insertions, 6 deletions
diff --git a/emacs/.emacs.d/lisp/my/my-net.el b/emacs/.emacs.d/lisp/my/my-net.el
index 1f1cbc6..b19ce68 100644
--- a/emacs/.emacs.d/lisp/my/my-net.el
+++ b/emacs/.emacs.d/lisp/my/my-net.el
@@ -29,12 +29,24 @@
 
 ;;; net utilities
 (defvar my-download-dir "~/Downloads")
+(defvar my-webpage-download-dir "~/Downloads")
 
-(defun my-make-file-name-from-url (url)
-  (file-name-nondirectory
-   (directory-file-name
-    (car (url-path-and-query (url-generic-parse-url
-                              (url-unhex-string url)))))))
+(defmacro my-url-as-googlebot (&rest body)
+  "Run BODY while spoofing as googlebot"
+  `(let ((url-request-extra-headers '(("X-Forwarded-For" . "66.249.66.1")))
+         (url-user-agent
+          "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"))
+     ,@body))
+
+(def-edebug-spec my-url-as-googlebot t)
+
+(defun my-make-file-name-from-url (url &optional extension)
+  (format "%s%s"
+          (file-name-nondirectory
+           (directory-file-name
+            (car (url-path-and-query (url-generic-parse-url
+                                      (url-unhex-string url))))))
+          (if extension (concat "." extension) "")))
 
 ;; stolen from `eww-make-unique-file-name'
 (defun my-make-unique-file-name (file directory)
@@ -119,6 +131,14 @@ It checks the STATUS, and if it is ok, saves the payload to FILE-NAME."
    decompression
    with-header))
 
+
+(defun my-url-fetch-raw (url &optional decompression with-header)
+  (my-url-fetch-internal
+   url
+   (lambda () (decode-coding-string (buffer-string) 'utf-8))
+   decompression
+   with-header))
+
 (defun my-url-fetch-internal (url buffer-processor decompression with-header)
   (with-current-buffer (get-buffer-create my-client-buffer-name)
     (goto-char (point-max))
@@ -141,7 +161,7 @@ It checks the STATUS, and if it is ok, saves the payload to FILE-NAME."
                 (list
                  (cons 'header fields)
                  (cons 'json (funcall buffer-processor)))
-              (funcall buffer-processor)))
+              (when buffer-processor (funcall buffer-processor))))
         (error "HTTP error: %s" (buffer-substring (point) (point-max)))))))
 
 (provide 'my-net)