From f97c84c2d4a0a8ef6e53333a0a5c528397ccaded Mon Sep 17 00:00:00 2001 From: Artur Malabarba Date: Mon, 8 Dec 2014 12:27:31 +0000 Subject: Move the script to a subdir,. So it's not caught by Melpa --- bot/sx-bot.el | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 bot/sx-bot.el (limited to 'bot/sx-bot.el') diff --git a/bot/sx-bot.el b/bot/sx-bot.el new file mode 100644 index 0000000..f7e0557 --- /dev/null +++ b/bot/sx-bot.el @@ -0,0 +1,54 @@ +;;; sx-bot.el --- Functions for viewing different tabs. -*- lexical-binding: t; -*- + +;; Copyright (C) 2014 Artur Malabarba + +;; Author: Artur Malabarba + +;; This program is free software; you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with this program. If not, see . + +;;; Commentary: + +;; + + +;;; Code: + +(require 'sx-site) +(require 'sx-tag) + +(defcustom sx-bot-out-dir "./data/tags/" + "Directory where output tag files are saved." + :type 'directory + :group 'sx) + + +;;; Printing +(defun sx-bot-write-to-file (data) + "Write (cdr DATA) to file named (car DATA). +File is savedd in `sx-bot-out-dir'." + (with-temp-file (expand-file-name (car data) sx-bot-out-dir) + (let (print-length) + (prin1 (cdr data) (current-buffer))))) + + +(defun sx-bot-fetch-and-write-tags () + "Get a list of all tags of all sites and save to disk." + (make-directory sx-bot-out-dir t) + (mapc #'sx-bot-write-to-file + ;; @TODO: Not yet implemented! + (mapcar #'sx-tag--get-all (sx-site-get-api-tokens)))) + +;;; Newest +(provide 'sx-bot) +;;; sx-bot.el ends here -- cgit v1.2.3 From 3ecbcf11346136134a977b0421488b001b479cc7 Mon Sep 17 00:00:00 2001 From: Sean Allred Date: Fri, 2 Jan 2015 19:55:51 -0500 Subject: Fix sx-bot-fetch-and-write-tags sx-bot-write-to-file expects a cons cell. --- bot/sx-bot.el | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'bot/sx-bot.el') diff --git a/bot/sx-bot.el b/bot/sx-bot.el index f7e0557..e80894f 100644 --- a/bot/sx-bot.el +++ b/bot/sx-bot.el @@ -46,8 +46,10 @@ File is savedd in `sx-bot-out-dir'." "Get a list of all tags of all sites and save to disk." (make-directory sx-bot-out-dir t) (mapc #'sx-bot-write-to-file - ;; @TODO: Not yet implemented! - (mapcar #'sx-tag--get-all (sx-site-get-api-tokens)))) + (mapcar + (lambda (site) (cons site (sx-tag--get-all site))) + (sx-site-get-api-tokens)))) + ;;; Newest (provide 'sx-bot) -- cgit v1.2.3 From 52a8682c762bdc5e2e8d3a9ccc2eb527556f58ad Mon Sep 17 00:00:00 2001 From: Sean Allred Date: Fri, 2 Jan 2015 22:15:44 -0500 Subject: Message actions --- bot/sx-bot.el | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'bot/sx-bot.el') diff --git a/bot/sx-bot.el b/bot/sx-bot.el index e80894f..be12089 100644 --- a/bot/sx-bot.el +++ b/bot/sx-bot.el @@ -37,9 +37,11 @@ (defun sx-bot-write-to-file (data) "Write (cdr DATA) to file named (car DATA). File is savedd in `sx-bot-out-dir'." - (with-temp-file (expand-file-name (car data) sx-bot-out-dir) - (let (print-length) - (prin1 (cdr data) (current-buffer))))) + (let ((file-name (expand-file-name (car data) sx-bot-out-dir))) + (message "Writing %S" file-name) + (with-temp-file file-name + (let (print-length) + (prin1 (cdr data) (current-buffer)))))) (defun sx-bot-fetch-and-write-tags () -- cgit v1.2.3 From f12ceed5050aebe3a73dd1d72c868d9da2a3ec66 Mon Sep 17 00:00:00 2001 From: Sean Allred Date: Fri, 2 Jan 2015 22:16:05 -0500 Subject: Fix get-all function for tag bot * write is done right after retrieval * progress updates after each site --- bot/sx-bot.el | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) (limited to 'bot/sx-bot.el') diff --git a/bot/sx-bot.el b/bot/sx-bot.el index be12089..8b17e68 100644 --- a/bot/sx-bot.el +++ b/bot/sx-bot.el @@ -47,10 +47,20 @@ File is savedd in `sx-bot-out-dir'." (defun sx-bot-fetch-and-write-tags () "Get a list of all tags of all sites and save to disk." (make-directory sx-bot-out-dir t) - (mapc #'sx-bot-write-to-file - (mapcar - (lambda (site) (cons site (sx-tag--get-all site))) - (sx-site-get-api-tokens)))) + (let* ((url-show-status nil) + (site-tokens (sx-site-get-api-tokens)) + (number-of-sites (length site-tokens)) + (current-site-number 0)) + (mapcar + (lambda (site) + (message "[%d/%d] Working on %S" + (cl-incf current-site-number) + number-of-sites + site) + (sx-bot-write-to-file + (cons (concat site ".el") + (sx-tag--get-all site)))) + site-tokens))) ;;; Newest -- cgit v1.2.3 From c55abef53e1876a47150acec30a387cb1797b775 Mon Sep 17 00:00:00 2001 From: Sean Allred Date: Fri, 2 Jan 2015 22:18:29 -0500 Subject: Initialization for tag bot --- bot/sx-bot.el | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'bot/sx-bot.el') diff --git a/bot/sx-bot.el b/bot/sx-bot.el index 8b17e68..a3acfe8 100644 --- a/bot/sx-bot.el +++ b/bot/sx-bot.el @@ -27,6 +27,11 @@ (require 'sx-site) (require 'sx-tag) +(setq sx-request-remaining-api-requests-message-threshold 50000) + +(require 'package) +(package-initialize) + (defcustom sx-bot-out-dir "./data/tags/" "Directory where output tag files are saved." :type 'directory -- cgit v1.2.3 From 1cc7f6ea074457bc0d6d1d917f082459710eb809 Mon Sep 17 00:00:00 2001 From: Sean Allred Date: Fri, 2 Jan 2015 22:36:39 -0500 Subject: Message after writing, not before --- bot/sx-bot.el | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'bot/sx-bot.el') diff --git a/bot/sx-bot.el b/bot/sx-bot.el index a3acfe8..2917a43 100644 --- a/bot/sx-bot.el +++ b/bot/sx-bot.el @@ -43,10 +43,10 @@ "Write (cdr DATA) to file named (car DATA). File is savedd in `sx-bot-out-dir'." (let ((file-name (expand-file-name (car data) sx-bot-out-dir))) - (message "Writing %S" file-name) (with-temp-file file-name (let (print-length) - (prin1 (cdr data) (current-buffer)))))) + (prin1 (cdr data) (current-buffer)))) + (message "Wrote %S" file-name))) (defun sx-bot-fetch-and-write-tags () -- cgit v1.2.3 From 8ef72fc0e00a397647a5240d8273e67fb58a278f Mon Sep 17 00:00:00 2001 From: Sean Allred Date: Sat, 3 Jan 2015 01:53:09 -0500 Subject: One tag per line This will make the diffs much shorter. --- bot/sx-bot.el | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'bot/sx-bot.el') diff --git a/bot/sx-bot.el b/bot/sx-bot.el index 2917a43..702d738 100644 --- a/bot/sx-bot.el +++ b/bot/sx-bot.el @@ -44,9 +44,15 @@ File is savedd in `sx-bot-out-dir'." (let ((file-name (expand-file-name (car data) sx-bot-out-dir))) (with-temp-file file-name - (let (print-length) - (prin1 (cdr data) (current-buffer)))) - (message "Wrote %S" file-name))) + (let* (print-length + (repr (prin1-to-string (cdr data)))) + (insert repr) + (insert "\n") + (goto-char (point-min)) + (while (search-forward "\" \"" nil t) + (replace-match "\"\n \"" nil t)))) + (message "Wrote %S" file-name) + file-name)) (defun sx-bot-fetch-and-write-tags () -- cgit v1.2.3 From 5cc2e6f9b6618af37d45395bb7b3aa9caba60005 Mon Sep 17 00:00:00 2001 From: Sean Allred Date: Sat, 3 Jan 2015 01:53:28 -0500 Subject: Increase tag retrieval speed --- bot/sx-bot.el | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'bot/sx-bot.el') diff --git a/bot/sx-bot.el b/bot/sx-bot.el index 702d738..c51822d 100644 --- a/bot/sx-bot.el +++ b/bot/sx-bot.el @@ -61,7 +61,8 @@ File is savedd in `sx-bot-out-dir'." (let* ((url-show-status nil) (site-tokens (sx-site-get-api-tokens)) (number-of-sites (length site-tokens)) - (current-site-number 0)) + (current-site-number 0) + (sx-request-all-items-delay 0.25)) (mapcar (lambda (site) (message "[%d/%d] Working on %S" -- cgit v1.2.3 From 09d018763d8b3cd89c56e1b50fbd8af88cd9addb Mon Sep 17 00:00:00 2001 From: Sean Allred Date: Sat, 3 Jan 2015 02:01:52 -0500 Subject: Sort tag list before writing This should ensure that diffs are as small as possible. --- bot/sx-bot.el | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'bot/sx-bot.el') diff --git a/bot/sx-bot.el b/bot/sx-bot.el index c51822d..8ec8f9c 100644 --- a/bot/sx-bot.el +++ b/bot/sx-bot.el @@ -45,7 +45,9 @@ File is savedd in `sx-bot-out-dir'." (let ((file-name (expand-file-name (car data) sx-bot-out-dir))) (with-temp-file file-name (let* (print-length - (repr (prin1-to-string (cdr data)))) + (repr (prin1-to-string + (sort (cdr data) + #'string-lessp)))) (insert repr) (insert "\n") (goto-char (point-min)) -- cgit v1.2.3 From fe43a453a93f3cdd4d5504fe55e61629cfa098e5 Mon Sep 17 00:00:00 2001 From: Sean Allred Date: Sat, 3 Jan 2015 22:29:40 -0500 Subject: Fix title commentary --- bot/sx-bot.el | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'bot/sx-bot.el') diff --git a/bot/sx-bot.el b/bot/sx-bot.el index 8ec8f9c..fe93beb 100644 --- a/bot/sx-bot.el +++ b/bot/sx-bot.el @@ -1,4 +1,4 @@ -;;; sx-bot.el --- Functions for viewing different tabs. -*- lexical-binding: t; -*- +;;; sx-bot.el --- Functions for automated maintanence -*- lexical-binding: t; -*- ;; Copyright (C) 2014 Artur Malabarba @@ -19,7 +19,10 @@ ;;; Commentary: -;; +;; This file defines the behavior of a bot. To allow completion for +;; tags, this bot runs through all sites in the network and retrieves +;; all of their tags. This data is then written to a directory which +;; is tracked by the git repository. ;;; Code: -- cgit v1.2.3 From 7069ab0b2374bf255425fca6705e85706645b643 Mon Sep 17 00:00:00 2001 From: Sean Allred Date: Sat, 3 Jan 2015 23:07:58 -0500 Subject: Miscellaneous changes * Use `sx-load' instead of the specific features we need * Use one `insert' form instead of two --- bot/sx-bot.el | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) (limited to 'bot/sx-bot.el') diff --git a/bot/sx-bot.el b/bot/sx-bot.el index fe93beb..b32a69c 100644 --- a/bot/sx-bot.el +++ b/bot/sx-bot.el @@ -27,14 +27,13 @@ ;;; Code: -(require 'sx-site) -(require 'sx-tag) - -(setq sx-request-remaining-api-requests-message-threshold 50000) - (require 'package) (package-initialize) +(require 'sx-load) + +(setq sx-request-remaining-api-requests-message-threshold 50000) + (defcustom sx-bot-out-dir "./data/tags/" "Directory where output tag files are saved." :type 'directory @@ -51,15 +50,13 @@ File is savedd in `sx-bot-out-dir'." (repr (prin1-to-string (sort (cdr data) #'string-lessp)))) - (insert repr) - (insert "\n") + (insert repr "\n") (goto-char (point-min)) (while (search-forward "\" \"" nil t) (replace-match "\"\n \"" nil t)))) (message "Wrote %S" file-name) file-name)) - (defun sx-bot-fetch-and-write-tags () "Get a list of all tags of all sites and save to disk." (make-directory sx-bot-out-dir t) -- cgit v1.2.3