1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
|
#!/usr/bin/env bash
# This file is part of license-reporter
# Copyright (C) 2017 David Hedlund
#
# license-reporter is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# license-reporter is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#if [ "$data" == "search" ]; then search_page_variable='.results[]'; fi
# Specifically for titles: en-US is sometimes unused, en-GB used to be used then.
# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.name.enUS" | sed "s|^\"||; s|\"$||;"
# sed "s|en-GB|enGB|g" "$json_file" | jq "$search_page_variable.name.enGB" | sed "s|^\"||; s|\"$||;"
# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.homepage.enUS" | sed "s|^\"||; s|\"$||;"
# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.support_email.enUS" | sed "s|^\"||; s|\"$||;"
# sed "s|en-US|enUS|g" "$json_file" | jq "$search_page_variable.support_url.enUS" | sed "s|^\"||; s|\"$||;"
# Licenses
# sed "s|en-US|enUS|g" "$slug.json" | jq '.license.name.enUS' | sed "s|^\"||; s|\"$||;"
if [ "$2" == "--debug" ]; then debug=true; else debug=false; fi
minimum_average_daily_users="10000";
SCRIPTSRC=$(readlink -f "$0" || echo "$0")
RUN_PATH=$(dirname "${SCRIPTSRC}" || echo .)
export -p SCRIPTSRC RUN_PATH minimum_average_daily_users debug
if [ ! -f /usr/bin/jq ]; then echo "/usr/bin/jq not found!"; exit=true; fi
if [ ! -f /usr/bin/wget ]; then echo "/usr/bin/wget not found!"; exit=true; fi
if [[ $exit == true ]]; then exit 1; fi
case "$1" in
--fresh-build)
if [ -d "build" ]; then
rm -fr build
fi
mkdir -p build
;;
--make-custom-list)
if [ ! -f "$RUN_PATH/license-reporter-custom" ]; then
echo "$RUN_PATH/license-reporter-custom don't exist. Skipping custom list.";
else
src/download-custom
fi
;;
--make-search-list)
cd build/ || exit
mkdir json
cd json || exit
# API documentation: https://addons-server.readthedocs.io/en/latest/topics/api/addons.html
function foo() {
# Get the most popular WebExtensions
wget "https://addons.mozilla.org/api/v3/addons/search/?sort=users&page=$page&tag=firefox57"
}
page="1"
foo
if [ "$debug" == false ]; then
while [ "$(jq ".results[].average_daily_users" "index.html?sort=users&page=$page&tag=firefox57" | tail -n 1)" -gt "$minimum_average_daily_users" ]; do
((page++))
foo
done
fi
find index.html* -type f -printf "%f\n" | while read -r file; do
line="0";
for average_daily_users in $(jq ".results[].average_daily_users" "$file"); do
# Exclude add-ons with to low average daily users from index.html*
if [ "$average_daily_users" -ge "$minimum_average_daily_users" ]; then
echo -e "$average_daily_users\t$(jq ".results[$line].id" "$file")\t$file" >> ../merged-search.txt
# wget -nc "https://addons.mozilla.org/api/v3/addons/addon/$(jq ".results[$line].slug" "$file" | sed "s|^\"||; s|\"$||;")/versions/$(jq ".results[$line].current_version.id" "$file")/" -O "current_versions/$(jq ".results[$line].current_version.id" "$file").json"
fi
((line++))
done
done
;;
--merge-lists)
cd build || exit
cat merged-search.txt >> MERGED-ALL.txt
cat merged-custom.txt >> MERGED-ALL.txt
;;
--download-free-webextensions-for-gnu-and-linux)
cd build || exit
rm -fr free_webextensions_for_GNU_and_Linux
mkdir free_webextensions_for_GNU_and_Linux
cd free_webextensions_for_GNU_and_Linux || exit
line="0";
while IFS= read -r license; do
((line++))
id="$(sed -n ${line}p ../id.txt)";
slug="$(sed -n ${line}p ../slug.txt)";
# This is the complete list of pre-defined licenses listed on AMO. Apache License 2.0 is not on the list so its not easy to detect because it has to be released under a "Custom License".
if
[ "$license" == "GNU General Public License, version 3.0" ] ||
[ "$license" == "GNU General Public License, version 2.0" ] ||
[ "$license" == "GNU Lesser General Public License, version 3.0" ] ||
[ "$license" == "GNU Lesser General Public License, version 2.1" ] ||
[ "$license" == "Mozilla Public License, version 2.0" ] ||
[ "$license" == "Mozilla Public License Version 1.1" ] ||
[ "$license" == "BSD License" ] ||
[ "$license" == "MIT/X11 License" ]
then
# Add-ons not avalible for GNU/Linux will be ignored (used to be very few dough).
wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/addon-$id-latest.xpi" || wget -nc "https://addons.mozilla.org/firefox/downloads/latest/$slug/platform:2/addon-$id-latest.xpi"
fi
done < ../license.name.enUS.txt
;;
--verify-license-copy)
cd build || exit
rm -fr VERIFY-LICENSE-COPY.txt
cd free_webextensions_for_GNU_and_Linux || exit
while IFS= read -r id; do
if [ ! -f "addon-$id-latest.xpi" ]; then
echo "Nonfree" >> ../VERIFY-LICENSE-COPY.txt
elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE$"; then
echo "LICENSE" >> ../VERIFY-LICENSE-COPY.txt
elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.txt$"; then
echo "LICENSE.txt" >> ../VERIFY-LICENSE-COPY.txt
elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^LICENSE.md$"; then
echo "LICENSE.md" >> ../VERIFY-LICENSE-COPY.txt
elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING$"; then
echo "COPYING" >> ../VERIFY-LICENSE-COPY.txt
elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.txt$"; then
echo "COPYING.txt" >> ../VERIFY-LICENSE-COPY.txt
elif unzip -l "addon-$id-latest.xpi" | awk '{ print $4 }' | grep -q "^COPYING.md$"; then
echo "COPYING.md" >> ../VERIFY-LICENSE-COPY.txt
else
echo "Request" >> ../VERIFY-LICENSE-COPY.txt
fi
done < ../id.txt
line="0";
while IFS= read -r status; do
((line++))
average_daily_users="$(sed -n ${line}p ../average_daily_users.txt)";
id="$(sed -n ${line}p ../id.txt)";
slug="$(sed -n ${line}p ../slug.txt)";
license_name_en_US="$(sed -n ${line}p ../license.name.enUS.txt)";
homepage_en_US="$(sed -n ${line}p ../homepage.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)";
support_email_en_US="$(sed -n ${line}p ../support_email.en-US.txt)";
support_url_en_US="$(sed -n ${line}p ../support_url.en-US.txt | sed "s|3A\\/\\/|\n|" | tail -n 1)";
if [ "$license_name_en_US" == "GNU General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/gpl-3.0.txt"; license_file_name="COPYING";
elif [ "$license_name_en_US" == "GNU General Public License, version 2.0" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/gpl-2.0.txt"; license_file_name="COPYING";
elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 3.0" ]; then license_txt="https://www.gnu.org/licenses/lgpl-3.0.txt"; license_file_name="COPYING";
elif [ "$license_name_en_US" == "GNU Lesser General Public License, version 2.1" ]; then license_txt="https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt"; license_file_name="COPYING";
elif [ "$license_name_en_US" == "Mozilla Public License, version 2.0" ]; then license_txt="https://www.mozilla.org/media/MPL/2.0/index.txt"; license_file_name="LICENSE";
elif [ "$license_name_en_US" == "Mozilla Public License Version 1.1" ]; then license_txt="https://www.mozilla.org/media/MPL/1.1/index.txt"; license_file_name="LICENSE";
elif [ "$license_name_en_US" == "BSD License" ]; then license_txt="https://directory.fsf.org/wiki?title=License:FreeBSD"; license_file_name="LICENSE";
elif [ "$license_name_en_US" == "MIT/X11 License" ]; then license_txt="https://directory.fsf.org/wiki/License:X11"; license_file_name="LICENSE";
fi
if [ "$(sed -n ${line}p ../name.en-US.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-US.txt)";
elif [ "$(sed -n ${line}p ../name.en-GB.txt)" != "null" ]; then name="$(sed -n ${line}p ../name.en-GB.txt)";
else name="$slug";
fi
if [ "$status" = "Request" ]; then
((count_reports++))
# Authors that don't provide contact information will be ignored, its impossible to track conversations in Reviews anyway, because they will be deleted.
if [ "$homepage_en_US" != "null" ] || [ "$support_url_en_US" != "null" ] || [ "$support_email_en_US" != "null" ]; then
decimals="$(echo "$average_daily_users" | sed -r ':L;s=\b([0-9]+)([0-9]{3})\b=\1,\2=g;t L')";
echo "
$decimals average daily users: $name - https://addons.mozilla.org/en-US/firefox/addon/$slug/
Request: \"You have distributed $name under $license_name_en_US on https://addons.mozilla.org/en-US/firefox/addon/$slug/, but you have not added a copy of the license. can you please download the text of the license from $license_txt and add save it to file name $license_file_name and put it in the root directory of you program?\" by contacting:";
if [ "$homepage_en_US" != "null" ]; then echo " * Homepage (please submit Pull Request for $license_file_name if possible): $homepage_en_US"; fi
if [ "$support_url_en_US" != "null" ]; then echo " * Support URL (please submit Pull Request for $license_file_name if possible): $support_url_en_US"; fi
if [ "$support_email_en_US" != "null" ]; then echo " * Support E-mail: $support_email_en_US"; fi
fi
fi
done < ../VERIFY-LICENSE-COPY.txt
echo "
#########################
$count_reports reports to submit";
;;
--bugs)
echo "
* Look in the GitHub repository for LICENSE or COPYING. Sometime the add-on is delayed or have not been released to addons.mozilla.org."
;;
--all)
$0 --fresh-build $2
$0 --make-custom-list $2
$0 --make-search-list $2
$0 --merge-lists $2
$0 --download-licenses-json $2
# $0 --download-free-webextensions-for-gnu-and-linux
# $0 --verify-license-copy
;;
esac
|
