1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
|
#!/usr/bin/env ruby
require 'json'
require 'nokogiri'
require 'slop'
# Returns a repo URL for a given package name.
def repo_url value
if value && value.start_with?('http')
value
elsif value
"https://dl.google.com/android/repository/#{value}"
else
nil
end
end
# Returns a system image URL for a given system image name.
def image_url value, dir
if dir == "default"
dir = "android"
end
if value && value.start_with?('http')
value
elsif value
"https://dl.google.com/android/repository/sys-img/#{dir}/#{value}"
else
nil
end
end
# Returns a JSON with the data and structure of the input XML
def to_json_collector doc
json = {}
index = 0
doc.element_children.each { |node|
if node.children.length == 1 and node.children.first.text?
json["#{node.name}:#{index}"] ||= node.content
index += 1
next
end
json["#{node.name}:#{index}"] ||= to_json_collector node
index += 1
}
element_attributes = {}
doc.attribute_nodes.each do |attr|
if attr.name == "type"
type = attr.value.split(':', 2).last
case attr.value
when 'generic:genericDetailsType'
element_attributes["xsi:type"] ||= "ns5:#{type}"
when 'addon:extraDetailsType'
element_attributes["xsi:type"] ||= "ns8:#{type}"
when 'addon:mavenType'
element_attributes["xsi:type"] ||= "ns8:#{type}"
when 'sdk:platformDetailsType'
element_attributes["xsi:type"] ||= "ns11:#{type}"
when 'sdk:sourceDetailsType'
element_attributes["xsi:type"] ||= "ns11:#{type}"
when 'sys-img:sysImgDetailsType'
element_attributes["xsi:type"] ||= "ns12:#{type}"
when 'addon:addonDetailsType' then
element_attributes["xsi:type"] ||= "ns8:#{type}"
end
else
element_attributes[attr.name] ||= attr.value
end
end
if !element_attributes.empty?
json['element-attributes'] ||= element_attributes
end
json
end
# Returns a tuple of [type, revision, revision components] for a package node.
def package_revision package
type_details = package.at_css('> type-details')
type = type_details.attributes['type']
type &&= type.value
revision = nil
components = nil
case type
when 'generic:genericDetailsType', 'addon:extraDetailsType', 'addon:mavenType'
major = text package.at_css('> revision > major')
minor = text package.at_css('> revision > minor')
micro = text package.at_css('> revision > micro')
preview = text package.at_css('> revision > preview')
revision = ''
components = []
unless empty?(major)
revision << major
components << major
end
unless empty?(minor)
revision << ".#{minor}"
components << minor
end
unless empty?(micro)
revision << ".#{micro}"
components << micro
end
unless empty?(preview)
revision << "-rc#{preview}"
components << preview
end
when 'sdk:platformDetailsType'
codename = text type_details.at_css('> codename')
api_level = text type_details.at_css('> api-level')
revision = empty?(codename) ? api_level : codename
components = [revision]
when 'sdk:sourceDetailsType'
api_level = text type_details.at_css('> api-level')
revision, components = api_level, [api_level]
when 'sys-img:sysImgDetailsType'
codename = text type_details.at_css('> codename')
api_level = text type_details.at_css('> api-level')
id = text type_details.at_css('> tag > id')
abi = text type_details.at_css('> abi')
revision = ''
components = []
if empty?(codename)
revision << api_level
components << api_level
else
revision << codename
components << codename
end
unless empty?(id)
revision << "-#{id}"
components << id
end
unless empty?(abi)
revision << "-#{abi}"
components << abi
end
when 'addon:addonDetailsType' then
api_level = text type_details.at_css('> api-level')
id = text type_details.at_css('> tag > id')
revision = api_level
components = [api_level, id]
end
[type, revision, components]
end
# Returns a hash of archives for the specified package node.
def package_archives package
archives = {}
package.css('> archives > archive').each do |archive|
host_os = text archive.at_css('> host-os')
host_os = 'all' if empty?(host_os)
archives[host_os] = {
'size' => Integer(text(archive.at_css('> complete > size'))),
'sha1' => text(archive.at_css('> complete > checksum')),
'url' => yield(text(archive.at_css('> complete > url')))
}
end
archives
end
# Returns the text from a node, or nil.
def text node
node ? node.text : nil
end
# Nil or empty helper.
def empty? value
!value || value.empty?
end
# Fixes up returned hashes by converting archives like
# (e.g. {'linux' => {'sha1' => ...}, 'macosx' => ...} to
# [{'os' => 'linux', 'sha1' => ...}, {'os' => 'macosx', ...}, ...].
def fixup value
Hash[value.map do |k, v|
if k == 'archives' && v.is_a?(Hash)
[k, v.map do |os, archive|
fixup({'os' => os}.merge(archive))
end]
elsif v.is_a?(Hash)
[k, fixup(v)]
else
[k, v]
end
end]
end
# Today since Unix Epoch, January 1, 1970.
def today
Time.now.utc.to_i / 24 / 60 / 60
end
# The expiration strategy. Expire if the last available day was before the `oldest_valid_day`.
def expire_records record, oldest_valid_day
if record.is_a?(Hash)
if record.has_key?('last-available-day') &&
record['last-available-day'] < oldest_valid_day
return nil
end
update = {}
# This should only happen in the first run of this scrip after adding the `expire_record` function.
if record.has_key?('displayName') &&
!record.has_key?('last-available-day')
update['last-available-day'] = today
end
record.each {|key, value|
v = expire_records value, oldest_valid_day
update[key] = v if v
}
update
else
record
end
end
# Normalize the specified license text.
# See: https://brash-snapper.glitch.me/ for how the munging works.
def normalize_license license
license = license.dup
license.gsub!(/([^\n])\n([^\n])/m, '\1 \2')
license.gsub!(/ +/, ' ')
license.strip!
license
end
# Gets all license texts, deduplicating them.
def get_licenses doc
licenses = {}
doc.css('license[type="text"]').each do |license_node|
license_id = license_node['id']
if license_id
licenses[license_id] ||= []
licenses[license_id] |= [normalize_license(text(license_node))]
end
end
licenses
end
def parse_package_xml doc
licenses = get_licenses doc
packages = {}
doc.css('remotePackage').each do |package|
name, _, version = package['path'].partition(';')
next if version == 'latest'
type, revision, _ = package_revision(package)
next unless revision
path = package['path'].tr(';', '/')
display_name = text package.at_css('> display-name')
uses_license = package.at_css('> uses-license')
uses_license &&= uses_license['ref']
obsolete ||= package['obsolete']
type_details = to_json_collector package.at_css('> type-details')
revision_details = to_json_collector package.at_css('> revision')
archives = package_archives(package) {|url| repo_url url}
dependencies_xml = package.at_css('> dependencies')
dependencies = to_json_collector dependencies_xml if dependencies_xml
target = (packages[name] ||= {})
target = (target[revision] ||= {})
target['name'] ||= name
target['path'] ||= path
target['revision'] ||= revision
target['displayName'] ||= display_name
target['license'] ||= uses_license if uses_license
target['obsolete'] ||= obsolete if obsolete == 'true'
target['type-details'] ||= type_details
target['revision-details'] ||= revision_details
target['dependencies'] ||= dependencies if dependencies
target['archives'] ||= {}
merge target['archives'], archives
target['last-available-day'] = today
end
[licenses, packages]
end
def parse_image_xml doc
licenses = get_licenses doc
images = {}
doc.css('remotePackage[path^="system-images;"]').each do |package|
type, revision, components = package_revision(package)
next unless revision
path = package['path'].tr(';', '/')
display_name = text package.at_css('> display-name')
uses_license = package.at_css('> uses-license')
uses_license &&= uses_license['ref']
obsolete &&= package['obsolete']
type_details = to_json_collector package.at_css('> type-details')
revision_details = to_json_collector package.at_css('> revision')
archives = package_archives(package) {|url| image_url url, components[-2]}
dependencies_xml = package.at_css('> dependencies')
dependencies = to_json_collector dependencies_xml if dependencies_xml
target = images
components.each do |component|
target[component] ||= {}
target = target[component]
end
target['name'] ||= "system-image-#{revision}"
target['path'] ||= path
target['revision'] ||= revision
target['displayName'] ||= display_name
target['license'] ||= uses_license if uses_license
target['obsolete'] ||= obsolete if obsolete
target['type-details'] ||= type_details
target['revision-details'] ||= revision_details
target['dependencies'] ||= dependencies if dependencies
target['archives'] ||= {}
merge target['archives'], archives
target['last-available-day'] = today
end
[licenses, images]
end
def parse_addon_xml doc
licenses = get_licenses doc
addons, extras = {}, {}
doc.css('remotePackage').each do |package|
type, revision, components = package_revision(package)
next unless revision
path = package['path'].tr(';', '/')
display_name = text package.at_css('> display-name')
uses_license = package.at_css('> uses-license')
uses_license &&= uses_license['ref']
obsolete &&= package['obsolete']
type_details = to_json_collector package.at_css('> type-details')
revision_details = to_json_collector package.at_css('> revision')
archives = package_archives(package) {|url| repo_url url}
dependencies_xml = package.at_css('> dependencies')
dependencies = to_json_collector dependencies_xml if dependencies_xml
case type
when 'addon:addonDetailsType'
name = components.last
target = addons
# Hack for Google APIs 25 r1, which displays as 23 for some reason
archive_name = text package.at_css('> archives > archive > complete > url')
if archive_name == 'google_apis-25_r1.zip'
path = 'add-ons/addon-google_apis-google-25'
revision = '25'
components = [revision, components.last]
end
when 'addon:extraDetailsType', 'addon:mavenType'
name = package['path'].tr(';', '-')
components = [package['path']]
target = extras
end
components.each do |component|
target = (target[component] ||= {})
end
target['name'] ||= name
target['path'] ||= path
target['revision'] ||= revision
target['displayName'] ||= display_name
target['license'] ||= uses_license if uses_license
target['obsolete'] ||= obsolete if obsolete
target['type-details'] ||= type_details
target['revision-details'] ||= revision_details
target['dependencies'] ||= dependencies if dependencies
target['archives'] ||= {}
merge target['archives'], archives
target['last-available-day'] = today
end
[licenses, addons, extras]
end
# Make the clean diff by always sorting the result before puting it in the stdout.
def sort_recursively value
if value.is_a?(Hash)
Hash[
value.map do |k, v|
[k, sort_recursively(v)]
end.sort_by {|(k, v)| k }
]
elsif value.is_a?(Array)
value.map do |v| sort_recursively(v) end
else
value
end
end
def merge_recursively a, b
a.merge!(b) {|key, a_item, b_item|
if a_item.is_a?(Hash) && b_item.is_a?(Hash)
merge_recursively(a_item, b_item)
elsif b_item != nil
b_item
end
}
a
end
def merge dest, src
merge_recursively dest, src
end
opts = Slop.parse do |o|
o.array '-p', '--packages', 'packages repo XMLs to parse'
o.array '-i', '--images', 'system image repo XMLs to parse'
o.array '-a', '--addons', 'addon repo XMLs to parse'
end
result = {}
result['licenses'] = {}
result['packages'] = {}
result['images'] = {}
result['addons'] = {}
result['extras'] = {}
opts[:packages].each do |filename|
licenses, packages = parse_package_xml(Nokogiri::XML(File.open(filename)) { |conf| conf.noblanks })
merge result['licenses'], licenses
merge result['packages'], packages
end
opts[:images].each do |filename|
licenses, images = parse_image_xml(Nokogiri::XML(File.open(filename)) { |conf| conf.noblanks })
merge result['licenses'], licenses
merge result['images'], images
end
opts[:addons].each do |filename|
licenses, addons, extras = parse_addon_xml(Nokogiri::XML(File.open(filename)) { |conf| conf.noblanks })
merge result['licenses'], licenses
merge result['addons'], addons
merge result['extras'], extras
end
# As we keep the old packages in the repo JSON file, we should have
# a strategy to remove them at some point!
# So with this variable we claim it's okay to remove them from the
# JSON after two years that they are not available.
two_years_ago = today - 2 * 365
input = {}
begin
input_json = (STDIN.tty?) ? "{}" : $stdin.read
if input_json != nil && !input_json.empty?
input = expire_records(JSON.parse(input_json), two_years_ago)
end
rescue JSON::ParserError => e
$stderr.write(e.message)
return
end
fixup_result = fixup(result)
# Regular installation of Android SDK would keep the previously installed packages even if they are not
# in the uptodate XML files, so here we try to support this logic by keeping un-available packages,
# therefore the old packages will work as long as the links are working on the Google servers.
output = merge input, fixup_result
puts JSON.pretty_generate(sort_recursively(output))
|