checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
---
2
2
SHA256:
3
- metadata.gz: '0382657017a7d0cc8bc41326d81f5c5bb4119fba404f6dbbdc903080ebb88722'
4
- data.tar.gz: 0ca907ef6d78f9212c1af2d421c8c0280614958c1e9c2694df3522230ae4c10a
3
+ metadata.gz: 5929841e7bf672bf499a73bbfd49a4f7b13db0238cfd57c3705d59513351fc30
4
+ data.tar.gz: f4a8809af8058bb43f24a0a0842b89e5158ca96ae16145d65f63dd8a6030f8bd
5
5
SHA512:
6
- metadata.gz: b1289b38c0ef0f39f66ca5ed913be6db354a50d8a4001dc86db098090c6f27513c62997205d370da769d517879b6f603df381ebfa3d016102b72657298794f9a
7
- data.tar.gz: eb9b7978b9b7a92f3757320256e42696062ed8a06f478ff2fddb987a911ca159836716ce66e0fc1a199a33f315c2acda4426829f41d47e11b6b1088f53d27fdd
6
+ metadata.gz: 867c37d6e1b39d2a7de7a001c0aec4cc0891338f0d3504e7de8062b3ff88aeef1e8794f312f97b038bd4b1a4936e58e5629ea97907c417d13e9ead01d502cfb4
7
+ data.tar.gz: d0eb4c0cdd073d4c34c8460a72e767347d52dcc0d03de007c829e7d6b2a20075e8013d1764db39e74e4d62a62e43537599b28d98b8ab0cbe163499663b578e64
data/CHANGELOG.md CHANGED
@@ -1,3 +1,14 @@
1
+ ## v4.1.0 (2019-08-12)
2
+
3
+ - Update dataset-catalog-services URL to only fetch current (*not* retired)
4
+ metadata records.
5
+ - Add a few more gem release notes to README.
6
+
7
+ ## v4.0.1 (2019-07-08)
8
+
9
+ - Update CHANGELOG and release instructions.
10
+ - Fix README typo.
11
+
1
12
## v4.0.0 (2019-07-08)
2
13
3
14
Changes:
data/README.md CHANGED
@@ -134,6 +134,19 @@ tagging, and publishing to RubyGems.
134
134
* the changes are pushed
135
135
* the tagged version is built and published to RubyGems
136
136
137
+ You will need to have a current Rubygems API key for the _NSIDC developer user_ account in
138
+ order to publish a new version of the gem to Rubygems. To get the lastest API key:
139
+
140
+ `curl -u <username> https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials; chmod 0600 ~/.gem/credentials`
141
+
142
+ ## Release steps (summary)
143
+
144
+ - Confirm no errors are returned by `bundle exec rubocop`
145
+ - Confirm all tests pass (`bundle exec rake spec:unit`)
146
+ - Update the version number and date manually in `CHANGELOG.md` and commit the
147
+ changes.
148
+ - Run the appropriate `bundle exec rake release:*` task
149
+
137
150
### SOLR
138
151
139
152
To harvest data utilizing the gem, you will need an installed instance of [Solr
data/bin/search_solr_tools CHANGED
@@ -12,13 +12,13 @@ class SolrHarvestCLI < Thor
12
12
puts SearchSolrTools::VERSION
13
13
end
14
14
15
- desc 'harvest', 'Harvest from one of the ADE harvesters'
15
+ desc 'harvest', 'Harvest from the specified data centers'
16
16
option :data_center, type: :array, required: true
17
17
option :environment, required: true
18
18
option :die_on_failure, type: :boolean
19
19
def harvest(die_on_failure = options[:die_on_failure] || false)
20
20
options[:data_center].each do |target|
21
- puts target
21
+ puts "Target: #{target}"
22
22
begin
23
23
harvest_class = get_harvester_class(target)
24
24
harvester = harvest_class.new(options[:environment], die_on_failure)
data/lib/search_solr_tools/config/environments.yaml CHANGED
@@ -36,39 +36,38 @@
36
36
- http://data.eol.ucar.edu/jedi/catalog/ucar.ncar.eol.project.ITEX.thredds.xml
37
37
:r2r_url: http://get.rvdata.us/services/cruise/
38
38
39
+ # Not using DCS API v2 here because not all retired datasets have their "retired"
40
+ # flag checked. For example, GLA01.033 is retired; GLA01.018 is not, but it
41
+ # should be. GLA01.018 will show up if we use DCS API v2.
42
+ :nsidc_oai_identifiers_url: oai?verb=ListIdentifiers&metadata_prefix=iso&retired=false
43
+
39
44
:local:
40
45
:host: localhost
41
46
:nsidc_dataset_metadata_url: http://integration.nsidc.org/api/dataset/metadata/
42
- :nsidc_oai_identifiers_url: http://integration.nsidc.org/api/dataset/metadata/oai?verb=ListIdentifiers&metadata_prefix=iso
43
47
44
48
:dev:
45
49
## For the below, you'll need to instantiate your own search-solr instance, and point host to that.
46
50
:host: dev.search-solr.USERNAME.dev.int.nsidc.org
47
- ## For these, either set up your own instance of dataset search catalog or change it to point to integration
51
+ ## For the metadata content, either set up your own instance of dataset-catalog-services
52
+ ## or change the URL below to point to integration
48
53
:nsidc_dataset_metadata_url: http://dev.dcs.USERNAME.dev.int.nsidc.org:1580/api/dataset/metadata/
49
- :nsidc_oai_identifiers_url: http://dev.dcs.USERNAME.dev.int.nsidc.org:1580/api/dataset/metadata/oai?verb=ListIdentifiers&metadata_prefix=iso
50
54
51
55
:integration:
52
56
:host: integration.search-solr.apps.int.nsidc.org
53
57
:nsidc_dataset_metadata_url: http://integration.nsidc.org/api/dataset/metadata/
54
- :nsidc_oai_identifiers_url: http://integration.nsidc.org/api/dataset/metadata/oai?verb=ListIdentifiers&metadata_prefix=iso
55
58
56
59
:qa:
57
60
:host: qa.search-solr.apps.int.nsidc.org
58
61
:nsidc_dataset_metadata_url: http://qa.nsidc.org/api/dataset/metadata/
59
- :nsidc_oai_identifiers_url: http://qa.nsidc.org/api/dataset/metadata/oai?verb=ListIdentifiers&metadata_prefix=iso
60
62
61
63
:staging:
62
64
:host: staging.search-solr.apps.int.nsidc.org
63
65
:nsidc_dataset_metadata_url: http://staging.nsidc.org/api/dataset/metadata/
64
- :nsidc_oai_identifiers_url: http://staging.nsidc.org/api/dataset/metadata/oai?verb=ListIdentifiers&metadata_prefix=iso
65
66
66
67
:blue:
67
68
:host: blue.search-solr.apps.int.nsidc.org
68
69
:nsidc_dataset_metadata_url: http://nsidc.org/api/dataset/metadata/
69
- :nsidc_oai_identifiers_url: http://nsidc.org/api/dataset/metadata/oai?verb=ListIdentifiers&metadata_prefix=iso
70
70
71
71
:production:
72
72
:host: search-solr.apps.int.nsidc.org
73
73
:nsidc_dataset_metadata_url: http://nsidc.org/api/dataset/metadata/
74
- :nsidc_oai_identifiers_url: http://nsidc.org/api/dataset/metadata/oai?verb=ListIdentifiers&metadata_prefix=iso
data/lib/search_solr_tools/harvesters/nsidc_json.rb CHANGED
@@ -31,9 +31,14 @@ module SearchSolrTools
31
31
end
32
32
33
33
def result_ids_from_nsidc
34
- get_results SolrEnvironments[@environment][:nsidc_oai_identifiers_url], '//xmlns:identifier'
34
+ url = SolrEnvironments[@environment][:nsidc_dataset_metadata_url] +
35
+ SolrEnvironments[@environment][:nsidc_oai_identifiers_url]
36
+ get_results url, '//xmlns:identifier'
35
37
end
36
38
39
+ # Fetch a JSON representation of a dataset's metadata
40
+ # @param id [String] NSIDC authoritative ID for the dataset
41
+ # @return [Hash] Parsed version of the JSON response
37
42
def fetch_json_from_nsidc(id)
38
43
json_response = RestClient.get(nsidc_json_url + id + '.json')
39
44
JSON.parse(json_response)
@@ -44,6 +49,8 @@ module SearchSolrTools
44
49
failure_ids = []
45
50
46
51
result_ids_from_nsidc.each do |r|
52
+ # Each result looks like:
53
+ # oai:nsidc.org/AE_L2A
47
54
id = r.text.split('/').last
48
55
begin
49
56
docs << { 'add' => { 'doc' => @translator.translate(fetch_json_from_nsidc(id)) } }
data/lib/search_solr_tools/helpers/facet_configuration.rb CHANGED
@@ -8,7 +8,7 @@ module SearchSolrTools
8
8
class FacetConfiguration
9
9
include Singleton
10
10
def self.import_bin_configuration(env)
11
- @bin_configuration = JSON.parse(RestClient.get(SolrEnvironments[env][:nsidc_dataset_metadata_url] + '/binConfiguration')) if @bin_configuration.nil?
11
+ @bin_configuration = JSON.parse(RestClient.get(SolrEnvironments[env][:nsidc_dataset_metadata_url] + 'binConfiguration')) if @bin_configuration.nil?
12
12
end
13
13
14
14
def self.get_facet_bin(facet_name)
data/lib/search_solr_tools/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
module SearchSolrTools
2
- VERSION = '4.0.1'
2
+ VERSION = '4.2.0'
3
3
end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
--- !ruby/object:Gem::Specification
2
2
name: search_solr_tools
3
3
version: !ruby/object:Gem::Version
4
- version: 4.0.1
4
+ version: 4.2.0
5
5
platform: ruby
6
6
authors:
7
7
- Chris Chalstrom
@@ -12,7 +12,7 @@ authors:
12
12
autorequire:
13
13
bindir: bin
14
14
cert_chain: []
15
- date: 2019-07-08 00:00:00.000000000 Z
15
+ date: 2019-08-13 00:00:00.000000000 Z
16
16
dependencies:
17
17
- !ruby/object:Gem::Dependency
18
18
name: ffi-geos