diff --git a/bin/refresh_title_summary_table.rb b/bin/refresh_title_summary_table.rb index fe70775..533a8c3 100755 --- a/bin/refresh_title_summary_table.rb +++ b/bin/refresh_title_summary_table.rb @@ -17,18 +17,20 @@ class RefreshTitleSummaryTable # hathifiles_database slice size, seems a little low DB_BATCH_SIZE = 100 - attr_reader :facets, :solr_to_database_fields, :summary_table, :tracker + attr_reader :solr_to_database_fields, :summary_table, :tracker, :formats def initialize # Do we need this? envfile = Pathname.new(__dir__).parent + ".env" Dotenv.load(envfile) @summary_table = TitleSummaryTable.new(logger: Services.logger) - @facets = SolrPivotFacets.new @tracker = PushMetrics.new( job_name: ENV.fetch("HATHIFILES_DATABASE_JOB_NAME", "title_summary_table_refresh"), logger: Services.logger ) - @solr_to_database_fields = YAML.load_file("data/title_summary_table.yaml")["solr_to_database_fields"] + + config = YAML.load_file("data/title_summary_table.yaml") + @solr_to_database_fields = config["solr_to_database_fields"] + @formats = config["formats"] end def insert_rows(rows) @@ -40,20 +42,15 @@ def count summary_table.dataset.count end - def run - Services.logger.info("Existing summary table count: #{count}") - Services.logger.info("Creating temporary database table") - summary_table.create(temp: true) - # truncate in case it was already present - summary_table.dataset(temp: true).truncate - - Services.logger.info("Getting summary data from Solr and collecting counts") + def summarize_format(format) + Services.logger.info("Getting summary data from Solr and collecting counts for format #{format}") rows = [] - facets.summarize do |row| + SolrPivotFacets.new(filter_query: "format:\"#{format}\"").summarize do |row| # Map the Solr fields to our database columns row = row.map do |key, value| [solr_to_database_fields.fetch(key, key), value] end.to_h + row[solr_to_database_fields.fetch('format','format')] = format rows << row if rows.count >= DB_BATCH_SIZE insert_rows(rows) @@ -63,6 +60,16 @@ def run end # Insert leftovers insert_rows(rows) + end + + def run + Services.logger.info("Existing summary table count: #{count}") + Services.logger.info("Creating temporary database table") + summary_table.create(temp: true) + # truncate in case it was already present + summary_table.dataset(temp: true).truncate + + formats.each { |format| summarize_format(format) } summary_table.swap tracker.log_final_line @@ -74,9 +81,7 @@ def run # Not covered because Simplecov doesn't instrument anything called w/ backticks or `open3` # and the integration test invokes this by class anyway. # Doing otherwise might involve brittle `$PROGRAM_NAME` shenanigans. - # Note: when moving from SimpleCov 0.22.0 to 1.X replace the :nocov: directives with - # simplecov:disable and simplecov:enable to avoid deprecation notices. - # :nocov: + # simplecov:disable RefreshTitleSummaryTable.new.run - # :nocov: + # simplecov:enable end diff --git a/config/hathitrust_contrib_configs b/config/hathitrust_contrib_configs index 702d5b5..d170be6 160000 --- a/config/hathitrust_contrib_configs +++ b/config/hathitrust_contrib_configs @@ -1 +1 @@ -Subproject commit 702d5b5b2094eb39fb200a3459ec1c63b4639edb +Subproject commit d170be66dd1f3ed0f37e10c4437e9d21f9be7a10 diff --git a/data/title_summary_table.yaml b/data/title_summary_table.yaml index c66b2d0..fd64864 100644 --- a/data/title_summary_table.yaml +++ b/data/title_summary_table.yaml @@ -1,4 +1,13 @@ --- +formats: + - Book + - Data File + - Map + - Mixed Material + - Music + - Serial + - Unknown + - Visual Material solr_to_database_fields: format: format language: language diff --git a/lib/solr_pivot_facets.rb b/lib/solr_pivot_facets.rb index 379cfcf..55f45a4 100644 --- a/lib/solr_pivot_facets.rb +++ b/lib/solr_pivot_facets.rb @@ -6,7 +6,11 @@ require "json" class SolrPivotFacets - FIELD_NAMES = %w[format language country_of_pub_facet publishDate] + FIELD_NAMES = %w[language country_of_pub_facet publishDate] + + def initialize(filter_query: "*:*") + @filter_query = filter_query + end # Retrieve pivots from Solr and parse. # Can be called before `run` if desired for more granular logging. @@ -42,7 +46,11 @@ def summarize(data = pivots, fields: {}, &block) private def solr_facets_url - "#{ENV["SOLR_URL"]}/select?q=*:*&facet.pivot=#{FIELD_NAMES.join(",")}&facet=true&rows=0&facet.pivot.mincount=1&wt=json" + # facet.limit = -1 - return all facets (do not limit # of returned facets) + # facet.pivot.mincount = 1 - return facets with at least one matching item + # facet.missing = true - include items where a facet value isn't set (i.e. + # include 'null' as a possible facet value) + "#{ENV["SOLR_URL"]}/select?q=*:*&fq=#{@filter_query}&facet.pivot=#{FIELD_NAMES.join(",")}&facet=true&rows=0&facet.pivot.mincount=1&facet.missing=true&facet.limit=-1&wt=json" end def solr_connection diff --git a/spec/jobs/refresh_title_summary_table_spec.rb b/spec/jobs/refresh_title_summary_table_spec.rb index ab46992..c915cbe 100644 --- a/spec/jobs/refresh_title_summary_table_spec.rb +++ b/spec/jobs/refresh_title_summary_table_spec.rb @@ -19,8 +19,8 @@ table.dataset.truncate # Now do it. RefreshTitleSummaryTable.new.run - # Expect about 1800 entries from solr-sdr-sample, subject to change - expect(table.dataset.count).to be_between(1500, 3000) + # Expect about 1300 entries from solr-sdr-sample, subject to change + expect(table.dataset.count).to be_between(1000, 2000) end it "creates the table(s) if necessary" do @@ -29,6 +29,28 @@ hfdb.drop_table?(temp_table_name) hfdb.drop_table?(old_table_name) RefreshTitleSummaryTable.new.run - expect(TitleSummaryTable.new.dataset.count).to be_between(1500, 3000) + expect(TitleSummaryTable.new.dataset.count).to be_between(1000, 2000) + end + + it "has rows for items missing language, pub year, or pub place" do + RefreshTitleSummaryTable.new.run + + dataset = TitleSummaryTable.new.dataset + + expect(dataset.where(language: nil).count).to be > 0 + expect(dataset.where(published_year: nil).count).to be > 0 + expect(dataset.where(publication_place: nil).count).to be > 0 + end + + it "has the expected number of formats" do + RefreshTitleSummaryTable.new.run + + # should match formats from title_summary_table.yaml; sample data won't + # have all of them, but should have more than one, and not have a bunch of + # extraneous formats + dataset = TitleSummaryTable.new.dataset + + expect(dataset.distinct(:format).to_a.map(&:values).flatten).to include("Book", "Serial") + expect(dataset.distinct(:format).count).to be < 8 end end diff --git a/spec/solr_pivot_facets_spec.rb b/spec/solr_pivot_facets_spec.rb index 8372738..7745358 100644 --- a/spec/solr_pivot_facets_spec.rb +++ b/spec/solr_pivot_facets_spec.rb @@ -29,14 +29,14 @@ end # See above, but the Solr catalog sample may undergo multiple iterations. - # The actual number is 1849 but I am leaving a fair amount of wiggle room. + # The actual number is 1319 but I am leaving a fair amount of wiggle room. context "with Solr data" do it "yields WAY more than 20 rows" do rows = Set.new facets.summarize do |row| rows << row end - expect(rows.count).to be_between(1500, 3000) + expect(rows.count).to be_between(1000, 2000) end end end