ETT-1840: Fixes for title summary table

bin/refresh_title_summary_table.rb

@@ -17,18 +17,20 @@ class RefreshTitleSummaryTable
   # hathifiles_database slice size, seems a little low
   DB_BATCH_SIZE = 100
 
-  attr_reader :facets, :solr_to_database_fields, :summary_table, :tracker
+  attr_reader :solr_to_database_fields, :summary_table, :tracker, :formats
   def initialize
     # Do we need this?
     envfile = Pathname.new(__dir__).parent + ".env"
     Dotenv.load(envfile)
     @summary_table = TitleSummaryTable.new(logger: Services.logger)
-    @facets = SolrPivotFacets.new
     @tracker = PushMetrics.new(
       job_name: ENV.fetch("HATHIFILES_DATABASE_JOB_NAME", "title_summary_table_refresh"),
       logger: Services.logger
     )
-    @solr_to_database_fields = YAML.load_file("data/title_summary_table.yaml")["solr_to_database_fields"]
+
+    config = YAML.load_file("data/title_summary_table.yaml")
+    @solr_to_database_fields = config["solr_to_database_fields"]
+    @formats = config["formats"]
   end
 
   def insert_rows(rows)
@@ -40,20 +42,15 @@ def count
     summary_table.dataset.count
   end
 
-  def run
-    Services.logger.info("Existing summary table count: #{count}")
-    Services.logger.info("Creating temporary database table")
-    summary_table.create(temp: true)
-    # truncate in case it was already present
-    summary_table.dataset(temp: true).truncate
-
-    Services.logger.info("Getting summary data from Solr and collecting counts")
+  def summarize_format(format)
+    Services.logger.info("Getting summary data from Solr and collecting counts for format #{format}")
     rows = []
-    facets.summarize do |row|
+    SolrPivotFacets.new(filter_query: "format:\"#{format}\"").summarize do |row|
       # Map the Solr fields to our database columns
       row = row.map do |key, value|
         [solr_to_database_fields.fetch(key, key), value]
       end.to_h
+      row[solr_to_database_fields.fetch('format','format')] = format
       rows << row
       if rows.count >= DB_BATCH_SIZE
         insert_rows(rows)
@@ -63,6 +60,16 @@ def run
     end
     # Insert leftovers
     insert_rows(rows)
+  end
+
+  def run
+    Services.logger.info("Existing summary table count: #{count}")
+    Services.logger.info("Creating temporary database table")
+    summary_table.create(temp: true)
+    # truncate in case it was already present
+    summary_table.dataset(temp: true).truncate
+
+    formats.each { |format| summarize_format(format) }
 
     summary_table.swap
     tracker.log_final_line
@@ -74,9 +81,7 @@ def run
   # Not covered because Simplecov doesn't instrument anything called w/ backticks or `open3`
   # and the integration test invokes this by class anyway.
   # Doing otherwise might involve brittle `$PROGRAM_NAME` shenanigans.
-  # Note: when moving from SimpleCov 0.22.0 to 1.X replace the :nocov: directives with
-  # simplecov:disable and simplecov:enable to avoid deprecation notices.
-  # :nocov:
+  # simplecov:disable
   RefreshTitleSummaryTable.new.run
-  # :nocov:
+  # simplecov:enable
 end

data/title_summary_table.yaml

@@ -1,4 +1,13 @@
 ---
+formats:
+  - Book
+  - Data File
+  - Map
+  - Mixed Material
+  - Music
+  - Serial
+  - Unknown
+  - Visual Material
 solr_to_database_fields:
   format: format
   language: language

lib/solr_pivot_facets.rb

@@ -6,7 +6,11 @@
 require "json"
 
 class SolrPivotFacets
-  FIELD_NAMES = %w[format language country_of_pub_facet publishDate]
+  FIELD_NAMES = %w[language country_of_pub_facet publishDate]
+
+  def initialize(filter_query: "*:*")
+    @filter_query = filter_query
+  end
 
   # Retrieve pivots from Solr and parse.
   # Can be called before `run` if desired for more granular logging.
@@ -42,7 +46,11 @@ def summarize(data = pivots, fields: {}, &block)
   private
 
   def solr_facets_url
-    "#{ENV["SOLR_URL"]}/select?q=*:*&facet.pivot=#{FIELD_NAMES.join(",")}&facet=true&rows=0&facet.pivot.mincount=1&wt=json"
+    # facet.limit = -1 - return all facets (do not limit # of returned facets)
+    # facet.pivot.mincount = 1 - return facets with at least one matching item
+    # facet.missing = true - include items where a facet value isn't set (i.e.
+    #                 include 'null' as a possible facet value)
+    "#{ENV["SOLR_URL"]}/select?q=*:*&fq=#{@filter_query}&facet.pivot=#{FIELD_NAMES.join(",")}&facet=true&rows=0&facet.pivot.mincount=1&facet.missing=true&facet.limit=-1&wt=json"
   end
 
   def solr_connection

spec/jobs/refresh_title_summary_table_spec.rb

@@ -19,8 +19,8 @@
     table.dataset.truncate
     # Now do it.
     RefreshTitleSummaryTable.new.run
-    # Expect about 1800 entries from solr-sdr-sample, subject to change
-    expect(table.dataset.count).to be_between(1500, 3000)
+    # Expect about 1300 entries from solr-sdr-sample, subject to change
+    expect(table.dataset.count).to be_between(1000, 2000)
   end
 
   it "creates the table(s) if necessary" do
@@ -29,6 +29,28 @@
     hfdb.drop_table?(temp_table_name)
     hfdb.drop_table?(old_table_name)
     RefreshTitleSummaryTable.new.run
-    expect(TitleSummaryTable.new.dataset.count).to be_between(1500, 3000)
+    expect(TitleSummaryTable.new.dataset.count).to be_between(1000, 2000)
+  end
+
+  it "has rows for items missing language, pub year, or pub place" do
+    RefreshTitleSummaryTable.new.run
+
+    dataset = TitleSummaryTable.new.dataset
+
+    expect(dataset.where(language: nil).count).to be > 0
+    expect(dataset.where(published_year: nil).count).to be > 0
+    expect(dataset.where(publication_place: nil).count).to be > 0
+  end
+
+  it "has the expected number of formats" do
+    RefreshTitleSummaryTable.new.run
+
+    # should match formats from title_summary_table.yaml; sample data won't
+    # have all of them, but should have more than one, and not have a bunch of
+    # extraneous formats
+    dataset = TitleSummaryTable.new.dataset
+
+    expect(dataset.distinct(:format).to_a.map(&:values).flatten).to include("Book", "Serial")
+    expect(dataset.distinct(:format).count).to be < 8
   end
 end

spec/solr_pivot_facets_spec.rb

@@ -29,14 +29,14 @@
     end
 
     # See above, but the Solr catalog sample may undergo multiple iterations.
-    # The actual number is 1849 but I am leaving a fair amount of wiggle room.
+    # The actual number is 1319 but I am leaving a fair amount of wiggle room.
     context "with Solr data" do
       it "yields WAY more than 20 rows" do
         rows = Set.new
         facets.summarize do |row|
           rows << row
         end
-        expect(rows.count).to be_between(1500, 3000)
+        expect(rows.count).to be_between(1000, 2000)
       end
     end
   end