diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f74abbb7..006e9bee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,8 +70,20 @@ jobs: spec: runs-on: ubuntu-latest - - # services: + services: + postgres: + image: postgres + env: + POSTGRES_DB: boxwalker + POSTGRES_USER: myuser + POSTGRES_PASSWORD: secret + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 # redis: # image: valkey/valkey:8 # ports: @@ -105,6 +117,20 @@ jobs: # ports: # - 6379:6379 # options: --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5 + services: + postgres: + image: postgres + env: + POSTGRES_DB: boxwalker + POSTGRES_USER: myuser + POSTGRES_PASSWORD: secret + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 steps: - name: Install packages run: sudo apt-get update && sudo apt-get install --no-install-recommends -y libvips node-gyp @@ -133,6 +159,20 @@ jobs: # ports: # - 6379:6379 # options: --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5 + services: + postgres: + image: postgres + env: + POSTGRES_DB: boxwalker + POSTGRES_USER: myuser + POSTGRES_PASSWORD: secret + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 steps: - name: Install packages run: sudo apt-get update && sudo apt-get install --no-install-recommends -y libvips node-gyp diff --git a/Gemfile b/Gemfile index badde0e3..e0ac314d 100644 --- a/Gemfile +++ b/Gemfile @@ -4,8 +4,8 @@ source "https://rubygems.org" gem "rails", "~> 8.1.3" # The modern asset pipeline for Rails [https://github.com/rails/propshaft] gem "propshaft" -# Use sqlite3 as the database for Active Record -gem "sqlite3", ">= 2.1" +# Use postgres for the default database +gem "pg" # Use the Puma web server [https://github.com/puma/puma] gem "puma", ">= 5.0" # Use JavaScript with ESM import maps [https://github.com/rails/importmap-rails] diff --git a/Gemfile.lock b/Gemfile.lock index ea3ab74f..52af03c8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -299,6 +299,12 @@ GEM parser (3.3.12.0) ast (~> 2.4.1) racc + pg (1.6.3) + pg (1.6.3-aarch64-linux) + pg (1.6.3-aarch64-linux-musl) + pg (1.6.3-arm64-darwin) + pg (1.6.3-x86_64-linux) + pg (1.6.3-x86_64-linux-musl) popper_js (2.11.8) pp (0.6.4) prettyprint @@ -488,13 +494,6 @@ GEM ostruct retriable ruby-progressbar - sqlite3 (2.9.6-aarch64-linux-gnu) - sqlite3 (2.9.6-aarch64-linux-musl) - sqlite3 (2.9.6-arm-linux-gnu) - sqlite3 (2.9.6-arm-linux-musl) - sqlite3 (2.9.6-arm64-darwin) - sqlite3 (2.9.6-x86_64-linux-gnu) - sqlite3 (2.9.6-x86_64-linux-musl) sshkit (1.25.1) base64 logger @@ -592,6 +591,7 @@ DEPENDENCIES importmap-rails jbuilder kamal + pg propshaft puma (>= 5.0) rails (~> 8.1.3) @@ -607,7 +607,6 @@ DEPENDENCIES solid_cache solid_queue solr_wrapper (>= 0.3) - sqlite3 (>= 2.1) stimulus-rails thruster turbo-rails @@ -735,6 +734,12 @@ CHECKSUMS ostruct (0.6.3) sha256=95a2ed4a4bd1d190784e666b47b2d3f078e4a9efda2fccf18f84ddc6538ed912 parallel (2.1.0) sha256=b35258865c2e31134c5ecb708beaaf6772adf9d5efae28e93e99260877b09356 parser (3.3.12.0) sha256=21a6d7f755d5a24dfbdc6e6b772e4e879a52e7631a88bc5a3a134606052c9828 + pg (1.6.3) sha256=1388d0563e13d2758c1089e35e973a3249e955c659592d10e5b77c468f628a99 + pg (1.6.3-aarch64-linux) sha256=0698ad563e02383c27510b76bf7d4cd2de19cd1d16a5013f375dd473e4be72ea + pg (1.6.3-aarch64-linux-musl) sha256=06a75f4ea04b05140146f2a10550b8e0d9f006a79cdaf8b5b130cde40e3ecc2c + pg (1.6.3-arm64-darwin) sha256=7240330b572e6355d7c75a7de535edb5dfcbd6295d9c7777df4d9dddfb8c0e5f + pg (1.6.3-x86_64-linux) sha256=5d9e188c8f7a0295d162b7b88a768d8452a899977d44f3274d1946d67920ae8d + pg (1.6.3-x86_64-linux-musl) sha256=9c9c90d98c72f78eb04c0f55e9618fe55d1512128e411035fe229ff427864009 popper_js (2.11.8) sha256=f4b0be717fc0d50bdb3dbbc55788525a9e0e8f640b76c9971fc34ee609eadbd2 pp (0.6.4) sha256=dfcb0fce700c41456265922884f9fe195d7fbb0674a3578e6c0f69588e82b570 prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193 @@ -790,13 +795,6 @@ CHECKSUMS solid_cache (1.0.10) sha256=bc05a2fb3ac78a6f43cbb5946679cf9db67dd30d22939ededc385cb93e120d41 solid_queue (1.6.0) sha256=b5fc3bb34162e09d8f960df6400d0f9304dc80fe66c2cfface31c32b228de6a1 solr_wrapper (4.4.0) sha256=1eaf0667611a90c623d05c3c2c765d7a2e4174427d03580396cc2124b10430bb - sqlite3 (2.9.6-aarch64-linux-gnu) sha256=d8b1f7d23efd7abac285775a9566562fc7debfef79d594e3a20354406fb7907c - sqlite3 (2.9.6-aarch64-linux-musl) sha256=3579e1c98cdc7ff5c3722847bb63ed4e1efb7ff675cb5e1e48ef2d4da5fb3bc9 - sqlite3 (2.9.6-arm-linux-gnu) sha256=33541500e3615da02afe54a9cc38b17a6985d3cf9d8b76d6d0a83002f114e7ec - sqlite3 (2.9.6-arm-linux-musl) sha256=c5490af48bb228fefa54314e9541375c3907e70f8109f3881b5ff97e1c93ae33 - sqlite3 (2.9.6-arm64-darwin) sha256=849b5d7f795e60fe25076d62c72dd722beb45b3850b516ad978d60ee848ec15b - sqlite3 (2.9.6-x86_64-linux-gnu) sha256=613188ce02f614126ddbc38c5e217ccffd6306d0dcd9adca9764547aa890a634 - sqlite3 (2.9.6-x86_64-linux-musl) sha256=d493b11818a3573387a1d56e1ee8fa00da23a683a7a1cc063e7a0feeed843abf sshkit (1.25.1) sha256=be3f10b9d6eb0b44d5eaba3f7cbe41bc6bb894bce4339688ac20124391455b78 stimulus-rails (1.3.4) sha256=765676ffa1f33af64ce026d26b48e8ffb2e0b94e0f50e9119e11d6107d67cb06 thor (1.5.0) sha256=e3a9e55fe857e44859ce104a84675ab6e8cd59c650a49106a05f55f136425e73 diff --git a/README.md b/README.md index f3f84ab1..89f1cb7d 100644 --- a/README.md +++ b/README.md @@ -15,15 +15,18 @@ docker compose --profile app up # Create the blacklight-collection /bin/bash ./solr/dev-init.sh ``` -#### Indexing a directory +#### Start ingest pipeline ```shell +# Add your data under ./data/ead/{repo_slug} +mkdir -p ./data/ead +cp -r ./sample-ead/scrc ./data/ead/ + # Inside the container docker compose exec app bash -DIR= REPOSITORY= rake um_arclight:index_dir +rake arclight:ingest_everything -# Outside the container -SOLR_URL=http://localhost:8983/solr/blacklight-collection DIR=./ REPOSITORY_ID= rake um_arclight:index_dir ``` + ### Docker Hybrid development Use hybrid for quick rebuilds of Rails (outside a container) without repeatedly spinning up Solr/Zookeeper for each reset. ##### Services shell @@ -31,42 +34,41 @@ Use hybrid for quick rebuilds of Rails (outside a container) without repeatedly ```shell # Build resque-web and resque images docker compose build -# Start zookeeper, solr, redis, resque and resque-web containers -docker compose up +# Start zookeeper, solr, postgres, redis, resque and resque-web containers +docker compose up ``` ##### Rails shell ```shell # Create the blacklight-collection -/bin/bash ./solr/dev-init.sh +/bin/bash ./solr/dev-init.sh # Bundle install the gems outside of the container bundle install -# Migrate the database and prepare it for GUI finding aid indexing -# NOTE: This command is run via bin/docker-entrypoint when resque is brought up +# You may also need to install vips: https://formulae.brew.sh/formula/vips. + +# NOTE: The migration and database preparation command here is run by the migrate service +# (see Notes - Database and Migration). # bin/rails db:prepare # Development Rails server using Solr container SOLR_URL=http://localhost:8983/solr/blacklight-collection bin/dev ``` -##### Browser Resque Web -Open http://localhost:5678 in your browser ##### Command shell ```shell -# List all available tasks -bin/rails --task -``` -```shell -# Index sample-ead +# Rake task to start ingest pipeline rsync -av --progress sample-ead/ data/ead/ -FINDING_AID_DATA=./data bin/rails arclight:ingest_everything +SOLR_URL=http://localhost:8983/solr/blacklight-collection \ + FINDING_AID_DATA=./data bin/rails \ + arclight:ingest_everything ``` -### Troubleshooting -If you encounter this indexing error post-Boxrunner merge -``` -. . . .rake aborted! -Bundler::GemRequireError: There was an error while trying to load the gem 'ruby-vips'. (Bundler::GemRequireError) -Gem Load Error is: Could not open library 'vips.42': dlopen(vips.42, 0x0005): tried: 'vips.42' (no such file), '/System/Volumes/Preboot/Cryptexes/OSvips.42' (no such file), '/usr/lib/vips.42' (no such file, not in dyld cache), 'vips.42' (no such file), '/usr/local/lib/vips.42' (no such file), '/usr/lib/vips.42' (no such file, not in dyld cache). -``` -I solved it by installing `vips` using `brew install vips`. -Then run `bundle install` +##### Browser Resque Web +Open http://localhost:5678 in your browser + +## Notes + +### Database and Migration + +The application is now setup to use a single PostgreSQL database (shared by the `app` and `resque` services) +in both all-Docker and hybrid usages. We use a service called migrate to run `db:prepare` +before both `app` and `resque` run, which helps us avoid race conditions during schema set up. \ No newline at end of file diff --git a/bin/docker-entrypoint b/bin/docker-entrypoint index c8f5e49f..16cae926 100755 --- a/bin/docker-entrypoint +++ b/bin/docker-entrypoint @@ -1,15 +1,9 @@ #!/bin/bash -e -# Prepare the database for the long-running services that depend on it: -# * the Rails web server (renders catalog pages), and -# * the Resque worker -- Package::Generator renders /catalog/:id in-process while -# building HTML/PDF, and in development the migration_error=:page_load middleware -# runs a pending-migration check on that request. The worker container has its own -# (image-baked, non-volume) SQLite DB, so it must be prepared here too or packaging -# jobs fail with a 500 on the internal render. -if { [ "${@: -2:1}" == "./bin/rails" ] && [ "${@: -1:1}" == "server" ]; } || \ - [[ "${*}" == *resque:work* ]]; then - ./bin/rails db:prepare -fi +# NOTE: database preparation is intentionally NOT done here. A dedicated, +# run-once "migrate" service (see compose.yml) owns `db:prepare` so exactly one +# process touches the schema, avoiding concurrent schema-load races between the +# web server and the Resque worker. This also mirrors the Kubernetes model where +# migrations run as a one-shot Job rather than on every pod/container start. exec "${@}" diff --git a/compose.yml b/compose.yml index aa0d97b3..7dc26ef3 100644 --- a/compose.yml +++ b/compose.yml @@ -4,10 +4,14 @@ services: profiles: - app depends_on: + migrate: + condition: service_completed_successfully solr: condition: service_healthy redis: condition: service_healthy + postgres: + condition: service_healthy build: context: . ports: @@ -16,10 +20,24 @@ services: - SOLR_URL=http://solr:8983/solr/blacklight-collection - REDIS_URL=redis://redis:6379/0 - FINDING_AID_DATA=/opt/app-data + - DB_HOST=postgres env_file: - .env + command: ./bin/rails server -b 0.0.0.0 -p 3000 volumes: - ./data:/opt/app-data + migrate: + build: + context: . + depends_on: + postgres: + condition: service_healthy + environment: + - DB_HOST=postgres + env_file: + - .env + command: ./bin/rails db:prepare + restart: "no" redis: image: redis:7 healthcheck: @@ -33,20 +51,40 @@ services: volumes: - redis-data:/data command: redis-server --appendonly yes + postgres: + healthcheck: + interval: 30s + timeout: 10s + start_period: 30s + retries: 5 + test: [ "CMD-SHELL", "pg_isready -h localhost -d boxwalker -p 5432 -U myuser" ] + image: 'postgres:latest' + environment: + - 'POSTGRES_DB=boxwalker' + - 'POSTGRES_USER=myuser' + - 'POSTGRES_PASSWORD=secret' + ports: + - '127.0.0.1:5432:5432' + volumes: + - postgres-data:/var/lib/postgresql resque: depends_on: + migrate: + condition: service_completed_successfully solr: condition: service_healthy redis: condition: service_healthy + postgres: + condition: service_healthy build: context: . environment: - - RAILS_ENV=development - SOLR_URL=http://solr:8983/solr/blacklight-collection - REDIS_URL=redis://redis:6379/0 - QUEUE=* - FINDING_AID_DATA=/opt/app-data + - DB_HOST=postgres env_file: - .env volumes: @@ -106,6 +144,7 @@ services: ZOO_SERVERS: server.1=0.0.0.0:2888:3888;2181 volumes: data: + postgres-data: solr-data: redis-data: zk-data: diff --git a/config/database.yml b/config/database.yml index 302d638c..bfc0ec62 100644 --- a/config/database.yml +++ b/config/database.yml @@ -1,40 +1,48 @@ -# SQLite. Versions 3.8.0 and up are supported. -# gem install sqlite3 -# -# Ensure the SQLite 3 gem is defined in your Gemfile -# gem "sqlite3" -# +--- default: &default - adapter: sqlite3 - max_connections: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> - timeout: 5000 + adapter: postgresql + encoding: unicode + pool: 5 + host: <%= ENV.fetch("DB_HOST", "127.0.0.1") %> + username: <%= ENV.fetch("DB_USERNAME", "myuser") %> + password: <%= ENV.fetch("DB_PASSWORD", "secret") %> + # Skip GSSAPI/Kerberos negotiation on connect. libpq defaults to + # gssencmode=prefer, which hangs ~34s in hybrid dev when the host has an + # expired Kerberos ticket (e.g. UMich ADSROOT). Postgres in Docker doesn't + # use GSSAPI, so disabling it is correct in every environment. + gssencmode: <%= ENV.fetch("DB_GSSENCMODE", "disable") %> + # Detect dead/half-open sockets quickly instead of blocking on the OS TCP + # timeout (~34s). This matters in hybrid dev (host Ruby -> Postgres in Docker + # Desktop), whose network proxy silently drops idle connections. The keepalive + # probes also keep pooled connections warm so they never go stale. + connect_timeout: <%= ENV.fetch("DB_CONNECT_TIMEOUT", 5) %> + keepalives: 1 + keepalives_idle: 10 + keepalives_interval: 5 + keepalives_count: 3 development: <<: *default - database: storage/development.sqlite3 + database: <%= ENV.fetch("DB_NAME", "boxwalker") %> -# Warning: The database defined as "test" will be erased and -# re-generated from your development database when you run "rake". -# Do not set this db to the same as development or production. test: <<: *default - database: storage/test.sqlite3 + database: <%= ENV.fetch("DB_NAME", "boxwalker") %> -# Store production database in the storage/ directory, which by default -# is mounted as a persistent Docker volume in config/deploy.yml. production: primary: <<: *default - database: storage/production.sqlite3 + database: <%= ENV.fetch("DB_NAME", "boxwalker") %> cache: <<: *default - database: storage/production_cache.sqlite3 + database: <%= ENV.fetch("CACHE_DB_NAME", "boxwalker_cache") %> migrations_paths: db/cache_migrate queue: <<: *default - database: storage/production_queue.sqlite3 + database: <%= ENV.fetch("QUEUE_DB_NAME", "boxwalker_queue") %> migrations_paths: db/queue_migrate cable: <<: *default - database: storage/production_cable.sqlite3 + database: <%= ENV.fetch("CABLE_DB_NAME", "boxwalker_cable") %> migrations_paths: db/cable_migrate + diff --git a/config/environments/development.rb b/config/environments/development.rb index f2b7d8a2..5936b2c2 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -26,7 +26,7 @@ end # Change to :null_store to avoid any caching. - config.cache_store = :memory_store + config.cache_store = :null_store # Store uploaded files on the local file system (see config/storage.yml for options). config.active_storage.service = :local diff --git a/db/schema.rb b/db/schema.rb index 7a75960f..344aaccf 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,6 +11,9 @@ # It's strongly recommended that you check this file into your version control system. ActiveRecord::Schema[8.1].define(version: 2026_06_05_172247) do + # These are extensions that must be enabled in order to support this database + enable_extension "pg_catalog.plpgsql" + create_table "bookmarks", force: :cascade do |t| t.datetime "created_at", precision: nil, null: false t.string "document_id"