feat: migrate to YAML config and add background server mode

.github/workflows/macos.yaml

@@ -17,6 +17,14 @@ jobs:
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
+        with:
+          path: bloom
+
+      - name: Check out daylily-tapdb (sibling dependency)
+        uses: actions/checkout@v4
+        with:
+          repository: Daylily-Informatics/daylily-tapdb
+          path: daylily-tapdb
 
       - name: Set up Python
         uses: actions/setup-python@v4
@@ -27,15 +35,16 @@ jobs:
         uses: conda-incubator/setup-miniconda@v2
         with:
           miniconda-version: "latest"
-          python-version: 3.12.0  # Update to your required Python version
+          python-version: 3.12.0
           activate-environment: BLOOM
-          environment-file: bloom_env.yaml  # If you have an environment file
+          environment-file: bloom/bloom_env.yaml
           auto-activate-base: false
 
       - name: Run custom PostgreSQL setup script (if applicable)
         shell: bash -l {0}
+        working-directory: bloom
         run: |
-         pip3 install pytest psycopg2-binary sqlalchemy zebra_day==0.3.0.4 pytz fedex_tracking_day==0.2.6
-         source bloom_lims/env/install_postgres.sh skip
-         pytest
+          pip3 install pytest psycopg2-binary sqlalchemy zebra_day==0.3.0.4 pytz fedex_tracking_day==0.2.6
+          source bloom_lims/env/install_postgres.sh skip
+          pytest
 

.github/workflows/ubuntu.yaml

@@ -17,6 +17,14 @@ jobs:
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
+        with:
+          path: bloom
+
+      - name: Check out daylily-tapdb (sibling dependency)
+        uses: actions/checkout@v4
+        with:
+          repository: Daylily-Informatics/daylily-tapdb
+          path: daylily-tapdb
 
       - name: Set up Python
         uses: actions/setup-python@v4
@@ -27,14 +35,15 @@ jobs:
         uses: conda-incubator/setup-miniconda@v2
         with:
           miniconda-version: "latest"
-          python-version: 3.12.0  # Update to your required Python version
+          python-version: 3.12.0
           activate-environment: BLOOM
-          environment-file: bloom_env.yaml  # If you have an environment file
+          environment-file: bloom/bloom_env.yaml
           auto-activate-base: false
 
       - name: Run custom PostgreSQL setup script (if applicable)
         shell: bash -l {0}
+        working-directory: bloom
         run: |
-          pip3 install pytest psycopg2-binary sqlalchemy  zebra_day==0.3.0.4 pytz fedex_tracking_day==0.2.6
+          pip3 install pytest psycopg2-binary sqlalchemy zebra_day==0.3.0.4 pytz fedex_tracking_day==0.2.6
           source bloom_lims/env/install_postgres.sh skip
           pytest

.gitignore

@@ -5,6 +5,7 @@ __pycache__/
 
 # C extensions
 *.so
+.env
 
 certs
 

README.md

@@ -1127,55 +1127,92 @@ s3_client.download_file(
 
 ## 13. Configuration
 
-### 13.1 Environment Variables
+BLOOM uses a YAML-based configuration system with environment variable overrides.
 
-Create a `.env` file with the following variables:
+### 13.1 Configuration Files
+
+**Configuration precedence** (highest to lowest):
+1. Environment variables with `BLOOM_*` prefix
+2. User config: `~/.config/bloom/bloom-config.yaml`
+3. Template defaults: `config/bloom-config-template.yaml`
+
+**Setup:**
 
 ```bash
+# Copy template to user config directory
+mkdir -p ~/.config/bloom
+cp config/bloom-config-template.yaml ~/.config/bloom/bloom-config.yaml
+
+# Edit with your settings
+$EDITOR ~/.config/bloom/bloom-config.yaml
+```
+
+### 13.2 Configuration Reference
+
+```yaml
+# ~/.config/bloom/bloom-config.yaml
+
+# Application settings
+environment: "development"  # development, staging, production, testing
+debug: false
+
 # Database
-BLOOM_DB_HOST=localhost
-BLOOM_DB_PORT=5432
-BLOOM_DB_NAME=bloom_lims
-BLOOM_DB_USER=bloom_user
-BLOOM_DB_PASSWORD=secure_password
-
-# Cognito (SSO)
-COGNITO_REGION=us-east-1
-COGNITO_USER_POOL_ID=us-east-1_XXXXXXXXX
-COGNITO_CLIENT_ID=your_app_client_id
-COGNITO_DOMAIN=your-custom-domain.auth.us-east-1.amazoncognito.com
-COGNITO_REDIRECT_URI=http://127.0.0.1:8000/
-COGNITO_LOGOUT_REDIRECT_URI=http://127.0.0.1:8000/
-COGNITO_WHITELIST_DOMAINS=all  # or comma-separated domains
-COGNITO_SCOPES="openid email profile"
-
-# AWS S3 (optional)
-AWS_ACCESS_KEY_ID=your-access-key
-AWS_SECRET_ACCESS_KEY=your-secret-key
-S3_BUCKET=bloom-files
-S3_REGION=us-east-1
-
-# FedEx API
-FEDEX_API_KEY=your-fedex-key
-FEDEX_SECRET=your-fedex-secret
-
-# Application
-FLASK_SECRET_KEY=your-flask-secret
-DEBUG=false
-LOG_LEVEL=INFO
+database:
+  host: "localhost"
+  port: 5445                    # Default BLOOM PostgreSQL port
+  database: "bloom"
+  user: "bloom"
+  password: ""                  # Leave empty for peer authentication
+
+# Authentication (Cognito)
+auth:
+  cognito_region: "us-east-1"
+  cognito_user_pool_id: "us-east-1_XXXXXXXXX"
+  cognito_client_id: "your_app_client_id"
+  cognito_domain: "your-domain.auth.us-east-1.amazoncognito.com"
+  cognito_redirect_uri: "http://127.0.0.1:8000/"
+  cognito_logout_redirect_uri: "http://127.0.0.1:8000/"
+  cognito_allowed_domains: []   # Empty = allow all
+
+# AWS settings
+aws:
+  profile: ""                   # AWS profile name (optional)
+  region: "us-west-2"
+
+# Storage (S3)
+storage:
+  s3_bucket: ""
+  s3_region: "us-east-1"
+
+# Logging
+logging:
+  level: "INFO"
 ```
 
-### 13.2 Database Connection String
+### 13.3 Environment Variable Overrides
+
+Override any YAML setting with environment variables using `BLOOM_` prefix and `__` for nesting:
+
+```bash
+export BLOOM_DATABASE__HOST=localhost
+export BLOOM_DATABASE__PORT=5445
+export BLOOM_AUTH__COGNITO_REGION=us-east-1
+export BLOOM_DEBUG=true
+```
+
+### 13.4 Database Connection
+
+The database URL is constructed automatically from configuration:
 
 ```python
-# Constructed from environment variables
-DATABASE_URL = f"postgresql://{BLOOM_DB_USER}:{BLOOM_DB_PASSWORD}@{BLOOM_DB_HOST}:{BLOOM_DB_PORT}/{BLOOM_DB_NAME}"
+from bloom_lims.config import get_settings
 
-# Or set directly
-DATABASE_URL = os.getenv("DATABASE_URL")
+settings = get_settings()
+db_url = settings.database.connection_string
+# postgresql://bloom@localhost:5445/bloom
 ```
 
-### 13.3 Printer Configuration
+### 13.5 Printer Configuration
 
 Printer configuration is stored in YAML files:
 
@@ -1206,7 +1243,7 @@ labs:
           ^XZ
 ```
 
-### 13.4 Assay Configuration
+### 13.6 Assay Configuration
 
 ```yaml
 # config/assay_config.yaml
@@ -1385,19 +1422,23 @@ cd bloom
 # This will attempt to build the conda env, install postgres, the database, build the schema and start postgres
 source bloom_lims/env/install_postgres.sh
 
-# conda activate BLOOM if it has not happened already.
+# Activate the BLOOM environment (use this for all future sessions)
+source bloom_activate.sh
 
 # Start the Bloom LIMS UI
-source run_bloomui.sh
+bloom gui
 ```
 
 ### (Optional) Install & Run pgadmin4 Database Admin UI
 
 ```bash
+# Ensure environment is activated first
+source bloom_activate.sh
+
 # RUN TESTS
 pytest
 
-# START THE UIs (on localhost:8080)
+# INSTALL pgadmin4 (on localhost:8080)
 source bloom_lims/env/install_pgadmin.sh
 ```
 
@@ -1406,7 +1447,7 @@ source bloom_lims/env/install_pgadmin.sh
 ## 16. Testing
 
 ```bash
-conda activate BLOOM
+source bloom_activate.sh
 pytest
 ```
 
@@ -1482,8 +1523,10 @@ All other relationships are subsets of this, and designing parts of the LIMS whi
 
 ## 19. Dev Tools
 
-__note:__ all commands below are expected to be run from a shell with conda activated.
-`conda activate BLOOM`
+__note:__ all commands below are expected to be run from a shell with the BLOOM environment activated:
+```bash
+source bloom_activate.sh
+```
 
 ### Drop The Entire Database (Loose All Data!) > Rebuild The Database / Re-seed With All Accessible JSON Templates
 
@@ -1495,7 +1538,7 @@ source clear_and_rebuild_postgres.sh
 ```
 
 #### Stop pgsql
-- `source bloom_lims/bin/stop_bloom_db.sh`
+- `bloom db stop` or `source bloom_lims/bin/stop_bloom_db.sh`
 
 #### Remove the db
 - `rm -rf bloom_lims/database/*`
@@ -1509,13 +1552,13 @@ Similar to `pytest`, but more extensive. Useful for development and smoke testin
 - Example: `python smoke_exams/accession_extract_qant.py 2 1` (runs 2 iterations with HLA-typing assay)
 
 #### Run the bloom UI
-- `source run_bloomui.sh`
+- `bloom gui` or `source run_bloomui.sh`
 
 #### Run the pgadmin UI
 - `source bloom_lims/env/install_pgadmin.sh`
 
 #### Start Interactive Shell w/Core Bloom Objects Instantiated
-`python bloom_shell.py`
+`bloom shell` or `python bloom_shell.py`
 
 ### Random Notes
 

analytics/README.md

@@ -51,30 +51,32 @@ docker-compose up -d
 
 ```bash
 # If using conda, Metabase can run alongside BLOOM
-conda activate BLOOM
+source bloom_activate.sh
 ./analytics/metabase/run_metabase.sh
 ```
 
 ## Configuration
 
 ### Environment Variables
 
-Add to your `.env` file:
+Set these environment variables before running Metabase:
 
 ```bash
 # Metabase Configuration
-METABASE_PORT=3000
-METABASE_DB_TYPE=postgres
-METABASE_DB_HOST=localhost
-METABASE_DB_PORT=5432
-METABASE_DB_NAME=bloom_lims
-METABASE_DB_USER=bloom_user
-METABASE_DB_PASS=your_password
+export METABASE_PORT=3000
+export METABASE_DB_TYPE=postgres
+export METABASE_DB_HOST=localhost
+export METABASE_DB_PORT=5445
+export METABASE_DB_NAME=bloom
+export METABASE_DB_USER=bloom
+export METABASE_DB_PASS=your_password
 
 # Optional: Embed dashboards in BLOOM
-METABASE_EMBED_SECRET=your-embed-secret-key
+export METABASE_EMBED_SECRET=your-embed-secret-key
 ```
 
+These can also be added to your shell profile (`~/.zshrc` or `~/.bashrc`).
+
 ## SQL Views for Analytics
 
 The `sql_views/` directory contains PostgreSQL views optimized for analytics:

analytics/metabase/run_metabase.sh

@@ -62,13 +62,8 @@ export MB_JETTY_PORT="$METABASE_PORT"
 export MB_SITE_NAME="BLOOM LIMS Analytics"
 export MB_ENABLE_EMBEDDING=true
 
-# Load .env if present
-if [ -f "$SCRIPT_DIR/../../.env" ]; then
-    echo -e "${YELLOW}Loading environment from .env${NC}"
-    set -a
-    source "$SCRIPT_DIR/../../.env"
-    set +a
-fi
+# Note: Configure Metabase via environment variables before running this script.
+# See analytics/README.md for configuration details.
 
 echo -e "${GREEN}Starting Metabase on port $METABASE_PORT...${NC}"
 echo -e "${YELLOW}First startup may take 1-2 minutes to initialize.${NC}"