From 683422cadeaea50128c74f8003f56fc41cabb6fd Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Mon, 27 Jul 2026 02:23:50 +0300 Subject: [PATCH] PMM-15238: Expose the built-in PostgreSQL to SEP Add an opt-in integration that lets SEP, running in a side container on a shared Docker bridge network, use PMM's embedded PostgreSQL for its persistence layer. When PMM_ENABLE_SEP is set, the entrypoint appends marker-delimited blocks to postgresql.conf (listen_addresses) and pg_hba.conf (one scram-sha-256 rule per attached Docker subnet, scoped to the sep database and role), then provisions a non-superuser sep role owning a dedicated sep database. Nothing is published on the host, and the postgres, pmm-managed and grafana accounts remain unreachable over the network. Unsetting the variable reverts the configuration on the next start and leaves the role and database intact. The postgres data directory, password file and binary directory are now declared once in the entrypoint and passed to the helper scripts via a subshell-scoped export, replacing the /usr/pgsql-14 literals that were duplicated across them. --- .env.example | 7 + .../roles/postgres/files/postgres-migration | 17 +-- .../ansible/roles/postgres/files/postgres-sep | 122 ++++++++++++++++++ build/docker/server/entrypoint.sh | 21 ++- docker-compose.yml | 3 + managed/utils/envvars/parser.go | 3 + managed/utils/envvars/parser_test.go | 12 ++ 7 files changed, 172 insertions(+), 13 deletions(-) create mode 100755 build/ansible/roles/postgres/files/postgres-sep diff --git a/.env.example b/.env.example index bca53acba2e..ec2f80a80bb 100644 --- a/.env.example +++ b/.env.example @@ -43,6 +43,13 @@ PMM_PORT_HTTPS=443 # PMM_POSTGRES_SSL_CERT_PATH=/tmp/certs/pmm-managed.crt # PMM_DISABLE_BUILTIN_POSTGRES=1 +# Expose the built-in PostgreSQL to SEP running in a side container on the same bridge +# network. PMM creates a dedicated `sep` database owned by a non-superuser `sep` role and +# accepts connections for it from the container's Docker subnets only; nothing is +# published on the host. Both variables are required to enable it. +# PMM_ENABLE_SEP=1 +# PMM_SEP_POSTGRES_PASSWORD= + # Use SSL certificates for PMM Server's internal database connection (PostgreSQL) # GF_DATABASE_SSL_MODE=verify-full # GF_DATABASE_CA_CERT_PATH=/tmp/certs/root.crt diff --git a/build/ansible/roles/postgres/files/postgres-migration b/build/ansible/roles/postgres/files/postgres-migration index 7ed3846fa4d..52c8c61de9a 100755 --- a/build/ansible/roles/postgres/files/postgres-migration +++ b/build/ansible/roles/postgres/files/postgres-migration @@ -2,8 +2,9 @@ set -o errexit set -o pipefail -declare POSTGRES_DATA_DIR="${POSTGRES_DATA_DIR:-/srv/postgres14}" -declare POSTGRES_PASSWORD_FILE="${POSTGRES_PASSWORD_FILE:-/srv/.postgres_password}" +declare POSTGRES_DATA_DIR="${POSTGRES_DATA_DIR:?must be exported by the entrypoint}" +declare POSTGRES_PASSWORD_FILE="${POSTGRES_PASSWORD_FILE:?must be exported by the entrypoint}" +declare POSTGRES_BIN_DIR="${POSTGRES_BIN_DIR:?must be exported by the entrypoint}" ensure_postgres_password() { # This check is to verify if the data directory is empty. @@ -20,19 +21,19 @@ ensure_postgres_password() { echo "Generating postgres superuser password..." POSTGRES_PASSWORD=$(openssl rand -hex 16) - if ! /usr/pgsql-14/bin/pg_ctl status -D "$POSTGRES_DATA_DIR" > /dev/null 2>&1; then - /usr/pgsql-14/bin/pg_ctl start -D "$POSTGRES_DATA_DIR" -o "-c logging_collector=off" + if ! "$POSTGRES_BIN_DIR/pg_ctl" status -D "$POSTGRES_DATA_DIR" > /dev/null 2>&1; then + "$POSTGRES_BIN_DIR/pg_ctl" start -D "$POSTGRES_DATA_DIR" -o "-c logging_collector=off" STARTED=1 fi - PGPASSWORD="$POSTGRES_PASSWORD" /usr/bin/psql -U postgres -h /run/postgresql -d postgres \ + PGPASSWORD="$POSTGRES_PASSWORD" "$POSTGRES_BIN_DIR/psql" -U postgres -h /run/postgresql -d postgres \ -c "ALTER USER postgres WITH PASSWORD '${POSTGRES_PASSWORD}'" echo -n "$POSTGRES_PASSWORD" > "$POSTGRES_PASSWORD_FILE" chmod 600 "$POSTGRES_PASSWORD_FILE" if [ "$STARTED" -eq 1 ]; then - /usr/pgsql-14/bin/pg_ctl stop -D "$POSTGRES_DATA_DIR" + "$POSTGRES_BIN_DIR/pg_ctl" stop -D "$POSTGRES_DATA_DIR" fi } @@ -53,8 +54,8 @@ update_pg_hba_auth() { sed -E 's/^([[:space:]]*host[[:space:]].*[[:space:]])trust([[:space:]]*)$/\1scram-sha-256\2/' "$hba" > "$tmp" mv "$tmp" "$hba" - if /usr/pgsql-14/bin/pg_ctl status -D "$POSTGRES_DATA_DIR" > /dev/null 2>&1; then - /usr/pgsql-14/bin/pg_ctl reload -D "$POSTGRES_DATA_DIR" + if "$POSTGRES_BIN_DIR/pg_ctl" status -D "$POSTGRES_DATA_DIR" > /dev/null 2>&1; then + "$POSTGRES_BIN_DIR/pg_ctl" reload -D "$POSTGRES_DATA_DIR" fi } diff --git a/build/ansible/roles/postgres/files/postgres-sep b/build/ansible/roles/postgres/files/postgres-sep new file mode 100755 index 00000000000..35af3dfd0c8 --- /dev/null +++ b/build/ansible/roles/postgres/files/postgres-sep @@ -0,0 +1,122 @@ +#!/bin/bash +# +# Makes the embedded PostgreSQL reachable by SEP running in a side container on a +# shared Docker network, and provisions a dedicated low-privilege role/database for it. +# +# Everything below is a no-op unless PMM_ENABLE_SEP is set. When it is unset again, +# the configuration this script added is reverted on the next start; the sep role and +# database are left untouched. + +set -o errexit +set -o pipefail + +declare POSTGRES_DATA_DIR="${POSTGRES_DATA_DIR:?must be exported by the entrypoint}" +declare POSTGRES_PASSWORD_FILE="${POSTGRES_PASSWORD_FILE:?must be exported by the entrypoint}" +declare POSTGRES_BIN_DIR="${POSTGRES_BIN_DIR:?must be exported by the entrypoint}" +declare PG_CONF="$POSTGRES_DATA_DIR/postgresql.conf" +declare PG_HBA="$POSTGRES_DATA_DIR/pg_hba.conf" +declare BEGIN_MARKER="# BEGIN PMM SEP" +declare END_MARKER="# END PMM SEP" + +is_enabled() { [ "$1" = "1" ] || [ "$1" = "true" ]; } + +# Replaces the marker-delimited block of $1 with $2, appending it at the end of the +# file so it takes precedence over anything set earlier. An empty $2 drops the block. +write_block() { + local file="$1" body="$2" tmp + + if [ ! -f "$file" ]; then + echo "FATAL: $file not found." >&2 + exit 1 + fi + + if [ -z "$body" ] && ! grep -q "^${BEGIN_MARKER}$" "$file"; then + return + fi + + tmp=$(mktemp "$POSTGRES_DATA_DIR/.pmm-sep.XXXXXX") + sed "/^${BEGIN_MARKER}$/,/^${END_MARKER}$/d" "$file" > "$tmp" + if [ -n "$body" ]; then + { + echo "$BEGIN_MARKER" + echo "$body" + echo "$END_MARKER" + } >> "$tmp" + fi + mv "$tmp" "$file" +} + +# Prints the IPv4 CIDR of every Docker network this container is attached to. +container_subnets() { + ip -o -4 route show scope link | awk '$1 ~ /\// && $1 !~ /^127\./ { print $1 }' +} + +psql_postgres() { + "$POSTGRES_BIN_DIR/psql" -X -q -v ON_ERROR_STOP=1 -U postgres -h /run/postgresql -d postgres "$@" +} + +provision_sep_role() { + local started=0 role_exists db_exists + + if ! "$POSTGRES_BIN_DIR/pg_ctl" status -D "$POSTGRES_DATA_DIR" > /dev/null 2>&1; then + "$POSTGRES_BIN_DIR/pg_ctl" start -D "$POSTGRES_DATA_DIR" -o "-c logging_collector=off -c listen_addresses=" + started=1 + fi + + PGPASSWORD=$(< "$POSTGRES_PASSWORD_FILE") + export PGPASSWORD + + role_exists=$(psql_postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname = 'sep'") + if [ -z "$role_exists" ]; then + psql_postgres -c "CREATE ROLE sep" + fi + + # Re-applied on every start so rotating PMM_SEP_POSTGRES_PASSWORD takes effect. + # Fed via stdin, not -c: psql only interpolates :'sep_password' when reading a script. + psql_postgres -v sep_password="$PMM_SEP_POSTGRES_PASSWORD" <<'SQL' +ALTER ROLE sep WITH LOGIN NOSUPERUSER NOCREATEROLE NOCREATEDB PASSWORD :'sep_password'; +SQL + + db_exists=$(psql_postgres -tAc "SELECT 1 FROM pg_database WHERE datname = 'sep'") + if [ -z "$db_exists" ]; then + psql_postgres -c "CREATE DATABASE sep OWNER sep" + fi + + unset PGPASSWORD + + if [ "$started" -eq 1 ]; then + "$POSTGRES_BIN_DIR/pg_ctl" stop -D "$POSTGRES_DATA_DIR" + fi +} + +if ! is_enabled "$PMM_ENABLE_SEP"; then + write_block "$PG_CONF" "" + write_block "$PG_HBA" "" + exit 0 +fi + +if [ -z "$PMM_SEP_POSTGRES_PASSWORD" ]; then + echo "FATAL: PMM_ENABLE_SEP is set but PMM_SEP_POSTGRES_PASSWORD is empty." >&2 + echo "Please set PMM_SEP_POSTGRES_PASSWORD to the password SEP will connect with and try again." >&2 + exit 1 +fi + +declare -a SUBNETS +mapfile -t SUBNETS < <(container_subnets) +if [ ${#SUBNETS[@]} -eq 0 ]; then + echo "FATAL: PMM_ENABLE_SEP is set but this container is not attached to any network." >&2 + echo "Please attach pmm-server to the bridge network shared with SEP and try again." >&2 + exit 1 +fi + +echo "Exposing PostgreSQL to SEP on ${SUBNETS[*]}..." + +write_block "$PG_CONF" "listen_addresses = '*'" + +declare HBA_BODY="" +for subnet in "${SUBNETS[@]}"; do + HBA_BODY+="host sep sep ${subnet} scram-sha-256"$'\n' +done +write_block "$PG_HBA" "${HBA_BODY%$'\n'}" + +provision_sep_role diff --git a/build/docker/server/entrypoint.sh b/build/docker/server/entrypoint.sh index 737cc2db37a..ec3f21c9cf2 100755 --- a/build/docker/server/entrypoint.sh +++ b/build/docker/server/entrypoint.sh @@ -8,6 +8,7 @@ declare CURRENT_GID CURRENT_UID CURRENT_USER is_enabled() { [ "$1" = "1" ] || [ "$1" = "true" ]; } declare POSTGRES_DATA_DIR="/srv/postgres14" declare POSTGRES_PASSWORD_FILE="/srv/.postgres_password" +declare POSTGRES_BIN_DIR="/usr/pgsql-14/bin" # Get current user info - handle cases where user doesn't exist in passwd CURRENT_UID=$(id -u) @@ -97,12 +98,12 @@ if [ ! -f "$DIST_FILE" ]; then chmod 600 "$POSTGRES_PASSWORD_FILE" # Initialize database with password authentication - /usr/pgsql-14/bin/initdb -D "$POSTGRES_DATA_DIR" --auth-host=scram-sha-256 --auth-local=trust --username=postgres --pwfile="$POSTGRES_PASSWORD_FILE" + "$POSTGRES_BIN_DIR/initdb" -D "$POSTGRES_DATA_DIR" --auth-host=scram-sha-256 --auth-local=trust --username=postgres --pwfile="$POSTGRES_PASSWORD_FILE" echo "Enabling pg_stat_statements extension for PostgreSQL..." - /usr/pgsql-14/bin/pg_ctl start -D "$POSTGRES_DATA_DIR" -o "-c logging_collector=off" - PGPASSWORD="$POSTGRES_PASSWORD" /usr/bin/psql -U postgres -h /run/postgresql -d postgres -c 'CREATE EXTENSION pg_stat_statements SCHEMA public' - /usr/pgsql-14/bin/pg_ctl stop -D "$POSTGRES_DATA_DIR" + "$POSTGRES_BIN_DIR/pg_ctl" start -D "$POSTGRES_DATA_DIR" -o "-c logging_collector=off" + PGPASSWORD="$POSTGRES_PASSWORD" "$POSTGRES_BIN_DIR/psql" -U postgres -h /run/postgresql -d postgres -c 'CREATE EXTENSION pg_stat_statements SCHEMA public' + "$POSTGRES_BIN_DIR/pg_ctl" stop -D "$POSTGRES_DATA_DIR" # Clean up password from environment unset POSTGRES_PASSWORD @@ -145,7 +146,17 @@ elif is_enabled "$PMM_DISABLE_BUILTIN_POSTGRES"; then else mkdir -p /run/postgresql chmod 750 "$POSTGRES_DATA_DIR" || true - bash /opt/ansible/roles/postgres/files/postgres-migration + # Scoped to this subshell so the helper scripts inherit them without polluting + # the environment that supervisord and its children are started with. + ( + export POSTGRES_DATA_DIR POSTGRES_PASSWORD_FILE POSTGRES_BIN_DIR + bash /opt/ansible/roles/postgres/files/postgres-migration + bash /opt/ansible/roles/postgres/files/postgres-sep + ) +fi + +if is_enabled "$PMM_ENABLE_SEP" && { is_enabled "$PMM_HA_ENABLE" || is_enabled "$PMM_DISABLE_BUILTIN_POSTGRES"; }; then + echo "WARNING: ignoring PMM_ENABLE_SEP, the embedded PostgreSQL is not in use." >&2 fi echo "Generating self-signed certificates for nginx..." diff --git a/docker-compose.yml b/docker-compose.yml index cb836b6b741..08153263ef9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,6 +44,9 @@ services: - PMM_POSTGRES_USERNAME - PMM_POSTGRES_DBPASSWORD - PMM_DISABLE_BUILTIN_POSTGRES + # To expose the built-in PostgreSQL to SEP running on the same bridge network + - PMM_ENABLE_SEP + - PMM_SEP_POSTGRES_PASSWORD # To discover RDS databases via UI - AWS_ACCESS_KEY - AWS_SECRET_KEY diff --git a/managed/utils/envvars/parser.go b/managed/utils/envvars/parser.go index fb78164b65a..3726771ce70 100644 --- a/managed/utils/envvars/parser.go +++ b/managed/utils/envvars/parser.go @@ -120,6 +120,9 @@ func ParseEnvVars(envs []string) (*models.ChangeSettingsParams, []error, []strin "PMM_DISABLE_BUILTIN_POSTGRES": // skip env variables for external postgres continue + case "PMM_ENABLE_SEP", "PMM_SEP_POSTGRES_PASSWORD": + // skip env variables consumed by the entrypoint to expose postgres to SEP + continue case "PERCONA_TELEMETRY_DISABLE": // skip the Pillars telemetry environment variable continue diff --git a/managed/utils/envvars/parser_test.go b/managed/utils/envvars/parser_test.go index 24211d072c7..6e48f2c617d 100644 --- a/managed/utils/envvars/parser_test.go +++ b/managed/utils/envvars/parser_test.go @@ -73,6 +73,18 @@ func TestEnvVarValidator(t *testing.T) { assert.Equal(t, expectedWarns, gotWarns) }) + t.Run("SEP env variables", func(t *testing.T) { + t.Parallel() + + envs := []string{"PMM_ENABLE_SEP=1", "PMM_SEP_POSTGRES_PASSWORD=s3cr3t"} + expectedEnvVars := &models.ChangeSettingsParams{} + + gotEnvVars, gotErrs, gotWarns := ParseEnvVars(envs) + assert.Equal(t, expectedEnvVars, gotEnvVars) + assert.Nil(t, gotErrs) + assert.Nil(t, gotWarns) + }) + t.Run("Default env vars", func(t *testing.T) { t.Parallel()