diff --git a/ENVIRONMENT.rst b/ENVIRONMENT.rst index ea2e3df81..054f3b517 100644 --- a/ENVIRONMENT.rst +++ b/ENVIRONMENT.rst @@ -93,6 +93,7 @@ Environment Configuration Settings - **LOG_S3_TAGS**: map of key value pairs to be used for tagging files uploaded to S3. Values should be referencing existing environment variables e.g. ``{"ClusterName": "SCOPE", "Namespace": "POD_NAMESPACE"}`` - **LOG_SHIP_HOURLY**: if true, log rotation in Postgres is set to 1h incl. foreign tables for every hour (schedule `1 */1 * * *`) - **LOG_SHIP_SCHEDULE**: cron schedule for shipping compressed logs from ``pg_log`` (``1 0 * * *`` by default) +- **LOG_FILE_EXTENSION**: (optional) extension of the files in PGDATA/pgroot/pg_log to be shipped to S3 (csv by default) - **LOG_ENV_DIR**: directory to store environment variables necessary for log shipping - **LOG_TMPDIR**: directory to store temporary compressed daily log files. PGROOT/../tmp by default. - **LOG_S3_ENDPOINT**: (optional) S3 Endpoint to use with Boto3 diff --git a/postgres-appliance/scripts/configure_spilo.py b/postgres-appliance/scripts/configure_spilo.py index 851af037c..a655e97d0 100755 --- a/postgres-appliance/scripts/configure_spilo.py +++ b/postgres-appliance/scripts/configure_spilo.py @@ -595,6 +595,7 @@ def get_placeholders(provider): placeholders.setdefault('LOG_GROUP_BY_DATE', False) placeholders.setdefault('LOG_SHIP_HOURLY', '') placeholders.setdefault('LOG_SHIP_SCHEDULE', '1 0 * * *') + placeholders.setdefault('LOG_FILE_EXTENSION', 'csv') placeholders.setdefault('LOG_S3_BUCKET', '') placeholders.setdefault('LOG_S3_ENDPOINT', '') placeholders.setdefault('LOG_S3_TAGS', '{}') @@ -802,7 +803,8 @@ def write_log_environment(placeholders): 'LOG_S3_KEY', 'LOG_S3_BUCKET', 'LOG_S3_TAGS', - 'PGLOG'): + 'PGLOG', + 'LOG_FILE_EXTENSION'): write_file(log_env[var], os.path.join(log_env['LOG_ENV_DIR'], var), True) diff --git a/postgres-appliance/scripts/upload_pg_log_to_s3.py b/postgres-appliance/scripts/upload_pg_log_to_s3.py index edc57e706..ced2304b5 100755 --- a/postgres-appliance/scripts/upload_pg_log_to_s3.py +++ b/postgres-appliance/scripts/upload_pg_log_to_s3.py @@ -26,8 +26,8 @@ def get_file_names(): prev_interval_number = prev_interval.strftime('%u-%H') upload_filename = prev_interval.strftime('%F-%H') - log_file = os.path.join(os.getenv('PGLOG'), 'postgresql-' + prev_interval_number + '.csv') - archived_log_file = os.path.join(os.getenv('LOG_TMPDIR'), upload_filename + '.csv.gz') + log_file = os.path.join(os.getenv('PGLOG'), 'postgresql-' + prev_interval_number + '.' + os.getenv('LOG_FILE_EXTENSION')) + archived_log_file = os.path.join(os.getenv('LOG_TMPDIR'), upload_filename + '.' + os.getenv('LOG_FILE_EXTENSION') + '.gz') return log_file, archived_log_file