diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml new file mode 100644 index 0000000..e9479f3 --- /dev/null +++ b/.github/workflows/publish-release.yml @@ -0,0 +1,410 @@ +name: publish-release + +on: + push: + tags: + - v* + workflow_dispatch: + inputs: + release_tag: + description: Release tag to publish, for example v0.1.0 or v0.1.0-beta.1 + required: true + type: string + share_scope: + description: Keep the SAR app private to the account or share it privately across the AWS Organization + required: false + default: account + type: choice + options: + - account + - organization + +permissions: {} + +env: + AWS_REGION: us-east-1 + SAM_CLI_TELEMETRY: "0" + RELEASE_ROOT: release + AWS_SAM_CLI_VERSION: "1.157.1" + AWS_SAM_CLI_LINUX_ARM64_SHA256: "c32cc1f7f8b1d794eaf223489e12a60b44a58517b58c7248a8b21e95e25e844f" + CARGO_LAMBDA_VERSION: "1.9.1" + ZIG_VERSION: "0.14.1" + ZIG_AARCH64_LINUX_SHA256: "f7a654acc967864f7a050ddacfaa778c7504a0eca8d2b678839c21eea47c992b" + +jobs: + publish: + name: Build and publish the SAR bootstrap app + runs-on: ubuntu-24.04-arm + permissions: + contents: read + id-token: write + + steps: + - name: Resolve release ref + id: release_ref + shell: bash + env: + RELEASE_TAG_INPUT: ${{ inputs.release_tag }} + run: | + set -euo pipefail + + if [[ "$GITHUB_EVENT_NAME" == "push" ]]; then + tag="$GITHUB_REF_NAME" + checkout_ref="$GITHUB_SHA" + else + tag="$RELEASE_TAG_INPUT" + checkout_ref="refs/tags/$tag" + fi + + if [[ ! "$tag" =~ ^v ]]; then + echo "::error::Release tags must start with 'v'." + exit 1 + fi + + { + echo "tag=$tag" + echo "version=${tag#v}" + echo "checkout_ref=$checkout_ref" + } >> "$GITHUB_OUTPUT" + + - name: Check out the release ref + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + persist-credentials: false + ref: ${{ steps.release_ref.outputs.checkout_ref }} + + - name: Validate release version metadata + shell: bash + env: + EXPECTED_VERSION: ${{ steps.release_ref.outputs.version }} + run: | + set -euo pipefail + + semver_regex='^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*|[0-9A-Za-z-][0-9A-Za-z-]*)(\.(0|[1-9][0-9]*|[0-9A-Za-z-][0-9A-Za-z-]*))*))?(\+([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' + if [[ ! "$EXPECTED_VERSION" =~ $semver_regex ]]; then + echo "::error::Release tags must use vX.Y.Z or an allowed prerelease form such as vX.Y.Z-beta.1." + exit 1 + fi + + file_version="$(tr -d '[:space:]' < VERSION)" + if [[ "$file_version" != "$EXPECTED_VERSION" ]]; then + echo "::error::VERSION ($file_version) does not match release tag version $EXPECTED_VERSION." + exit 1 + fi + + sam_version_raw="$( + sed -nE 's/^[[:space:]]*SemanticVersion:[[:space:]]*(.+)$/\1/p' bootstrap/template.yaml | head -n1 + )" + sam_version="$(scripts/normalize-yaml-scalar.sh "$sam_version_raw")" + if [[ "$sam_version" != "$EXPECTED_VERSION" ]]; then + echo "::error::bootstrap/template.yaml SemanticVersion ($sam_version) does not match release tag version $EXPECTED_VERSION." + exit 1 + fi + + - name: Validate sharing scope + shell: bash + env: + SHARE_SCOPE: ${{ inputs.share_scope || 'account' }} + run: | + set -euo pipefail + + case "$SHARE_SCOPE" in + account|organization) ;; + *) + echo "::error::share_scope must be account or organization." + exit 1 + ;; + esac + + - name: Set up Rust + shell: bash + run: | + set -euo pipefail + + rustup toolchain install stable --profile minimal + rustup default stable + rustc --version + cargo --version + + - name: Install cargo-lambda + shell: bash + run: | + set -euo pipefail + + cargo install cargo-lambda --locked --version "$CARGO_LAMBDA_VERSION" --force + + installed_version="$(cargo lambda --version | awk '{print $2}')" + if [[ "$installed_version" != "$CARGO_LAMBDA_VERSION" ]]; then + echo "::error::Expected cargo-lambda $CARGO_LAMBDA_VERSION but found $installed_version." + exit 1 + fi + + - name: Install Zig + shell: bash + run: | + set -euo pipefail + + install_dir="$HOME/.local/zig/$ZIG_VERSION" + if [[ -x "$install_dir/zig" ]]; then + if installed_version="$(PATH="$install_dir:$PATH" zig version 2>/dev/null)"; then + if [[ "$installed_version" == "$ZIG_VERSION" ]]; then + echo "$install_dir" >> "$GITHUB_PATH" + exit 0 + fi + fi + fi + + archive="zig-aarch64-linux-${ZIG_VERSION}.tar.xz" + base_url="https://ziglang.org/download/${ZIG_VERSION}" + extract_dir="zig-aarch64-linux-${ZIG_VERSION}" + tmp_dir="$(mktemp -d)" + trap 'rm -rf "$tmp_dir"' EXIT + + curl -fsSL --retry 5 --retry-all-errors --retry-delay 2 "$base_url/$archive" -o "$tmp_dir/$archive" + echo "$ZIG_AARCH64_LINUX_SHA256 $tmp_dir/$archive" | sha256sum -c - + + tar -xJf "$tmp_dir/$archive" -C "$tmp_dir" + mkdir -p "$(dirname "$install_dir")" + rm -rf "$install_dir" + mv "$tmp_dir/$extract_dir" "$install_dir" + echo "$install_dir" >> "$GITHUB_PATH" + + installed_version="$(PATH="$install_dir:$PATH" zig version)" + if [[ "$installed_version" != "$ZIG_VERSION" ]]; then + echo "::error::Expected zig $ZIG_VERSION but found $installed_version." + exit 1 + fi + + - name: Build the gateway Lambda zip + shell: bash + run: cargo lambda build --release --arm64 --output-format zip -p khone-gateway + + - name: Install AWS SAM CLI + shell: bash + run: | + set -euo pipefail + + archive="aws-sam-cli-linux-arm64.zip" + base_url="https://github.com/aws/aws-sam-cli/releases/download/v${AWS_SAM_CLI_VERSION}" + install_dir="$HOME/.local/aws-sam-cli" + bin_dir="$HOME/.local/bin" + tmp_dir="$(mktemp -d)" + trap 'rm -rf "$tmp_dir"' EXIT + + curl -fsSL --retry 5 --retry-all-errors --retry-delay 2 "$base_url/$archive" -o "$tmp_dir/$archive" + echo "$AWS_SAM_CLI_LINUX_ARM64_SHA256 $tmp_dir/$archive" | sha256sum -c - + + unzip -q "$tmp_dir/$archive" -d "$tmp_dir" + rm -rf "$install_dir" + rm -f "$bin_dir/sam" + "$tmp_dir/install" --bin-dir "$bin_dir" --install-dir "$install_dir" + + export PATH="$bin_dir:$PATH" + echo "$bin_dir" >> "$GITHUB_PATH" + + installed_version="$(sam --version | sed -nE 's/^SAM CLI, version ([^[:space:]]+)$/\1/p')" + if [[ "$installed_version" != "$AWS_SAM_CLI_VERSION" ]]; then + echo "::error::Expected AWS SAM CLI $AWS_SAM_CLI_VERSION but found ${installed_version:-unknown}." + exit 1 + fi + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1 + with: + aws-region: ${{ env.AWS_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + + - name: Upload versioned gateway artifact + id: gateway_artifact + shell: bash + env: + GATEWAY_ARTIFACT_BUCKET: ${{ secrets.GATEWAY_ARTIFACT_BUCKET }} + SAR_ARTIFACT_BUCKET: ${{ secrets.SAR_ARTIFACT_BUCKET }} + VERSION: ${{ steps.release_ref.outputs.version }} + run: | + set -euo pipefail + + gateway_bucket="${GATEWAY_ARTIFACT_BUCKET:-$SAR_ARTIFACT_BUCKET}" + if [[ -z "$gateway_bucket" ]]; then + echo "::error::Set GATEWAY_ARTIFACT_BUCKET or SAR_ARTIFACT_BUCKET for release artifacts." + exit 1 + fi + + zip_path="target/lambda/khone-gateway/bootstrap.zip" + if [[ ! -f "$zip_path" ]]; then + echo "::error::Expected cargo-lambda output at $zip_path." + exit 1 + fi + + gateway_key="khone/releases/$VERSION/gateway/bootstrap.zip" + aws s3 cp "$zip_path" "s3://$gateway_bucket/$gateway_key" \ + --metadata "khone-version=$VERSION" + + object_version="$( + aws s3api head-object \ + --bucket "$gateway_bucket" \ + --key "$gateway_key" \ + --query VersionId \ + --output text + )" + case "$object_version" in + None|null) object_version="" ;; + esac + + { + echo "bucket=$gateway_bucket" + echo "key=$gateway_key" + echo "object_version=$object_version" + } >> "$GITHUB_OUTPUT" + + - name: Render release bootstrap template + id: release_template + shell: bash + env: + VERSION: ${{ steps.release_ref.outputs.version }} + GATEWAY_BUCKET: ${{ steps.gateway_artifact.outputs.bucket }} + GATEWAY_KEY: ${{ steps.gateway_artifact.outputs.key }} + GATEWAY_OBJECT_VERSION: ${{ steps.gateway_artifact.outputs.object_version }} + run: | + set -euo pipefail + + output="bootstrap/template.release.yaml" + python3 scripts/render-bootstrap-release-template.py \ + --template bootstrap/template.yaml \ + --output "$output" \ + --version "$VERSION" \ + --gateway-code-s3-bucket "$GATEWAY_BUCKET" \ + --gateway-code-s3-key "$GATEWAY_KEY" \ + --gateway-code-s3-object-version "$GATEWAY_OBJECT_VERSION" + + echo "path=$output" >> "$GITHUB_OUTPUT" + + - name: Validate rendered SAR template + shell: bash + run: sam validate --template-file "${{ steps.release_template.outputs.path }}" --lint + + - name: Build the SAR application + shell: bash + run: sam build --template-file "${{ steps.release_template.outputs.path }}" --beta-features + + - name: Package the SAR application + id: packaged_template + shell: bash + env: + PACKAGE_BUCKET: ${{ secrets.SAR_ARTIFACT_BUCKET }} + VERSION: ${{ steps.release_ref.outputs.version }} + run: | + set -euo pipefail + + if [[ -z "${PACKAGE_BUCKET:-}" ]]; then + echo "::error::SAR_ARTIFACT_BUCKET secret is required." + exit 1 + fi + + artifact_dir="$RELEASE_ROOT/$VERSION/bootstrap" + mkdir -p "$artifact_dir" + packaged_template="$artifact_dir/packaged.yaml" + + sam package \ + --template-file .aws-sam/build/template.yaml \ + --s3-bucket "$PACKAGE_BUCKET" \ + --s3-prefix "khone/releases/$VERSION/bootstrap" \ + --output-template-file "$packaged_template" + + packaged_semantic_version_raw="$( + sed -nE 's/^[[:space:]]*SemanticVersion:[[:space:]]*(.+)$/\1/p' "$packaged_template" | head -n1 + )" + packaged_readme_url_raw="$( + sed -nE 's/^[[:space:]]*ReadmeUrl:[[:space:]]*(.+)$/\1/p' "$packaged_template" | head -n1 + )" + packaged_semantic_version="$(scripts/normalize-yaml-scalar.sh "$packaged_semantic_version_raw")" + packaged_readme_url="$(scripts/normalize-yaml-scalar.sh "$packaged_readme_url_raw")" + + if [[ "$packaged_semantic_version" != "$VERSION" ]]; then + echo "::error::Packaged template SemanticVersion '$packaged_semantic_version' does not match $VERSION." + exit 1 + fi + + if [[ ! "$packaged_readme_url" =~ ^s3:// ]]; then + echo "::error::Packaged template ReadmeUrl must be an s3:// URL, got '$packaged_readme_url'." + exit 1 + fi + + echo "path=$packaged_template" >> "$GITHUB_OUTPUT" + + - name: Resolve SAR application identity + id: sar_application + shell: bash + run: | + set -euo pipefail + + app_name_raw="$( + sed -nE '/AWS::ServerlessRepo::Application:/,/^[^[:space:]]/ s/^[[:space:]]*Name:[[:space:]]*(.+)$/\1/p' bootstrap/template.yaml | head -n1 + )" + app_name="$(scripts/normalize-yaml-scalar.sh "$app_name_raw")" + if [[ -z "$app_name" ]]; then + echo "::error::bootstrap/template.yaml is missing Metadata.AWS::ServerlessRepo::Application.Name." + exit 1 + fi + + account_id="$(aws sts get-caller-identity --query Account --output text)" + echo "application_id=arn:aws:serverlessrepo:${AWS_REGION}:${account_id}:applications/${app_name}" >> "$GITHUB_OUTPUT" + + - name: Publish to Serverless Application Repository + shell: bash + env: + VERSION: ${{ steps.release_ref.outputs.version }} + TEMPLATE_PATH: ${{ steps.packaged_template.outputs.path }} + run: | + set -euo pipefail + + sam publish \ + --template "$TEMPLATE_PATH" \ + --region "$AWS_REGION" \ + --semantic-version "$VERSION" + + - name: Share the published application with the AWS Organization + if: ${{ inputs.share_scope == 'organization' }} + shell: bash + env: + APPLICATION_ID: ${{ steps.sar_application.outputs.application_id }} + run: | + set -euo pipefail + + org_id="$( + aws organizations describe-organization \ + --query 'Organization.Id' \ + --output text + )" + if [[ ! "$org_id" =~ ^o-[a-z0-9]{10,32}$ ]]; then + echo "::error::Discovered AWS Organization ID is invalid: '$org_id'." + exit 1 + fi + + existing_statements="$( + aws serverlessrepo get-application-policy \ + --region "$AWS_REGION" \ + --application-id "$APPLICATION_ID" \ + --query 'Statements' \ + --output json + )" + statements="$( + jq -cn --argjson existing "$existing_statements" --arg org_id "$org_id" ' + ($existing // []) + | map(select(.StatementId != "share-org")) + + [ + { + StatementId: "share-org", + Actions: ["Deploy", "UnshareApplication"], + PrincipalOrgIDs: [$org_id], + Principals: ["*"] + } + ] + ' + )" + + aws serverlessrepo put-application-policy \ + --region "$AWS_REGION" \ + --application-id "$APPLICATION_ID" \ + --statements "$statements" diff --git a/.gitignore b/.gitignore index d077de1..eec02ff 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,8 @@ __pycache__/ # AWS SAM .aws-sam/ +/bootstrap/template.release.yaml +/release/ # Benchmark output benchmark-results/ diff --git a/Makefile b/Makefile index d4ff28a..e12b9b4 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,9 @@ SAM_DEPLOY_FLAGS ?= --resolve-s3 --capabilities CAPABILITY_IAM --no-confirm-chan BOOTSTRAP_STACK_NAME ?= khone-bootstrap BOOTSTRAP_TEMPLATE ?= bootstrap/template.yaml BOOTSTRAP_BUCKET ?= +GATEWAY_CODE_S3_BUCKET ?= +GATEWAY_CODE_S3_KEY ?= +GATEWAY_CODE_S3_OBJECT_VERSION ?= .PHONY: help help: @@ -60,7 +63,10 @@ print-vars: check "EXAMPLE_TEMPLATE_DIR=$(EXAMPLE_TEMPLATE_DIR)" \ "BOOTSTRAP_STACK_NAME=$(BOOTSTRAP_STACK_NAME)" \ "BOOTSTRAP_TEMPLATE=$(BOOTSTRAP_TEMPLATE)" \ - "BOOTSTRAP_BUCKET=$(BOOTSTRAP_BUCKET)" + "BOOTSTRAP_BUCKET=$(BOOTSTRAP_BUCKET)" \ + "GATEWAY_CODE_S3_BUCKET=$(GATEWAY_CODE_S3_BUCKET)" \ + "GATEWAY_CODE_S3_KEY=$(GATEWAY_CODE_S3_KEY)" \ + "GATEWAY_CODE_S3_OBJECT_VERSION=$(GATEWAY_CODE_S3_OBJECT_VERSION)" .PHONY: check-example-template check-example-template: @@ -115,6 +121,9 @@ bootstrap-deploy: check bootstrap-build export AWS_REGION="$(AWS_REGION)" AWS_DEFAULT_REGION="$(AWS_REGION)"; \ params=(); \ if [[ -n "$(BOOTSTRAP_BUCKET)" ]]; then params+=("UseExistingBucket=$(BOOTSTRAP_BUCKET)"); fi; \ + if [[ -n "$(GATEWAY_CODE_S3_BUCKET)" ]]; then params+=("GatewayCodeS3Bucket=$(GATEWAY_CODE_S3_BUCKET)"); fi; \ + if [[ -n "$(GATEWAY_CODE_S3_KEY)" ]]; then params+=("GatewayCodeS3Key=$(GATEWAY_CODE_S3_KEY)"); fi; \ + if [[ -n "$(GATEWAY_CODE_S3_OBJECT_VERSION)" ]]; then params+=("GatewayCodeS3ObjectVersion=$(GATEWAY_CODE_S3_OBJECT_VERSION)"); fi; \ deploy_args=( \ --stack-name "$(BOOTSTRAP_STACK_NAME)" \ --template-file bootstrap/.aws-sam/build/template.yaml \ diff --git a/README.md b/README.md index eecbb51..46c0836 100644 --- a/README.md +++ b/README.md @@ -72,19 +72,24 @@ caveats, and links to the sanitized reports. ## Current Deployment Model -- The `KhoneGateway` macro publishes the gateway config/spec artifact to S3. -- User templates define the gateway as an explicit `AWS::Serverless::Function`. -- The gateway reads `KHONE_CONFIG_URI` from `!GetAtt .ConfigS3Uri`. -- SAM `CapacityProviderConfig` attaches an existing LMI capacity provider. -- `FunctionUrlConfig.InvokeMode: RESPONSE_STREAM` exposes the HTTP interface. +- The `KhoneGateway` macro expands `Khone::Gateway::Service` into the gateway Lambda, + Function URL, execution role, and config publisher. +- Application stacks still bring an existing LMI capacity provider ARN. +- The SAR-installed bootstrap stack carries the versioned gateway Lambda zip location. +- `Khone::Gateway::Service` configures gateway memory, scaling, environment, and route spec. +- The generated Function URL uses `InvokeMode: RESPONSE_STREAM` and defaults to `AuthType: NONE`. -Deployment resources stay explicit in your SAM template; the macro is only responsible for the -gateway config artifact. +The gateway source package no longer needs to be part of each application deployment path. ## Quick Start ```bash -make bootstrap-deploy +cargo lambda build --release --arm64 --output-format zip -p khone-gateway +aws s3 cp target/lambda/khone-gateway/bootstrap.zip \ + "s3://$GATEWAY_ARTIFACT_BUCKET/khone/dev/gateway/bootstrap.zip" +make bootstrap-deploy \ + GATEWAY_CODE_S3_BUCKET="$GATEWAY_ARTIFACT_BUCKET" \ + GATEWAY_CODE_S3_KEY="khone/dev/gateway/bootstrap.zip" make examples-sam-deploy GATEWAY_CAPACITY_PROVIDER_ARN=arn:aws:lambda:... ``` @@ -92,7 +97,9 @@ make examples-sam-deploy GATEWAY_CAPACITY_PROVIDER_ARN=arn:aws:lambda:... `EXAMPLE_TEMPLATE` to `adapter-node`, `adapter-rust`, `layer-proxy-node`, `layer-proxy-python`, or `native-batch-node` to deploy a specific example. -SAM Rust builds require `cargo-lambda` and `SAM_CLI_BETA_RUST_CARGO_LAMBDA=1`. +Released SAR bootstrap installs already set the gateway artifact location, so the source-built zip +step is only needed when deploying the bootstrap stack directly from this checkout. SAM Rust builds +require `cargo-lambda` and `SAM_CLI_BETA_RUST_CARGO_LAMBDA=1`. ## Documentation diff --git a/benchmark/sam/template.yaml b/benchmark/sam/template.yaml index e5db40d..c64a910 100644 --- a/benchmark/sam/template.yaml +++ b/benchmark/sam/template.yaml @@ -326,6 +326,27 @@ Resources: GatewayService: Type: Khone::Gateway::Service Properties: + CapacityProviderArn: !Ref GatewayCapacityProviderArn + FunctionName: !Sub '${AWS::StackName}-gateway' + Description: Khone benchmark router running on Lambda Managed Instances. + MemorySize: 2048 + Timeout: 90 + ExecutionEnvironmentMemoryGiBPerVCpu: 2.0 + PerExecutionEnvironmentMaxConcurrency: 64 + MinExecutionEnvironments: 4 + MaxExecutionEnvironments: 4 + Environment: + RUST_LOG: info + KHONE_DEBUG_RESPONSE_HEADERS: "1" + KHONE_EMF_METRICS: !If [EmfMetricsEnabledCondition, "1", !Ref AWS::NoValue] + KHONE_EMF_NAMESPACE: !If [EmfMetricsEnabledCondition, KhoneBenchmark, !Ref AWS::NoValue] + KHONE_EMF_HIGH_RES: !If [EmfMetricsEnabledCondition, "1", !Ref AWS::NoValue] + KHONE_OBSERVABILITY_VENDOR: !If [ObservabilityEnabledCondition, AWSXRAY, !Ref AWS::NoValue] + OTEL_PROPAGATORS: !If [ObservabilityEnabledCondition, "xray,tracecontext,baggage", !Ref AWS::NoValue] + OTEL_METRICS_EXPORTER: !If [ObservabilityEnabledCondition, "none", !Ref AWS::NoValue] + OTEL_SERVICE_NAME: !If [ObservabilityEnabledCondition, !Sub '${AWS::StackName}-gateway', !Ref AWS::NoValue] + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: !If [ObservabilityEnabledCondition, !Ref OtelTracesEndpoint, !Ref AWS::NoValue] + OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: !If [ObservabilityEnabledCondition, !Ref OtelTracesProtocol, !Ref AWS::NoValue] ConfigPrefix: !Sub 'khone/${AWS::StackName}/gateway/' GatewayConfig: DefaultTimeoutMs: 2000 @@ -376,65 +397,6 @@ Resources: smoothingSamples: 10 warmupProbes: 3 - GatewayFunction: - Type: AWS::Serverless::Function - Metadata: - BuildMethod: rust-cargolambda - Properties: - CodeUri: ../../gateway - Handler: bootstrap - Runtime: provided.al2023 - PackageType: Zip - FunctionName: !Sub '${AWS::StackName}-gateway' - Description: Khone benchmark router running on Lambda Managed Instances. - Architectures: - - arm64 - MemorySize: 2048 - Timeout: 90 - CapacityProviderConfig: - Arn: !Ref GatewayCapacityProviderArn - ExecutionEnvironmentMemoryGiBPerVCpu: 2.0 - PerExecutionEnvironmentMaxConcurrency: 64 - FunctionScalingConfig: - MinExecutionEnvironments: 4 - MaxExecutionEnvironments: 4 - FunctionUrlConfig: - AuthType: NONE - InvokeMode: RESPONSE_STREAM - Environment: - Variables: - RUST_LOG: info - KHONE_CONFIG_URI: !GetAtt GatewayService.ConfigS3Uri - KHONE_DEBUG_RESPONSE_HEADERS: "1" - KHONE_EMF_METRICS: !If [EmfMetricsEnabledCondition, "1", !Ref AWS::NoValue] - KHONE_EMF_NAMESPACE: !If [EmfMetricsEnabledCondition, KhoneBenchmark, !Ref AWS::NoValue] - KHONE_EMF_HIGH_RES: !If [EmfMetricsEnabledCondition, "1", !Ref AWS::NoValue] - KHONE_OBSERVABILITY_VENDOR: !If [ObservabilityEnabledCondition, AWSXRAY, !Ref AWS::NoValue] - OTEL_PROPAGATORS: !If [ObservabilityEnabledCondition, "xray,tracecontext,baggage", !Ref AWS::NoValue] - OTEL_METRICS_EXPORTER: !If [ObservabilityEnabledCondition, "none", !Ref AWS::NoValue] - OTEL_SERVICE_NAME: !If [ObservabilityEnabledCondition, !Sub '${AWS::StackName}-gateway', !Ref AWS::NoValue] - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: !If [ObservabilityEnabledCondition, !Ref OtelTracesEndpoint, !Ref AWS::NoValue] - OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: !If [ObservabilityEnabledCondition, !Ref OtelTracesProtocol, !Ref AWS::NoValue] - Policies: - - AWSLambdaBasicExecutionRole - - Statement: - - Sid: ReadGatewayConfig - Effect: Allow - Action: s3:GetObject - Resource: !Sub - - 'arn:${AWS::Partition}:s3:::${Bucket}/${Prefix}*' - - Bucket: !GetAtt GatewayService.BucketName - Prefix: !GetAtt GatewayService.Prefix - - Sid: InvokeTargetLambdas - Effect: Allow - Action: - - lambda:InvokeFunction - - lambda:InvokeWithResponseStream - Resource: - - !GetAtt SteadyFunction.Arn - - !GetAtt AdaptiveFunction.Arn - - !GetAtt TargetAwareFunction.Arn - Outputs: BenchmarkBackendUrl: Value: !GetAtt BackendFunctionUrl.FunctionUrl @@ -443,7 +405,7 @@ Outputs: Value: "steady,adaptive,target-aware,standard" BenchmarkTargetsJson: - Value: !Sub '[{"name":"steady","url":"${GatewayFunctionUrl.FunctionUrl}steady"},{"name":"adaptive","url":"${GatewayFunctionUrl.FunctionUrl}adaptive"},{"name":"target-aware","url":"${GatewayFunctionUrl.FunctionUrl}target-aware"},{"name":"standard","url":"https://${DirectHttpApi}.execute-api.${AWS::Region}.amazonaws.com/std"}]' + Value: !Sub '[{"name":"steady","url":"${GatewayServiceKhoneFunctionUrl.FunctionUrl}steady"},{"name":"adaptive","url":"${GatewayServiceKhoneFunctionUrl.FunctionUrl}adaptive"},{"name":"target-aware","url":"${GatewayServiceKhoneFunctionUrl.FunctionUrl}target-aware"},{"name":"standard","url":"https://${DirectHttpApi}.execute-api.${AWS::Region}.amazonaws.com/std"}]' SteadyFunctionArn: Value: !GetAtt SteadyFunction.Arn @@ -458,19 +420,19 @@ Outputs: Value: !GetAtt STDFunction.Arn GatewayFunctionUrl: - Value: !GetAtt GatewayFunctionUrl.FunctionUrl + Value: !GetAtt GatewayServiceKhoneFunctionUrl.FunctionUrl GatewayServiceConfigS3Uri: - Value: !GetAtt GatewayService.ConfigS3Uri + Value: !GetAtt GatewayServiceKhoneConfigPublisher.ConfigS3Uri SteadyUrl: - Value: !Sub "${GatewayFunctionUrl.FunctionUrl}steady" + Value: !Sub "${GatewayServiceKhoneFunctionUrl.FunctionUrl}steady" AdaptiveUrl: - Value: !Sub "${GatewayFunctionUrl.FunctionUrl}adaptive" + Value: !Sub "${GatewayServiceKhoneFunctionUrl.FunctionUrl}adaptive" TargetAwareUrl: - Value: !Sub "${GatewayFunctionUrl.FunctionUrl}target-aware" + Value: !Sub "${GatewayServiceKhoneFunctionUrl.FunctionUrl}target-aware" StandardUrl: Value: !Sub "https://${DirectHttpApi}.execute-api.${AWS::Region}.amazonaws.com/std" diff --git a/benchmark/src/charts/echarts/specs.ts b/benchmark/src/charts/echarts/specs.ts index c58b89b..ea09e40 100644 --- a/benchmark/src/charts/echarts/specs.ts +++ b/benchmark/src/charts/echarts/specs.ts @@ -338,7 +338,7 @@ function backendDelayLabel(metrics: MetricsBundle): string | null { } function gatewayLmiLabel(metrics: MetricsBundle): string | null { - const gateway = functionMetadata(metrics, ['GatewayFunction']); + const gateway = functionMetadata(metrics, ['GatewayService']); if (!gateway) { return null; } diff --git a/benchmark/test/render.test.ts b/benchmark/test/render.test.ts index 5a93aec..cba380e 100644 --- a/benchmark/test/render.test.ts +++ b/benchmark/test/render.test.ts @@ -172,7 +172,7 @@ test('latency distribution title includes run configuration context', async () = }, functions: [ { - logical_id: 'GatewayFunction', + logical_id: 'GatewayService', function_name: 'test-gateway', function_arn: null, runtime: 'provided.al2023', diff --git a/benchmark/test/report.test.ts b/benchmark/test/report.test.ts index 6c71b7a..654485b 100644 --- a/benchmark/test/report.test.ts +++ b/benchmark/test/report.test.ts @@ -87,7 +87,7 @@ test('report includes workload and deployed Lambda configuration', async () => { }, functions: [ { - logical_id: 'GatewayFunction', + logical_id: 'GatewayService', function_name: 'test-gateway', function_arn: 'arn:aws:lambda:us-east-1:123456789012:function:test-gateway', runtime: 'provided.al2023', @@ -139,7 +139,7 @@ test('report includes workload and deployed Lambda configuration', async () => { assert.match(text, /From rps/); assert.match(text, /BenchmarkHandlerMemorySize/); assert.match(text, /512/); - assert.match(text, /GatewayFunction/); + assert.match(text, /GatewayService/); assert.match(text, /2048 MB/); assert.match(text, /1\/4 envs/); assert.match(text, /64 conc\/env/); @@ -224,7 +224,7 @@ test('metrics json redacts infrastructure identifiers from public report data', }, functions: [ { - logical_id: 'GatewayFunction', + logical_id: 'GatewayService', function_name: 'test-gateway', function_arn: 'arn:aws:lambda:us-east-1:123456789012:function:test-gateway', runtime: 'provided.al2023', diff --git a/bootstrap/README.md b/bootstrap/README.md index 8630069..79af178 100644 --- a/bootstrap/README.md +++ b/bootstrap/README.md @@ -6,10 +6,14 @@ The bootstrap stack installs the account/region resources used by application st - `Custom::KhoneConfigPublisher`. - `KhoneGateway` CloudFormation macro. - shared Mode A runtime API proxy layers. +- versioned gateway Lambda artifact settings used by the macro. See [Bootstrap macro](../docs/reference/bootstrap-macro.md) for the resource contract and [SAM gateway](../docs/deploy/sam-gateway.md) for the application-stack pattern. +SAR release templates set the gateway artifact bucket/key defaults. Source deployments can pass +`GatewayCodeS3Bucket`, `GatewayCodeS3Key`, and optional `GatewayCodeS3ObjectVersion` explicitly. + Deploy from the repository root: ```bash diff --git a/bootstrap/gateway_macro/app.py b/bootstrap/gateway_macro/app.py index 31805c2..c25c081 100644 --- a/bootstrap/gateway_macro/app.py +++ b/bootstrap/gateway_macro/app.py @@ -1,5 +1,8 @@ import copy +import json import logging +import os +import re from typing import Any, Mapping, MutableMapping @@ -7,15 +10,39 @@ logger.setLevel(logging.INFO) KHONE_GATEWAY_RESOURCE_TYPE = "Khone::Gateway::Service" +EXPORT_CONFIG_BUCKET_NAME = "KhoneConfigBucketName" EXPORT_CONFIG_PUBLISHER_SERVICE_TOKEN = "KhoneConfigPublisherServiceToken" -ALLOWED_PROPERTIES = {"ConfigPrefix", "GatewayConfig", "Spec"} +GATEWAY_CODE_BUCKET_ENV_VAR = "KHONE_GATEWAY_CODE_S3_BUCKET" +GATEWAY_CODE_KEY_ENV_VAR = "KHONE_GATEWAY_CODE_S3_KEY" +GATEWAY_CODE_OBJECT_VERSION_ENV_VAR = "KHONE_GATEWAY_CODE_S3_OBJECT_VERSION" + +KHONE_CONFIG_URI_ENV_VAR = "KHONE_CONFIG_URI" + +ALLOWED_PROPERTIES = { + "CapacityProviderArn", + "ConfigPrefix", + "Description", + "Environment", + "ExecutionEnvironmentMemoryGiBPerVCpu", + "FunctionName", + "FunctionUrlAuthType", + "GatewayConfig", + "LogRetentionInDays", + "LoggingConfig", + "MaxExecutionEnvironments", + "MemorySize", + "MinExecutionEnvironments", + "PerExecutionEnvironmentMaxConcurrency", + "Spec", + "Timeout", + "TracingConfig", +} REMOVED_APP_RUNNER_PROPERTIES = { "AutoDeploymentsEnabled", "AutoScalingConfiguration", "AutoScalingConfigurationArn", "EmfMetrics", - "Environment", "EnvironmentSecrets", "ImageIdentifier", "InstanceConfiguration", @@ -36,6 +63,10 @@ "Metadata", "UpdateReplacePolicy", } +GENERATED_RESOURCE_TOP_LEVEL_ATTRIBUTES = PRESERVED_TOP_LEVEL_ATTRIBUTES +LAMBDA_FUNCTION_ARN_RE = re.compile( + r"^arn:[A-Za-z0-9-]+:lambda:[a-z0-9-]+:[0-9]{12}:function:[A-Za-z0-9-_]+(?::[A-Za-z0-9-_$]+)?$" +) def _default_prefix_for(logical_id: str) -> Any: @@ -44,10 +75,44 @@ def _default_prefix_for(logical_id: str) -> Any: return {"Fn::Sub": f"khone/${{AWS::StackName}}/{logical_id}/"} +def _normalize_prefix_literal(prefix: Any) -> Any: + if not isinstance(prefix, str): + return prefix + prefix = prefix.strip() + if not prefix: + return "" + if prefix.startswith("/"): + prefix = prefix[1:] + if prefix and not prefix.endswith("/"): + prefix += "/" + return prefix + + def _import_value(name: str) -> dict[str, Any]: return {"Fn::ImportValue": name} +def _get_att(logical_id: str, attr: str) -> dict[str, Any]: + return {"Fn::GetAtt": [logical_id, attr]} + + +def _sub(template: str, variables: dict[str, Any] | None = None) -> Any: + if variables is None: + return {"Fn::Sub": template} + return {"Fn::Sub": [template, variables]} + + +def _ref(logical_id: str) -> dict[str, str]: + return {"Ref": logical_id} + + +def _read_required_env(name: str) -> str: + value = os.environ.get(name, "").strip() + if not value: + raise ValueError(f"Macro is missing environment variable {name}.") + return value + + def _reject_unsupported_properties(logical_id: str, props: Mapping[str, Any]) -> None: unsupported = sorted(set(props.keys()) - ALLOWED_PROPERTIES) if not unsupported: @@ -63,8 +128,8 @@ def _reject_unsupported_properties(logical_id: str, props: Mapping[str, Any]) -> joined = ", ".join(removed) raise ValueError( f"{logical_id}.Properties contains App Runner-era properties that are no longer " - f"supported by {KHONE_GATEWAY_RESOURCE_TYPE}: {joined}. Define the gateway as an " - "AWS::Serverless::Function and set KHONE_CONFIG_URI from this resource's ConfigS3Uri." + f"supported by {KHONE_GATEWAY_RESOURCE_TYPE}: {joined}. Use the Lambda/LMI gateway " + "properties on Khone::Gateway::Service instead." ) joined = ", ".join(unsupported) @@ -78,6 +143,214 @@ def _copy_preserved_top_level_attributes(original: Mapping[str, Any]) -> dict[st return {k: copy.deepcopy(v) for k, v in original.items() if k in PRESERVED_TOP_LEVEL_ATTRIBUTES} +def _copy_generated_top_level_attributes(original: Mapping[str, Any]) -> dict[str, Any]: + return {k: copy.deepcopy(v) for k, v in original.items() if k in GENERATED_RESOURCE_TOP_LEVEL_ATTRIBUTES} + + +def _generated_resource(original: Mapping[str, Any], resource: dict[str, Any]) -> dict[str, Any]: + out = _copy_generated_top_level_attributes(original) + out.update(resource) + return out + + +def _ensure_no_collision(resources: Mapping[str, Any], logical_id: str) -> None: + if logical_id in resources: + raise ValueError(f"Macro expansion would overwrite an existing resource '{logical_id}'.") + + +def _validate_object_prop( + *, + logical_id: str, + props: Mapping[str, Any], + name: str, + required: bool = False, +) -> dict[str, Any] | None: + value = props.get(name) + if value is None: + if required: + raise ValueError(f"{logical_id}.Properties.{name} is required and must be an object.") + return None + if not isinstance(value, dict): + raise ValueError(f"{logical_id}.Properties.{name} must be an object.") + return value + + +def _validate_string_or_intrinsic( + *, + logical_id: str, + props: Mapping[str, Any], + name: str, + required: bool = False, + default: Any = None, + allow_empty: bool = True, +) -> Any: + value = props.get(name, default) + if value is None: + if required: + raise ValueError( + f"{logical_id}.Properties.{name} is required and must be a string or intrinsic function object." + ) + return None + if not isinstance(value, (str, dict)): + raise ValueError( + f"{logical_id}.Properties.{name} must be a string or intrinsic function object." + ) + if isinstance(value, str) and required and not value: + raise ValueError(f"{logical_id}.Properties.{name} must not be empty.") + if isinstance(value, str) and not allow_empty and not value.strip(): + raise ValueError(f"{logical_id}.Properties.{name} must not be empty.") + return value + + +def _validate_int_or_intrinsic( + *, + logical_id: str, + props: Mapping[str, Any], + name: str, + default: int | None = None, + minimum: int | None = None, +) -> Any: + value = props.get(name, default) + if value is None: + return None + if isinstance(value, dict): + return value + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{logical_id}.Properties.{name} must be an integer or intrinsic function object.") + if minimum is not None and value < minimum: + raise ValueError(f"{logical_id}.Properties.{name} must be >= {minimum}.") + return value + + +def _validate_number_or_intrinsic( + *, + logical_id: str, + props: Mapping[str, Any], + name: str, + default: int | float | None = None, + minimum: int | float | None = None, +) -> Any: + value = props.get(name, default) + if value is None: + return None + if isinstance(value, dict): + return value + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{logical_id}.Properties.{name} must be a number or intrinsic function object.") + if minimum is not None and value < minimum: + raise ValueError(f"{logical_id}.Properties.{name} must be >= {minimum}.") + return value + + +def _validate_environment(logical_id: str, props: Mapping[str, Any]) -> dict[str, Any]: + env = props.get("Environment") or {} + if not isinstance(env, dict): + raise ValueError(f"{logical_id}.Properties.Environment must be an object.") + out: dict[str, Any] = {} + for key, value in env.items(): + if not isinstance(key, str) or not key: + raise ValueError("Environment keys must be non-empty strings.") + if key == KHONE_CONFIG_URI_ENV_VAR: + raise ValueError(f"{logical_id}.Properties.Environment cannot define {KHONE_CONFIG_URI_ENV_VAR}.") + if not isinstance(value, (str, dict)): + raise ValueError(f"{logical_id}.Properties.Environment.{key} must be a string or intrinsic function object.") + out[key] = value + return out + + +def _is_positive_integer_literal(value: Any) -> bool: + if isinstance(value, bool): + return False + if isinstance(value, int): + return value > 0 + if isinstance(value, str): + stripped = value.strip() + return stripped.isdigit() and int(stripped) > 0 + return False + + +def _is_lambda_function_arn(value: str) -> bool: + return bool(LAMBDA_FUNCTION_ARN_RE.fullmatch(value)) + + +def _collect_target_lambda_arns(spec: Mapping[str, Any]) -> list[Any]: + paths = spec.get("paths") or {} + if not isinstance(paths, dict): + raise ValueError("Spec.paths must be an object.") + + found: list[Any] = [] + for path_name, path_item in paths.items(): + if not isinstance(path_item, dict): + continue + for method_name, op in path_item.items(): + if not isinstance(op, dict): + continue + operation_path = f"Spec.paths[{path_name!r}].{method_name}" + + x_khone = op.get("x-khone") + if x_khone is not None: + if not isinstance(x_khone, dict): + raise ValueError(f"{operation_path}.x-khone must be an object.") + for field in ("maxWaitMs", "maxBatchSize"): + if field not in x_khone: + continue + field_value = x_khone[field] + if isinstance(field_value, dict): + continue + if not _is_positive_integer_literal(field_value): + raise ValueError( + f"{operation_path}.x-khone.{field} must be a positive integer, positive integer string, " + "or intrinsic function object." + ) + + if "x-target-lambda" not in op: + continue + target = op["x-target-lambda"] + if not isinstance(target, (str, dict)): + raise ValueError( + f"{operation_path}.x-target-lambda must be a string or an intrinsic function object." + ) + if isinstance(target, str) and not _is_lambda_function_arn(target): + raise ValueError( + f"{operation_path}.x-target-lambda must be a Lambda function ARN (got: {target!r})." + ) + found.append(target) + + out: list[Any] = [] + seen: set[str] = set() + for value in found: + key = value if isinstance(value, str) else json.dumps(value, sort_keys=True) + if key in seen: + continue + seen.add(key) + out.append(value) + return out + + +def _gateway_code() -> dict[str, Any]: + code: dict[str, Any] = { + "S3Bucket": _read_required_env(GATEWAY_CODE_BUCKET_ENV_VAR), + "S3Key": _read_required_env(GATEWAY_CODE_KEY_ENV_VAR), + } + object_version = os.environ.get(GATEWAY_CODE_OBJECT_VERSION_ENV_VAR, "").strip() + if object_version: + code["S3ObjectVersion"] = object_version + return code + + +def _gateway_log_group_arn(logical_id: str, function_name: Any, suffix: str = "") -> Any: + if function_name is None: + return _sub( + f"arn:${{AWS::Partition}}:logs:${{AWS::Region}}:${{AWS::AccountId}}:" + f"log-group:/aws/lambda/${{AWS::StackName}}-{logical_id}-*{suffix}" + ) + return _sub( + f"arn:${{AWS::Partition}}:logs:${{AWS::Region}}:${{AWS::AccountId}}:" + f"log-group:/aws/lambda/${{FunctionName}}{suffix}", + {"FunctionName": function_name}, + ) + + def _expand_gateway_service( *, resources: MutableMapping[str, Any], @@ -90,22 +363,113 @@ def _expand_gateway_service( _reject_unsupported_properties(logical_id, props) - gateway_config = props.get("GatewayConfig") - if not isinstance(gateway_config, dict): - raise ValueError(f"{logical_id}.Properties.GatewayConfig is required and must be an object.") + capacity_provider_arn = _validate_string_or_intrinsic( + logical_id=logical_id, + props=props, + name="CapacityProviderArn", + required=True, + allow_empty=False, + ) + gateway_config = _validate_object_prop( + logical_id=logical_id, + props=props, + name="GatewayConfig", + required=True, + ) + spec = _validate_object_prop(logical_id=logical_id, props=props, name="Spec", required=True) + assert gateway_config is not None + assert spec is not None - spec = props.get("Spec") - if not isinstance(spec, dict): - raise ValueError(f"{logical_id}.Properties.Spec is required and must be an object.") + config_prefix = _validate_string_or_intrinsic( + logical_id=logical_id, + props=props, + name="ConfigPrefix", + default=_default_prefix_for(logical_id), + ) + config_prefix = _normalize_prefix_literal(config_prefix) + function_name = _validate_string_or_intrinsic( + logical_id=logical_id, + props=props, + name="FunctionName", + allow_empty=False, + ) + description = _validate_string_or_intrinsic(logical_id=logical_id, props=props, name="Description") + memory_size = _validate_int_or_intrinsic( + logical_id=logical_id, + props=props, + name="MemorySize", + default=2048, + minimum=128, + ) + timeout = _validate_int_or_intrinsic( + logical_id=logical_id, + props=props, + name="Timeout", + default=30, + minimum=1, + ) + execution_environment_memory = _validate_number_or_intrinsic( + logical_id=logical_id, + props=props, + name="ExecutionEnvironmentMemoryGiBPerVCpu", + default=2.0, + minimum=2, + ) + per_environment_concurrency = _validate_int_or_intrinsic( + logical_id=logical_id, + props=props, + name="PerExecutionEnvironmentMaxConcurrency", + default=64, + minimum=1, + ) + min_execution_environments = _validate_int_or_intrinsic( + logical_id=logical_id, + props=props, + name="MinExecutionEnvironments", + default=1, + minimum=0, + ) + max_execution_environments = _validate_int_or_intrinsic( + logical_id=logical_id, + props=props, + name="MaxExecutionEnvironments", + default=4, + minimum=0, + ) + log_retention = _validate_int_or_intrinsic( + logical_id=logical_id, + props=props, + name="LogRetentionInDays", + minimum=1, + ) + tracing_config = _validate_object_prop(logical_id=logical_id, props=props, name="TracingConfig") + logging_config = _validate_object_prop(logical_id=logical_id, props=props, name="LoggingConfig") + environment = _validate_environment(logical_id, props) - config_prefix = props.get("ConfigPrefix", _default_prefix_for(logical_id)) - if not isinstance(config_prefix, (str, dict)): + function_url_auth_type = props.get("FunctionUrlAuthType", "NONE") + if function_url_auth_type not in ("NONE", "AWS_IAM"): raise ValueError( - f"{logical_id}.Properties.ConfigPrefix must be a string or intrinsic function object." + f"{logical_id}.Properties.FunctionUrlAuthType must be NONE or AWS_IAM." ) - transformed = _copy_preserved_top_level_attributes(original) - transformed.update( + target_lambda_arns = _collect_target_lambda_arns(spec) + + config_publisher_id = f"{logical_id}KhoneConfigPublisher" + execution_role_id = f"{logical_id}KhoneExecutionRole" + function_url_id = f"{logical_id}KhoneFunctionUrl" + function_url_permission_id = f"{logical_id}KhoneFunctionUrlPermission" + log_group_id = f"{logical_id}KhoneLogGroup" + + generated_ids = [config_publisher_id, execution_role_id, function_url_id] + if function_url_auth_type == "NONE": + generated_ids.append(function_url_permission_id) + if log_retention is not None: + generated_ids.append(log_group_id) + for generated_id in generated_ids: + _ensure_no_collision(resources, generated_id) + + resources[config_publisher_id] = _generated_resource( + original, { "Type": "Custom::KhoneConfigPublisher", "Properties": { @@ -114,10 +478,149 @@ def _expand_gateway_service( "GatewayConfig": gateway_config, "Spec": spec, }, - } + }, + ) + + config_object_arn = _sub( + "arn:${AWS::Partition}:s3:::${Bucket}/${Prefix}*", + {"Bucket": _import_value(EXPORT_CONFIG_BUCKET_NAME), "Prefix": config_prefix}, ) + log_group_arn = _gateway_log_group_arn(logical_id, function_name) + log_stream_arn = _gateway_log_group_arn(logical_id, function_name, ":*") + policy_statements: list[dict[str, Any]] = [ + { + "Sid": "CreateLogGroup", + "Effect": "Allow", + "Action": ["logs:CreateLogGroup"], + "Resource": log_group_arn, + }, + { + "Sid": "WriteLogs", + "Effect": "Allow", + "Action": ["logs:CreateLogStream", "logs:PutLogEvents"], + "Resource": log_stream_arn, + }, + { + "Sid": "ReadGatewayConfig", + "Effect": "Allow", + "Action": ["s3:GetObject"], + "Resource": [config_object_arn], + }, + ] + if target_lambda_arns: + policy_statements.append( + { + "Sid": "InvokeTargetLambdas", + "Effect": "Allow", + "Action": ["lambda:InvokeFunction", "lambda:InvokeWithResponseStream"], + "Resource": target_lambda_arns, + } + ) + + resources[execution_role_id] = _generated_resource( + original, + { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": {"Service": "lambda.amazonaws.com"}, + "Action": "sts:AssumeRole", + } + ], + }, + "Policies": [ + { + "PolicyName": "KhoneGatewayExecutionPolicy", + "PolicyDocument": { + "Version": "2012-10-17", + "Statement": policy_statements, + }, + } + ], + }, + }, + ) + + if log_retention is not None: + resources[log_group_id] = _generated_resource( + original, + { + "Type": "AWS::Logs::LogGroup", + "Properties": { + "LogGroupName": _sub("/aws/lambda/${FunctionName}", {"FunctionName": _ref(logical_id)}), + "RetentionInDays": log_retention, + }, + }, + ) + + runtime_env = {"RUST_LOG": "info", **environment} + runtime_env[KHONE_CONFIG_URI_ENV_VAR] = _get_att(config_publisher_id, "ConfigS3Uri") + + lambda_props: dict[str, Any] = { + "Architectures": ["arm64"], + "Code": _gateway_code(), + "Handler": "bootstrap", + "Role": _get_att(execution_role_id, "Arn"), + "Runtime": "provided.al2023", + "PackageType": "Zip", + "MemorySize": memory_size, + "Timeout": timeout, + "CapacityProviderConfig": { + "LambdaManagedInstancesCapacityProviderConfig": { + "CapacityProviderArn": capacity_provider_arn, + "ExecutionEnvironmentMemoryGiBPerVCpu": execution_environment_memory, + "PerExecutionEnvironmentMaxConcurrency": per_environment_concurrency, + } + }, + "FunctionScalingConfig": { + "MinExecutionEnvironments": min_execution_environments, + "MaxExecutionEnvironments": max_execution_environments, + }, + "Environment": {"Variables": runtime_env}, + } + if function_name is not None: + lambda_props["FunctionName"] = function_name + if description is not None: + lambda_props["Description"] = description + if tracing_config is not None: + lambda_props["TracingConfig"] = tracing_config + if logging_config is not None: + lambda_props["LoggingConfig"] = logging_config + + transformed = _copy_preserved_top_level_attributes(original) + transformed.update({"Type": "AWS::Lambda::Function", "Properties": lambda_props}) resources[logical_id] = transformed + resources[function_url_id] = _generated_resource( + original, + { + "Type": "AWS::Lambda::Url", + "Properties": { + "AuthType": function_url_auth_type, + "InvokeMode": "RESPONSE_STREAM", + "TargetFunctionArn": _get_att(logical_id, "Arn"), + }, + }, + ) + + if function_url_auth_type == "NONE": + resources[function_url_permission_id] = _generated_resource( + original, + { + "Type": "AWS::Lambda::Permission", + "Properties": { + "Action": "lambda:InvokeFunctionUrl", + "FunctionName": _ref(logical_id), + "FunctionUrlAuthType": "NONE", + "Principal": "*", + }, + }, + ) + def handler(event: Mapping[str, Any], context: Any) -> dict[str, Any]: request_id = event.get("requestId") @@ -141,7 +644,7 @@ def handler(event: Mapping[str, Any], context: Any) -> dict[str, Any]: continue if resource.get("Type") != KHONE_GATEWAY_RESOURCE_TYPE: continue - logger.info("Expanding %s %s to Custom::KhoneConfigPublisher", KHONE_GATEWAY_RESOURCE_TYPE, logical_id) + logger.info("Expanding %s %s to Lambda gateway resources", KHONE_GATEWAY_RESOURCE_TYPE, logical_id) _expand_gateway_service(resources=resources, logical_id=logical_id, original=resource) return {"requestId": request_id, "status": "success", "fragment": out} diff --git a/bootstrap/template.yaml b/bootstrap/template.yaml index d703a19..6c24f57 100644 --- a/bootstrap/template.yaml +++ b/bootstrap/template.yaml @@ -1,12 +1,48 @@ AWSTemplateFormatVersion: "2010-09-09" Transform: AWS::Serverless-2016-10-31 -Description: Khone bootstrap (shared config bucket + macro + config publisher custom resource) +Description: Khone bootstrap (shared config bucket, macro, config publisher, runtime layers, and gateway artifact settings) + +Metadata: + AWS::ServerlessRepo::Application: + Name: khone-bootstrap + Description: Installs the Khone CloudFormation macro, config publisher, runtime proxy layers, and gateway artifact settings. + Author: dev7a + SpdxLicenseId: MIT + ReadmeUrl: README.md + Labels: + - lambda + - lmi + - gateway + - serverless + HomePageUrl: https://github.com/dev7a/khone + SourceCodeUrl: https://github.com/dev7a/khone + SemanticVersion: 0.1.0 Parameters: UseExistingBucket: Type: String Default: "" Description: Optional existing S3 bucket name to use for gateway config/spec objects. Leave empty to create a bucket named khone-config--. + GatewayCodeS3Bucket: + Type: String + Default: "" + Description: S3 bucket containing the versioned khone-gateway Lambda zip. Release templates set this default; source deployments may pass it explicitly. + GatewayCodeS3Key: + Type: String + Default: "" + Description: S3 key for the versioned khone-gateway Lambda zip. Release templates set this default; source deployments may pass it explicitly. + GatewayCodeS3ObjectVersion: + Type: String + Default: "" + Description: Optional S3 object version for the versioned khone-gateway Lambda zip. + +Rules: + GatewayCodeArtifactRequired: + Assertions: + - Assert: !And + - !Not [!Equals [!Ref GatewayCodeS3Bucket, ""]] + - !Not [!Equals [!Ref GatewayCodeS3Key, ""]] + AssertDescription: GatewayCodeS3Bucket and GatewayCodeS3Key are required for the gateway macro to deploy gateway Lambdas. Conditions: CreateConfigBucket: !Equals [!Ref UseExistingBucket, ""] @@ -106,11 +142,16 @@ Resources: Runtime: python3.13 Handler: app.handler CodeUri: gateway_macro/ - Description: CloudFormation macro that expands Khone::Gateway::Service into a config publisher custom resource. + Description: CloudFormation macro that expands Khone::Gateway::Service into a Lambda gateway and config publisher custom resource. MemorySize: 256 Timeout: 30 Policies: - AWSLambdaBasicExecutionRole + Environment: + Variables: + KHONE_GATEWAY_CODE_S3_BUCKET: !Ref GatewayCodeS3Bucket + KHONE_GATEWAY_CODE_S3_KEY: !Ref GatewayCodeS3Key + KHONE_GATEWAY_CODE_S3_OBJECT_VERSION: !Ref GatewayCodeS3ObjectVersion GatewayMacro: Type: AWS::CloudFormation::Macro diff --git a/bootstrap/tests/test_gateway_macro.py b/bootstrap/tests/test_gateway_macro.py index ecf43e5..afe4cfa 100644 --- a/bootstrap/tests/test_gateway_macro.py +++ b/bootstrap/tests/test_gateway_macro.py @@ -1,4 +1,5 @@ import importlib.util +import os import unittest from pathlib import Path @@ -22,18 +23,80 @@ def _event(resources): } +def _gateway_props(**overrides): + props = { + "CapacityProviderArn": "arn:aws:lambda:us-east-1:123456789012:capacity-provider:test", + "GatewayConfig": {"DefaultTimeoutMs": 2000}, + "Spec": { + "openapi": "3.0.0", + "paths": { + "/hello": { + "get": { + "x-target-lambda": {"Fn::GetAtt": ["HelloFunction", "Arn"]}, + "x-khone": {"maxWaitMs": 25, "maxBatchSize": 4}, + } + }, + "/hello-again": { + "get": { + "x-target-lambda": {"Fn::GetAtt": ["HelloFunction", "Arn"]}, + "x-khone": {"maxWaitMs": 25, "maxBatchSize": 4}, + } + }, + "/stream": { + "get": { + "x-target-lambda": "arn:aws:lambda:us-east-1:123456789012:function:stream", + "x-khone": {"maxWaitMs": 10, "maxBatchSize": 2}, + } + }, + }, + }, + } + props.update(overrides) + return props + + class GatewayMacroTests(unittest.TestCase): - def test_expands_gateway_service_to_config_publisher_with_same_logical_id(self) -> None: + def setUp(self) -> None: + self._old_env = { + name: os.environ.get(name) + for name in ( + "KHONE_GATEWAY_CODE_S3_BUCKET", + "KHONE_GATEWAY_CODE_S3_KEY", + "KHONE_GATEWAY_CODE_S3_OBJECT_VERSION", + ) + } + os.environ["KHONE_GATEWAY_CODE_S3_BUCKET"] = "khone-artifacts" + os.environ["KHONE_GATEWAY_CODE_S3_KEY"] = "khone/releases/0.1.0/gateway/bootstrap.zip" + os.environ.pop("KHONE_GATEWAY_CODE_S3_OBJECT_VERSION", None) + + def tearDown(self) -> None: + for name, old in self._old_env.items(): + if old is None: + os.environ.pop(name, None) + else: + os.environ[name] = old + + def test_expands_gateway_service_to_lambda_gateway_resources(self) -> None: out = app.handler( _event( { "Gateway": { "Type": "Khone::Gateway::Service", - "Properties": { - "ConfigPrefix": "khone/demo/", - "GatewayConfig": {"DefaultTimeoutMs": 2000}, - "Spec": {"openapi": "3.0.0", "paths": {}}, - }, + "Properties": _gateway_props( + ConfigPrefix="khone/demo/", + FunctionName={"Fn::Sub": "${AWS::StackName}-gateway"}, + Description="Khone gateway", + MemorySize=4096, + Timeout=90, + ExecutionEnvironmentMemoryGiBPerVCpu=4.0, + PerExecutionEnvironmentMaxConcurrency=128, + MinExecutionEnvironments=4, + MaxExecutionEnvironments=4, + Environment={"RUST_LOG": "debug", "KHONE_EMF_METRICS": "1"}, + TracingConfig={"Mode": "Active"}, + LoggingConfig={"LogFormat": "JSON", "ApplicationLogLevel": "INFO"}, + LogRetentionInDays=7, + ), } } ), @@ -42,27 +105,109 @@ def test_expands_gateway_service_to_config_publisher_with_same_logical_id(self) self.assertEqual(out["status"], "success") resources = out["fragment"]["Resources"] - self.assertEqual(set(resources.keys()), {"Gateway"}) - gateway = resources["Gateway"] - self.assertEqual(gateway["Type"], "Custom::KhoneConfigPublisher") + self.assertEqual(resources["Gateway"]["Type"], "AWS::Lambda::Function") + self.assertEqual(resources["GatewayKhoneConfigPublisher"]["Type"], "Custom::KhoneConfigPublisher") + self.assertEqual(resources["GatewayKhoneExecutionRole"]["Type"], "AWS::IAM::Role") + self.assertEqual(resources["GatewayKhoneFunctionUrl"]["Type"], "AWS::Lambda::Url") + self.assertEqual(resources["GatewayKhoneFunctionUrlPermission"]["Type"], "AWS::Lambda::Permission") + self.assertEqual(resources["GatewayKhoneLogGroup"]["Type"], "AWS::Logs::LogGroup") + + publisher = resources["GatewayKhoneConfigPublisher"] self.assertEqual( - gateway["Properties"]["ServiceToken"], + publisher["Properties"]["ServiceToken"], {"Fn::ImportValue": "KhoneConfigPublisherServiceToken"}, ) - self.assertEqual(gateway["Properties"]["Prefix"], "khone/demo/") - self.assertEqual(gateway["Properties"]["GatewayConfig"], {"DefaultTimeoutMs": 2000}) - self.assertEqual(gateway["Properties"]["Spec"], {"openapi": "3.0.0", "paths": {}}) + self.assertEqual(publisher["Properties"]["Prefix"], "khone/demo/") - def test_default_prefix_is_deterministic(self) -> None: + function_props = resources["Gateway"]["Properties"] + self.assertEqual(function_props["Code"], { + "S3Bucket": "khone-artifacts", + "S3Key": "khone/releases/0.1.0/gateway/bootstrap.zip", + }) + self.assertEqual(function_props["FunctionName"], {"Fn::Sub": "${AWS::StackName}-gateway"}) + self.assertEqual(function_props["Description"], "Khone gateway") + self.assertEqual(function_props["Architectures"], ["arm64"]) + self.assertEqual(function_props["Runtime"], "provided.al2023") + self.assertEqual(function_props["Handler"], "bootstrap") + self.assertEqual(function_props["MemorySize"], 4096) + self.assertEqual(function_props["Timeout"], 90) + self.assertEqual( + function_props["CapacityProviderConfig"], + { + "LambdaManagedInstancesCapacityProviderConfig": { + "CapacityProviderArn": "arn:aws:lambda:us-east-1:123456789012:capacity-provider:test", + "ExecutionEnvironmentMemoryGiBPerVCpu": 4.0, + "PerExecutionEnvironmentMaxConcurrency": 128, + } + }, + ) + self.assertEqual( + function_props["FunctionScalingConfig"], + {"MinExecutionEnvironments": 4, "MaxExecutionEnvironments": 4}, + ) + env = function_props["Environment"]["Variables"] + self.assertEqual(env["RUST_LOG"], "debug") + self.assertEqual(env["KHONE_EMF_METRICS"], "1") + self.assertEqual(env["KHONE_CONFIG_URI"], {"Fn::GetAtt": ["GatewayKhoneConfigPublisher", "ConfigS3Uri"]}) + + policy_doc = resources["GatewayKhoneExecutionRole"]["Properties"]["Policies"][0]["PolicyDocument"] + create_log_stmt = next(s for s in policy_doc["Statement"] if s["Sid"] == "CreateLogGroup") + write_log_stmt = next(s for s in policy_doc["Statement"] if s["Sid"] == "WriteLogs") + invoke_stmt = next(s for s in policy_doc["Statement"] if s["Sid"] == "InvokeTargetLambdas") + self.assertEqual( + create_log_stmt["Resource"], + { + "Fn::Sub": [ + "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:" + "log-group:/aws/lambda/${FunctionName}", + {"FunctionName": {"Fn::Sub": "${AWS::StackName}-gateway"}}, + ] + }, + ) + self.assertEqual( + write_log_stmt["Resource"], + { + "Fn::Sub": [ + "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:" + "log-group:/aws/lambda/${FunctionName}:*", + {"FunctionName": {"Fn::Sub": "${AWS::StackName}-gateway"}}, + ] + }, + ) + self.assertEqual(invoke_stmt["Action"], ["lambda:InvokeFunction", "lambda:InvokeWithResponseStream"]) + self.assertEqual( + invoke_stmt["Resource"], + [ + {"Fn::GetAtt": ["HelloFunction", "Arn"]}, + "arn:aws:lambda:us-east-1:123456789012:function:stream", + ], + ) + + self.assertEqual( + resources["GatewayKhoneFunctionUrl"]["Properties"], + { + "AuthType": "NONE", + "InvokeMode": "RESPONSE_STREAM", + "TargetFunctionArn": {"Fn::GetAtt": ["Gateway", "Arn"]}, + }, + ) + self.assertEqual( + resources["GatewayKhoneFunctionUrlPermission"]["Properties"], + { + "Action": "lambda:InvokeFunctionUrl", + "FunctionName": {"Ref": "Gateway"}, + "FunctionUrlAuthType": "NONE", + "Principal": "*", + }, + ) + + def test_default_prefix_and_lambda_defaults_are_deterministic(self) -> None: out = app.handler( _event( { "Gateway": { "Type": "Khone::Gateway::Service", - "Properties": { - "GatewayConfig": {}, - "Spec": {"paths": {}}, - }, + "Properties": _gateway_props(Spec={"paths": {}}), } } ), @@ -70,24 +215,72 @@ def test_default_prefix_is_deterministic(self) -> None: ) self.assertEqual(out["status"], "success") + resources = out["fragment"]["Resources"] self.assertEqual( - out["fragment"]["Resources"]["Gateway"]["Properties"]["Prefix"], + resources["GatewayKhoneConfigPublisher"]["Properties"]["Prefix"], {"Fn::Sub": "khone/${AWS::StackName}/Gateway/"}, ) + function_props = resources["Gateway"]["Properties"] + self.assertEqual(function_props["MemorySize"], 2048) + self.assertEqual(function_props["Timeout"], 30) + self.assertEqual( + function_props["FunctionScalingConfig"], + {"MinExecutionEnvironments": 1, "MaxExecutionEnvironments": 4}, + ) + self.assertEqual(function_props["Environment"]["Variables"]["RUST_LOG"], "info") + policy_doc = resources["GatewayKhoneExecutionRole"]["Properties"]["Policies"][0]["PolicyDocument"] + write_log_stmt = next(s for s in policy_doc["Statement"] if s["Sid"] == "WriteLogs") + self.assertEqual( + write_log_stmt["Resource"], + { + "Fn::Sub": ( + "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:" + "log-group:/aws/lambda/${AWS::StackName}-Gateway-*:*" + ) + }, + ) - def test_preserves_safe_top_level_resource_attributes(self) -> None: + def test_normalizes_literal_config_prefix_for_publisher_and_policy(self) -> None: out = app.handler( _event( { "Gateway": { "Type": "Khone::Gateway::Service", - "Condition": "UseGateway", - "DependsOn": ["ConfigBucket"], - "Metadata": {"Comment": "kept"}, - "Properties": { - "GatewayConfig": {}, - "Spec": {"paths": {}}, + "Properties": _gateway_props(ConfigPrefix=" /khone/demo "), + } + } + ), + context=None, + ) + + self.assertEqual(out["status"], "success") + resources = out["fragment"]["Resources"] + self.assertEqual(resources["GatewayKhoneConfigPublisher"]["Properties"]["Prefix"], "khone/demo/") + policy_doc = resources["GatewayKhoneExecutionRole"]["Properties"]["Policies"][0]["PolicyDocument"] + read_stmt = next(s for s in policy_doc["Statement"] if s["Sid"] == "ReadGatewayConfig") + self.assertEqual( + read_stmt["Resource"], + [ + { + "Fn::Sub": [ + "arn:${AWS::Partition}:s3:::${Bucket}/${Prefix}*", + { + "Bucket": {"Fn::ImportValue": "KhoneConfigBucketName"}, + "Prefix": "khone/demo/", }, + ] + } + ], + ) + + def test_includes_code_object_version_when_macro_env_is_set(self) -> None: + os.environ["KHONE_GATEWAY_CODE_S3_OBJECT_VERSION"] = "object-version" + out = app.handler( + _event( + { + "Gateway": { + "Type": "Khone::Gateway::Service", + "Properties": _gateway_props(Spec={"paths": {}}), } } ), @@ -95,47 +288,78 @@ def test_preserves_safe_top_level_resource_attributes(self) -> None: ) self.assertEqual(out["status"], "success") - gateway = out["fragment"]["Resources"]["Gateway"] - self.assertEqual(gateway["Condition"], "UseGateway") - self.assertEqual(gateway["DependsOn"], ["ConfigBucket"]) - self.assertEqual(gateway["Metadata"], {"Comment": "kept"}) + self.assertEqual( + out["fragment"]["Resources"]["Gateway"]["Properties"]["Code"], + { + "S3Bucket": "khone-artifacts", + "S3Key": "khone/releases/0.1.0/gateway/bootstrap.zip", + "S3ObjectVersion": "object-version", + }, + ) - def test_rejects_removed_app_runner_properties(self) -> None: + def test_aws_iam_function_url_does_not_generate_public_permission(self) -> None: out = app.handler( _event( { "Gateway": { "Type": "Khone::Gateway::Service", - "Properties": { - "ImageIdentifier": "public.ecr.aws/example/gateway:1", - "GatewayConfig": {}, - "Spec": {"paths": {}}, - }, + "Properties": _gateway_props(FunctionUrlAuthType="AWS_IAM", Spec={"paths": {}}), } } ), context=None, ) - self.assertEqual(out["status"], "failed") - self.assertIn("App Runner-era properties", out["errorMessage"]) - self.assertIn("ImageIdentifier", out["errorMessage"]) + self.assertEqual(out["status"], "success") + resources = out["fragment"]["Resources"] + self.assertEqual(resources["GatewayKhoneFunctionUrl"]["Properties"]["AuthType"], "AWS_IAM") + self.assertNotIn("GatewayKhoneFunctionUrlPermission", resources) - def test_rejects_app_runner_property_prefixes(self) -> None: + def test_preserves_safe_top_level_resource_attributes(self) -> None: out = app.handler( _event( { "Gateway": { "Type": "Khone::Gateway::Service", - "Properties": { - "GatewayConfig": {}, - "Spec": {"paths": {}}, - "AutoScalingConfigurationRevision": 3, - "ObservabilityConfigurationArn": ( - "arn:aws:apprunner:us-east-1:123456789012:" - "observabilityconfiguration/example/1/hash" - ), - }, + "Condition": "UseGateway", + "DeletionPolicy": "Retain", + "DependsOn": ["ConfigBucket"], + "Metadata": {"Comment": "kept"}, + "UpdateReplacePolicy": "Retain", + "Properties": _gateway_props(Spec={"paths": {}}), + } + } + ), + context=None, + ) + + self.assertEqual(out["status"], "success") + gateway = out["fragment"]["Resources"]["Gateway"] + self.assertEqual(gateway["Condition"], "UseGateway") + self.assertEqual(gateway["DeletionPolicy"], "Retain") + self.assertEqual(gateway["DependsOn"], ["ConfigBucket"]) + self.assertEqual(gateway["Metadata"], {"Comment": "kept"}) + self.assertEqual(gateway["UpdateReplacePolicy"], "Retain") + for generated_id in ( + "GatewayKhoneConfigPublisher", + "GatewayKhoneExecutionRole", + "GatewayKhoneFunctionUrl", + "GatewayKhoneFunctionUrlPermission", + ): + generated = out["fragment"]["Resources"][generated_id] + self.assertEqual(generated["Condition"], "UseGateway") + self.assertEqual(generated["DeletionPolicy"], "Retain") + self.assertEqual(generated["DependsOn"], ["ConfigBucket"]) + self.assertEqual(generated["Metadata"], {"Comment": "kept"}) + self.assertEqual(generated["UpdateReplacePolicy"], "Retain") + + def test_rejects_removed_app_runner_properties(self) -> None: + out = app.handler( + _event( + { + "Gateway": { + "Type": "Khone::Gateway::Service", + "Properties": _gateway_props(ImageIdentifier="public.ecr.aws/example/gateway:1"), } } ), @@ -144,8 +368,7 @@ def test_rejects_app_runner_property_prefixes(self) -> None: self.assertEqual(out["status"], "failed") self.assertIn("App Runner-era properties", out["errorMessage"]) - self.assertIn("AutoScalingConfigurationRevision", out["errorMessage"]) - self.assertIn("ObservabilityConfigurationArn", out["errorMessage"]) + self.assertIn("ImageIdentifier", out["errorMessage"]) def test_rejects_unknown_properties(self) -> None: out = app.handler( @@ -153,11 +376,7 @@ def test_rejects_unknown_properties(self) -> None: { "Gateway": { "Type": "Khone::Gateway::Service", - "Properties": { - "GatewayConfig": {}, - "Spec": {"paths": {}}, - "Unknown": True, - }, + "Properties": _gateway_props(Unknown=True), } } ), @@ -167,12 +386,14 @@ def test_rejects_unknown_properties(self) -> None: self.assertEqual(out["status"], "failed") self.assertIn("unsupported keys: Unknown", out["errorMessage"]) - def test_requires_gateway_config_and_spec_objects(self) -> None: + def test_requires_capacity_provider_gateway_config_and_spec_objects(self) -> None: for props, expected in [ - ({"Spec": {"paths": {}}}, "GatewayConfig is required"), - ({"GatewayConfig": {}}, "Spec is required"), - ({"GatewayConfig": [], "Spec": {"paths": {}}}, "GatewayConfig is required"), - ({"GatewayConfig": {}, "Spec": []}, "Spec is required"), + ({"GatewayConfig": {}, "Spec": {"paths": {}}}, "CapacityProviderArn is required"), + ({"CapacityProviderArn": " ", "GatewayConfig": {}, "Spec": {"paths": {}}}, "CapacityProviderArn must not be empty"), + ({"CapacityProviderArn": "arn", "Spec": {"paths": {}}}, "GatewayConfig is required"), + ({"CapacityProviderArn": "arn", "GatewayConfig": {}}, "Spec is required"), + ({"CapacityProviderArn": "arn", "GatewayConfig": [], "Spec": {"paths": {}}}, "GatewayConfig must be an object"), + ({"CapacityProviderArn": "arn", "GatewayConfig": {}, "Spec": []}, "Spec must be an object"), ]: out = app.handler( _event({"Gateway": {"Type": "Khone::Gateway::Service", "Properties": props}}), @@ -181,6 +402,215 @@ def test_requires_gateway_config_and_spec_objects(self) -> None: self.assertEqual(out["status"], "failed") self.assertIn(expected, out["errorMessage"]) + def test_rejects_environment_override_of_config_uri(self) -> None: + out = app.handler( + _event( + { + "Gateway": { + "Type": "Khone::Gateway::Service", + "Properties": _gateway_props( + Environment={"KHONE_CONFIG_URI": "s3://elsewhere"}, + Spec={"paths": {}}, + ), + } + } + ), + context=None, + ) + + self.assertEqual(out["status"], "failed") + self.assertIn("cannot define KHONE_CONFIG_URI", out["errorMessage"]) + + def test_rejects_empty_function_name(self) -> None: + out = app.handler( + _event( + { + "Gateway": { + "Type": "Khone::Gateway::Service", + "Properties": _gateway_props(FunctionName=""), + } + } + ), + context=None, + ) + + self.assertEqual(out["status"], "failed") + self.assertIn("Gateway.Properties.FunctionName must not be empty", out["errorMessage"]) + + def test_detects_generated_resource_collision(self) -> None: + out = app.handler( + _event( + { + "GatewayKhoneExecutionRole": {"Type": "AWS::IAM::Role"}, + "Gateway": { + "Type": "Khone::Gateway::Service", + "Properties": _gateway_props(Spec={"paths": {}}), + }, + } + ), + context=None, + ) + + self.assertEqual(out["status"], "failed") + self.assertIn("GatewayKhoneExecutionRole", out["errorMessage"]) + + def test_reports_path_in_x_target_lambda_validation_error(self) -> None: + out = app.handler( + _event( + { + "Gateway": { + "Type": "Khone::Gateway::Service", + "Properties": _gateway_props( + Spec={ + "paths": { + "/hello": { + "get": { + "x-target-lambda": "not-an-arn", + "x-khone": {"maxWaitMs": 1, "maxBatchSize": 1}, + } + } + } + } + ), + } + } + ), + context=None, + ) + + self.assertEqual(out["status"], "failed") + self.assertIn("Spec.paths['/hello'].get.x-target-lambda", out["errorMessage"]) + + def test_rejects_non_lambda_target_arn(self) -> None: + out = app.handler( + _event( + { + "Gateway": { + "Type": "Khone::Gateway::Service", + "Properties": _gateway_props( + Spec={ + "paths": { + "/hello": { + "get": { + "x-target-lambda": "arn:aws:s3:::not-a-lambda", + "x-khone": {"maxWaitMs": 1, "maxBatchSize": 1}, + } + } + } + } + ), + } + } + ), + context=None, + ) + + self.assertEqual(out["status"], "failed") + self.assertIn("Spec.paths['/hello'].get.x-target-lambda", out["errorMessage"]) + + def test_accepts_qualified_lambda_target_arn(self) -> None: + out = app.handler( + _event( + { + "Gateway": { + "Type": "Khone::Gateway::Service", + "Properties": _gateway_props( + Spec={ + "paths": { + "/hello": { + "get": { + "x-target-lambda": ( + "arn:aws-us-gov:lambda:us-gov-west-1:123456789012:" + "function:hello:prod" + ), + "x-khone": {"maxWaitMs": 1, "maxBatchSize": 1}, + } + } + } + } + ), + } + } + ), + context=None, + ) + + self.assertEqual(out["status"], "success") + + def test_accepts_string_route_numeric_settings(self) -> None: + out = app.handler( + _event( + { + "Gateway": { + "Type": "Khone::Gateway::Service", + "Properties": _gateway_props( + Spec={ + "paths": { + "/hello": { + "get": { + "x-target-lambda": "arn:aws:lambda:us-east-1:123456789012:function:hello", + "x-khone": {"maxWaitMs": "25", "maxBatchSize": "4"}, + } + } + } + } + ), + } + } + ), + context=None, + ) + + self.assertEqual(out["status"], "success") + spec = out["fragment"]["Resources"]["GatewayKhoneConfigPublisher"]["Properties"]["Spec"] + x_khone = spec["paths"]["/hello"]["get"]["x-khone"] + self.assertEqual(x_khone["maxWaitMs"], "25") + self.assertEqual(x_khone["maxBatchSize"], "4") + + def test_rejects_invalid_string_route_numeric_settings(self) -> None: + out = app.handler( + _event( + { + "Gateway": { + "Type": "Khone::Gateway::Service", + "Properties": _gateway_props( + Spec={ + "paths": { + "/hello": { + "get": { + "x-target-lambda": "arn:aws:lambda:us-east-1:123456789012:function:hello", + "x-khone": {"maxWaitMs": "soon", "maxBatchSize": "4"}, + } + } + } + } + ), + } + } + ), + context=None, + ) + + self.assertEqual(out["status"], "failed") + self.assertIn("Spec.paths['/hello'].get.x-khone.maxWaitMs", out["errorMessage"]) + + def test_missing_gateway_artifact_env_fails(self) -> None: + os.environ.pop("KHONE_GATEWAY_CODE_S3_BUCKET", None) + out = app.handler( + _event( + { + "Gateway": { + "Type": "Khone::Gateway::Service", + "Properties": _gateway_props(Spec={"paths": {}}), + } + } + ), + context=None, + ) + + self.assertEqual(out["status"], "failed") + self.assertIn("KHONE_GATEWAY_CODE_S3_BUCKET", out["errorMessage"]) + def test_leaves_other_resources_untouched(self) -> None: original = {"Type": "AWS::S3::Bucket", "Properties": {"BucketName": "example"}} out = app.handler(_event({"Bucket": original}), context=None) diff --git a/docs/deploy/bootstrap-sar.md b/docs/deploy/bootstrap-sar.md new file mode 100644 index 0000000..fbb5d10 --- /dev/null +++ b/docs/deploy/bootstrap-sar.md @@ -0,0 +1,48 @@ +--- +title: Bootstrap from SAR +description: Install the Khone bootstrap application from Serverless Application Repository. +--- + +# Bootstrap from SAR + +The released Khone bootstrap application installs the shared account/region resources: + +- `KhoneGateway` CloudFormation macro. +- `Custom::KhoneConfigPublisher`. +- Config artifact bucket. +- Mode A runtime API proxy layers. +- Versioned gateway Lambda zip coordinates used by the macro. + +Install the published SAR application before deploying application stacks that use +`Khone::Gateway::Service`. + +```yaml +KhoneBootstrap: + Type: AWS::Serverless::Application + Properties: + Location: + ApplicationId: arn:aws:serverlessrepo:us-east-1::applications/khone-bootstrap + SemanticVersion: 0.1.0 +``` + +The release template sets `GatewayCodeS3Bucket`, `GatewayCodeS3Key`, and optional +`GatewayCodeS3ObjectVersion` defaults to the versioned gateway artifact uploaded during release. +Application stacks do not need `CodeUri` access to the gateway source tree. + +## Source checkout installs + +When deploying the bootstrap stack directly from this repository, provide your own gateway artifact +coordinates: + +```bash +cargo lambda build --release --arm64 --output-format zip -p khone-gateway +aws s3 cp target/lambda/khone-gateway/bootstrap.zip \ + "s3://$GATEWAY_ARTIFACT_BUCKET/khone/dev/gateway/bootstrap.zip" + +make bootstrap-deploy \ + GATEWAY_CODE_S3_BUCKET="$GATEWAY_ARTIFACT_BUCKET" \ + GATEWAY_CODE_S3_KEY="khone/dev/gateway/bootstrap.zip" +``` + +The artifact bucket policy must allow every account that will deploy gateway Lambdas to read the +versioned zip object. diff --git a/docs/deploy/examples.md b/docs/deploy/examples.md index 2349a71..cd74cba 100644 --- a/docs/deploy/examples.md +++ b/docs/deploy/examples.md @@ -18,7 +18,7 @@ Lambda functions needed to show a specific integration mode or language. ## Prerequisites -- Bootstrap stack deployed with `make bootstrap-deploy`. +- Bootstrap stack deployed from the SAR release, or from source with gateway artifact parameters. - Existing LMI capacity provider ARN. - SAM CLI, Rust, `cargo-lambda`, and `SAM_CLI_BETA_RUST_CARGO_LAMBDA=1`. @@ -36,8 +36,9 @@ make examples-sam-deploy \ GATEWAY_CAPACITY_PROVIDER_ARN=arn:aws:lambda:... ``` -Layer proxy templates also need the bootstrap layer export. The Makefile resolves -`KhoneLayerArm64Arn` automatically when a template declares that parameter. +The Makefile resolves bootstrap exports such as `KhoneLayerArm64Arn` automatically when a template +declares the corresponding parameter. The gateway Lambda code comes from the macro's versioned +artifact settings. Or run SAM directly from one template directory: diff --git a/docs/deploy/index.md b/docs/deploy/index.md index 9367b40..d1f467f 100644 --- a/docs/deploy/index.md +++ b/docs/deploy/index.md @@ -5,11 +5,14 @@ description: Deploy Khone bootstrap resources, example stacks, and application g # Deploy -Deployment is split between account-level bootstrap resources and application-owned gateway stacks. +Deployment is split between account-level bootstrap resources and application gateway stacks. +- [Bootstrap from SAR](bootstrap-sar.md): install the released macro, config publisher, layers, and + gateway artifact settings. - [LMI deployment model](lmi-deployment-model.md): how the bootstrap stack, gateway Lambda, Function URL, and LMI capacity provider fit together. - [Example templates](examples.md): deploy one included example by integration mode and language. -- [SAM gateway](sam-gateway.md): define a gateway function and config resource in your own stack. +- [SAM gateway](sam-gateway.md): define a gateway resource in your own stack. +- [Release](release.md): maintainer workflow for publishing the SAR bootstrap app. If this is your first deployment, start with the [Quickstart](../start/quickstart.md). diff --git a/docs/deploy/lmi-deployment-model.md b/docs/deploy/lmi-deployment-model.md index f9d174b..f5de511 100644 --- a/docs/deploy/lmi-deployment-model.md +++ b/docs/deploy/lmi-deployment-model.md @@ -1,6 +1,6 @@ --- title: LMI deployment model -description: How Khone uses bootstrap resources, an explicit gateway function, a Function URL, and Lambda Managed Instances. +description: How Khone uses bootstrap resources, a macro-owned gateway function, a Function URL, and Lambda Managed Instances. --- # LMI deployment model @@ -27,21 +27,25 @@ Per-request responses -> gateway demux -> client ## Two stacks, different owners The bootstrap stack is shared per account and region. It installs the config bucket, config -publisher, CloudFormation macro, and Mode A layer artifacts. +publisher, CloudFormation macro, Mode A layer artifacts, and versioned gateway Lambda artifact +settings. -Application stacks own the actual gateway function and target functions. The `Khone::Gateway::Service` -resource publishes a config artifact; it does not create the gateway compute. +Application stacks own target functions and supply an existing LMI capacity provider ARN. The +`Khone::Gateway::Service` resource creates the gateway Lambda, Function URL, execution role, and +config artifact. ## Gateway function -Application templates define the gateway as an explicit `AWS::Serverless::Function`: +Application templates define the gateway with `Khone::Gateway::Service`. The macro emits a native +`AWS::Lambda::Function` with: - `Runtime: provided.al2023` - `PackageType: Zip` - `Architectures: [arm64]` -- `FunctionUrlConfig.InvokeMode: RESPONSE_STREAM` -- `CapacityProviderConfig` attached to an existing LMI capacity provider -- `KHONE_CONFIG_URI` set from `!GetAtt .ConfigS3Uri` +- `AWS::Lambda::Url` using `InvokeMode: RESPONSE_STREAM` +- `CapacityProviderConfig.LambdaManagedInstancesCapacityProviderConfig` +- `FunctionScalingConfig` +- `KHONE_CONFIG_URI` set from the generated config publisher Use response streaming on the Function URL even when a route invokes buffered targets. The gateway needs the client-facing response stream for routes that do stream. diff --git a/docs/deploy/release.md b/docs/deploy/release.md new file mode 100644 index 0000000..c53b711 --- /dev/null +++ b/docs/deploy/release.md @@ -0,0 +1,32 @@ +--- +title: Release +description: Maintain the SAR bootstrap release and versioned gateway artifact. +--- + +# Release + +Khone releases publish two related artifacts: + +- The `khone-gateway` arm64 Lambda zip at a versioned S3 key. +- The packaged SAR bootstrap application whose macro environment points at that zip. + +The release workflow is `.github/workflows/publish-release.yml`. It runs for `v*` tags or manual +dispatch, checks the tag against `VERSION` and `bootstrap/template.yaml`, builds the gateway zip, +uploads it, renders `bootstrap/template.release.yaml`, packages the SAR app, and publishes it. + +Manual release inputs: + +| Input | Description | +| --- | --- | +| `release_tag` | Existing tag to publish, such as `v0.1.0`. | +| `share_scope` | `account` or `organization`. Organization sharing requires `organizations:DescribeOrganization` and `serverlessrepo:PutApplicationPolicy`. | + +Required GitHub secrets: + +| Secret | Description | +| --- | --- | +| `AWS_ROLE_TO_ASSUME` | Release role assumed through GitHub OIDC. | +| `SAR_ARTIFACT_BUCKET` | S3 bucket for SAM-packaged SAR assets. | +| `GATEWAY_ARTIFACT_BUCKET` | Optional S3 bucket for the gateway zip. Defaults to `SAR_ARTIFACT_BUCKET` when unset. | + +Use `scripts/set-version.sh ` to update the repository version metadata before tagging. diff --git a/docs/deploy/sam-gateway.md b/docs/deploy/sam-gateway.md index cdc307e..0fecc36 100644 --- a/docs/deploy/sam-gateway.md +++ b/docs/deploy/sam-gateway.md @@ -1,12 +1,13 @@ --- title: SAM gateway -description: Define a Khone gateway config resource and gateway Lambda in your own SAM application stack. +description: Define a Khone gateway resource in your SAM application stack. --- # SAM gateway -Use this guide to deploy an application gateway rather than the demo or benchmark stack. The gateway -function is owned by your SAM template; the macro only publishes the config artifact. +Use this guide to deploy an application gateway rather than the demo or benchmark stack. The +bootstrap stack installs the SAR-versioned macro and gateway artifact settings; your application +stack supplies target functions and an existing LMI capacity provider ARN. ## 1. Add the transform @@ -18,12 +19,23 @@ Transform: Deploy the bootstrap stack first so the macro and config publisher exist in the account and region. -## 2. Publish the gateway config +## 2. Define the gateway ```yaml -GatewayConfig: +GatewayService: Type: Khone::Gateway::Service Properties: + CapacityProviderArn: !Ref GatewayCapacityProviderArn + FunctionName: !Sub "${AWS::StackName}-gateway" + Description: Khone router running on Lambda Managed Instances. + MemorySize: 2048 + Timeout: 30 + ExecutionEnvironmentMemoryGiBPerVCpu: 2.0 + PerExecutionEnvironmentMaxConcurrency: 64 + MinExecutionEnvironments: 1 + MaxExecutionEnvironments: 4 + Environment: + RUST_LOG: info ConfigPrefix: !Sub "khone/${AWS::StackName}/gateway/" GatewayConfig: MaxInflightRequests: 4096 @@ -40,61 +52,36 @@ GatewayConfig: invokeMode: buffered ``` -See [Configuration](../reference/configuration.md) for field defaults and validation rules. +The macro emits a native `AWS::Lambda::Function` using the same logical ID, plus: -## 3. Define the gateway function - -```yaml -GatewayFunction: - Type: AWS::Serverless::Function - Metadata: - BuildMethod: rust-cargolambda - Properties: - CodeUri: ../../gateway - Handler: bootstrap - Runtime: provided.al2023 - PackageType: Zip - Architectures: [arm64] - MemorySize: 2048 - Timeout: 30 - CapacityProviderConfig: - Arn: !Ref GatewayCapacityProviderArn - ExecutionEnvironmentMemoryGiBPerVCpu: 2.0 - PerExecutionEnvironmentMaxConcurrency: 64 - FunctionScalingConfig: - MinExecutionEnvironments: 1 - MaxExecutionEnvironments: 4 - FunctionUrlConfig: - AuthType: NONE - InvokeMode: RESPONSE_STREAM - Environment: - Variables: - KHONE_CONFIG_URI: !GetAtt GatewayConfig.ConfigS3Uri -``` +- `GatewayServiceKhoneConfigPublisher` +- `GatewayServiceKhoneExecutionRole` +- `GatewayServiceKhoneFunctionUrl` +- `GatewayServiceKhoneFunctionUrlPermission` when `FunctionUrlAuthType: NONE` +- `GatewayServiceKhoneLogGroup` when `LogRetentionInDays` is set -The function URL is the HTTP interface. Use `InvokeMode: RESPONSE_STREAM` even for buffered target -routes so the gateway can stream client responses when routes need it. +See [Configuration](../reference/configuration.md) for gateway config fields and +[Bootstrap macro](../reference/bootstrap-macro.md) for the complete resource contract. -## 4. Grant gateway permissions +## 3. Permissions -The gateway execution role needs: +The macro generates the gateway execution role. It grants: -- `s3:GetObject` for the config artifact bucket and prefix. -- `lambda:InvokeFunction` for buffered target routes. -- `lambda:InvokeWithResponseStream` for response-streaming target routes. -- CloudWatch Logs permissions for the gateway function. +- CloudWatch Logs write permissions. +- `s3:GetObject` for the generated config artifact prefix. +- `lambda:InvokeFunction` and `lambda:InvokeWithResponseStream` for each + `x-target-lambda` found under `Spec.paths`. -If the Function URL uses `AuthType: NONE`, place public access controls in front of it or in the -target application protocol. The demo and benchmark stacks use unauthenticated URLs for simplicity. +Use a literal Lambda ARN or an intrinsic object such as `!GetAtt HelloFunction.Arn` for +`x-target-lambda`. Literal values must be Lambda ARNs. -## 5. Output the function URL +## 4. Output the function URL ```yaml Outputs: GatewayFunctionUrl: - Value: !GetAtt GatewayFunctionUrl.FunctionUrl + Value: !GetAtt GatewayServiceKhoneFunctionUrl.FunctionUrl ``` -SAM auto-creates a `Url` resource when `FunctionUrlConfig` is set. The -`!GetAtt GatewayFunctionUrl.FunctionUrl` reference works only because the function logical ID is -exactly `GatewayFunction`. Adjust the resource name if you rename the function. +`GatewayService` itself is the Lambda function after macro expansion. `!Ref GatewayService` returns +the function name, and `!GetAtt GatewayService.Arn` returns the function ARN. diff --git a/docs/reference/bootstrap-macro.md b/docs/reference/bootstrap-macro.md index e5a1e20..40540d2 100644 --- a/docs/reference/bootstrap-macro.md +++ b/docs/reference/bootstrap-macro.md @@ -1,6 +1,6 @@ --- title: Bootstrap macro -description: Khone bootstrap resources, gateway config publisher behavior, custom resource attributes, and exported outputs. +description: Khone bootstrap resources, gateway macro behavior, generated resources, and exported outputs. --- # Bootstrap macro @@ -11,17 +11,26 @@ The bootstrap stack installs the per-account/per-region resources that applicati - `Custom::KhoneConfigPublisher` Lambda (writes the manifest). - `KhoneGateway` CloudFormation macro (expands `Khone::Gateway::Service`). - Shared Mode A runtime API proxy layers (arm64 and amd64). +- Versioned gateway Lambda artifact settings used by the macro. ## `Khone::Gateway::Service` -`Khone::Gateway::Service` is a config-artifact resource. The macro replaces it with a -`Custom::KhoneConfigPublisher` using the same logical ID, so callers can reference the original -logical ID (for example `!GetAtt GatewayConfig.ConfigS3Uri`). +`Khone::Gateway::Service` is the deployable gateway resource. The macro replaces the original +logical ID with a native `AWS::Lambda::Function`, then generates the config publisher, execution +role, Function URL, and optional log group around it. ```yaml -GatewayConfig: +GatewayService: Type: Khone::Gateway::Service Properties: + CapacityProviderArn: !Ref GatewayCapacityProviderArn + FunctionName: !Sub "${AWS::StackName}-gateway" + MemorySize: 2048 + Timeout: 30 + ExecutionEnvironmentMemoryGiBPerVCpu: 2.0 + PerExecutionEnvironmentMaxConcurrency: 64 + MinExecutionEnvironments: 1 + MaxExecutionEnvironments: 4 ConfigPrefix: !Sub "khone/${AWS::StackName}/gateway/" GatewayConfig: DefaultTimeoutMs: 2000 @@ -34,14 +43,41 @@ Supported properties: | Property | Required | Description | | --- | --- | --- | +| `CapacityProviderArn` | Yes | Existing Lambda Managed Instances capacity provider ARN. | | `GatewayConfig` | Yes | Runtime settings excluding `Spec`. Must be an object. | | `Spec` | Yes | OpenAPI-ish route document embedded into the manifest. Must be an object. | | `ConfigPrefix` | No | S3 key prefix. Defaults to `khone/${AWS::StackName}//`. | +| `FunctionName` | No | Gateway Lambda function name. | +| `Description` | No | Gateway Lambda description. | +| `MemorySize` | No | Gateway Lambda memory in MB. Defaults to `2048`. | +| `Timeout` | No | Gateway Lambda timeout in seconds. Defaults to `30`. | +| `ExecutionEnvironmentMemoryGiBPerVCpu` | No | LMI execution environment memory per vCPU. Defaults to `2.0`. | +| `PerExecutionEnvironmentMaxConcurrency` | No | LMI max concurrency per execution environment. Defaults to `64`. | +| `MinExecutionEnvironments` | No | LMI minimum execution environments. Defaults to `1`. | +| `MaxExecutionEnvironments` | No | LMI maximum execution environments. Defaults to `4`. | +| `FunctionUrlAuthType` | No | Function URL auth type, `NONE` or `AWS_IAM`. Defaults to `NONE`. | +| `Environment` | No | Gateway environment variables as a map of strings or intrinsics. `KHONE_CONFIG_URI` is reserved. | +| `TracingConfig` | No | Native Lambda tracing config. | +| `LoggingConfig` | No | Native Lambda logging config. | +| `LogRetentionInDays` | No | Creates a generated CloudWatch log group with the requested retention. | The macro additionally preserves `Condition`, `DeletionPolicy`, `DependsOn`, `Metadata`, and `UpdateReplacePolicy` from the original resource fragment. -Returned attributes (via `!GetAtt`): +The original logical ID becomes the gateway Lambda. `!Ref GatewayService` returns the function name, +and `!GetAtt GatewayService.Arn` returns the gateway Lambda ARN. + +Generated logical IDs: + +| Logical ID | Resource | +| --- | --- | +| `KhoneConfigPublisher` | `Custom::KhoneConfigPublisher` | +| `KhoneExecutionRole` | `AWS::IAM::Role` | +| `KhoneFunctionUrl` | `AWS::Lambda::Url` | +| `KhoneFunctionUrlPermission` | `AWS::Lambda::Permission` when `FunctionUrlAuthType: NONE` | +| `KhoneLogGroup` | `AWS::Logs::LogGroup` when `LogRetentionInDays` is set | + +Config publisher attributes are available from `KhoneConfigPublisher`: | Attribute | Description | | --- | --- | @@ -53,8 +89,9 @@ Returned attributes (via `!GetAtt`): ## Deployment ownership -`Khone::Gateway::Service` only publishes configuration. Define gateway compute, IAM, environment -variables, observability, and scaling directly on the explicit SAM gateway function. +`Khone::Gateway::Service` owns gateway compute, IAM, environment variables, observability, Function +URL, and scaling. Capacity providers remain external: pass the existing capacity provider ARN into +`CapacityProviderArn`. ## Bootstrap outputs diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 98f6a8a..accf9fb 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -27,8 +27,8 @@ Gateway config is YAML with PascalCase top-level fields. The config publisher em Numeric fields and the boolean `profiling` may be written as numbers/booleans or as strings. The gateway reads the manifest at startup from the `KHONE_CONFIG_URI` environment variable -(`s3:///`). The macro exposes the `ConfigS3Uri` attribute so that application templates -can set it as `KHONE_CONFIG_URI` on the gateway function; see [Bootstrap macro](bootstrap-macro.md). +(`s3:///`). The macro wires this environment variable automatically from the generated +config publisher; see [Bootstrap macro](bootstrap-macro.md). ## Header forwarding @@ -64,7 +64,9 @@ Spec: ``` Supported HTTP methods are `get`, `post`, `put`, `delete`, `patch`, `head`, and `options`. -`x-target-lambda` must be a Lambda function ARN. Paths use `{name}` placeholders. +`x-target-lambda` must be a Lambda function ARN when written as a literal string. Intrinsic +function objects such as `!GetAtt HelloFunction.Arn` are also supported. Paths use `{name}` +placeholders. ## `x-khone` diff --git a/docs/start/quickstart.md b/docs/start/quickstart.md index 9e4a280..1ffe70e 100644 --- a/docs/start/quickstart.md +++ b/docs/start/quickstart.md @@ -16,6 +16,8 @@ response-streaming Function URL, plus a working `curl` request against a sample - IAM permissions to create IAM roles, Lambda functions, CloudFormation macros, S3 buckets, and custom resources. - AWS SAM CLI, AWS CLI, Rust, and [`cargo-lambda`](https://www.cargo-lambda.info/). +- A released SAR bootstrap app, or an S3 bucket where you can upload a source-built + `khone-gateway` zip. - An existing LMI capacity provider ARN. CloudFormation accepts the `arn:aws:lambda:::capacity-provider:` form; the Makefile also accepts and normalizes the `capacity-provider/` form. @@ -25,8 +27,17 @@ running `sam build` directly. ## 1. Deploy bootstrap resources +Released bootstrap installs already carry the gateway artifact location. If you are deploying from +this source checkout, upload a gateway zip and pass its S3 coordinates: + ```bash -make bootstrap-deploy +cargo lambda build --release --arm64 --output-format zip -p khone-gateway +aws s3 cp target/lambda/khone-gateway/bootstrap.zip \ + "s3://$GATEWAY_ARTIFACT_BUCKET/khone/dev/gateway/bootstrap.zip" + +make bootstrap-deploy \ + GATEWAY_CODE_S3_BUCKET="$GATEWAY_ARTIFACT_BUCKET" \ + GATEWAY_CODE_S3_KEY="khone/dev/gateway/bootstrap.zip" ``` The bootstrap stack is shared per account and region. It creates: @@ -35,6 +46,7 @@ The bootstrap stack is shared per account and region. It creates: - the `Custom::KhoneConfigPublisher` Lambda - the `KhoneGateway` CloudFormation macro - arm64 and amd64 Mode A runtime API proxy layers +- versioned gateway Lambda artifact settings It also exports `KhoneLayerArm64Arn`, `KhoneLayerAmd64Arn`, `KhoneConfigBucketName`, and `KhoneConfigPublisherServiceToken`. @@ -45,8 +57,9 @@ It also exports `KhoneLayerArm64Arn`, `KhoneLayerAmd64Arn`, `KhoneConfigBucketNa make examples-sam-deploy GATEWAY_CAPACITY_PROVIDER_ARN=arn:aws:lambda:... ``` -This builds the gateway and sample target handlers, normalizes the capacity provider ARN if needed, -and deploys the default `adapter-node` stack named `khone-adapter-node`. +This builds the sample target handlers, normalizes the capacity provider ARN if needed, and deploys +the default `adapter-node` stack named `khone-adapter-node`. The gateway Lambda code comes from the +bootstrap macro's versioned artifact settings. Choose another example with `EXAMPLE_TEMPLATE`: diff --git a/examples/sam/templates/adapter-node/template.yaml b/examples/sam/templates/adapter-node/template.yaml index 6d55f31..45842e0 100644 --- a/examples/sam/templates/adapter-node/template.yaml +++ b/examples/sam/templates/adapter-node/template.yaml @@ -94,6 +94,23 @@ Resources: GatewayService: Type: Khone::Gateway::Service Properties: + CapacityProviderArn: !Ref GatewayCapacityProviderArn + FunctionName: !Sub '${AWS::StackName}-gateway' + Description: Khone router running on Lambda Managed Instances. + MemorySize: 2048 + Timeout: 30 + ExecutionEnvironmentMemoryGiBPerVCpu: 2.0 + PerExecutionEnvironmentMaxConcurrency: 64 + MinExecutionEnvironments: 1 + MaxExecutionEnvironments: 4 + Environment: + RUST_LOG: info + KHONE_OBSERVABILITY_VENDOR: !If [OtelEnabled, AWSXRAY, !Ref AWS::NoValue] + OTEL_PROPAGATORS: !If [OtelEnabled, "xray,tracecontext,baggage", !Ref AWS::NoValue] + OTEL_METRICS_EXPORTER: !If [OtelEnabled, "none", !Ref AWS::NoValue] + OTEL_SERVICE_NAME: !If [OtelEnabled, !Sub '${AWS::StackName}-gateway', !Ref AWS::NoValue] + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: !If [OtelEnabled, !Ref OtelTracesEndpoint, !Ref AWS::NoValue] + OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: !If [OtelEnabled, !Ref OtelTracesProtocol, !Ref AWS::NoValue] ConfigPrefix: !Sub 'khone/${AWS::StackName}/gateway/' GatewayConfig: DefaultTimeoutMs: 2000 @@ -128,61 +145,6 @@ Resources: invokeMode: response_stream timeoutMs: 8000 - GatewayFunction: - Type: AWS::Serverless::Function - Metadata: - BuildMethod: rust-cargolambda - Properties: - CodeUri: ../../../../gateway - Handler: bootstrap - Runtime: provided.al2023 - PackageType: Zip - FunctionName: !Sub '${AWS::StackName}-gateway' - Description: Khone router running on Lambda Managed Instances. - Architectures: - - arm64 - MemorySize: 2048 - Timeout: 30 - CapacityProviderConfig: - Arn: !Ref GatewayCapacityProviderArn - ExecutionEnvironmentMemoryGiBPerVCpu: 2.0 - PerExecutionEnvironmentMaxConcurrency: 64 - FunctionScalingConfig: - MinExecutionEnvironments: 1 - MaxExecutionEnvironments: 4 - FunctionUrlConfig: - AuthType: NONE - InvokeMode: RESPONSE_STREAM - Environment: - Variables: - RUST_LOG: info - KHONE_CONFIG_URI: !GetAtt GatewayService.ConfigS3Uri - KHONE_OBSERVABILITY_VENDOR: !If [OtelEnabled, AWSXRAY, !Ref AWS::NoValue] - OTEL_PROPAGATORS: !If [OtelEnabled, "xray,tracecontext,baggage", !Ref AWS::NoValue] - OTEL_METRICS_EXPORTER: !If [OtelEnabled, "none", !Ref AWS::NoValue] - OTEL_SERVICE_NAME: !If [OtelEnabled, !Sub '${AWS::StackName}-gateway', !Ref AWS::NoValue] - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: !If [OtelEnabled, !Ref OtelTracesEndpoint, !Ref AWS::NoValue] - OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: !If [OtelEnabled, !Ref OtelTracesProtocol, !Ref AWS::NoValue] - Policies: - - AWSLambdaBasicExecutionRole - - Statement: - - Sid: ReadGatewayConfig - Effect: Allow - Action: s3:GetObject - Resource: !Sub - - 'arn:${AWS::Partition}:s3:::${Bucket}/${Prefix}*' - - Bucket: !GetAtt GatewayService.BucketName - Prefix: !GetAtt GatewayService.Prefix - - Sid: InvokeTargetLambdas - Effect: Allow - Action: - - lambda:InvokeFunction - - lambda:InvokeWithResponseStream - Resource: - - !GetAtt AdapterNodeBufferedFunction.Arn - - !GetAtt AdapterNodeStreamingFunction.Arn - - !GetAtt AdapterNodeSseFunction.Arn - Outputs: AdapterNodeBufferedFunctionArn: Value: !GetAtt AdapterNodeBufferedFunction.Arn @@ -191,12 +153,12 @@ Outputs: AdapterNodeSseFunctionArn: Value: !GetAtt AdapterNodeSseFunction.Arn GatewayFunctionUrl: - Value: !GetAtt GatewayFunctionUrl.FunctionUrl + Value: !GetAtt GatewayServiceKhoneFunctionUrl.FunctionUrl GatewayServiceConfigS3Uri: - Value: !GetAtt GatewayService.ConfigS3Uri + Value: !GetAtt GatewayServiceKhoneConfigPublisher.ConfigS3Uri AdapterNodeBufferedUrl: - Value: !Sub "${GatewayFunctionUrl.FunctionUrl}adapter/node/buffered/hello" + Value: !Sub "${GatewayServiceKhoneFunctionUrl.FunctionUrl}adapter/node/buffered/hello" AdapterNodeStreamingUrl: - Value: !Sub "${GatewayFunctionUrl.FunctionUrl}adapter/node/streaming/hello" + Value: !Sub "${GatewayServiceKhoneFunctionUrl.FunctionUrl}adapter/node/streaming/hello" AdapterNodeSseUrl: - Value: !Sub "${GatewayFunctionUrl.FunctionUrl}adapter/node/sse" + Value: !Sub "${GatewayServiceKhoneFunctionUrl.FunctionUrl}adapter/node/sse" diff --git a/examples/sam/templates/adapter-rust/template.yaml b/examples/sam/templates/adapter-rust/template.yaml index f7dc514..924d183 100644 --- a/examples/sam/templates/adapter-rust/template.yaml +++ b/examples/sam/templates/adapter-rust/template.yaml @@ -52,6 +52,23 @@ Resources: GatewayService: Type: Khone::Gateway::Service Properties: + CapacityProviderArn: !Ref GatewayCapacityProviderArn + FunctionName: !Sub '${AWS::StackName}-gateway' + Description: Khone router running on Lambda Managed Instances. + MemorySize: 2048 + Timeout: 30 + ExecutionEnvironmentMemoryGiBPerVCpu: 2.0 + PerExecutionEnvironmentMaxConcurrency: 64 + MinExecutionEnvironments: 1 + MaxExecutionEnvironments: 4 + Environment: + RUST_LOG: info + KHONE_OBSERVABILITY_VENDOR: !If [OtelEnabled, AWSXRAY, !Ref AWS::NoValue] + OTEL_PROPAGATORS: !If [OtelEnabled, "xray,tracecontext,baggage", !Ref AWS::NoValue] + OTEL_METRICS_EXPORTER: !If [OtelEnabled, "none", !Ref AWS::NoValue] + OTEL_SERVICE_NAME: !If [OtelEnabled, !Sub '${AWS::StackName}-gateway', !Ref AWS::NoValue] + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: !If [OtelEnabled, !Ref OtelTracesEndpoint, !Ref AWS::NoValue] + OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: !If [OtelEnabled, !Ref OtelTracesProtocol, !Ref AWS::NoValue] ConfigPrefix: !Sub 'khone/${AWS::StackName}/gateway/' GatewayConfig: DefaultTimeoutMs: 2000 @@ -70,64 +87,12 @@ Resources: invokeMode: buffered timeoutMs: 8000 - GatewayFunction: - Type: AWS::Serverless::Function - Metadata: - BuildMethod: rust-cargolambda - Properties: - CodeUri: ../../../../gateway - Handler: bootstrap - Runtime: provided.al2023 - PackageType: Zip - FunctionName: !Sub '${AWS::StackName}-gateway' - Description: Khone router running on Lambda Managed Instances. - Architectures: - - arm64 - MemorySize: 2048 - Timeout: 30 - CapacityProviderConfig: - Arn: !Ref GatewayCapacityProviderArn - ExecutionEnvironmentMemoryGiBPerVCpu: 2.0 - PerExecutionEnvironmentMaxConcurrency: 64 - FunctionScalingConfig: - MinExecutionEnvironments: 1 - MaxExecutionEnvironments: 4 - FunctionUrlConfig: - AuthType: NONE - InvokeMode: RESPONSE_STREAM - Environment: - Variables: - RUST_LOG: info - KHONE_CONFIG_URI: !GetAtt GatewayService.ConfigS3Uri - KHONE_OBSERVABILITY_VENDOR: !If [OtelEnabled, AWSXRAY, !Ref AWS::NoValue] - OTEL_PROPAGATORS: !If [OtelEnabled, "xray,tracecontext,baggage", !Ref AWS::NoValue] - OTEL_METRICS_EXPORTER: !If [OtelEnabled, "none", !Ref AWS::NoValue] - OTEL_SERVICE_NAME: !If [OtelEnabled, !Sub '${AWS::StackName}-gateway', !Ref AWS::NoValue] - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: !If [OtelEnabled, !Ref OtelTracesEndpoint, !Ref AWS::NoValue] - OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: !If [OtelEnabled, !Ref OtelTracesProtocol, !Ref AWS::NoValue] - Policies: - - AWSLambdaBasicExecutionRole - - Statement: - - Sid: ReadGatewayConfig - Effect: Allow - Action: s3:GetObject - Resource: !Sub - - 'arn:${AWS::Partition}:s3:::${Bucket}/${Prefix}*' - - Bucket: !GetAtt GatewayService.BucketName - Prefix: !GetAtt GatewayService.Prefix - - Sid: InvokeTargetLambda - Effect: Allow - Action: - - lambda:InvokeFunction - Resource: - - !GetAtt AdapterRustBufferedFunction.Arn - Outputs: AdapterRustBufferedFunctionArn: Value: !GetAtt AdapterRustBufferedFunction.Arn GatewayFunctionUrl: - Value: !GetAtt GatewayFunctionUrl.FunctionUrl + Value: !GetAtt GatewayServiceKhoneFunctionUrl.FunctionUrl GatewayServiceConfigS3Uri: - Value: !GetAtt GatewayService.ConfigS3Uri + Value: !GetAtt GatewayServiceKhoneConfigPublisher.ConfigS3Uri AdapterRustBufferedUrl: - Value: !Sub "${GatewayFunctionUrl.FunctionUrl}adapter/rust/hello" + Value: !Sub "${GatewayServiceKhoneFunctionUrl.FunctionUrl}adapter/rust/hello" diff --git a/examples/sam/templates/layer-proxy-node/template.yaml b/examples/sam/templates/layer-proxy-node/template.yaml index 18de4f3..ffb27eb 100644 --- a/examples/sam/templates/layer-proxy-node/template.yaml +++ b/examples/sam/templates/layer-proxy-node/template.yaml @@ -69,6 +69,23 @@ Resources: GatewayService: Type: Khone::Gateway::Service Properties: + CapacityProviderArn: !Ref GatewayCapacityProviderArn + FunctionName: !Sub '${AWS::StackName}-gateway' + Description: Khone router running on Lambda Managed Instances. + MemorySize: 2048 + Timeout: 30 + ExecutionEnvironmentMemoryGiBPerVCpu: 2.0 + PerExecutionEnvironmentMaxConcurrency: 64 + MinExecutionEnvironments: 1 + MaxExecutionEnvironments: 4 + Environment: + RUST_LOG: info + KHONE_OBSERVABILITY_VENDOR: !If [OtelEnabled, AWSXRAY, !Ref AWS::NoValue] + OTEL_PROPAGATORS: !If [OtelEnabled, "xray,tracecontext,baggage", !Ref AWS::NoValue] + OTEL_METRICS_EXPORTER: !If [OtelEnabled, "none", !Ref AWS::NoValue] + OTEL_SERVICE_NAME: !If [OtelEnabled, !Sub '${AWS::StackName}-gateway', !Ref AWS::NoValue] + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: !If [OtelEnabled, !Ref OtelTracesEndpoint, !Ref AWS::NoValue] + OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: !If [OtelEnabled, !Ref OtelTracesProtocol, !Ref AWS::NoValue] ConfigPrefix: !Sub 'khone/${AWS::StackName}/gateway/' GatewayConfig: DefaultTimeoutMs: 2000 @@ -87,64 +104,12 @@ Resources: invokeMode: response_stream timeoutMs: 8000 - GatewayFunction: - Type: AWS::Serverless::Function - Metadata: - BuildMethod: rust-cargolambda - Properties: - CodeUri: ../../../../gateway - Handler: bootstrap - Runtime: provided.al2023 - PackageType: Zip - FunctionName: !Sub '${AWS::StackName}-gateway' - Description: Khone router running on Lambda Managed Instances. - Architectures: - - arm64 - MemorySize: 2048 - Timeout: 30 - CapacityProviderConfig: - Arn: !Ref GatewayCapacityProviderArn - ExecutionEnvironmentMemoryGiBPerVCpu: 2.0 - PerExecutionEnvironmentMaxConcurrency: 64 - FunctionScalingConfig: - MinExecutionEnvironments: 1 - MaxExecutionEnvironments: 4 - FunctionUrlConfig: - AuthType: NONE - InvokeMode: RESPONSE_STREAM - Environment: - Variables: - RUST_LOG: info - KHONE_CONFIG_URI: !GetAtt GatewayService.ConfigS3Uri - KHONE_OBSERVABILITY_VENDOR: !If [OtelEnabled, AWSXRAY, !Ref AWS::NoValue] - OTEL_PROPAGATORS: !If [OtelEnabled, "xray,tracecontext,baggage", !Ref AWS::NoValue] - OTEL_METRICS_EXPORTER: !If [OtelEnabled, "none", !Ref AWS::NoValue] - OTEL_SERVICE_NAME: !If [OtelEnabled, !Sub '${AWS::StackName}-gateway', !Ref AWS::NoValue] - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: !If [OtelEnabled, !Ref OtelTracesEndpoint, !Ref AWS::NoValue] - OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: !If [OtelEnabled, !Ref OtelTracesProtocol, !Ref AWS::NoValue] - Policies: - - AWSLambdaBasicExecutionRole - - Statement: - - Sid: ReadGatewayConfig - Effect: Allow - Action: s3:GetObject - Resource: !Sub - - 'arn:${AWS::Partition}:s3:::${Bucket}/${Prefix}*' - - Bucket: !GetAtt GatewayService.BucketName - Prefix: !GetAtt GatewayService.Prefix - - Sid: InvokeTargetLambda - Effect: Allow - Action: - - lambda:InvokeWithResponseStream - Resource: - - !GetAtt LayerProxyNodeHelloFunction.Arn - Outputs: LayerProxyNodeHelloFunctionArn: Value: !GetAtt LayerProxyNodeHelloFunction.Arn GatewayFunctionUrl: - Value: !GetAtt GatewayFunctionUrl.FunctionUrl + Value: !GetAtt GatewayServiceKhoneFunctionUrl.FunctionUrl GatewayServiceConfigS3Uri: - Value: !GetAtt GatewayService.ConfigS3Uri + Value: !GetAtt GatewayServiceKhoneConfigPublisher.ConfigS3Uri LayerProxyNodeHelloUrl: - Value: !Sub "${GatewayFunctionUrl.FunctionUrl}layer-proxy/node/hello" + Value: !Sub "${GatewayServiceKhoneFunctionUrl.FunctionUrl}layer-proxy/node/hello" diff --git a/examples/sam/templates/layer-proxy-python/template.yaml b/examples/sam/templates/layer-proxy-python/template.yaml index dce4e60..7c13d38 100644 --- a/examples/sam/templates/layer-proxy-python/template.yaml +++ b/examples/sam/templates/layer-proxy-python/template.yaml @@ -58,6 +58,23 @@ Resources: GatewayService: Type: Khone::Gateway::Service Properties: + CapacityProviderArn: !Ref GatewayCapacityProviderArn + FunctionName: !Sub '${AWS::StackName}-gateway' + Description: Khone router running on Lambda Managed Instances. + MemorySize: 2048 + Timeout: 30 + ExecutionEnvironmentMemoryGiBPerVCpu: 2.0 + PerExecutionEnvironmentMaxConcurrency: 64 + MinExecutionEnvironments: 1 + MaxExecutionEnvironments: 4 + Environment: + RUST_LOG: info + KHONE_OBSERVABILITY_VENDOR: !If [OtelEnabled, AWSXRAY, !Ref AWS::NoValue] + OTEL_PROPAGATORS: !If [OtelEnabled, "xray,tracecontext,baggage", !Ref AWS::NoValue] + OTEL_METRICS_EXPORTER: !If [OtelEnabled, "none", !Ref AWS::NoValue] + OTEL_SERVICE_NAME: !If [OtelEnabled, !Sub '${AWS::StackName}-gateway', !Ref AWS::NoValue] + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: !If [OtelEnabled, !Ref OtelTracesEndpoint, !Ref AWS::NoValue] + OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: !If [OtelEnabled, !Ref OtelTracesProtocol, !Ref AWS::NoValue] ConfigPrefix: !Sub 'khone/${AWS::StackName}/gateway/' GatewayConfig: DefaultTimeoutMs: 2000 @@ -76,64 +93,12 @@ Resources: invokeMode: response_stream timeoutMs: 8000 - GatewayFunction: - Type: AWS::Serverless::Function - Metadata: - BuildMethod: rust-cargolambda - Properties: - CodeUri: ../../../../gateway - Handler: bootstrap - Runtime: provided.al2023 - PackageType: Zip - FunctionName: !Sub '${AWS::StackName}-gateway' - Description: Khone router running on Lambda Managed Instances. - Architectures: - - arm64 - MemorySize: 2048 - Timeout: 30 - CapacityProviderConfig: - Arn: !Ref GatewayCapacityProviderArn - ExecutionEnvironmentMemoryGiBPerVCpu: 2.0 - PerExecutionEnvironmentMaxConcurrency: 64 - FunctionScalingConfig: - MinExecutionEnvironments: 1 - MaxExecutionEnvironments: 4 - FunctionUrlConfig: - AuthType: NONE - InvokeMode: RESPONSE_STREAM - Environment: - Variables: - RUST_LOG: info - KHONE_CONFIG_URI: !GetAtt GatewayService.ConfigS3Uri - KHONE_OBSERVABILITY_VENDOR: !If [OtelEnabled, AWSXRAY, !Ref AWS::NoValue] - OTEL_PROPAGATORS: !If [OtelEnabled, "xray,tracecontext,baggage", !Ref AWS::NoValue] - OTEL_METRICS_EXPORTER: !If [OtelEnabled, "none", !Ref AWS::NoValue] - OTEL_SERVICE_NAME: !If [OtelEnabled, !Sub '${AWS::StackName}-gateway', !Ref AWS::NoValue] - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: !If [OtelEnabled, !Ref OtelTracesEndpoint, !Ref AWS::NoValue] - OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: !If [OtelEnabled, !Ref OtelTracesProtocol, !Ref AWS::NoValue] - Policies: - - AWSLambdaBasicExecutionRole - - Statement: - - Sid: ReadGatewayConfig - Effect: Allow - Action: s3:GetObject - Resource: !Sub - - 'arn:${AWS::Partition}:s3:::${Bucket}/${Prefix}*' - - Bucket: !GetAtt GatewayService.BucketName - Prefix: !GetAtt GatewayService.Prefix - - Sid: InvokeTargetLambda - Effect: Allow - Action: - - lambda:InvokeWithResponseStream - Resource: - - !GetAtt LayerProxyPythonHelloFunction.Arn - Outputs: LayerProxyPythonHelloFunctionArn: Value: !GetAtt LayerProxyPythonHelloFunction.Arn GatewayFunctionUrl: - Value: !GetAtt GatewayFunctionUrl.FunctionUrl + Value: !GetAtt GatewayServiceKhoneFunctionUrl.FunctionUrl GatewayServiceConfigS3Uri: - Value: !GetAtt GatewayService.ConfigS3Uri + Value: !GetAtt GatewayServiceKhoneConfigPublisher.ConfigS3Uri LayerProxyPythonHelloUrl: - Value: !Sub "${GatewayFunctionUrl.FunctionUrl}layer-proxy/python/hello" + Value: !Sub "${GatewayServiceKhoneFunctionUrl.FunctionUrl}layer-proxy/python/hello" diff --git a/examples/sam/templates/native-batch-node/template.yaml b/examples/sam/templates/native-batch-node/template.yaml index 18d59ab..a573e42 100644 --- a/examples/sam/templates/native-batch-node/template.yaml +++ b/examples/sam/templates/native-batch-node/template.yaml @@ -76,6 +76,23 @@ Resources: GatewayService: Type: Khone::Gateway::Service Properties: + CapacityProviderArn: !Ref GatewayCapacityProviderArn + FunctionName: !Sub '${AWS::StackName}-gateway' + Description: Khone router running on Lambda Managed Instances. + MemorySize: 2048 + Timeout: 30 + ExecutionEnvironmentMemoryGiBPerVCpu: 2.0 + PerExecutionEnvironmentMaxConcurrency: 64 + MinExecutionEnvironments: 1 + MaxExecutionEnvironments: 4 + Environment: + RUST_LOG: info + KHONE_OBSERVABILITY_VENDOR: !If [OtelEnabled, AWSXRAY, !Ref AWS::NoValue] + OTEL_PROPAGATORS: !If [OtelEnabled, "xray,tracecontext,baggage", !Ref AWS::NoValue] + OTEL_METRICS_EXPORTER: !If [OtelEnabled, "none", !Ref AWS::NoValue] + OTEL_SERVICE_NAME: !If [OtelEnabled, !Sub '${AWS::StackName}-gateway', !Ref AWS::NoValue] + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: !If [OtelEnabled, !Ref OtelTracesEndpoint, !Ref AWS::NoValue] + OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: !If [OtelEnabled, !Ref OtelTracesProtocol, !Ref AWS::NoValue] ConfigPrefix: !Sub 'khone/${AWS::StackName}/gateway/' GatewayConfig: DefaultTimeoutMs: 2000 @@ -131,74 +148,20 @@ Resources: invokeMode: response_stream timeoutMs: 8000 - GatewayFunction: - Type: AWS::Serverless::Function - Metadata: - BuildMethod: rust-cargolambda - Properties: - CodeUri: ../../../../gateway - Handler: bootstrap - Runtime: provided.al2023 - PackageType: Zip - FunctionName: !Sub '${AWS::StackName}-gateway' - Description: Khone router running on Lambda Managed Instances. - Architectures: - - arm64 - MemorySize: 2048 - Timeout: 30 - CapacityProviderConfig: - Arn: !Ref GatewayCapacityProviderArn - ExecutionEnvironmentMemoryGiBPerVCpu: 2.0 - PerExecutionEnvironmentMaxConcurrency: 64 - FunctionScalingConfig: - MinExecutionEnvironments: 1 - MaxExecutionEnvironments: 4 - FunctionUrlConfig: - AuthType: NONE - InvokeMode: RESPONSE_STREAM - Environment: - Variables: - RUST_LOG: info - KHONE_CONFIG_URI: !GetAtt GatewayService.ConfigS3Uri - KHONE_OBSERVABILITY_VENDOR: !If [OtelEnabled, AWSXRAY, !Ref AWS::NoValue] - OTEL_PROPAGATORS: !If [OtelEnabled, "xray,tracecontext,baggage", !Ref AWS::NoValue] - OTEL_METRICS_EXPORTER: !If [OtelEnabled, "none", !Ref AWS::NoValue] - OTEL_SERVICE_NAME: !If [OtelEnabled, !Sub '${AWS::StackName}-gateway', !Ref AWS::NoValue] - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: !If [OtelEnabled, !Ref OtelTracesEndpoint, !Ref AWS::NoValue] - OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: !If [OtelEnabled, !Ref OtelTracesProtocol, !Ref AWS::NoValue] - Policies: - - AWSLambdaBasicExecutionRole - - Statement: - - Sid: ReadGatewayConfig - Effect: Allow - Action: s3:GetObject - Resource: !Sub - - 'arn:${AWS::Partition}:s3:::${Bucket}/${Prefix}*' - - Bucket: !GetAtt GatewayService.BucketName - Prefix: !GetAtt GatewayService.Prefix - - Sid: InvokeTargetLambdas - Effect: Allow - Action: - - lambda:InvokeFunction - - lambda:InvokeWithResponseStream - Resource: - - !GetAtt NativeBatchNodeBufferedFunction.Arn - - !GetAtt NativeBatchNodeStreamingFunction.Arn - Outputs: NativeBatchNodeBufferedFunctionArn: Value: !GetAtt NativeBatchNodeBufferedFunction.Arn NativeBatchNodeStreamingFunctionArn: Value: !GetAtt NativeBatchNodeStreamingFunction.Arn GatewayFunctionUrl: - Value: !GetAtt GatewayFunctionUrl.FunctionUrl + Value: !GetAtt GatewayServiceKhoneFunctionUrl.FunctionUrl GatewayServiceConfigS3Uri: - Value: !GetAtt GatewayService.ConfigS3Uri + Value: !GetAtt GatewayServiceKhoneConfigPublisher.ConfigS3Uri NativeBatchNodeBufferedUrl: - Value: !Sub "${GatewayFunctionUrl.FunctionUrl}native-batch/node/buffered/hello" + Value: !Sub "${GatewayServiceKhoneFunctionUrl.FunctionUrl}native-batch/node/buffered/hello" NativeBatchNodeAdaptiveUrl: - Value: !Sub "${GatewayFunctionUrl.FunctionUrl}native-batch/node/adaptive/hello" + Value: !Sub "${GatewayServiceKhoneFunctionUrl.FunctionUrl}native-batch/node/adaptive/hello" NativeBatchNodeTargetAwareUrl: - Value: !Sub "${GatewayFunctionUrl.FunctionUrl}native-batch/node/target-aware/hello" + Value: !Sub "${GatewayServiceKhoneFunctionUrl.FunctionUrl}native-batch/node/target-aware/hello" NativeBatchNodeStreamingUrl: - Value: !Sub "${GatewayFunctionUrl.FunctionUrl}native-batch/node/streaming/hello" + Value: !Sub "${GatewayServiceKhoneFunctionUrl.FunctionUrl}native-batch/node/streaming/hello" diff --git a/scripts/normalize-yaml-scalar.sh b/scripts/normalize-yaml-scalar.sh new file mode 100755 index 0000000..f7e9927 --- /dev/null +++ b/scripts/normalize-yaml-scalar.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +ruby -e ' + require "yaml" + + scalar = ARGV.fetch(0) + document = YAML.safe_load("value: #{scalar}\n", permitted_classes: [], aliases: false) || {} + value = document["value"] + puts(value.nil? ? "" : value.to_s) +' "$1" diff --git a/scripts/render-bootstrap-release-template.py b/scripts/render-bootstrap-release-template.py new file mode 100755 index 0000000..773fd77 --- /dev/null +++ b/scripts/render-bootstrap-release-template.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Render the bootstrap SAR template with immutable gateway artifact defaults.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path + + +SEMVER_RE = re.compile( + r"^(0|[1-9][0-9]*)\." + r"(0|[1-9][0-9]*)\." + r"(0|[1-9][0-9]*)" + r"(?:-((?:0|[1-9][0-9]*|[0-9A-Za-z-][0-9A-Za-z-]*)(?:\.(?:0|[1-9][0-9]*|[0-9A-Za-z-][0-9A-Za-z-]*))*))?" + r"(?:\+([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?$" +) + + +def yaml_scalar(value: str) -> str: + return json.dumps(value) + + +def replace_parameter_default(lines: list[str], parameter_name: str, value: str) -> None: + header = f" {parameter_name}:\n" + try: + start = lines.index(header) + except ValueError as exc: + raise SystemExit(f"Parameter {parameter_name} was not found.") from exc + + for index in range(start + 1, len(lines)): + line = lines[index] + if line.startswith(" ") and not line.startswith(" ") and line.strip(): + break + if line.startswith(" Default:"): + lines[index] = f" Default: {yaml_scalar(value)}\n" + return + + raise SystemExit(f"Parameter {parameter_name} does not have a Default field.") + + +def replace_semantic_version(lines: list[str], version: str) -> None: + for index, line in enumerate(lines): + if line.startswith(" SemanticVersion:"): + lines[index] = f" SemanticVersion: {yaml_scalar(version)}\n" + return + + raise SystemExit("Metadata.AWS::ServerlessRepo::Application.SemanticVersion was not found.") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--template", type=Path, default=Path("bootstrap/template.yaml")) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--version", required=True) + parser.add_argument("--gateway-code-s3-bucket", required=True) + parser.add_argument("--gateway-code-s3-key", required=True) + parser.add_argument("--gateway-code-s3-object-version", default="") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + if not SEMVER_RE.fullmatch(args.version): + raise SystemExit("--version must be a semantic version without a leading v.") + if not args.gateway_code_s3_bucket: + raise SystemExit("--gateway-code-s3-bucket is required.") + if not args.gateway_code_s3_key: + raise SystemExit("--gateway-code-s3-key is required.") + + lines = args.template.read_text(encoding="utf-8").splitlines(keepends=True) + replace_semantic_version(lines, args.version) + replace_parameter_default(lines, "GatewayCodeS3Bucket", args.gateway_code_s3_bucket) + replace_parameter_default(lines, "GatewayCodeS3Key", args.gateway_code_s3_key) + replace_parameter_default(lines, "GatewayCodeS3ObjectVersion", args.gateway_code_s3_object_version) + + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text("".join(lines), encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/scripts/set-version.sh b/scripts/set-version.sh new file mode 100755 index 0000000..eb86cba --- /dev/null +++ b/scripts/set-version.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +version="$1" +semver_regex='^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*|[0-9A-Za-z-][0-9A-Za-z-]*)(\.(0|[1-9][0-9]*|[0-9A-Za-z-][0-9A-Za-z-]*))*))?(\+([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' +if [[ ! "$version" =~ $semver_regex ]]; then + echo "version must be semantic version without a leading v" >&2 + exit 2 +fi + +printf '%s\n' "$version" > VERSION + +python3 - "$version" <<'PY' +import sys +from pathlib import Path + +version = sys.argv[1] +path = Path("bootstrap/template.yaml") +lines = path.read_text(encoding="utf-8").splitlines(keepends=True) +for index, line in enumerate(lines): + if line.startswith(" SemanticVersion:"): + lines[index] = f" SemanticVersion: {version}\n" + break +else: + raise SystemExit("Metadata.AWS::ServerlessRepo::Application.SemanticVersion was not found.") + +path.write_text("".join(lines), encoding="utf-8") +PY + +echo "Updated Khone version to $version." diff --git a/website/app/page.tsx b/website/app/page.tsx index 0280dce..9bd8ecf 100644 --- a/website/app/page.tsx +++ b/website/app/page.tsx @@ -175,9 +175,9 @@ export default function HomePage() {

- The KhoneGateway macro publishes the gateway config and spec to S3. Your - stack defines the gateway as an explicit AWS::Serverless::Function backed - by an LMI capacity provider. + The KhoneGateway macro creates the gateway Lambda, Function URL, execution + role, and config artifact. Your stack supplies target functions and an existing LMI + capacity provider.

@@ -193,9 +193,14 @@ export default function HomePage() { - KhoneGateway Resources: - GatewayConfig: + GatewayService: Type: Khone::Gateway::Service Properties: + CapacityProviderArn: !Ref GatewayCapacityProviderArn + MemorySize: 2048 + Timeout: 30 + ExecutionEnvironmentMemoryGiBPerVCpu: 2.0 + PerExecutionEnvironmentMaxConcurrency: 64 ConfigPrefix: !Sub "khone/\${AWS::StackName}/gateway/" GatewayConfig: DefaultTimeoutMs: 2000 @@ -208,25 +213,7 @@ Resources: x-khone: maxBatchSize: 16 maxWaitMs: 35 - invokeMode: response_stream - - GatewayFunction: - Type: AWS::Serverless::Function - Metadata: - BuildMethod: rust-cargolambda - Properties: - CodeUri: ../../gateway - Handler: bootstrap - Runtime: provided.al2023 - Architectures: [arm64] - CapacityProviderConfig: - Arn: !Ref GatewayCapacityProviderArn - Environment: - Variables: - KHONE_CONFIG_URI: !GetAtt GatewayConfig.ConfigS3Uri - FunctionUrlConfig: - AuthType: NONE - InvokeMode: RESPONSE_STREAM`} + invokeMode: response_stream`}