From 466f792ee4721827ebb7f6fe7023e4bbd39097b9 Mon Sep 17 00:00:00 2001 From: Marco Ieni <11428655+marcoieni@users.noreply.github.com> Date: Wed, 12 Aug 2026 18:16:37 +0200 Subject: [PATCH] Provision the Graviton 5 rustc-perf collector Allocate an M9g Dedicated Host in us-east-2 and place an m9g.12xlarge partition on its isolated no-ingress network. Use Session Manager for administration, require an approved host quota, and smoke-test cycles, instructions, branches, and branch-miss PMU counters during boot. --- .../collector/.terraform.lock.hcl | 26 +++++++ .../rustc-perf-prod/collector/terragrunt.hcl | 20 ++++++ .../rustc-perf-collector/_terraform.tf | 10 +++ .../modules/rustc-perf-collector/data.tf | 52 ++++++++++++++ .../rustc-perf-collector/dedicated-host.tf | 42 +++++++++++ .../modules/rustc-perf-collector/iam.tf | 70 +++++++++++++++++++ .../modules/rustc-perf-collector/instance.tf | 67 ++++++++++++++++++ .../modules/rustc-perf-collector/network.tf | 68 ++++++++++++++++++ .../modules/rustc-perf-collector/outputs.tf | 24 +++++++ .../modules/rustc-perf-collector/user-data.sh | 44 ++++++++++++ .../modules/rustc-perf-collector/variables.tf | 9 +++ 11 files changed, 432 insertions(+) create mode 100644 terragrunt/accounts/rustc-perf-prod/collector/.terraform.lock.hcl create mode 100644 terragrunt/accounts/rustc-perf-prod/collector/terragrunt.hcl create mode 100644 terragrunt/modules/rustc-perf-collector/_terraform.tf create mode 100644 terragrunt/modules/rustc-perf-collector/data.tf create mode 100644 terragrunt/modules/rustc-perf-collector/dedicated-host.tf create mode 100644 terragrunt/modules/rustc-perf-collector/iam.tf create mode 100644 terragrunt/modules/rustc-perf-collector/instance.tf create mode 100644 terragrunt/modules/rustc-perf-collector/network.tf create mode 100644 terragrunt/modules/rustc-perf-collector/outputs.tf create mode 100644 terragrunt/modules/rustc-perf-collector/user-data.sh create mode 100644 terragrunt/modules/rustc-perf-collector/variables.tf diff --git a/terragrunt/accounts/rustc-perf-prod/collector/.terraform.lock.hcl b/terragrunt/accounts/rustc-perf-prod/collector/.terraform.lock.hcl new file mode 100644 index 000000000..bd3bce1d6 --- /dev/null +++ b/terragrunt/accounts/rustc-perf-prod/collector/.terraform.lock.hcl @@ -0,0 +1,26 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.58.0" + constraints = "~> 6.27" + hashes = [ + "h1:2kpake4zZKRX5437QVIRU3qFYH6Bjw/QE1fgVCPOrUg=", + "zh:1221253beee5629fb503d79cebc9bc661279cbc4be5d01db9ab4c1b702108250", + "zh:132bd0925bdc4b72446ac750b7ccb1e19b9ba8fbb6df57b2c1423314d2195d4f", + "zh:18cda250b9e82b753808715893c8927f132273c00ffae7a697d65ac1cb577e48", + "zh:204c944f1fb7f440a335bb2083c9691a9d1f677aea9701025dd5816aee41f0ba", + "zh:2dc41df289f2b10a01e650cdd73699955f0ab0645d09cfb114a8cd0f4cc4ede7", + "zh:345633dfa9a234659d52aadd126e6dce658518c3ab5cbf6d871221287ed5ec56", + "zh:4dadcced73e742903158bc9838936d911f3fa4c2c37b5591c1a28f8f2a1902a6", + "zh:5bc60cc2b8c093da98b211d9f6c21c9ecec0f21b944d9ecbe3961fba33086e80", + "zh:6cc8f084938b0033a9c0c910989919dad6b1683e76e0afa1a5c604e39f398a75", + "zh:7db214647f79de9a033b5dfd6cbfaa42d53c4d056b32cb3acc7ad99306dd548a", + "zh:9078589ec881cee7ed9403af262c98ff257fb3e1baae72ff6429a398b1c730af", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:bd5bce6aec4d4922b1127b8575688bd4bc4279670ee28d198ede404709826c7c", + "zh:cd900ecf56d21023873898b06e40234f3f4d350f2343b7d9b980d6c5cb604fae", + "zh:dbe93b276a84421026b956c3c5b4eb8897da6cbb54b93cb89f5d6ebbd30805ca", + "zh:f6b6c7bb2dbf04ee085e5c22f7a65b3ccaebf368ed95584dc0dfee8a22771056", + ] +} diff --git a/terragrunt/accounts/rustc-perf-prod/collector/terragrunt.hcl b/terragrunt/accounts/rustc-perf-prod/collector/terragrunt.hcl new file mode 100644 index 000000000..b5ca6e3f9 --- /dev/null +++ b/terragrunt/accounts/rustc-perf-prod/collector/terragrunt.hcl @@ -0,0 +1,20 @@ +terraform { + source = "../../../modules//rustc-perf-collector" +} + +include { + path = find_in_parent_folders() + merge_strategy = "deep" +} + +dependency "quota" { + config_path = "../ec2-quota" +} + +inputs = { + # Reading an output creates a Terragrunt dependency on the separately + # applied quota request. The module also checks the account's *current* + # quota, so a requested-but-not-yet-approved increase cannot allocate the + # Dedicated Host and start billing it. + required_dedicated_hosts = dependency.quota.outputs.requested_value +} diff --git a/terragrunt/modules/rustc-perf-collector/_terraform.tf b/terragrunt/modules/rustc-perf-collector/_terraform.tf new file mode 100644 index 000000000..931e83bf8 --- /dev/null +++ b/terragrunt/modules/rustc-perf-collector/_terraform.tf @@ -0,0 +1,10 @@ +terraform { + required_version = "~> 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.27" + } + } +} diff --git a/terragrunt/modules/rustc-perf-collector/data.tf b/terragrunt/modules/rustc-perf-collector/data.tf new file mode 100644 index 000000000..358742b64 --- /dev/null +++ b/terragrunt/modules/rustc-perf-collector/data.tf @@ -0,0 +1,52 @@ +locals { + // The project requested the M9g (Graviton 5) family on a Dedicated Host. + // Start with a 12xlarge partition: it provides 48 vCPUs and 192 GiB while + // using one of the eight 12xlarge slots listed for an empty M9g host, leaving + // capacity available for future M9g collectors. + instance_family = "m9g" + instance_type = "m9g.12xlarge" + + // New instance families are not necessarily offered in every AZ. Select a + // stable AZ ID from AWS's live offerings instead of hard-coding an AZ name + // that can map differently between accounts. + availability_zone_id = try(sort(data.aws_ec2_instance_type_offerings.collector.locations)[0], null) +} + +// Use Canonical's official arm64 image: Graviton cannot boot the repository's +// usual amd64 AMIs. most_recent is safe here because instance.tf ignores later +// AMI changes to avoid silently replacing a benchmark machine. +data "aws_ami" "ubuntu" { + most_recent = true + owners = ["099720109477"] # Canonical + + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-arm64-server-*"] + } + + filter { + name = "architecture" + values = ["arm64"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } +} + +data "aws_ec2_instance_type_offerings" "collector" { + filter { + name = "instance-type" + values = [local.instance_type] + } + + location_type = "availability-zone-id" +} + +// The quota request records the desired value; this data source reads the +// actually approved value used by the host-allocation precondition. +data "aws_servicequotas_service_quota" "m9g_hosts" { + service_code = "ec2" + quota_code = "L-9F9F275C" +} diff --git a/terragrunt/modules/rustc-perf-collector/dedicated-host.tf b/terragrunt/modules/rustc-perf-collector/dedicated-host.tf new file mode 100644 index 000000000..c015026dc --- /dev/null +++ b/terragrunt/modules/rustc-perf-collector/dedicated-host.tf @@ -0,0 +1,42 @@ +// A Dedicated Host is an entire physical EC2 server allocated to this AWS +// account. Selecting the family rather than one instance type lets us divide +// its capacity among supported M9g sizes instead of fixing the host layout at +// allocation time. aws_ec2_host allocates an On-Demand host; it deliberately +// does not purchase a one- or three-year Dedicated Host Reservation because +// that is a separate, irreversible billing commitment with payment-term input. +resource "aws_ec2_host" "collector" { + availability_zone = aws_subnet.collector.availability_zone + instance_family = local.instance_family + + // Require every instance to name this host explicitly. This prevents an + // unrelated M9g launch in the account from silently consuming benchmark + // capacity through EC2 auto-placement. + auto_placement = "off" + + // M9g supports Dedicated Host recovery, so EC2 can allocate replacement + // hardware after supported power or network failures. It does not cover + // every failure mode (notably scheduled host retirement), which still needs + // operator action. + host_recovery = "on" + + tags = { + Name = "rustc-perf-graviton5" + Environment = "prod" + Service = "rustc-perf" + } + + lifecycle { + precondition { + condition = local.availability_zone_id != null + error_message = "${local.instance_type} is not offered in this AWS region." + } + + // A quota request can exist in state while AWS is still reviewing it. + // Gate host allocation on the applied quota because billing begins when + // the Dedicated Host is allocated, even if it contains no instances. + precondition { + condition = data.aws_servicequotas_service_quota.m9g_hosts.value >= var.required_dedicated_hosts + error_message = "The Running Dedicated m9g Hosts quota increase must be approved before allocating the host." + } + } +} diff --git a/terragrunt/modules/rustc-perf-collector/iam.tf b/terragrunt/modules/rustc-perf-collector/iam.tf new file mode 100644 index 000000000..eb2af2125 --- /dev/null +++ b/terragrunt/modules/rustc-perf-collector/iam.tf @@ -0,0 +1,70 @@ +// The instance role is intentionally limited to Systems Manager. Workload +// access to rustc-perf storage or secrets should be added separately and with +// narrower policies once the collector is enrolled. +resource "aws_iam_role" "collector" { + name = "rustc-perf-collector" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = "sts:AssumeRole" + Principal = { + Service = "ec2.amazonaws.com" + } + } + ] + }) +} + +resource "aws_iam_role_policy_attachment" "ssm" { + // This managed policy lets the *agent* establish its outbound SSM control + // channel. Human permission to start a session comes from Identity Center. + role = aws_iam_role.collector.name + policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" +} + +// EC2 receives IAM roles through an instance-profile wrapper rather than by +// attaching aws_iam_role directly to aws_instance. +resource "aws_iam_instance_profile" "collector" { + name = "rustc-perf-collector" + role = aws_iam_role.collector.name +} + +// SSM documents are account-and-region-local configuration consumed by +// Systems Manager. This one defines the default interactive shell used by +// `aws ssm start-session`; it is not a startup script for the instance. +// Managing it here makes the no-SSH access path available without a manual +// console step, runs operators as Ubuntu's normal sudo-capable user, and puts +// finite idle and total limits on forgotten sessions. +resource "aws_ssm_document" "session_preferences" { + name = "SSM-SessionManagerRunShell" + document_format = "JSON" + document_type = "Session" + + content = jsonencode({ + schemaVersion = "1.0" + description = "Regional Session Manager settings for rustc-perf-prod" + sessionType = "Standard_Stream" + inputs = { + // There is no account-local log destination yet, so session contents are + // not streamed. Session API activity is still captured by CloudTrail. + s3BucketName = "" + s3KeyPrefix = "" + s3EncryptionEnabled = true + cloudWatchLogGroupName = "" + cloudWatchEncryptionEnabled = true + cloudWatchStreamingEnabled = false + kmsKeyId = "" + runAsEnabled = true + runAsDefaultUser = "ubuntu" + idleSessionTimeout = "20" + maxSessionDuration = "240" + shellProfile = { + windows = "" + linux = "cd /home/ubuntu" + } + } + }) +} diff --git a/terragrunt/modules/rustc-perf-collector/instance.tf b/terragrunt/modules/rustc-perf-collector/instance.tf new file mode 100644 index 000000000..56cfce979 --- /dev/null +++ b/terragrunt/modules/rustc-perf-collector/instance.tf @@ -0,0 +1,67 @@ +resource "aws_instance" "collector" { + ami = data.aws_ami.ubuntu.id + instance_type = local.instance_type + // host tenancy plus the explicit host ID places this instance on the + // account's M9g physical server instead of the normal shared EC2 fleet. + host_id = aws_ec2_host.collector.id + tenancy = "host" + subnet_id = aws_subnet.collector.id + vpc_security_group_ids = [aws_security_group.collector.id] + // The public address is only for outbound package, toolchain, and benchmark + // downloads. network.tf defines no ingress rules; administration goes + // through SSM. This avoids a continuously billed NAT gateway for one host. + associate_public_ip_address = true + iam_instance_profile = aws_iam_instance_profile.collector.name + + // Accidental deletion would discard a calibrated benchmark environment. A + // guest shutdown stops the partition rather than terminating its EBS volume; + // it does not stop Dedicated Host billing while the physical host is + // allocated. Detailed EC2 monitoring is unrelated to PMU counters. + disable_api_termination = true + ebs_optimized = true + instance_initiated_shutdown_behavior = "stop" + monitoring = false + + // EC2 gives this script to cloud-init on the first boot. It installs the + // native perf tooling, enables unprivileged PMU access, starts the SSM agent, + // and runs a counter smoke test. It is bootstrap configuration, not a script + // rerun by every Terraform apply. + user_data = file("${path.module}/user-data.sh") + + // Require IMDSv2 and keep its packets local to the host, reducing the chance + // that a benchmark process can accidentally expose instance-role credentials. + metadata_options { + http_endpoint = "enabled" + http_protocol_ipv6 = "disabled" + http_put_response_hop_limit = 1 + http_tokens = "required" + instance_metadata_tags = "disabled" + } + + // M9g is EBS-only. Provisioned gp3 performance reduces storage variance + // during compilation, while 500 GiB leaves room for toolchains, sources, and + // build artifacts. Encryption protects the persistent volume when stopped. + root_block_device { + delete_on_termination = true + encrypted = true + iops = 12000 + throughput = 500 + volume_size = 500 + volume_type = "gp3" + } + + tags = { + Name = "rustc-perf-graviton5" + Environment = "prod" + Service = "rustc-perf" + } + + lifecycle { + # Do not replace the collector just because Canonical published a new AMI. + ignore_changes = [ami] + } + + // Ensure the documented SSM access method is configured before EC2 can + // register as a managed node and an operator attempts the first session. + depends_on = [aws_ssm_document.session_preferences] +} diff --git a/terragrunt/modules/rustc-perf-collector/network.tf b/terragrunt/modules/rustc-perf-collector/network.tf new file mode 100644 index 000000000..60abf2bdd --- /dev/null +++ b/terragrunt/modules/rustc-perf-collector/network.tf @@ -0,0 +1,68 @@ +// This service needs one outbound-only host, so a small dedicated VPC is easier +// to reason about than adopting a default VPC or deploying the multi-AZ/NAT +// topology used by web services. +resource "aws_vpc" "collector" { + cidr_block = "10.0.0.0/24" + enable_dns_hostnames = true + enable_dns_support = true + + tags = { + Name = "rustc-perf-prod" + } +} + +resource "aws_internet_gateway" "collector" { + vpc_id = aws_vpc.collector.id + + tags = { + Name = "rustc-perf-prod" + } +} + +resource "aws_subnet" "collector" { + availability_zone_id = local.availability_zone_id + cidr_block = "10.0.0.0/26" + map_public_ip_on_launch = true + vpc_id = aws_vpc.collector.id + + tags = { + Name = "rustc-perf-prod" + } +} + +resource "aws_route_table" "collector" { + vpc_id = aws_vpc.collector.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.collector.id + } + + tags = { + Name = "rustc-perf-prod" + } +} + +resource "aws_route_table_association" "collector" { + route_table_id = aws_route_table.collector.id + subnet_id = aws_subnet.collector.id +} + +// Deliberately define no ingress rules. SSM establishes its management channel +// outbound, so neither SSH nor a bastion needs to be exposed to the internet. +resource "aws_security_group" "collector" { + name = "rustc-perf-collector" + description = "No-ingress security group for the rustc-perf collector" + vpc_id = aws_vpc.collector.id + + tags = { + Name = "rustc-perf-collector" + } +} + +resource "aws_vpc_security_group_egress_rule" "collector" { + security_group_id = aws_security_group.collector.id + cidr_ipv4 = "0.0.0.0/0" + ip_protocol = "-1" + description = "Allow the collector to fetch toolchains and benchmark sources" +} diff --git a/terragrunt/modules/rustc-perf-collector/outputs.tf b/terragrunt/modules/rustc-perf-collector/outputs.tf new file mode 100644 index 000000000..0317bd261 --- /dev/null +++ b/terragrunt/modules/rustc-perf-collector/outputs.tf @@ -0,0 +1,24 @@ +output "instance_id" { + description = "EC2 instance ID of the Graviton 5 collector" + value = aws_instance.collector.id +} + +output "dedicated_host_id" { + description = "EC2 Dedicated Host containing the collector" + value = aws_ec2_host.collector.id +} + +output "instance_type" { + description = "EC2 instance type of the collector" + value = aws_instance.collector.instance_type +} + +output "availability_zone" { + description = "Availability Zone selected for the collector" + value = aws_instance.collector.availability_zone +} + +output "perf_check_marker" { + description = "File created by cloud-init after hardware counters pass their smoke test" + value = "/var/lib/rustc-perf/perf-counters-ready" +} diff --git a/terragrunt/modules/rustc-perf-collector/user-data.sh b/terragrunt/modules/rustc-perf-collector/user-data.sh new file mode 100644 index 000000000..aed92d0c7 --- /dev/null +++ b/terragrunt/modules/rustc-perf-collector/user-data.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euxo pipefail + +export DEBIAN_FRONTEND=noninteractive + +# EC2 executes user data as root through cloud-init on the first boot. Install +# both rustc-perf build prerequisites and the perf binary matching the running +# AWS kernel; a generic perf package can point at an incompatible binary. +apt-get update +apt-get install --yes \ + build-essential \ + cmake \ + git \ + libssl-dev \ + "linux-tools-$(uname -r)" \ + pkg-config \ + python3-venv \ + unzip + +tee /etc/sysctl.d/90-rustc-perf.conf >/dev/null <<'EOF' +# Allow the unprivileged collector process to use all PMU events. +kernel.perf_event_paranoid = -1 +# Symbol addresses are useful when correlating counter samples with profiles. +kernel.kptr_restrict = 0 +EOF + +sysctl --system + +# Canonical's AWS images ship the agent as a snap. Keep this fallback for an +# image where it is not preinstalled yet. +if ! snap list amazon-ssm-agent >/dev/null 2>&1; then + snap install amazon-ssm-agent --classic +fi +systemctl enable --now snap.amazon-ssm-agent.amazon-ssm-agent.service + +install -d -m 0755 /var/lib/rustc-perf +# Fail cloud-init if EC2 does not expose the core PMU events rustc-perf relies +# on to this M9g guest. Dedicated Host tenancy reserves the physical server, +# but this runtime check is what verifies the counters are actually usable. +# The marker distinguishes verified bootstrap from a merely running instance. +perf stat \ + --event cycles,instructions,branches,branch-misses \ + -- sleep 1 +date --iso-8601=seconds > /var/lib/rustc-perf/perf-counters-ready diff --git a/terragrunt/modules/rustc-perf-collector/variables.tf b/terragrunt/modules/rustc-perf-collector/variables.tf new file mode 100644 index 000000000..235748b44 --- /dev/null +++ b/terragrunt/modules/rustc-perf-collector/variables.tf @@ -0,0 +1,9 @@ +variable "required_dedicated_hosts" { + description = "Approved regional M9g Dedicated Host quota required by the collector" + type = number + + validation { + condition = var.required_dedicated_hosts >= 1 + error_message = "The collector requires at least one M9g Dedicated Host." + } +}