diff --git a/src/machine/machine_rp2040_flashsafe.go b/src/machine/machine_rp2040_flashsafe.go new file mode 100644 index 0000000000..d071dca1c1 --- /dev/null +++ b/src/machine/machine_rp2040_flashsafe.go @@ -0,0 +1,20 @@ +//go:build tinygo && rp2040 + +// "Flash safe" follows the RP2040/Pico SDK terminology: flash operations +// must run while the other core is not executing from XIP flash. +// +// Use linkname to call runtime hooks from package machine without creating +// an import cycle. + +package machine + +import ( + "runtime/interrupt" + _ "unsafe" +) + +//go:linkname rp2040EnterFlashSafeSection runtime.rp2040EnterFlashSafeSection +func rp2040EnterFlashSafeSection() interrupt.State + +//go:linkname rp2040ExitFlashSafeSection runtime.rp2040ExitFlashSafeSection +func rp2040ExitFlashSafeSection(state interrupt.State) diff --git a/src/machine/machine_rp2040_rom.go b/src/machine/machine_rp2040_rom.go index 0f7c6819a5..3afef547a3 100644 --- a/src/machine/machine_rp2040_rom.go +++ b/src/machine/machine_rp2040_rom.go @@ -3,7 +3,6 @@ package machine import ( - "runtime/interrupt" "unsafe" ) @@ -206,6 +205,12 @@ func doFlashCommand(tx []byte, rx []byte) error { if len(tx) != len(rx) { return errFlashInvalidWriteLength } + if len(tx) == 0 { + return nil + } + + state := rp2040EnterFlashSafeSection() + defer rp2040ExitFlashSafeSection(state) C.flash_do_cmd( (*C.uint8_t)(unsafe.Pointer(&tx[0])), @@ -223,14 +228,17 @@ func (f flashBlockDevice) writeAt(p []byte, off int64) (n int, err error) { return 0, errFlashCannotWritePastEOF } - state := interrupt.Disable() - defer interrupt.Restore(state) - // rp2040 writes to offset, not actual address // e.g. real address 0x10003000 is written to at // 0x00003000 address := writeAddress(off) padded := flashPad(p, int(f.WriteBlockSize())) + if len(padded) == 0 { + return 0, nil + } + + state := rp2040EnterFlashSafeSection() + defer rp2040ExitFlashSafeSection(state) C.flash_range_write(C.uint32_t(address), (*C.uint8_t)(unsafe.Pointer(&padded[0])), @@ -245,8 +253,8 @@ func (f flashBlockDevice) eraseBlocks(start, length int64) error { return errFlashCannotErasePastEOF } - state := interrupt.Disable() - defer interrupt.Restore(state) + state := rp2040EnterFlashSafeSection() + defer rp2040ExitFlashSafeSection(state) C.flash_erase_blocks(C.uint32_t(address), C.ulong(length*f.EraseBlockSize())) diff --git a/src/runtime/runtime_rp2.go b/src/runtime/runtime_rp2.go index f6d31f9806..e2c2559e35 100644 --- a/src/runtime/runtime_rp2.go +++ b/src/runtime/runtime_rp2.go @@ -12,6 +12,11 @@ import ( "unsafe" ) +const ( + rp2SIOFIFOCommandGC uint32 = iota + 1 + rp2SIOFIFOCommandFlashSafe +) + const numCPU = 2 const numSpinlocks = 32 @@ -250,7 +255,7 @@ func gcInterruptHandler(hartID uint32) { // Pause the given core by sending it an interrupt. func gcPauseCore(core uint32) { - rp.SIO.FIFO_WR.Set(1) + rp.SIO.FIFO_WR.Set(rp2SIOFIFOCommandGC) } // Signal the given core that it can resume one step. @@ -281,6 +286,15 @@ var ( schedulerLock = spinLock{id: 21} atomicsLock = spinLock{id: 22} futexLock = spinLock{id: 23} + + // flashSafeLock serializes RP2040 flash-safe Enter/Exit so that only one + // core owns the flash-safe state machine at a time. The other core can + // still participate as a victim through the FIFO interrupt while spinning + // on this lock. + // + // It is defined here with the other RP2 spinLocks so spinLock IDs remain + // visible in one place. + flashSafeLock = spinLock{id: 24} ) func resetSpinLocks() { diff --git a/src/runtime/runtime_rp2040.go b/src/runtime/runtime_rp2040.go index 56c7da1028..d0f61b9093 100644 --- a/src/runtime/runtime_rp2040.go +++ b/src/runtime/runtime_rp2040.go @@ -29,14 +29,18 @@ func enableSIOFifoInterruptCore1() { func handleSIOFifoInterruptCore0(intr interrupt.Interrupt) { switch rp.SIO.FIFO_RD.Get() { - case 1: + case rp2SIOFIFOCommandGC: gcInterruptHandler(0) + case rp2SIOFIFOCommandFlashSafe: + rp2FlashSafeInterruptHandler(0) } } func handleSIOFifoInterruptCore1(intr interrupt.Interrupt) { switch rp.SIO.FIFO_RD.Get() { - case 1: + case rp2SIOFIFOCommandGC: gcInterruptHandler(1) + case rp2SIOFIFOCommandFlashSafe: + rp2FlashSafeInterruptHandler(1) } } diff --git a/src/runtime/runtime_rp2040_flashsafe_cores.go b/src/runtime/runtime_rp2040_flashsafe_cores.go new file mode 100644 index 0000000000..427fc2e3c4 --- /dev/null +++ b/src/runtime/runtime_rp2040_flashsafe_cores.go @@ -0,0 +1,99 @@ +//go:build rp2040 && scheduler.cores + +package runtime + +import ( + "device/arm" + "device/rp" + "runtime/interrupt" + "runtime/volatile" + _ "unsafe" // required for //go:section +) + +const ( + rp2040FlashSafeIdle uint8 = iota + rp2040FlashSafeLocked + rp2040FlashSafeRelease +) + +// rp2040FlashSafeState is used to synchronize the core that performs a flash +// operation with the other core that must stop executing from XIP flash. +var rp2040FlashSafeState volatile.Register8 + +// rp2040EnterFlashSafeSection enters a section in which RP2040 flash operations +// may temporarily disable XIP. +// +// The multicore path asks the other core to enter the flash-safe interrupt +// handler and waits until it acknowledges that it is parked. Local interrupts +// are disabled after the other core is parked. +func rp2040EnterFlashSafeSection() interrupt.State { + if !secondaryCoresStarted { + return interrupt.Disable() + } + + flashSafeLock.Lock() + + core := currentCPU() + rp2040FlashSafeState.Set(rp2040FlashSafeIdle) + + for i := uint32(0); i < numCPU; i++ { + if i == core { + continue + } + rp2040FlashSafePauseCore(i) + } + + for rp2040FlashSafeState.Get() != rp2040FlashSafeLocked { + spinLoopWait() + } + + return interrupt.Disable() +} + +// rp2040ExitFlashSafeSection exits a section entered by +// rp2040EnterFlashSafeSection. +func rp2040ExitFlashSafeSection(state interrupt.State) { + if secondaryCoresStarted { + rp2040FlashSafeState.Set(rp2040FlashSafeRelease) + arm.Asm("sev") + + for rp2040FlashSafeState.Get() != rp2040FlashSafeIdle { + spinLoopWait() + } + + flashSafeLock.Unlock() + } + + interrupt.Restore(state) +} + +func rp2040FlashSafePauseCore(core uint32) { + // RP2040 SIO FIFO writes to the other core. + rp.SIO.FIFO_WR.Set(rp2SIOFIFOCommandFlashSafe) + arm.Asm("sev") +} + +// rp2FlashSafeInterruptHandler runs on the other core while this core is +// performing a flash operation that temporarily disables XIP. +// +// This function MUST be placed in RAM (.ramfuncs section). During the +// flash operation the QSPI flash is in non-XIP mode and instruction +// fetches from the 0x10000000 region will fail. The wait loop below +// runs entirely from RAM so that the parked core can keep executing. +// +//go:section .ramfuncs +func rp2FlashSafeInterruptHandler(core uint32) { + state := interrupt.Disable() + + rp2040FlashSafeState.Set(rp2040FlashSafeLocked) + arm.Asm("sev") + + for rp2040FlashSafeState.Get() == rp2040FlashSafeLocked { + arm.Asm("wfe") + } + + interrupt.Restore(state) + + rp2040FlashSafeState.Set(rp2040FlashSafeIdle) + arm.Asm("sev") +} diff --git a/src/runtime/runtime_rp2040_flashsafe_single.go b/src/runtime/runtime_rp2040_flashsafe_single.go new file mode 100644 index 0000000000..3f5ec7bcd1 --- /dev/null +++ b/src/runtime/runtime_rp2040_flashsafe_single.go @@ -0,0 +1,17 @@ +//go:build rp2040 && !scheduler.cores + +package runtime + +import "runtime/interrupt" + +func rp2040EnterFlashSafeSection() interrupt.State { + return interrupt.Disable() +} + +func rp2040ExitFlashSafeSection(state interrupt.State) { + interrupt.Restore(state) +} + +func rp2FlashSafeInterruptHandler(uint32) { + // No-op on single-core schedulers. +} diff --git a/src/runtime/runtime_rp2350.go b/src/runtime/runtime_rp2350.go index 02c00b61ea..2ec90138a8 100644 --- a/src/runtime/runtime_rp2350.go +++ b/src/runtime/runtime_rp2350.go @@ -33,7 +33,9 @@ func enableSIOFifoInterruptCore1() { func handleSIOFifoInterrupt(intr interrupt.Interrupt) { switch rp.SIO.FIFO_RD.Get() { - case 1: + case rp2SIOFIFOCommandGC: gcInterruptHandler(currentCPU()) + case rp2SIOFIFOCommandFlashSafe: + rp2FlashSafeInterruptHandler(currentCPU()) } } diff --git a/src/runtime/runtime_rp2350_flashsafe_stub.go b/src/runtime/runtime_rp2350_flashsafe_stub.go new file mode 100644 index 0000000000..630ffac9a0 --- /dev/null +++ b/src/runtime/runtime_rp2350_flashsafe_stub.go @@ -0,0 +1,7 @@ +//go:build rp2350 + +package runtime + +func rp2FlashSafeInterruptHandler(uint32) { + // No-op on RP2350. RP2350 flash-safe handling is intentionally unchanged. +}