diff --git a/pkg/config/base.go b/pkg/config/base.go index c3a93acc..2b17b4a4 100644 --- a/pkg/config/base.go +++ b/pkg/config/base.go @@ -64,6 +64,7 @@ type BaseConfig struct { ChromeFlags map[string]interface{} `yaml:"chrome_flags"` // additional flags to pass to Chrome Latency LatencyConfig `yaml:"latency"` // gstreamer latencies, modifying these may break the service LatencyOverrides map[types.RequestType]LatencyConfig `yaml:"latency_overrides"` // latency overrides for different request types, experimental only, will be removed + EnableTemplateSDK bool `yaml:"enable_template_sdk"` // use GStreamer compositor instead of Chrome for default template layouts EnableOneShotSenderReportSync bool `yaml:"enable_one_shot_sender_report_sync"` // temporary rollout flag enabling one-shot sender report correction for room composite / track requests that previously used audio PTS adjustment disabling EnableSyncEngine bool `yaml:"enable_sync_engine"` // use Chrome-inspired sync engine for improved cross-participant alignment and A/V sync AudioTempoController AudioTempoController `yaml:"audio_tempo_controller"` // audio tempo controller diff --git a/pkg/config/pipeline.go b/pkg/config/pipeline.go index b31a2974..20ed585c 100644 --- a/pkg/config/pipeline.go +++ b/pkg/config/pipeline.go @@ -93,9 +93,10 @@ type SDKSourceParams struct { TrackSource string TrackKind string ScreenShare bool + Compositing bool VideoInCodec types.MimeType AudioTracks []*TrackSource - VideoTrack *TrackSource + VideoTracks []*TrackSource AudioRoutes []AudioRouteConfig CaptureAudioAll bool } @@ -112,16 +113,18 @@ type AudioRouteMatch struct { } type TrackSource struct { - TrackID string - TrackKind lksdk.TrackKind - ParticipantKind lksdk.ParticipantKind - AudioChannel *livekit.AudioChannel - AppSrc *app.Source - MimeType types.MimeType - PayloadType webrtc.PayloadType - ClockRate uint32 - TempoController *tempo.Controller - OnKeyframeRequired func() + TrackID string + TrackKind lksdk.TrackKind + ParticipantIdentity string + PublicationSource livekit.TrackSource + ParticipantKind lksdk.ParticipantKind + AudioChannel *livekit.AudioChannel + AppSrc *app.Source + MimeType types.MimeType + PayloadType webrtc.PayloadType + ClockRate uint32 + TempoController *tempo.Controller + OnKeyframeRequired func() } type AudioConfig struct { @@ -228,7 +231,11 @@ func (p *PipelineConfig) Update(request *rpc.StartEgressRequest) error { } egress.RedactEncodedOutputs(clone) - if ShouldUseSDKSource(req.RoomComposite) { + if p.EnableTemplateSDK && req.RoomComposite.CustomBaseUrl == "" { + p.AudioMixing = req.RoomComposite.AudioMixing + p.SourceType = types.SourceTypeSDK + p.Compositing = !req.RoomComposite.AudioOnly + } else if ShouldUseSDKSource(req.RoomComposite) { p.AudioMixing = req.RoomComposite.AudioMixing p.SourceType = types.SourceTypeSDK } else { @@ -579,7 +586,10 @@ func (p *PipelineConfig) applyV2Source(req egress.EgressRequest) (connectionInfo tmpl := req.GetTemplate() p.RequestType = types.RequestTypeTemplate - if ShouldUseSDKSource(tmpl) { + if p.EnableTemplateSDK && tmpl.CustomBaseUrl == "" { + p.SourceType = types.SourceTypeSDK + p.Compositing = !tmpl.AudioOnly + } else if ShouldUseSDKSource(tmpl) { p.SourceType = types.SourceTypeSDK } else { p.SourceType = types.SourceTypeWeb diff --git a/pkg/gstreamer/callbacks.go b/pkg/gstreamer/callbacks.go index fd3a436a..d14825fb 100644 --- a/pkg/gstreamer/callbacks.go +++ b/pkg/gstreamer/callbacks.go @@ -19,6 +19,7 @@ import ( "github.com/linkdata/deadlock" "github.com/livekit/egress/pkg/config" "github.com/livekit/egress/pkg/errors" + lksdk "github.com/livekit/server-sdk-go/v2" ) type Callbacks struct { @@ -32,11 +33,12 @@ type Callbacks struct { onDebugDotRequest func(string) // source callbacks - onTrackAdded []func(*config.TrackSource) - onTrackMuted []func(string) - onTrackUnmuted []func(string) - onTrackRemoved []func(string) - onEOSSent func() + onTrackAdded []func(*config.TrackSource) + onTrackMuted []func(string) + onTrackUnmuted []func(string) + onTrackRemoved []func(string) + onActiveSpeakersChanged []func([]lksdk.Participant) + onEOSSent func() pipelinePaused core.Fuse } @@ -163,6 +165,22 @@ func (c *Callbacks) OnTrackRemoved(trackID string) { } } +func (c *Callbacks) AddOnActiveSpeakersChanged(f func([]lksdk.Participant)) { + c.mu.Lock() + c.onActiveSpeakersChanged = append(c.onActiveSpeakersChanged, f) + c.mu.Unlock() +} + +func (c *Callbacks) OnActiveSpeakersChanged(speakers []lksdk.Participant) { + c.mu.RLock() + handlers := c.onActiveSpeakersChanged + c.mu.RUnlock() + + for _, f := range handlers { + f(speakers) + } +} + func (c *Callbacks) SetOnEOSSent(f func()) { c.mu.Lock() c.onEOSSent = f diff --git a/pkg/pipeline/builder/image.go b/pkg/pipeline/builder/image.go index 800e6a7a..06e9e94b 100644 --- a/pkg/pipeline/builder/image.go +++ b/pkg/pipeline/builder/image.go @@ -122,6 +122,13 @@ func BuildImageBin(c *config.ImageConfig, pipeline *gstreamer.Pipeline, p *confi if err != nil { return nil, err } + // async=true here would gate pipeline preroll on the first composited frame, stalling PAUSED→PLAYING. + if err = sink.SetProperty("sync", false); err != nil { + return nil, err + } + if err = sink.SetProperty("async", false); err != nil { + return nil, err + } // File will be renamed if the TS prefix is configured location := fmt.Sprintf("%s_%%05d%s", path.Join(c.LocalDir, c.ImagePrefix), types.FileExtensionForOutputType[c.OutputType]) diff --git a/pkg/pipeline/builder/layout.go b/pkg/pipeline/builder/layout.go new file mode 100644 index 00000000..69a618ee --- /dev/null +++ b/pkg/pipeline/builder/layout.go @@ -0,0 +1,581 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package builder + +import ( + "sort" + "time" + + "github.com/linkdata/deadlock" +) + +type TrackSource int + +const ( + TrackSourceCamera TrackSource = iota + TrackSourceScreenShare + + LayoutGrid = "grid" + LayoutGridLight = "grid-light" + LayoutGridDark = "grid-dark" + LayoutSingleSpeaker = "single-speaker" + LayoutSingleSpeakerLight = "single-speaker-light" + LayoutSingleSpeakerDark = "single-speaker-dark" + LayoutSpeaker = "speaker" + LayoutSpeakerLight = "speaker-light" + LayoutSpeakerDark = "speaker-dark" +) + +type PadLayout struct { + TrackID string + X, Y int + W, H int + ZOrder uint + Alpha float64 +} + +type SpeakerInfo struct { + Identity string + AudioLevel float32 + IsSpeaking bool +} + +type trackInfo struct { + TrackID string + Identity string + Source TrackSource + addOrder int + isSpeaking bool + audioLevel float32 + lastSpokeAt int64 +} + +// gridGap is the Chrome template's --grid-gap (0.5rem at 16px base); applied +// both between cells and as outer padding on .lk-grid-layout / .lk-focus-layout. +const gridGap = 8 + +const minCarouselTileHeight = 130 + +// maxCarouselTiles mirrors CarouselLayout.tsx vertical sizing (see +// template-default/node_modules/@livekit/components-react CarouselLayout.tsx). +func maxCarouselTiles(carouselW, innerH int) int { + tileSpan := carouselW * 6 / 10 + if tileSpan < minCarouselTileHeight { + tileSpan = minCarouselTileHeight + } + n := (innerH + tileSpan/2) / tileSpan + if n < 1 { + n = 1 + } + return n +} + +type LayoutManager struct { + mu deadlock.Mutex + layout string + origLayout string + width, height int + tracks []trackInfo + displayOrder []string + addCounter int +} + +func NewLayoutManager(layout string, width, height int) *LayoutManager { + return &LayoutManager{ + layout: layout, + origLayout: layout, + width: width, + height: height, + } +} + +func (l *LayoutManager) AddTrack(trackID, identity string, source TrackSource) []PadLayout { + l.mu.Lock() + defer l.mu.Unlock() + + l.addCounter++ + l.tracks = append(l.tracks, trackInfo{ + TrackID: trackID, + Identity: identity, + Source: source, + addOrder: l.addCounter, + }) + l.displayOrder = append(l.displayOrder, trackID) + + if source == TrackSourceScreenShare { + l.onScreenShareChanged() + } + + idealOrder := l.sortTracks() + l.displayOrder = l.stabilize(idealOrder) + + return l.recalculate() +} + +func (l *LayoutManager) RemoveTrack(trackID string) []PadLayout { + l.mu.Lock() + defer l.mu.Unlock() + + var removedSource TrackSource + for i, t := range l.tracks { + if t.TrackID == trackID { + removedSource = t.Source + l.tracks = append(l.tracks[:i], l.tracks[i+1:]...) + break + } + } + for i, id := range l.displayOrder { + if id == trackID { + l.displayOrder = append(l.displayOrder[:i], l.displayOrder[i+1:]...) + break + } + } + + if removedSource == TrackSourceScreenShare { + l.onScreenShareChanged() + } + + return l.recalculate() +} + +func (l *LayoutManager) onScreenShareChanged() { + hasScreenShare := false + for _, t := range l.tracks { + if t.Source == TrackSourceScreenShare { + hasScreenShare = true + break + } + } + + if hasScreenShare { + if isGridLayout(l.origLayout) { + l.layout = gridToSpeaker(l.origLayout) + } + } else { + l.layout = l.origLayout + } +} + +func isGridLayout(layout string) bool { + return layout == LayoutGrid || layout == LayoutGridLight || layout == LayoutGridDark +} + +func gridToSpeaker(layout string) string { + switch layout { + case LayoutGridLight: + return LayoutSpeakerLight + case LayoutGridDark: + return LayoutSpeakerDark + default: + return LayoutSpeaker + } +} + +func (l *LayoutManager) Layout() []PadLayout { + l.mu.Lock() + defer l.mu.Unlock() + + return l.recalculate() +} + +func (l *LayoutManager) recalculate() []PadLayout { + switch l.layout { + case LayoutSpeaker, LayoutSpeakerLight, LayoutSpeakerDark: + return l.calculateSpeakerLayout() + case LayoutSingleSpeaker, LayoutSingleSpeakerLight, LayoutSingleSpeakerDark: + return l.calculateSingleSpeakerLayout() + default: + return l.calculateGridLayout() + } +} + +// gridColumns mirrors the LiveKit components GridLayout breakpoint algorithm. +func (l *LayoutManager) gridColumns(count int) int { + switch { + case count <= 1: + return 1 + case count <= 4: + if l.width >= 560 { + return 2 + } + return 1 + case count <= 9: + if l.width >= 700 { + return 3 + } + return 2 + case count <= 16: + if l.width >= 960 { + return 4 + } + return 3 + default: + if l.width >= 1100 { + return 5 + } + return 4 + } +} + +func (l *LayoutManager) calculateGridLayout() []PadLayout { + count := len(l.tracks) + if count == 0 { + return nil + } + + // Single participant fills the canvas: matches Chrome's single-tile render + // and keeps the avsync test pattern's top stripe at y=0. + if count == 1 { + return []PadLayout{{ + TrackID: l.tracks[0].TrackID, + X: 0, Y: 0, + W: l.width, H: l.height, + ZOrder: 1, Alpha: 1.0, + }} + } + + cols := l.gridColumns(count) + rows := (count + cols - 1) / cols + innerW := l.width - 2*gridGap + innerH := l.height - 2*gridGap + cellW := (innerW - (cols-1)*gridGap) / cols + cellH := (innerH - (rows-1)*gridGap) / rows + + pads := make([]PadLayout, 0, count) + for i, t := range l.tracks { + col := i % cols + row := i / cols + pads = append(pads, PadLayout{ + TrackID: t.TrackID, + X: gridGap + col*(cellW+gridGap), + Y: gridGap + row*(cellH+gridGap), + W: cellW, + H: cellH, + ZOrder: 1, + Alpha: 1.0, + }) + } + return pads +} + +func (l *LayoutManager) calculateSpeakerLayout() []PadLayout { + count := len(l.tracks) + if count == 0 { + return nil + } + + ordered := l.orderedTracks() + + if count == 1 { + return []PadLayout{{ + TrackID: ordered[0].TrackID, + X: 0, Y: 0, + W: l.width, H: l.height, + ZOrder: 1, Alpha: 1.0, + }} + } + + // .lk-focus-layout: 1fr carousel + 5fr stage CSS grid, gridGap padding/gap. + const totalFr = 6 + innerW := l.width - 2*gridGap + innerH := l.height - 2*gridGap + availableForCols := innerW - gridGap + carouselW := availableForCols / totalFr + stageW := availableForCols - carouselW + stageX := gridGap + carouselW + gridGap + + // thumbH always divides by maxTiles so tile size stays stable as participants join/leave. + maxTiles := maxCarouselTiles(carouselW, innerH) + thumbH := (innerH - gridGap*(maxTiles-1)) / maxTiles + visibleThumbs := maxTiles + if count-1 < visibleThumbs { + visibleThumbs = count - 1 + } + + pads := make([]PadLayout, 0, count) + pads = append(pads, PadLayout{ + TrackID: ordered[0].TrackID, + X: stageX, Y: gridGap, + W: stageW, H: innerH, + ZOrder: 1, Alpha: 1.0, + }) + + for i := 1; i <= visibleThumbs; i++ { + pads = append(pads, PadLayout{ + TrackID: ordered[i].TrackID, + X: gridGap, + Y: gridGap + (i-1)*(thumbH+gridGap), + W: carouselW, + H: thumbH, + ZOrder: 1, + Alpha: 1.0, + }) + } + + for i := visibleThumbs + 1; i < count; i++ { + pads = append(pads, PadLayout{ + TrackID: ordered[i].TrackID, + X: 0, Y: 0, + W: 0, H: 0, + ZOrder: 0, Alpha: 0.0, + }) + } + + return pads +} + +func (l *LayoutManager) calculateSingleSpeakerLayout() []PadLayout { + if len(l.tracks) == 0 { + return nil + } + + ordered := l.orderedTracks() + pads := make([]PadLayout, 0, len(l.tracks)) + + pads = append(pads, PadLayout{ + TrackID: ordered[0].TrackID, + X: 0, Y: 0, + W: l.width, H: l.height, + ZOrder: 1, Alpha: 1.0, + }) + + for i := 1; i < len(ordered); i++ { + pads = append(pads, PadLayout{ + TrackID: ordered[i].TrackID, + X: 0, Y: 0, + W: 0, H: 0, + ZOrder: 0, Alpha: 0.0, + }) + } + + return pads +} + +func (l *LayoutManager) orderedTracks() []trackInfo { + byID := make(map[string]trackInfo, len(l.tracks)) + for _, t := range l.tracks { + byID[t.TrackID] = t + } + + result := make([]trackInfo, 0, len(l.tracks)) + seen := make(map[string]bool) + for _, id := range l.displayOrder { + if t, ok := byID[id]; ok { + result = append(result, t) + seen[id] = true + } + } + for _, t := range l.tracks { + if !seen[t.TrackID] { + result = append(result, t) + } + } + return result +} + +// UpdateSpeakers returns nil if the stabilized display order is unchanged. +func (l *LayoutManager) UpdateSpeakers(speakers []SpeakerInfo) []PadLayout { + l.mu.Lock() + defer l.mu.Unlock() + + now := time.Now().UnixNano() + + speakerMap := make(map[string]SpeakerInfo, len(speakers)) + for _, s := range speakers { + speakerMap[s.Identity] = s + } + + for i := range l.tracks { + if s, ok := speakerMap[l.tracks[i].Identity]; ok { + l.tracks[i].isSpeaking = s.IsSpeaking + l.tracks[i].audioLevel = s.AudioLevel + if s.IsSpeaking { + l.tracks[i].lastSpokeAt = now + } + } else { + l.tracks[i].isSpeaking = false + l.tracks[i].audioLevel = 0 + } + } + + idealOrder := l.sortTracks() + newDisplayOrder := l.stabilize(idealOrder) + + changed := false + if len(newDisplayOrder) != len(l.displayOrder) { + changed = true + } else { + for i := range newDisplayOrder { + if newDisplayOrder[i] != l.displayOrder[i] { + changed = true + break + } + } + } + + if !changed { + return nil + } + + l.displayOrder = newDisplayOrder + return l.recalculate() +} + +func (l *LayoutManager) sortTracks() []string { + var screenShares []trackInfo + var cameras []trackInfo + + for _, t := range l.tracks { + if t.Source == TrackSourceScreenShare { + screenShares = append(screenShares, t) + } else { + cameras = append(cameras, t) + } + } + + sort.SliceStable(screenShares, func(i, j int) bool { + return screenShares[i].addOrder < screenShares[j].addOrder + }) + + sort.SliceStable(cameras, func(i, j int) bool { + a, b := cameras[i], cameras[j] + if a.isSpeaking && b.isSpeaking { + return a.audioLevel > b.audioLevel + } + if a.isSpeaking != b.isSpeaking { + return a.isSpeaking + } + if a.lastSpokeAt != b.lastSpokeAt { + return a.lastSpokeAt > b.lastSpokeAt + } + return a.addOrder < b.addOrder + }) + + result := make([]string, 0, len(l.tracks)) + for _, t := range screenShares { + result = append(result, t.TrackID) + } + for _, t := range cameras { + result = append(result, t.TrackID) + } + return result +} + +func (l *LayoutManager) stabilize(idealOrder []string) []string { + if len(l.displayOrder) == 0 { + return idealOrder + } + + maxOnPage := l.maxItemsOnPage() + + current := l.refreshOrder(idealOrder) + + currentSet := make(map[string]bool, len(current)) + for _, id := range current { + currentSet[id] = true + } + for _, id := range idealOrder { + if !currentSet[id] { + current = append(current, id) + } + } + + idealSet := make(map[string]bool, len(idealOrder)) + for _, id := range idealOrder { + idealSet[id] = true + } + filtered := current[:0] + for _, id := range current { + if idealSet[id] { + filtered = append(filtered, id) + } + } + current = filtered + + if maxOnPage >= len(current) { + return idealOrder + } + + visibleIdeal := idealOrder + if len(visibleIdeal) > maxOnPage { + visibleIdeal = visibleIdeal[:maxOnPage] + } + + visibleCurrent := current + if len(visibleCurrent) > maxOnPage { + visibleCurrent = visibleCurrent[:maxOnPage] + } + visibleSet := make(map[string]bool, len(visibleCurrent)) + for _, id := range visibleCurrent { + visibleSet[id] = true + } + + for _, shouldBeVisible := range visibleIdeal { + if visibleSet[shouldBeVisible] { + continue + } + for i := 0; i < maxOnPage && i < len(current); i++ { + shouldBeHere := false + for _, vid := range visibleIdeal { + if current[i] == vid { + shouldBeHere = true + break + } + } + if !shouldBeHere { + for j := range current { + if current[j] == shouldBeVisible { + current[i], current[j] = current[j], current[i] + break + } + } + break + } + } + } + + return current +} + +func (l *LayoutManager) refreshOrder(idealOrder []string) []string { + idealSet := make(map[string]bool, len(idealOrder)) + for _, id := range idealOrder { + idealSet[id] = true + } + result := make([]string, 0, len(l.displayOrder)) + for _, id := range l.displayOrder { + if idealSet[id] { + result = append(result, id) + } + } + return result +} + +func (l *LayoutManager) maxItemsOnPage() int { + switch l.layout { + case LayoutSingleSpeaker, LayoutSingleSpeakerLight, LayoutSingleSpeakerDark: + return 1 + case LayoutSpeaker, LayoutSpeakerLight, LayoutSpeakerDark: + innerW := l.width - 2*gridGap + innerH := l.height - 2*gridGap + carouselW := (innerW - gridGap) / 6 + return 1 + maxCarouselTiles(carouselW, innerH) + default: + count := len(l.tracks) + cols := l.gridColumns(count) + rows := (count + cols - 1) / cols + return cols * rows + } +} diff --git a/pkg/pipeline/builder/layout_test.go b/pkg/pipeline/builder/layout_test.go new file mode 100644 index 00000000..83e8f44b --- /dev/null +++ b/pkg/pipeline/builder/layout_test.go @@ -0,0 +1,350 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package builder + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGridLayout_SingleParticipant(t *testing.T) { + lm := NewLayoutManager("grid", 1920, 1080) + pads := lm.AddTrack("track1", "alice", TrackSourceCamera) + + require.Len(t, pads, 1) + require.Equal(t, PadLayout{ + TrackID: "track1", + X: 0, Y: 0, + W: 1920, H: 1080, + ZOrder: 1, Alpha: 1.0, + }, pads[0]) +} + +func TestGridLayout_TwoParticipants(t *testing.T) { + lm := NewLayoutManager("grid", 1920, 1080) + lm.AddTrack("t1", "alice", TrackSourceCamera) + pads := lm.AddTrack("t2", "bob", TrackSourceCamera) + + require.Len(t, pads, 2) + // innerW=1904, innerH=1064, 2 cells, 1 row → cellW = (1904-8)/2 = 948 + require.Equal(t, 8, pads[0].X) + require.Equal(t, 8, pads[0].Y) + require.Equal(t, 948, pads[0].W) + require.Equal(t, 1064, pads[0].H) + + require.Equal(t, 964, pads[1].X) + require.Equal(t, 8, pads[1].Y) + require.Equal(t, 948, pads[1].W) + require.Equal(t, 1064, pads[1].H) +} + +func TestGridLayout_ThreeParticipants(t *testing.T) { + lm := NewLayoutManager("grid", 1920, 1080) + lm.AddTrack("t1", "alice", TrackSourceCamera) + lm.AddTrack("t2", "bob", TrackSourceCamera) + pads := lm.AddTrack("t3", "carol", TrackSourceCamera) + + require.Len(t, pads, 3) + // 3 participants → 2 cols × 2 rows, last cell empty (matches Chrome) + // innerW=1904, innerH=1064 → cellW=948, cellH=(1064-8)/2=528 + require.Equal(t, 948, pads[0].W) + require.Equal(t, 528, pads[0].H) + require.Equal(t, 8, pads[0].X) + require.Equal(t, 8, pads[0].Y) + + require.Equal(t, 964, pads[1].X) + require.Equal(t, 8, pads[1].Y) + + require.Equal(t, 8, pads[2].X) + require.Equal(t, 544, pads[2].Y) +} + +func TestGridLayout_FourParticipants(t *testing.T) { + lm := NewLayoutManager("grid", 1920, 1080) + lm.AddTrack("t1", "alice", TrackSourceCamera) + lm.AddTrack("t2", "bob", TrackSourceCamera) + lm.AddTrack("t3", "carol", TrackSourceCamera) + pads := lm.AddTrack("t4", "dave", TrackSourceCamera) + + require.Len(t, pads, 4) + require.Equal(t, 948, pads[0].W) + require.Equal(t, 528, pads[0].H) + require.Equal(t, 8, pads[0].X) + require.Equal(t, 8, pads[0].Y) + + require.Equal(t, 964, pads[1].X) + require.Equal(t, 8, pads[1].Y) + + require.Equal(t, 8, pads[2].X) + require.Equal(t, 544, pads[2].Y) + + require.Equal(t, 964, pads[3].X) + require.Equal(t, 544, pads[3].Y) +} + +func TestGridLayout_NineParticipants(t *testing.T) { + lm := NewLayoutManager("grid", 1920, 1080) + for i := 0; i < 8; i++ { + lm.AddTrack( + "t"+string(rune('1'+i)), + string(rune('a'+i)), + TrackSourceCamera, + ) + } + pads := lm.AddTrack("t9", "i", TrackSourceCamera) + + require.Len(t, pads, 9) + // 9 participants @ 1920w → 3 cols × 3 rows. + // innerW=1904, innerH=1064 → cellW=(1904-16)/3=629, cellH=(1064-16)/3=349 + require.Equal(t, 629, pads[0].W) + require.Equal(t, 349, pads[0].H) +} + +func TestGridLayout_RemoveTrack(t *testing.T) { + lm := NewLayoutManager("grid", 1920, 1080) + lm.AddTrack("t1", "alice", TrackSourceCamera) + lm.AddTrack("t2", "bob", TrackSourceCamera) + lm.AddTrack("t3", "carol", TrackSourceCamera) + + pads := lm.RemoveTrack("t2") + require.Len(t, pads, 2) + // Back to 2 participants → 2 cols × 1 row + require.Equal(t, 948, pads[0].W) + require.Equal(t, 1064, pads[0].H) +} + +func TestGridLayout_NarrowWidth(t *testing.T) { + // width < 560 → 1 col for 2-4 participants (stacked) + lm := NewLayoutManager("grid", 400, 1280) + lm.AddTrack("t1", "alice", TrackSourceCamera) + pads := lm.AddTrack("t2", "bob", TrackSourceCamera) + + require.Len(t, pads, 2) + // innerW=384, innerH=1264 → cellW=384, cellH=(1264-8)/2=628 + require.Equal(t, 384, pads[0].W) + require.Equal(t, 628, pads[0].H) + require.Equal(t, 8, pads[0].Y) + + require.Equal(t, 384, pads[1].W) + require.Equal(t, 628, pads[1].H) + require.Equal(t, 644, pads[1].Y) +} + +func TestSpeakerLayout_SingleTrack(t *testing.T) { + lm := NewLayoutManager("speaker", 1920, 1080) + pads := lm.AddTrack("t1", "alice", TrackSourceCamera) + + require.Len(t, pads, 1) + require.Equal(t, 0, pads[0].X) + require.Equal(t, 0, pads[0].Y) + require.Equal(t, 1920, pads[0].W) + require.Equal(t, 1080, pads[0].H) +} + +func TestSpeakerLayout_TwoTracks(t *testing.T) { + lm := NewLayoutManager("speaker", 1920, 1080) + lm.AddTrack("t1", "alice", TrackSourceCamera) + pads := lm.AddTrack("t2", "bob", TrackSourceCamera) + + require.Len(t, pads, 2) + // innerW=1904, innerH=1064. + // carouselW = (1904-8)/6 = 316, stageW = 1896-316 = 1580, stageX = 8+316+8 = 332. + require.Equal(t, 332, pads[0].X) + require.Equal(t, 8, pads[0].Y) + require.Equal(t, 1580, pads[0].W) + require.Equal(t, 1064, pads[0].H) + + // maxTiles = round(1064 / max(316*0.6, 130)) = round(1064/189) = 6 + // thumbH = (1064 - 8*5)/6 = 170 + require.Equal(t, 8, pads[1].X) + require.Equal(t, 8, pads[1].Y) + require.Equal(t, 316, pads[1].W) + require.Equal(t, 170, pads[1].H) +} + +func TestSpeakerLayout_FourTracks(t *testing.T) { + lm := NewLayoutManager("speaker", 1920, 1080) + lm.AddTrack("t1", "alice", TrackSourceCamera) + lm.AddTrack("t2", "bob", TrackSourceCamera) + lm.AddTrack("t3", "carol", TrackSourceCamera) + pads := lm.AddTrack("t4", "dave", TrackSourceCamera) + + require.Len(t, pads, 4) + require.Equal(t, 332, pads[0].X) + require.Equal(t, 1580, pads[0].W) + require.Equal(t, 1064, pads[0].H) + + require.Equal(t, 8, pads[1].X) + require.Equal(t, 316, pads[1].W) + require.Equal(t, 170, pads[1].H) + require.Equal(t, 8, pads[1].Y) + + // Thumbs stack with 8px gap: Y[i] = 8 + (i-1)*(170+8) + require.Equal(t, 186, pads[2].Y) + require.Equal(t, 364, pads[3].Y) +} + +func TestSpeakerLayout_MoreThanFitTracks(t *testing.T) { + // At 1920×1080 the carousel fits 6 tiles. With 9 participants → 1 stage + + // 6 visible carousel + 2 hidden. + lm := NewLayoutManager("speaker", 1920, 1080) + for i := 0; i < 9; i++ { + lm.AddTrack( + "t"+string(rune('1'+i)), + string(rune('a'+i)), + TrackSourceCamera, + ) + } + pads := lm.recalculate() + + visible := 0 + hidden := 0 + for _, p := range pads { + if p.Alpha > 0 { + visible++ + } else { + hidden++ + } + } + require.Equal(t, 7, visible) // 1 stage + 6 carousel + require.Equal(t, 2, hidden) +} + +func TestSingleSpeakerLayout(t *testing.T) { + lm := NewLayoutManager("single-speaker", 1920, 1080) + lm.AddTrack("t1", "alice", TrackSourceCamera) + pads := lm.AddTrack("t2", "bob", TrackSourceCamera) + + require.Len(t, pads, 2) + require.Equal(t, 1.0, pads[0].Alpha) + require.Equal(t, 1920, pads[0].W) + require.Equal(t, 1080, pads[0].H) + + require.Equal(t, 0.0, pads[1].Alpha) +} + +func TestUpdateSpeakers_SortsBySpeaking(t *testing.T) { + lm := NewLayoutManager("speaker", 1920, 1080) + lm.AddTrack("t1", "alice", TrackSourceCamera) + lm.AddTrack("t2", "bob", TrackSourceCamera) + lm.AddTrack("t3", "carol", TrackSourceCamera) + + pads := lm.UpdateSpeakers([]SpeakerInfo{ + {Identity: "bob", AudioLevel: 0.9, IsSpeaking: true}, + }) + + require.NotNil(t, pads) + require.Equal(t, "t2", pads[0].TrackID) + require.Equal(t, 1580, pads[0].W) // stage width +} + +func TestUpdateSpeakers_AudioLevelTiebreak(t *testing.T) { + lm := NewLayoutManager("speaker", 1920, 1080) + lm.AddTrack("t1", "alice", TrackSourceCamera) + lm.AddTrack("t2", "bob", TrackSourceCamera) + lm.AddTrack("t3", "carol", TrackSourceCamera) + + pads := lm.UpdateSpeakers([]SpeakerInfo{ + {Identity: "bob", AudioLevel: 0.5, IsSpeaking: true}, + {Identity: "carol", AudioLevel: 0.9, IsSpeaking: true}, + }) + + require.NotNil(t, pads) + require.Equal(t, "t3", pads[0].TrackID) +} + +func TestUpdateSpeakers_Stabilization(t *testing.T) { + lm := NewLayoutManager("grid", 1920, 1080) + lm.AddTrack("t1", "alice", TrackSourceCamera) + lm.AddTrack("t2", "bob", TrackSourceCamera) + lm.AddTrack("t3", "carol", TrackSourceCamera) + lm.AddTrack("t4", "dave", TrackSourceCamera) + + initialPads := lm.recalculate() + require.Equal(t, "t1", initialPads[0].TrackID) + + // Dave speaks — all 4 fit on one page (2x2), so no swap needed + pads := lm.UpdateSpeakers([]SpeakerInfo{ + {Identity: "dave", AudioLevel: 0.8, IsSpeaking: true}, + }) + require.Len(t, pads, 4) +} + +func TestUpdateSpeakers_LastSpokeAtTiebreak(t *testing.T) { + lm := NewLayoutManager("speaker", 1920, 1080) + lm.AddTrack("t1", "alice", TrackSourceCamera) + lm.AddTrack("t2", "bob", TrackSourceCamera) + lm.AddTrack("t3", "carol", TrackSourceCamera) + + // Bob speaks + lm.UpdateSpeakers([]SpeakerInfo{ + {Identity: "bob", AudioLevel: 0.8, IsSpeaking: true}, + }) + // Bob stops, carol speaks + pads := lm.UpdateSpeakers([]SpeakerInfo{ + {Identity: "carol", AudioLevel: 0.7, IsSpeaking: true}, + }) + + require.NotNil(t, pads) + require.Equal(t, "t3", pads[0].TrackID) + require.Equal(t, "t2", pads[1].TrackID) +} + +func TestScreenShare_GridSwitchesToSpeaker(t *testing.T) { + lm := NewLayoutManager("grid", 1920, 1080) + lm.AddTrack("cam1", "alice", TrackSourceCamera) + lm.AddTrack("cam2", "bob", TrackSourceCamera) + + pads := lm.AddTrack("ss1", "bob", TrackSourceScreenShare) + + require.Equal(t, "ss1", pads[0].TrackID) + require.Equal(t, 1580, pads[0].W) // speaker stage width +} + +func TestScreenShare_RevertsToGrid(t *testing.T) { + lm := NewLayoutManager("grid", 1920, 1080) + lm.AddTrack("cam1", "alice", TrackSourceCamera) + lm.AddTrack("cam2", "bob", TrackSourceCamera) + lm.AddTrack("ss1", "bob", TrackSourceScreenShare) + + pads := lm.RemoveTrack("ss1") + + require.Len(t, pads, 2) + require.Equal(t, 948, pads[0].W) // grid 2x1 cell width +} + +func TestScreenShare_SpeakerLayoutStays(t *testing.T) { + lm := NewLayoutManager("speaker", 1920, 1080) + lm.AddTrack("cam1", "alice", TrackSourceCamera) + lm.AddTrack("cam2", "bob", TrackSourceCamera) + + pads := lm.AddTrack("ss1", "bob", TrackSourceScreenShare) + + require.Equal(t, "ss1", pads[0].TrackID) + require.Equal(t, 1580, pads[0].W) +} + +func TestScreenShare_MultipleScreenShares(t *testing.T) { + lm := NewLayoutManager("grid", 1920, 1080) + lm.AddTrack("cam1", "alice", TrackSourceCamera) + lm.AddTrack("cam2", "bob", TrackSourceCamera) + lm.AddTrack("ss1", "bob", TrackSourceScreenShare) + + pads := lm.AddTrack("ss2", "alice", TrackSourceScreenShare) + + require.Equal(t, "ss1", pads[0].TrackID) + require.Equal(t, 1580, pads[0].W) +} diff --git a/pkg/pipeline/builder/video.go b/pkg/pipeline/builder/video.go index 5f4c8d99..d9fc6510 100644 --- a/pkg/pipeline/builder/video.go +++ b/pkg/pipeline/builder/video.go @@ -26,47 +26,40 @@ import ( "github.com/livekit/egress/pkg/errors" "github.com/livekit/egress/pkg/gstreamer" "github.com/livekit/egress/pkg/types" + "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" lksdk "github.com/livekit/server-sdk-go/v2" ) -const ( - videoTestSrcName = "video_test_src" -) +const videoTestSrcName = "video_test_src" type VideoBin struct { bin *gstreamer.Bin conf *config.PipelineConfig - mu deadlock.Mutex - nextID int + mu deadlock.Mutex + nextID int + pads map[string]*gst.Pad + names map[string]string + selector *gst.Element + rawVideoTee *gst.Element + layout *LayoutManager // nil when not compositing + setVideoDimensions func(trackID string, width, height int) + + // input-selector state (only used when !Compositing) selectedPad string lastPTS uint64 - pads map[string]*gst.Pad - names map[string]string - selector *gst.Element - rawVideoTee *gst.Element probesMu deadlock.Mutex probes map[string]*keyframeProbe } -// buildVideoQueue creates a queue for the video pipeline. For live sources the -// queue is leaky (drops old buffers when full) to handle real-time overrun. For -// non-live replay the queue is blocking so backpressure throttles the source. -func (b *VideoBin) buildVideoQueue(name string) (*gst.Element, error) { - queue, err := gstreamer.BuildQueue(name, b.conf.Latency.PipelineLatency, b.conf.Live) - if err != nil { - return nil, errors.ErrGstPipelineError(err) - } - return queue, nil -} - -func BuildVideoBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) error { +func BuildVideoBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig, setVideoDimensions func(string, int, int)) error { b := &VideoBin{ - bin: pipeline.NewBin("video"), - conf: p, - probes: make(map[string]*keyframeProbe), + bin: pipeline.NewBin("video"), + conf: p, + setVideoDimensions: setVideoDimensions, + probes: make(map[string]*keyframeProbe), } switch p.SourceType { @@ -84,6 +77,7 @@ func BuildVideoBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) error pipeline.AddOnTrackRemoved(b.onTrackRemoved) pipeline.AddOnTrackMuted(b.onTrackMuted) pipeline.AddOnTrackUnmuted(b.onTrackUnmuted) + pipeline.AddOnActiveSpeakersChanged(b.onActiveSpeakersChanged) } var getPad func() *gst.Pad @@ -137,6 +131,16 @@ func (b *VideoBin) onTrackAdded(ts *config.TrackSource) { if err := b.addAppSrcBin(ts); err != nil { logger.Errorw("failed to add video app src bin", err, "trackID", ts.TrackID) b.bin.OnError(err) + return + } + + if b.layout != nil { + source := TrackSourceCamera + if ts.PublicationSource == livekit.TrackSource_SCREEN_SHARE { + source = TrackSourceScreenShare + } + pads := b.layout.AddTrack(ts.TrackID, ts.ParticipantIdentity, source) + b.applyLayout(pads) } } } @@ -156,7 +160,7 @@ func (b *VideoBin) onTrackRemoved(trackID string) { delete(b.pads, name) b.closeProbe(name) - if b.selectedPad == name { + if !b.conf.Compositing && b.selectedPad == name { if err := b.setSelectorPadLocked(videoTestSrcName); err != nil { b.mu.Unlock() b.bin.OnError(err) @@ -165,25 +169,39 @@ func (b *VideoBin) onTrackRemoved(trackID string) { } b.mu.Unlock() + if b.layout != nil { + pads := b.layout.RemoveTrack(trackID) + b.applyLayout(pads) + } + if err := b.bin.RemoveSourceBin(name); err != nil { b.bin.OnError(err) } } +func (b *VideoBin) buildVideoQueue(name string) (*gst.Element, error) { + queue, err := gstreamer.BuildQueue(name, b.conf.Latency.PipelineLatency, b.conf.Live) + if err != nil { + return nil, errors.ErrGstPipelineError(err) + } + return queue, nil +} + func (b *VideoBin) onTrackMuted(trackID string) { if b.bin.GetState() > gstreamer.StateRunning { return } b.mu.Lock() - if name, ok := b.names[trackID]; ok && b.selectedPad == name { - if err := b.setSelectorPadLocked(videoTestSrcName); err != nil { - b.mu.Unlock() - b.bin.OnError(err) - return - } - } + name, ok := b.names[trackID] b.mu.Unlock() + if !ok { + return + } + + if err := b.setTrackVisible(name, false); err != nil { + b.bin.OnError(err) + } } func (b *VideoBin) onTrackUnmuted(trackID string) { @@ -192,14 +210,69 @@ func (b *VideoBin) onTrackUnmuted(trackID string) { } b.mu.Lock() - if name, ok := b.names[trackID]; ok { - if err := b.setSelectorPadLocked(name); err != nil { - b.mu.Unlock() - b.bin.OnError(err) - return + name, ok := b.names[trackID] + b.mu.Unlock() + if !ok { + return + } + + if err := b.setTrackVisible(name, true); err != nil { + b.bin.OnError(err) + } +} + +func (b *VideoBin) onActiveSpeakersChanged(speakers []lksdk.Participant) { + if b.bin.GetState() > gstreamer.StateRunning { + return + } + + if b.layout == nil { + return + } + + speakerInfos := make([]SpeakerInfo, len(speakers)) + for i, s := range speakers { + speakerInfos[i] = SpeakerInfo{ + Identity: s.Identity(), + AudioLevel: s.AudioLevel(), + IsSpeaking: s.IsSpeaking(), + } + } + + pads := b.layout.UpdateSpeakers(speakerInfos) + if pads != nil { + b.applyLayout(pads) + } +} + +func (b *VideoBin) applyLayout(pads []PadLayout) { + b.mu.Lock() + defer b.mu.Unlock() + + b.applyLayoutLocked(pads) +} + +func (b *VideoBin) applyLayoutLocked(pads []PadLayout) { + for _, pl := range pads { + name, ok := b.names[pl.TrackID] + if !ok { + continue + } + pad, ok := b.pads[name] + if !ok { + continue + } + pad.SetProperty("xpos", pl.X) + pad.SetProperty("ypos", pl.Y) + pad.SetProperty("width", pl.W) + pad.SetProperty("height", pl.H) + pad.SetProperty("alpha", pl.Alpha) + pad.SetProperty("zorder", pl.ZOrder) + + if b.setVideoDimensions != nil && pl.W > 0 && pl.H > 0 { + b.setVideoDimensions(pl.TrackID, pl.W, pl.H) } } - b.mu.Unlock() } func (b *VideoBin) buildWebInput() error { @@ -258,33 +331,49 @@ func (b *VideoBin) buildSDKInput() error { b.pads = make(map[string]*gst.Pad) b.names = make(map[string]string) - // add selector first so pads can be created if b.conf.VideoDecoding { - if err := b.addSelector(); err != nil { + if b.conf.Compositing { + if err := b.addCompositor(); err != nil { + return err + } + } else { + if err := b.addSelector(); err != nil { + return err + } + } + if err := b.addVideoTestSrcBin(); err != nil { return err } } - if b.conf.VideoTrack != nil { - if err := b.addAppSrcBin(b.conf.VideoTrack); err != nil { + if b.conf.Compositing { + b.layout = NewLayoutManager(b.conf.Layout, int(b.conf.Width), int(b.conf.Height)) + } + + for _, vt := range b.conf.VideoTracks { + if err := b.addAppSrcBin(vt); err != nil { return err } + if b.layout != nil { + source := TrackSourceCamera + if vt.PublicationSource == livekit.TrackSource_SCREEN_SHARE { + source = TrackSourceScreenShare + } + pads := b.layout.AddTrack(vt.TrackID, vt.ParticipantIdentity, source) + b.applyLayout(pads) + } } if b.conf.VideoDecoding { b.bin.SetGetSrcPad(b.getSrcPad) - - if err := b.addVideoTestSrcBin(); err != nil { + if err := b.addDecodedVideoSink(); err != nil { return err } - if b.conf.VideoTrack == nil { + if !b.conf.Compositing && len(b.conf.VideoTracks) == 0 { if err := b.setSelectorPad(videoTestSrcName); err != nil { return err } } - if err := b.addDecodedVideoSink(); err != nil { - return err - } } return nil @@ -308,7 +397,7 @@ func (b *VideoBin) addAppSrcBin(ts *config.TrackSource) error { } if b.conf.VideoDecoding { - return b.setSelectorPad(name) + return b.setTrackVisible(name, true) } return nil @@ -503,26 +592,24 @@ func (b *VideoBin) buildAppSrcBin(ts *config.TrackSource, name string) (*gstream return appSrcBin, nil } -func (b *VideoBin) addVideoTestSrcBin() error { - testSrcBin := b.bin.NewBin(videoTestSrcName) - if err := b.bin.AddSourceBin(testSrcBin); err != nil { - return err - } - - videoTestSrc, err := gst.NewElement("videotestsrc") +func (b *VideoBin) addCompositor() error { + compositor, err := gst.NewElement("compositor") if err != nil { return errors.ErrGstPipelineError(err) } - if err = videoTestSrc.SetProperty("is-live", true); err != nil { + compositor.SetArg("background", "black") + if err = compositor.SetProperty("latency", uint64(b.conf.Latency.JitterBufferLatency)); err != nil { + return errors.ErrGstPipelineError(err) + } + if err = compositor.SetProperty("min-upstream-latency", uint64(b.conf.Latency.JitterBufferLatency)); err != nil { return errors.ErrGstPipelineError(err) } - videoTestSrc.SetArg("pattern", "black") - queue, err := gstreamer.BuildQueue("video_test_src_queue", b.conf.Latency.PipelineLatency, false) + videoRate, err := gst.NewElement("videorate") if err != nil { - return err + return errors.ErrGstPipelineError(err) } - if err = queue.SetProperty("min-threshold-time", uint64(2e9)); err != nil { + if err = videoRate.SetProperty("skip-to-first", true); err != nil { return errors.ErrGstPipelineError(err) } @@ -531,11 +618,11 @@ func (b *VideoBin) addVideoTestSrcBin() error { return errors.ErrGstPipelineError(err) } - if err = testSrcBin.AddElements(videoTestSrc, queue, caps); err != nil { + if err = b.bin.AddElements(compositor, videoRate, caps); err != nil { return err } - b.createTestSrcPad() + b.selector = compositor return nil } @@ -566,6 +653,60 @@ func (b *VideoBin) addSelector() error { return nil } +func (b *VideoBin) addVideoTestSrcBin() error { + testSrcBin := b.bin.NewBin(videoTestSrcName) + if err := b.bin.AddSourceBin(testSrcBin); err != nil { + return err + } + + videoTestSrc, err := gst.NewElement("videotestsrc") + if err != nil { + return errors.ErrGstPipelineError(err) + } + if err = videoTestSrc.SetProperty("is-live", true); err != nil { + return errors.ErrGstPipelineError(err) + } + videoTestSrc.SetArg("pattern", "black") + + queue, err := gstreamer.BuildQueue("video_test_src_queue", b.conf.Latency.PipelineLatency, false) + if err != nil { + return err + } + + caps, err := b.newVideoCapsFilter(true) + if err != nil { + return errors.ErrGstPipelineError(err) + } + + if err = testSrcBin.AddElements(videoTestSrc, queue, caps); err != nil { + return err + } + + pad := b.selector.GetRequestPad("sink_%u") + if b.conf.Compositing { + pad.SetProperty("xpos", 0) + pad.SetProperty("ypos", 0) + pad.SetProperty("width", int(b.conf.Width)) + pad.SetProperty("height", int(b.conf.Height)) + pad.SetProperty("zorder", uint(0)) + pad.SetProperty("alpha", 1.0) + } else { + pad.AddProbe(gst.PadProbeTypeBuffer, func(_ *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { + pts := uint64(info.GetBuffer().PresentationTimestamp()) + b.mu.Lock() + if pts < b.lastPTS || b.selectedPad != videoTestSrcName { + b.mu.Unlock() + return gst.PadProbeDrop + } + b.lastPTS = pts + b.mu.Unlock() + return gst.PadProbeOK + }) + } + b.pads[videoTestSrcName] = pad + return nil +} + func (b *VideoBin) addEncoder() error { videoQueue, err := gstreamer.BuildQueue("video_encoder_queue", b.conf.Latency.PipelineLatency, false) if err != nil { @@ -717,26 +858,21 @@ func (b *VideoBin) addVideoConverter(bin *gstreamer.Bin) error { return errors.ErrGstPipelineError(err) } - elements := []*gst.Element{videoQueue, videoConvert, videoScale} - - if !b.conf.VideoDecoding { - videoRate, err := gst.NewElement("videorate") - if err != nil { - return errors.ErrGstPipelineError(err) - } - if err = videoRate.SetProperty("skip-to-first", true); err != nil { - return errors.ErrGstPipelineError(err) - } - elements = append(elements, videoRate) + videoRate, err := gst.NewElement("videorate") + if err != nil { + return errors.ErrGstPipelineError(err) + } + if err = videoRate.SetProperty("skip-to-first", true); err != nil { + return errors.ErrGstPipelineError(err) } - caps, err := b.newVideoCapsFilter(!b.conf.VideoDecoding) + // Compositor downstream requires framerate-locked inputs. + caps, err := b.newVideoCapsFilter(true) if err != nil { return errors.ErrGstPipelineError(err) } - elements = append(elements, caps) - return bin.AddElements(elements...) + return bin.AddElements(videoQueue, videoConvert, videoScale, videoRate, caps) } func (b *VideoBin) newVideoCapsFilter(includeFramerate bool) (*gst.Element, error) { @@ -779,39 +915,62 @@ func (b *VideoBin) createSrcPadLocked(trackID, name string) { b.names[trackID] = name pad := b.selector.GetRequestPad("sink_%u") - pad.AddProbe(gst.PadProbeTypeBuffer, func(_ *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { - pts := uint64(info.GetBuffer().PresentationTimestamp()) - b.mu.Lock() - if pts < b.lastPTS || (b.selectedPad != videoTestSrcName && b.selectedPad != name) { + if b.conf.Compositing { + pad.SetProperty("xpos", 0) + pad.SetProperty("ypos", 0) + pad.SetProperty("width", int(b.conf.Width)) + pad.SetProperty("height", int(b.conf.Height)) + pad.SetProperty("zorder", uint(1)) + } else { + pad.AddProbe(gst.PadProbeTypeBuffer, func(_ *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { + pts := uint64(info.GetBuffer().PresentationTimestamp()) + b.mu.Lock() + if pts < b.lastPTS || (b.selectedPad != videoTestSrcName && b.selectedPad != name) { + b.mu.Unlock() + return gst.PadProbeDrop + } + b.lastPTS = pts b.mu.Unlock() - return gst.PadProbeDrop - } - b.lastPTS = pts - b.mu.Unlock() - return gst.PadProbeOK - }) + return gst.PadProbeOK + }) + } b.pads[name] = pad } -func (b *VideoBin) createTestSrcPad() { +func (b *VideoBin) setTrackVisible(name string, visible bool) error { b.mu.Lock() defer b.mu.Unlock() - pad := b.selector.GetRequestPad("sink_%u") - pad.AddProbe(gst.PadProbeTypeBuffer, func(_ *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { - pts := uint64(info.GetBuffer().PresentationTimestamp()) - b.mu.Lock() - if pts < b.lastPTS || (b.selectedPad != videoTestSrcName) { - b.mu.Unlock() - return gst.PadProbeDrop + return b.setTrackVisibleLocked(name, visible) +} + +func (b *VideoBin) setTrackVisibleLocked(name string, visible bool) error { + if b.conf.Compositing { + pad, ok := b.pads[name] + if !ok { + return errors.New("pad not found: " + name) } - b.lastPTS = pts - b.mu.Unlock() - return gst.PadProbeOK - }) - b.pads[videoTestSrcName] = pad + alpha := 0.0 + if visible { + alpha = 1.0 + } + if err := pad.SetProperty("alpha", alpha); err != nil { + return errors.ErrGstPipelineError(err) + } + + logger.Debugw("track visibility changed", "name", name, "visible", visible) + return nil + } + + if visible { + return b.setSelectorPadLocked(name) + } + if b.selectedPad == name { + return b.setSelectorPadLocked(videoTestSrcName) + } + return nil } func (b *VideoBin) setSelectorPad(name string) error { @@ -822,9 +981,11 @@ func (b *VideoBin) setSelectorPad(name string) error { } func (b *VideoBin) setSelectorPadLocked(name string) error { - pad := b.pads[name] + pad, ok := b.pads[name] + if !ok { + return errors.New("pad not found: " + name) + } - // drop until the next keyframe pad.AddProbe(gst.PadProbeTypeBuffer, func(_ *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { buffer := info.GetBuffer() if buffer.HasFlags(gst.BufferFlagDeltaUnit) { diff --git a/pkg/pipeline/controller.go b/pkg/pipeline/controller.go index a1deaca4..f8fa8b8f 100644 --- a/pkg/pipeline/controller.go +++ b/pkg/pipeline/controller.go @@ -30,6 +30,8 @@ import ( "go.uber.org/atomic" "go.uber.org/zap" + "go.opentelemetry.io/otel" + "github.com/livekit/egress/pkg/config" "github.com/livekit/egress/pkg/errors" "github.com/livekit/egress/pkg/gstreamer" @@ -42,7 +44,6 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/psrpc" - "go.opentelemetry.io/otel" ) const ( @@ -211,7 +212,13 @@ func (c *Controller) BuildPipeline() error { } } if c.VideoEnabled { - if err = builder.BuildVideoBin(p, c.PipelineConfig); err != nil { + var setDims func(string, int, int) + if c.Compositing { + if sdkSrc, ok := c.src.(*source.SDKSource); ok { + setDims = sdkSrc.UpdateTrackDimensions + } + } + if err = builder.BuildVideoBin(p, c.PipelineConfig, setDims); err != nil { return err } } diff --git a/pkg/pipeline/source/sdk.go b/pkg/pipeline/source/sdk.go index 524ad246..ccc2e339 100644 --- a/pkg/pipeline/source/sdk.go +++ b/pkg/pipeline/source/sdk.go @@ -50,7 +50,6 @@ type SDKSource struct { mu deadlock.Mutex initialized core.Fuse filenameReplacements map[string]string - audioChannels map[string]livekit.AudioChannel workersMu deadlock.RWMutex workers map[string]*trackWorker @@ -86,7 +85,6 @@ func NewSDKSource(ctx context.Context, p *config.PipelineConfig, callbacks *gstr PipelineConfig: p, callbacks: callbacks, filenameReplacements: make(map[string]string), - audioChannels: make(map[string]livekit.AudioChannel), workers: make(map[string]*trackWorker), } logger.Debugw("latency config", "latency", p.Latency) @@ -263,6 +261,10 @@ func (s *SDKSource) joinRoom() error { OnDisconnectedWithReason: s.onDisconnectedWithReason, } + if s.Compositing { + cb.OnActiveSpeakersChanged = s.onActiveSpeakersChanged + } + switch s.RequestType { case types.RequestTypeRoomComposite, types.RequestTypeTemplate, types.RequestTypeMedia: cb.OnTrackPublished = s.onTrackPublished @@ -716,13 +718,7 @@ func (s *SDKSource) shouldSubscribe(pub lksdk.TrackPublication) bool { func (s *SDKSource) shouldSubscribeMedia(pub lksdk.TrackPublication, rp *lksdk.RemoteParticipant) bool { switch pub.Kind() { case lksdk.TrackKindAudio: - if route := s.matchesAudioRoute(pub, rp); route != nil { - s.mu.Lock() - s.audioChannels[pub.SID()] = route.Channel - s.mu.Unlock() - return true - } - return s.CaptureAudioAll + return s.matchesAudioRoute(pub, rp) != nil case lksdk.TrackKindVideo: return s.matchesMediaVideo(pub, rp) default: @@ -749,6 +745,9 @@ func (s *SDKSource) matchesAudioRoute(pub lksdk.TrackPublication, rp *lksdk.Remo } } } + if s.CaptureAudioAll { + return &config.AudioRouteConfig{} + } return nil } @@ -863,7 +862,9 @@ func (s *SDKSource) disconnectRoom() { } func (s *SDKSource) shouldUseOneShotSenderReportSync() bool { - return s.RequestType == types.RequestTypeRoomComposite // one-shot correction is only useful when the audio mixer can drop late audio + // one-shot correction is only useful when the audio mixer can drop late audio + return !s.Compositing && + (s.RequestType == types.RequestTypeRoomComposite || s.RequestType == types.RequestTypeTemplate) } func (s *SDKSource) shouldEnableOneShotSenderReportSync() bool { @@ -871,12 +872,35 @@ func (s *SDKSource) shouldEnableOneShotSenderReportSync() bool { } func (s *SDKSource) shouldDisableAudioPTSAdjustment() bool { - return s.RequestType == types.RequestTypeRoomComposite || // SDK room composites are audio only - no need to adjust audio timestamps - s.RequestType == types.RequestTypeTemplate || // SDK templates are audio only - same as room composite + return (s.RequestType == types.RequestTypeRoomComposite && !s.Compositing) || // SDK room composites are audio only unless compositing - no need to adjust audio timestamps + (s.RequestType == types.RequestTypeTemplate && !s.Compositing) || // SDK templates are audio only - same as room composite s.RequestType == types.RequestTypeTrack || // no A/V sync needed for single track requests s.AudioTempoController.Enabled } +func (s *SDKSource) onActiveSpeakersChanged(speakers []lksdk.Participant) { + s.callbacks.OnActiveSpeakersChanged(speakers) +} + +// UpdateTrackDimensions sets the preferred video dimensions for a track's publication. +// Called when layout changes to request appropriate simulcast layers. +func (s *SDKSource) UpdateTrackDimensions(trackID string, width, height int) { + room := s.room.Load() + if room == nil { + return + } + for _, rp := range room.GetRemoteParticipants() { + for _, pub := range rp.TrackPublications() { + if pub.SID() == trackID { + if remotePub, ok := pub.(*lksdk.RemoteTrackPublication); ok { + remotePub.SetVideoDimensions(uint32(width), uint32(height)) + } + return + } + } + } +} + func shouldWarnOnRoomDisconnect(reason lksdk.DisconnectionReason, protoReason livekit.DisconnectReason) bool { return reason != lksdk.RoomClosed && reason != lksdk.LeaveRequested && diff --git a/pkg/pipeline/source/sdk/appwriter.go b/pkg/pipeline/source/sdk/appwriter.go index 883920dc..61676722 100644 --- a/pkg/pipeline/source/sdk/appwriter.go +++ b/pkg/pipeline/source/sdk/appwriter.go @@ -751,7 +751,7 @@ func isDiscontinuity(lastPTS time.Duration, pts time.Duration) bool { func (w *AppWriter) shouldRemoveBeforeDrain() bool { return w.track.Kind() == webrtc.RTPCodecTypeVideo && - (w.conf.RequestType == types.RequestTypeParticipant || w.conf.RequestType == types.RequestTypeRoomComposite || w.conf.RequestType == types.RequestTypeMedia) + (w.conf.RequestType == types.RequestTypeParticipant || w.conf.RequestType == types.RequestTypeRoomComposite || w.conf.RequestType == types.RequestTypeTemplate || w.conf.RequestType == types.RequestTypeMedia) } func (w *AppWriter) ensureRemovedBeforeDrain() { diff --git a/pkg/pipeline/source/track_worker.go b/pkg/pipeline/source/track_worker.go index 49e3aa02..b5ad0b9f 100644 --- a/pkg/pipeline/source/track_worker.go +++ b/pkg/pipeline/source/track_worker.go @@ -231,7 +231,7 @@ func (s *SDKSource) updatePreInitStateLocked(op Operation, ts *config.TrackSourc s.VideoEncoding = true } } - s.VideoTrack = ts + s.VideoTracks = append(s.VideoTracks, ts) } // Set identity and filename replacements based on request type @@ -463,20 +463,23 @@ func (s *SDKSource) createWriterForOp(op Operation) (*sdk.AppWriter, *config.Tra } ts := &config.TrackSource{ - TrackID: pub.SID(), - TrackKind: pub.Kind(), - ParticipantKind: rp.Kind(), - MimeType: types.MimeType(strings.ToLower(track.Codec().MimeType)), - PayloadType: track.Codec().PayloadType, - ClockRate: track.Codec().ClockRate, + TrackID: pub.SID(), + TrackKind: pub.Kind(), + ParticipantIdentity: rp.Identity(), + PublicationSource: pub.Source(), + ParticipantKind: rp.Kind(), + MimeType: types.MimeType(strings.ToLower(track.Codec().MimeType)), + PayloadType: track.Codec().PayloadType, + ClockRate: track.Codec().ClockRate, } - // Set audio channel from route match (RequestTypeMedia) - s.mu.Lock() - if ch, ok := s.audioChannels[pub.SID()]; ok { - ts.AudioChannel = &ch + // Match here (not at subscribe time): rp.Kind() can race with OnTrackPublished but is settled by writer creation. + if s.RequestType == types.RequestTypeMedia { + if route := s.matchesAudioRoute(pub, rp); route != nil { + ch := route.Channel + ts.AudioChannel = &ch + } } - s.mu.Unlock() ts.AppSrc = app.SrcFromElement(src) diff --git a/test/ffprobe.go b/test/ffprobe.go index e39607b7..a14158de 100644 --- a/test/ffprobe.go +++ b/test/ffprobe.go @@ -112,15 +112,15 @@ func verify(t *testing.T, in string, p *config.PipelineConfig, res *livekit.Egre info, err := ffprobe(in) require.NoError(t, err) - // Check source type if res != nil { - if (p.RequestType == types.RequestTypeRoomComposite || p.RequestType == types.RequestTypeTemplate) && (p.VideoEnabled || p.Layout != "") { - require.Equal(t, livekit.EgressSourceType_EGRESS_SOURCE_TYPE_WEB, res.SourceType) - } else if p.RequestType == types.RequestTypeWeb { - require.Equal(t, livekit.EgressSourceType_EGRESS_SOURCE_TYPE_WEB, res.SourceType) - } else { - require.Equal(t, livekit.EgressSourceType_EGRESS_SOURCE_TYPE_SDK, res.SourceType) + var expected livekit.EgressSourceType + switch p.SourceType { + case types.SourceTypeWeb: + expected = livekit.EgressSourceType_EGRESS_SOURCE_TYPE_WEB + case types.SourceTypeSDK: + expected = livekit.EgressSourceType_EGRESS_SOURCE_TYPE_SDK } + require.Equal(t, expected, res.SourceType) } switch egressType { diff --git a/test/layout.go b/test/layout.go index a3ab9ed2..6ad253dc 100644 --- a/test/layout.go +++ b/test/layout.go @@ -25,18 +25,29 @@ import ( ) const ( - // gridGap is --grid-gap: 0.5rem = 8px at 16px base font size. - // Used between grid cells; the grid itself sits flush against the frame. + // gridGap is --grid-gap: 0.5rem = 8px; applied both as cell gap and outer padding. gridGap = 8 - // regionInset is the margin applied to the sides and bottom of each region - // to avoid sampling at tile edges (compression artifacts, borders, etc.). - // The top is intentionally NOT inset: the test pattern's flash bar lives at - // the top of each tile, so insetting the top would cut it off — this is - // especially critical for carousel thumbnails where the scaled flash bar is - // only ~12px tall. + // regionInset insets sampling regions to avoid edge compression artifacts. + // Top is intentionally not inset: the flash bar lives at the tile top and is + // only ~12px tall on carousel thumbnails. regionInset = 20 + // minCarouselTileHeight mirrors CarouselLayout.tsx MIN_HEIGHT. + minCarouselTileHeight = 130 ) +// maxCarouselTiles mirrors CarouselLayout.tsx vertical sizing. +func maxCarouselTiles(carouselW, innerH int) int { + tileSpan := carouselW * 6 / 10 + if tileSpan < minCarouselTileHeight { + tileSpan = minCarouselTileHeight + } + n := (innerH + tileSpan/2) / tileSpan + if n < 1 { + n = 1 + } + return n +} + // insetRect shrinks r by margin on the sides and bottom only. The top edge is // preserved so the flash strip remains within the region. func insetRect(r image.Rectangle, margin int) image.Rectangle { @@ -60,18 +71,17 @@ func insetRect(r image.Rectangle, margin int) image.Rectangle { // numParticipants is the total number of participants in the room (including // the dominant speaker). func SpeakerLayoutRegions(width, height, numParticipants int) []avsync.Region { - // CSS Grid only applies grid-gap between cells, not around the grid, so the - // rendered output has no outer page padding. Tiles start at the frame edge. - - // Two columns: 1fr + 5fr = 6fr total, with one gap between them. totalFr := 6 - availableForCols := width - gridGap // subtract the single inter-column gap + innerW := width - 2*gridGap + innerH := height - 2*gridGap + availableForCols := innerW - gridGap carouselW := availableForCols / totalFr stageW := availableForCols - carouselW + stageX := gridGap + carouselW + gridGap stage := image.Rectangle{ - Min: image.Pt(carouselW+gridGap, 0), - Max: image.Pt(carouselW+gridGap+stageW, height), + Min: image.Pt(stageX, gridGap), + Max: image.Pt(stageX+stageW, gridGap+innerH), } regions := []avsync.Region{ @@ -81,16 +91,18 @@ func SpeakerLayoutRegions(width, height, numParticipants int) []avsync.Region { }, } - // Carousel renders numParticipants-1 thumbnails (the active speaker shows - // on stage, not in the carousel — see template-default/SpeakerLayout.tsx). - // Stacked vertically, no row gap (CSS grid-row-gap doesn't visibly apply). + maxTiles := maxCarouselTiles(carouselW, innerH) + thumbCount := numParticipants - 1 + if thumbCount > maxTiles { + thumbCount = maxTiles + } thumbW := carouselW - thumbH := thumbW * 10 / 16 - for i := 0; i < numParticipants-1; i++ { - thumbY := i * thumbH + thumbH := (innerH - gridGap*(maxTiles-1)) / maxTiles + for i := 0; i < thumbCount; i++ { + thumbY := gridGap + i*(thumbH+gridGap) thumb := image.Rectangle{ - Min: image.Pt(0, thumbY), - Max: image.Pt(thumbW, thumbY+thumbH), + Min: image.Pt(gridGap, thumbY), + Max: image.Pt(gridGap+thumbW, thumbY+thumbH), } regions = append(regions, avsync.Region{ Name: fmt.Sprintf("thumb%d", i), @@ -115,20 +127,29 @@ func SpeakerLayoutRegions(width, height, numParticipants int) []avsync.Region { // Rows are determined by ceil(numParticipants / cols). All cells are equal in // size, separated by --grid-gap (8px), with 8px padding around the grid. func GridLayoutRegions(width, height, numParticipants int) []avsync.Region { + // Single participant fills the canvas (no padding); see calculateGridLayout. + if numParticipants == 1 { + full := image.Rectangle{Min: image.Pt(0, 0), Max: image.Pt(width, height)} + return []avsync.Region{{Name: "cell0", Rect: insetRect(full, regionInset)}} + } + cols := gridColumns(width, numParticipants) rows := int(math.Ceil(float64(numParticipants) / float64(cols))) - // No outer page padding (see SpeakerLayoutRegions for rationale). - cellW := (width - (cols-1)*gridGap) / cols - cellH := (height - (rows-1)*gridGap) / rows + innerW := width - 2*gridGap + innerH := height - 2*gridGap + cellW := (innerW - (cols-1)*gridGap) / cols + cellH := (innerH - (rows-1)*gridGap) / rows regions := make([]avsync.Region, 0, numParticipants) for i := 0; i < numParticipants; i++ { col := i % cols row := i / cols + x := gridGap + col*(cellW+gridGap) + y := gridGap + row*(cellH+gridGap) cell := image.Rectangle{ - Min: image.Pt(col*(cellW+gridGap), row*(cellH+gridGap)), - Max: image.Pt(col*(cellW+gridGap)+cellW, row*(cellH+gridGap)+cellH), + Min: image.Pt(x, y), + Max: image.Pt(x+cellW, y+cellH), } regions = append(regions, avsync.Region{ Name: fmt.Sprintf("cell%d", i), diff --git a/test/runner.go b/test/runner.go index aa51190d..2f1b797e 100644 --- a/test/runner.go +++ b/test/runner.go @@ -168,6 +168,7 @@ func NewRunner(t *testing.T) *Runner { require.NoError(t, err) conf.EnableSyncEngine = true + conf.EnableTemplateSDK = true conf.AudioTempoController.Enabled = true conf.AudioTempoController.AdjustmentRate = 0.05 // short grace so the pulse sink reaper edge case completes quickly