diff --git a/.changes/adaptive-stream-manual-quality-merge b/.changes/adaptive-stream-manual-quality-merge new file mode 100644 index 000000000..083a3def6 --- /dev/null +++ b/.changes/adaptive-stream-manual-quality-merge @@ -0,0 +1 @@ +patch type="improved" "Allow manual video quality selection with adaptive stream enabled" diff --git a/lib/src/publication/remote.dart b/lib/src/publication/remote.dart index f9e30bb5e..aa74cb87d 100644 --- a/lib/src/publication/remote.dart +++ b/lib/src/publication/remote.dart @@ -33,6 +33,7 @@ import '../types/other.dart'; import '../types/video_dimensions.dart'; import '../utils.dart'; import 'track_publication.dart'; +import 'track_settings.dart'; /// Represents a track publication from a RemoteParticipant. Provides methods to /// control if we should subscribe to the track, and its quality (for video). @@ -48,11 +49,15 @@ class RemoteTrackPublication extends TrackPublication int? _fps; int get fps => _fps ?? 0; - VideoQuality? _videoQuality = VideoQuality.HIGH; - VideoQuality get videoQuality => _videoQuality ?? VideoQuality.HIGH; + // Manual settings (set by user via setVideoQuality / setVideoDimensions) + VideoSettings? _userPreference; - VideoDimensions? _videoDimensions; - VideoDimensions? get videoDimensions => _videoDimensions; + // Adaptive stream state (set automatically by visibility observer) + VideoDimensions? _adaptiveStreamDimensions; + bool _adaptiveStreamEnabled = true; + + VideoQuality get videoQuality => _userPreference?.quality ?? VideoQuality.HIGH; + VideoDimensions? get videoDimensions => _userPreference?.dimensions; /// The server may pause the track when they are bandwidth limitations and resume /// when there is more capacity. This property will be updated when the track is @@ -144,11 +149,6 @@ class RemoteTrackPublication extends TrackPublication final videoTrack = track as VideoTrack; - final settings = lk_rtc.UpdateTrackSettings( - trackSids: [sid], - disabled: true, - ); - // filter visible build contexts final viewSizes = videoTrack.viewKeys .map((e) => e.currentContext) @@ -161,15 +161,19 @@ class RemoteTrackPublication extends TrackPublication logger.finer('[Visibility] ${track?.sid} watching ${viewSizes.length} views...'); if (viewSizes.isNotEmpty) { - // compute largest size final largestSize = viewSizes.reduce((value, element) => maxOfSizes(value, element)); - - settings - ..disabled = false - ..width = largestSize.width.ceil() - ..height = largestSize.height.ceil(); + _adaptiveStreamDimensions = VideoDimensions( + largestSize.width.ceil(), + largestSize.height.ceil(), + ); + _adaptiveStreamEnabled = true; + } else { + _adaptiveStreamDimensions = null; + _adaptiveStreamEnabled = false; } + final settings = _buildTrackSettings(); + // Only send new settings to server if it changed if (settings != _lastSentTrackSettings) { _lastSentTrackSettings = settings; @@ -229,7 +233,7 @@ class RemoteTrackPublication extends TrackPublication return didUpdate; } - bool _canUpdateManualVideoSettings() { + bool _isManualOperationAllowed() { if (kind != TrackType.VIDEO) { logger.warning('Manual video setting updates are only supported for video tracks'); return false; @@ -240,55 +244,57 @@ class RemoteTrackPublication extends TrackPublication return false; } - if (participant.room.roomOptions.adaptiveStream) { - logger.warning('Manual video setting update ignored because adaptive stream is enabled'); - return false; - } - return true; } + /// For tracks that support simulcasting, adjust subscribed quality. + /// + /// This indicates the highest quality the client can accept. If network + /// bandwidth does not allow, the server will automatically reduce quality to + /// optimize for uninterrupted video. + /// + /// When adaptive stream is enabled, the server will use the smaller of + /// this setting and the adaptive stream dimensions. Future setVideoQuality(VideoQuality newValue) async { - if (newValue == _videoQuality) return; - if (!_canUpdateManualVideoSettings()) return; - _videoQuality = newValue; - _videoDimensions = null; - sendUpdateTrackSettings(); + if (newValue == _userPreference?.quality) return; + if (!_isManualOperationAllowed()) return; + _userPreference = VideoSettings.quality(newValue); + _emitTrackUpdate(); } /// Set preferred video dimensions for this track. /// /// Server will choose the appropriate layer based on these dimensions. /// Will override previous calls to [setVideoQuality]. + /// + /// When adaptive stream is enabled, the server will use the smaller of + /// this setting and the adaptive stream dimensions. Future setVideoDimensions(VideoDimensions newValue) async { - if (newValue.width == _videoDimensions?.width && newValue.height == _videoDimensions?.height) { - return; - } - if (!_canUpdateManualVideoSettings()) return; - _videoDimensions = newValue; - _videoQuality = null; - sendUpdateTrackSettings(); + if (newValue == _userPreference?.dimensions) return; + if (!_isManualOperationAllowed()) return; + _userPreference = VideoSettings.dimensions(newValue); + _emitTrackUpdate(); } /// Set desired FPS, server will do its best to return FPS close to this. /// It's only supported for video codecs that support SVC currently. Future setVideoFPS(int newValue) async { if (newValue == _fps) return; - if (!_canUpdateManualVideoSettings()) return; + if (!_isManualOperationAllowed()) return; _fps = newValue; - sendUpdateTrackSettings(); + _emitTrackUpdate(); } Future enable() async { if (_enabled) return; _enabled = true; - sendUpdateTrackSettings(); + _emitTrackUpdate(); } Future disable() async { if (!_enabled) return; _enabled = false; - sendUpdateTrackSettings(); + _emitTrackUpdate(); } Future subscribe() async { @@ -333,26 +339,48 @@ class RemoteTrackPublication extends TrackPublication participant.room.engine.signalClient.sendUpdateSubscription(subscription); } - @internal - void sendUpdateTrackSettings() { + lk_rtc.UpdateTrackSettings _buildTrackSettings() { + // disabled if manually disabled or adaptive stream says no views visible + final isDisabled = !_enabled || !_adaptiveStreamEnabled; + final settings = lk_rtc.UpdateTrackSettings( trackSids: [sid], - disabled: !_enabled, + disabled: isDisabled, ); + if (kind == TrackType.VIDEO) { - if (_videoDimensions != null) { - settings.width = _videoDimensions!.width; - settings.height = _videoDimensions!.height; - } else if (_videoQuality != null) { - settings.quality = _videoQuality!.toPBType(); - } else { - settings.quality = VideoQuality.HIGH.toPBType(); + final resolved = resolveVideoSettings( + adaptiveStreamDimensions: _adaptiveStreamDimensions, + userPreference: _userPreference, + layerDimensionsForQuality: (quality) { + final pbQuality = quality.toPBType(); + final layer = latestInfo?.layers.where((l) => l.quality == pbQuality).firstOrNull; + if (layer == null) return null; + return VideoDimensions(layer.width, layer.height); + }, + ); + + if (resolved.dimensions != null) { + settings.width = resolved.dimensions!.width; + settings.height = resolved.dimensions!.height; + } else if (resolved.quality != null) { + settings.quality = resolved.quality!.toPBType(); } if (_fps != null) settings.fps = _fps!; } + return settings; + } + + void _emitTrackUpdate() { + final settings = _buildTrackSettings(); + _lastSentTrackSettings = settings; participant.room.engine.signalClient.sendUpdateTrackSettings(settings); } + @internal + @Deprecated('Use _emitTrackUpdate instead') + void sendUpdateTrackSettings() => _emitTrackUpdate(); + @internal // Update internal var and return true if changed Future updateSubscriptionAllowed(bool allowed) async { diff --git a/lib/src/publication/track_settings.dart b/lib/src/publication/track_settings.dart new file mode 100644 index 000000000..0853969a0 --- /dev/null +++ b/lib/src/publication/track_settings.dart @@ -0,0 +1,72 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'package:meta/meta.dart' show immutable, internal; + +import '../types/other.dart'; +import '../types/video_dimensions.dart'; + +/// Represents a video quality setting — either explicit dimensions or a +/// quality level (LOW/MEDIUM/HIGH), never both. +/// +/// Used for both user-requested settings and the resolved merge result. +@internal +@immutable +class VideoSettings { + final VideoDimensions? dimensions; + final VideoQuality? quality; + + const VideoSettings.dimensions(VideoDimensions this.dimensions) : quality = null; + + const VideoSettings.quality(VideoQuality this.quality) : dimensions = null; + + static const high = VideoSettings.quality(VideoQuality.HIGH); +} + +/// Merges adaptive stream dimensions with manual [VideoSettings], +/// always picking the more conservative (smaller) of the two. +/// +/// This matches the JS SDK's merge behavior in `emitTrackUpdate()`. +@internal +VideoSettings resolveVideoSettings({ + VideoDimensions? adaptiveStreamDimensions, + VideoSettings? userPreference, + VideoDimensions? Function(VideoQuality quality)? layerDimensionsForQuality, +}) { + VideoDimensions? minDimensions = userPreference?.dimensions; + + if (adaptiveStreamDimensions != null) { + if (minDimensions != null) { + // Use the smaller of adaptive vs manually requested dimensions + if (adaptiveStreamDimensions.area() < minDimensions.area()) { + minDimensions = adaptiveStreamDimensions; + } + } else if (userPreference?.quality != null) { + // Compare adaptive dimensions with the max quality layer dimensions + final maxQualityLayer = layerDimensionsForQuality?.call(userPreference!.quality!); + if (maxQualityLayer != null && adaptiveStreamDimensions.area() < maxQualityLayer.area()) { + minDimensions = adaptiveStreamDimensions; + } + } else { + minDimensions = adaptiveStreamDimensions; + } + } + + if (minDimensions != null) { + return VideoSettings.dimensions(minDimensions); + } else if (userPreference?.quality != null) { + return VideoSettings.quality(userPreference!.quality!); + } + return VideoSettings.high; +} diff --git a/test/publication/track_settings_test.dart b/test/publication/track_settings_test.dart new file mode 100644 index 000000000..b85336dd9 --- /dev/null +++ b/test/publication/track_settings_test.dart @@ -0,0 +1,135 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import 'package:flutter_test/flutter_test.dart'; + +import 'package:livekit_client/src/publication/track_settings.dart'; +import 'package:livekit_client/src/types/other.dart'; +import 'package:livekit_client/src/types/video_dimensions.dart'; + +/// Test helper: returns layer dimensions for a standard 3-layer SVC/simulcast track. +VideoDimensions? _testLayerDimensions(VideoQuality quality) { + return { + VideoQuality.LOW: VideoDimensions(320, 180), + VideoQuality.MEDIUM: VideoDimensions(640, 360), + VideoQuality.HIGH: VideoDimensions(1280, 720), + }[quality]; +} + +void main() { + group('resolveVideoSettings', () { + group('no adaptive stream', () { + test('defaults to HIGH quality when nothing set', () { + final r = resolveVideoSettings(); + expect(r.quality, VideoQuality.HIGH); + expect(r.dimensions, isNull); + }); + + test('uses preferred quality', () { + final r = resolveVideoSettings( + userPreference: VideoSettings.quality(VideoQuality.LOW), + ); + expect(r.quality, VideoQuality.LOW); + expect(r.dimensions, isNull); + }); + + test('uses preferred dimensions', () { + final r = resolveVideoSettings( + userPreference: VideoSettings.dimensions(VideoDimensions(800, 600)), + ); + expect(r.dimensions, VideoDimensions(800, 600)); + expect(r.quality, isNull); + }); + }); + + group('adaptive stream only', () { + test('uses adaptive stream dimensions', () { + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(480, 270), + ); + expect(r.dimensions, VideoDimensions(480, 270)); + expect(r.quality, isNull); + }); + }); + + group('adaptive stream + preferred dimensions', () { + test('adaptive wins when smaller', () { + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(320, 180), + userPreference: VideoSettings.dimensions(VideoDimensions(1280, 720)), + ); + expect(r.dimensions, VideoDimensions(320, 180)); + }); + + test('preferred wins when smaller', () { + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(1920, 1080), + userPreference: VideoSettings.dimensions(VideoDimensions(640, 360)), + ); + expect(r.dimensions, VideoDimensions(640, 360)); + }); + + test('equal areas keep preferred', () { + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(640, 360), + userPreference: VideoSettings.dimensions(VideoDimensions(640, 360)), + ); + expect(r.dimensions, VideoDimensions(640, 360)); + }); + }); + + group('adaptive stream + preferred quality', () { + test('adaptive wins when smaller than quality layer', () { + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(320, 180), + userPreference: VideoSettings.quality(VideoQuality.HIGH), + layerDimensionsForQuality: _testLayerDimensions, + ); + // adaptive 320*180 < HIGH 1280*720 → sends adaptive dimensions + expect(r.dimensions, VideoDimensions(320, 180)); + expect(r.quality, isNull); + }); + + test('quality wins when adaptive is larger than quality layer', () { + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(1920, 1080), + userPreference: VideoSettings.quality(VideoQuality.LOW), + layerDimensionsForQuality: _testLayerDimensions, + ); + // adaptive 1920*1080 > LOW 320*180 → sends quality directly + expect(r.quality, VideoQuality.LOW); + expect(r.dimensions, isNull); + }); + + test('quality sent directly when no layer info available', () { + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(320, 180), + userPreference: VideoSettings.quality(VideoQuality.LOW), + ); + expect(r.quality, VideoQuality.LOW); + expect(r.dimensions, isNull); + }); + + test('quality sent when layer lookup returns null', () { + final r = resolveVideoSettings( + adaptiveStreamDimensions: VideoDimensions(320, 180), + userPreference: VideoSettings.quality(VideoQuality.MEDIUM), + layerDimensionsForQuality: (_) => null, + ); + expect(r.quality, VideoQuality.MEDIUM); + expect(r.dimensions, isNull); + }); + }); + }); +}