diff --git a/CMakeLists.txt b/CMakeLists.txt index 86eb1e49..504e9cc1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -76,6 +76,7 @@ set(FFI_PROTO_FILES ${FFI_PROTO_DIR}/e2ee.proto ${FFI_PROTO_DIR}/stats.proto ${FFI_PROTO_DIR}/data_stream.proto + ${FFI_PROTO_DIR}/data_track.proto ${FFI_PROTO_DIR}/rpc.proto ${FFI_PROTO_DIR}/track_publication.proto ) @@ -323,7 +324,10 @@ add_library(livekit SHARED src/audio_processing_module.cpp src/audio_source.cpp src/audio_stream.cpp + src/data_frame.cpp src/data_stream.cpp + src/data_track_error.cpp + src/data_track_subscription.cpp src/e2ee.cpp src/ffi_handle.cpp src/ffi_client.cpp @@ -331,7 +335,9 @@ add_library(livekit SHARED src/livekit.cpp src/logging.cpp src/local_audio_track.cpp + src/local_data_track.cpp src/remote_audio_track.cpp + src/remote_data_track.cpp src/room.cpp src/room_proto_converter.cpp src/room_proto_converter.h @@ -683,10 +689,6 @@ install(FILES # Build the LiveKit C++ bridge before examples (human_robot depends on it) add_subdirectory(bridge) -# ---- Examples ---- -# add_subdirectory(examples) - - if(LIVEKIT_BUILD_EXAMPLES) add_subdirectory(examples) endif() diff --git a/README.md b/README.md index eb473f26..b318e0d5 100644 --- a/README.md +++ b/README.md @@ -447,6 +447,35 @@ CPP SDK is using clang C++ format brew install clang-format ``` + +#### Memory Checks +Run valgrind on various examples or tests to check for memory leaks and other issues. +```bash +valgrind --leak-check=full ./build-debug/bin/livekit_integration_tests +valgrind --leak-check=full ./build-debug/bin/livekit_stress_tests +``` + +# Running locally +1. Install the livekit-server +https://docs.livekit.io/transport/self-hosting/local/ + +Start the livekit-server with data tracks enabled: +```bash +LIVEKIT_CONFIG="enable_data_tracks: true" livekit-server --dev +``` + +```bash +# generate tokens, do for all participants +lk token create \ + --api-key devkey \ + --api-secret secret \ + -i robot \ + --join \ + --valid-for 99999h \ + --room robo_room \ + --grant '{"canPublish":true,"canSubscribe":true,"canPublishData":true}' +``` +
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 19a37781..56cdf129 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -39,10 +39,18 @@ set(EXAMPLES_COMMON_DIR ${CMAKE_CURRENT_SOURCE_DIR}/common) # All example executables (used for copying livekit_ffi DLL/shared lib) set(EXAMPLES_ALL SimpleRoom + DataStampingProducer + DataStampingConsumer + UserTimestampedVideoProducer + UserTimestampedVideoConsumer SimpleRpc SimpleJoystickSender SimpleJoystickReceiver SimpleDataStream + PingPongPing + PingPongPong + HelloLivekitSender + HelloLivekitReceiver LoggingLevelsBasicUsage LoggingLevelsCustomSinks BridgeRobot @@ -234,6 +242,56 @@ target_link_libraries(SimpleDataStream spdlog::spdlog ) +# --- UserTimestampedVideo example --- + +add_executable(DataStampingProducer + data_stamping/producer.cpp +) + +target_include_directories(DataStampingProducer PRIVATE ${EXAMPLES_PRIVATE_INCLUDE_DIRS}) + +target_link_libraries(DataStampingProducer + PRIVATE + livekit + spdlog::spdlog +) + +add_executable(DataStampingConsumer + data_stamping/consumer.cpp +) + +target_include_directories(DataStampingConsumer PRIVATE ${EXAMPLES_PRIVATE_INCLUDE_DIRS}) + +target_link_libraries(DataStampingConsumer + PRIVATE + livekit + spdlog::spdlog +) + +add_executable(UserTimestampedVideoProducer + user_timestamped_video/producer.cpp +) + +target_include_directories(UserTimestampedVideoProducer PRIVATE ${EXAMPLES_PRIVATE_INCLUDE_DIRS}) + +target_link_libraries(UserTimestampedVideoProducer + PRIVATE + livekit + spdlog::spdlog +) + +add_executable(UserTimestampedVideoConsumer + user_timestamped_video/consumer.cpp +) + +target_include_directories(UserTimestampedVideoConsumer PRIVATE ${EXAMPLES_PRIVATE_INCLUDE_DIRS}) + +target_link_libraries(UserTimestampedVideoConsumer + PRIVATE + livekit + spdlog::spdlog +) + add_custom_command( TARGET SimpleDataStream POST_BUILD @@ -242,6 +300,77 @@ add_custom_command( $/data ) +# --- ping_pong (request/response latency measurement over data tracks) --- + +add_library(ping_pong_support STATIC + ping_pong/json_converters.cpp + ping_pong/json_converters.h + ping_pong/constants.h + ping_pong/messages.h + ping_pong/utils.h +) + +target_include_directories(ping_pong_support PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/ping_pong +) + +target_link_libraries(ping_pong_support + PRIVATE + nlohmann_json::nlohmann_json +) + +add_executable(PingPongPing + ping_pong/ping.cpp +) + +target_include_directories(PingPongPing PRIVATE ${EXAMPLES_PRIVATE_INCLUDE_DIRS}) + +target_link_libraries(PingPongPing + PRIVATE + ping_pong_support + livekit + spdlog::spdlog +) + +add_executable(PingPongPong + ping_pong/pong.cpp +) + +target_include_directories(PingPongPong PRIVATE ${EXAMPLES_PRIVATE_INCLUDE_DIRS}) + +target_link_libraries(PingPongPong + PRIVATE + ping_pong_support + livekit + spdlog::spdlog +) + +# --- hello_livekit (minimal synthetic video + data publish / subscribe) --- + +add_executable(HelloLivekitSender + hello_livekit/sender.cpp +) + +target_include_directories(HelloLivekitSender PRIVATE ${EXAMPLES_PRIVATE_INCLUDE_DIRS}) + +target_link_libraries(HelloLivekitSender + PRIVATE + livekit + spdlog::spdlog +) + +add_executable(HelloLivekitReceiver + hello_livekit/receiver.cpp +) + +target_include_directories(HelloLivekitReceiver PRIVATE ${EXAMPLES_PRIVATE_INCLUDE_DIRS}) + +target_link_libraries(HelloLivekitReceiver + PRIVATE + livekit + spdlog::spdlog +) + # --- bridge_human_robot examples (robot + human; use livekit_bridge and SDL3) --- add_executable(BridgeRobot @@ -398,4 +527,4 @@ if(UNIX) foreach(EXAMPLE ${EXAMPLES_BRIDGE}) add_dependencies(${EXAMPLE} copy_bridge_to_bin) endforeach() -endif() \ No newline at end of file +endif() diff --git a/examples/bridge_human_robot/human.cpp b/examples/bridge_human_robot/human.cpp index 81989eb5..3e8c553d 100644 --- a/examples/bridge_human_robot/human.cpp +++ b/examples/bridge_human_robot/human.cpp @@ -103,6 +103,11 @@ static void renderFrame(const livekit::VideoFrame &frame) { static std::atomic g_audio_frames{0}; static std::atomic g_video_frames{0}; +constexpr const char *kRobotMicTrackName = "robot-mic"; +constexpr const char *kRobotSimAudioTrackName = "robot-sim-audio"; +constexpr const char *kRobotCamTrackName = "robot-cam"; +constexpr const char *kRobotSimVideoTrackName = "robot-sim-frame"; + int main(int argc, char *argv[]) { // ----- Parse args / env ----- bool no_audio = false; @@ -232,7 +237,7 @@ int main(int argc, char *argv[]) { // ----- Set audio callbacks using Room::setOnAudioFrameCallback ----- room->setOnAudioFrameCallback( - "robot", livekit::TrackSource::SOURCE_MICROPHONE, + "robot", kRobotMicTrackName, [playAudio, no_audio](const livekit::AudioFrame &frame) { g_audio_frames.fetch_add(1, std::memory_order_relaxed); if (!no_audio && g_selected_source.load(std::memory_order_relaxed) == @@ -242,7 +247,7 @@ int main(int argc, char *argv[]) { }); room->setOnAudioFrameCallback( - "robot", livekit::TrackSource::SOURCE_SCREENSHARE_AUDIO, + "robot", kRobotSimAudioTrackName, [playAudio, no_audio](const livekit::AudioFrame &frame) { g_audio_frames.fetch_add(1, std::memory_order_relaxed); if (!no_audio && g_selected_source.load(std::memory_order_relaxed) == @@ -253,7 +258,7 @@ int main(int argc, char *argv[]) { // ----- Set video callbacks using Room::setOnVideoFrameCallback ----- room->setOnVideoFrameCallback( - "robot", livekit::TrackSource::SOURCE_CAMERA, + "robot", kRobotCamTrackName, [](const livekit::VideoFrame &frame, std::int64_t /*timestamp_us*/) { g_video_frames.fetch_add(1, std::memory_order_relaxed); if (g_selected_source.load(std::memory_order_relaxed) == @@ -263,7 +268,7 @@ int main(int argc, char *argv[]) { }); room->setOnVideoFrameCallback( - "robot", livekit::TrackSource::SOURCE_SCREENSHARE, + "robot", kRobotSimVideoTrackName, [](const livekit::VideoFrame &frame, std::int64_t /*timestamp_us*/) { g_video_frames.fetch_add(1, std::memory_order_relaxed); if (g_selected_source.load(std::memory_order_relaxed) == diff --git a/examples/common/sdl_media_manager.cpp b/examples/common/sdl_media_manager.cpp index f44c60ae..59b451ba 100644 --- a/examples/common/sdl_media_manager.cpp +++ b/examples/common/sdl_media_manager.cpp @@ -399,4 +399,4 @@ void SDLMediaManager::render() { if (renderer_running_.load(std::memory_order_relaxed) && sdl_renderer_) { sdl_renderer_->render(); } -} \ No newline at end of file +} diff --git a/examples/data_stamping/README.md b/examples/data_stamping/README.md new file mode 100644 index 00000000..ed19758a --- /dev/null +++ b/examples/data_stamping/README.md @@ -0,0 +1,28 @@ +# DataStamping + +Minimal two-process example: + +- `DataStampingProducer` publishes a synthetic camera track with + `metadata.user_timestamp_us` set on every frame. +- `DataStampingProducer` also publishes a data track named `imu` with JSON + payloads shaped like: + +```json +{ + "angular": {"x": 0.0, "y": 0.0, "z": 0.0}, + "linear": {"x": 0.0, "y": 0.0, "z": 0.0} +} +``` + +The IMU values are simulated sine waves. + +- `DataStampingConsumer` only registers: + - `addOnDataFrameCallback()` + - `setOnVideoFrameCallback()` + +Run them in the same room with different participant identities: + +```sh +LIVEKIT_URL=ws://localhost:7880 LIVEKIT_TOKEN= ./DataStampingProducer +LIVEKIT_URL=ws://localhost:7880 LIVEKIT_TOKEN= ./DataStampingConsumer +``` diff --git a/examples/data_stamping/consumer.cpp b/examples/data_stamping/consumer.cpp new file mode 100644 index 00000000..ead84041 --- /dev/null +++ b/examples/data_stamping/consumer.cpp @@ -0,0 +1,134 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "data_alignment_manager.h" +#include "livekit/livekit.h" + +using namespace livekit; +using livekit_examples::data_stamping::DataAlignmentManager; + +namespace { + +constexpr char kDataTrackName[] = "imu"; +constexpr char kProducerParticipantId[] = "producer"; + +std::atomic g_running{true}; + +void handleSignal(int) { g_running.store(false); } + +std::string getenvOrEmpty(const char *name) { + const char *value = std::getenv(name); + return value ? std::string(value) : std::string{}; +} + +void printUsage(const char *program) { + LK_LOG_INFO("Usage:\n {} \nor:\n LIVEKIT_URL=... " + "LIVEKIT_TOKEN=... {}", + program, program); +} + +bool parseArgs(int argc, char *argv[], std::string &url, std::string &token) { + if (argc > 1) { + const std::string first = argv[1]; + if (first == "-h" || first == "--help") { + return false; + } + } + + url = getenvOrEmpty("LIVEKIT_URL"); + token = getenvOrEmpty("LIVEKIT_TOKEN"); + + if (argc >= 3) { + url = argv[1]; + token = argv[2]; + } + + return !(url.empty() || token.empty()); +} +} // namespace + +int main(int argc, char *argv[]) { + std::string url; + std::string token; + + if (!parseArgs(argc, argv, url, token)) { + printUsage(argv[0]); + return 1; + } + + std::signal(SIGINT, handleSignal); +#ifdef SIGTERM + std::signal(SIGTERM, handleSignal); +#endif + + std::shared_ptr alignment_manager = + std::make_shared(2'000'000); + + livekit::initialize(livekit::LogLevel::Info, livekit::LogSink::kConsole); + + std::unique_ptr room = std::make_unique(); + RoomOptions options; + options.auto_subscribe = true; + options.dynacast = false; + + LK_LOG_INFO("[consumer] connecting to {}", url); + if (!room->Connect(url, token, options)) { + LK_LOG_ERROR("[consumer] failed to connect"); + return 1; + } + LK_LOG_INFO("[consumer] connected as {} to room '{}'", + room->localParticipant()->identity(), room->room_info().name); + + room->setOnVideoFrameEventCallback( + kProducerParticipantId, TrackSource::SOURCE_CAMERA, + [alignment_manager](const VideoFrameEvent &event) { + if (!event.metadata || !event.metadata->user_timestamp_us.has_value()) { + return; + } + alignment_manager->addVideoFrame(*event.metadata->user_timestamp_us); + }, + VideoStream::Options{}); + + room->addOnDataFrameCallback( + kProducerParticipantId, kDataTrackName, + [alignment_manager](const std::vector &, + std::optional user_timestamp) { + if (!user_timestamp.has_value()) { + return; + } + + alignment_manager->addImuFrame(*user_timestamp); + }); + + while (g_running.load(std::memory_order_relaxed)) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + + livekit::shutdown(); + return 0; +} diff --git a/examples/data_stamping/data_alignment_manager.h b/examples/data_stamping/data_alignment_manager.h new file mode 100644 index 00000000..a848c157 --- /dev/null +++ b/examples/data_stamping/data_alignment_manager.h @@ -0,0 +1,155 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace livekit_examples::data_stamping { + +enum class FrameType { + Video, + Imu, +}; + +/** + * Aligns recent video and IMU frames by user timestamp. + * + * This demo stamps video with `VideoFrameMetadata::user_timestamp_us` and + * IMU data with the same microseconds-since-epoch convention. The constructor + * accepts an alignment window in nanoseconds for convenience, but comparisons + * are performed in microseconds to match the stamped values. + * + * The default window is +/- 5 ms. At 100 Hz, IMU samples arrive roughly every + * 10 ms, so a 5 ms half-window is a reasonable default for pairing each ~30 Hz + * video frame with its nearest IMU sample without being overly permissive. + */ +class DataAlignmentManager { +public: + // Video comes in at ~30Hz, imu comes in at ~100Hz + static constexpr std::size_t kDefaultHistorySize = 30; + // We allow up to 5ms of difference between video and imu timestamps for them + // to be considered aligned. + static constexpr std::uint64_t kDefaultAlignmentWindowNs = 5'000'000; + + explicit DataAlignmentManager( + std::uint64_t alignment_window_ns = kDefaultAlignmentWindowNs, + std::size_t history_size = kDefaultHistorySize) + : aligned_frame_count_(0), alignment_window_us_(std::max( + 1, (alignment_window_ns + 999) / 1000)), + history_size_(std::max(1, history_size)) {} + + void addVideoFrame(std::uint64_t user_timestamp) { + addFrame(video_timestamps_, imu_timestamps_, user_timestamp, + FrameType::Video); + } + + void addImuFrame(std::uint64_t user_timestamp) { + addFrame(imu_timestamps_, video_timestamps_, user_timestamp, + FrameType::Imu); + } + + std::uint64_t numberAlignedDataFrames() const { + std::lock_guard lock(mutex_); + return aligned_frame_count_; + } + +private: + struct TimestampEntry { + std::uint64_t user_timestamp; + bool matched; + }; + + static std::uint64_t absoluteDifference(std::uint64_t lhs, + std::uint64_t rhs) { + return lhs >= rhs ? (lhs - rhs) : (rhs - lhs); + } + + void addFrame(std::deque &own_timestamps, + std::deque &other_timestamps, + std::uint64_t user_timestamp, FrameType frame_type) { + std::lock_guard lock(mutex_); + + own_timestamps.push_back(TimestampEntry{user_timestamp, false}); + trim(own_timestamps); + + const auto match_index = findBestMatch(other_timestamps, user_timestamp); + if (!match_index.has_value()) { + return; + } + + own_timestamps.back().matched = true; + other_timestamps[*match_index].matched = true; + ++aligned_frame_count_; + + LK_LOG_INFO( + "Aligned frames: {}={} {}={} " + "delta_us={}", + frame_type == FrameType::Video ? "Video" : "IMU", user_timestamp, + frame_type == FrameType::Video ? "IMU" : "Video", + other_timestamps[*match_index].user_timestamp, + absoluteDifference(user_timestamp, + other_timestamps[*match_index].user_timestamp)); + } + + void trim(std::deque ×tamps) const { + while (timestamps.size() > history_size_) { + timestamps.pop_front(); + } + } + + std::optional + findBestMatch(const std::deque ×tamps, + std::uint64_t user_timestamp) const { + std::optional best_index; + std::uint64_t best_delta = std::numeric_limits::max(); + + for (std::size_t i = 0; i < timestamps.size(); ++i) { + const auto &candidate = timestamps[i]; + if (candidate.matched) { + continue; + } + + const std::uint64_t delta = + absoluteDifference(candidate.user_timestamp, user_timestamp); + if (delta > alignment_window_us_ || delta >= best_delta) { + continue; + } + + best_index = i; + best_delta = delta; + } + + return best_index; + } + + mutable std::mutex mutex_; + std::deque video_timestamps_; + std::deque imu_timestamps_; + std::uint64_t aligned_frame_count_; + std::uint64_t alignment_window_us_; + std::size_t history_size_; +}; + +} // namespace livekit_examples::data_stamping diff --git a/examples/data_stamping/producer.cpp b/examples/data_stamping/producer.cpp new file mode 100644 index 00000000..c0270712 --- /dev/null +++ b/examples/data_stamping/producer.cpp @@ -0,0 +1,266 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "livekit/livekit.h" + +using namespace livekit; + +namespace { + +constexpr char kVideoTrackName[] = "stamped-camera"; +constexpr char kDataTrackName[] = "imu"; +constexpr int kFrameWidth = 640; +constexpr int kFrameHeight = 360; +constexpr int kVideoIntervalMs = 33; // ~30Hz +constexpr int kDataIntervalMs = 10; // 100Hz +constexpr double kWaveHz = 0.5; +constexpr double kAngularAmplitude = 1.0; +constexpr double kLinearAmplitude = 2.0; + +std::atomic g_running{true}; + +void handleSignal(int) { g_running.store(false); } + +std::string getenvOrEmpty(const char *name) { + const char *value = std::getenv(name); + return value ? std::string(value) : std::string{}; +} + +std::uint64_t nowEpochUs() { + return static_cast( + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count()); +} + +double sampleWave(double elapsed_seconds, double phase) { + constexpr double kTau = 6.28318530717958647692; + return std::sin((kTau * kWaveHz * elapsed_seconds) + phase); +} + +std::string buildImuJson(double elapsed_seconds) { + const double angular_x = kAngularAmplitude * sampleWave(elapsed_seconds, 0.0); + const double angular_y = + kAngularAmplitude * sampleWave(elapsed_seconds, 2.09439510239); + const double angular_z = + kAngularAmplitude * sampleWave(elapsed_seconds, 4.18879020479); + + const double linear_x = kLinearAmplitude * sampleWave(elapsed_seconds, 0.5); + const double linear_y = kLinearAmplitude * sampleWave(elapsed_seconds, 1.5); + const double linear_z = kLinearAmplitude * sampleWave(elapsed_seconds, 2.5); + + std::ostringstream stream; + stream << std::fixed << std::setprecision(4) << "{" + << "\"angular\":{\"x\":" << angular_x << ",\"y\":" << angular_y + << ",\"z\":" << angular_z << "}," + << "\"linear\":{\"x\":" << linear_x << ",\"y\":" << linear_y + << ",\"z\":" << linear_z << "}" + << "}"; + return stream.str(); +} + +void fillFrame(VideoFrame &frame, double elapsed_seconds) { + const double brightness = 0.5 + 0.5 * sampleWave(elapsed_seconds, 0.0); + const std::uint8_t blue = + static_cast(40 + (brightness * 120.0)); + const std::uint8_t green = + static_cast(60 + (brightness * 140.0)); + const std::uint8_t red = static_cast(80 + (brightness * 100.0)); + + std::uint8_t *data = frame.data(); + for (std::size_t i = 0; i < frame.dataSize(); i += 4) { + data[i + 0] = blue; + data[i + 1] = green; + data[i + 2] = red; + data[i + 3] = 255; + } +} + +void printUsage(const char *program) { + LK_LOG_INFO("Usage:\n {} \nor:\n LIVEKIT_URL=... " + "LIVEKIT_TOKEN=... {}", + program, program); +} + +bool parseArgs(int argc, char *argv[], std::string &url, std::string &token) { + if (argc > 1) { + const std::string first = argv[1]; + if (first == "-h" || first == "--help") { + return false; + } + } + + url = getenvOrEmpty("LIVEKIT_URL"); + token = getenvOrEmpty("LIVEKIT_TOKEN"); + + if (argc >= 3) { + url = argv[1]; + token = argv[2]; + } + + return !(url.empty() || token.empty()); +} + +void runVideoPublisher( + const std::shared_ptr &video_source, + const std::chrono::steady_clock::time_point &start_time) { + + uint64_t frame_count = 0; + VideoFrame frame = + VideoFrame::create(kFrameWidth, kFrameHeight, VideoBufferType::BGRA); + auto next_frame_at = std::chrono::steady_clock::now(); + + while (g_running.load(std::memory_order_relaxed)) { + const auto now = std::chrono::steady_clock::now(); + const double elapsed_seconds = + std::chrono::duration(now - start_time).count(); + + fillFrame(frame, elapsed_seconds); + + VideoCaptureOptions capture_options; + capture_options.timestamp_us = static_cast( + std::chrono::duration_cast(now - start_time) + .count()); + capture_options.rotation = VideoRotation::VIDEO_ROTATION_0; + capture_options.metadata = VideoFrameMetadata{}; + capture_options.metadata->user_timestamp_us = nowEpochUs(); + + video_source->captureFrame(frame, capture_options); + if (frame_count % 300 == 0) { + LK_LOG_INFO("[producer] captured frame {} timestamp={} user_timestamp={}", + frame_count, capture_options.timestamp_us, + capture_options.metadata->user_timestamp_us.value_or(0)); + } + ++frame_count; + next_frame_at += std::chrono::milliseconds(kVideoIntervalMs); + std::this_thread::sleep_until(next_frame_at); + } +} + +void runImuPublisher(const std::shared_ptr &data_track, + const std::chrono::steady_clock::time_point &start_time) { + uint64_t frame_count = 0; + auto next_frame_at = std::chrono::steady_clock::now(); + + while (g_running.load(std::memory_order_relaxed)) { + const auto now = std::chrono::steady_clock::now(); + const double elapsed_seconds = + std::chrono::duration(now - start_time).count(); + + const std::string payload = buildImuJson(elapsed_seconds); + auto push_result = data_track->tryPush( + std::vector(payload.begin(), payload.end()), + nowEpochUs()); + if (!push_result) { + const auto &error = push_result.error(); + LK_LOG_ERROR("[producer] failed to push data frame: {}", error.message); + } + + if (frame_count % 1000 == 0) { + LK_LOG_INFO("[producer] pushed IMU frame {} timestamp={}", frame_count, + nowEpochUs()); + } + ++frame_count; + next_frame_at += std::chrono::milliseconds(kDataIntervalMs); + std::this_thread::sleep_until(next_frame_at); + } +} + +} // namespace + +int main(int argc, char *argv[]) { + std::string url; + std::string token; + + if (!parseArgs(argc, argv, url, token)) { + printUsage(argv[0]); + return 1; + } + + std::signal(SIGINT, handleSignal); +#ifdef SIGTERM + std::signal(SIGTERM, handleSignal); +#endif + + livekit::initialize(livekit::LogLevel::Info, livekit::LogSink::kConsole); + + std::unique_ptr room = std::make_unique(); + RoomOptions options; + options.auto_subscribe = true; + options.dynacast = false; + + LK_LOG_INFO("[producer] connecting to {}", url); + if (!room->Connect(url, token, options)) { + LK_LOG_ERROR("[producer] failed to connect"); + livekit::shutdown(); + return 1; + } + + auto *participant = room->localParticipant(); + assert(participant != nullptr); + LK_LOG_INFO("[producer] connected as {} to room '{}'", + participant->identity(), room->room_info().name); + + auto video_source = std::make_shared(kFrameWidth, kFrameHeight); + auto video_track = + LocalVideoTrack::createLocalVideoTrack(kVideoTrackName, video_source); + + auto data_track_result = participant->publishDataTrack(kDataTrackName); + if (!data_track_result) { + const auto &error = data_track_result.error(); + LK_LOG_ERROR("[producer] failed to publish data track: {}", error.message); + return 1; + } + auto data_track = data_track_result.value(); + + try { + TrackPublishOptions publish_options; + publish_options.source = TrackSource::SOURCE_CAMERA; + publish_options.packet_trailer_features.user_timestamp = true; + participant->publishTrack(video_track, publish_options); + + LK_LOG_INFO("[producer] publishing stamped video track '{}' and data track " + "'{}'", + kVideoTrackName, kDataTrackName); + + const auto steady_start = std::chrono::steady_clock::now(); + std::thread video_thread(runVideoPublisher, video_source, steady_start); + std::thread imu_thread(runImuPublisher, data_track, steady_start); + + video_thread.join(); + imu_thread.join(); + } catch (const std::exception &error) { + LK_LOG_ERROR("[producer] error: {}", error.what()); + return 1; + } + + room.reset(); // reset the room to cleanup pubs/subs/rpc/etc before shutdown + livekit::shutdown(); + return 0; +} diff --git a/examples/hello_livekit/receiver.cpp b/examples/hello_livekit/receiver.cpp new file mode 100644 index 00000000..bc05e5f2 --- /dev/null +++ b/examples/hello_livekit/receiver.cpp @@ -0,0 +1,130 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// Subscribes to the sender's camera video and data track. Run +/// HelloLivekitSender first; use the identity it prints, or the sender's known +/// participant name. +/// +/// Usage: +/// HelloLivekitReceiver +/// +/// Or via environment variables: +/// LIVEKIT_URL, LIVEKIT_RECEIVER_TOKEN, LIVEKIT_SENDER_IDENTITY + +#include "livekit/livekit.h" + +#include +#include +#include +#include +#include + +using namespace livekit; + +constexpr const char *kDataTrackName = "app-data"; +constexpr const char *kVideoTrackName = "camera0"; + +std::atomic g_running{true}; + +void handleSignal(int) { g_running.store(false); } + +std::string getenvOrEmpty(const char *name) { + const char *v = std::getenv(name); + return v ? std::string(v) : std::string{}; +} + +int main(int argc, char *argv[]) { + std::string url = getenvOrEmpty("LIVEKIT_URL"); + std::string receiver_token = getenvOrEmpty("LIVEKIT_RECEIVER_TOKEN"); + std::string sender_identity = getenvOrEmpty("LIVEKIT_SENDER_IDENTITY"); + + if (argc >= 4) { + url = argv[1]; + receiver_token = argv[2]; + sender_identity = argv[3]; + } + + if (url.empty() || receiver_token.empty() || sender_identity.empty()) { + LK_LOG_ERROR("Usage: HelloLivekitReceiver " + "\n" + " or set LIVEKIT_URL, LIVEKIT_RECEIVER_TOKEN, " + "LIVEKIT_SENDER_IDENTITY"); + return 1; + } + + std::signal(SIGINT, handleSignal); +#ifdef SIGTERM + std::signal(SIGTERM, handleSignal); +#endif + + livekit::initialize(livekit::LogLevel::Info, livekit::LogSink::kConsole); + + auto room = std::make_unique(); + RoomOptions options; + options.auto_subscribe = true; + options.dynacast = false; + + if (!room->Connect(url, receiver_token, options)) { + LK_LOG_ERROR("[receiver] Failed to connect"); + livekit::shutdown(); + return 1; + } + + LocalParticipant *lp = room->localParticipant(); + assert(lp); + + LK_LOG_INFO("[receiver] Connected as identity='{}' room='{}'; subscribing " + "to sender identity='{}'", + lp->identity(), room->room_info().name, sender_identity); + + int video_frame_count = 0; + room->setOnVideoFrameCallback( + sender_identity, kVideoTrackName, + [&video_frame_count](const VideoFrame &frame, std::int64_t timestamp_us) { + const auto ts_ms = + std::chrono::duration(timestamp_us).count(); + const int n = video_frame_count++; + if (n % 10 == 0) { + LK_LOG_INFO("[receiver] Video frame #{} {}x{} ts_ms={}", n, + frame.width(), frame.height(), ts_ms); + } + }); + + int data_frame_count = 0; + room->addOnDataFrameCallback( + sender_identity, kDataTrackName, + [&data_frame_count](const std::vector &payload, + std::optional user_ts) { + const int n = data_frame_count++; + if (n % 10 == 0) { + LK_LOG_INFO("[receiver] Data frame #{}", n); + } + }); + + LK_LOG_INFO("[receiver] Listening for video track '{}' + data track '{}'; " + "Ctrl-C to exit", + kVideoTrackName, kDataTrackName); + + while (g_running.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + + LK_LOG_INFO("[receiver] Shutting down"); + room.reset(); + + livekit::shutdown(); + return 0; +} diff --git a/examples/hello_livekit/sender.cpp b/examples/hello_livekit/sender.cpp new file mode 100644 index 00000000..253091fd --- /dev/null +++ b/examples/hello_livekit/sender.cpp @@ -0,0 +1,142 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// Publishes synthetic RGBA video and a data track. Run the receiver in another +/// process and pass this participant's identity (printed after connect). +/// +/// Usage: +/// HelloLivekitSender +/// +/// Or via environment variables: +/// LIVEKIT_URL, LIVEKIT_SENDER_TOKEN + +#include "livekit/livekit.h" + +#include +#include +#include +#include +#include +#include +#include + +using namespace livekit; + +constexpr int kWidth = 640; +constexpr int kHeight = 480; +constexpr const char *kVideoTrackName = "camera0"; +constexpr const char *kDataTrackName = "app-data"; + +std::atomic g_running{true}; + +void handleSignal(int) { g_running.store(false); } + +std::string getenvOrEmpty(const char *name) { + const char *v = std::getenv(name); + return v ? std::string(v) : std::string{}; +} + +int main(int argc, char *argv[]) { + std::string url = getenvOrEmpty("LIVEKIT_URL"); + std::string sender_token = getenvOrEmpty("LIVEKIT_SENDER_TOKEN"); + + if (argc >= 3) { + url = argv[1]; + sender_token = argv[2]; + } + + if (url.empty() || sender_token.empty()) { + LK_LOG_ERROR("Usage: HelloLivekitSender \n" + " or set LIVEKIT_URL, LIVEKIT_SENDER_TOKEN"); + return 1; + } + + std::signal(SIGINT, handleSignal); +#ifdef SIGTERM + std::signal(SIGTERM, handleSignal); +#endif + + livekit::initialize(livekit::LogLevel::Info, livekit::LogSink::kConsole); + + auto room = std::make_unique(); + RoomOptions options; + options.auto_subscribe = true; + options.dynacast = false; + + if (!room->Connect(url, sender_token, options)) { + LK_LOG_ERROR("[sender] Failed to connect"); + livekit::shutdown(); + return 1; + } + + LocalParticipant *lp = room->localParticipant(); + assert(lp); + + LK_LOG_INFO("[sender] Connected as identity='{}' room='{}' — pass this " + "identity to HelloLivekitReceiver", + lp->identity(), room->room_info().name); + + auto video_source = std::make_shared(kWidth, kHeight); + + std::shared_ptr video_track = lp->publishVideoTrack( + kVideoTrackName, video_source, TrackSource::SOURCE_CAMERA); + + auto publish_result = lp->publishDataTrack(kDataTrackName); + if (!publish_result) { + const auto &error = publish_result.error(); + LK_LOG_ERROR("Failed to publish data track: code={} message={}", + static_cast(error.code), error.message); + room.reset(); + livekit::shutdown(); + return 1; + } + std::shared_ptr data_track = publish_result.value(); + + const auto t0 = std::chrono::steady_clock::now(); + std::uint64_t count = 0; + + LK_LOG_INFO( + "[sender] Publishing synthetic video + data on '{}'; Ctrl-C to exit", + kDataTrackName); + + while (g_running.load()) { + VideoFrame vf = VideoFrame::create(kWidth, kHeight, VideoBufferType::RGBA); + video_source->captureFrame(std::move(vf)); + + const auto now = std::chrono::steady_clock::now(); + const double ms = + std::chrono::duration(now - t0).count(); + std::ostringstream oss; + oss << std::fixed << std::setprecision(2) << ms << " ms, count: " << count; + const std::string msg = oss.str(); + auto push_result = + data_track->tryPush(std::vector(msg.begin(), msg.end())); + if (!push_result) { + const auto &error = push_result.error(); + LK_LOG_WARN("Failed to push data frame: code={} message={}", + static_cast(error.code), error.message); + } + + ++count; + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + LK_LOG_INFO("[sender] Disconnecting"); + room.reset(); + + livekit::shutdown(); + return 0; +} diff --git a/examples/ping_pong/constants.h b/examples/ping_pong/constants.h new file mode 100644 index 00000000..da3c9b53 --- /dev/null +++ b/examples/ping_pong/constants.h @@ -0,0 +1,36 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace ping_pong { + +inline constexpr char kPingParticipantIdentity[] = "ping"; +inline constexpr char kPongParticipantIdentity[] = "pong"; + +inline constexpr char kPingTrackName[] = "ping"; +inline constexpr char kPongTrackName[] = "pong"; + +inline constexpr char kPingIdKey[] = "id"; +inline constexpr char kReceivedIdKey[] = "rec_id"; +inline constexpr char kTimestampKey[] = "ts"; + +inline constexpr auto kPingPeriod = std::chrono::seconds(1); +inline constexpr auto kPollPeriod = std::chrono::milliseconds(50); + +} // namespace ping_pong diff --git a/examples/ping_pong/json_converters.cpp b/examples/ping_pong/json_converters.cpp new file mode 100644 index 00000000..24f89b14 --- /dev/null +++ b/examples/ping_pong/json_converters.cpp @@ -0,0 +1,69 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "json_converters.h" + +#include "constants.h" + +#include + +#include + +namespace ping_pong { + +std::string pingMessageToJson(const PingMessage &message) { + nlohmann::json json; + json[kPingIdKey] = message.id; + json[kTimestampKey] = message.ts_ns; + return json.dump(); +} + +PingMessage pingMessageFromJson(const std::string &json_text) { + try { + const auto json = nlohmann::json::parse(json_text); + + PingMessage message; + message.id = json.at(kPingIdKey).get(); + message.ts_ns = json.at(kTimestampKey).get(); + return message; + } catch (const nlohmann::json::exception &e) { + throw std::runtime_error(std::string("Failed to parse ping JSON: ") + + e.what()); + } +} + +std::string pongMessageToJson(const PongMessage &message) { + nlohmann::json json; + json[kReceivedIdKey] = message.rec_id; + json[kTimestampKey] = message.ts_ns; + return json.dump(); +} + +PongMessage pongMessageFromJson(const std::string &json_text) { + try { + const auto json = nlohmann::json::parse(json_text); + + PongMessage message; + message.rec_id = json.at(kReceivedIdKey).get(); + message.ts_ns = json.at(kTimestampKey).get(); + return message; + } catch (const nlohmann::json::exception &e) { + throw std::runtime_error(std::string("Failed to parse pong JSON: ") + + e.what()); + } +} + +} // namespace ping_pong diff --git a/examples/ping_pong/json_converters.h b/examples/ping_pong/json_converters.h new file mode 100644 index 00000000..3491ef6c --- /dev/null +++ b/examples/ping_pong/json_converters.h @@ -0,0 +1,31 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "messages.h" + +#include + +namespace ping_pong { + +std::string pingMessageToJson(const PingMessage &message); +PingMessage pingMessageFromJson(const std::string &json); + +std::string pongMessageToJson(const PongMessage &message); +PongMessage pongMessageFromJson(const std::string &json); + +} // namespace ping_pong diff --git a/examples/ping_pong/messages.h b/examples/ping_pong/messages.h new file mode 100644 index 00000000..d4212ed6 --- /dev/null +++ b/examples/ping_pong/messages.h @@ -0,0 +1,48 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace ping_pong { + +struct PingMessage { + std::uint64_t id = 0; + std::int64_t ts_ns = 0; +}; + +struct PongMessage { + std::uint64_t rec_id = 0; + std::int64_t ts_ns = 0; +}; + +struct LatencyMetrics { + std::uint64_t id = 0; + std::int64_t ping_sent_ts_ns = 0; + std::int64_t pong_sent_ts_ns = 0; + std::int64_t ping_received_ts_ns = 0; + std::int64_t round_trip_time_ns = 0; + std::int64_t pong_to_ping_time_ns = 0; + std::int64_t ping_to_pong_and_processing_ns = 0; + double estimated_one_way_latency_ns = 0.0; + double round_trip_time_ms = 0.0; + double pong_to_ping_time_ms = 0.0; + double ping_to_pong_and_processing_ms = 0.0; + double estimated_one_way_latency_ms = 0.0; +}; + +} // namespace ping_pong diff --git a/examples/ping_pong/ping.cpp b/examples/ping_pong/ping.cpp new file mode 100644 index 00000000..c46f941c --- /dev/null +++ b/examples/ping_pong/ping.cpp @@ -0,0 +1,209 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// Ping participant: publishes on the "ping" data track, listens on "pong", +/// and logs latency metrics for each matched response. Use a token whose +/// identity is `ping`. + +#include "constants.h" +#include "json_converters.h" +#include "livekit/livekit.h" +#include "livekit/lk_log.h" +#include "messages.h" +#include "utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace livekit; + +namespace { + +std::atomic g_running{true}; + +void handleSignal(int) { g_running.store(false); } + +ping_pong::LatencyMetrics +calculateLatencyMetrics(const ping_pong::PingMessage &ping_message, + const ping_pong::PongMessage &pong_message, + std::int64_t received_ts_ns) { + ping_pong::LatencyMetrics metrics; + metrics.id = ping_message.id; + metrics.pong_sent_ts_ns = pong_message.ts_ns; + metrics.ping_received_ts_ns = received_ts_ns; + metrics.round_trip_time_ns = received_ts_ns - ping_message.ts_ns; + metrics.pong_to_ping_time_ns = received_ts_ns - pong_message.ts_ns; + metrics.ping_to_pong_and_processing_ns = + pong_message.ts_ns - ping_message.ts_ns; + metrics.estimated_one_way_latency_ns = + static_cast(metrics.round_trip_time_ns) / 2.0; + metrics.round_trip_time_ms = + static_cast(metrics.round_trip_time_ns) / 1'000'000.0; + metrics.pong_to_ping_time_ms = + static_cast(metrics.pong_to_ping_time_ns) / 1'000'000.0; + metrics.ping_to_pong_and_processing_ms = + static_cast(metrics.ping_to_pong_and_processing_ns) / 1'000'000.0; + metrics.estimated_one_way_latency_ms = + metrics.estimated_one_way_latency_ns / 1'000'000.0; + return metrics; +} + +} // namespace + +int main(int argc, char *argv[]) { + std::string url = ping_pong::getenvOrEmpty("LIVEKIT_URL"); + std::string token = ping_pong::getenvOrEmpty("LIVEKIT_TOKEN"); + + if (argc >= 3) { + url = argv[1]; + token = argv[2]; + } + + if (url.empty() || token.empty()) { + LK_LOG_ERROR("LIVEKIT_URL and LIVEKIT_TOKEN (or ) are " + "required"); + return 1; + } + + std::signal(SIGINT, handleSignal); +#ifdef SIGTERM + std::signal(SIGTERM, handleSignal); +#endif + + livekit::initialize(livekit::LogLevel::Info, livekit::LogSink::kConsole); + + auto room = std::make_unique(); + RoomOptions options; + options.auto_subscribe = true; + options.dynacast = false; + + if (!room->Connect(url, token, options)) { + LK_LOG_ERROR("Failed to connect to room"); + livekit::shutdown(); + return 1; + } + + LocalParticipant *local_participant = room->localParticipant(); + assert(local_participant); + + LK_LOG_INFO("ping connected as identity='{}' room='{}'", + local_participant->identity(), room->room_info().name); + + auto publish_result = + local_participant->publishDataTrack(ping_pong::kPingTrackName); + if (!publish_result) { + const auto &error = publish_result.error(); + LK_LOG_ERROR("Failed to publish ping data track: code={} message={}", + static_cast(error.code), error.message); + room->setDelegate(nullptr); + room.reset(); + livekit::shutdown(); + return 1; + } + + std::shared_ptr ping_track = publish_result.value(); + std::unordered_map sent_messages; + std::mutex sent_messages_mutex; + + const auto callback_id = room->addOnDataFrameCallback( + ping_pong::kPongParticipantIdentity, ping_pong::kPongTrackName, + [&sent_messages, + &sent_messages_mutex](const std::vector &payload, + std::optional /*user_timestamp*/) { + try { + if (payload.empty()) { + LK_LOG_DEBUG("Ignoring empty pong payload"); + return; + } + + const auto pong_message = + ping_pong::pongMessageFromJson(ping_pong::toString(payload)); + const auto received_ts_ns = ping_pong::timeSinceEpochNs(); + + ping_pong::PingMessage ping_message; + { + std::lock_guard lock(sent_messages_mutex); + const auto it = sent_messages.find(pong_message.rec_id); + if (it == sent_messages.end()) { + LK_LOG_WARN("Received pong for unknown id={}", + pong_message.rec_id); + return; + } + ping_message = it->second; + sent_messages.erase(it); + } + + const auto metrics = calculateLatencyMetrics( + ping_message, pong_message, received_ts_ns); + + LK_LOG_INFO("pong id={} rtt_ms={:.3f} " + "pong_to_ping_ms={:.3f} " + "ping_to_pong_and_processing_ms={:.3f} " + "estimated_one_way_latency_ms={:.3f}", + metrics.id, metrics.round_trip_time_ms, + metrics.pong_to_ping_time_ms, + metrics.ping_to_pong_and_processing_ms, + metrics.estimated_one_way_latency_ms); + } catch (const std::exception &e) { + LK_LOG_WARN("Failed to process pong payload: {}", e.what()); + } + }); + + LK_LOG_INFO("published data track '{}' and listening for '{}' from '{}'", + ping_pong::kPingTrackName, ping_pong::kPongTrackName, + ping_pong::kPongParticipantIdentity); + + std::uint64_t next_id = 1; + auto next_deadline = std::chrono::steady_clock::now(); + + while (g_running.load()) { + ping_pong::PingMessage ping_message; + ping_message.id = next_id++; + ping_message.ts_ns = ping_pong::timeSinceEpochNs(); + + const std::string json = ping_pong::pingMessageToJson(ping_message); + auto push_result = ping_track->tryPush(ping_pong::toPayload(json)); + if (!push_result) { + const auto &error = push_result.error(); + LK_LOG_WARN("Failed to push ping data frame: code={} message={}", + static_cast(error.code), error.message); + } else { + { + std::lock_guard lock(sent_messages_mutex); + sent_messages.emplace(ping_message.id, ping_message); + } + LK_LOG_INFO("sent ping id={} ts_ns={}", ping_message.id, + ping_message.ts_ns); + } + + next_deadline += ping_pong::kPingPeriod; + std::this_thread::sleep_until(next_deadline); + } + + LK_LOG_INFO("shutting down ping participant"); + room.reset(); + livekit::shutdown(); + return 0; +} diff --git a/examples/ping_pong/pong.cpp b/examples/ping_pong/pong.cpp new file mode 100644 index 00000000..34bdbd54 --- /dev/null +++ b/examples/ping_pong/pong.cpp @@ -0,0 +1,147 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// Pong participant: listens on the "ping" data track and publishes responses +/// on the "pong" data track. Use a token whose identity is `pong`. + +#include "constants.h" +#include "json_converters.h" +#include "livekit/livekit.h" +#include "livekit/lk_log.h" +#include "messages.h" +#include "utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace livekit; + +namespace { + +std::atomic g_running{true}; + +void handleSignal(int) { g_running.store(false); } + +} // namespace + +int main(int argc, char *argv[]) { + std::string url = ping_pong::getenvOrEmpty("LIVEKIT_URL"); + std::string token = ping_pong::getenvOrEmpty("LIVEKIT_TOKEN"); + + if (argc >= 3) { + url = argv[1]; + token = argv[2]; + } + + if (url.empty() || token.empty()) { + LK_LOG_ERROR("LIVEKIT_URL and LIVEKIT_TOKEN (or ) are " + "required"); + return 1; + } + + std::signal(SIGINT, handleSignal); +#ifdef SIGTERM + std::signal(SIGTERM, handleSignal); +#endif + + livekit::initialize(livekit::LogLevel::Info, livekit::LogSink::kConsole); + + auto room = std::make_unique(); + RoomOptions options; + options.auto_subscribe = true; + options.dynacast = false; + + if (!room->Connect(url, token, options)) { + LK_LOG_ERROR("Failed to connect to room"); + livekit::shutdown(); + return 1; + } + + LocalParticipant *local_participant = room->localParticipant(); + assert(local_participant); + + LK_LOG_INFO("pong connected as identity='{}' room='{}'", + local_participant->identity(), room->room_info().name); + + auto publish_result = + local_participant->publishDataTrack(ping_pong::kPongTrackName); + if (!publish_result) { + const auto &error = publish_result.error(); + LK_LOG_ERROR("Failed to publish pong data track: code={} message={}", + static_cast(error.code), error.message); + room->setDelegate(nullptr); + room.reset(); + livekit::shutdown(); + return 1; + } + + std::shared_ptr pong_track = publish_result.value(); + + const auto callback_id = room->addOnDataFrameCallback( + ping_pong::kPingParticipantIdentity, ping_pong::kPingTrackName, + [pong_track](const std::vector &payload, + std::optional /*user_timestamp*/) { + try { + if (payload.empty()) { + LK_LOG_DEBUG("Ignoring empty ping payload"); + return; + } + + const auto ping_message = + ping_pong::pingMessageFromJson(ping_pong::toString(payload)); + + ping_pong::PongMessage pong_message; + pong_message.rec_id = ping_message.id; + pong_message.ts_ns = ping_pong::timeSinceEpochNs(); + + const std::string json = ping_pong::pongMessageToJson(pong_message); + auto push_result = pong_track->tryPush(ping_pong::toPayload(json)); + if (!push_result) { + const auto &error = push_result.error(); + LK_LOG_WARN("Failed to push pong data frame: code={} message={}", + static_cast(error.code), error.message); + return; + } + + LK_LOG_INFO("received ping id={} ts_ns={} and sent pong rec_id={} " + "ts_ns={}", + ping_message.id, ping_message.ts_ns, pong_message.rec_id, + pong_message.ts_ns); + } catch (const std::exception &e) { + LK_LOG_WARN("Failed to process ping payload: {}", e.what()); + } + }); + + LK_LOG_INFO("published data track '{}' and listening for '{}' from '{}'", + ping_pong::kPongTrackName, ping_pong::kPingTrackName, + ping_pong::kPingParticipantIdentity); + + while (g_running.load()) { + std::this_thread::sleep_for(ping_pong::kPollPeriod); + } + + LK_LOG_INFO("shutting down pong participant"); + room.reset(); + livekit::shutdown(); + return 0; +} diff --git a/examples/ping_pong/utils.h b/examples/ping_pong/utils.h new file mode 100644 index 00000000..56c915b9 --- /dev/null +++ b/examples/ping_pong/utils.h @@ -0,0 +1,45 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +namespace ping_pong { + +inline std::string getenvOrEmpty(const char *name) { + const char *value = std::getenv(name); + return value ? std::string(value) : std::string{}; +} + +inline std::int64_t timeSinceEpochNs() { + const auto now = std::chrono::system_clock::now().time_since_epoch(); + return std::chrono::duration_cast(now).count(); +} + +inline std::vector toPayload(const std::string &json) { + return std::vector(json.begin(), json.end()); +} + +inline std::string toString(const std::vector &payload) { + return std::string(payload.begin(), payload.end()); +} + +} // namespace ping_pong diff --git a/examples/simple_joystick/receiver.cpp b/examples/simple_joystick/receiver.cpp index aa9814f3..d62785a4 100644 --- a/examples/simple_joystick/receiver.cpp +++ b/examples/simple_joystick/receiver.cpp @@ -108,8 +108,9 @@ int main(int argc, char *argv[]) { if (!g_running.load()) { std::cout << "[Receiver] Interrupted by signal. Shutting down.\n"; } else if (!g_sender_connected.load()) { - std::cerr << "[Receiver] Timed out after 2 minutes with no sender connection. " - << "Exiting as failure.\n"; + std::cerr + << "[Receiver] Timed out after 2 minutes with no sender connection. " + << "Exiting as failure.\n"; room->setDelegate(nullptr); room.reset(); livekit::shutdown(); diff --git a/examples/simple_joystick/sender.cpp b/examples/simple_joystick/sender.cpp index 9af5b4ee..a235c3da 100644 --- a/examples/simple_joystick/sender.cpp +++ b/examples/simple_joystick/sender.cpp @@ -165,10 +165,12 @@ int main(int argc, char *argv[]) { bool receiver_present = (room->remoteParticipant("robot") != nullptr); if (receiver_present && !receiver_connected) { - std::cout << "[Sender] Receiver connected! Use keys to send commands.\n"; + std::cout + << "[Sender] Receiver connected! Use keys to send commands.\n"; receiver_connected = true; } else if (!receiver_present && receiver_connected) { - std::cout << "[Sender] Receiver disconnected. Waiting for reconnect...\n"; + std::cout + << "[Sender] Receiver disconnected. Waiting for reconnect...\n"; receiver_connected = false; } } @@ -236,7 +238,8 @@ int main(int argc, char *argv[]) { simple_joystick::JoystickCommand cmd{x, y, z}; std::string payload = simple_joystick::joystick_to_json(cmd); - std::cout << "[Sender] Sending: x=" << x << " y=" << y << " z=" << z << "\n"; + std::cout << "[Sender] Sending: x=" << x << " y=" << y << " z=" << z + << "\n"; try { std::string response = @@ -246,7 +249,8 @@ int main(int argc, char *argv[]) { std::cerr << "[Sender] RPC error: " << e.message() << "\n"; if (static_cast(e.code()) == RpcError::ErrorCode::RECIPIENT_DISCONNECTED) { - std::cout << "[Sender] Receiver disconnected. Waiting for reconnect...\n"; + std::cout + << "[Sender] Receiver disconnected. Waiting for reconnect...\n"; receiver_connected = false; } } catch (const std::exception &e) { diff --git a/examples/tokens/README.md b/examples/tokens/README.md new file mode 100644 index 00000000..ebed99c1 --- /dev/null +++ b/examples/tokens/README.md @@ -0,0 +1,8 @@ +# Overview +Examples of generating tokens + +## gen_and_set.bash +Generate tokens and then set them as env vars for the current terminal session + +## set_data_track_test_tokens.bash +Generate tokens for data track integration tests and set them as env vars for the current terminal session. \ No newline at end of file diff --git a/examples/tokens/gen_and_set.bash b/examples/tokens/gen_and_set.bash new file mode 100755 index 00000000..b933a24f --- /dev/null +++ b/examples/tokens/gen_and_set.bash @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# Copyright 2026 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generate a LiveKit access token via `lk` and set LIVEKIT_TOKEN (and LIVEKIT_URL) +# for your current shell session. +# +# source examples/tokens/gen_and_set.bash --id PARTICIPANT_ID --room ROOM_NAME [--view-token] +# eval "$(bash examples/tokens/gen_and_set.bash --id ID --room ROOM [--view-token])" +# +# Optional env: LIVEKIT_API_KEY, LIVEKIT_API_SECRET, LIVEKIT_VALID_FOR. + +# When sourced, we must NOT enable errexit/pipefail on the interactive shell — a +# failing pipeline (e.g. sed|head SIGPIPE) or any error would close your terminal. + +_sourced=0 +if [[ -n "${BASH_VERSION:-}" ]] && [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then + _sourced=1 +elif [[ -n "${ZSH_VERSION:-}" ]] && [[ "${ZSH_EVAL_CONTEXT:-}" == *:file* ]]; then + _sourced=1 +fi + +_fail() { + echo "gen_and_set.bash: $1" >&2 + if [[ "$_sourced" -eq 1 ]]; then + return "${2:-1}" + fi + exit "${2:-1}" +} + +_usage() { + echo "Usage: ${0##*/} --id PARTICIPANT_IDENTITY --room ROOM_NAME [--view-token]" >&2 + echo " --id LiveKit participant identity (required)" >&2 + echo " --room Room name (required; not read from env)" >&2 + echo " --view-token Print the JWT to stderr after generating" >&2 +} + +if [[ "$_sourced" -eq 0 ]]; then + set -euo pipefail +fi + +_view_token=0 +LIVEKIT_IDENTITY="" +LIVEKIT_ROOM="robo_room" +while [[ $# -gt 0 ]]; do + case "$1" in + --view-token) + _view_token=1 + shift + ;; + --id) + if [[ $# -lt 2 ]]; then + _usage + _fail "--id requires a value" 2 + fi + LIVEKIT_IDENTITY="$2" + shift 2 + ;; + --room) + if [[ $# -lt 2 ]]; then + _usage + _fail "--room requires a value" 2 + fi + LIVEKIT_ROOM="$2" + shift 2 + ;; + -h | --help) + _usage + if [[ "$_sourced" -eq 1 ]]; then + return 0 + fi + exit 0 + ;; + *) + _usage + _fail "unknown argument: $1" 2 + ;; + esac +done + +if [[ -z "$LIVEKIT_IDENTITY" ]]; then + _usage + _fail "--id is required" 2 +fi +if [[ -z "$LIVEKIT_ROOM" ]]; then + _usage + _fail "--room is required" 2 +fi + +LIVEKIT_API_KEY="${LIVEKIT_API_KEY:-devkey}" +LIVEKIT_API_SECRET="${LIVEKIT_API_SECRET:-secret}" +LIVEKIT_VALID_FOR="${LIVEKIT_VALID_FOR:-99999h}" +_grant_json='{"canPublish":true,"canSubscribe":true,"canPublishData":true}' + +if ! command -v lk >/dev/null 2>&1; then + _fail "'lk' CLI not found. Install: https://docs.livekit.io/home/cli/" 2 +fi + +# Run lk inside bash so --grant JSON (with embedded ") is safe when this file is +# sourced from zsh; zsh misparses --grant "$json" on the same line. +_out="$( + bash -c ' + lk token create \ + --api-key "$1" \ + --api-secret "$2" \ + -i "$3" \ + --join \ + --valid-for "$4" \ + --room "$5" \ + --grant "$6" 2>&1 + ' _ "$LIVEKIT_API_KEY" "$LIVEKIT_API_SECRET" "$LIVEKIT_IDENTITY" \ + "$LIVEKIT_VALID_FOR" "$LIVEKIT_ROOM" "$_grant_json" +)" +_lk_st=$? +if [[ "$_lk_st" -ne 0 ]]; then + echo "$_out" >&2 + _fail "lk token create failed" 1 +fi + +# Avoid sed|head pipelines (pipefail + SIGPIPE can kill a sourced shell). +LIVEKIT_TOKEN="" +LIVEKIT_URL="" +while IFS= read -r _line || [[ -n "${_line}" ]]; do + if [[ "$_line" == "Access token: "* ]]; then + LIVEKIT_TOKEN="${_line#Access token: }" + elif [[ "$_line" == "Project URL: "* ]]; then + LIVEKIT_URL="${_line#Project URL: }" + fi +done <<< "$_out" + +if [[ -z "$LIVEKIT_TOKEN" ]]; then + echo "gen_and_set.bash: could not parse Access token from lk output:" >&2 + echo "$_out" >&2 + _fail "missing Access token line" 1 +fi + +if [[ "$_view_token" -eq 1 ]]; then + echo "$LIVEKIT_TOKEN" >&2 +fi + +_apply() { + export LIVEKIT_TOKEN + export LIVEKIT_URL +} + +_emit_eval() { + printf 'export LIVEKIT_TOKEN=%q\n' "$LIVEKIT_TOKEN" + [[ -n "$LIVEKIT_URL" ]] && printf 'export LIVEKIT_URL=%q\n' "$LIVEKIT_URL" +} + +if [[ "$_sourced" -eq 1 ]]; then + _apply + echo "LIVEKIT_TOKEN and LIVEKIT_URL set for this shell." >&2 + [[ -n "$LIVEKIT_URL" ]] || echo "gen_and_set.bash: warning: no Project URL in output; set LIVEKIT_URL manually." >&2 +else + _emit_eval + echo "gen_and_set.bash: for this shell run: source $0 --id ... --room ... or: eval \"\$(bash $0 ...)\"" >&2 +fi diff --git a/examples/tokens/set_data_track_test_tokens.bash b/examples/tokens/set_data_track_test_tokens.bash new file mode 100755 index 00000000..1cc8bb56 --- /dev/null +++ b/examples/tokens/set_data_track_test_tokens.bash @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# Copyright 2026 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generate two LiveKit access tokens via `lk` and set the environment variables +# required by src/tests/integration/test_data_track.cpp. +# +# source examples/tokens/set_data_track_test_tokens.bash +# eval "$(bash examples/tokens/set_data_track_test_tokens.bash)" +# +# Exports: +# LK_TOKEN_TEST_A +# LK_TOKEN_TEST_B +# LIVEKIT_URL=ws://localhost:7880 +# + +_sourced=0 +if [[ -n "${BASH_VERSION:-}" ]] && [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then + _sourced=1 +elif [[ -n "${ZSH_VERSION:-}" ]] && [[ "${ZSH_EVAL_CONTEXT:-}" == *:file* ]]; then + _sourced=1 +fi + +_fail() { + echo "set_data_track_test_tokens.bash: $1" >&2 + if [[ "$_sourced" -eq 1 ]]; then + return "${2:-1}" + fi + exit "${2:-1}" +} + +if [[ "$_sourced" -eq 0 ]]; then + set -euo pipefail +fi + +LIVEKIT_ROOM="cpp_data_track_test" +LIVEKIT_IDENTITY_A="cpp-test-a" +LIVEKIT_IDENTITY_B="cpp-test-b" + +if [[ $# -ne 0 ]]; then + _fail "this script is hard-coded and does not accept arguments" 2 +fi + +LIVEKIT_API_KEY="devkey" +LIVEKIT_API_SECRET="secret" +LIVEKIT_VALID_FOR="99999h" +LIVEKIT_URL="ws://localhost:7880" +_grant_json='{"canPublish":true,"canSubscribe":true,"canPublishData":true}' + +if ! command -v lk >/dev/null 2>&1; then + _fail "'lk' CLI not found. Install: https://docs.livekit.io/home/cli/" 2 +fi + +_create_token() { + local identity="$1" + local output="" + local command_status=0 + local token="" + + output="$( + bash -c ' + lk token create \ + --api-key "$1" \ + --api-secret "$2" \ + -i "$3" \ + --join \ + --valid-for "$4" \ + --room "$5" \ + --grant "$6" 2>&1 + ' _ "$LIVEKIT_API_KEY" "$LIVEKIT_API_SECRET" "$identity" \ + "$LIVEKIT_VALID_FOR" "$LIVEKIT_ROOM" "$_grant_json" + )" + command_status=$? + if [[ "$command_status" -ne 0 ]]; then + echo "$output" >&2 + _fail "lk token create failed for identity '$identity'" 1 + fi + + while IFS= read -r line || [[ -n "${line}" ]]; do + if [[ "$line" == "Access token: "* ]]; then + token="${line#Access token: }" + break + fi + done <<< "$output" + + if [[ -z "$token" ]]; then + echo "$output" >&2 + _fail "could not parse Access token for identity '$identity'" 1 + fi + + printf '%s' "$token" +} + +LK_TOKEN_TEST_A="$(_create_token "$LIVEKIT_IDENTITY_A")" +LK_TOKEN_TEST_B="$(_create_token "$LIVEKIT_IDENTITY_B")" + +_apply() { + export LK_TOKEN_TEST_A + export LK_TOKEN_TEST_B + export LIVEKIT_URL +} + +_emit_eval() { + printf 'export LK_TOKEN_TEST_A=%q\n' "$LK_TOKEN_TEST_A" + printf 'export LK_TOKEN_TEST_B=%q\n' "$LK_TOKEN_TEST_B" + printf 'export LIVEKIT_URL=%q\n' "$LIVEKIT_URL" +} + +if [[ "$_sourced" -eq 1 ]]; then + _apply + echo "LK_TOKEN_TEST_A, LK_TOKEN_TEST_B, and LIVEKIT_URL set for this shell." >&2 +else + _emit_eval + echo "set_data_track_test_tokens.bash: for this shell run: source $0 or: eval \"\$(bash $0 ...)\"" >&2 +fi diff --git a/examples/user_timestamped_video/README.md b/examples/user_timestamped_video/README.md new file mode 100644 index 00000000..ca9e8814 --- /dev/null +++ b/examples/user_timestamped_video/README.md @@ -0,0 +1,53 @@ +# UserTimestampedVideo + +This example is split into two executables and can demonstrate all four +producer/consumer combinations: + +- `UserTimestampedVideoProducer` publishes a synthetic camera track and stamps + each frame with `VideoCaptureOptions::metadata.user_timestamp_us`. +- `UserTimestampedVideoConsumer` subscribes to remote camera frames with + either the rich or legacy callback path. + +Run them in the same room with different participant identities: + +```sh +LIVEKIT_URL=ws://localhost:7880 LIVEKIT_TOKEN= ./UserTimestampedVideoProducer +LIVEKIT_URL=ws://localhost:7880 LIVEKIT_TOKEN= ./UserTimestampedVideoConsumer +``` + +Flags: + +- Producer default: sends user timestamps +- Producer `--without-user-timestamp`: does not send user timestamps +- Consumer default: reads user timestamps through `setOnVideoFrameEventCallback` +- Consumer `--ignore-user-timestamp`: ignores metadata through the legacy + `setOnVideoFrameCallback` + +Matrix: + +```sh +# 1. Producer sends, consumer reads +./UserTimestampedVideoProducer +./UserTimestampedVideoConsumer + +# 2. Producer sends, consumer ignores +./UserTimestampedVideoProducer +./UserTimestampedVideoConsumer --ignore-user-timestamp + +# 3. Producer does not send, consumer ignores +./UserTimestampedVideoProducer --without-user-timestamp +./UserTimestampedVideoConsumer --ignore-user-timestamp + +# 4. Producer does not send, consumer reads +./UserTimestampedVideoProducer --without-user-timestamp +./UserTimestampedVideoConsumer +``` + +Timestamp note: + +- `user_ts_us` is application metadata and is the value to compare end to end. +- `capture_ts_us` on the producer is the timestamp submitted to `captureFrame`. +- `capture_ts_us` on the consumer is the received WebRTC frame timestamp. +- Producer and consumer `capture_ts_us` values are not expected to match exactly, + because WebRTC may translate frame timestamps onto its own internal + capture-time timeline before delivery. diff --git a/examples/user_timestamped_video/consumer.cpp b/examples/user_timestamped_video/consumer.cpp new file mode 100644 index 00000000..191a1156 --- /dev/null +++ b/examples/user_timestamped_video/consumer.cpp @@ -0,0 +1,247 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * UserTimestampedVideoConsumer + * + * Receives remote camera frames via Room::setOnVideoFrameEventCallback() and + * logs any VideoFrameMetadata::user_timestamp_us values that arrive. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "livekit/livekit.h" + +using namespace livekit; + +namespace { + +std::atomic g_running{true}; + +void handleSignal(int) { g_running.store(false); } + +std::string getenvOrEmpty(const char *name) { + const char *value = std::getenv(name); + return value ? std::string(value) : std::string{}; +} + +std::string +formatUserTimestamp(const std::optional &metadata) { + if (!metadata || !metadata->user_timestamp_us.has_value()) { + return "n/a"; + } + + return std::to_string(*metadata->user_timestamp_us); +} + +void printUsage(const char *program) { + std::cerr << "Usage:\n" + << " " << program << " [--ignore-user-timestamp]\n" + << "or:\n" + << " LIVEKIT_URL=... LIVEKIT_TOKEN=... " << program + << " [--ignore-user-timestamp]\n"; +} + +bool parseArgs(int argc, char *argv[], std::string &url, std::string &token, + bool &read_user_timestamp) { + read_user_timestamp = true; + std::vector positional; + + for (int i = 1; i < argc; ++i) { + const std::string arg = argv[i]; + if (arg == "-h" || arg == "--help") { + return false; + } + if (arg == "--ignore-user-timestamp") { + read_user_timestamp = false; + continue; + } + if (arg == "--read-user-timestamp") { + read_user_timestamp = true; + continue; + } + + positional.push_back(arg); + } + + url = getenvOrEmpty("LIVEKIT_URL"); + token = getenvOrEmpty("LIVEKIT_TOKEN"); + + if (positional.size() >= 2) { + url = positional[0]; + token = positional[1]; + } + + return !(url.empty() || token.empty()); +} + +class UserTimestampedVideoConsumerDelegate : public RoomDelegate { +public: + UserTimestampedVideoConsumerDelegate(Room &room, bool read_user_timestamp) + : room_(room), read_user_timestamp_(read_user_timestamp) {} + + void registerExistingParticipants() { + for (const auto &participant : room_.remoteParticipants()) { + if (participant) { + registerRemoteCameraCallback(participant->identity()); + } + } + } + + void onParticipantConnected(Room &, + const ParticipantConnectedEvent &event) override { + if (!event.participant) { + return; + } + + std::cout << "[consumer] participant connected: " + << event.participant->identity() << "\n"; + registerRemoteCameraCallback(event.participant->identity()); + } + + void onParticipantDisconnected( + Room &, const ParticipantDisconnectedEvent &event) override { + if (!event.participant) { + return; + } + + const std::string identity = event.participant->identity(); + room_.clearOnVideoFrameCallback(identity, TrackSource::SOURCE_CAMERA); + + { + std::lock_guard lock(mutex_); + registered_identities_.erase(identity); + } + + std::cout << "[consumer] participant disconnected: " << identity << "\n"; + } + +private: + void registerRemoteCameraCallback(const std::string &identity) { + { + std::lock_guard lock(mutex_); + if (!registered_identities_.insert(identity).second) { + return; + } + } + + VideoStream::Options stream_options; + stream_options.format = VideoBufferType::RGBA; + + if (read_user_timestamp_) { + room_.setOnVideoFrameEventCallback( + identity, TrackSource::SOURCE_CAMERA, + [identity](const VideoFrameEvent &event) { + std::cout << "[consumer] from=" << identity + << " size=" << event.frame.width() << "x" + << event.frame.height() + << " capture_ts_us=" << event.timestamp_us + << " user_ts_us=" << formatUserTimestamp(event.metadata) + << " rotation=" << static_cast(event.rotation) + << "\n"; + }, + stream_options); + } else { + room_.setOnVideoFrameCallback( + identity, TrackSource::SOURCE_CAMERA, + [identity](const VideoFrame &frame, const std::int64_t timestamp_us) { + std::cout << "[consumer] from=" << identity + << " size=" << frame.width() << "x" << frame.height() + << " capture_ts_us=" << timestamp_us + << " user_ts_us=ignored\n"; + }, + stream_options); + } + + std::cout << "[consumer] listening for camera frames from " << identity + << " with user timestamp " + << (read_user_timestamp_ ? "enabled" : "ignored") << "\n"; + } + + Room &room_; + bool read_user_timestamp_; + std::mutex mutex_; + std::unordered_set registered_identities_; +}; + +} // namespace + +int main(int argc, char *argv[]) { + std::string url; + std::string token; + bool read_user_timestamp = true; + + if (!parseArgs(argc, argv, url, token, read_user_timestamp)) { + printUsage(argv[0]); + return 1; + } + + std::signal(SIGINT, handleSignal); +#ifdef SIGTERM + std::signal(SIGTERM, handleSignal); +#endif + + livekit::initialize(livekit::LogLevel::Info, livekit::LogSink::kConsole); + int exit_code = 0; + + { + Room room; + RoomOptions options; + options.auto_subscribe = true; + options.dynacast = false; + + UserTimestampedVideoConsumerDelegate delegate(room, read_user_timestamp); + room.setDelegate(&delegate); + + std::cout << "[consumer] connecting to " << url << "\n"; + if (!room.Connect(url, token, options)) { + std::cerr << "[consumer] failed to connect\n"; + exit_code = 1; + } else { + std::cout << "[consumer] connected as " + << room.localParticipant()->identity() << " to room '" + << room.room_info().name << "' with user timestamp " + << (read_user_timestamp ? "enabled" : "ignored") << "\n"; + + delegate.registerExistingParticipants(); + + while (g_running.load(std::memory_order_relaxed)) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + + for (const auto &participant : room.remoteParticipants()) { + if (participant) { + room.clearOnVideoFrameCallback(participant->identity(), + TrackSource::SOURCE_CAMERA); + } + } + } + + room.setDelegate(nullptr); + } + + livekit::shutdown(); + return exit_code; +} diff --git a/examples/user_timestamped_video/producer.cpp b/examples/user_timestamped_video/producer.cpp new file mode 100644 index 00000000..98473682 --- /dev/null +++ b/examples/user_timestamped_video/producer.cpp @@ -0,0 +1,219 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * UserTimestampedVideoProducer + * + * Publishes a synthetic camera track and stamps each frame with + * VideoCaptureOptions::metadata.user_timestamp_us. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "livekit/livekit.h" + +using namespace livekit; + +namespace { + +constexpr int kFrameWidth = 640; +constexpr int kFrameHeight = 360; +constexpr int kFrameIntervalMs = 200; + +std::atomic g_running{true}; + +void handleSignal(int) { g_running.store(false); } + +std::string getenvOrEmpty(const char *name) { + const char *value = std::getenv(name); + return value ? std::string(value) : std::string{}; +} + +std::uint64_t nowEpochUs() { + return static_cast( + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count()); +} + +void fillFrame(VideoFrame &frame, std::uint32_t frame_index) { + const std::uint8_t blue = static_cast((frame_index * 7) % 255); + const std::uint8_t green = + static_cast((frame_index * 13) % 255); + const std::uint8_t red = static_cast((frame_index * 29) % 255); + + std::uint8_t *data = frame.data(); + for (std::size_t i = 0; i < frame.dataSize(); i += 4) { + data[i + 0] = blue; + data[i + 1] = green; + data[i + 2] = red; + data[i + 3] = 255; + } +} + +void printUsage(const char *program) { + std::cerr << "Usage:\n" + << " " << program + << " [--without-user-timestamp]\n" + << "or:\n" + << " LIVEKIT_URL=... LIVEKIT_TOKEN=... " << program + << " [--without-user-timestamp]\n"; +} + +bool parseArgs(int argc, char *argv[], std::string &url, std::string &token, + bool &send_user_timestamp) { + send_user_timestamp = true; + std::vector positional; + + for (int i = 1; i < argc; ++i) { + const std::string arg = argv[i]; + if (arg == "-h" || arg == "--help") { + return false; + } + if (arg == "--without-user-timestamp") { + send_user_timestamp = false; + continue; + } + if (arg == "--with-user-timestamp") { + send_user_timestamp = true; + continue; + } + + positional.push_back(arg); + } + + url = getenvOrEmpty("LIVEKIT_URL"); + token = getenvOrEmpty("LIVEKIT_TOKEN"); + + if (positional.size() >= 2) { + url = positional[0]; + token = positional[1]; + } + + return !(url.empty() || token.empty()); +} + +} // namespace + +int main(int argc, char *argv[]) { + std::string url; + std::string token; + bool send_user_timestamp = true; + + if (!parseArgs(argc, argv, url, token, send_user_timestamp)) { + printUsage(argv[0]); + return 1; + } + + std::signal(SIGINT, handleSignal); +#ifdef SIGTERM + std::signal(SIGTERM, handleSignal); +#endif + + livekit::initialize(livekit::LogLevel::Info, livekit::LogSink::kConsole); + int exit_code = 0; + + { + Room room; + RoomOptions options; + options.auto_subscribe = true; + options.dynacast = false; + + std::cout << "[producer] connecting to " << url << "\n"; + if (!room.Connect(url, token, options)) { + std::cerr << "[producer] failed to connect\n"; + exit_code = 1; + } else { + std::cout << "[producer] connected as " + << room.localParticipant()->identity() << " to room '" + << room.room_info().name << "'\n"; + + auto source = std::make_shared(kFrameWidth, kFrameHeight); + auto track = + LocalVideoTrack::createLocalVideoTrack("timestamped-camera", source); + + try { + TrackPublishOptions publish_options; + publish_options.source = TrackSource::SOURCE_CAMERA; + publish_options.packet_trailer_features.user_timestamp = + send_user_timestamp; + + room.localParticipant()->publishTrack(track, publish_options); + std::cout << "[producer] published camera track with user timestamp " + << (send_user_timestamp ? "enabled" : "disabled") << "\n"; + + VideoFrame frame = VideoFrame::create(kFrameWidth, kFrameHeight, + VideoBufferType::BGRA); + const auto capture_start = std::chrono::steady_clock::now(); + std::uint32_t frame_index = 0; + auto next_frame_at = std::chrono::steady_clock::now(); + + while (g_running.load(std::memory_order_relaxed)) { + fillFrame(frame, frame_index); + + VideoCaptureOptions capture_options; + + // a steady_clock to align with other data/video frames + capture_options.timestamp_us = static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - capture_start) + .count()); + capture_options.rotation = VideoRotation::VIDEO_ROTATION_0; + if (send_user_timestamp) { + capture_options.metadata = VideoFrameMetadata{}; + capture_options.metadata->user_timestamp_us = nowEpochUs(); + } + + source->captureFrame(frame, capture_options); + + if (frame_index % 5 == 0) { + std::cout << "[producer] frame=" << frame_index + << " capture_ts_us=" << capture_options.timestamp_us + << " user_ts_us=" + << (send_user_timestamp + ? std::to_string( + *capture_options.metadata->user_timestamp_us) + : std::string("disabled")) + << "\n"; + } + + ++frame_index; + next_frame_at += std::chrono::milliseconds(kFrameIntervalMs); + std::this_thread::sleep_until(next_frame_at); + } + } catch (const std::exception &error) { + std::cerr << "[producer] error: " << error.what() << "\n"; + exit_code = 1; + } + + if (track->publication()) { + room.localParticipant()->unpublishTrack(track->publication()->sid()); + } + } + } + + livekit::shutdown(); + return exit_code; +} diff --git a/include/livekit/data_frame.h b/include/livekit/data_frame.h new file mode 100644 index 00000000..ac830fee --- /dev/null +++ b/include/livekit/data_frame.h @@ -0,0 +1,60 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +namespace livekit { + +namespace proto { +class DataTrackFrame; +} // namespace proto + +/** + * A single frame of data published or received on a data track. + * + * Carries an arbitrary binary payload and an optional user-specified + * timestamp. The unit is application-defined; the SDK examples use + * microseconds since the Unix epoch (system_clock). + */ +struct DataFrame { + /** Arbitrary binary payload (the frame contents). */ + std::vector payload; + + /** + * Optional application-defined timestamp. + * + * The proto field is a bare uint64 with no prescribed unit. + * By convention the SDK examples use microseconds since the Unix epoch. + */ + std::optional user_timestamp; + DataFrame() = default; + DataFrame(const DataFrame &) = default; + DataFrame(DataFrame &&) noexcept = default; + DataFrame &operator=(const DataFrame &) = default; + DataFrame &operator=(DataFrame &&) noexcept = default; + + explicit DataFrame(std::vector &&p, + std::optional ts = std::nullopt) noexcept + : payload(std::move(p)), user_timestamp(ts) {} + + static DataFrame fromOwnedInfo(const proto::DataTrackFrame &owned); +}; + +} // namespace livekit diff --git a/include/livekit/data_track_error.h b/include/livekit/data_track_error.h new file mode 100644 index 00000000..5ec8f97f --- /dev/null +++ b/include/livekit/data_track_error.h @@ -0,0 +1,88 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an “AS IS” BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIVEKIT_DATA_TRACK_ERROR_H +#define LIVEKIT_DATA_TRACK_ERROR_H + +#include +#include + +namespace livekit { + +namespace proto { +class PublishDataTrackError; +class LocalDataTrackTryPushError; +class SubscribeDataTrackError; +} // namespace proto + +enum class PublishDataTrackErrorCode : std::uint32_t { + UNKNOWN = 0, + INVALID_HANDLE = 1, + DUPLICATE_NAME = 2, + TIMEOUT = 3, + DISCONNECTED = 4, + NOT_ALLOWED = 5, + INVALID_NAME = 6, + LIMIT_REACHED = 7, + PROTOCOL_ERROR = 8, + INTERNAL = 9, +}; + +struct PublishDataTrackError { + PublishDataTrackErrorCode code{PublishDataTrackErrorCode::UNKNOWN}; + std::string message; + + static PublishDataTrackError + fromProto(const proto::PublishDataTrackError &error); +}; + +enum class LocalDataTrackTryPushErrorCode : std::uint32_t { + UNKNOWN = 0, + INVALID_HANDLE = 1, + TRACK_UNPUBLISHED = 2, + QUEUE_FULL = 3, + INTERNAL = 4, +}; + +struct LocalDataTrackTryPushError { + LocalDataTrackTryPushErrorCode code{LocalDataTrackTryPushErrorCode::UNKNOWN}; + std::string message; + + static LocalDataTrackTryPushError + fromProto(const proto::LocalDataTrackTryPushError &error); +}; + +enum class SubscribeDataTrackErrorCode : std::uint32_t { + UNKNOWN = 0, + INVALID_HANDLE = 1, + UNPUBLISHED = 2, + TIMEOUT = 3, + DISCONNECTED = 4, + PROTOCOL_ERROR = 5, + INTERNAL = 6, +}; + +struct SubscribeDataTrackError { + SubscribeDataTrackErrorCode code{SubscribeDataTrackErrorCode::UNKNOWN}; + std::string message; + + static SubscribeDataTrackError + fromProto(const proto::SubscribeDataTrackError &error); +}; + +} // namespace livekit + +#endif // LIVEKIT_DATA_TRACK_ERROR_H diff --git a/include/livekit/data_track_info.h b/include/livekit/data_track_info.h new file mode 100644 index 00000000..45c4fc5f --- /dev/null +++ b/include/livekit/data_track_info.h @@ -0,0 +1,40 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace livekit { + +/** + * Metadata about a published data track. + * + * Unlike audio/video tracks, data tracks are not part of the Track class + * hierarchy. They carry their own lightweight info struct. + */ +struct DataTrackInfo { + /** Publisher-assigned track name (unique per publisher). */ + std::string name; + + /** SFU-assigned track identifier. */ + std::string sid; + + /** Whether frames on this track use end-to-end encryption. */ + bool uses_e2ee = false; +}; + +} // namespace livekit diff --git a/include/livekit/data_track_subscription.h b/include/livekit/data_track_subscription.h new file mode 100644 index 00000000..cfac2b24 --- /dev/null +++ b/include/livekit/data_track_subscription.h @@ -0,0 +1,130 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "livekit/data_frame.h" +#include "livekit/ffi_handle.h" + +#include +#include +#include +#include +#include +#include + +namespace livekit { + +namespace proto { +class FfiEvent; +} + +/** + * An active subscription to a remote data track. + * + * Provides a blocking read() interface similar to AudioStream / VideoStream. + * Frames are delivered via FfiEvent callbacks and stored internally. + * + * Dropping (destroying) the subscription automatically unsubscribes from the + * remote track by releasing the underlying FFI handle. + * + * Typical usage: + * + * auto sub_result = remoteDataTrack->subscribe(); + * if (sub_result) { + * auto sub = sub_result.value(); + * DataFrame frame; + * while (sub->read(frame)) { + * // process frame.payload + * } + * } + */ +class DataTrackSubscription { +public: + struct Options { + /// Maximum frames buffered on the Rust side. Rust defaults to 16. + std::optional buffer_size{std::nullopt}; + }; + + virtual ~DataTrackSubscription(); + + DataTrackSubscription(const DataTrackSubscription &) = delete; + DataTrackSubscription &operator=(const DataTrackSubscription &) = delete; + // The FFI listener captures `this`, so moving the object would leave the + // registered callback pointing at the old address. + DataTrackSubscription(DataTrackSubscription &&) noexcept = delete; + // Instances are created and returned as std::shared_ptr, so value-move + // support is not required by the current API. + DataTrackSubscription &operator=(DataTrackSubscription &&) noexcept = delete; + + /** + * Blocking read: waits until a DataFrame is available, or the + * subscription reaches EOS / is closed. + * + * @param out On success, filled with the next data frame. + * @return true if a frame was delivered; false if the subscription ended. + */ + bool read(DataFrame &out); + + /** + * End the subscription early. + * + * Releases the FFI handle (which unsubscribes from the remote track), + * unregisters the event listener, and wakes any blocking read(). + */ + void close(); + +private: + friend class RemoteDataTrack; + + DataTrackSubscription() = default; + /// Internal init helper, called by RemoteDataTrack. + void init(FfiHandle subscription_handle); + + /// FFI event handler, called by FfiClient. + void onFfiEvent(const proto::FfiEvent &event); + + /// Push a received DataFrame to the internal storage. + void pushFrame(DataFrame &&frame); + + /// Push an end-of-stream signal (EOS). + void pushEos(); + + /** Protects all mutable state below. */ + mutable std::mutex mutex_; + + /** Signalled when a frame is pushed or the subscription ends. */ + std::condition_variable cv_; + + /** Received frame awaiting read(). + NOTE: the rust side handles buffering, so we should only really ever have one + item*/ + std::optional frame_; + + /** True once the remote side signals end-of-stream. */ + bool eof_{false}; + + /** True after close() has been called by the consumer. */ + bool closed_{false}; + + /** RAII handle for the Rust-owned subscription resource. */ + FfiHandle subscription_handle_; + + /** FfiClient listener id for routing FfiEvent callbacks to this object. */ + std::int64_t listener_id_{0}; +}; + +} // namespace livekit diff --git a/include/livekit/local_data_track.h b/include/livekit/local_data_track.h new file mode 100644 index 00000000..bfb8e4c1 --- /dev/null +++ b/include/livekit/local_data_track.h @@ -0,0 +1,110 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "livekit/data_frame.h" +#include "livekit/data_track_error.h" +#include "livekit/data_track_info.h" +#include "livekit/ffi_handle.h" +#include "livekit/result.h" + +#include +#include +#include +#include +#include + +namespace livekit { + +namespace proto { +class OwnedLocalDataTrack; +} + +/** + * Represents a locally published data track. + * + * Unlike audio/video tracks, data tracks do not extend the Track base class. + * They use a separate publish/unpublish lifecycle and carry arbitrary binary + * frames instead of media. + * + * Created via LocalParticipant::publishDataTrack(). + * + * Typical usage: + * + * auto lp = room->localParticipant(); + * auto result = lp->publishDataTrack("sensor-data"); + * if (result) { + * auto dt = result.value(); + * DataFrame frame; + * frame.payload = {0x01, 0x02, 0x03}; + * (void)dt->tryPush(frame); + * dt->unpublishDataTrack(); + * } + */ +class LocalDataTrack { +public: + ~LocalDataTrack() = default; + + LocalDataTrack(const LocalDataTrack &) = delete; + LocalDataTrack &operator=(const LocalDataTrack &) = delete; + + /// Metadata about this data track. + const DataTrackInfo &info() const noexcept { return info_; } + + /** + * Try to push a frame to all subscribers of this track. + * + * @return success on delivery acceptance, or a typed error describing why + * the frame could not be queued. + */ + Result tryPush(const DataFrame &frame); + + /** + * Try to push a frame to all subscribers of this track. + * + * @return success on delivery acceptance, or a typed error describing why + * the frame could not be queued. + */ + Result + tryPush(std::vector &&payload, + std::optional user_timestamp = std::nullopt); + + /// Whether the track is still published in the room. + bool isPublished() const; + + /** + * Unpublish this data track from the room. + * + * After this call, tryPush() fails and the track cannot be re-published. + */ + void unpublishDataTrack(); + +private: + friend class LocalParticipant; + + explicit LocalDataTrack(const proto::OwnedLocalDataTrack &owned); + + uintptr_t ffi_handle_id() const noexcept { return handle_.get(); } + + /** RAII wrapper for the Rust-owned FFI resource. */ + FfiHandle handle_; + + /** Metadata snapshot taken at construction time. */ + DataTrackInfo info_; +}; + +} // namespace livekit diff --git a/include/livekit/local_participant.h b/include/livekit/local_participant.h index edd7c945..a7b855d2 100644 --- a/include/livekit/local_participant.h +++ b/include/livekit/local_participant.h @@ -18,6 +18,7 @@ #include "livekit/ffi_handle.h" #include "livekit/local_audio_track.h" +#include "livekit/local_data_track.h" #include "livekit/local_video_track.h" #include "livekit/participant.h" #include "livekit/room_event_types.h" @@ -101,7 +102,13 @@ class LocalParticipant : public Participant { const std::string &topic = {}); /** - * Publish SIP DTMF message. + * Publish a SIP DTMF (phone keypad) tone into the room. + * + * Only meaningful when a SIP trunk is bridging a phone call into the + * room. See SipDtmfData for background on SIP and DTMF. + * + * @param code DTMF code (0-15). + * @param digit Human-readable digit string (e.g. "5", "#"). */ void publishDtmf(int code, const std::string &digit); @@ -164,6 +171,32 @@ class LocalParticipant : public Participant { */ void unpublishTrack(const std::string &track_sid); + /** + * Publish a data track to the room. + * + * Data tracks carry arbitrary binary frames and are independent of the + * audio/video track hierarchy. The returned LocalDataTrack can push + * frames via tryPush() and be unpublished via + * LocalDataTrack::unpublishDataTrack() or + * LocalParticipant::unpublishDataTrack(). + * + * @param name Unique track name visible to other participants. + * @return The published track on success, or a typed error describing why + * publication failed. + */ + Result, PublishDataTrackError> + publishDataTrack(const std::string &name); + + /** + * Unpublish a data track from the room. + * + * Delegates to LocalDataTrack::unpublishDataTrack(). After this call, + * tryPush() on the track will fail and the track cannot be re-published. + * + * @param track The data track to unpublish. Null is ignored. + */ + void unpublishDataTrack(const std::shared_ptr &track); + /** * Initiate an RPC call to a remote participant. * @@ -244,6 +277,7 @@ class LocalParticipant : public Participant { /// cached publication). \c mutable so \ref trackPublications() const can /// prune expired \c weak_ptr entries. mutable TrackMap published_tracks_by_sid_; + std::unordered_map rpc_handlers_; // Shared state for RPC invocation tracking. Using shared_ptr so the state diff --git a/include/livekit/remote_data_track.h b/include/livekit/remote_data_track.h new file mode 100644 index 00000000..02a76094 --- /dev/null +++ b/include/livekit/remote_data_track.h @@ -0,0 +1,101 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "livekit/data_track_error.h" +#include "livekit/data_track_info.h" +#include "livekit/data_track_subscription.h" +#include "livekit/ffi_handle.h" +#include "livekit/result.h" + +#include +#include + +namespace livekit { + +namespace proto { +class OwnedRemoteDataTrack; +} + +/** + * Represents a data track published by a remote participant. + * + * Discovered via the DataTrackPublishedEvent room event. Unlike + * audio/video tracks, remote data tracks require an explicit subscribe() + * call to begin receiving frames. + * + * Typical usage: + * + * // In RoomDelegate::onDataTrackPublished callback: + * auto sub_result = remoteDataTrack->subscribe(); + * if (sub_result) { + * auto sub = sub_result.value(); + * DataFrame frame; + * while (sub->read(frame)) { + * // process frame + * } + * } + */ +class RemoteDataTrack { +public: + ~RemoteDataTrack() = default; + + RemoteDataTrack(const RemoteDataTrack &) = delete; + RemoteDataTrack &operator=(const RemoteDataTrack &) = delete; + + /// Metadata about this data track. + const DataTrackInfo &info() const noexcept { return info_; } + + /// Identity of the remote participant who published this track. + const std::string &publisherIdentity() const noexcept { + return publisher_identity_; + } + + /// Whether the track is still published by the remote participant. + bool isPublished() const; + +#ifdef LIVEKIT_TEST_ACCESS + /// Test-only accessor for exercising lower-level FFI subscription paths. + uintptr_t testFfiHandleId() const noexcept { return ffi_handle_id(); } +#endif + + /** + * Subscribe to this remote data track. + * + * Returns a DataTrackSubscription that delivers frames via blocking + * read(). Destroy the subscription to unsubscribe. + */ + Result, SubscribeDataTrackError> + subscribe(const DataTrackSubscription::Options &options = {}); + +private: + friend class Room; + + explicit RemoteDataTrack(const proto::OwnedRemoteDataTrack &owned); + + uintptr_t ffi_handle_id() const noexcept { return handle_.get(); } + /** RAII wrapper for the Rust-owned FFI resource. */ + FfiHandle handle_; + + /** Metadata snapshot taken at construction time. */ + DataTrackInfo info_; + + /** Identity string of the remote participant who published this track. */ + std::string publisher_identity_; +}; + +} // namespace livekit diff --git a/include/livekit/result.h b/include/livekit/result.h new file mode 100644 index 00000000..bc7f1338 --- /dev/null +++ b/include/livekit/result.h @@ -0,0 +1,181 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an “AS IS” BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIVEKIT_RESULT_H +#define LIVEKIT_RESULT_H + +#include +#include +#include +#include +#include + +namespace livekit { + +/** + * Lightweight success-or-error return type for non-exceptional API failures. + * + * This is intended for SDK operations where callers are expected to branch on + * success vs. failure, such as back-pressure or an unpublished track. + * + * `Result` stores either: + * - a success value of type `T`, or + * - an error value of type `E` + * + * Accessors are intentionally non-throwing. Calling `value()` on an error + * result, or `error()` on a success result, is a programmer error and will + * trip the debug assertion. + */ +template class [[nodiscard]] Result { +public: + /// Construct a successful result containing a value. + template ::value>> + static Result success(U &&value) { + return Result( + std::variant(std::in_place_index<0>, std::forward(value))); + } + + /// Construct a failed result containing an error. + template ::value>> + static Result failure(F &&error) { + return Result( + std::variant(std::in_place_index<1>, std::forward(error))); + } + + /// True when the result contains a success value. + bool ok() const noexcept { return storage_.index() == 0; } + /// True when the result contains an error. + bool has_error() const noexcept { return !ok(); } + /// Allows `if (result)` style success checks. + explicit operator bool() const noexcept { return ok(); } + + /// Access the success value. Requires `ok() == true`. + T &value() & noexcept { + assert(ok()); + return std::get<0>(storage_); + } + + /// Access the success value. Requires `ok() == true`. + const T &value() const & noexcept { + assert(ok()); + return std::get<0>(storage_); + } + + /// Move the success value out. Requires `ok() == true`. + T &&value() && noexcept { + assert(ok()); + return std::get<0>(std::move(storage_)); + } + + /// Move the success value out. Requires `ok() == true`. + const T &&value() const && noexcept { + assert(ok()); + return std::get<0>(std::move(storage_)); + } + + /// Access the error value. Requires `has_error() == true`. + E &error() & noexcept { + assert(has_error()); + return std::get<1>(storage_); + } + + /// Access the error value. Requires `has_error() == true`. + const E &error() const & noexcept { + assert(has_error()); + return std::get<1>(storage_); + } + + /// Move the error value out. Requires `has_error() == true`. + E &&error() && noexcept { + assert(has_error()); + return std::get<1>(std::move(storage_)); + } + + /// Move the error value out. Requires `has_error() == true`. + const E &&error() const && noexcept { + assert(has_error()); + return std::get<1>(std::move(storage_)); + } + +private: + explicit Result(std::variant storage) : storage_(std::move(storage)) {} + + std::variant storage_; +}; + +/** + * `void` specialization for operations that only report success or failure. + * + * This keeps the same calling style as `Result` without forcing callers + * to invent a dummy success payload. + */ +template class [[nodiscard]] Result { +public: + /// Construct a successful result with no payload. + static Result success() { return Result(std::nullopt); } + + /// Construct a failed result containing an error. + template ::value>> + static Result failure(F &&error) { + return Result(std::optional(std::forward(error))); + } + + /// True when the operation succeeded. + bool ok() const noexcept { return !error_.has_value(); } + /// True when the operation failed. + bool has_error() const noexcept { return error_.has_value(); } + /// Allows `if (result)` style success checks. + explicit operator bool() const noexcept { return ok(); } + + /// Validates success in debug builds. Mirrors the `value()` API shape. + void value() const noexcept { assert(ok()); } + + /// Access the error value. Requires `has_error() == true`. + E &error() & noexcept { + assert(has_error()); + return *error_; + } + + /// Access the error value. Requires `has_error() == true`. + const E &error() const & noexcept { + assert(has_error()); + return *error_; + } + + /// Move the error value out. Requires `has_error() == true`. + E &&error() && noexcept { + assert(has_error()); + return std::move(*error_); + } + + /// Move the error value out. Requires `has_error() == true`. + const E &&error() const && noexcept { + assert(has_error()); + return std::move(*error_); + } + +private: + explicit Result(std::optional error) : error_(std::move(error)) {} + + std::optional error_; +}; + +} // namespace livekit + +#endif // LIVEKIT_RESULT_H diff --git a/include/livekit/room.h b/include/livekit/room.h index d808ecd4..52d55d5e 100644 --- a/include/livekit/room.h +++ b/include/livekit/room.h @@ -241,62 +241,82 @@ class Room { // --------------------------------------------------------------- /** - * Set a callback for audio frames from a specific remote participant and - * track source. - * - * A dedicated reader thread is spawned for each (participant, source) pair - * when the track is subscribed. If the track is already subscribed, the - * reader starts immediately. If not, it starts when the track arrives. - * - * Only one callback may exist per (participant, source) pair. Re-calling - * with the same pair replaces the previous callback. - * - * @param participant_identity Identity of the remote participant. - * @param source Track source (e.g. SOURCE_MICROPHONE). - * @param callback Function invoked per audio frame. - * @param opts AudioStream options (capacity, noise - * cancellation). + * @brief Sets the audio frame callback via SubscriptionThreadDispatcher. */ void setOnAudioFrameCallback(const std::string &participant_identity, TrackSource source, AudioFrameCallback callback, AudioStream::Options opts = {}); /** - * Set a callback for video frames from a specific remote participant and - * track source. - * - * @see setOnAudioFrameCallback for threading and lifecycle semantics. - * - * @param participant_identity Identity of the remote participant. - * @param source Track source (e.g. SOURCE_CAMERA). - * @param callback Function invoked per video frame. - * @param opts VideoStream options (capacity, pixel format). + * @brief Sets the audio frame callback via SubscriptionThreadDispatcher. + */ + void setOnAudioFrameCallback(const std::string &participant_identity, + const std::string &track_name, + AudioFrameCallback callback, + AudioStream::Options opts = {}); + + /** + * @brief Sets the video frame callback via SubscriptionThreadDispatcher. */ void setOnVideoFrameCallback(const std::string &participant_identity, TrackSource source, VideoFrameCallback callback, VideoStream::Options opts = {}); /** - * Clear the audio frame callback for a specific (participant, source) pair. - * Stops and joins any active reader thread. - * No-op if no callback is registered for this key. - * @param participant_identity Identity of the remote participant. - * @param source Track source (e.g. SOURCE_MICROPHONE). + * @brief Sets the video frame callback via SubscriptionThreadDispatcher. + */ + void setOnVideoFrameCallback(const std::string &participant_identity, + const std::string &track_name, + VideoFrameCallback callback, + VideoStream::Options opts = {}); + + /** + * @brief Sets the video frame callback via SubscriptionThreadDispatcher. + */ + void setOnVideoFrameEventCallback(const std::string &participant_identity, + TrackSource source, + VideoFrameEventCallback callback, + VideoStream::Options opts); + + /** + * @brief Clears the audio frame callback via SubscriptionThreadDispatcher. */ void clearOnAudioFrameCallback(const std::string &participant_identity, TrackSource source); + /** + * @brief Clears the audio frame callback via SubscriptionThreadDispatcher. + */ + void clearOnAudioFrameCallback(const std::string &participant_identity, + const std::string &track_name); /** - * Clear the video frame callback for a specific (participant, source) pair. - * Stops and joins any active reader thread. - * No-op if no callback is registered for this key. - * @param participant_identity Identity of the remote participant. - * @param source Track source (e.g. SOURCE_CAMERA). + * @brief Clears the video frame callback via SubscriptionThreadDispatcher. */ void clearOnVideoFrameCallback(const std::string &participant_identity, TrackSource source); + /** + * @brief Clears the video frame callback via SubscriptionThreadDispatcher. + */ + void clearOnVideoFrameCallback(const std::string &participant_identity, + const std::string &track_name); + + /** + * @brief Adds a data frame callback via SubscriptionThreadDispatcher. + */ + DataFrameCallbackId + addOnDataFrameCallback(const std::string &participant_identity, + const std::string &track_name, + DataFrameCallback callback); + + /** + * @brief Removes the data frame callback via SubscriptionThreadDispatcher. + */ + void removeOnDataFrameCallback(DataFrameCallbackId id); + private: + friend class RoomCallbackTest; + mutable std::mutex lock_; ConnectionState connection_state_ = ConnectionState::Disconnected; RoomDelegate *delegate_ = nullptr; // Not owned diff --git a/include/livekit/room_delegate.h b/include/livekit/room_delegate.h index 04474a9f..2621c92c 100644 --- a/include/livekit/room_delegate.h +++ b/include/livekit/room_delegate.h @@ -287,6 +287,24 @@ class RoomDelegate { */ virtual void onTextStreamOpened(Room &, const TextStreamOpenedEvent &) {} + // ------------------------------------------------------------------ + // Data tracks + // ------------------------------------------------------------------ + + /** + * Called when a remote participant publishes a data track. + * + * Data tracks are independent of the audio/video track hierarchy and + * require an explicit subscribe() call to start receiving frames. + */ + virtual void onDataTrackPublished(Room &, const DataTrackPublishedEvent &) {} + + /** + * Called when a remote participant unpublishes a data track. + */ + virtual void onDataTrackUnpublished(Room &, + const DataTrackUnpublishedEvent &) {} + // ------------------------------------------------------------------ // Participants snapshot // ------------------------------------------------------------------ diff --git a/include/livekit/room_event_types.h b/include/livekit/room_event_types.h index 63c75140..809f9efc 100644 --- a/include/livekit/room_event_types.h +++ b/include/livekit/room_event_types.h @@ -29,6 +29,7 @@ namespace livekit { class Track; class Participant; class RemoteParticipant; +class RemoteDataTrack; class LocalTrackPublication; class RemoteTrackPublication; class TrackPublication; @@ -100,7 +101,7 @@ enum class DisconnectReason { RoomClosed, UserUnavailable, UserRejected, - SipTrunkFailure, + SipTrunkFailure, ///< SIP (telephony) trunk connection failed ConnectionTimeout, MediaFailure }; @@ -117,10 +118,17 @@ struct UserPacketData { }; /** - * SIP DTMF payload carried via data packets. + * SIP (Session Initiation Protocol) DTMF payload carried via data packets. + * + * SIP is a signalling protocol used in VoIP telephony. LiveKit supports + * SIP trunking, which bridges traditional phone calls into LiveKit rooms. + * DTMF (Dual-Tone Multi-Frequency) tones are the signals generated when + * phone keypad buttons are pressed (0-9, *, #). This struct surfaces + * those tones so that applications handling SIP-bridged calls can react + * to caller input (e.g. IVR menu selection). */ struct SipDtmfData { - /** DTMF code value. */ + /** Numeric DTMF code (0-15, mapping to 0-9, *, #, A-D). */ std::uint32_t code = 0; /** Human-readable digit representation (e.g. "1", "#"). */ @@ -299,6 +307,17 @@ struct AudioEncodingOptions { std::uint64_t max_bitrate = 0; }; +/** + * Optional RTP packet-trailer features for published video tracks. + */ +struct PacketTrailerFeatures { + /** Embed a user-supplied wall-clock timestamp. */ + bool user_timestamp = false; + + /** Embed a monotonically increasing frame identifier. */ + bool frame_id = false; +}; + /** * Options for publishing a track to the room. */ @@ -329,6 +348,9 @@ struct TrackPublishOptions { /** Enable pre-connect buffering for lower startup latency. */ std::optional preconnect_buffer; + + /** Optional packet-trailer features to enable for published video. */ + PacketTrailerFeatures packet_trailer_features{}; }; // --------------------------------------------------------- @@ -719,4 +741,24 @@ struct E2eeStateChangedEvent { EncryptionState state = EncryptionState::New; }; +/** + * Fired when a participant publishes a data track. + * + * Data tracks are independent of the audio/video track hierarchy. + * The application must call RemoteDataTrack::subscribe() to start + * receiving frames. + */ +struct DataTrackPublishedEvent { + /** The newly published remote data track. */ + std::shared_ptr track; +}; + +/** + * Fired when a remote participant unpublishes a data track. + */ +struct DataTrackUnpublishedEvent { + /** SID of the track that was unpublished. */ + std::string sid; +}; + } // namespace livekit diff --git a/include/livekit/subscription_thread_dispatcher.h b/include/livekit/subscription_thread_dispatcher.h index 3e843541..2ef5dceb 100644 --- a/include/livekit/subscription_thread_dispatcher.h +++ b/include/livekit/subscription_thread_dispatcher.h @@ -24,13 +24,17 @@ #include #include #include +#include #include #include #include +#include namespace livekit { class AudioFrame; +class DataTrackSubscription; +class RemoteDataTrack; class Track; class VideoFrame; @@ -43,6 +47,18 @@ using AudioFrameCallback = std::function; using VideoFrameCallback = std::function; +/// Callback type for incoming data track frames. +/// Invoked on a dedicated reader thread per subscription. +/// @param payload Raw binary data received. +/// @param user_timestamp Optional application-defined timestamp from sender. +using DataFrameCallback = + std::function &payload, + std::optional user_timestamp)>; + +/// Opaque identifier returned by addOnDataFrameCallback, used to remove an +/// individual subscription via removeOnDataFrameCallback. +using DataFrameCallbackId = std::uint64_t; + /** * Owns subscription callback registration and per-subscription reader threads. * @@ -90,6 +106,24 @@ class SubscriptionThreadDispatcher { TrackSource source, AudioFrameCallback callback, AudioStream::Options opts = {}); + /** + * Register or replace an audio frame callback for a remote subscription. + * + * The callback is keyed by remote participant identity plus \p track_name. + * If the matching remote audio track is already subscribed, \ref Room may + * immediately call \ref handleTrackSubscribed to start a reader. + * + * @param participant_identity Identity of the remote participant. + * @param track_name Track name to match. + * @param callback Function invoked for each decoded audio frame. + * @param opts Options used when creating the backing + * \ref AudioStream. + */ + void setOnAudioFrameCallback(const std::string &participant_identity, + const std::string &track_name, + AudioFrameCallback callback, + AudioStream::Options opts = {}); + /** * Register or replace a video frame callback for a remote subscription. * @@ -107,6 +141,41 @@ class SubscriptionThreadDispatcher { TrackSource source, VideoFrameCallback callback, VideoStream::Options opts = {}); + /** + * Register or replace a legacy video frame callback for a remote + * subscription matched by track name. + * + * @param participant_identity Identity of the remote participant. + * @param track_name Track name to match. + * @param callback Function invoked for each decoded video frame. + * @param opts Options used when creating the backing + * \ref VideoStream. + */ + void setOnVideoFrameCallback(const std::string &participant_identity, + const std::string &track_name, + VideoFrameCallback callback, + VideoStream::Options opts = {}); + + /** + * Register or replace a rich video frame event callback for a remote + * subscription. + * + * The callback is keyed by remote participant identity plus \p source. + * If the matching remote video track is already subscribed, \ref Room may + * immediately call \ref handleTrackSubscribed to start a reader. + * + * @param participant_identity Identity of the remote participant. + * @param source Track source to match. + * @param callback Function invoked for each decoded video frame + * event, including optional metadata. + * @param opts Options used when creating the backing + * \ref VideoStream. + */ + void setOnVideoFrameEventCallback(const std::string &participant_identity, + TrackSource source, + VideoFrameEventCallback callback, + VideoStream::Options opts = {}); + /** * Remove an audio callback registration and stop any active reader. * @@ -119,6 +188,18 @@ class SubscriptionThreadDispatcher { void clearOnAudioFrameCallback(const std::string &participant_identity, TrackSource source); + /** + * Remove an audio callback registration and stop any active reader. + * + * If an audio reader thread is active for the given key, its stream is + * closed and the thread is joined before this call returns. + * + * @param participant_identity Identity of the remote participant. + * @param track_name Track name to clear. + */ + void clearOnAudioFrameCallback(const std::string &participant_identity, + const std::string &track_name); + /** * Remove a video callback registration and stop any active reader. * @@ -131,22 +212,35 @@ class SubscriptionThreadDispatcher { void clearOnVideoFrameCallback(const std::string &participant_identity, TrackSource source); + /** + * Remove a video callback registration and stop any active reader. + * + * If a video reader thread is active for the given key, its stream is + * closed and the thread is joined before this call returns. + * + * @param participant_identity Identity of the remote participant. + * @param track_name Track name to clear. + */ + void clearOnVideoFrameCallback(const std::string &participant_identity, + const std::string &track_name); + /** * Start or restart reader dispatch for a newly subscribed remote track. * - * \ref Room calls this after it has processed a track-subscription event and - * updated its publication state. If a matching callback registration exists, - * the dispatcher creates the appropriate stream type and launches a reader - * thread for the `(participant, source)` key. + * \ref Room calls this after it has processed a track-subscription event + * and updated its publication state. If a matching callback registration + * exists, the dispatcher creates the appropriate stream type and launches a + * reader thread for the `(participant, source)` key. * * If no matching callback is registered, this is a no-op. * * @param participant_identity Identity of the remote participant. - * @param source Track source associated with the subscription. + * @param source Track source associated with the + * subscription. * @param track Subscribed remote track to read from. */ void handleTrackSubscribed(const std::string &participant_identity, - TrackSource source, + TrackSource source, const std::string &track_name, const std::shared_ptr &track); /** @@ -158,10 +252,70 @@ class SubscriptionThreadDispatcher { * re-subscription can start dispatch again automatically. * * @param participant_identity Identity of the remote participant. - * @param source Track source associated with the subscription. + * @param source Track source associated with the + * subscription. + * @param track_name Track name associated with the subscription. */ void handleTrackUnsubscribed(const std::string &participant_identity, - TrackSource source); + TrackSource source, + const std::string &track_name); + + // --------------------------------------------------------------- + // Data track callbacks + // --------------------------------------------------------------- + + /** + * Add a callback for data frames from a specific remote participant's + * data track. + * + * Multiple callbacks may be registered for the same (participant, + * track_name) pair; each one creates an independent FFI subscription. + * + * The callback fires on a dedicated background thread. If the remote + * data track has not yet been published, the callback is stored and + * auto-wired when the track appears (via handleDataTrackPublished). + * + * @param participant_identity Identity of the remote participant. + * @param track_name Name of the remote data track. + * @param callback Function to invoke per data frame. + * @return An opaque ID that can later be passed to + * removeOnDataFrameCallback() to tear down this subscription. + */ + DataFrameCallbackId + addOnDataFrameCallback(const std::string &participant_identity, + const std::string &track_name, + DataFrameCallback callback); + + /** + * Remove a data frame callback previously registered via + * addOnDataFrameCallback(). Stops and joins the active reader thread + * for this subscription. + * No-op if the ID is not (or no longer) registered. + * + * @param id The identifier returned by addOnDataFrameCallback(). + */ + void removeOnDataFrameCallback(DataFrameCallbackId id); + + /** + * Notify the dispatcher that a remote data track has been published. + * + * \ref Room calls this when it receives a kDataTrackPublished event. + * For every registered callback whose (participant, track_name) matches, + * a reader thread is launched. + * + * @param track The newly published remote data track. + */ + void handleDataTrackPublished(const std::shared_ptr &track); + + /** + * Notify the dispatcher that a remote data track has been unpublished. + * + * \ref Room calls this when it receives a kDataTrackUnpublished event. + * Any active data reader threads for this track SID are closed and joined. + * + * @param sid The SID of the unpublished data track. + */ + void handleDataTrackUnpublished(const std::string &sid); /** * Stop all readers and clear all callback registrations. @@ -174,14 +328,17 @@ class SubscriptionThreadDispatcher { private: friend class SubscriptionThreadDispatcherTest; - /// Compound lookup key for a remote participant identity and track source. + /// Compound lookup key for callback dispatch: + /// either `(participant, source, "")` or `(participant, SOURCE_UNKNOWN, + /// track_name)`. struct CallbackKey { std::string participant_identity; TrackSource source; + std::string track_name; bool operator==(const CallbackKey &o) const { return participant_identity == o.participant_identity && - source == o.source; + source == o.source && track_name == o.track_name; } }; @@ -190,17 +347,54 @@ class SubscriptionThreadDispatcher { std::size_t operator()(const CallbackKey &k) const { auto h1 = std::hash{}(k.participant_identity); auto h2 = std::hash{}(static_cast(k.source)); - return h1 ^ (h2 << 1); + auto h3 = std::hash{}(k.track_name); + return h1 ^ (h2 << 1) ^ (h3 << 2); } }; - /// Active read-side resources for one subscription dispatch slot. + /// Active read-side resources for one audio/video subscription dispatch + /// slot. struct ActiveReader { std::shared_ptr audio_stream; std::shared_ptr video_stream; std::thread thread; }; + /// Compound lookup key for a remote participant identity and data track + /// name. + struct DataCallbackKey { + std::string participant_identity; + std::string track_name; + + bool operator==(const DataCallbackKey &o) const { + return participant_identity == o.participant_identity && + track_name == o.track_name; + } + }; + + /// Hash function for \ref DataCallbackKey. + struct DataCallbackKeyHash { + std::size_t operator()(const DataCallbackKey &k) const { + auto h1 = std::hash{}(k.participant_identity); + auto h2 = std::hash{}(k.track_name); + return h1 ^ (h2 << 1); + } + }; + + /// Stored data callback registration. + struct RegisteredDataCallback { + DataCallbackKey key; + DataFrameCallback callback; + }; + + /// Active read-side resources for one data track subscription. + struct ActiveDataReader { + std::shared_ptr remote_track; + std::mutex sub_mutex; + std::shared_ptr subscription; // guarded by sub_mutex + std::thread thread; + }; + /// Stored audio callback registration plus stream-construction options. struct RegisteredAudioCallback { AudioFrameCallback callback; @@ -209,7 +403,8 @@ class SubscriptionThreadDispatcher { /// Stored video callback registration plus stream-construction options. struct RegisteredVideoCallback { - VideoFrameCallback callback; + VideoFrameCallback legacy_callback; + VideoFrameEventCallback event_callback; VideoStream::Options options; }; @@ -227,8 +422,9 @@ class SubscriptionThreadDispatcher { /// Start an audio reader thread for \p key using \p track. /// - /// Must be called with \ref lock_ held. Any previous reader for the same key - /// is extracted and returned to the caller for joining outside the lock. + /// Must be called with \ref lock_ held. Any previous reader for the same + /// key is extracted and returned to the caller for joining outside the + /// lock. std::thread startAudioReaderLocked(const CallbackKey &key, const std::shared_ptr &track, AudioFrameCallback cb, @@ -236,28 +432,59 @@ class SubscriptionThreadDispatcher { /// Start a video reader thread for \p key using \p track. /// - /// Must be called with \ref lock_ held. Any previous reader for the same key - /// is extracted and returned to the caller for joining outside the lock. + /// Must be called with \ref lock_ held. Any previous reader for the same + /// key is extracted and returned to the caller for joining outside the + /// lock. std::thread startVideoReaderLocked(const CallbackKey &key, const std::shared_ptr &track, - VideoFrameCallback cb, - const VideoStream::Options &opts); + const RegisteredVideoCallback &callback); + + /// Extract and close the data reader for a given callback ID, returning its + /// thread. Must be called with \ref lock_ held. + std::thread extractDataReaderThreadLocked(DataFrameCallbackId id); + + /// Extract and close the data reader for a given (participant, track_name) + /// key, returning its thread. Must be called with \ref lock_ held. + std::thread extractDataReaderThreadLocked(const DataCallbackKey &key); + + /// Start a data reader thread for the given callback ID, key, and track. + /// Must be called with \ref lock_ held. + std::thread + startDataReaderLocked(DataFrameCallbackId id, const DataCallbackKey &key, + const std::shared_ptr &track, + DataFrameCallback cb); /// Protects callback registration maps and active reader state. mutable std::mutex lock_; - /// Registered audio frame callbacks keyed by `(participant, source)`. + /// Registered audio frame callbacks keyed by \ref CallbackKey. std::unordered_map audio_callbacks_; - /// Registered video frame callbacks keyed by `(participant, source)`. + /// Registered video frame callbacks keyed by \ref CallbackKey. std::unordered_map video_callbacks_; - /// Active stream/thread state keyed by `(participant, source)`. + /// Active stream/thread state keyed by \ref CallbackKey. std::unordered_map active_readers_; + /// Next auto-increment ID for data frame callbacks. + DataFrameCallbackId next_data_callback_id_; + + /// Registered data frame callbacks keyed by opaque callback ID. + std::unordered_map + data_callbacks_; + + /// Active data reader threads keyed by callback ID. + std::unordered_map> + active_data_readers_; + + /// Currently published remote data tracks, keyed by (participant, name). + std::unordered_map, + DataCallbackKeyHash> + remote_data_tracks_; + /// Hard limit on concurrently active per-subscription reader threads. static constexpr int kMaxActiveReaders = 20; }; diff --git a/include/livekit/video_source.h b/include/livekit/video_source.h index 47715dc2..89703479 100644 --- a/include/livekit/video_source.h +++ b/include/livekit/video_source.h @@ -17,6 +17,7 @@ #pragma once #include +#include #include "livekit/ffi_handle.h" @@ -36,6 +37,26 @@ enum class VideoRotation { VIDEO_ROTATION_270 = 270, }; +/** + * Optional packet-trailer metadata carried alongside a video frame. + * + * Each field is independently optional because the corresponding transport + * feature can be negotiated separately. + */ +struct VideoFrameMetadata { + std::optional user_timestamp_us; + std::optional frame_id; +}; + +/** + * Capture options for a single outbound video frame. + */ +struct VideoCaptureOptions { + std::int64_t timestamp_us = 0; + VideoRotation rotation = VideoRotation::VIDEO_ROTATION_0; + std::optional metadata; +}; + /** * Represents a real-time video source that can accept frames from the * application and feed them into the LiveKit core. @@ -69,14 +90,14 @@ class VideoSource { /** * Push a VideoFrame into the FFI video source. * - * @param frame Video frame to send. - * @param timestamp_us Optional timestamp in microseconds. - * @param rotation Video rotation enum. - * @param timeout_ms Controls waiting behavior: - * - * Notes: - * - Fire-and-forget to send a frame to FFI - * lifetime correctly (e.g., persistent frame pools, GPU buffers, etc.). + * @param frame Video frame to send. + * @param options Timestamp, rotation, and optional metadata for this frame. + */ + void captureFrame(const VideoFrame &frame, + const VideoCaptureOptions &options); + + /** + * Backward-compatible convenience overload for timestamp + rotation only. */ void captureFrame(const VideoFrame &frame, std::int64_t timestamp_us = 0, VideoRotation rotation = VideoRotation::VIDEO_ROTATION_0); diff --git a/include/livekit/video_stream.h b/include/livekit/video_stream.h index 0f666083..9350f52a 100644 --- a/include/livekit/video_stream.h +++ b/include/livekit/video_stream.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -34,10 +35,17 @@ namespace livekit { // A single video frame event delivered by VideoStream::read(). struct VideoFrameEvent { VideoFrame frame; + // WebRTC frame timestamp in microseconds. + // This may be translated onto WebRTC's internal capture-time timeline and + // should not be expected to match application-provided metadata such as + // VideoFrameMetadata::user_timestamp_us exactly. std::int64_t timestamp_us; VideoRotation rotation; + std::optional metadata; }; +using VideoFrameEventCallback = std::function; + namespace proto { class FfiEvent; } diff --git a/src/data_frame.cpp b/src/data_frame.cpp new file mode 100644 index 00000000..17a82722 --- /dev/null +++ b/src/data_frame.cpp @@ -0,0 +1,20 @@ +#include "livekit/data_frame.h" + +#include "data_track.pb.h" + +namespace livekit { + +DataFrame DataFrame::fromOwnedInfo(const proto::DataTrackFrame &owned) { + DataFrame frame; + const auto &payload_str = owned.payload(); + frame.payload.assign( + reinterpret_cast(payload_str.data()), + reinterpret_cast(payload_str.data()) + + payload_str.size()); + if (owned.has_user_timestamp()) { + frame.user_timestamp = owned.user_timestamp(); + } + return frame; +} + +} // namespace livekit diff --git a/src/data_track_error.cpp b/src/data_track_error.cpp new file mode 100644 index 00000000..9926493c --- /dev/null +++ b/src/data_track_error.cpp @@ -0,0 +1,107 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an “AS IS” BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "livekit/data_track_error.h" + +#include "data_track.pb.h" + +namespace livekit { + +namespace { + +PublishDataTrackErrorCode fromProtoCode(proto::PublishDataTrackErrorCode code) { + switch (code) { + case proto::PUBLISH_DATA_TRACK_ERROR_CODE_INVALID_HANDLE: + return PublishDataTrackErrorCode::INVALID_HANDLE; + case proto::PUBLISH_DATA_TRACK_ERROR_CODE_DUPLICATE_NAME: + return PublishDataTrackErrorCode::DUPLICATE_NAME; + case proto::PUBLISH_DATA_TRACK_ERROR_CODE_TIMEOUT: + return PublishDataTrackErrorCode::TIMEOUT; + case proto::PUBLISH_DATA_TRACK_ERROR_CODE_DISCONNECTED: + return PublishDataTrackErrorCode::DISCONNECTED; + case proto::PUBLISH_DATA_TRACK_ERROR_CODE_NOT_ALLOWED: + return PublishDataTrackErrorCode::NOT_ALLOWED; + case proto::PUBLISH_DATA_TRACK_ERROR_CODE_INVALID_NAME: + return PublishDataTrackErrorCode::INVALID_NAME; + case proto::PUBLISH_DATA_TRACK_ERROR_CODE_LIMIT_REACHED: + return PublishDataTrackErrorCode::LIMIT_REACHED; + case proto::PUBLISH_DATA_TRACK_ERROR_CODE_PROTOCOL_ERROR: + return PublishDataTrackErrorCode::PROTOCOL_ERROR; + case proto::PUBLISH_DATA_TRACK_ERROR_CODE_INTERNAL: + return PublishDataTrackErrorCode::INTERNAL; + case proto::PUBLISH_DATA_TRACK_ERROR_CODE_UNKNOWN: + default: + return PublishDataTrackErrorCode::UNKNOWN; + } +} + +LocalDataTrackTryPushErrorCode +fromProtoCode(proto::LocalDataTrackTryPushErrorCode code) { + switch (code) { + case proto::LOCAL_DATA_TRACK_TRY_PUSH_ERROR_CODE_INVALID_HANDLE: + return LocalDataTrackTryPushErrorCode::INVALID_HANDLE; + case proto::LOCAL_DATA_TRACK_TRY_PUSH_ERROR_CODE_TRACK_UNPUBLISHED: + return LocalDataTrackTryPushErrorCode::TRACK_UNPUBLISHED; + case proto::LOCAL_DATA_TRACK_TRY_PUSH_ERROR_CODE_QUEUE_FULL: + return LocalDataTrackTryPushErrorCode::QUEUE_FULL; + case proto::LOCAL_DATA_TRACK_TRY_PUSH_ERROR_CODE_INTERNAL: + return LocalDataTrackTryPushErrorCode::INTERNAL; + case proto::LOCAL_DATA_TRACK_TRY_PUSH_ERROR_CODE_UNKNOWN: + default: + return LocalDataTrackTryPushErrorCode::UNKNOWN; + } +} + +SubscribeDataTrackErrorCode +fromProtoCode(proto::SubscribeDataTrackErrorCode code) { + switch (code) { + case proto::SUBSCRIBE_DATA_TRACK_ERROR_CODE_INVALID_HANDLE: + return SubscribeDataTrackErrorCode::INVALID_HANDLE; + case proto::SUBSCRIBE_DATA_TRACK_ERROR_CODE_UNPUBLISHED: + return SubscribeDataTrackErrorCode::UNPUBLISHED; + case proto::SUBSCRIBE_DATA_TRACK_ERROR_CODE_TIMEOUT: + return SubscribeDataTrackErrorCode::TIMEOUT; + case proto::SUBSCRIBE_DATA_TRACK_ERROR_CODE_DISCONNECTED: + return SubscribeDataTrackErrorCode::DISCONNECTED; + case proto::SUBSCRIBE_DATA_TRACK_ERROR_CODE_PROTOCOL_ERROR: + return SubscribeDataTrackErrorCode::PROTOCOL_ERROR; + case proto::SUBSCRIBE_DATA_TRACK_ERROR_CODE_INTERNAL: + return SubscribeDataTrackErrorCode::INTERNAL; + case proto::SUBSCRIBE_DATA_TRACK_ERROR_CODE_UNKNOWN: + default: + return SubscribeDataTrackErrorCode::UNKNOWN; + } +} + +} // namespace + +PublishDataTrackError +PublishDataTrackError::fromProto(const proto::PublishDataTrackError &error) { + return PublishDataTrackError{fromProtoCode(error.code()), error.message()}; +} + +LocalDataTrackTryPushError LocalDataTrackTryPushError::fromProto( + const proto::LocalDataTrackTryPushError &error) { + return LocalDataTrackTryPushError{fromProtoCode(error.code()), + error.message()}; +} + +SubscribeDataTrackError SubscribeDataTrackError::fromProto( + const proto::SubscribeDataTrackError &error) { + return SubscribeDataTrackError{fromProtoCode(error.code()), error.message()}; +} + +} // namespace livekit diff --git a/src/data_track_subscription.cpp b/src/data_track_subscription.cpp new file mode 100644 index 00000000..fc186bcb --- /dev/null +++ b/src/data_track_subscription.cpp @@ -0,0 +1,140 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "livekit/data_track_subscription.h" + +#include "data_track.pb.h" +#include "ffi.pb.h" +#include "ffi_client.h" +#include "livekit/lk_log.h" + +#include + +namespace livekit { + +using proto::FfiEvent; + +DataTrackSubscription::~DataTrackSubscription() { close(); } + +void DataTrackSubscription::init(FfiHandle subscription_handle) { + subscription_handle_ = std::move(subscription_handle); + + listener_id_ = FfiClient::instance().AddListener( + [this](const FfiEvent &e) { this->onFfiEvent(e); }); +} + +bool DataTrackSubscription::read(DataFrame &out) { + { + std::lock_guard lock(mutex_); + if (closed_ || eof_) { + return false; + } + + const auto subscription_handle = + static_cast(subscription_handle_.get()); + + // Signal the Rust side that we're ready to receive the next frame. + // The Rust SubscriptionTask uses a demand-driven protocol: it won't pull + // from the underlying stream until notified via this request. + proto::FfiRequest req; + auto *msg = req.mutable_data_track_subscription_read(); + msg->set_subscription_handle(subscription_handle); + FfiClient::instance().sendRequest(req); + } + + std::unique_lock lock(mutex_); + cv_.wait(lock, [this] { return frame_.has_value() || eof_ || closed_; }); + + if (closed_ || (!frame_.has_value() && eof_)) { + return false; + } + + out = std::move(*frame_); + frame_.reset(); + return true; +} + +void DataTrackSubscription::close() { + std::int64_t listener_id = -1; + { + std::lock_guard lock(mutex_); + if (closed_) { + return; + } + closed_ = true; + subscription_handle_.reset(); + listener_id = listener_id_; + listener_id_ = 0; + } + + if (listener_id != -1) { + FfiClient::instance().RemoveListener(listener_id); + } + + cv_.notify_all(); +} + +void DataTrackSubscription::onFfiEvent(const FfiEvent &event) { + if (event.message_case() != FfiEvent::kDataTrackSubscriptionEvent) { + return; + } + + const auto &dts = event.data_track_subscription_event(); + { + std::lock_guard lock(mutex_); + if (closed_ || dts.subscription_handle() != + static_cast(subscription_handle_.get())) { + return; + } + } + + if (dts.has_frame_received()) { + const auto &fr = dts.frame_received().frame(); + DataFrame frame = DataFrame::fromOwnedInfo(fr); + pushFrame(std::move(frame)); + } else if (dts.has_eos()) { + pushEos(); + } +} + +void DataTrackSubscription::pushFrame(DataFrame &&frame) { + std::lock_guard lock(mutex_); + + if (closed_ || eof_) { + return; + } + + // rust side handles buffering, so we should only really ever have one item + assert(!frame_.has_value()); + + frame_ = std::move(frame); + + // notify no matter what since we got a new frame + cv_.notify_one(); +} + +void DataTrackSubscription::pushEos() { + { + std::lock_guard lock(mutex_); + if (eof_) { + return; + } + eof_ = true; + } + cv_.notify_all(); +} + +} // namespace livekit diff --git a/src/e2ee.cpp b/src/e2ee.cpp index dc95252f..ae46bf79 100644 --- a/src/e2ee.cpp +++ b/src/e2ee.cpp @@ -166,6 +166,7 @@ void E2EEManager::setEnabled(bool enabled) { req.mutable_e2ee()->set_room_handle(room_handle_); req.mutable_e2ee()->mutable_manager_set_enabled()->set_enabled(enabled); FfiClient::instance().sendRequest(req); + enabled_ = enabled; } E2EEManager::KeyProvider *E2EEManager::keyProvider() { return &key_provider_; } diff --git a/src/ffi_client.cpp b/src/ffi_client.cpp index eba73821..36e0f53a 100644 --- a/src/ffi_client.cpp +++ b/src/ffi_client.cpp @@ -16,6 +16,7 @@ #include +#include "data_track.pb.h" #include "e2ee.pb.h" #include "ffi.pb.h" #include "ffi_client.h" @@ -114,6 +115,10 @@ std::optional ExtractAsyncId(const proto::FfiEvent &event) { case E::kSendBytes: return event.send_bytes().async_id(); + // data track async completions + case E::kPublishDataTrack: + return event.publish_data_track().async_id(); + // NOT async completion: case E::kRoomEvent: case E::kTrackEvent: @@ -121,6 +126,7 @@ std::optional ExtractAsyncId(const proto::FfiEvent &event) { case E::kAudioStreamEvent: case E::kByteStreamReaderEvent: case E::kTextStreamReaderEvent: + case E::kDataTrackSubscriptionEvent: case E::kRpcMethodInvocation: case E::kLogs: case E::kPanic: @@ -318,6 +324,11 @@ FfiClient::connectAsync(const std::string &url, const std::string &token, opts->set_dynacast(options.dynacast); opts->set_single_peer_connection(options.single_peer_connection); + LK_LOG_DEBUG("[FfiClient] connectAsync: auto_subscribe={}, dynacast={}, " + "single_peer_connection={}", + options.auto_subscribe, options.dynacast, + options.single_peer_connection); + // --- E2EE / encryption (optional) --- if (options.encryption.has_value()) { const E2EEOptions &e2ee = *options.encryption; @@ -608,6 +619,117 @@ std::future FfiClient::publishDataAsync( return fut; } +std::future> +FfiClient::publishDataTrackAsync(std::uint64_t local_participant_handle, + const std::string &track_name) { + const AsyncId async_id = generateAsyncId(); + + auto fut = registerAsync< + Result>( + async_id, + [async_id](const proto::FfiEvent &event) { + return event.has_publish_data_track() && + event.publish_data_track().async_id() == async_id; + }, + [](const proto::FfiEvent &event, + std::promise> + &pr) { + const auto &cb = event.publish_data_track(); + if (cb.has_error()) { + pr.set_value( + Result:: + failure(PublishDataTrackError::fromProto(cb.error()))); + return; + } + if (!cb.has_track()) { + pr.set_value( + Result::failure(PublishDataTrackError{ + PublishDataTrackErrorCode::PROTOCOL_ERROR, + "PublishDataTrackCallback missing track"})); + return; + } + proto::OwnedLocalDataTrack track = cb.track(); + pr.set_value( + Result::success( + std::move(track))); + }); + + proto::FfiRequest req; + auto *msg = req.mutable_publish_data_track(); + msg->set_local_participant_handle(local_participant_handle); + msg->mutable_options()->set_name(track_name); + msg->set_request_async_id(async_id); + + try { + proto::FfiResponse resp = sendRequest(req); + if (!resp.has_publish_data_track()) { + cancelPendingByAsyncId(async_id); + std::promise> + pr; + pr.set_value( + Result::failure( + PublishDataTrackError{PublishDataTrackErrorCode::PROTOCOL_ERROR, + "FfiResponse missing publish_data_track"})); + return pr.get_future(); + } + } catch (...) { + cancelPendingByAsyncId(async_id); + std::promise> pr; + try { + throw; + } catch (const std::exception &e) { + pr.set_value( + Result::failure( + PublishDataTrackError{PublishDataTrackErrorCode::INTERNAL, + e.what()})); + } + return pr.get_future(); + } + + return fut; +} + +Result +FfiClient::subscribeDataTrack(std::uint64_t track_handle, + std::optional buffer_size) { + proto::FfiRequest req; + auto *msg = req.mutable_subscribe_data_track(); + msg->set_track_handle(track_handle); + auto *opts = msg->mutable_options(); + if (buffer_size.has_value()) { + opts->set_buffer_size(buffer_size.value()); + } + + try { + proto::FfiResponse resp = sendRequest(req); + if (!resp.has_subscribe_data_track()) { + return Result::failure(SubscribeDataTrackError{ + SubscribeDataTrackErrorCode::PROTOCOL_ERROR, + "FfiResponse missing subscribe_data_track"}); + } + if (!resp.subscribe_data_track().has_subscription()) { + return Result::failure(SubscribeDataTrackError{ + SubscribeDataTrackErrorCode::PROTOCOL_ERROR, + "FfiResponse subscribe_data_track missing subscription"}); + } + proto::OwnedDataTrackSubscription sub = + resp.subscribe_data_track().subscription(); + return Result::success(std::move(sub)); + } catch (...) { + try { + throw; + } catch (const std::exception &e) { + return Result::failure(SubscribeDataTrackError{ + SubscribeDataTrackErrorCode::INTERNAL, e.what()}); + } + } +} + std::future FfiClient::publishSipDtmfAsync( std::uint64_t local_participant_handle, std::uint32_t code, const std::string &digit, diff --git a/src/ffi_client.h b/src/ffi_client.h index 667100ea..275b2fb0 100644 --- a/src/ffi_client.h +++ b/src/ffi_client.h @@ -18,14 +18,19 @@ #define LIVEKIT_FFI_CLIENT_H #include +#include #include #include #include #include #include +#include #include #include +#include "data_track.pb.h" +#include "livekit/data_track_error.h" +#include "livekit/result.h" #include "livekit/stats.h" #include "room.pb.h" @@ -38,6 +43,8 @@ class FfiEvent; class FfiResponse; class FfiRequest; class OwnedTrackPublication; +class OwnedLocalDataTrack; +class OwnedDataTrackSubscription; class DataStream; } // namespace proto @@ -123,6 +130,15 @@ class FfiClient { const std::string &payload, std::optional response_timeout_ms = std::nullopt); + // Data Track APIs + std::future> + publishDataTrackAsync(std::uint64_t local_participant_handle, + const std::string &track_name); + + Result + subscribeDataTrack(std::uint64_t track_handle, + std::optional buffer_size = std::nullopt); + // Data stream functionalities std::future sendStreamHeaderAsync(std::uint64_t local_participant_handle, diff --git a/src/local_data_track.cpp b/src/local_data_track.cpp new file mode 100644 index 00000000..88603a66 --- /dev/null +++ b/src/local_data_track.cpp @@ -0,0 +1,111 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "livekit/local_data_track.h" + +#include "livekit/lk_log.h" + +#include "data_track.pb.h" +#include "ffi.pb.h" +#include "ffi_client.h" + +namespace livekit { + +namespace { + +LocalDataTrackTryPushError +makeInternalDataTrackTryPushError(const std::string &message) { + return LocalDataTrackTryPushError{LocalDataTrackTryPushErrorCode::INTERNAL, + message}; +} + +} // namespace + +LocalDataTrack::LocalDataTrack(const proto::OwnedLocalDataTrack &owned) + : handle_(static_cast(owned.handle().id())) { + const auto &pi = owned.info(); + info_.name = pi.name(); + info_.sid = pi.sid(); + info_.uses_e2ee = pi.uses_e2ee(); +} + +Result +LocalDataTrack::tryPush(const DataFrame &frame) { + if (!handle_.valid()) { + return Result::failure( + LocalDataTrackTryPushError{ + LocalDataTrackTryPushErrorCode::INVALID_HANDLE, + "LocalDataTrack::tryPush: invalid FFI handle"}); + } + + try { + proto::FfiRequest req; + auto *msg = req.mutable_local_data_track_try_push(); + msg->set_track_handle(static_cast(handle_.get())); + auto *pf = msg->mutable_frame(); + pf->set_payload(frame.payload.data(), frame.payload.size()); + if (frame.user_timestamp.has_value()) { + pf->set_user_timestamp(frame.user_timestamp.value()); + } + + proto::FfiResponse resp = FfiClient::instance().sendRequest(req); + const auto &r = resp.local_data_track_try_push(); + if (r.has_error()) { + return Result::failure( + LocalDataTrackTryPushError::fromProto(r.error())); + } + return Result::success(); + } catch (const std::exception &e) { + return Result::failure( + makeInternalDataTrackTryPushError(e.what())); + } +} + +Result +LocalDataTrack::tryPush(std::vector &&payload, + std::optional user_timestamp) { + DataFrame frame; + frame.payload = std::move(payload); + frame.user_timestamp = user_timestamp; + return tryPush(frame); +} + +bool LocalDataTrack::isPublished() const { + if (!handle_.valid()) { + return false; + } + + proto::FfiRequest req; + auto *msg = req.mutable_local_data_track_is_published(); + msg->set_track_handle(static_cast(handle_.get())); + + proto::FfiResponse resp = FfiClient::instance().sendRequest(req); + return resp.local_data_track_is_published().is_published(); +} + +void LocalDataTrack::unpublishDataTrack() { + if (!handle_.valid()) { + return; + } + + proto::FfiRequest req; + auto *msg = req.mutable_local_data_track_unpublish(); + msg->set_track_handle(static_cast(handle_.get())); + + (void)FfiClient::instance().sendRequest(req); +} + +} // namespace livekit diff --git a/src/local_participant.cpp b/src/local_participant.cpp index 8aea35ff..8fe31129 100644 --- a/src/local_participant.cpp +++ b/src/local_participant.cpp @@ -18,11 +18,13 @@ #include "livekit/ffi_handle.h" #include "livekit/local_audio_track.h" +#include "livekit/local_data_track.h" #include "livekit/local_track_publication.h" #include "livekit/local_video_track.h" #include "livekit/room_delegate.h" #include "livekit/track.h" +#include "data_track.pb.h" #include "ffi.pb.h" #include "ffi_client.h" #include "participant.pb.h" @@ -286,6 +288,40 @@ LocalParticipant::PublicationMap LocalParticipant::trackPublications() const { return out; } +Result, PublishDataTrackError> +LocalParticipant::publishDataTrack(const std::string &name) { + auto handle_id = ffiHandleId(); + if (handle_id == 0) { + return Result, + PublishDataTrackError>::failure(PublishDataTrackError{ + PublishDataTrackErrorCode::INVALID_HANDLE, + "LocalParticipant::publishDataTrack: invalid FFI " + "handle"}); + } + + auto fut = FfiClient::instance().publishDataTrackAsync( + static_cast(handle_id), name); + + auto result = fut.get(); + if (!result) { + return Result, + PublishDataTrackError>::failure(std::move(result).error()); + } + + return Result, PublishDataTrackError>:: + success( + std::shared_ptr(new LocalDataTrack(result.value()))); +} + +void LocalParticipant::unpublishDataTrack( + const std::shared_ptr &track) { + if (!track) { + return; + } + + track->unpublishDataTrack(); +} + std::string LocalParticipant::performRpc( const std::string &destination_identity, const std::string &method, const std::string &payload, const std::optional &response_timeout) { diff --git a/src/remote_data_track.cpp b/src/remote_data_track.cpp new file mode 100644 index 00000000..4bea7ca1 --- /dev/null +++ b/src/remote_data_track.cpp @@ -0,0 +1,77 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "livekit/remote_data_track.h" + +#include "data_track.pb.h" +#include "ffi.pb.h" +#include "ffi_client.h" + +#include + +namespace livekit { + +RemoteDataTrack::RemoteDataTrack(const proto::OwnedRemoteDataTrack &owned) + : handle_(static_cast(owned.handle().id())), + publisher_identity_(owned.publisher_identity()) { + const auto &pi = owned.info(); + info_.name = pi.name(); + info_.sid = pi.sid(); + info_.uses_e2ee = pi.uses_e2ee(); +} + +bool RemoteDataTrack::isPublished() const { + if (!handle_.valid()) { + return false; + } + + proto::FfiRequest req; + auto *msg = req.mutable_remote_data_track_is_published(); + msg->set_track_handle(static_cast(handle_.get())); + + proto::FfiResponse resp = FfiClient::instance().sendRequest(req); + return resp.remote_data_track_is_published().is_published(); +} + +Result, SubscribeDataTrackError> +RemoteDataTrack::subscribe(const DataTrackSubscription::Options &options) { + if (!handle_.valid()) { + return Result, + SubscribeDataTrackError>::failure(SubscribeDataTrackError{ + SubscribeDataTrackErrorCode::INVALID_HANDLE, + "RemoteDataTrack::subscribe: invalid FFI " + "handle"}); + } + + auto result = FfiClient::instance().subscribeDataTrack( + static_cast(handle_.get()), options.buffer_size); + if (!result) { + return Result, + SubscribeDataTrackError>::failure(std::move(result).error()); + } + + proto::OwnedDataTrackSubscription owned_sub = result.value(); + + FfiHandle sub_handle(static_cast(owned_sub.handle().id())); + + auto subscription = + std::shared_ptr(new DataTrackSubscription()); + subscription->init(std::move(sub_handle)); + return Result, + SubscribeDataTrackError>::success(std::move(subscription)); +} + +} // namespace livekit diff --git a/src/room.cpp b/src/room.cpp index ab7ab286..8b87b64b 100644 --- a/src/room.cpp +++ b/src/room.cpp @@ -18,15 +18,18 @@ #include "livekit/audio_stream.h" #include "livekit/e2ee.h" +#include "livekit/local_data_track.h" #include "livekit/local_participant.h" #include "livekit/local_track_publication.h" #include "livekit/remote_audio_track.h" +#include "livekit/remote_data_track.h" #include "livekit/remote_participant.h" #include "livekit/remote_track_publication.h" #include "livekit/remote_video_track.h" #include "livekit/room_delegate.h" #include "livekit/room_event_types.h" +#include "data_track.pb.h" #include "ffi.pb.h" #include "ffi_client.h" #include "livekit/lk_log.h" @@ -168,8 +171,8 @@ bool Room::Connect(const std::string &url, const std::string &token, std::unique_ptr new_e2ee_manager; if (options.encryption) { LK_LOG_INFO("creating E2eeManager"); - e2ee_manager_ = std::unique_ptr( - new E2EEManager(room_handle_->get(), options.encryption.value())); + new_e2ee_manager = std::unique_ptr( + new E2EEManager(new_room_handle->get(), options.encryption.value())); } // Publish all state atomically under lock @@ -227,6 +230,11 @@ Room::remoteParticipants() const { return out; } +E2EEManager *Room::e2eeManager() const { + std::lock_guard g(lock_); + return e2ee_manager_.get(); +} + void Room::registerTextStreamHandler(const std::string &topic, TextStreamHandler handler) { std::lock_guard g(lock_); @@ -273,6 +281,16 @@ void Room::setOnAudioFrameCallback(const std::string &participant_identity, } } +void Room::setOnAudioFrameCallback(const std::string &participant_identity, + const std::string &track_name, + AudioFrameCallback callback, + AudioStream::Options opts) { + if (subscription_thread_dispatcher_) { + subscription_thread_dispatcher_->setOnAudioFrameCallback( + participant_identity, track_name, std::move(callback), std::move(opts)); + } +} + void Room::setOnVideoFrameCallback(const std::string &participant_identity, TrackSource source, VideoFrameCallback callback, @@ -283,6 +301,26 @@ void Room::setOnVideoFrameCallback(const std::string &participant_identity, } } +void Room::setOnVideoFrameCallback(const std::string &participant_identity, + const std::string &track_name, + VideoFrameCallback callback, + VideoStream::Options opts) { + if (subscription_thread_dispatcher_) { + subscription_thread_dispatcher_->setOnVideoFrameCallback( + participant_identity, track_name, std::move(callback), std::move(opts)); + } +} + +void Room::setOnVideoFrameEventCallback(const std::string &participant_identity, + TrackSource source, + VideoFrameEventCallback callback, + VideoStream::Options opts) { + if (subscription_thread_dispatcher_) { + subscription_thread_dispatcher_->setOnVideoFrameEventCallback( + participant_identity, source, std::move(callback), std::move(opts)); + } +} + void Room::clearOnAudioFrameCallback(const std::string &participant_identity, TrackSource source) { if (subscription_thread_dispatcher_) { @@ -291,6 +329,14 @@ void Room::clearOnAudioFrameCallback(const std::string &participant_identity, } } +void Room::clearOnAudioFrameCallback(const std::string &participant_identity, + const std::string &track_name) { + if (subscription_thread_dispatcher_) { + subscription_thread_dispatcher_->clearOnAudioFrameCallback( + participant_identity, track_name); + } +} + void Room::clearOnVideoFrameCallback(const std::string &participant_identity, TrackSource source) { if (subscription_thread_dispatcher_) { @@ -299,6 +345,31 @@ void Room::clearOnVideoFrameCallback(const std::string &participant_identity, } } +void Room::clearOnVideoFrameCallback(const std::string &participant_identity, + const std::string &track_name) { + if (subscription_thread_dispatcher_) { + subscription_thread_dispatcher_->clearOnVideoFrameCallback( + participant_identity, track_name); + } +} + +DataFrameCallbackId +Room::addOnDataFrameCallback(const std::string &participant_identity, + const std::string &track_name, + DataFrameCallback callback) { + if (subscription_thread_dispatcher_) { + return subscription_thread_dispatcher_->addOnDataFrameCallback( + participant_identity, track_name, std::move(callback)); + } + return std::numeric_limits::max(); +} + +void Room::removeOnDataFrameCallback(DataFrameCallbackId id) { + if (subscription_thread_dispatcher_) { + subscription_thread_dispatcher_->removeOnDataFrameCallback(id); + } +} + void Room::OnEvent(const FfiEvent &event) { // Take a snapshot of the delegate under lock, but do NOT call it under the // lock. @@ -586,7 +657,8 @@ void Room::OnEvent(const FfiEvent &event) { if (subscription_thread_dispatcher_ && remote_track && rpublication) { subscription_thread_dispatcher_->handleTrackSubscribed( - identity, rpublication->source(), remote_track); + identity, rpublication->source(), rpublication->name(), + remote_track); } break; } @@ -630,8 +702,9 @@ void Room::OnEvent(const FfiEvent &event) { if (subscription_thread_dispatcher_ && unsub_source != TrackSource::SOURCE_UNKNOWN) { - subscription_thread_dispatcher_->handleTrackUnsubscribed(unsub_identity, - unsub_source); + subscription_thread_dispatcher_->handleTrackUnsubscribed( + unsub_identity, unsub_source, + ev.publication ? ev.publication->name() : ""); } break; } @@ -656,6 +729,36 @@ void Room::OnEvent(const FfiEvent &event) { } break; } + case proto::RoomEvent::kDataTrackPublished: { + const auto &rdtp = re.data_track_published(); + auto remote_track = + std::shared_ptr(new RemoteDataTrack(rdtp.track())); + + if (subscription_thread_dispatcher_) { + subscription_thread_dispatcher_->handleDataTrackPublished(remote_track); + } + + DataTrackPublishedEvent ev; + ev.track = remote_track; + if (delegate_snapshot) { + delegate_snapshot->onDataTrackPublished(*this, ev); + } + break; + } + case proto::RoomEvent::kDataTrackUnpublished: { + const auto &dtu = re.data_track_unpublished(); + + if (subscription_thread_dispatcher_) { + subscription_thread_dispatcher_->handleDataTrackUnpublished(dtu.sid()); + } + + DataTrackUnpublishedEvent ev; + ev.sid = dtu.sid(); + if (delegate_snapshot) { + delegate_snapshot->onDataTrackUnpublished(*this, ev); + } + break; + } case proto::RoomEvent::kTrackMuted: { TrackMutedEvent ev; bool success = false; diff --git a/src/room_proto_converter.cpp b/src/room_proto_converter.cpp index 989d05bf..b3d5aec5 100644 --- a/src/room_proto_converter.cpp +++ b/src/room_proto_converter.cpp @@ -22,6 +22,40 @@ namespace livekit { +namespace { + +std::vector +toProto(const PacketTrailerFeatures &features) { + std::vector out; + if (features.user_timestamp) { + out.push_back(proto::PacketTrailerFeature::PTF_USER_TIMESTAMP); + } + if (features.frame_id) { + out.push_back(proto::PacketTrailerFeature::PTF_FRAME_ID); + } + return out; +} + +PacketTrailerFeatures +fromProto(const google::protobuf::RepeatedField &features) { + PacketTrailerFeatures out; + for (int feature : features) { + switch (static_cast(feature)) { + case proto::PacketTrailerFeature::PTF_USER_TIMESTAMP: + out.user_timestamp = true; + break; + case proto::PacketTrailerFeature::PTF_FRAME_ID: + out.frame_id = true; + break; + default: + break; + } + } + return out; +} + +} // namespace + // --------- enum conversions --------- ConnectionQuality toConnectionQuality(proto::ConnectionQuality in) { @@ -340,6 +374,9 @@ proto::TrackPublishOptions toProto(const TrackPublishOptions &in) { if (in.preconnect_buffer) { msg.set_preconnect_buffer(*in.preconnect_buffer); } + for (proto::PacketTrailerFeature feature : toProto(in.packet_trailer_features)) { + msg.add_packet_trailer_features(feature); + } return msg; } @@ -372,6 +409,7 @@ TrackPublishOptions fromProto(const proto::TrackPublishOptions &in) { if (in.has_preconnect_buffer()) { out.preconnect_buffer = in.preconnect_buffer(); } + out.packet_trailer_features = fromProto(in.packet_trailer_features()); return out; } diff --git a/src/subscription_thread_dispatcher.cpp b/src/subscription_thread_dispatcher.cpp index a7f9a2a7..2f8e60da 100644 --- a/src/subscription_thread_dispatcher.cpp +++ b/src/subscription_thread_dispatcher.cpp @@ -16,7 +16,10 @@ #include "livekit/subscription_thread_dispatcher.h" +#include "livekit/data_frame.h" +#include "livekit/data_track_subscription.h" #include "livekit/lk_log.h" +#include "livekit/remote_data_track.h" #include "livekit/track.h" #include @@ -40,7 +43,8 @@ const char *trackKindName(TrackKind kind) { } // namespace -SubscriptionThreadDispatcher::SubscriptionThreadDispatcher() = default; +SubscriptionThreadDispatcher::SubscriptionThreadDispatcher() + : next_data_callback_id_(1) {} SubscriptionThreadDispatcher::~SubscriptionThreadDispatcher() { LK_LOG_DEBUG("Destroying SubscriptionThreadDispatcher"); @@ -50,7 +54,7 @@ SubscriptionThreadDispatcher::~SubscriptionThreadDispatcher() { void SubscriptionThreadDispatcher::setOnAudioFrameCallback( const std::string &participant_identity, TrackSource source, AudioFrameCallback callback, AudioStream::Options opts) { - CallbackKey key{participant_identity, source}; + CallbackKey key{participant_identity, source, ""}; std::lock_guard lock(lock_); const bool replacing = audio_callbacks_.find(key) != audio_callbacks_.end(); audio_callbacks_[key] = @@ -61,14 +65,64 @@ void SubscriptionThreadDispatcher::setOnAudioFrameCallback( audio_callbacks_.size()); } +void SubscriptionThreadDispatcher::setOnAudioFrameCallback( + const std::string &participant_identity, const std::string &track_name, + AudioFrameCallback callback, AudioStream::Options opts) { + CallbackKey key{participant_identity, TrackSource::SOURCE_UNKNOWN, + track_name}; + std::lock_guard lock(lock_); + const bool replacing = audio_callbacks_.find(key) != audio_callbacks_.end(); + audio_callbacks_[key] = + RegisteredAudioCallback{std::move(callback), std::move(opts)}; + LK_LOG_DEBUG( + "Registered audio frame callback for participant={} track_name={} " + "replacing_existing={} total_audio_callbacks={}", + participant_identity, track_name, replacing, audio_callbacks_.size()); +} + void SubscriptionThreadDispatcher::setOnVideoFrameCallback( const std::string &participant_identity, TrackSource source, VideoFrameCallback callback, VideoStream::Options opts) { - CallbackKey key{participant_identity, source}; + CallbackKey key{participant_identity, source, ""}; std::lock_guard lock(lock_); const bool replacing = video_callbacks_.find(key) != video_callbacks_.end(); - video_callbacks_[key] = - RegisteredVideoCallback{std::move(callback), std::move(opts)}; + video_callbacks_[key] = RegisteredVideoCallback{ + std::move(callback), + VideoFrameEventCallback{}, + std::move(opts), + }; + LK_LOG_DEBUG("Registered legacy video frame callback for participant={} " + "source={} replacing_existing={} total_video_callbacks={}", + participant_identity, static_cast(source), replacing, + video_callbacks_.size()); +} + +void SubscriptionThreadDispatcher::setOnVideoFrameCallback( + const std::string &participant_identity, const std::string &track_name, + VideoFrameCallback callback, VideoStream::Options opts) { + CallbackKey key{participant_identity, TrackSource::SOURCE_UNKNOWN, + track_name}; + std::lock_guard lock(lock_); + const bool replacing = video_callbacks_.find(key) != video_callbacks_.end(); + video_callbacks_[key] = RegisteredVideoCallback{ + std::move(callback), VideoFrameEventCallback{}, std::move(opts)}; + LK_LOG_DEBUG( + "Registered video frame callback for participant={} track_name={} " + "replacing_existing={} total_video_callbacks={}", + participant_identity, track_name, replacing, video_callbacks_.size()); +} + +void SubscriptionThreadDispatcher::setOnVideoFrameEventCallback( + const std::string &participant_identity, TrackSource source, + VideoFrameEventCallback callback, VideoStream::Options opts) { + CallbackKey key{participant_identity, source, ""}; + std::lock_guard lock(lock_); + const bool replacing = video_callbacks_.find(key) != video_callbacks_.end(); + video_callbacks_[key] = RegisteredVideoCallback{ + VideoFrameCallback{}, + std::move(callback), + std::move(opts), + }; LK_LOG_DEBUG("Registered video frame callback for participant={} source={} " "replacing_existing={} total_video_callbacks={}", participant_identity, static_cast(source), replacing, @@ -77,7 +131,7 @@ void SubscriptionThreadDispatcher::setOnVideoFrameCallback( void SubscriptionThreadDispatcher::clearOnAudioFrameCallback( const std::string &participant_identity, TrackSource source) { - CallbackKey key{participant_identity, source}; + CallbackKey key{participant_identity, source, ""}; std::thread old_thread; bool removed_callback = false; { @@ -95,9 +149,30 @@ void SubscriptionThreadDispatcher::clearOnAudioFrameCallback( } } +void SubscriptionThreadDispatcher::clearOnAudioFrameCallback( + const std::string &participant_identity, const std::string &track_name) { + CallbackKey key{participant_identity, TrackSource::SOURCE_UNKNOWN, + track_name}; + std::thread old_thread; + bool removed_callback = false; + { + std::lock_guard lock(lock_); + removed_callback = audio_callbacks_.erase(key) > 0; + old_thread = extractReaderThreadLocked(key); + LK_LOG_DEBUG( + "Clearing audio frame callback for participant={} track_name={} " + "removed_callback={} stopped_reader={} remaining_audio_callbacks={}", + participant_identity, track_name, removed_callback, + old_thread.joinable(), audio_callbacks_.size()); + } + if (old_thread.joinable()) { + old_thread.join(); + } +} + void SubscriptionThreadDispatcher::clearOnVideoFrameCallback( const std::string &participant_identity, TrackSource source) { - CallbackKey key{participant_identity, source}; + CallbackKey key{participant_identity, source, ""}; std::thread old_thread; bool removed_callback = false; { @@ -115,9 +190,30 @@ void SubscriptionThreadDispatcher::clearOnVideoFrameCallback( } } +void SubscriptionThreadDispatcher::clearOnVideoFrameCallback( + const std::string &participant_identity, const std::string &track_name) { + CallbackKey key{participant_identity, TrackSource::SOURCE_UNKNOWN, + track_name}; + std::thread old_thread; + bool removed_callback = false; + { + std::lock_guard lock(lock_); + removed_callback = video_callbacks_.erase(key) > 0; + old_thread = extractReaderThreadLocked(key); + LK_LOG_DEBUG( + "Clearing video frame callback for participant={} track_name={} " + "removed_callback={} stopped_reader={} remaining_video_callbacks={}", + participant_identity, track_name, removed_callback, + old_thread.joinable(), video_callbacks_.size()); + } + if (old_thread.joinable()) { + old_thread.join(); + } +} + void SubscriptionThreadDispatcher::handleTrackSubscribed( const std::string &participant_identity, TrackSource source, - const std::shared_ptr &track) { + const std::string &track_name, const std::shared_ptr &track) { if (!track) { LK_LOG_WARN( "Ignoring subscribed track dispatch for participant={} source={} " @@ -130,10 +226,19 @@ void SubscriptionThreadDispatcher::handleTrackSubscribed( participant_identity, static_cast(source), trackKindName(track->kind())); - CallbackKey key{participant_identity, source}; + CallbackKey key{participant_identity, TrackSource::SOURCE_UNKNOWN, + track_name}; + CallbackKey fallback_key{participant_identity, source, ""}; std::thread old_thread; { std::lock_guard lock(lock_); + if (track->kind() == TrackKind::KIND_AUDIO && + audio_callbacks_.find(key) == audio_callbacks_.end()) { + key = fallback_key; + } else if (track->kind() == TrackKind::KIND_VIDEO && + video_callbacks_.find(key) == video_callbacks_.end()) { + key = fallback_key; + } old_thread = startReaderLocked(key, track); } if (old_thread.joinable()) { @@ -142,39 +247,151 @@ void SubscriptionThreadDispatcher::handleTrackSubscribed( } void SubscriptionThreadDispatcher::handleTrackUnsubscribed( - const std::string &participant_identity, TrackSource source) { - CallbackKey key{participant_identity, source}; + const std::string &participant_identity, TrackSource source, + const std::string &track_name) { + CallbackKey key{participant_identity, TrackSource::SOURCE_UNKNOWN, + track_name}; + CallbackKey fallback_key{participant_identity, source, ""}; std::thread old_thread; + std::thread fallback_old_thread; { std::lock_guard lock(lock_); old_thread = extractReaderThreadLocked(key); + fallback_old_thread = extractReaderThreadLocked(fallback_key); LK_LOG_DEBUG("Handling unsubscribed track for participant={} source={} " - "stopped_reader={}", - participant_identity, static_cast(source), - old_thread.joinable()); + "track_name={} stopped_reader={} fallback_stopped_reader={}", + participant_identity, static_cast(source), track_name, + old_thread.joinable(), fallback_old_thread.joinable()); + } + if (old_thread.joinable()) { + old_thread.join(); + } + if (fallback_old_thread.joinable()) { + fallback_old_thread.join(); + } +} + +// ------------------------------------------------------------------- +// Data track callback registration +// ------------------------------------------------------------------- + +DataFrameCallbackId SubscriptionThreadDispatcher::addOnDataFrameCallback( + const std::string &participant_identity, const std::string &track_name, + DataFrameCallback callback) { + std::thread old_thread; + DataFrameCallbackId id; + { + std::lock_guard lock(lock_); + id = next_data_callback_id_++; + DataCallbackKey key{participant_identity, track_name}; + data_callbacks_[id] = RegisteredDataCallback{key, std::move(callback)}; + + auto track_it = remote_data_tracks_.find(key); + if (track_it != remote_data_tracks_.end()) { + old_thread = startDataReaderLocked(id, key, track_it->second, + data_callbacks_[id].callback); + } } if (old_thread.joinable()) { old_thread.join(); } + return id; +} + +void SubscriptionThreadDispatcher::removeOnDataFrameCallback( + DataFrameCallbackId id) { + std::thread old_thread; + { + std::lock_guard lock(lock_); + data_callbacks_.erase(id); + old_thread = extractDataReaderThreadLocked(id); + } + if (old_thread.joinable()) { + old_thread.join(); + } +} + +void SubscriptionThreadDispatcher::handleDataTrackPublished( + const std::shared_ptr &track) { + if (!track) { + LK_LOG_WARN("handleDataTrackPublished called with null track"); + return; + } + + LK_LOG_INFO("Handling data track published: \"{}\" from \"{}\" (sid={})", + track->info().name, track->publisherIdentity(), + track->info().sid); + + std::vector old_threads; + { + std::lock_guard lock(lock_); + DataCallbackKey key{track->publisherIdentity(), track->info().name}; + remote_data_tracks_[key] = track; + + for (auto &[id, reg] : data_callbacks_) { + if (reg.key == key) { + auto t = startDataReaderLocked(id, key, track, reg.callback); + if (t.joinable()) { + old_threads.push_back(std::move(t)); + } + } + } + } + for (auto &t : old_threads) { + t.join(); + } +} + +void SubscriptionThreadDispatcher::handleDataTrackUnpublished( + const std::string &sid) { + LK_LOG_INFO("Handling data track unpublished: sid={}", sid); + + std::vector old_threads; + { + std::lock_guard lock(lock_); + for (auto it = active_data_readers_.begin(); + it != active_data_readers_.end();) { + auto &reader = it->second; + if (reader->remote_track && reader->remote_track->info().sid == sid) { + { + std::lock_guard sub_guard(reader->sub_mutex); + if (reader->subscription) { + reader->subscription->close(); + } + } + if (reader->thread.joinable()) { + old_threads.push_back(std::move(reader->thread)); + } + it = active_data_readers_.erase(it); + } else { + ++it; + } + } + for (auto it = remote_data_tracks_.begin(); it != remote_data_tracks_.end(); + ++it) { + if (it->second && it->second->info().sid == sid) { + remote_data_tracks_.erase(it); + break; + } + } + } + for (auto &t : old_threads) { + t.join(); + } } void SubscriptionThreadDispatcher::stopAll() { std::vector threads; - std::size_t active_reader_count = 0; - std::size_t audio_callback_count = 0; - std::size_t video_callback_count = 0; { std::lock_guard lock(lock_); - active_reader_count = active_readers_.size(); - audio_callback_count = audio_callbacks_.size(); - video_callback_count = video_callbacks_.size(); LK_LOG_DEBUG("Stopping all subscription readers active_readers={} " - "audio_callbacks={} video_callbacks={}", - active_reader_count, audio_callback_count, - video_callback_count); + "active_data_readers={} audio_callbacks={} " + "video_callbacks={} data_callbacks={}", + active_readers_.size(), active_data_readers_.size(), + audio_callbacks_.size(), video_callbacks_.size(), + data_callbacks_.size()); + for (auto &[key, reader] : active_readers_) { - LK_LOG_TRACE("Closing active reader for participant={} source={}", - key.participant_identity, static_cast(key.source)); if (reader.audio_stream) { reader.audio_stream->close(); } @@ -188,6 +405,21 @@ void SubscriptionThreadDispatcher::stopAll() { active_readers_.clear(); audio_callbacks_.clear(); video_callbacks_.clear(); + + for (auto &[id, reader] : active_data_readers_) { + { + std::lock_guard sub_guard(reader->sub_mutex); + if (reader->subscription) { + reader->subscription->close(); + } + } + if (reader->thread.joinable()) { + threads.push_back(std::move(reader->thread)); + } + } + active_data_readers_.clear(); + data_callbacks_.clear(); + remote_data_tracks_.clear(); } for (auto &thread : threads) { thread.join(); @@ -199,13 +431,17 @@ std::thread SubscriptionThreadDispatcher::extractReaderThreadLocked( const CallbackKey &key) { auto it = active_readers_.find(key); if (it == active_readers_.end()) { - LK_LOG_TRACE("No active reader to extract for participant={} source={}", - key.participant_identity, static_cast(key.source)); + LK_LOG_TRACE("No active reader to extract for participant={} source={} " + "track_name={}", + key.participant_identity, static_cast(key.source), + key.track_name); return {}; } - LK_LOG_DEBUG("Extracting active reader for participant={} source={}", - key.participant_identity, static_cast(key.source)); + LK_LOG_DEBUG("Extracting active reader for participant={} source={} " + "track_name={}", + key.participant_identity, static_cast(key.source), + key.track_name); ActiveReader reader = std::move(it->second); active_readers_.erase(it); @@ -239,8 +475,7 @@ std::thread SubscriptionThreadDispatcher::startReaderLocked( key.participant_identity, static_cast(key.source)); return {}; } - return startVideoReaderLocked(key, track, it->second.callback, - it->second.options); + return startVideoReaderLocked(key, track, it->second); } if (track->kind() == TrackKind::KIND_UNKNOWN) { LK_LOG_WARN( @@ -310,7 +545,7 @@ std::thread SubscriptionThreadDispatcher::startAudioReaderLocked( std::thread SubscriptionThreadDispatcher::startVideoReaderLocked( const CallbackKey &key, const std::shared_ptr &track, - VideoFrameCallback cb, const VideoStream::Options &opts) { + const RegisteredVideoCallback &callback) { LK_LOG_DEBUG("Starting video reader for participant={} source={}", key.participant_identity, static_cast(key.source)); auto old_thread = extractReaderThreadLocked(key); @@ -324,7 +559,7 @@ std::thread SubscriptionThreadDispatcher::startVideoReaderLocked( return old_thread; } - auto stream = VideoStream::fromTrack(track, opts); + auto stream = VideoStream::fromTrack(track, callback.options); if (!stream) { LK_LOG_ERROR("Failed to create VideoStream for {} source={}", key.participant_identity, static_cast(key.source)); @@ -334,16 +569,22 @@ std::thread SubscriptionThreadDispatcher::startVideoReaderLocked( ActiveReader reader; reader.video_stream = stream; auto stream_copy = stream; + auto legacy_cb = callback.legacy_callback; + auto event_cb = callback.event_callback; const std::string participant_identity = key.participant_identity; const TrackSource source = key.source; - reader.thread = - std::thread([stream_copy, cb, participant_identity, source]() { + reader.thread = std::thread( + [stream_copy, legacy_cb, event_cb, participant_identity, source]() { LK_LOG_DEBUG("Video reader thread started for participant={} source={}", participant_identity, static_cast(source)); VideoFrameEvent ev; while (stream_copy->read(ev)) { try { - cb(ev.frame, ev.timestamp_us); + if (event_cb) { + event_cb(ev); + } else if (legacy_cb) { + legacy_cb(ev.frame, ev.timestamp_us); + } } catch (const std::exception &e) { LK_LOG_ERROR("Video frame callback exception: {}", e.what()); } @@ -359,4 +600,106 @@ std::thread SubscriptionThreadDispatcher::startVideoReaderLocked( return old_thread; } +// ------------------------------------------------------------------- +// Data track reader helpers +// ------------------------------------------------------------------- + +std::thread SubscriptionThreadDispatcher::extractDataReaderThreadLocked( + DataFrameCallbackId id) { + auto it = active_data_readers_.find(id); + if (it == active_data_readers_.end()) { + return {}; + } + auto reader = std::move(it->second); + active_data_readers_.erase(it); + { + std::lock_guard guard(reader->sub_mutex); + if (reader->subscription) { + reader->subscription->close(); + } + } + return std::move(reader->thread); +} + +std::thread SubscriptionThreadDispatcher::extractDataReaderThreadLocked( + const DataCallbackKey &key) { + for (auto it = active_data_readers_.begin(); it != active_data_readers_.end(); + ++it) { + if (it->second && it->second->remote_track && + it->second->remote_track->publisherIdentity() == + key.participant_identity && + it->second->remote_track->info().name == key.track_name) { + auto reader = std::move(it->second); + active_data_readers_.erase(it); + { + std::lock_guard guard(reader->sub_mutex); + if (reader->subscription) { + reader->subscription->close(); + } + } + return std::move(reader->thread); + } + } + return {}; +} + +std::thread SubscriptionThreadDispatcher::startDataReaderLocked( + DataFrameCallbackId id, const DataCallbackKey &key, + const std::shared_ptr &track, DataFrameCallback cb) { + auto old_thread = extractDataReaderThreadLocked(id); + + int total_active = static_cast(active_readers_.size()) + + static_cast(active_data_readers_.size()); + if (total_active >= kMaxActiveReaders) { + LK_LOG_ERROR("Cannot start data reader for {} track={}: active reader " + "limit ({}) reached", + key.participant_identity, key.track_name, kMaxActiveReaders); + return old_thread; + } + + LK_LOG_INFO("Starting data reader for \"{}\" track=\"{}\"", + key.participant_identity, key.track_name); + + auto reader = std::make_shared(); + reader->remote_track = track; + auto identity = key.participant_identity; + auto track_name = key.track_name; + reader->thread = std::thread([reader, track, cb, identity, track_name]() { + LK_LOG_INFO("Data reader thread: subscribing to \"{}\" track=\"{}\"", + identity, track_name); + std::shared_ptr subscription; + auto subscribe_result = track->subscribe(); + if (!subscribe_result) { + const auto &error = subscribe_result.error(); + LK_LOG_ERROR( + "Failed to subscribe to data track \"{}\" from \"{}\": code={} " + "message={}", + track_name, identity, static_cast(error.code), + error.message); + return; + } + subscription = subscribe_result.value(); + LK_LOG_INFO("Data reader thread: subscribed to \"{}\" track=\"{}\"", + identity, track_name); + + { + std::lock_guard guard(reader->sub_mutex); + reader->subscription = subscription; + } + + DataFrame frame; + while (subscription->read(frame)) { + try { + cb(frame.payload, frame.user_timestamp); + } catch (const std::exception &e) { + LK_LOG_ERROR("Data frame callback exception: {}", e.what()); + } + } + LK_LOG_INFO("Data reader thread exiting for \"{}\" track=\"{}\"", identity, + track_name); + }); + active_data_readers_[id] = reader; + return old_thread; +} + } // namespace livekit diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 6ff68ef9..18e8f73a 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -49,10 +49,22 @@ if(INTEGRATION_TEST_SOURCES) PRIVATE ${LIVEKIT_ROOT_DIR}/include ${LIVEKIT_ROOT_DIR}/src + ${LIVEKIT_BINARY_DIR}/generated + ${LIVEKIT_BINARY_DIR}/_deps/livekit_protobuf-src/src + ${LIVEKIT_BINARY_DIR}/_deps/livekit_abseil-src ) + if(TARGET absl::base) + get_target_property(_livekit_test_absl_inc absl::base INTERFACE_INCLUDE_DIRECTORIES) + if(_livekit_test_absl_inc) + target_include_directories(livekit_integration_tests PRIVATE + ${_livekit_test_absl_inc} + ) + endif() + endif() target_compile_definitions(livekit_integration_tests PRIVATE + LIVEKIT_TEST_ACCESS LIVEKIT_ROOT_DIR="${LIVEKIT_ROOT_DIR}" SPDLOG_ACTIVE_LEVEL=${_SPDLOG_ACTIVE_LEVEL} ) @@ -123,6 +135,9 @@ if(STRESS_TEST_SOURCES) PRIVATE ${LIVEKIT_ROOT_DIR}/include ${LIVEKIT_ROOT_DIR}/src + ${LIVEKIT_BINARY_DIR}/generated + ${LIVEKIT_BINARY_DIR}/_deps/livekit_protobuf-src/src + ${LIVEKIT_BINARY_DIR}/_deps/livekit_abseil-src ) # Copy shared libraries to test executable directory diff --git a/src/tests/common/test_common.h b/src/tests/common/test_common.h index 0298e1f6..3133adc9 100644 --- a/src/tests/common/test_common.h +++ b/src/tests/common/test_common.h @@ -17,6 +17,7 @@ #pragma once #include +#include #include #include #include @@ -25,8 +26,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -46,6 +49,9 @@ constexpr int kDefaultTestIterations = 10; // Default stress test duration in seconds constexpr int kDefaultStressDurationSeconds = 600; // 10 minutes +// Local SFU URL used by end-to-end data track tests. +constexpr char kLocalTestLiveKitUrl[] = "ws://localhost:7880"; + // ============================================================================= // Common Test Configuration // ============================================================================= @@ -97,6 +103,11 @@ struct TestConfig { } }; +struct TestRoomConnectionOptions { + RoomOptions room_options; + RoomDelegate *delegate = nullptr; +}; + // ============================================================================= // Utility Functions // ============================================================================= @@ -121,6 +132,113 @@ inline bool waitForParticipant(Room *room, const std::string &identity, return false; } +inline std::array getDataTrackTestTokens() { + const char *token_a = std::getenv("LK_TOKEN_TEST_A"); + if (token_a == nullptr || std::string(token_a).empty()) { + throw std::runtime_error( + "LK_TOKEN_TEST_A must be present and non-empty for data track E2E " + "tests"); + } + + const char *token_b = std::getenv("LK_TOKEN_TEST_B"); + if (token_b == nullptr || std::string(token_b).empty()) { + throw std::runtime_error( + "LK_TOKEN_TEST_B must be present and non-empty for data track E2E " + "tests"); + } + + return {token_a, token_b}; +} + +inline void +waitForParticipantVisibility(const std::vector> &rooms, + std::chrono::milliseconds timeout = 5s) { + std::vector participant_identities; + participant_identities.reserve(rooms.size()); + for (const auto &room : rooms) { + if (!room || room->localParticipant() == nullptr) { + throw std::runtime_error( + "Test room is missing a local participant after connect"); + } + participant_identities.push_back(room->localParticipant()->identity()); + } + + auto start = std::chrono::steady_clock::now(); + while (std::chrono::steady_clock::now() - start < timeout) { + bool all_visible = true; + for (size_t i = 0; i < rooms.size(); ++i) { + const auto &room = rooms[i]; + if (!room || room->localParticipant() == nullptr) { + throw std::runtime_error( + "Test room is missing a local participant after connect"); + } + + for (size_t j = 0; j < participant_identities.size(); ++j) { + if (i == j) { + continue; + } + + if (room->remoteParticipant(participant_identities[j]) == nullptr) { + all_visible = false; + break; + } + } + + if (!all_visible) { + break; + } + } + + if (all_visible) { + return; + } + + std::this_thread::sleep_for(10ms); + } + + throw std::runtime_error("Not all test participants became visible"); +} + +inline std::vector> +testRooms(const std::vector &room_configs) { + if (room_configs.empty()) { + throw std::invalid_argument("testRooms requires at least one room"); + } + + if (room_configs.size() > 2) { + throw std::invalid_argument( + "testRooms supports at most two rooms with LK_TOKEN_TEST_A/B"); + } + + auto tokens = getDataTrackTestTokens(); + + std::vector> rooms; + rooms.reserve(room_configs.size()); + + for (size_t i = 0; i < room_configs.size(); ++i) { + auto room = std::make_unique(); + if (room_configs[i].delegate != nullptr) { + room->setDelegate(room_configs[i].delegate); + } + + if (!room->Connect(kLocalTestLiveKitUrl, tokens[i], + room_configs[i].room_options)) { + throw std::runtime_error("Failed to connect test room " + + std::to_string(i)); + } + + rooms.push_back(std::move(room)); + } + + waitForParticipantVisibility(rooms); + return rooms; +} + +inline std::vector> testRooms(size_t count) { + std::vector room_configs(count); + return testRooms(room_configs); +} + // ============================================================================= // Statistics Collection // ============================================================================= diff --git a/src/tests/integration/test_data_track.cpp b/src/tests/integration/test_data_track.cpp new file mode 100644 index 00000000..dd8ee8e9 --- /dev/null +++ b/src/tests/integration/test_data_track.cpp @@ -0,0 +1,848 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This test is used to verify that data tracks are published and received +// correctly. It is the same implementation as the rust +// client-sdk-rust/livekit/tests/data_track_test.rs test. To run this test, run +// a local SFU, set credentials examples/tokens/set_data_track_test_tokens.bash, +// and run: +// ./build-debug/bin/livekit_integration_tests + +#include "../common/test_common.h" + +#include "ffi_client.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace livekit { +namespace test { + +using namespace std::chrono_literals; + +namespace { + +constexpr char kTrackNamePrefix[] = "data_track_e2e"; +constexpr auto kPublishDuration = 5s; +constexpr auto kTrackWaitTimeout = 10s; +constexpr auto kReadTimeout = 30s; +constexpr auto kPollingInterval = 10ms; +constexpr float kMinimumReceivedPercent = 0.95f; +constexpr int kResubscribeIterations = 10; +constexpr int kPublishManyTrackCount = 256; +constexpr auto kPublishManyTimeout = 5s; +constexpr std::size_t kLargeFramePayloadBytes = 196608; +constexpr char kE2EESharedSecret[] = "password"; +constexpr int kE2EEFrameCount = 5; + +std::string makeTrackName(const std::string &suffix) { + return std::string(kTrackNamePrefix) + "_" + suffix + "_" + + std::to_string(getTimestampUs()); +} + +std::vector e2eeSharedKey() { + return std::vector( + kE2EESharedSecret, kE2EESharedSecret + sizeof(kE2EESharedSecret) - 1); +} + +std::size_t parseTestTrackIndex(const std::string &track_name) { + constexpr char kPrefix[] = "test_"; + if (track_name.rfind(kPrefix, 0) != 0) { + throw std::runtime_error("Unexpected test track name: " + track_name); + } + return static_cast( + std::stoul(track_name.substr(sizeof(kPrefix) - 1))); +} + +E2EEOptions makeE2EEOptions() { + E2EEOptions options; + options.key_provider_options.shared_key = e2eeSharedKey(); + return options; +} + +std::vector +encryptedRoomConfigs(RoomDelegate *subscriber_delegate) { + std::vector room_configs(2); + room_configs[0].room_options.encryption = makeE2EEOptions(); + room_configs[1].room_options.encryption = makeE2EEOptions(); + room_configs[1].delegate = subscriber_delegate; + return room_configs; +} + +template +bool waitForCondition(Predicate &&predicate, std::chrono::milliseconds timeout, + std::chrono::milliseconds interval = kPollingInterval) { + auto start = std::chrono::steady_clock::now(); + while (std::chrono::steady_clock::now() - start < timeout) { + if (predicate()) { + return true; + } + std::this_thread::sleep_for(interval); + } + return false; +} + +template +std::string describeDataTrackError(const Error &error) { + return "code=" + std::to_string(static_cast(error.code)) + + " message=" + error.message; +} + +std::shared_ptr +requirePublishedTrack(LocalParticipant *participant, const std::string &name) { + auto result = participant->publishDataTrack(name); + if (!result) { + throw std::runtime_error("Failed to publish data track: " + + describeDataTrackError(result.error())); + } + return result.value(); +} + +std::shared_ptr +requireSubscription(const std::shared_ptr &track) { + auto result = track->subscribe(); + if (!result) { + throw std::runtime_error("Failed to subscribe to data track: " + + describeDataTrackError(result.error())); + } + return result.value(); +} + +void requirePushSuccess(const Result &result, + const std::string &context) { + if (!result) { + throw std::runtime_error(context + ": " + + describeDataTrackError(result.error())); + } +} + +class DataTrackPublishedDelegate : public RoomDelegate { +public: + void onDataTrackPublished(Room &, + const DataTrackPublishedEvent &event) override { + if (!event.track) { + return; + } + + std::lock_guard lock(mutex_); + tracks_.push_back(event.track); + cv_.notify_all(); + } + + std::shared_ptr + waitForTrack(std::chrono::milliseconds timeout) { + std::unique_lock lock(mutex_); + if (!cv_.wait_for(lock, timeout, [this] { return !tracks_.empty(); })) { + return nullptr; + } + return tracks_.front(); + } + + std::vector> + waitForTracks(std::size_t count, std::chrono::milliseconds timeout) { + std::unique_lock lock(mutex_); + if (!cv_.wait_for(lock, timeout, + [this, count] { return tracks_.size() >= count; })) { + return {}; + } + return {tracks_.begin(), + tracks_.begin() + static_cast(count)}; + } + +private: + std::mutex mutex_; + std::condition_variable cv_; + std::vector> tracks_; +}; + +DataFrame +readFrameWithTimeout(const std::shared_ptr &subscription, + std::chrono::milliseconds timeout) { + std::promise frame_promise; + auto future = frame_promise.get_future(); + + std::thread reader([subscription, + promise = std::move(frame_promise)]() mutable { + try { + DataFrame frame; + if (!subscription->read(frame)) { + throw std::runtime_error("Subscription ended before a frame arrived"); + } + promise.set_value(std::move(frame)); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }); + + if (future.wait_for(timeout) != std::future_status::ready) { + subscription->close(); + } + + reader.join(); + return future.get(); +} + +} // namespace + +class DataTrackE2ETest : public LiveKitTestBase {}; + +class DataTrackTransportTest + : public DataTrackE2ETest, + public ::testing::WithParamInterface> {}; + +TEST_P(DataTrackTransportTest, PublishesAndReceivesFramesEndToEnd) { + const auto publish_fps = std::get<0>(GetParam()); + const auto payload_len = std::get<1>(GetParam()); + const auto track_name = makeTrackName("transport"); + const auto frame_count = static_cast(std::llround( + std::chrono::duration(kPublishDuration).count() * publish_fps)); + + DataTrackPublishedDelegate subscriber_delegate; + std::vector room_configs(2); + room_configs[1].delegate = &subscriber_delegate; + + auto rooms = testRooms(room_configs); + auto &publisher_room = rooms[0]; + const auto publisher_identity = + publisher_room->localParticipant()->identity(); + + std::exception_ptr publish_error; + std::thread publisher([&]() { + try { + auto track = + requirePublishedTrack(publisher_room->localParticipant(), track_name); + if (!track->isPublished()) { + throw std::runtime_error("Publisher failed to publish data track"); + } + if (track->info().uses_e2ee) { + throw std::runtime_error("Unexpected E2EE on test data track"); + } + if (track->info().name != track_name) { + throw std::runtime_error("Published track name mismatch"); + } + + const auto frame_interval = + std::chrono::duration_cast( + std::chrono::duration(1.0 / publish_fps)); + auto next_send = std::chrono::steady_clock::now(); + + std::cout << "Publishing " << frame_count + << " frames with payload length " << payload_len << std::endl; + for (size_t index = 0; index < frame_count; ++index) { + std::vector payload(payload_len, + static_cast(index)); + requirePushSuccess(track->tryPush(std::move(payload)), + "Failed to push data frame"); + + next_send += frame_interval; + std::this_thread::sleep_until(next_send); + } + + track->unpublishDataTrack(); + } catch (...) { + publish_error = std::current_exception(); + } + }); + + auto remote_track = subscriber_delegate.waitForTrack(kTrackWaitTimeout); + ASSERT_NE(remote_track, nullptr) << "Timed out waiting for remote data track"; + EXPECT_TRUE(remote_track->isPublished()); + EXPECT_FALSE(remote_track->info().uses_e2ee); + EXPECT_EQ(remote_track->info().name, track_name); + EXPECT_EQ(remote_track->publisherIdentity(), publisher_identity); + + auto subscribe_result = remote_track->subscribe(); + if (!subscribe_result) { + FAIL() << describeDataTrackError(subscribe_result.error()); + } + auto subscription = subscribe_result.value(); + + std::promise receive_count_promise; + auto receive_count_future = receive_count_promise.get_future(); + std::exception_ptr subscribe_error; + std::thread subscriber([&]() { + try { + size_t received_count = 0; + DataFrame frame; + while (subscription->read(frame) && received_count < frame_count) { + if (frame.payload.empty()) { + throw std::runtime_error("Received empty data frame"); + } + + const auto first_byte = frame.payload.front(); + if (!std::all_of(frame.payload.begin(), frame.payload.end(), + [first_byte](std::uint8_t byte) { + return byte == first_byte; + })) { + throw std::runtime_error("Received frame with inconsistent payload"); + } + if (frame.user_timestamp.has_value()) { + throw std::runtime_error( + "Received unexpected user timestamp in transport test"); + } + + ++received_count; + } + + receive_count_promise.set_value(received_count); + } catch (...) { + subscribe_error = std::current_exception(); + receive_count_promise.set_exception(std::current_exception()); + } + }); + + if (receive_count_future.wait_for(kReadTimeout) != + std::future_status::ready) { + subscription->close(); + ADD_FAILURE() << "Timed out waiting for data frames"; + } + + subscriber.join(); + publisher.join(); + + if (publish_error) { + std::rethrow_exception(publish_error); + } + if (subscribe_error) { + std::rethrow_exception(subscribe_error); + } + + const auto received_count = receive_count_future.get(); + const auto received_percent = + static_cast(received_count) / static_cast(frame_count); + std::cout << "Received " << received_count << "/" << frame_count + << " frames (" << received_percent * 100.0f << "%)" << std::endl; + + EXPECT_GE(received_percent, kMinimumReceivedPercent) + << "Received " << received_count << "/" << frame_count << " frames"; +} + +TEST_F(DataTrackE2ETest, UnpublishUpdatesPublishedStateEndToEnd) { + const auto track_name = makeTrackName("published_state"); + + DataTrackPublishedDelegate subscriber_delegate; + std::vector room_configs(2); + room_configs[1].delegate = &subscriber_delegate; + + auto rooms = testRooms(room_configs); + auto &publisher_room = rooms[0]; + + auto publish_result = + publisher_room->localParticipant()->publishDataTrack(track_name); + if (!publish_result) { + FAIL() << describeDataTrackError(publish_result.error()); + } + auto local_track = publish_result.value(); + ASSERT_TRUE(local_track->isPublished()); + + auto remote_track = subscriber_delegate.waitForTrack(kTrackWaitTimeout); + ASSERT_NE(remote_track, nullptr) << "Timed out waiting for remote data track"; + EXPECT_TRUE(remote_track->isPublished()); + + std::this_thread::sleep_for(500ms); + local_track->unpublishDataTrack(); + + EXPECT_FALSE(local_track->isPublished()); + EXPECT_TRUE( + waitForCondition([&]() { return !remote_track->isPublished(); }, 2s)) + << "Remote track did not report unpublished state"; +} + +TEST_F(DataTrackE2ETest, PublishManyTracks) { + auto rooms = testRooms(1); + auto &room = rooms[0]; + + std::vector> tracks; + tracks.reserve(kPublishManyTrackCount); + + const auto start = std::chrono::steady_clock::now(); + for (int index = 0; index < kPublishManyTrackCount; ++index) { + const auto track_name = "track_" + std::to_string(index); + auto publish_result = + room->localParticipant()->publishDataTrack(track_name); + if (!publish_result) { + FAIL() << "Failed to publish track " << track_name << ": " + << describeDataTrackError(publish_result.error()); + } + auto track = publish_result.value(); + EXPECT_TRUE(track->isPublished()) + << "Track was not published: " << track_name; + EXPECT_EQ(track->info().name, track_name); + + tracks.push_back(std::move(track)); + } + const auto elapsed = std::chrono::steady_clock::now() - start; + + std::cout + << "Publishing " << kPublishManyTrackCount << " tracks took " + << std::chrono::duration_cast(elapsed).count() + << " ms" << std::endl; + EXPECT_LT(elapsed, kPublishManyTimeout); + + // This test intentionally creates bursty data-track traffic by pushing a + // large frame on every published track in quick succession. The RTC sender + // path uses bounded queues, so under this load not every packet is expected + // to make it onto the transport and "Failed to enqueue data track packet" + // logs are expected. The purpose of this test is to verify publish/push + // behavior and local track state, not end-to-end delivery of every packet. + for (const auto &track : tracks) { + auto push_result = track->tryPush( + std::vector(kLargeFramePayloadBytes, 0xFA)); + if (!push_result) { + ADD_FAILURE() << "Failed to push large frame on track " + << track->info().name << ": " + << describeDataTrackError(push_result.error()); + } + std::this_thread::sleep_for(50ms); + } + + for (const auto &track : tracks) { + track->unpublishDataTrack(); + EXPECT_FALSE(track->isPublished()); + } +} + +TEST_F(DataTrackE2ETest, PublishDuplicateName) { + auto rooms = testRooms(1); + auto &room = rooms[0]; + + auto first_track_result = room->localParticipant()->publishDataTrack("first"); + if (!first_track_result) { + FAIL() << describeDataTrackError(first_track_result.error()); + } + auto first_track = first_track_result.value(); + ASSERT_TRUE(first_track->isPublished()); + + auto duplicate_result = room->localParticipant()->publishDataTrack("first"); + ASSERT_FALSE(duplicate_result) + << "Expected duplicate data-track name to be rejected"; + EXPECT_EQ(duplicate_result.error().code, + PublishDataTrackErrorCode::DUPLICATE_NAME); + EXPECT_FALSE(duplicate_result.error().message.empty()); + + first_track->unpublishDataTrack(); +} + +TEST_F(DataTrackE2ETest, CanResubscribeToRemoteDataTrack) { + const auto track_name = makeTrackName("resubscribe"); + + DataTrackPublishedDelegate subscriber_delegate; + std::vector room_configs(2); + room_configs[1].delegate = &subscriber_delegate; + + auto rooms = testRooms(room_configs); + auto &publisher_room = rooms[0]; + + std::atomic keep_publishing{true}; + std::exception_ptr publish_error; + std::thread publisher([&]() { + try { + auto track = + requirePublishedTrack(publisher_room->localParticipant(), track_name); + if (!track->isPublished()) { + throw std::runtime_error("Publisher failed to publish data track"); + } + + while (keep_publishing.load()) { + requirePushSuccess(track->tryPush(std::vector(64, 0xFA)), + "Failed to push resubscribe test frame"); + std::this_thread::sleep_for(50ms); + } + + track->unpublishDataTrack(); + } catch (...) { + publish_error = std::current_exception(); + } + }); + + auto remote_track = subscriber_delegate.waitForTrack(kTrackWaitTimeout); + ASSERT_NE(remote_track, nullptr) << "Timed out waiting for remote data track"; + + for (int iteration = 0; iteration < kResubscribeIterations; ++iteration) { + auto subscribe_result = remote_track->subscribe(); + if (!subscribe_result) { + FAIL() << describeDataTrackError(subscribe_result.error()); + } + auto subscription = subscribe_result.value(); + + auto frame = readFrameWithTimeout(subscription, 5s); + EXPECT_FALSE(frame.payload.empty()) << "Iteration " << iteration; + + subscription->close(); + std::this_thread::sleep_for(50ms); + } + + keep_publishing.store(false); + publisher.join(); + + if (publish_error) { + std::rethrow_exception(publish_error); + } +} + +TEST_F(DataTrackE2ETest, FfiClientSubscribeDataTrackReturnsSyncResult) { + constexpr std::size_t kTopicCount = 20; + + DataTrackPublishedDelegate subscriber_delegate; + std::vector room_configs(2); + room_configs[1].delegate = &subscriber_delegate; + + auto rooms = testRooms(room_configs); + auto &publisher_room = rooms[0]; + + std::vector> local_tracks; + local_tracks.reserve(kTopicCount); + + for (std::size_t idx = 0; idx < kTopicCount; ++idx) { + const auto track_name = "test_" + std::to_string(idx); + auto publish_result = + publisher_room->localParticipant()->publishDataTrack(track_name); + if (!publish_result) { + FAIL() << "Failed to publish " << track_name << ": " + << describeDataTrackError(publish_result.error()); + } + auto local_track = publish_result.value(); + ASSERT_TRUE(local_track->isPublished()) << track_name; + local_tracks.push_back(std::move(local_track)); + } + + auto remote_tracks = + subscriber_delegate.waitForTracks(kTopicCount, kTrackWaitTimeout); + ASSERT_EQ(remote_tracks.size(), kTopicCount) + << "Timed out waiting for all remote data tracks"; + + std::sort(remote_tracks.begin(), remote_tracks.end(), + [](const std::shared_ptr &lhs, + const std::shared_ptr &rhs) { + return parseTestTrackIndex(lhs->info().name) < + parseTestTrackIndex(rhs->info().name); + }); + + std::vector subscription_handles; + subscription_handles.reserve(kTopicCount); + + for (std::size_t idx = 0; idx < remote_tracks.size(); ++idx) { + const auto &remote_track = remote_tracks[idx]; + const auto expected_name = "test_" + std::to_string(idx); + ASSERT_NE(remote_track, nullptr); + EXPECT_TRUE(remote_track->isPublished()) << expected_name; + EXPECT_EQ(remote_track->info().name, expected_name); + + const auto subscribe_start = std::chrono::steady_clock::now(); + auto subscribe_result = FfiClient::instance().subscribeDataTrack( + static_cast(remote_track->testFfiHandleId())); + const auto subscribe_elapsed = + std::chrono::steady_clock::now() - subscribe_start; + const auto subscribe_elapsed_ns = + std::chrono::duration_cast(subscribe_elapsed) + .count(); + + std::cout << "FfiClient::subscribeDataTrack(" << expected_name + << ") completed in " << subscribe_elapsed_ns << " ns" + << std::endl; + + if (!subscribe_result) { + FAIL() << "Failed to subscribe to " << expected_name << ": " + << describeDataTrackError(subscribe_result.error()); + } + + const auto subscription_handle_id = + static_cast(subscribe_result.value().handle().id()); + EXPECT_NE(subscription_handle_id, 0u) << expected_name; + subscription_handles.emplace_back(subscription_handle_id); + EXPECT_TRUE(subscription_handles.back().valid()) << expected_name; + } + + for (auto &local_track : local_tracks) { + local_track->unpublishDataTrack(); + } +} + +TEST_F(DataTrackE2ETest, PreservesUserTimestampEndToEnd) { + const auto track_name = makeTrackName("user_timestamp"); + const auto sent_timestamp = getTimestampUs(); + + DataTrackPublishedDelegate subscriber_delegate; + std::vector room_configs(2); + room_configs[1].delegate = &subscriber_delegate; + + auto rooms = testRooms(room_configs); + auto &publisher_room = rooms[0]; + + auto publish_result = + publisher_room->localParticipant()->publishDataTrack(track_name); + if (!publish_result) { + FAIL() << describeDataTrackError(publish_result.error()); + } + auto local_track = publish_result.value(); + ASSERT_TRUE(local_track->isPublished()); + + auto remote_track = subscriber_delegate.waitForTrack(kTrackWaitTimeout); + ASSERT_NE(remote_track, nullptr) << "Timed out waiting for remote data track"; + + auto subscribe_result = remote_track->subscribe(); + if (!subscribe_result) { + FAIL() << describeDataTrackError(subscribe_result.error()); + } + auto subscription = subscribe_result.value(); + + std::promise frame_promise; + auto frame_future = frame_promise.get_future(); + std::thread reader([&]() { + try { + DataFrame frame; + if (!subscription->read(frame)) { + throw std::runtime_error( + "Subscription ended before timestamped frame arrived"); + } + frame_promise.set_value(std::move(frame)); + } catch (...) { + frame_promise.set_exception(std::current_exception()); + } + }); + + const auto push_result = + local_track->tryPush(std::vector(64, 0xFA), sent_timestamp); + const auto frame_status = frame_future.wait_for(5s); + + if (frame_status != std::future_status::ready) { + subscription->close(); + } + + subscription->close(); + reader.join(); + local_track->unpublishDataTrack(); + + if (!push_result) { + FAIL() << "Failed to push timestamped data frame: " + << describeDataTrackError(push_result.error()); + } + ASSERT_EQ(frame_status, std::future_status::ready) + << "Timed out waiting for timestamped frame"; + + DataFrame frame; + try { + frame = frame_future.get(); + } catch (const std::exception &e) { + FAIL() << e.what(); + } + + ASSERT_FALSE(frame.payload.empty()); + ASSERT_TRUE(frame.user_timestamp.has_value()); + EXPECT_EQ(frame.user_timestamp.value(), sent_timestamp); +} + +TEST_F(DataTrackE2ETest, PublishesAndReceivesEncryptedFramesEndToEnd) { + const auto track_name = makeTrackName("e2ee_transport"); + + DataTrackPublishedDelegate subscriber_delegate; + auto room_configs = encryptedRoomConfigs(&subscriber_delegate); + auto rooms = testRooms(room_configs); + auto &publisher_room = rooms[0]; + auto &subscriber_room = rooms[1]; + + ASSERT_NE(publisher_room->e2eeManager(), nullptr); + ASSERT_NE(subscriber_room->e2eeManager(), nullptr); + ASSERT_NE(publisher_room->e2eeManager()->keyProvider(), nullptr); + ASSERT_NE(subscriber_room->e2eeManager()->keyProvider(), nullptr); + publisher_room->e2eeManager()->setEnabled(true); + subscriber_room->e2eeManager()->setEnabled(true); + EXPECT_EQ(publisher_room->e2eeManager()->keyProvider()->exportSharedKey(), + e2eeSharedKey()); + EXPECT_EQ(subscriber_room->e2eeManager()->keyProvider()->exportSharedKey(), + e2eeSharedKey()); + + auto publish_result = + publisher_room->localParticipant()->publishDataTrack(track_name); + if (!publish_result) { + FAIL() << describeDataTrackError(publish_result.error()); + } + auto local_track = publish_result.value(); + ASSERT_TRUE(local_track->isPublished()); + EXPECT_TRUE(local_track->info().uses_e2ee); + + auto remote_track = subscriber_delegate.waitForTrack(kTrackWaitTimeout); + ASSERT_NE(remote_track, nullptr) << "Timed out waiting for remote data track"; + EXPECT_TRUE(remote_track->isPublished()); + EXPECT_TRUE(remote_track->info().uses_e2ee); + EXPECT_EQ(remote_track->info().name, track_name); + + auto subscribe_result = remote_track->subscribe(); + if (!subscribe_result) { + FAIL() << describeDataTrackError(subscribe_result.error()); + } + auto subscription = subscribe_result.value(); + + std::promise frame_promise; + auto frame_future = frame_promise.get_future(); + std::thread reader([&]() { + try { + DataFrame frame; + if (!subscription->read(frame)) { + throw std::runtime_error( + "Subscription ended before an encrypted frame arrived"); + } + frame_promise.set_value(std::move(frame)); + } catch (...) { + frame_promise.set_exception(std::current_exception()); + } + }); + + bool pushed = false; + for (int index = 0; index < 200; ++index) { + std::vector payload(kLargeFramePayloadBytes, + static_cast(index + 1)); + auto push_result = local_track->tryPush(std::move(payload)); + pushed = static_cast(push_result) || pushed; + if (frame_future.wait_for(25ms) == std::future_status::ready) { + break; + } + } + + const auto frame_status = frame_future.wait_for(5s); + if (frame_status != std::future_status::ready) { + subscription->close(); + } + reader.join(); + ASSERT_TRUE(pushed) << "Failed to push encrypted data frames"; + ASSERT_EQ(frame_status, std::future_status::ready) + << "Timed out waiting for encrypted frame delivery"; + + DataFrame frame; + try { + frame = frame_future.get(); + } catch (const std::exception &e) { + FAIL() << e.what(); + } + ASSERT_FALSE(frame.payload.empty()); + const auto first_byte = frame.payload.front(); + EXPECT_TRUE(std::all_of( + frame.payload.begin(), frame.payload.end(), + [first_byte](std::uint8_t byte) { return byte == first_byte; })) + << "Encrypted payload is not byte-consistent"; + EXPECT_FALSE(frame.user_timestamp.has_value()) + << "Unexpected user timestamp on encrypted frame"; + + subscription->close(); + local_track->unpublishDataTrack(); +} + +TEST_F(DataTrackE2ETest, PreservesUserTimestampOnEncryptedDataTrack) { + const auto track_name = makeTrackName("e2ee_user_timestamp"); + const auto sent_timestamp = getTimestampUs(); + const std::vector payload(64, 0xFA); + + DataTrackPublishedDelegate subscriber_delegate; + auto room_configs = encryptedRoomConfigs(&subscriber_delegate); + auto rooms = testRooms(room_configs); + auto &publisher_room = rooms[0]; + auto &subscriber_room = rooms[1]; + + ASSERT_NE(publisher_room->e2eeManager(), nullptr); + ASSERT_NE(subscriber_room->e2eeManager(), nullptr); + publisher_room->e2eeManager()->setEnabled(true); + subscriber_room->e2eeManager()->setEnabled(true); + + auto publish_result = + publisher_room->localParticipant()->publishDataTrack(track_name); + if (!publish_result) { + FAIL() << describeDataTrackError(publish_result.error()); + } + auto local_track = publish_result.value(); + ASSERT_TRUE(local_track->isPublished()); + EXPECT_TRUE(local_track->info().uses_e2ee); + + auto remote_track = subscriber_delegate.waitForTrack(kTrackWaitTimeout); + ASSERT_NE(remote_track, nullptr) << "Timed out waiting for remote data track"; + EXPECT_TRUE(remote_track->info().uses_e2ee); + + auto subscribe_result = remote_track->subscribe(); + if (!subscribe_result) { + FAIL() << describeDataTrackError(subscribe_result.error()); + } + auto subscription = subscribe_result.value(); + + std::promise frame_promise; + auto frame_future = frame_promise.get_future(); + std::thread reader([&]() { + try { + DataFrame incoming_frame; + if (!subscription->read(incoming_frame)) { + throw std::runtime_error( + "Subscription ended before timestamped encrypted frame arrived"); + } + frame_promise.set_value(std::move(incoming_frame)); + } catch (...) { + frame_promise.set_exception(std::current_exception()); + } + }); + + bool pushed = false; + for (int attempt = 0; attempt < 200; ++attempt) { + auto payload_copy = payload; + auto push_result = + local_track->tryPush(std::move(payload_copy), sent_timestamp); + pushed = static_cast(push_result) || pushed; + if (frame_future.wait_for(25ms) == std::future_status::ready) { + break; + } + } + const auto frame_status = frame_future.wait_for(5s); + if (frame_status != std::future_status::ready) { + subscription->close(); + } + + reader.join(); + ASSERT_TRUE(pushed) << "Failed to push timestamped encrypted frame"; + ASSERT_EQ(frame_status, std::future_status::ready) + << "Timed out waiting for timestamped encrypted frame"; + + DataFrame frame; + try { + frame = frame_future.get(); + } catch (const std::exception &e) { + FAIL() << e.what(); + } + EXPECT_EQ(frame.payload, payload); + ASSERT_TRUE(frame.user_timestamp.has_value()); + EXPECT_EQ(frame.user_timestamp.value(), sent_timestamp); + + subscription->close(); + local_track->unpublishDataTrack(); +} + +std::string dataTrackParamName( + const ::testing::TestParamInfo> &info) { + if (std::get<0>(info.param) > 100.0) { + return "HighFpsSinglePacket"; + } + return "LowFpsMultiPacket"; +} + +INSTANTIATE_TEST_SUITE_P(DataTrackScenarios, DataTrackTransportTest, + ::testing::Values(std::make_tuple(120.0, size_t{8192}), + std::make_tuple(10.0, + size_t{196608})), + dataTrackParamName); + +} // namespace test +} // namespace livekit diff --git a/src/tests/integration/test_room_callbacks.cpp b/src/tests/integration/test_room_callbacks.cpp new file mode 100644 index 00000000..fe759214 --- /dev/null +++ b/src/tests/integration/test_room_callbacks.cpp @@ -0,0 +1,270 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// @file test_room_callbacks.cpp +/// @brief Public API tests for Room callback registration. + +#include +#include + +#include +#include +#include +#include + +namespace livekit { + +class RoomCallbackTest : public ::testing::Test { +protected: + void SetUp() override { + livekit::initialize(livekit::LogLevel::Info, livekit::LogSink::kConsole); + } + + void TearDown() override { livekit::shutdown(); } +}; + +TEST_F(RoomCallbackTest, AudioCallbackRegistrationIsAccepted) { + Room room; + + EXPECT_NO_THROW(room.setOnAudioFrameCallback( + "alice", TrackSource::SOURCE_MICROPHONE, [](const AudioFrame &) {})); +} + +TEST_F(RoomCallbackTest, VideoCallbackRegistrationIsAccepted) { + Room room; + + EXPECT_NO_THROW( + room.setOnVideoFrameCallback("alice", TrackSource::SOURCE_CAMERA, + [](const VideoFrame &, std::int64_t) {})); +} + +TEST_F(RoomCallbackTest, AudioCallbackRegistrationByTrackNameIsAccepted) { + Room room; + + EXPECT_NO_THROW(room.setOnAudioFrameCallback("alice", "mic-main", + [](const AudioFrame &) {})); +} + +TEST_F(RoomCallbackTest, VideoCallbackRegistrationByTrackNameIsAccepted) { + Room room; + + EXPECT_NO_THROW(room.setOnVideoFrameCallback( + "alice", "cam-main", [](const VideoFrame &, std::int64_t) {})); +} + +TEST_F(RoomCallbackTest, ClearingMissingCallbacksIsNoOp) { + Room room; + + EXPECT_NO_THROW( + room.clearOnAudioFrameCallback("nobody", TrackSource::SOURCE_MICROPHONE)); + EXPECT_NO_THROW( + room.clearOnVideoFrameCallback("nobody", TrackSource::SOURCE_CAMERA)); + EXPECT_NO_THROW(room.clearOnAudioFrameCallback("nobody", "missing-audio")); + EXPECT_NO_THROW(room.clearOnVideoFrameCallback("nobody", "missing-video")); +} + +TEST_F(RoomCallbackTest, ReRegisteringSameAudioKeyDoesNotThrow) { + Room room; + std::atomic counter1{0}; + std::atomic counter2{0}; + + EXPECT_NO_THROW(room.setOnAudioFrameCallback( + "alice", TrackSource::SOURCE_MICROPHONE, + [&counter1](const AudioFrame &) { counter1++; })); + EXPECT_NO_THROW(room.setOnAudioFrameCallback( + "alice", TrackSource::SOURCE_MICROPHONE, + [&counter2](const AudioFrame &) { counter2++; })); +} + +TEST_F(RoomCallbackTest, ReRegisteringSameVideoKeyDoesNotThrow) { + Room room; + + EXPECT_NO_THROW( + room.setOnVideoFrameCallback("alice", TrackSource::SOURCE_CAMERA, + [](const VideoFrame &, std::int64_t) {})); + EXPECT_NO_THROW( + room.setOnVideoFrameCallback("alice", TrackSource::SOURCE_CAMERA, + [](const VideoFrame &, std::int64_t) {})); +} + +TEST_F(RoomCallbackTest, DistinctAudioAndVideoCallbacksCanCoexist) { + Room room; + + EXPECT_NO_THROW(room.setOnAudioFrameCallback( + "alice", TrackSource::SOURCE_MICROPHONE, [](const AudioFrame &) {})); + EXPECT_NO_THROW( + room.setOnVideoFrameCallback("alice", TrackSource::SOURCE_CAMERA, + [](const VideoFrame &, std::int64_t) {})); + EXPECT_NO_THROW(room.setOnAudioFrameCallback( + "bob", TrackSource::SOURCE_MICROPHONE, [](const AudioFrame &) {})); + EXPECT_NO_THROW( + room.setOnVideoFrameCallback("bob", TrackSource::SOURCE_CAMERA, + [](const VideoFrame &, std::int64_t) {})); +} + +TEST_F(RoomCallbackTest, SameSourceDifferentTrackNamesAreAccepted) { + Room room; + + EXPECT_NO_THROW(room.setOnVideoFrameCallback( + "alice", "cam-main", [](const VideoFrame &, std::int64_t) {})); + EXPECT_NO_THROW(room.setOnVideoFrameCallback( + "alice", "cam-backup", [](const VideoFrame &, std::int64_t) {})); +} + +TEST_F(RoomCallbackTest, ClearingTrackNameCallbackIsAccepted) { + Room room; + + EXPECT_NO_THROW(room.setOnAudioFrameCallback("alice", "mic-main", + [](const AudioFrame &) {})); + EXPECT_NO_THROW(room.clearOnAudioFrameCallback("alice", "mic-main")); +} + +TEST_F(RoomCallbackTest, SourceAndTrackNameCallbacksCanCoexist) { + Room room; + + EXPECT_NO_THROW(room.setOnAudioFrameCallback( + "alice", TrackSource::SOURCE_MICROPHONE, [](const AudioFrame &) {})); + EXPECT_NO_THROW(room.setOnAudioFrameCallback("alice", "mic-main", + [](const AudioFrame &) {})); +} + +TEST_F(RoomCallbackTest, DataCallbackRegistrationReturnsUsableIds) { + Room room; + + const auto id1 = room.addOnDataFrameCallback( + "alice", "track-a", + [](const std::vector &, std::optional) {}); + const auto id2 = room.addOnDataFrameCallback( + "alice", "track-a", + [](const std::vector &, std::optional) {}); + + EXPECT_NE(id1, std::numeric_limits::max()); + EXPECT_NE(id2, std::numeric_limits::max()); + EXPECT_NE(id1, id2); + + EXPECT_NO_THROW(room.removeOnDataFrameCallback(id1)); + EXPECT_NO_THROW(room.removeOnDataFrameCallback(id2)); +} + +TEST_F(RoomCallbackTest, RemovingUnknownDataCallbackIsNoOp) { + Room room; + + EXPECT_NO_THROW(room.removeOnDataFrameCallback( + std::numeric_limits::max())); +} + +TEST_F(RoomCallbackTest, DestroyRoomWithRegisteredCallbacksIsSafe) { + EXPECT_NO_THROW({ + Room room; + room.setOnAudioFrameCallback("alice", TrackSource::SOURCE_MICROPHONE, + [](const AudioFrame &) {}); + room.setOnVideoFrameCallback("bob", TrackSource::SOURCE_CAMERA, + [](const VideoFrame &, std::int64_t) {}); + room.addOnDataFrameCallback( + "carol", "track", + [](const std::vector &, std::optional) {}); + }); +} + +TEST_F(RoomCallbackTest, DestroyRoomAfterClearingCallbacksIsSafe) { + EXPECT_NO_THROW({ + Room room; + room.setOnAudioFrameCallback("alice", TrackSource::SOURCE_MICROPHONE, + [](const AudioFrame &) {}); + room.clearOnAudioFrameCallback("alice", TrackSource::SOURCE_MICROPHONE); + + const auto id = room.addOnDataFrameCallback( + "alice", "track", + [](const std::vector &, std::optional) {}); + room.removeOnDataFrameCallback(id); + }); +} + +TEST_F(RoomCallbackTest, ConcurrentRegistrationDoesNotCrash) { + Room room; + constexpr int kThreads = 8; + constexpr int kIterations = 100; + + std::vector threads; + threads.reserve(kThreads); + + for (int t = 0; t < kThreads; ++t) { + threads.emplace_back([&room, t]() { + for (int i = 0; i < kIterations; ++i) { + const std::string id = "participant-" + std::to_string(t); + room.setOnAudioFrameCallback(id, TrackSource::SOURCE_MICROPHONE, + [](const AudioFrame &) {}); + room.clearOnAudioFrameCallback(id, TrackSource::SOURCE_MICROPHONE); + } + }); + } + + for (auto &thread : threads) { + thread.join(); + } + + SUCCEED(); +} + +TEST_F(RoomCallbackTest, ConcurrentMixedRegistrationDoesNotCrash) { + Room room; + constexpr int kThreads = 4; + constexpr int kIterations = 50; + + std::vector threads; + threads.reserve(kThreads); + + for (int t = 0; t < kThreads; ++t) { + threads.emplace_back([&room, t]() { + const std::string id = "p-" + std::to_string(t); + for (int i = 0; i < kIterations; ++i) { + room.setOnAudioFrameCallback(id, TrackSource::SOURCE_MICROPHONE, + [](const AudioFrame &) {}); + room.setOnVideoFrameCallback(id, TrackSource::SOURCE_CAMERA, + [](const VideoFrame &, std::int64_t) {}); + const auto data_id = + room.addOnDataFrameCallback(id, "track", + [](const std::vector &, + std::optional) {}); + room.removeOnDataFrameCallback(data_id); + } + }); + } + + for (auto &thread : threads) { + thread.join(); + } + + SUCCEED(); +} + +TEST_F(RoomCallbackTest, ManyDistinctAudioCallbacksCanBeRegisteredAndCleared) { + Room room; + constexpr int kCount = 50; + + for (int i = 0; i < kCount; ++i) { + EXPECT_NO_THROW(room.setOnAudioFrameCallback( + "participant-" + std::to_string(i), TrackSource::SOURCE_MICROPHONE, + [](const AudioFrame &) {})); + } + + for (int i = 0; i < kCount; ++i) { + EXPECT_NO_THROW(room.clearOnAudioFrameCallback( + "participant-" + std::to_string(i), TrackSource::SOURCE_MICROPHONE)); + } +} + +} // namespace livekit diff --git a/src/tests/integration/test_subscription_thread_dispatcher.cpp b/src/tests/integration/test_subscription_thread_dispatcher.cpp index 71601a18..26c2185e 100644 --- a/src/tests/integration/test_subscription_thread_dispatcher.cpp +++ b/src/tests/integration/test_subscription_thread_dispatcher.cpp @@ -36,6 +36,8 @@ class SubscriptionThreadDispatcherTest : public ::testing::Test { using CallbackKey = SubscriptionThreadDispatcher::CallbackKey; using CallbackKeyHash = SubscriptionThreadDispatcher::CallbackKeyHash; + using DataCallbackKey = SubscriptionThreadDispatcher::DataCallbackKey; + using DataCallbackKeyHash = SubscriptionThreadDispatcher::DataCallbackKeyHash; static auto &audioCallbacks(SubscriptionThreadDispatcher &dispatcher) { return dispatcher.audio_callbacks_; @@ -46,6 +48,15 @@ class SubscriptionThreadDispatcherTest : public ::testing::Test { static auto &activeReaders(SubscriptionThreadDispatcher &dispatcher) { return dispatcher.active_readers_; } + static auto &dataCallbacks(SubscriptionThreadDispatcher &dispatcher) { + return dispatcher.data_callbacks_; + } + static auto &activeDataReaders(SubscriptionThreadDispatcher &dispatcher) { + return dispatcher.active_data_readers_; + } + static auto &remoteDataTracks(SubscriptionThreadDispatcher &dispatcher) { + return dispatcher.remote_data_tracks_; + } static int maxActiveReaders() { return SubscriptionThreadDispatcher::kMaxActiveReaders; } @@ -56,21 +67,27 @@ class SubscriptionThreadDispatcherTest : public ::testing::Test { // ============================================================================ TEST_F(SubscriptionThreadDispatcherTest, CallbackKeyEqualKeysCompareEqual) { - CallbackKey a{"alice", TrackSource::SOURCE_MICROPHONE}; - CallbackKey b{"alice", TrackSource::SOURCE_MICROPHONE}; + CallbackKey a{"alice", TrackSource::SOURCE_MICROPHONE, ""}; + CallbackKey b{"alice", TrackSource::SOURCE_MICROPHONE, ""}; EXPECT_TRUE(a == b); } -TEST_F(SubscriptionThreadDispatcherTest, - CallbackKeyDifferentIdentityNotEqual) { - CallbackKey a{"alice", TrackSource::SOURCE_MICROPHONE}; - CallbackKey b{"bob", TrackSource::SOURCE_MICROPHONE}; +TEST_F(SubscriptionThreadDispatcherTest, CallbackKeyDifferentIdentityNotEqual) { + CallbackKey a{"alice", TrackSource::SOURCE_MICROPHONE, ""}; + CallbackKey b{"bob", TrackSource::SOURCE_MICROPHONE, ""}; EXPECT_FALSE(a == b); } TEST_F(SubscriptionThreadDispatcherTest, CallbackKeyDifferentSourceNotEqual) { - CallbackKey a{"alice", TrackSource::SOURCE_MICROPHONE}; - CallbackKey b{"alice", TrackSource::SOURCE_CAMERA}; + CallbackKey a{"alice", TrackSource::SOURCE_MICROPHONE, ""}; + CallbackKey b{"alice", TrackSource::SOURCE_CAMERA, ""}; + EXPECT_FALSE(a == b); +} + +TEST_F(SubscriptionThreadDispatcherTest, + CallbackKeyDifferentTrackNameNotEqual) { + CallbackKey a{"alice", TrackSource::SOURCE_UNKNOWN, "cam-main"}; + CallbackKey b{"alice", TrackSource::SOURCE_UNKNOWN, "cam-backup"}; EXPECT_FALSE(a == b); } @@ -80,8 +97,8 @@ TEST_F(SubscriptionThreadDispatcherTest, CallbackKeyDifferentSourceNotEqual) { TEST_F(SubscriptionThreadDispatcherTest, CallbackKeyHashEqualKeysProduceSameHash) { - CallbackKey a{"alice", TrackSource::SOURCE_MICROPHONE}; - CallbackKey b{"alice", TrackSource::SOURCE_MICROPHONE}; + CallbackKey a{"alice", TrackSource::SOURCE_MICROPHONE, ""}; + CallbackKey b{"alice", TrackSource::SOURCE_MICROPHONE, ""}; CallbackKeyHash hasher; EXPECT_EQ(hasher(a), hasher(b)); } @@ -89,20 +106,22 @@ TEST_F(SubscriptionThreadDispatcherTest, TEST_F(SubscriptionThreadDispatcherTest, CallbackKeyHashDifferentKeysLikelyDifferentHash) { CallbackKeyHash hasher; - CallbackKey mic{"alice", TrackSource::SOURCE_MICROPHONE}; - CallbackKey cam{"alice", TrackSource::SOURCE_CAMERA}; - CallbackKey bob{"bob", TrackSource::SOURCE_MICROPHONE}; + CallbackKey mic{"alice", TrackSource::SOURCE_MICROPHONE, ""}; + CallbackKey cam{"alice", TrackSource::SOURCE_CAMERA, ""}; + CallbackKey bob{"bob", TrackSource::SOURCE_MICROPHONE, ""}; + CallbackKey named{"alice", TrackSource::SOURCE_UNKNOWN, "mic-main"}; EXPECT_NE(hasher(mic), hasher(cam)); EXPECT_NE(hasher(mic), hasher(bob)); + EXPECT_NE(hasher(mic), hasher(named)); } TEST_F(SubscriptionThreadDispatcherTest, CallbackKeyWorksAsUnorderedMapKey) { std::unordered_map map; - CallbackKey k1{"alice", TrackSource::SOURCE_MICROPHONE}; - CallbackKey k2{"bob", TrackSource::SOURCE_CAMERA}; - CallbackKey k3{"alice", TrackSource::SOURCE_CAMERA}; + CallbackKey k1{"alice", TrackSource::SOURCE_MICROPHONE, ""}; + CallbackKey k2{"bob", TrackSource::SOURCE_CAMERA, ""}; + CallbackKey k3{"alice", TrackSource::SOURCE_CAMERA, ""}; map[k1] = 1; map[k2] = 2; @@ -123,8 +142,8 @@ TEST_F(SubscriptionThreadDispatcherTest, CallbackKeyWorksAsUnorderedMapKey) { } TEST_F(SubscriptionThreadDispatcherTest, CallbackKeyEmptyIdentityWorks) { - CallbackKey a{"", TrackSource::SOURCE_UNKNOWN}; - CallbackKey b{"", TrackSource::SOURCE_UNKNOWN}; + CallbackKey a{"", TrackSource::SOURCE_UNKNOWN, ""}; + CallbackKey b{"", TrackSource::SOURCE_UNKNOWN, ""}; CallbackKeyHash hasher; EXPECT_TRUE(a == b); EXPECT_EQ(hasher(a), hasher(b)); @@ -150,6 +169,19 @@ TEST_F(SubscriptionThreadDispatcherTest, SetAudioCallbackStoresRegistration) { EXPECT_EQ(audioCallbacks(dispatcher).size(), 1u); } +TEST_F(SubscriptionThreadDispatcherTest, + SetAudioCallbackByTrackNameStoresRegistration) { + SubscriptionThreadDispatcher dispatcher; + dispatcher.setOnAudioFrameCallback("alice", "mic-main", + [](const AudioFrame &) {}); + + EXPECT_EQ(audioCallbacks(dispatcher).size(), 1u); + EXPECT_EQ( + audioCallbacks(dispatcher) + .count(CallbackKey{"alice", TrackSource::SOURCE_UNKNOWN, "mic-main"}), + 1u); +} + TEST_F(SubscriptionThreadDispatcherTest, SetVideoCallbackStoresRegistration) { SubscriptionThreadDispatcher dispatcher; dispatcher.setOnVideoFrameCallback("alice", TrackSource::SOURCE_CAMERA, @@ -158,7 +190,21 @@ TEST_F(SubscriptionThreadDispatcherTest, SetVideoCallbackStoresRegistration) { EXPECT_EQ(videoCallbacks(dispatcher).size(), 1u); } -TEST_F(SubscriptionThreadDispatcherTest, ClearAudioCallbackRemovesRegistration) { +TEST_F(SubscriptionThreadDispatcherTest, + SetVideoCallbackByTrackNameStoresRegistration) { + SubscriptionThreadDispatcher dispatcher; + dispatcher.setOnVideoFrameCallback("alice", "cam-main", + [](const VideoFrame &, std::int64_t) {}); + + EXPECT_EQ(videoCallbacks(dispatcher).size(), 1u); + EXPECT_EQ( + videoCallbacks(dispatcher) + .count(CallbackKey{"alice", TrackSource::SOURCE_UNKNOWN, "cam-main"}), + 1u); +} + +TEST_F(SubscriptionThreadDispatcherTest, + ClearAudioCallbackRemovesRegistration) { SubscriptionThreadDispatcher dispatcher; dispatcher.setOnAudioFrameCallback("alice", TrackSource::SOURCE_MICROPHONE, [](const AudioFrame &) {}); @@ -168,7 +214,19 @@ TEST_F(SubscriptionThreadDispatcherTest, ClearAudioCallbackRemovesRegistration) EXPECT_EQ(audioCallbacks(dispatcher).size(), 0u); } -TEST_F(SubscriptionThreadDispatcherTest, ClearVideoCallbackRemovesRegistration) { +TEST_F(SubscriptionThreadDispatcherTest, + ClearAudioCallbackByTrackNameRemovesRegistration) { + SubscriptionThreadDispatcher dispatcher; + dispatcher.setOnAudioFrameCallback("alice", "mic-main", + [](const AudioFrame &) {}); + ASSERT_EQ(audioCallbacks(dispatcher).size(), 1u); + + dispatcher.clearOnAudioFrameCallback("alice", "mic-main"); + EXPECT_EQ(audioCallbacks(dispatcher).size(), 0u); +} + +TEST_F(SubscriptionThreadDispatcherTest, + ClearVideoCallbackRemovesRegistration) { SubscriptionThreadDispatcher dispatcher; dispatcher.setOnVideoFrameCallback("alice", TrackSource::SOURCE_CAMERA, [](const VideoFrame &, std::int64_t) {}); @@ -178,15 +236,29 @@ TEST_F(SubscriptionThreadDispatcherTest, ClearVideoCallbackRemovesRegistration) EXPECT_EQ(videoCallbacks(dispatcher).size(), 0u); } +TEST_F(SubscriptionThreadDispatcherTest, + ClearVideoCallbackByTrackNameRemovesRegistration) { + SubscriptionThreadDispatcher dispatcher; + dispatcher.setOnVideoFrameCallback("alice", "cam-main", + [](const VideoFrame &, std::int64_t) {}); + ASSERT_EQ(videoCallbacks(dispatcher).size(), 1u); + + dispatcher.clearOnVideoFrameCallback("alice", "cam-main"); + EXPECT_EQ(videoCallbacks(dispatcher).size(), 0u); +} + TEST_F(SubscriptionThreadDispatcherTest, ClearNonExistentCallbackIsNoOp) { SubscriptionThreadDispatcher dispatcher; EXPECT_NO_THROW(dispatcher.clearOnAudioFrameCallback( "nobody", TrackSource::SOURCE_MICROPHONE)); - EXPECT_NO_THROW( - dispatcher.clearOnVideoFrameCallback("nobody", TrackSource::SOURCE_CAMERA)); + EXPECT_NO_THROW(dispatcher.clearOnVideoFrameCallback( + "nobody", TrackSource::SOURCE_CAMERA)); + EXPECT_NO_THROW(dispatcher.clearOnAudioFrameCallback("nobody", "missing")); + EXPECT_NO_THROW(dispatcher.clearOnVideoFrameCallback("nobody", "missing")); } -TEST_F(SubscriptionThreadDispatcherTest, OverwriteAudioCallbackKeepsSingleEntry) { +TEST_F(SubscriptionThreadDispatcherTest, + OverwriteAudioCallbackKeepsSingleEntry) { SubscriptionThreadDispatcher dispatcher; std::atomic counter1{0}; std::atomic counter2{0}; @@ -202,7 +274,8 @@ TEST_F(SubscriptionThreadDispatcherTest, OverwriteAudioCallbackKeepsSingleEntry) << "Re-registering with the same key should overwrite, not add"; } -TEST_F(SubscriptionThreadDispatcherTest, OverwriteVideoCallbackKeepsSingleEntry) { +TEST_F(SubscriptionThreadDispatcherTest, + OverwriteVideoCallbackKeepsSingleEntry) { SubscriptionThreadDispatcher dispatcher; dispatcher.setOnVideoFrameCallback("alice", TrackSource::SOURCE_CAMERA, [](const VideoFrame &, std::int64_t) {}); @@ -212,6 +285,17 @@ TEST_F(SubscriptionThreadDispatcherTest, OverwriteVideoCallbackKeepsSingleEntry) EXPECT_EQ(videoCallbacks(dispatcher).size(), 1u); } +TEST_F(SubscriptionThreadDispatcherTest, + OverwriteTrackNameAudioCallbackKeepsSingleEntry) { + SubscriptionThreadDispatcher dispatcher; + dispatcher.setOnAudioFrameCallback("alice", "mic-main", + [](const AudioFrame &) {}); + dispatcher.setOnAudioFrameCallback("alice", "mic-main", + [](const AudioFrame &) {}); + + EXPECT_EQ(audioCallbacks(dispatcher).size(), 1u); +} + TEST_F(SubscriptionThreadDispatcherTest, MultipleDistinctCallbacksAreIndependent) { SubscriptionThreadDispatcher dispatcher; @@ -227,8 +311,7 @@ TEST_F(SubscriptionThreadDispatcherTest, EXPECT_EQ(audioCallbacks(dispatcher).size(), 2u); EXPECT_EQ(videoCallbacks(dispatcher).size(), 2u); - dispatcher.clearOnAudioFrameCallback("alice", - TrackSource::SOURCE_MICROPHONE); + dispatcher.clearOnAudioFrameCallback("alice", TrackSource::SOURCE_MICROPHONE); EXPECT_EQ(audioCallbacks(dispatcher).size(), 1u); EXPECT_EQ(videoCallbacks(dispatcher).size(), 2u); } @@ -242,14 +325,30 @@ TEST_F(SubscriptionThreadDispatcherTest, ClearingOneSourceDoesNotAffectOther) { [](const AudioFrame &) {}); ASSERT_EQ(audioCallbacks(dispatcher).size(), 2u); - dispatcher.clearOnAudioFrameCallback("alice", - TrackSource::SOURCE_MICROPHONE); + dispatcher.clearOnAudioFrameCallback("alice", TrackSource::SOURCE_MICROPHONE); EXPECT_EQ(audioCallbacks(dispatcher).size(), 1u); - CallbackKey remaining{"alice", TrackSource::SOURCE_SCREENSHARE_AUDIO}; + CallbackKey remaining{"alice", TrackSource::SOURCE_SCREENSHARE_AUDIO, ""}; EXPECT_EQ(audioCallbacks(dispatcher).count(remaining), 1u); } +TEST_F(SubscriptionThreadDispatcherTest, + SourceAndTrackNameAudioCallbacksAreIndependent) { + SubscriptionThreadDispatcher dispatcher; + dispatcher.setOnAudioFrameCallback("alice", TrackSource::SOURCE_MICROPHONE, + [](const AudioFrame &) {}); + dispatcher.setOnAudioFrameCallback("alice", "mic-main", + [](const AudioFrame &) {}); + ASSERT_EQ(audioCallbacks(dispatcher).size(), 2u); + + dispatcher.clearOnAudioFrameCallback("alice", "mic-main"); + EXPECT_EQ(audioCallbacks(dispatcher).size(), 1u); + EXPECT_EQ( + audioCallbacks(dispatcher) + .count(CallbackKey{"alice", TrackSource::SOURCE_MICROPHONE, ""}), + 1u); +} + // ============================================================================ // Active readers state (no real streams, just map state) // ============================================================================ @@ -341,9 +440,9 @@ TEST_F(SubscriptionThreadDispatcherTest, for (int i = 0; i < kIterations; ++i) { dispatcher.setOnAudioFrameCallback(id, TrackSource::SOURCE_MICROPHONE, [](const AudioFrame &) {}); - dispatcher.setOnVideoFrameCallback(id, TrackSource::SOURCE_CAMERA, - [](const VideoFrame &, - std::int64_t) {}); + dispatcher.setOnVideoFrameCallback( + id, TrackSource::SOURCE_CAMERA, + [](const VideoFrame &, std::int64_t) {}); } }); } @@ -380,4 +479,259 @@ TEST_F(SubscriptionThreadDispatcherTest, ManyDistinctCallbacksCanBeRegistered) { EXPECT_EQ(audioCallbacks(dispatcher).size(), 0u); } +// ============================================================================ +// DataCallbackKey equality +// ============================================================================ + +TEST_F(SubscriptionThreadDispatcherTest, DataCallbackKeyEqualKeysCompareEqual) { + DataCallbackKey a{"alice", "my-track"}; + DataCallbackKey b{"alice", "my-track"}; + EXPECT_TRUE(a == b); +} + +TEST_F(SubscriptionThreadDispatcherTest, + DataCallbackKeyDifferentIdentityNotEqual) { + DataCallbackKey a{"alice", "my-track"}; + DataCallbackKey b{"bob", "my-track"}; + EXPECT_FALSE(a == b); +} + +TEST_F(SubscriptionThreadDispatcherTest, + DataCallbackKeyDifferentTrackNameNotEqual) { + DataCallbackKey a{"alice", "track-a"}; + DataCallbackKey b{"alice", "track-b"}; + EXPECT_FALSE(a == b); +} + +// ============================================================================ +// DataCallbackKeyHash +// ============================================================================ + +TEST_F(SubscriptionThreadDispatcherTest, + DataCallbackKeyHashEqualKeysProduceSameHash) { + DataCallbackKey a{"alice", "my-track"}; + DataCallbackKey b{"alice", "my-track"}; + DataCallbackKeyHash hasher; + EXPECT_EQ(hasher(a), hasher(b)); +} + +TEST_F(SubscriptionThreadDispatcherTest, + DataCallbackKeyHashDifferentKeysLikelyDifferentHash) { + DataCallbackKeyHash hasher; + DataCallbackKey a{"alice", "track-a"}; + DataCallbackKey b{"alice", "track-b"}; + DataCallbackKey c{"bob", "track-a"}; + EXPECT_NE(hasher(a), hasher(b)); + EXPECT_NE(hasher(a), hasher(c)); +} + +TEST_F(SubscriptionThreadDispatcherTest, + DataCallbackKeyWorksAsUnorderedMapKey) { + std::unordered_map map; + + DataCallbackKey k1{"alice", "track-a"}; + DataCallbackKey k2{"bob", "track-b"}; + DataCallbackKey k3{"alice", "track-b"}; + + map[k1] = 1; + map[k2] = 2; + map[k3] = 3; + + EXPECT_EQ(map.size(), 3u); + EXPECT_EQ(map[k1], 1); + EXPECT_EQ(map[k2], 2); + EXPECT_EQ(map[k3], 3); + + map[k1] = 42; + EXPECT_EQ(map[k1], 42); + EXPECT_EQ(map.size(), 3u); + + map.erase(k2); + EXPECT_EQ(map.size(), 2u); + EXPECT_EQ(map.count(k2), 0u); +} + +// ============================================================================ +// Data callback registration and clearing +// ============================================================================ + +TEST_F(SubscriptionThreadDispatcherTest, + AddDataFrameCallbackStoresRegistration) { + SubscriptionThreadDispatcher dispatcher; + auto id = dispatcher.addOnDataFrameCallback( + "alice", "my-track", + [](const std::vector &, std::optional) {}); + + EXPECT_NE(id, 0u); + EXPECT_EQ(dataCallbacks(dispatcher).size(), 1u); +} + +TEST_F(SubscriptionThreadDispatcherTest, + RemoveDataFrameCallbackRemovesRegistration) { + SubscriptionThreadDispatcher dispatcher; + auto id = dispatcher.addOnDataFrameCallback( + "alice", "my-track", + [](const std::vector &, std::optional) {}); + ASSERT_EQ(dataCallbacks(dispatcher).size(), 1u); + + dispatcher.removeOnDataFrameCallback(id); + EXPECT_EQ(dataCallbacks(dispatcher).size(), 0u); +} + +TEST_F(SubscriptionThreadDispatcherTest, RemoveNonExistentDataCallbackIsNoOp) { + SubscriptionThreadDispatcher dispatcher; + EXPECT_NO_THROW(dispatcher.removeOnDataFrameCallback(999)); +} + +TEST_F(SubscriptionThreadDispatcherTest, + MultipleDataCallbacksForSameKeyAreIndependent) { + SubscriptionThreadDispatcher dispatcher; + auto cb = [](const std::vector &, + std::optional) {}; + auto id1 = dispatcher.addOnDataFrameCallback("alice", "track", cb); + auto id2 = dispatcher.addOnDataFrameCallback("alice", "track", cb); + + EXPECT_NE(id1, id2); + EXPECT_EQ(dataCallbacks(dispatcher).size(), 2u); + + dispatcher.removeOnDataFrameCallback(id1); + EXPECT_EQ(dataCallbacks(dispatcher).size(), 1u); +} + +TEST_F(SubscriptionThreadDispatcherTest, + DataCallbackIdsAreMonotonicallyIncreasing) { + SubscriptionThreadDispatcher dispatcher; + auto cb = [](const std::vector &, + std::optional) {}; + auto id1 = dispatcher.addOnDataFrameCallback("alice", "t1", cb); + auto id2 = dispatcher.addOnDataFrameCallback("bob", "t2", cb); + auto id3 = dispatcher.addOnDataFrameCallback("carol", "t3", cb); + + EXPECT_LT(id1, id2); + EXPECT_LT(id2, id3); +} + +// ============================================================================ +// Data track active readers (no real tracks, just map state) +// ============================================================================ + +TEST_F(SubscriptionThreadDispatcherTest, NoActiveDataReadersInitially) { + SubscriptionThreadDispatcher dispatcher; + EXPECT_TRUE(activeDataReaders(dispatcher).empty()); +} + +TEST_F(SubscriptionThreadDispatcherTest, + ActiveDataReadersEmptyAfterCallbackRegistration) { + SubscriptionThreadDispatcher dispatcher; + dispatcher.addOnDataFrameCallback( + "alice", "my-track", + [](const std::vector &, std::optional) {}); + EXPECT_TRUE(activeDataReaders(dispatcher).empty()) + << "Registering a callback without a published track should not spawn " + "readers"; +} + +TEST_F(SubscriptionThreadDispatcherTest, NoRemoteDataTracksInitially) { + SubscriptionThreadDispatcher dispatcher; + EXPECT_TRUE(remoteDataTracks(dispatcher).empty()); +} + +// ============================================================================ +// Data track destruction safety +// ============================================================================ + +TEST_F(SubscriptionThreadDispatcherTest, + DestroyDispatcherWithDataCallbacksIsSafe) { + EXPECT_NO_THROW({ + SubscriptionThreadDispatcher dispatcher; + dispatcher.addOnDataFrameCallback( + "alice", "track-a", + [](const std::vector &, std::optional) {}); + dispatcher.addOnDataFrameCallback( + "bob", "track-b", + [](const std::vector &, std::optional) {}); + }); +} + +TEST_F(SubscriptionThreadDispatcherTest, + DestroyDispatcherAfterRemovingDataCallbacksIsSafe) { + EXPECT_NO_THROW({ + SubscriptionThreadDispatcher dispatcher; + auto id = dispatcher.addOnDataFrameCallback( + "alice", "track-a", + [](const std::vector &, std::optional) {}); + dispatcher.removeOnDataFrameCallback(id); + }); +} + +// ============================================================================ +// Mixed audio/video/data registration +// ============================================================================ + +TEST_F(SubscriptionThreadDispatcherTest, + MixedAudioVideoDataCallbacksAreIndependent) { + SubscriptionThreadDispatcher dispatcher; + dispatcher.setOnAudioFrameCallback("alice", TrackSource::SOURCE_MICROPHONE, + [](const AudioFrame &) {}); + dispatcher.setOnVideoFrameCallback("alice", TrackSource::SOURCE_CAMERA, + [](const VideoFrame &, std::int64_t) {}); + dispatcher.addOnDataFrameCallback( + "alice", "data-track", + [](const std::vector &, std::optional) {}); + + EXPECT_EQ(audioCallbacks(dispatcher).size(), 1u); + EXPECT_EQ(videoCallbacks(dispatcher).size(), 1u); + EXPECT_EQ(dataCallbacks(dispatcher).size(), 1u); +} + +TEST_F(SubscriptionThreadDispatcherTest, StopAllClearsDataCallbacksAndReaders) { + SubscriptionThreadDispatcher dispatcher; + dispatcher.addOnDataFrameCallback( + "alice", "track-a", + [](const std::vector &, std::optional) {}); + dispatcher.addOnDataFrameCallback( + "bob", "track-b", + [](const std::vector &, std::optional) {}); + + dispatcher.stopAll(); + + EXPECT_EQ(dataCallbacks(dispatcher).size(), 0u); + EXPECT_TRUE(activeDataReaders(dispatcher).empty()); + EXPECT_TRUE(remoteDataTracks(dispatcher).empty()); +} + +// ============================================================================ +// Concurrent data callback registration +// ============================================================================ + +TEST_F(SubscriptionThreadDispatcherTest, + ConcurrentDataCallbackRegistrationDoesNotCrash) { + SubscriptionThreadDispatcher dispatcher; + constexpr int kThreads = 8; + constexpr int kIterations = 100; + + std::vector threads; + threads.reserve(kThreads); + + for (int t = 0; t < kThreads; ++t) { + threads.emplace_back([&dispatcher, t]() { + for (int i = 0; i < kIterations; ++i) { + auto id = dispatcher.addOnDataFrameCallback( + "participant-" + std::to_string(t), "track", + [](const std::vector &, + std::optional) {}); + dispatcher.removeOnDataFrameCallback(id); + } + }); + } + + for (auto &thread : threads) { + thread.join(); + } + + EXPECT_TRUE(dataCallbacks(dispatcher).empty()) + << "All data callbacks should be cleared after concurrent " + "register/remove"; +} + } // namespace livekit diff --git a/src/tests/integration/test_video_frame_metadata.cpp b/src/tests/integration/test_video_frame_metadata.cpp new file mode 100644 index 00000000..f7fa86df --- /dev/null +++ b/src/tests/integration/test_video_frame_metadata.cpp @@ -0,0 +1,185 @@ +/* + * Copyright 2025 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "room_proto_converter.h" +#include "tests/common/test_common.h" +#include "video_utils.h" + +#include +#include +#include +#include +#include + +namespace livekit { +namespace test { + +TEST(VideoFrameMetadataTest, EmptyMetadataIsOmittedFromProto) { + std::optional metadata = VideoFrameMetadata{}; + EXPECT_FALSE(toProto(metadata).has_value()); +} + +TEST(VideoFrameMetadataTest, UserTimestampOnlyIsPreserved) { + VideoFrameMetadata metadata; + metadata.user_timestamp_us = 123; + + auto proto_metadata = toProto(metadata); + ASSERT_TRUE(proto_metadata.has_value()); + EXPECT_TRUE(proto_metadata->has_user_timestamp_us()); + EXPECT_EQ(proto_metadata->user_timestamp_us(), 123u); + EXPECT_FALSE(proto_metadata->has_frame_id()); + + auto round_trip = fromProto(*proto_metadata); + ASSERT_TRUE(round_trip.has_value()); + EXPECT_EQ(round_trip->user_timestamp_us, std::optional(123)); + EXPECT_EQ(round_trip->frame_id, std::nullopt); +} + +TEST(VideoFrameMetadataTest, FrameIdOnlyIsPreserved) { + VideoFrameMetadata metadata; + metadata.frame_id = 456; + + auto proto_metadata = toProto(metadata); + ASSERT_TRUE(proto_metadata.has_value()); + EXPECT_FALSE(proto_metadata->has_user_timestamp_us()); + EXPECT_TRUE(proto_metadata->has_frame_id()); + EXPECT_EQ(proto_metadata->frame_id(), 456u); + + auto round_trip = fromProto(*proto_metadata); + ASSERT_TRUE(round_trip.has_value()); + EXPECT_EQ(round_trip->user_timestamp_us, std::nullopt); + EXPECT_EQ(round_trip->frame_id, std::optional(456)); +} + +TEST(VideoFrameMetadataTest, BothFieldsArePreserved) { + VideoFrameMetadata metadata; + metadata.user_timestamp_us = 123; + metadata.frame_id = 456; + + auto proto_metadata = toProto(metadata); + ASSERT_TRUE(proto_metadata.has_value()); + EXPECT_TRUE(proto_metadata->has_user_timestamp_us()); + EXPECT_TRUE(proto_metadata->has_frame_id()); + + auto round_trip = fromProto(*proto_metadata); + ASSERT_TRUE(round_trip.has_value()); + EXPECT_EQ(round_trip->user_timestamp_us, std::optional(123)); + EXPECT_EQ(round_trip->frame_id, std::optional(456)); +} + +TEST(VideoFrameMetadataTest, EmptyProtoMetadataIsIgnored) { + proto::FrameMetadata metadata; + EXPECT_FALSE(fromProto(metadata).has_value()); +} + +TEST(TrackPublishOptionsTest, PacketTrailerFeaturesRoundTrip) { + TrackPublishOptions options; + options.packet_trailer_features.user_timestamp = true; + options.packet_trailer_features.frame_id = true; + + proto::TrackPublishOptions proto_options = toProto(options); + ASSERT_EQ(proto_options.packet_trailer_features_size(), 2); + EXPECT_EQ(proto_options.packet_trailer_features(0), + proto::PacketTrailerFeature::PTF_USER_TIMESTAMP); + EXPECT_EQ(proto_options.packet_trailer_features(1), + proto::PacketTrailerFeature::PTF_FRAME_ID); + + TrackPublishOptions round_trip = fromProto(proto_options); + EXPECT_TRUE(round_trip.packet_trailer_features.user_timestamp); + EXPECT_TRUE(round_trip.packet_trailer_features.frame_id); +} + +class VideoFrameMetadataServerTest : public LiveKitTestBase {}; + +TEST_F(VideoFrameMetadataServerTest, + UserTimestampRoundTripsToReceiverEventCallback) { + skipIfNotConfigured(); + + Room sender_room; + Room receiver_room; + RoomOptions options; + + ASSERT_TRUE(receiver_room.Connect(config_.url, config_.receiver_token, options)); + ASSERT_TRUE(sender_room.Connect(config_.url, config_.caller_token, options)); + ASSERT_NE(sender_room.localParticipant(), nullptr); + ASSERT_NE(receiver_room.localParticipant(), nullptr); + + const std::string sender_identity = sender_room.localParticipant()->identity(); + ASSERT_FALSE(sender_identity.empty()); + ASSERT_TRUE(waitForParticipant(&receiver_room, sender_identity, 10s)); + + std::mutex mutex; + std::condition_variable cv; + std::optional received_user_timestamp_us; + + receiver_room.setOnVideoFrameEventCallback( + sender_identity, TrackSource::SOURCE_CAMERA, + [&mutex, &cv, &received_user_timestamp_us](const VideoFrameEvent &event) { + std::lock_guard lock(mutex); + if (event.metadata && event.metadata->user_timestamp_us.has_value()) { + received_user_timestamp_us = event.metadata->user_timestamp_us; + cv.notify_all(); + } + }); + + auto source = std::make_shared(16, 16); + auto track = LocalVideoTrack::createLocalVideoTrack("metadata-track", source); + + TrackPublishOptions publish_options; + publish_options.source = TrackSource::SOURCE_CAMERA; + publish_options.simulcast = false; + publish_options.packet_trailer_features.user_timestamp = true; + + ASSERT_NO_THROW(sender_room.localParticipant()->publishTrack(track, publish_options)); + + VideoFrame frame = VideoFrame::create(16, 16, VideoBufferType::RGBA); + std::fill(frame.data(), frame.data() + frame.dataSize(), 0x7f); + + const std::uint64_t expected_user_timestamp_us = getTimestampUs(); + VideoCaptureOptions capture_options; + capture_options.timestamp_us = + static_cast(expected_user_timestamp_us); + capture_options.metadata = VideoFrameMetadata{}; + capture_options.metadata->user_timestamp_us = expected_user_timestamp_us; + + const auto deadline = std::chrono::steady_clock::now() + 10s; + while (std::chrono::steady_clock::now() < deadline) { + source->captureFrame(frame, capture_options); + + std::unique_lock lock(mutex); + if (cv.wait_for(lock, 100ms, [&received_user_timestamp_us] { + return received_user_timestamp_us.has_value(); + })) { + break; + } + } + + { + std::unique_lock lock(mutex); + ASSERT_TRUE(received_user_timestamp_us.has_value()) + << "Timed out waiting for user timestamp metadata"; + EXPECT_EQ(*received_user_timestamp_us, expected_user_timestamp_us); + } + + receiver_room.clearOnVideoFrameCallback(sender_identity, + TrackSource::SOURCE_CAMERA); + if (track->publication()) { + sender_room.localParticipant()->unpublishTrack(track->publication()->sid()); + } +} + +} // namespace test +} // namespace livekit diff --git a/src/tests/stress/test_latency_measurement.cpp b/src/tests/stress/test_latency_measurement.cpp index ed988b79..c60e9680 100644 --- a/src/tests/stress/test_latency_measurement.cpp +++ b/src/tests/stress/test_latency_measurement.cpp @@ -343,6 +343,9 @@ TEST_F(LatencyMeasurementTest, AudioLatency) { } // Clean up + ASSERT_NE(audio_track, nullptr) << "Audio track is null"; + ASSERT_NE(audio_track->publication(), nullptr) + << "Audio track publication is null"; sender_room->localParticipant()->unpublishTrack( audio_track->publication()->sid()); diff --git a/src/video_source.cpp b/src/video_source.cpp index 09e4858c..e593e884 100644 --- a/src/video_source.cpp +++ b/src/video_source.cpp @@ -16,7 +16,6 @@ #include "livekit/video_source.h" -#include #include #include "ffi.pb.h" @@ -45,8 +44,7 @@ VideoSource::VideoSource(int width, int height) } void VideoSource::captureFrame(const VideoFrame &frame, - std::int64_t timestamp_us, - VideoRotation rotation) { + const VideoCaptureOptions &options) { if (!handle_) { return; } @@ -56,12 +54,24 @@ void VideoSource::captureFrame(const VideoFrame &frame, auto *msg = req.mutable_capture_video_frame(); msg->set_source_handle(handle_.get()); msg->mutable_buffer()->CopyFrom(buf); - msg->set_timestamp_us(timestamp_us); - msg->set_rotation(static_cast(rotation)); + msg->set_timestamp_us(options.timestamp_us); + msg->set_rotation(static_cast(options.rotation)); + if (auto metadata = toProto(options.metadata)) { + msg->mutable_metadata()->CopyFrom(*metadata); + } proto::FfiResponse resp = FfiClient::instance().sendRequest(req); if (!resp.has_capture_video_frame()) { throw std::runtime_error("FfiResponse missing capture_video_frame"); } } -} // namespace livekit \ No newline at end of file +void VideoSource::captureFrame(const VideoFrame &frame, + std::int64_t timestamp_us, + VideoRotation rotation) { + VideoCaptureOptions options; + options.timestamp_us = timestamp_us; + options.rotation = rotation; + captureFrame(frame, options); +} + +} // namespace livekit diff --git a/src/video_stream.cpp b/src/video_stream.cpp index 657d4db1..3ea9c8aa 100644 --- a/src/video_stream.cpp +++ b/src/video_stream.cpp @@ -184,8 +184,13 @@ void VideoStream::onFfiEvent(const proto::FfiEvent &event) { // You should implement this static function in your VideoFrame class. VideoFrame frame = VideoFrame::fromOwnedInfo(fr.buffer()); - VideoFrameEvent ev{std::move(frame), fr.timestamp_us(), - static_cast(fr.rotation())}; + VideoFrameEvent ev; + ev.frame = std::move(frame); + ev.timestamp_us = fr.timestamp_us(); + ev.rotation = static_cast(fr.rotation()); + if (fr.has_metadata()) { + ev.metadata = fromProto(fr.metadata()); + } pushFrame(std::move(ev)); } else if (vse.has_eos()) { pushEos(); diff --git a/src/video_utils.cpp b/src/video_utils.cpp index 8b034de9..9ac551bd 100644 --- a/src/video_utils.cpp +++ b/src/video_utils.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "livekit/video_frame.h" +#include "video_utils.h" #include #include @@ -86,6 +86,43 @@ VideoBufferType fromProto(proto::VideoBufferType t) { } } +std::optional +toProto(const std::optional &metadata) { + if (!metadata.has_value()) { + return std::nullopt; + } + + proto::FrameMetadata proto_metadata; + if (metadata->user_timestamp_us.has_value()) { + proto_metadata.set_user_timestamp_us(*metadata->user_timestamp_us); + } + if (metadata->frame_id.has_value()) { + proto_metadata.set_frame_id(*metadata->frame_id); + } + + if (!proto_metadata.has_user_timestamp_us() && !proto_metadata.has_frame_id()) { + return std::nullopt; + } + + return proto_metadata; +} + +std::optional fromProto(const proto::FrameMetadata &metadata) { + VideoFrameMetadata out; + if (metadata.has_user_timestamp_us()) { + out.user_timestamp_us = metadata.user_timestamp_us(); + } + if (metadata.has_frame_id()) { + out.frame_id = metadata.frame_id(); + } + + if (!out.user_timestamp_us.has_value() && !out.frame_id.has_value()) { + return std::nullopt; + } + + return out; +} + proto::VideoBufferInfo toProto(const VideoFrame &frame) { proto::VideoBufferInfo info; diff --git a/src/video_utils.h b/src/video_utils.h index 69664154..eb02a6d2 100644 --- a/src/video_utils.h +++ b/src/video_utils.h @@ -17,6 +17,7 @@ #pragma once #include "livekit/video_frame.h" +#include "livekit/video_source.h" #include "video_frame.pb.h" namespace livekit { @@ -28,5 +29,8 @@ VideoFrame convertViaFfi(const VideoFrame &frame, VideoBufferType dst, bool flip_y); proto::VideoBufferType toProto(VideoBufferType t); VideoBufferType fromProto(proto::VideoBufferType t); +std::optional +toProto(const std::optional &metadata); +std::optional fromProto(const proto::FrameMetadata &metadata); } // namespace livekit
LiveKit Ecosystem