From 03418b090b1652cf92da5ebb93a950633cfe11cf Mon Sep 17 00:00:00 2001 From: Przemyslaw Gorszkowski Date: Thu, 28 May 2026 14:56:58 +0200 Subject: [PATCH 1/2] [HolePunch][westerossink] pull current pts from video sink and provide it with requestVideoFrameCallback --- .../gstreamer/MediaPlayerPrivateGStreamer.cpp | 3 ++ .../GStreamerHolePunchQuirkWesteros.cpp | 43 +++++++++++++++++++ .../GStreamerHolePunchQuirkWesteros.h | 1 + .../platform/gstreamer/GStreamerQuirks.cpp | 8 ++++ .../platform/gstreamer/GStreamerQuirks.h | 3 ++ 5 files changed, 58 insertions(+) diff --git a/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp b/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp index aef56943a2ecf..b462f9c6eb5b3 100644 --- a/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp +++ b/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp @@ -4894,6 +4894,9 @@ WTFLogChannel& MediaPlayerPrivateGStreamer::logChannel() const std::optional MediaPlayerPrivateGStreamer::videoFrameMetadata() { Locker sampleLocker { m_sampleMutex }; + if (isHolePunchRenderingEnabled() && m_videoSink) + return GStreamerQuirksManager::singleton().videoFrameMetadata(m_videoSink, m_lastVideoFrameMetadataSampleCount); + if (!GST_IS_SAMPLE(m_sample.get())) return { }; diff --git a/Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkWesteros.cpp b/Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkWesteros.cpp index 4d7a6f4a95d05..a1f4a4acc2651 100644 --- a/Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkWesteros.cpp +++ b/Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkWesteros.cpp @@ -61,6 +61,49 @@ bool GStreamerHolePunchQuirkWesteros::setHolePunchVideoRectangle(GstElement* vid return true; } +std::optional GStreamerHolePunchQuirkWesteros::videoFrameMetadata(GRefPtr videoSink, uint64_t& lastVideoFrameMetadataSampleCount) +{ + if (UNLIKELY(!gstObjectHasProperty(videoSink.get(), "video_pts") + || !gstObjectHasProperty(videoSink.get(), "stats"))) + return { }; + + gint64 pts90kHz = 0; + GstStructure* stats = nullptr; + gint width = 0, height = 0; + g_object_get(videoSink.get(), + "video_pts", &pts90kHz, + "stats", &stats, + "video_width", &width, + "video_height", &height, + nullptr); + + if (!pts90kHz || !stats) + return { }; + + guint64 rendered = 0; + gst_structure_get_uint64(stats, "rendered", &rendered); + gst_structure_free(stats); + + if (rendered == lastVideoFrameMetadataSampleCount) + return { }; + lastVideoFrameMetadataSampleCount = rendered; + + double now = MonotonicTime::now().secondsSinceEpoch().seconds(); + + VideoFrameMetadata metadata; + // Convert 90kHz ticks to seconds + metadata.mediaTime = MediaTime(pts90kHz, 90000.0).toDouble(); + metadata.width = width; + metadata.height = height; + metadata.presentedFrames = rendered; + + // FIXME: presentationTime and expectedDisplayTime might not always have the same value, we should try getting more precise values. + metadata.presentationTime = now; + metadata.expectedDisplayTime = metadata.presentationTime; + + return metadata; +} + #undef GST_CAT_DEFAULT } // namespace WebCore diff --git a/Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkWesteros.h b/Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkWesteros.h index 15bdb1cf664ea..869a5e0d38fa3 100644 --- a/Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkWesteros.h +++ b/Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkWesteros.h @@ -33,6 +33,7 @@ class GStreamerHolePunchQuirkWesteros final : public GStreamerHolePunchQuirk { GstElement* createHolePunchVideoSink(bool, const MediaPlayer*) final; bool setHolePunchVideoRectangle(GstElement*, const IntRect&) final; bool requiresClockSynchronization() const final { return false; } + std::optional videoFrameMetadata(GRefPtr, uint64_t&) final; }; } // namespace WebCore diff --git a/Source/WebCore/platform/gstreamer/GStreamerQuirks.cpp b/Source/WebCore/platform/gstreamer/GStreamerQuirks.cpp index e3a31d1d5b0a2..796b5242beff0 100644 --- a/Source/WebCore/platform/gstreamer/GStreamerQuirks.cpp +++ b/Source/WebCore/platform/gstreamer/GStreamerQuirks.cpp @@ -201,6 +201,14 @@ bool GStreamerQuirksManager::sinksRequireClockSynchronization() const return m_holePunchQuirk->requiresClockSynchronization(); } +std::optional GStreamerQuirksManager::videoFrameMetadata(GRefPtr videoSink, uint64_t& lastVideoFrameMetadataSampleCount) +{ + if (!m_holePunchQuirk) + return { }; + + return m_holePunchQuirk->videoFrameMetadata(videoSink, lastVideoFrameMetadataSampleCount); +} + void GStreamerQuirksManager::configureElement(GstElement* element, OptionSet&& characteristics) { GST_DEBUG("Configuring element %" GST_PTR_FORMAT, element); diff --git a/Source/WebCore/platform/gstreamer/GStreamerQuirks.h b/Source/WebCore/platform/gstreamer/GStreamerQuirks.h index cdb656faa8ae6..2c228ec121f52 100644 --- a/Source/WebCore/platform/gstreamer/GStreamerQuirks.h +++ b/Source/WebCore/platform/gstreamer/GStreamerQuirks.h @@ -24,6 +24,7 @@ #include "GStreamerCommon.h" #include "MediaPlayer.h" +#include "VideoFrameMetadata.h" #include #include #include @@ -116,6 +117,7 @@ class GStreamerHolePunchQuirk : public GStreamerQuirkBase { virtual GstElement* createHolePunchVideoSink(bool, const MediaPlayer*) { return nullptr; } virtual bool setHolePunchVideoRectangle(GstElement*, const IntRect&) { return false; } virtual bool requiresClockSynchronization() const { return true; } + virtual std::optional videoFrameMetadata(GRefPtr, uint64_t&) { return { }; } }; class GStreamerQuirksManager : public RefCounted { @@ -143,6 +145,7 @@ class GStreamerQuirksManager : public RefCounted { GstElement* createHolePunchVideoSink(bool isLegacyPlaybin, const MediaPlayer*); void setHolePunchVideoRectangle(GstElement*, const IntRect&); bool sinksRequireClockSynchronization() const; + std::optional videoFrameMetadata(GRefPtr, uint64_t&); void setHolePunchEnabledForTesting(bool); From 026acfb60d57e8580c3178e7808684fdbd23c16e Mon Sep 17 00:00:00 2001 From: Przemyslaw Gorszkowski Date: Fri, 12 Jun 2026 14:28:55 +0200 Subject: [PATCH 2/2] [HolePunch][rialtosink] pull current pts from video sink and provide it with requestVideoFrameCallback --- .../GStreamerHolePunchQuirkRialto.cpp | 49 +++++++++++++++++++ .../gstreamer/GStreamerHolePunchQuirkRialto.h | 1 + 2 files changed, 50 insertions(+) diff --git a/Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkRialto.cpp b/Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkRialto.cpp index 903f5bf85acf3..d419c1e8a2d90 100644 --- a/Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkRialto.cpp +++ b/Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkRialto.cpp @@ -58,6 +58,55 @@ bool GStreamerHolePunchQuirkRialto::setHolePunchVideoRectangle(GstElement* video return true; } +std::optional GStreamerHolePunchQuirkRialto::videoFrameMetadata(GRefPtr videoSink, uint64_t& lastVideoFrameMetadataSampleCount) +{ + if (UNLIKELY(!gstObjectHasProperty(videoSink.get(), "video_pts") + || !gstObjectHasProperty(videoSink.get(), "stats"))) + return { }; + + gint64 pts90kHz = 0; + GstStructure* stats = nullptr; + g_object_get(videoSink.get(), "video_pts", &pts90kHz, "stats", &stats, nullptr); + + if (!pts90kHz || !stats) + return { }; + + guint64 rendered = 0; + gst_structure_get_uint64(stats, "rendered", &rendered); + gst_structure_free(stats); + + if (rendered == lastVideoFrameMetadataSampleCount) + return { }; + lastVideoFrameMetadataSampleCount = rendered; + + gint width = 0; + gint height = 0; + if (GstPad *sinkPad = gst_element_get_static_pad(GST_ELEMENT(videoSink.get()), "sink")) { + if (GstCaps *caps = gst_pad_get_current_caps(sinkPad)){ + GstStructure *s = gst_caps_get_structure(caps, 0); + gst_structure_get_int(s, "width", &width); + gst_structure_get_int(s, "height", &height); + gst_caps_unref(caps); + } + gst_object_unref(sinkPad); + } + + const auto now = MonotonicTime::now().secondsSinceEpoch().seconds(); + + VideoFrameMetadata metadata; + // Convert 90kHz ticks to seconds + metadata.mediaTime = MediaTime(pts90kHz, 90000.0).toDouble(); + metadata.width = width; + metadata.height = height; + metadata.presentedFrames = rendered; + + // FIXME: presentationTime and expectedDisplayTime might not always have the same value, we should try getting more precise values. + metadata.presentationTime = now; + metadata.expectedDisplayTime = metadata.presentationTime; + + return metadata; +} + #undef GST_CAT_DEFAULT } // namespace WebCore diff --git a/Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkRialto.h b/Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkRialto.h index a33e34b9dec06..d856713c98412 100644 --- a/Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkRialto.h +++ b/Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkRialto.h @@ -38,6 +38,7 @@ class GStreamerHolePunchQuirkRialto final : public GStreamerHolePunchQuirk { GstElement* createHolePunchVideoSink(bool, const MediaPlayer*) final; bool setHolePunchVideoRectangle(GstElement*, const IntRect&) final; bool requiresClockSynchronization() const final { return false; } + std::optional videoFrameMetadata(GRefPtr, uint64_t&) final; }; } // namespace WebCore