Attachment 516083 Details for Bug 580531

[patch] Patch: Implement mozFrameDelay and mozPaintCount

580531-paint-delay.patch (text/plain), 14.24 KB, created by Chris Pearce [:cpearce (Not reading bugmail)]

(hide)

Description:

Filename:

MIME Type:

Creator: Chris Pearce [:cpearce (Not reading bugmail)]

Size: 14.24 KB

patch

obsolete

># HG changeset patch
># User Chris Pearce <chris@pearce.org.nz>
># Parent c4de363ca5971acec175205939734c6f391cf68b
>Bug 580531 - Implement HTMLMediaElement.mozPaintedFrames and mozFrameDelay. r=?
>
>diff --git a/content/html/content/src/nsHTMLVideoElement.cpp b/content/html/content/src/nsHTMLVideoElement.cpp
>--- a/content/html/content/src/nsHTMLVideoElement.cpp
>+++ b/content/html/content/src/nsHTMLVideoElement.cpp
>@@ -199,8 +199,21 @@ NS_IMETHODIMP nsHTMLVideoElement::GetMoz
> }
> 
> NS_IMETHODIMP nsHTMLVideoElement::GetMozPresentedFrames(PRUint32 *aMozPresentedFrames)
> {
>   NS_ASSERTION(NS_IsMainThread(), "Should be on main thread.");
>   *aMozPresentedFrames = mDecoder ? mDecoder->GetFrameStatistics().GetPresentedFrames() : 0;
>   return NS_OK;
> }
>+
>+NS_IMETHODIMP nsHTMLVideoElement::GetMozPaintedFrames(PRUint32 *aMozPaintedFrames)
>+{
>+  NS_ASSERTION(NS_IsMainThread(), "Should be on main thread.");
>+  *aMozPaintedFrames = (!mDecoder || !GetImageContainer()) ? 0 : GetImageContainer()->GetPaintCount();
>+  return NS_OK;
>+}
>+
>+NS_IMETHODIMP nsHTMLVideoElement::GetMozFrameDelay(double *aMozFrameDelay) {
>+  NS_ASSERTION(NS_IsMainThread(), "Should be on main thread.");
>+  *aMozFrameDelay = mDecoder ? mDecoder->GetFrameDelay() : 0;
>+  return NS_OK;
>+}
>diff --git a/content/media/nsBuiltinDecoderStateMachine.cpp b/content/media/nsBuiltinDecoderStateMachine.cpp
>--- a/content/media/nsBuiltinDecoderStateMachine.cpp
>+++ b/content/media/nsBuiltinDecoderStateMachine.cpp
>@@ -980,17 +980,17 @@ nsresult nsBuiltinDecoderStateMachine::R
>         LoadMetadata();
>         if (mState == DECODER_STATE_SHUTDOWN) {
>           continue;
>         }
> 
>         VideoData* videoData = FindStartTime();
>         if (videoData) {
>           MonitorAutoExit exitMon(mDecoder->GetMonitor());
>-          RenderVideoFrame(videoData);
>+          RenderVideoFrame(videoData, TimeStamp::Now());
>         }
> 
>         // Start the decode threads, so that we can pre buffer the streams.
>         // and calculate the start time in order to determine the duration.
>         if (NS_FAILED(StartDecodeThreads())) {
>           continue;
>         }
> 
>@@ -1102,17 +1102,17 @@ nsresult nsBuiltinDecoderStateMachine::R
>             PRInt64 startTime = (audio && audio->mTime < seekTime) ? audio->mTime : seekTime;
>             mAudioStartTime = startTime;
>             mPlayDuration = TimeDuration::FromMilliseconds(startTime - mStartTime);
>             if (HasVideo()) {
>               nsAutoPtr<VideoData> video(mReader->mVideoQueue.PeekFront());
>               if (video) {
>                 NS_ASSERTION(video->mTime <= seekTime && seekTime <= video->mEndTime,
>                              "Seek target should lie inside the first frame after seek");
>-                RenderVideoFrame(video);
>+                RenderVideoFrame(video, TimeStamp::Now());
>                 mReader->mVideoQueue.PopFront();
>                 nsCOMPtr<nsIRunnable> event =
>                   NS_NewRunnableMethod(mDecoder, &nsBuiltinDecoder::Invalidate);
>                 NS_DispatchToMainThread(event, NS_DISPATCH_NORMAL);
>               }
>             }
>           }
>         }
>@@ -1265,28 +1265,32 @@ nsresult nsBuiltinDecoderStateMachine::R
>       }
>       break;
>     }
>   }
> 
>   return NS_OK;
> }
> 
>-void nsBuiltinDecoderStateMachine::RenderVideoFrame(VideoData* aData)
>+void nsBuiltinDecoderStateMachine::RenderVideoFrame(VideoData* aData,
>+                                                    TimeStamp aTarget)
> {
>   NS_ASSERTION(IsCurrentThread(mDecoder->mStateMachineThread), "Should be on state machine thread.");
> 
>   if (aData->mDuplicate) {
>     return;
>   }
> 
>   nsRefPtr<Image> image = aData->mImage;
>   if (image) {
>     const nsVideoInfo& info = mReader->GetInfo();
>-    mDecoder->SetVideoData(gfxIntSize(info.mDisplay.width, info.mDisplay.height), info.mPixelAspectRatio, image);
>+    mDecoder->SetVideoData(gfxIntSize(info.mDisplay.width, info.mDisplay.height),
>+                           info.mPixelAspectRatio,
>+                           image,
>+                           aTarget);
>   }
> }
> 
> PRInt64
> nsBuiltinDecoderStateMachine::GetAudioClock()
> {
>   NS_ASSERTION(IsCurrentThread(mDecoder->mStateMachineThread), "Should be on state machine thread.");
>   if (!mAudioStream || !HasAudio())
>@@ -1357,22 +1361,24 @@ void nsBuiltinDecoderStateMachine::Advan
>       if (frame && !currentFrame) {
>         PRInt64 now = (TimeStamp::Now() - mPlayStartTime + mPlayDuration).ToMilliseconds();
>         remainingTime = frame->mTime - mStartTime - now;
>       }
>     }
> 
>     if (currentFrame) {
>       // Decode one frame and display it
>+      TimeStamp presTime = mPlayStartTime - mPlayDuration +
>+        TimeDuration::FromMilliseconds(currentFrame->mTime - mStartTime);
>       NS_ASSERTION(currentFrame->mTime >= mStartTime, "Should have positive frame time");
>       {
>         MonitorAutoExit exitMon(mDecoder->GetMonitor());
>         // If we have video, we want to increment the clock in steps of the frame
>         // duration.
>-        RenderVideoFrame(currentFrame);
>+        RenderVideoFrame(currentFrame, presTime);
>       }
>       mDecoder->GetFrameStatistics().NotifyPresentedFrame();
>       PRInt64 now = (TimeStamp::Now() - mPlayStartTime + mPlayDuration).ToMilliseconds();
>       remainingTime = currentFrame->mEndTime - mStartTime - now;
>       currentFrame = nsnull;
>     }
> 
>     // Kick the decode thread in case it filled its buffers and put itself
>diff --git a/content/media/nsBuiltinDecoderStateMachine.h b/content/media/nsBuiltinDecoderStateMachine.h
>--- a/content/media/nsBuiltinDecoderStateMachine.h
>+++ b/content/media/nsBuiltinDecoderStateMachine.h
>@@ -303,17 +303,18 @@ protected:
>   // Update only the state machine's current playback position (and duration,
>   // if unknown).  Does not update the playback position on the decoder or
>   // media element -- use UpdatePlaybackPosition for that.  Called on the state
>   // machine thread, caller must hold the decoder lock.
>   void UpdatePlaybackPositionInternal(PRInt64 aTime);
> 
>   // Performs YCbCr to RGB conversion, and pushes the image down the
>   // rendering pipeline. Called on the state machine thread.
>-  void RenderVideoFrame(VideoData* aData);
>+  void RenderVideoFrame(VideoData* aData,
>+                        TimeStamp aTarget);
> 
>   // If we have video, display a video frame if it's time for display has
>   // arrived, otherwise sleep until it's time for the next sample. Update
>   // the current frame time as appropriate, and trigger ready state update.
>   // The decoder monitor must be held with exactly one lock count. Called
>   // on the state machine thread.
>   void AdvanceFrame();
> 
>diff --git a/content/media/nsMediaDecoder.cpp b/content/media/nsMediaDecoder.cpp
>--- a/content/media/nsMediaDecoder.cpp
>+++ b/content/media/nsMediaDecoder.cpp
>@@ -250,35 +250,50 @@ void nsMediaDecoder::FireTimeUpdate()
> {
>   if (!mElement)
>     return;
>   mElement->FireTimeUpdate(PR_TRUE);
> }
> 
> void nsMediaDecoder::SetVideoData(const gfxIntSize& aSize,
>                                   float aPixelAspectRatio,
>-                                  Image* aImage)
>+                                  Image* aImage,
>+                                  TimeStamp aTarget)
> {
>   nsAutoLock lock(mVideoUpdateLock);
> 
>   if (mRGBWidth != aSize.width || mRGBHeight != aSize.height ||
>       mPixelAspectRatio != aPixelAspectRatio) {
>     mRGBWidth = aSize.width;
>     mRGBHeight = aSize.height;
>     mPixelAspectRatio = aPixelAspectRatio;
>     mSizeChanged = PR_TRUE;
>   }
>   if (mImageContainer && aImage) {
>     gfxIntSize oldFrameSize = mImageContainer->GetCurrentSize();
>+
>+    TimeStamp paintTime = mImageContainer->GetPaintTime();
>+    if (!paintTime.IsNull() && !mPaintTarget.IsNull()) {
>+      mPaintDelay = paintTime - mPaintTarget;
>+    }
>+
>     mImageContainer->SetCurrentImage(aImage);
>     gfxIntSize newFrameSize = mImageContainer->GetCurrentSize();
>     if (oldFrameSize != newFrameSize) {
>       mImageContainerSizeChanged = PR_TRUE;
>     }
>   }
>+
>+  mPaintTarget = aTarget;
>+}
>+
>+double nsMediaDecoder::GetFrameDelay()
>+{
>+  nsAutoLock lock(mVideoUpdateLock);
>+  return mPaintDelay.ToSeconds();
> }
> 
> void nsMediaDecoder::PinForSeek()
> {
>   nsMediaStream* stream = GetCurrentStream();
>   if (!stream || mPinnedForSeek) {
>     return;
>   }
>diff --git a/content/media/nsMediaDecoder.h b/content/media/nsMediaDecoder.h
>--- a/content/media/nsMediaDecoder.h
>+++ b/content/media/nsMediaDecoder.h
>@@ -261,16 +261,21 @@ public:
>       mDecoder->GetFrameStatistics().NotifyDecodedFrames(mParsed, mDecoded);
>     }
>   private:
>     nsMediaDecoder* mDecoder;
>     PRUint32& mParsed;
>     PRUint32& mDecoded;
>   };
> 
>+  // Time in seconds by which the last painted video frame was late by.
>+  // E.g. if the last painted frame should have been painted at time t,
>+  // but was actually painted at t+n, this returns n in seconds. Threadsafe.
>+  double GetFrameDelay();
>+
>   // Return statistics. This is used for progress events and other things.
>   // This can be called from any thread. It's only a snapshot of the
>   // current state, since other threads might be changing the state
>   // at any time.
>   virtual Statistics GetStatistics() = 0;
>   
>   // Return the frame decode/paint related statistics.
>   FrameStatistics& GetFrameStatistics() { return mFrameStats; }
>@@ -356,21 +361,23 @@ public:
>   // their nsMediaStream.
>   virtual void MoveLoadsToBackground()=0;
> 
>   // Gets the image container for the media element. Will return null if
>   // the element is not a video element. This can be called from any
>   // thread; ImageContainers can be used from any thread.
>   ImageContainer* GetImageContainer() { return mImageContainer; }
> 
>-  // Set the video width, height, pixel aspect ratio, and current image.
>-  // Ownership of the image is transferred to the decoder.
>+  // Set the video width, height, pixel aspect ratio, current image and
>+  // target paint time of the next video frame to be displayed.
>+  // Ownership of the image is transferred to the layers subsystem.
>   void SetVideoData(const gfxIntSize& aSize,
>                     float aPixelAspectRatio,
>-                    Image* aImage);
>+                    Image* aImage,
>+                    TimeStamp aTarget);
> 
>   // Constructs the time ranges representing what segments of the media
>   // are buffered and playable.
>   virtual nsresult GetBuffered(nsTimeRanges* aBuffered) = 0;
> 
>   // Returns PR_TRUE if we can play the entire media through without stopping
>   // to buffer, given the current download and playback rates.
>   PRBool CanPlayThrough();
>@@ -403,16 +410,25 @@ protected:
>   nsHTMLMediaElement* mElement;
> 
>   PRInt32 mRGBWidth;
>   PRInt32 mRGBHeight;
> 
>   // Counters related to decode and presentation of frames.
>   FrameStatistics mFrameStats;
> 
>+  // The time at which the current video frame should have been painted.
>+  // Access protected by mVideoUpdateLock.
>+  TimeStamp mPaintTarget;
>+
>+  // The delay between the last video frame being presented and it being
>+  // painted. This is time elapsed after mPaintTarget until the most recently
>+  // painted frame appeared on screen. Access protected by mVideoUpdateLock.
>+  TimeDuration mPaintDelay;
>+
>   nsRefPtr<ImageContainer> mImageContainer;
> 
>   // Time that the last progress event was fired. Read/Write from the
>   // main thread only.
>   TimeStamp mProgressTime;
> 
>   // Time that data was last read from the media resource. Used for
>   // computing if the download has stalled and to rate limit progress events
>diff --git a/content/media/ogg/nsOggReader.cpp b/content/media/ogg/nsOggReader.cpp
>--- a/content/media/ogg/nsOggReader.cpp
>+++ b/content/media/ogg/nsOggReader.cpp
>@@ -38,16 +38,17 @@
>  * ***** END LICENSE BLOCK ***** */
> #include "nsError.h"
> #include "nsBuiltinDecoderStateMachine.h"
> #include "nsBuiltinDecoder.h"
> #include "nsOggReader.h"
> #include "VideoUtils.h"
> #include "theora/theoradec.h"
> #include "nsTimeRanges.h"
>+#include "mozilla/TimeStamp.h"
> 
> using namespace mozilla;
> 
> // Un-comment to enable logging of seek bisections.
> //#define SEEK_LOGGING
> 
> #ifdef PR_LOGGING
> extern PRLogModuleInfo* gBuiltinDecoderLog;
>@@ -285,17 +286,17 @@ nsresult nsOggReader::ReadMetadata()
> 
>   // Initialize the first Theora and Vorbis bitstreams. According to the
>   // Theora spec these can be considered the 'primary' bitstreams for playback.
>   // Extract the metadata needed from these streams.
>   // Set a default callback period for if we have no video data
>   if (mTheoraState && mTheoraState->Init()) {
>     gfxIntSize sz(mTheoraState->mInfo.pic_width,
>                   mTheoraState->mInfo.pic_height);
>-    mDecoder->SetVideoData(sz, mTheoraState->mPixelAspectRatio, nsnull);
>+    mDecoder->SetVideoData(sz, mTheoraState->mPixelAspectRatio, nsnull, TimeStamp::Now());
>   }
>   if (mVorbisState) {
>     mVorbisState->Init();
>   }
> 
>   if (!HasAudio() && !HasVideo() && mSkeletonState) {
>     // We have a skeleton track, but no audio or video, may as well disable
>     // the skeleton, we can't do anything useful with this media.
>diff --git a/dom/interfaces/html/nsIDOMHTMLVideoElement.idl b/dom/interfaces/html/nsIDOMHTMLVideoElement.idl
>--- a/dom/interfaces/html/nsIDOMHTMLVideoElement.idl
>+++ b/dom/interfaces/html/nsIDOMHTMLVideoElement.idl
>@@ -43,17 +43,17 @@
>  * <video> element.
>  *
>  * For more information on this interface, please see
>  * http://www.whatwg.org/specs/web-apps/current-work/#video
>  *
>  * @status UNDER_DEVELOPMENT
>  */
> 
>-[scriptable, uuid(8a2e5756-e9a3-404f-9592-4e5b5830de85)]
>+[scriptable, uuid(e1f52aa5-9962-4019-b7b3-af3aee6e4d48)]
> interface nsIDOMHTMLVideoElement : nsIDOMHTMLMediaElement
> {
>            attribute long width; 
>            attribute long height;
>   readonly attribute unsigned long videoWidth;
>   readonly attribute unsigned long videoHeight;
>            attribute DOMString poster;
>            
>@@ -65,10 +65,15 @@ interface nsIDOMHTMLVideoElement : nsIDO
>   // A count of the number of frames that have been decoded. We may drop
>   // frames if the decode is taking too much time.
>   readonly attribute unsigned long mozDecodedFrames;
> 
>   // A count of the number of frames that have been presented to the rendering
>   // pipeline. We may drop frames if they arrive late at the renderer.
>   readonly attribute unsigned long mozPresentedFrames;
> 
>+  // Number of presented frames which were painted on screen.
>+  readonly attribute unsigned long mozPaintedFrames;
>+
>+  // Time which the last painted video frame was late by, in seconds.
>+  readonly attribute double mozFrameDelay;
> };
>

Flags:

roc: review+

Actions: View | Diff | Review

Attachments on bug 580531: 458935 | 458937 | 465555 | 465556 | 468611 | 468612 | 468617 | 495942 | 496253 | 496254 | 496258 | 514667 | 515506 | 515799 | 516082 | 516083 | 516105 | 516139 | 516418 | 516419