Attachment #495942 for bug #580531

Attachment #495942: Patch: playback statistics v2 for bug #580531

View | Details | Raw Unified | Return to bug 580531
Collapse All | Expand All






  // Returns the current video frame width and height.
  // If there is no video frame, returns the given default size.
  nsIntSize GetVideoSize(nsIntSize defaultSize);

  virtual nsresult SetAcceptHeader(nsIHttpChannel* aChannel);

  virtual nsXPCClassInfo* GetClassInfo();

  // Dispatches an event to increment the counter of the number of frames
  // painted. Called on the main thread by the nsVideoFrame when a new image
  // is painted.
  void NotifyPaintedFrame();
};

#endif

-      Line
+  Link Here
-       Lines 178-185
      nsresult nsHTMLVideoElement::SetAcceptHe
+  Link Here
         "audio/*;q=0.6,*/*;q=0.5");
     return aChannel->SetRequestHeader(NS_LITERAL_CSTRING("Accept"),
                                       value,
                                       PR_FALSE);
+}
 NS_IMPL_URI_ATTR(nsHTMLVideoElement, Poster, poster)
+/* readonly attribute unsigned long mozDecodedFrames; */
+NS_IMETHODIMP nsHTMLVideoElement::GetMozParsedFrames(PRUint32 *aMozDecodedFrames)
+{
+  NS_ASSERTION(NS_IsMainThread(), "Should be on main thread.");
+  *aMozDecodedFrames = mDecoder ? mDecoder->GetParsedFrames() : 0;
+  return NS_OK;
+}
+/* readonly attribute unsigned long mozDroppedFrames; */
+NS_IMETHODIMP nsHTMLVideoElement::GetMozDecodedFrames(PRUint32 *aMozDroppedFrames)
+{
+  NS_ASSERTION(NS_IsMainThread(), "Should be on main thread.");
+  *aMozDroppedFrames = mDecoder ? mDecoder->GetDecodedFrames() : 0;
+  return NS_OK;
+}
+NS_IMETHODIMP nsHTMLVideoElement::GetMozPresentedFrames(PRUint32 *aMozDroppedFrames)
+{
+  NS_ASSERTION(NS_IsMainThread(), "Should be on main thread.");
+  *aMozDroppedFrames = mDecoder ? mDecoder->GetPresentedFrames() : 0;
+  return NS_OK;
+}
+NS_IMETHODIMP nsHTMLVideoElement::GetMozPaintedFrames(PRUint32 *aMozDroppedFrames)
+{
+  NS_ASSERTION(NS_IsMainThread(), "Should be on main thread.");
+  *aMozDroppedFrames = mDecoder ? mDecoder->GetPaintedFrames() : 0;
+  return NS_OK;
+}
+NS_IMETHODIMP nsHTMLVideoElement::GetMozFrameDelay(float *aMozFrameDelay)
+{
+  NS_ASSERTION(NS_IsMainThread(), "Should be on main thread.");
+  *aMozFrameDelay = mImageContainer ? mImageContainer->GetPaintDelay().ToSeconds() : 0;
+  return NS_OK;
+}
+void nsHTMLVideoElement::NotifyPaintedFrame()
+{
+  NS_ASSERTION(NS_IsMainThread(), "Should be on main thread.");
+  if (mDecoder) {
+    mDecoder->NotifyPaintedFrame();
+  }
+}





{
  // Decode forward to the target frame. Start with video, if we have it.
  if (HasVideo()) {
    PRBool eof = PR_FALSE;
    PRInt64 startTime = -1;
    while (HasVideo() && !eof) {
      while (mVideoQueue.GetSize() == 0 && !eof) {
        PRBool skip = PR_FALSE;
        PRUint32 parsed=0, decoded=0;
        eof = !DecodeVideoFrame(skip, 0, parsed, decoded);
        {
          MonitorAutoExit exitReaderMon(mMonitor);
          MonitorAutoEnter decoderMon(mDecoder->GetMonitor());
          if (mDecoder->GetDecodeState() == nsBuiltinDecoderStateMachine::DECODER_STATE_SHUTDOWN) {
            return NS_ERROR_FAILURE;
          }
        }
      }





  // in mAudioQueue. Returns PR_TRUE when there's more audio to decode,
  // PR_FALSE if the audio is finished, end of file has been reached,
  // or an un-recoverable read error has occured.
  virtual PRBool DecodeAudioData() = 0;

  // Reads and decodes one video frame. Packets with a timestamp less
  // than aTimeThreshold will be decoded (unless they're not keyframes
  // and aKeyframeSkip is PR_TRUE), but will not be added to the queue.
  virtual PRBool DecodeVideoFrame(PRBool& aKeyframeSkip,
                                  PRInt64 aTimeThreshold,
                                  PRUint32& aParsed,
                                  PRUint32& aDecoded) = 0;

  virtual PRBool HasAudio() = 0;
  virtual PRBool HasVideo() = 0;

  // Read header data for all bitstreams in the file. Fills mInfo with
  // the data required to present the media. Returns NS_OK on success,
  // or NS_ERROR_FAILURE on failure.
  virtual nsresult ReadMetadata() = 0;

  template<class Data>
  Data* DecodeToFirstData(DecodeFn aDecodeFn,
                          MediaQueue<Data>& aQueue);

  // Wrapper so that DecodeVideoFrame(PRBool&,PRInt64) can be called from
  // DecodeToFirstData().
  PRBool DecodeVideoFrame() {
    PRBool f = PR_FALSE;
    // Ignore these, DecodeVideoFrame() is only called during seeking, so
    // these aren't relevant to performance.
    PRUint32 parsed=0, decoded=0;
    return DecodeVideoFrame(f, 0, parsed, decoded);
  }

  // Fills aRanges with ByteRanges denoting the sections of the media which
  // have been downloaded and are stored in the media cache. The reader
  // monitor must must be held with exactly one lock count. The nsMediaStream
  // must be pinned while calling this.
  nsresult GetBufferedBytes(nsTArray<ByteRange>& aRanges);






        ((!audioPump && audioPlaying && audioDecoded < lowAudioThreshold) ||
         (!videoPump && videoQueueSize < LOW_VIDEO_FRAMES)))
    {
      skipToNextKeyframe = PR_TRUE;
      LOG(PR_LOG_DEBUG, ("Skipping video decode to the next keyframe"));
    }

    // Video decode.
    PRUint32 parsed = 0, decoded = 0;
    if (videoPlaying && !videoWait) {
      // Time the video decode, so that if it's slow, we can increase our low
      // audio threshold to reduce the chance of an audio underrun while we're
      // waiting for a video decode to complete.
      TimeStamp start = TimeStamp::Now();
      videoPlaying = mReader->DecodeVideoFrame(skipToNextKeyframe,
                                               currentTime,
                                               parsed,
                                               decoded);
      TimeDuration decodeTime = TimeStamp::Now() - start;
      if (!decodeCloseToDownload &&
          THRESHOLD_FACTOR * decodeTime.ToMilliseconds() > lowAudioThreshold)
      {
        lowAudioThreshold =
          NS_MIN(static_cast<PRInt64>(THRESHOLD_FACTOR * decodeTime.ToMilliseconds()),
                 static_cast<PRInt64>(AMPLE_AUDIO_MS));
        ampleAudioThreshold = NS_MAX(THRESHOLD_FACTOR * lowAudioThreshold,
                                     ampleAudioThreshold);
        LOG(PR_LOG_DEBUG,
            ("Slow video decode, set lowAudioThreshold=%lld ampleAudioThreshold=%lld",
             lowAudioThreshold, ampleAudioThreshold));
      }
    }
    if (parsed || decoded) {
      mDecoder->NotifyDecodedFrames(parsed, decoded);
    }
    {
      MonitorAutoEnter mon(mDecoder->GetMonitor());
      mDecoder->GetMonitor().NotifyAll();
    }

    // Audio decode.
    if (audioPlaying && !audioWait) {
      audioPlaying = mReader->DecodeAudioData();

        LoadMetadata();
        if (mState == DECODER_STATE_SHUTDOWN) {
          continue;
        }

        VideoData* videoData = FindStartTime();
        if (videoData) {
          MonitorAutoExit exitMon(mDecoder->GetMonitor());
          RenderVideoFrame(videoData, TimeStamp::Now());
        }

        // Start the decode threads, so that we can pre buffer the streams.
        // and calculate the start time in order to determine the duration.
        if (NS_FAILED(StartDecodeThreads())) {
          continue;
        }


          }
          if (NS_SUCCEEDED(res)){
            SoundData* audio = HasAudio() ? mReader->mAudioQueue.PeekFront() : nsnull;
            NS_ASSERTION(!audio || (audio->mTime <= seekTime &&
                                    seekTime <= audio->mTime + audio->mDuration),
                         "Seek target should lie inside the first audio block after seek");
            PRInt64 startTime = (audio && audio->mTime < seekTime) ? audio->mTime : seekTime;
            mAudioStartTime = startTime;
            mPlayDuration = TimeDuration::FromMilliseconds(startTime - mStartTime);
            if (HasVideo()) {
              nsAutoPtr<VideoData> video(mReader->mVideoQueue.PeekFront());
              if (video) {
                NS_ASSERTION(video->mTime <= seekTime && seekTime <= video->mEndTime,
                             "Seek target should lie inside the first frame after seek");
                RenderVideoFrame(video, TimeStamp::Now());
                mReader->mVideoQueue.PopFront();
              }
            }
          }
        }
        mDecoder->StartProgressUpdates();
        if (mState == DECODER_STATE_SHUTDOWN)
          continue;

      }
      break;
    }
  }

  return NS_OK;
}

void nsBuiltinDecoderStateMachine::RenderVideoFrame(VideoData* aData, TimeStamp aTarget)
{
  NS_ASSERTION(IsCurrentThread(mDecoder->mStateMachineThread), "Should be on state machine thread.");

  if (aData->mDuplicate) {
    return;
  }

  nsRefPtr<Image> image = aData->mImage;
  if (image) {
    const nsVideoInfo& info = mReader->GetInfo();
    mDecoder->SetVideoData(gfxIntSize(info.mPicture.width, info.mPicture.height),
                           info.mPixelAspectRatio,
                           image,
                           aTarget);
  }
}

PRInt64
nsBuiltinDecoderStateMachine::GetAudioClock()
{
  NS_ASSERTION(IsCurrentThread(mDecoder->mStateMachineThread), "Should be on state machine thread.");
  if (!mAudioStream || !HasAudio())

    // the end of the audio, use the audio clock. However if we've finished
    // audio, or don't have audio, use the system clock.
    PRInt64 clock_time = -1;
    PRInt64 audio_time = GetAudioClock();
    if (HasAudio() && !mAudioCompleted && audio_time != -1) {
      clock_time = audio_time;
      // Resync against the audio clock, while we're trusting the
      // audio clock. This ensures no "drift", particularly on Linux.
      mPlayDuration = TimeDuration::FromMilliseconds(clock_time);
      mPlayStartTime = TimeStamp::Now();
    } else {
      // Sound is disabled on this system. Sync to the system clock.
      TimeDuration t = TimeStamp::Now() - mPlayStartTime + mPlayDuration;
      clock_time = (PRInt64)(1000 * t.ToSeconds());
      // Ensure the clock can never go backwards.
      NS_ASSERTION(mCurrentFrameTime <= clock_time, "Clock should go forwards");
      clock_time = NS_MAX(mCurrentFrameTime, clock_time) + mStartTime;
    }

    // Skip video frames up to the current playback position.
    NS_ASSERTION(clock_time >= mStartTime, "Should have positive clock time.");
    nsAutoPtr<VideoData> videoData;
    if (mReader->mVideoQueue.GetSize() > 0) {
      VideoData* data = mReader->mVideoQueue.PeekFront();
      while (clock_time >= data->mTime) {
        mVideoFrameEndTime = data->mEndTime;
        videoData = data;
        mReader->mVideoQueue.PopFront();
        mDecoder->UpdatePlaybackOffset(data->mOffset);
        if (mReader->mVideoQueue.GetSize() == 0)
          break;
        data = mReader->mVideoQueue.PeekFront();
      }
    }

    // The timestamp of the next frame to be displayed.
    TimeStamp nextFrameTime;

    if (videoData) {
      // Display the next frame.
      NS_ASSERTION(videoData->mTime >= mStartTime,
                   "Should have positive frame time");

      // Calculate the system-clock time at which the frame should start being displayed.
      TimeStamp presTime = mPlayStartTime - mPlayDuration +
        TimeDuration::FromMilliseconds(videoData->mTime - mStartTime);

      // Frame end system-clock time.
      nextFrameTime = mPlayStartTime - mPlayDuration +
        TimeDuration::FromMilliseconds(videoData->mEndTime - mStartTime);

      NS_ASSERTION(presTime <= TimeStamp::Now(),
                   "Frame must start before or at now");
      {
        MonitorAutoExit exitMon(mDecoder->GetMonitor());
        RenderVideoFrame(videoData, presTime);
        // duration.
        RenderVideoFrame(videoData);
      }
      mDecoder->NotifyPresentedFrame();

      videoData = nsnull;
    } else {
      // We don't have video, just assume an arbitrary "frame" duration.
      nextFrameTime =
        TimeStamp::Now() + TimeDuration::FromMilliseconds(AUDIO_DURATION_MS);
    }

    // Kick the decode thread in case it filled its buffers and put itself
    // to sleep.
    mDecoder->GetMonitor().NotifyAll();

    // Cap the current time to the larger of the audio and video end time.
    // This ensures that if we're running off the system clock, we don't

    }

    // If the number of audio/video samples queued has changed, either by
    // this function popping and playing a video sample, or by the audio
    // thread popping and playing an audio sample, we may need to update our
    // ready state. Post an update to do so.
    UpdateReadyState();

    // Wait until it's about time to display the next frame.
    TimeStamp now = TimeStamp::Now();
    if (now < nextFrameTime) {
      Wait((nextFrameTime - now).ToMilliseconds());
    }

  } else {
    if (IsPlaying()) {
      StopPlayback(AUDIO_PAUSE);
      mDecoder->GetMonitor().NotifyAll();
    }

    if (mState == DECODER_STATE_DECODING ||
        mState == DECODER_STATE_COMPLETED) {






  // Finds the end time of the last frame of data in the file, storing the value
  // in mEndTime if successful. The decoder must be held with exactly one lock
  // count. Called on the state machine thread.
  void FindEndTime();

  // Performs YCbCr to RGB conversion, and pushes the image down the
  // rendering pipeline. Called on the state machine thread.
  void RenderVideoFrame(VideoData* aData, TimeStamp aTarget);

  // If we have video, display a video frame if it's time for display has
  // arrived, otherwise sleep until it's time for the next sample. Update
  // the current frame time as appropriate, and trigger ready state update.
  // The decoder monitor must be held with exactly one lock count. Called
  // on the state machine thread.
  void AdvanceFrame();






// nsMediaDecoder::CanPlayThrough() calculation more stable in the case of
// fluctuating bitrates.
#define CAN_PLAY_THROUGH_MARGIN 20

nsMediaDecoder::nsMediaDecoder() :
  mElement(0),
  mRGBWidth(-1),
  mRGBHeight(-1),
  mStatsMonitor("nsMediaDecoder.stats"),
  mParsedFrames(0),
  mDecodedFrames(0),
  mPresentedFrames(0),
  mPaintedFrames(0),
  mVideoUpdateLock(nsnull),
  mPixelAspectRatio(1.0),
  mFrameBufferLength(0),
  mPinnedForSeek(PR_FALSE),
  mSizeChanged(PR_FALSE),
  mShuttingDown(PR_FALSE)
{
  MOZ_COUNT_CTOR(nsMediaDecoder);

{
  if (!mElement)
    return;
  mElement->FireTimeUpdate(PR_TRUE);
}

void nsMediaDecoder::SetVideoData(const gfxIntSize& aSize,
                                  float aPixelAspectRatio,
                                  Image* aImage,
                                  TimeStamp aTarget)
{
  nsAutoLock lock(mVideoUpdateLock);

  if (mRGBWidth != aSize.width || mRGBHeight != aSize.height ||
      mPixelAspectRatio != aPixelAspectRatio) {
    mRGBWidth = aSize.width;
    mRGBHeight = aSize.height;
    mPixelAspectRatio = aPixelAspectRatio;
    mSizeChanged = PR_TRUE;
  }
  if (mImageContainer && aImage) {
    mImageContainer->SetCurrentImage(aImage);
    mImageContainer->SetPaintTarget(aTarget);
  }
}

void nsMediaDecoder::PinForSeek()
{
  nsMediaStream* stream = GetCurrentStream();
  if (!stream || mPinnedForSeek) {
    return;

  // playback position, so that if the bitrate of the media fluctuates, or if
  // our download rate or decode rate estimation is otherwise inaccurate,
  // we don't suddenly discover that we need to buffer. This is particularly
  // required near the start of the media, when not much data is downloaded.
  PRInt64 readAheadMargin = stats.mPlaybackRate * CAN_PLAY_THROUGH_MARGIN;
  return stats.mTotalBytes == stats.mDownloadPosition ||
         stats.mDownloadPosition > stats.mPlaybackPosition + readAheadMargin;
}

void nsMediaDecoder::NotifyPaintedFrame() {
  mozilla::MonitorAutoEnter mon(mStatsMonitor);
  ++mPaintedFrames;
}

void nsMediaDecoder::NotifyPresentedFrame() {
  mozilla::MonitorAutoEnter mon(mStatsMonitor);
  ++mPresentedFrames;
}

void nsMediaDecoder::NotifyDecodedFrames(PRUint32 aParsed, PRUint32 aDecoded) {
  mozilla::MonitorAutoEnter mon(mStatsMonitor);
  mParsedFrames += aParsed;
  mDecodedFrames += aDecoded;
}

#define IMPL_GET_STAT_METHOD(X) \
PRUint32 nsMediaDecoder::Get##X##Frames() { \
  mozilla::MonitorAutoEnter mon(mStatsMonitor); \
  return m##X##Frames; \
}

IMPL_GET_STAT_METHOD(Parsed);
IMPL_GET_STAT_METHOD(Decoded);
IMPL_GET_STAT_METHOD(Presented);
IMPL_GET_STAT_METHOD(Painted);







#include "nsIPrincipal.h"
#include "nsSize.h"
#include "prlog.h"
#include "gfxContext.h"
#include "gfxRect.h"
#include "nsITimer.h"
#include "ImageLayers.h"
#include "mozilla/Monitor.h"

class nsHTMLMediaElement;
class nsMediaStream;
class nsIStreamListener;
class nsTimeRanges;

// The size to use for audio data frames in MozAudioAvailable events.
// This value is per channel, and is chosen to give ~43 fps of events,

// which can be called from any thread.
class nsMediaDecoder : public nsIObserver
{
public:
  typedef mozilla::TimeStamp TimeStamp;
  typedef mozilla::TimeDuration TimeDuration;
  typedef mozilla::layers::ImageContainer ImageContainer;
  typedef mozilla::layers::Image Image;
  typedef mozilla::Monitor Monitor;

  nsMediaDecoder();
  virtual ~nsMediaDecoder();

  // Create a new decoder of the same type as this one.
  virtual nsMediaDecoder* Clone() = 0;

  // Perform any initialization required for the decoder.

  // the element is not a video element. This can be called from any
  // thread; ImageContainers can be used from any thread.
  ImageContainer* GetImageContainer() { return mImageContainer; }

  // Set the video width, height, pixel aspect ratio, and current image.
  // Ownership of the image is transferred to the decoder.
  void SetVideoData(const gfxIntSize& aSize,
                    float aPixelAspectRatio,
                    Image* aImage,
                    TimeStamp aTarget);

  // Constructs the time ranges representing what segments of the media
  // are buffered and playable.
  virtual nsresult GetBuffered(nsTimeRanges* aBuffered) = 0;

  // Returns PR_TRUE if we can play the entire media through without stopping
  // to buffer, given the current download and playback rates.
  PRBool CanPlayThrough();

  // Returns number of frames which have been parsed from the media.
  // Can be called on any thread.
  PRUint32 GetParsedFrames();

  // Returns the number of parsed frames which have been decoded.
  // Can be called on any thread.
  PRUint32 GetDecodedFrames();

  // Returns the number of decoded frames which have been sent to the rendering
  // pipeline for painting ("presented").
  // Can be called on any thread.
  PRUint32 GetPresentedFrames();

  // Returns the number of presented frames which ended up being painted.
  // Can be called on any thread.
  PRUint32 GetPaintedFrames();

  // Playback statistics gathering functions. Called when frames reach various
  // stages through the decode/rendering pipeline. Can be called on any thread.
  void NotifyDecodedFrames(PRUint32 aParsed, PRUint32 aDecoded);
  void NotifyPresentedFrame();
  void NotifyPaintedFrame();

protected:

  // Start timer to update download progress information.
  nsresult StartProgress();

  // Stop progress information timer.
  nsresult StopProgress();


  // This should only ever be accessed from the main thread.
  // It is set in Init and cleared in Shutdown when the element goes away.
  // The decoder does not add a reference the element.
  nsHTMLMediaElement* mElement;

  PRInt32 mRGBWidth;
  PRInt32 mRGBHeight;

  // Monitor to protect access of playback statistics.
  Monitor mStatsMonitor;

  // Playback statistics counters. Access protected by mStatsMonitor;
  PRUint32 mParsedFrames;
  PRUint32 mDecodedFrames;
  PRUint32 mPresentedFrames;
  PRUint32 mPaintedFrames;
  
  nsRefPtr<ImageContainer> mImageContainer;

  // Time that the last progress event was fired. Read/Write from the
  // main thread only.
  TimeStamp mProgressTime;

  // Time that data was last read from the media resource. Used for
  // computing if the download has stalled and to rate limit progress events






  // Initialize the first Theora and Vorbis bitstreams. According to the
  // Theora spec these can be considered the 'primary' bitstreams for playback.
  // Extract the metadata needed from these streams.
  // Set a default callback period for if we have no video data
  if (mTheoraState && mTheoraState->Init()) {
    gfxIntSize sz(mTheoraState->mInfo.pic_width,
                  mTheoraState->mInfo.pic_height);
    mDecoder->SetVideoData(sz, mTheoraState->mPixelAspectRatio, nsnull, TimeStamp::Now());
  }
  if (mVorbisState) {
    mVorbisState->Init();
  }

  if (!HasAudio() && !HasVideo() && mSkeletonState) {
    // We have a skeleton track, but no audio or video, may as well disable
    // the skeleton, we can't do anything useful with this media.

    }
    if (!aFrames.AppendElement(v)) {
      delete v;
    }
  }
  return NS_OK;
}

PRBool nsOggReader::DecodeVideoFrame(PRBool& aKeyframeSkip,
                                     PRInt64 aTimeThreshold,
                                     PRUint32& aParsed,
                                     PRUint32& aDecoded)
{
  aParsed = aDecoded = 0;
  MonitorAutoEnter mon(mMonitor);
  NS_ASSERTION(mDecoder->OnStateMachineThread() || mDecoder->OnDecodeThread(),
               "Should be on state machine or AV thread.");
  // We chose to keep track of the Theora granulepos ourselves, rather than
  // rely on th_decode_packetin() to do it for us. This is because
  // th_decode_packetin() simply works by incrementing a counter every time
  // it's called, so if we drop frames and don't call it, subsequent granulepos
  // will be wrong. Whenever we read a packet which has a granulepos, we use

        // Failed to read another page, must be the end of file. We can't have
        // already encountered an end of bitstream packet, else we wouldn't be
        // here, so this bitstream must be missing its end of stream packet, or
        // is otherwise corrupt (oggz-chop can output files like this). Inform
        // the queue that there will be no more frames.
        mVideoQueue.Finish();
        return PR_FALSE;
      }
      aParsed++;

      if (packet.granulepos > 0) {
        // We've found a packet with a granulepos, we can now determine the
        // buffered packet's timestamps, as well as the timestamps for any
        // packets we read subsequently.
        mTheoraGranulepos = packet.granulepos;
      }
    

    NS_ASSERTION(mTheoraGranulepos > 0, "We must Theora granulepos!");
    
    if (!ReadOggPacket(mTheoraState, &packet)) {
      // Failed to read from file, so EOF or other premature failure.
      // Inform the queue that there will be no more frames.
      mVideoQueue.Finish();
      return PR_FALSE;
    }
    aParsed++;

    endOfStream = packet.e_o_s != 0;

    // Maintain the Theora granulepos. We must do this even if we drop frames,
    // otherwise our clock will be wrong after we've skipped frames.
    if (packet.granulepos != -1) {
      // Incoming packet has a granulepos, use that as it's granulepos.
      mTheoraGranulepos = packet.granulepos;

  for (PRUint32 i = 0; i < frames.Length(); i++) {
    nsAutoPtr<VideoData> data(frames[i].forget());
    if (aKeyframeSkip && data->mKeyframe) {
      aKeyframeSkip = PR_FALSE;
    }

    if (!aKeyframeSkip) {
      mVideoQueue.Push(data.forget());
      aDecoded++;
    }
  }

  if (endOfStream) {
    // We've encountered an end of bitstream packet. Inform the queue that
    // there will be no more frames.
    mVideoQueue.Finish();
  }

    return res;
  }

  // We have an active Theora bitstream. Decode the next Theora frame, and
  // extract its keyframe's time.
  PRBool eof;
  do {
    PRBool skip = PR_FALSE;
    PRUint32 parsed, decoded;
    eof = !DecodeVideoFrame(skip, 0, parsed, decoded);
    {
      MonitorAutoExit exitReaderMon(mMonitor);
      MonitorAutoEnter decoderMon(mDecoder->GetMonitor());
      if (mDecoder->GetDecodeState() == nsBuiltinDecoderStateMachine::DECODER_STATE_SHUTDOWN) {
        return NS_ERROR_FAILURE;
      }
    }
  } while (!eof &&






  virtual nsresult Init(nsBuiltinDecoderReader* aCloneDonor);
  virtual nsresult ResetDecode();
  virtual PRBool DecodeAudioData();

  // If the Theora granulepos has not been captured, it may read several packets
  // until one with a granulepos has been captured, to ensure that all packets
  // read have valid time info.  
  virtual PRBool DecodeVideoFrame(PRBool& aKeyframeSkip,
                                  PRInt64 aTimeThreshold,
                                  PRUint32& aParsed,
                                  PRUint32& aDecoded);

  virtual VideoData* FindStartTime(PRInt64 aOffset,
                                   PRInt64& aOutStartTime);

  // Get the end time of aEndOffset. This is the playback position we'd reach
  // after playback finished at aEndOffset.
  virtual PRInt64 FindEndTime(PRInt64 aEndOffset);






    aLength -= bytesRead;
    aBuf += bytesRead;
  }

  return PR_TRUE;
}

PRBool nsRawReader::DecodeVideoFrame(PRBool &aKeyframeSkip,
                                     PRInt64 aTimeThreshold,
                                     PRUint32& aParsed,
                                     PRUint32& aDecoded)
{
  aParsed = aDecoded = 0;
  mozilla::MonitorAutoEnter autoEnter(mMonitor);
  NS_ASSERTION(mDecoder->OnStateMachineThread() || mDecoder->OnDecodeThread(),
               "Should be on state machine thread or decode thread.");

  if (!mFrameSize)
    return PR_FALSE; // Metadata read failed.  We should refuse to play.

  PRInt64 currentFrameTime = 1000 * mCurrentFrame / mFrameRate;

        !(header.packetID == 0xFF && header.codecID == RAW_ID /* "YUV" */)) {
      return PR_FALSE;
    }

    if (!ReadFromStream(stream, buffer, length)) {
      return PR_FALSE;
    }

    aParsed++;

    if (currentFrameTime >= aTimeThreshold)
      break;

    mCurrentFrame++;
    currentFrameTime += 1000.0 / mFrameRate;
  }

  VideoData::YCbCrBuffer b;

                                   b,
                                   1, // In raw video every frame is a keyframe
                                   -1);
  if (!v)
    return PR_FALSE;

  mVideoQueue.Push(v);
  mCurrentFrame++;
  aDecoded++;
  currentFrameTime += 1000 / mFrameRate;

  return PR_TRUE;
}

nsresult nsRawReader::Seek(PRInt64 aTime, PRInt64 aStartTime, PRInt64 aEndTime, PRInt64 aCurrentTime)
{
  mozilla::MonitorAutoEnter autoEnter(mMonitor);


  nsresult rv = stream->Seek(nsISeekableStream::NS_SEEK_SET, offset);
  NS_ENSURE_SUCCESS(rv, rv);

  mVideoQueue.Erase();

  while(mVideoQueue.GetSize() == 0) {
    PRBool keyframeSkip = PR_FALSE;
    PRUint32 parsed, decoded;
    if (!DecodeVideoFrame(keyframeSkip, 0, parsed, decoded)) {
      mCurrentFrame = frame;
      return NS_ERROR_FAILURE;
    }

    {
      mozilla::MonitorAutoExit autoMonitorExit(mMonitor);
      mozilla::MonitorAutoEnter autoMonitor(mDecoder->GetMonitor());
      if (mDecoder->GetDecodeState() ==





public:
  nsRawReader(nsBuiltinDecoder* aDecoder);
  ~nsRawReader();

  virtual nsresult Init(nsBuiltinDecoderReader* aCloneDonor);
  virtual nsresult ResetDecode();
  virtual PRBool DecodeAudioData();

  virtual PRBool DecodeVideoFrame(PRBool& aKeyframeSkip,
                                  PRInt64 aTimeThreshold,
                                  PRUint32& aParsed,
                                  PRUint32& aDecoded);

  virtual PRBool HasAudio()
  {
    return PR_FALSE;
  }

  virtual PRBool HasVideo()
  {





  if (!holder) {
    mAudioQueue.Finish();
    return PR_FALSE;
  }

  return DecodeAudioPacket(holder->mPacket, holder->mOffset);
}

PRBool nsWebMReader::DecodeVideoFrame(PRBool& aKeyframeSkip,
                                      PRInt64 aTimeThreshold,
                                      PRUint32& aParsed,
                                      PRUint32& aDecoded)
{
  aParsed = aDecoded = 0;
  MonitorAutoEnter mon(mMonitor);
  NS_ASSERTION(mDecoder->OnStateMachineThread() || mDecoder->OnDecodeThread(),
               "Should be on state machine or decode thread.");

  nsAutoRef<NesteggPacketHolder> holder(NextPacket(VIDEO));
  if (!holder) {
    mVideoQueue.Finish();
    return PR_FALSE;

    if (r == -1) {
      return PR_FALSE;
    }

    vpx_codec_stream_info_t si;
    memset(&si, 0, sizeof(si));
    si.sz = sizeof(si);
    vpx_codec_peek_stream_info(&vpx_codec_vp8_dx_algo, data, length, &si);
    if (aKeyframeSkip && (!si.is_kf || tstamp_ms < aTimeThreshold)) {
      // Skipping to next keyframe...
      aParsed++; // Assume 1 frame per chunk.
      continue;
    }

    if (aKeyframeSkip && si.is_kf) {
      aKeyframeSkip = PR_FALSE;
    }

    if (vpx_codec_decode(&mVP8, data, length, NULL, 0)) {
      return PR_FALSE;
    }

    // If the timestamp of the video frame is less than
    // the time threshold required then it is not added
    // to the video queue and won't be displayed.
    if (tstamp_ms < aTimeThreshold) {
      aParsed++; // Assume 1 frame per chunk.
      continue;
    }

    vpx_codec_iter_t  iter = NULL;
    vpx_image_t      *img;

    while ((img = vpx_codec_get_frame(&mVP8, &iter))) {
      NS_ASSERTION(mInfo.mPicture.width == static_cast<PRInt32>(img->d_w), 
                   "WebM picture width from header does not match decoded frame");
      NS_ASSERTION(mInfo.mPicture.height == static_cast<PRInt32>(img->d_h),
                   "WebM picture height from header does not match decoded frame");
      NS_ASSERTION(img->fmt == IMG_FMT_I420, "WebM image format is not I420");

      // Chroma shifts are rounded down as per the decoding examples in the VP8 SDK
      VideoData::YCbCrBuffer b;

                                       tstamp_ms,
                                       next_tstamp / NS_PER_MS,
                                       b,
                                       si.is_kf,
                                       -1);
      if (!v) {
        return PR_FALSE;
      }
      aParsed++;
      aDecoded++;
      NS_ASSERTION(aDecoded <= aParsed,
        "Expect only 1 frame per chunk per packet in WebM...");
      mVideoQueue.Push(v);
    }
  }

  return PR_TRUE;
}

nsresult nsWebMReader::Seek(PRInt64 aTarget, PRInt64 aStartTime, PRInt64 aEndTime,






  virtual nsresult Init(nsBuiltinDecoderReader* aCloneDonor);
  virtual nsresult ResetDecode();
  virtual PRBool DecodeAudioData();

  // If the Theora granulepos has not been captured, it may read several packets
  // until one with a granulepos has been captured, to ensure that all packets
  // read have valid time info.  
  virtual PRBool DecodeVideoFrame(PRBool& aKeyframeSkip,
                                  PRInt64 aTimeThreshold,
                                  PRUint32& aParsed,
                                  PRUint32& aDecoded);

  virtual PRBool HasAudio()
  {
    mozilla::MonitorAutoEnter mon(mMonitor);
    return mHasAudio;
  }

  virtual PRBool HasVideo()





 * <video> element.
 *
 * For more information on this interface, please see
 * http://www.whatwg.org/specs/web-apps/current-work/#video
 *
 * @status UNDER_DEVELOPMENT
 */

[scriptable, uuid(e1f52aa5-9962-4019-b7b3-af3aee6e4d48)]
interface nsIDOMHTMLVideoElement : nsIDOMHTMLMediaElement
{
           attribute long width; 
           attribute long height;
  readonly attribute unsigned long videoWidth;
  readonly attribute unsigned long videoHeight;
           attribute DOMString poster;
           
  // A count of the number of video frames that have demuxed from the media
  // resource. If we were playing perfectly, we'd be able to paint this many
  // frames.
  readonly attribute unsigned long mozParsedFrames;

  // A count of the number of frames that have been decoded. We may drop
  // frames if the decode is taking too much time.
  readonly attribute unsigned long mozDecodedFrames;

  // A count of the number of frames that have been presented to the rendering
  // pipeline. We may drop frames if they arrive late at the renderer.
  readonly attribute unsigned long mozPresentedFrames;
  
  // Number of presented frames which were drawn on screen.
  readonly attribute unsigned long mozPaintedFrames;
  
  // Time which the last painted video frame was late by, in seconds.
  readonly attribute float mozFrameDelay;
};






 * the terms of any one of the MPL, the GPL or the LGPL.
 *
 * ***** END LICENSE BLOCK ***** */

#ifndef GFX_IMAGELAYER_H
#define GFX_IMAGELAYER_H

#include "Layers.h"
#include "mozilla/Monitor.h"

#include "gfxPattern.h"
#include "nsThreadUtils.h"
#include "mozilla/TimeStamp.h"

namespace mozilla {
namespace layers {

/**
 * A class representing a buffer of pixel data. The data can be in one
 * of various formats including YCbCr.
 * 

 * we need a separate class here is that ImageLayers aren't threadsafe
 * (because layers can only be used on the main thread) and we want to
 * be able to set the current Image from any thread, to facilitate
 * video playback without involving the main thread, for example.
 */
class THEBES_API ImageContainer {
  THEBES_INLINE_DECL_THREADSAFE_REFCOUNTING(ImageContainer)

  typedef mozilla::Monitor Monitor;

public:
  ImageContainer() : mTimeMonitor("ImageContainer"), mImagePainted(PR_FALSE) {}
  virtual ~ImageContainer() {}

  /**
   * Create an Image in one of the given formats.
   * Picks the "best" format from the list and creates an Image of that
   * format.
   * Returns null if this backend does not support any of the formats.
   */


  /**
   * Sets a size that the image is expected to be rendered at.
   * This is a hint for image backends to optimize scaling.
   * Default implementation in this class is to ignore the hint.
   */
  virtual void SetScaleHint(const gfxIntSize& /* aScaleHint */) { }

  /**
   * Returns the duration which the paint was late by, if a paint target
   * was specified.
   */
  TimeDuration GetPaintDelay();

  /**
   * Notifies the ImageLayer that it's been painted, so it can calculate
   * the delay between the target paint time, and the achieved paint time.
   */
  void NotifyPainted(TimeStamp aPaintTime);

  /**
   * Sets the time at which we'd like the contained to be image painted.
   */
  void SetPaintTarget(TimeStamp aTime);

  /**
   * Returns the target time at which we'd like the contained image
   * to be painted, as previously set by SetPaintTarget().
   */
  TimeStamp GetPaintTarget();

protected:
  LayerManager* mManager;

  /**
   * Protects acces to mTargetTime, mDelay, and mImagePainted.
   */
  Monitor mTimeMonitor;

  /**
   * Time at which we aim to paint the image. Set by SetPaintTarget(). This
   * is typically set on video frames in the video decoder.
   */
  TimeStamp mTargetTime;

  /**
   * Duration by which the paint was late by. This is only valid if a target
   * paint time was specified, and if the image has actually been painted.
   */
  TimeDuration mDelay;

  /**
   * Set to PR_TRUE if the currently contained image has been painted at
   * least once.
   */
  PRBool mImagePainted;

  ImageContainer(LayerManager* aManager) : mManager(aManager), mTimeMonitor("ImageContainer") {}
};

/**
 * A Layer which renders an Image.
 */
class THEBES_API ImageLayer : public Layer {
public:
  /**





already_AddRefed<gfxASurface>
LayerManager::CreateOptimalSurface(const gfxIntSize &aSize,
                                   gfxASurface::gfxImageFormat aFormat)
{
  return gfxPlatform::GetPlatform()->
    CreateOffscreenSurface(aSize, gfxASurface::ContentFromFormat(aFormat));
}

static void NotifyPainted(Layer* aLayer, TimeStamp aTimeStamp) {
  NS_ASSERTION(aLayer, "Must have specified a non-null layer");
  NS_ASSERTION(!aTimeStamp.IsNull(), "Must have a valid timestamp");
  if (aLayer->GetType() == Layer::TYPE_IMAGE) {
    ImageLayer* imgLayer = static_cast<ImageLayer*>(aLayer);
    ImageContainer* container = imgLayer->GetContainer();
    container->NotifyPainted(aTimeStamp);
  }
  Layer* child = aLayer->GetFirstChild();
  for (; child; child = child->GetNextSibling()) {
    NotifyPainted(child, aTimeStamp);
  }
}

void
LayerManager::NotifyPainted()
{
  if (mRoot)
    ::NotifyPainted(mRoot, TimeStamp::Now());
}

//--------------------------------------------------
// Layer

PRBool
Layer::CanUseOpaqueSurface()
{
  // If the visible content in the layer is opaque, there is no need
  // for an alpha channel.


/*static*/ void LayerManager::InitLog() {}
/*static*/ bool LayerManager::IsLogEnabled() { return false; }

#endif // MOZ_LAYERS_HAVE_LOG

PRLogModuleInfo* LayerManager::sLog;

TimeDuration ImageContainer::GetPaintDelay() {
  MonitorAutoEnter mon(mTimeMonitor);
  return mDelay;
}

void ImageContainer::NotifyPainted(TimeStamp aPaintTime) {
  MonitorAutoEnter mon(mTimeMonitor);
  if (mImagePainted || mTargetTime.IsNull() || mTargetTime > aPaintTime)
    return;
  mDelay = aPaintTime - mTargetTime;
  // Remember that we've painted this image, so that we won't recalculate
  // and assume a larger delay if we paint this image again.
  mImagePainted = PR_TRUE;
}

TimeStamp ImageContainer::GetPaintTarget() {
  MonitorAutoEnter mon(mTimeMonitor);
  return mTargetTime;
}

void ImageContainer::SetPaintTarget(TimeStamp aTime) {
  MonitorAutoEnter mon(mTimeMonitor);
  mTargetTime = aTime;
  // Reset our "has been painted" flag, so we know to recalculate the paint
  // delay the first time this frame is painted.
  mImagePainted = PR_FALSE;
}

} // namespace layers 
} // namespace mozilla





   * Log information about just this layer manager itself to the NSPR
   * log (if enabled for "Layers").
   */
  void LogSelf(const char* aPrefix="");

  static bool IsLogEnabled();
  static PRLogModuleInfo* GetLog() { return sLog; }

  /**
   * Notifies all ImageLayers in the layer tree that they've been painted, so
   * that they can record paint-delay statistics. Call this at the end of every
   * EndTransaction() implementation.
   */
  void NotifyPainted();

protected:
  nsRefPtr<Layer> mRoot;
  LayerUserDataSet mUserData;
  PRPackedBool mDestroyed;
  PRPackedBool mSnapEffectiveTransforms;

  // Print interesting information about this into aTo.  Internally
  // used to implement Dump*() and Log*().





    if (useDoubleBuffering) {
      finalTarget->SetOperator(gfxContext::OPERATOR_SOURCE);
      PopGroupWithCachedSurface(finalTarget, cachedSurfaceOffset);
    }

    mTarget = nsnull;
  }

  NotifyPainted();

#ifdef MOZ_LAYERS_HAVE_LOG
  Log();
  MOZ_LAYERS_LOG(("]----- EndTransaction"));
#endif

#ifdef DEBUG
  mPhase = PHASE_NONE;
#endif

      default:
        NS_RUNTIMEABORT("not reached");
      }
    }
  } else if (HasShadowManager()) {
    NS_WARNING("failed to forward Layers transaction");
  }

  NotifyPainted();

#ifdef DEBUG
  mPhase = PHASE_NONE;
#endif

  // this may result in Layers being deleted, which results in
  // PLayer::Send__delete__() and DeallocShmem()
  mKeepAlive.Clear();
}





  // The results of our drawing always go directly into a pixel buffer,
  // so we don't need to pass any global transform here.
  mRoot->ComputeEffectiveTransforms(gfx3DMatrix());

  Render();
  mCurrentCallbackInfo.Callback = nsnull;
  mCurrentCallbackInfo.CallbackData = nsnull;
  mTarget = nsnull;
  NotifyPainted();
}

already_AddRefed<ThebesLayer>
LayerManagerD3D10::CreateThebesLayer()
{
  nsRefPtr<ThebesLayer> layer = new ThebesLayerD3D10(this);
  return layer.forget();
}





  mRoot->ComputeEffectiveTransforms(gfx3DMatrix());

  Render();
  /* Clean this out for sanity */
  mCurrentCallbackInfo.Callback = NULL;
  mCurrentCallbackInfo.CallbackData = NULL;
  // Clear mTarget, next transaction could have no target
  mTarget = NULL;

  NotifyPainted();
}

void
LayerManagerD3D9::SetRoot(Layer *aLayer)
{
  mRoot = aLayer;
}






    Render();
  }

  mThebesLayerCallback = nsnull;
  mThebesLayerCallbackData = nsnull;

  mTarget = NULL;

  NotifyPainted();

#ifdef MOZ_LAYERS_HAVE_LOG
  Log();
  MOZ_LAYERS_LOG(("]----- EndTransaction"));
#endif
}

already_AddRefed<ThebesLayer>
LayerManagerOGL::CreateThebesLayer()






  layer->SetContainer(container);
  layer->SetFilter(nsLayoutUtils::GetGraphicsFilterForFrame(this));
  // Set a transform on the layer to draw the video in the right place
  gfxMatrix transform;
  transform.Translate(r.pos);
  transform.Scale(r.Width()/frameSize.width, r.Height()/frameSize.height);
  layer->SetTransform(gfx3DMatrix::From2D(transform));

  if (HasVideoElement()) {
    TimeStamp target = container->GetPaintTarget();
    if (!target.IsNull() &&
        (mLastPaintedTarget.IsNull() || target != mLastPaintedTarget))
    {
      // This is the first time we've painted this frame, count it.
      mLastPaintedTarget = target;
      nsHTMLVideoElement* element = static_cast<nsHTMLVideoElement*>(GetContent());
      element->NotifyPaintedFrame();
    }
  }

  nsRefPtr<Layer> result = layer.forget();
  return result.forget();
}

NS_IMETHODIMP
nsVideoFrame::Reflow(nsPresContext*           aPresContext,
                     nsHTMLReflowMetrics&     aMetrics,
                     const nsHTMLReflowState& aReflowState,






nsIFrame* NS_NewVideoFrame (nsIPresShell* aPresShell, nsStyleContext* aContext);

class nsVideoFrame : public nsContainerFrame, public nsIAnonymousContentCreator
{
public:
  typedef mozilla::layers::Layer Layer;
  typedef mozilla::layers::LayerManager LayerManager;
  typedef mozilla::TimeStamp TimeStamp;

  nsVideoFrame(nsStyleContext* aContext);

  NS_DECL_QUERYFRAME
  NS_DECL_FRAMEARENA_HELPERS

  NS_IMETHOD BuildDisplayList(nsDisplayListBuilder*   aBuilder,
                              const nsRect&           aDirtyRect,


  nsMargin mBorderPadding;
  
  // Anonymous child which is bound via XBL to the video controls.
  nsCOMPtr<nsIContent> mVideoControls;
  
  // Anonymous child which is the image element of the poster frame.
  nsCOMPtr<nsIContent> mPosterImage;

  // Target timestamp of the last frame we painted.
  TimeStamp mLastPaintedTarget;
};

#endif /* nsVideoFrame_h___ */

Return to bug 580531