Attachment #468617 for bug #580531

Attachment #468617: Implements playback statistics on video and media elements for bug #580531

View | Details | Raw Unified | Return to bug 580531
Collapse All | Expand All





NS_INTERFACE_MAP_END_INHERITING(nsGenericHTMLElement)

// nsIDOMHTMLMediaElement
NS_IMPL_URI_ATTR(nsHTMLMediaElement, Src, src)
NS_IMPL_BOOL_ATTR(nsHTMLMediaElement, Controls, controls)
NS_IMPL_BOOL_ATTR(nsHTMLMediaElement, Autoplay, autoplay)
NS_IMPL_STRING_ATTR(nsHTMLMediaElement, Preload, preload)

/* readonly attribute float mozDownloadRate; */
NS_IMETHODIMP nsHTMLMediaElement::GetMozDownloadRate(float *aMozDownloadRate)
{
  nsMediaDecoder::Statistics stats = mDecoder->GetStatistics();
  *aMozDownloadRate = stats.mDownloadRateReliable ? 
                        stats.mDownloadRate :
                        std::numeric_limits<float>::quiet_NaN();
  return NS_OK;
}

/* readonly attribute float mozDecodeRate; */
NS_IMETHODIMP nsHTMLMediaElement::GetMozDecodeRate(float *aMozDecodeRate)
{
  nsMediaDecoder::Statistics stats = mDecoder->GetStatistics();
  *aMozDecodeRate = stats.mPlaybackRateReliable ? 
                      stats.mPlaybackRate :
                      std::numeric_limits<float>::quiet_NaN();
  return NS_OK;
}

/* readonly attribute nsIDOMHTMLMediaElement mozAutoplayEnabled; */
NS_IMETHODIMP nsHTMLMediaElement::GetMozAutoplayEnabled(PRBool *aAutoplayEnabled)
{
  *aAutoplayEnabled = mAutoplayEnabled;

  return NS_OK;
}







/* readonly attribute unsigned long videoHeight; */
NS_IMETHODIMP nsHTMLVideoElement::GetVideoHeight(PRUint32 *aVideoHeight)
{
  *aVideoHeight = mMediaSize.height == -1 ? 0 : mMediaSize.height;
  return NS_OK;
}

/* readonly attribute unsigned long mozDecodedFrames; */
NS_IMETHODIMP nsHTMLVideoElement::GetMozDecodedFrames(PRUint32 *aMozDecodedFrames)
{
  *aMozDecodedFrames = mDecoder ? mDecoder->GetDecodedFrames() : 0;
  return NS_OK;
}

/* readonly attribute unsigned long mozDroppedFrames; */
NS_IMETHODIMP nsHTMLVideoElement::GetMozDroppedFrames(PRUint32 *aMozDroppedFrames)
{
  *aMozDroppedFrames = mDecoder ? mDecoder->GetDroppedFrames() : 0;
  return NS_OK;
}

nsHTMLVideoElement::nsHTMLVideoElement(already_AddRefed<nsINodeInfo> aNodeInfo,
                                       PRUint32 aFromParser)
  : nsHTMLMediaElement(aNodeInfo, aFromParser)
{
}

nsHTMLVideoElement::~nsHTMLVideoElement()
{





{
  // Decode forward to the target frame. Start with video, if we have it.
  if (HasVideo()) {
    PRBool eof = PR_FALSE;
    PRInt64 startTime = -1;
    while (HasVideo() && !eof) {
      while (mVideoQueue.GetSize() == 0 && !eof) {
        PRBool skip = PR_FALSE;
        PRUint32 droppedFrames = 0;
        eof = !DecodeVideoFrame(skip, 0, droppedFrames);
        {
          MonitorAutoExit exitReaderMon(mMonitor);
          MonitorAutoEnter decoderMon(mDecoder->GetMonitor());
          if (mDecoder->GetDecodeState() == nsBuiltinDecoderStateMachine::DECODER_STATE_SHUTDOWN) {
            return NS_ERROR_FAILURE;
          }
        }
      }





  // in mAudioQueue. Returns PR_TRUE when there's more audio to decode,
  // PR_FALSE if the audio is finished, end of file has been reached,
  // or an un-recoverable read error has occured.
  virtual PRBool DecodeAudioData() = 0;

  // Reads and decodes one video frame. Packets with a timestamp less
  // than aTimeThreshold will be decoded (unless they're not keyframes
  // and aKeyframeSkip is PR_TRUE), but will not be added to the queue.
  // 'aDroppedFrames' will be incremented everytime a video frame is
  // dropped.
  virtual PRBool DecodeVideoFrame(PRBool &aKeyframeSkip,
                                  PRInt64 aTimeThreshold,
                                  PRUint32 &aDroppedFrames) = 0;

  virtual PRBool HasAudio() = 0;
  virtual PRBool HasVideo() = 0;

  // Read header data for all bitstreams in the file. Fills mInfo with
  // the data required to present the media. Returns NS_OK on success,
  // or NS_ERROR_FAILURE on failure.
  virtual nsresult ReadMetadata() = 0;

  template<class Data>
  Data* DecodeToFirstData(DecodeFn aDecodeFn,
                          MediaQueue<Data>& aQueue);

  // Wrapper so that DecodeVideoFrame(PRBool&,PRInt64) can be called from
  // DecodeToFirstData().
  PRBool DecodeVideoFrame() {
    PRBool f = PR_FALSE;
    PRUint32 droppedFrames = 0;
    return DecodeVideoFrame(f, 0, droppedFrames);
  }

  // Fills aRanges with ByteRanges denoting the sections of the media which
  // have been downloaded and are stored in the media cache. The reader
  // monitor must must be held with exactly one lock count. The nsMediaStream
  // must be pinned while calling this.
  nsresult GetBufferedBytes(nsTArray<ByteRange>& aRanges);






    if (audioPump && audioDecoded > audioPumpThresholdMs) {
      audioPump = PR_FALSE;
    }
    if (!audioPump && audioPlaying && audioDecoded < LOW_AUDIO_MS) {
      skipToNextKeyframe = PR_TRUE;
    }

    if (videoPlaying && !videoWait) {
      PRUint32 droppedFrames = 0;
      videoPlaying = mReader->DecodeVideoFrame(skipToNextKeyframe, currentTime, droppedFrames);
      {
        MonitorAutoEnter mon(mDecoder->GetMonitor());
        mDecoder->NotifyDroppedFrames(droppedFrames);
        if (mDecoder->mDecoderPosition > initialDownloadPosition) {
          mBufferExhausted = PR_TRUE;
        }
      }
    }
    {
      MonitorAutoEnter mon(mDecoder->GetMonitor());
      initialDownloadPosition =





  mElement(0),
  mRGBWidth(-1),
  mRGBHeight(-1),
  mProgressTime(),
  mDataTime(),
  mVideoUpdateLock(nsnull),
  mPixelAspectRatio(1.0),
  mPinnedForSeek(PR_FALSE),
  mDecodeDecodedFrames(0),
  mDecodeDroppedFrames(0),
  mSizeChanged(PR_FALSE),
  mShuttingDown(PR_FALSE)
{
  MOZ_COUNT_CTOR(nsMediaDecoder);
}

nsMediaDecoder::~nsMediaDecoder()
{

      mPixelAspectRatio != aPixelAspectRatio) {
    mRGBWidth = aSize.width;
    mRGBHeight = aSize.height;
    mPixelAspectRatio = aPixelAspectRatio;
    mSizeChanged = PR_TRUE;
  }
  if (mImageContainer && aImage) {
    mImageContainer->SetCurrentImage(aImage);
    mDecodeDecodedFrames++;
  }
}

void nsMediaDecoder::PinForSeek()
{
  nsMediaStream* stream = GetCurrentStream();
  if (!stream || mPinnedForSeek) {
    return;

  }
  PRInt64 bytesToDownload = stats.mTotalBytes - stats.mDownloadPosition;
  PRInt64 bytesToPlayback = stats.mTotalBytes - stats.mPlaybackPosition;
  double timeToDownload =
    (bytesToDownload + gDownloadSizeSafetyMargin)/stats.mDownloadRate;
  double timeToPlay = bytesToPlayback/stats.mPlaybackRate;
  return timeToDownload <= timeToPlay;
}

PRUint32 nsMediaDecoder::GetDecodedFrames()
{
  return mDecodeDecodedFrames;
}

PRUint32 nsMediaDecoder::GetDroppedFrames()
{
  return mDecodeDroppedFrames;
}

void nsMediaDecoder::IncrementDroppedFrames(PRUint32 aDroppedFrames)
{
  NS_ASSERTION(NS_IsMainThread(), "Don't call on non-main thread");

  mDecodeDroppedFrames += aDroppedFrames;
}

// An event to update the dropped frames count on the main thread
class DroppedFrames : public nsRunnable {
public:
  DroppedFrames(nsMediaDecoder* aDecoder, PRUint32 aDroppedFrames) :
    mDecoder(aDecoder), mDroppedFrames(aDroppedFrames) {}
  NS_IMETHOD Run() {
    mDecoder->IncrementDroppedFrames(mDroppedFrames);
    return NS_OK;
  }
private:
  nsRefPtr<nsMediaDecoder> mDecoder;
  PRUint32                 mDroppedFrames;
};

void nsMediaDecoder::NotifyDroppedFrames(PRUint32 aDroppedFrames)
{
  NS_ASSERTION(!NS_IsMainThread(), "Don't call on main thread");

  if (aDroppedFrames > 0) {
    nsCOMPtr<nsIRunnable> event = new DroppedFrames(this, aDroppedFrames);
    NS_DispatchToMainThread(event, NS_DISPATCH_NORMAL);
  }
}






  // Constructs the time ranges representing what segments of the media
  // are buffered and playable.
  virtual nsresult GetBuffered(nsHTMLTimeRanges* aBuffered) = 0;

  // Returns PR_TRUE if we can play the entire media through without stopping
  // to buffer, given the current download and playback rates.
  PRBool CanPlayThrough();

  // Return the count of frames that have been decoded
  PRUint32 GetDecodedFrames();

  // Return the count of frames that have been dropped
  PRUint32 GetDroppedFrames();

  // Increment the dropped frame count by the given amount. This must
  // only be called on the main thread. use 'NotifyDroppedFrames' to
  // increment dropped frames on non-main threads.
  void IncrementDroppedFrames(PRUint32 aDroppedFrames);

  // Raises an event that is dispatched to the main thread to increment
  // the number of dropped frames. Do not call on the main thread (use
  // IncrementDroppedFrames instead).
  void NotifyDroppedFrames(PRUint32 aDroppedFrames);

protected:

  // Start timer to update download progress information.
  nsresult StartProgress();

  // Stop progress information timer.
  nsresult StopProgress();



  // Pixel aspect ratio (ratio of the pixel width to pixel height)
  float mPixelAspectRatio;

  // PR_TRUE when our media stream has been pinned. We pin the stream
  // while seeking.
  PRPackedBool mPinnedForSeek;

  // A count of the number of frames that have been decoded and ready
  // to be displayed. This can be used for computing the decoding framerate
  PRUint32 mDecodeDecodedFrames;

  // A count of the number of frames that have been dropped for performance
  // reasons during playback.
  PRUint32 mDecodeDroppedFrames;

  // Has our size changed since the last repaint?
  PRPackedBool mSizeChanged;

  // True if the decoder is being shutdown. At this point all events that
  // are currently queued need to return immediately to prevent javascript
  // being run that operates on the element and decoder during shutdown.
  // Read/Write from the main thread only.
  PRPackedBool mShuttingDown;





      return NS_ERROR_OUT_OF_MEMORY;
    }
    aFrames.AppendElement(v);
  }
  return NS_OK;
}

PRBool nsOggReader::DecodeVideoFrame(PRBool &aKeyframeSkip,
                                     PRInt64 aTimeThreshold,
                                     PRUint32 &aDroppedFrames)
{
  MonitorAutoEnter mon(mMonitor);
  NS_ASSERTION(mDecoder->OnStateMachineThread() || mDecoder->OnDecodeThread(),
               "Should be on state machine or AV thread.");
  // We chose to keep track of the Theora granulepos ourselves, rather than
  // rely on th_decode_packetin() to do it for us. This is because
  // th_decode_packetin() simply works by incrementing a counter every time
  // it's called, so if we drop frames and don't call it, subsequent granulepos

    if (!aKeyframeSkip ||
        (th_packet_iskeyframe(&packet) == 1 && time >= aTimeThreshold))
    {
      if (DecodeTheora(frames, &packet) == NS_ERROR_OUT_OF_MEMORY) {
        NS_WARNING("Theora decode memory allocation failure");
        return PR_FALSE;
      }
    }
    else {
      aDroppedFrames++;
    }
  }

  // Push decoded data into the video frame queue.
  for (PRUint32 i = 0; i < frames.Length(); i++) {
    nsAutoPtr<VideoData> data(frames[i]);
    if (!aKeyframeSkip || (aKeyframeSkip && frames[i]->mKeyframe)) {
      mVideoQueue.Push(data.forget());
      if (aKeyframeSkip && frames[i]->mKeyframe) {
        aKeyframeSkip = PR_FALSE;
      }
    } else {
      frames[i] = nsnull;
      data = nsnull;
      aDroppedFrames++;
    }
  }

  if (endOfStream) {
    // We've encountered an end of bitstream packet. Inform the queue that
    // there will be no more frames.
    mVideoQueue.Finish();
  }

    return res;
  }

  // We have an active Theora bitstream. Decode the next Theora frame, and
  // extract its keyframe's time.
  PRBool eof;
  do {
    PRBool skip = PR_FALSE;
    PRUint32 droppedFrames = 0;
    eof = !DecodeVideoFrame(skip, 0, droppedFrames);
    {
      MonitorAutoExit exitReaderMon(mMonitor);
      MonitorAutoEnter decoderMon(mDecoder->GetMonitor());
      if (mDecoder->GetDecodeState() == nsBuiltinDecoderStateMachine::DECODER_STATE_SHUTDOWN) {
        return NS_ERROR_FAILURE;
      }
    }
  } while (!eof &&





  virtual nsresult Init();
  virtual nsresult ResetDecode();
  virtual PRBool DecodeAudioData();

  // If the Theora granulepos has not been captured, it may read several packets
  // until one with a granulepos has been captured, to ensure that all packets
  // read have valid time info.  
  virtual PRBool DecodeVideoFrame(PRBool &aKeyframeSkip,
                                  PRInt64 aTimeThreshold,
                                  PRUint32 &aDroppedFrames);

  virtual VideoData* FindStartTime(PRInt64 aOffset,
                                   PRInt64& aOutStartTime);

  // Get the end time of aEndOffset. This is the playback position we'd reach
  // after playback finished at aEndOffset.
  virtual PRInt64 FindEndTime(PRInt64 aEndOffset);






    aLength -= bytesRead;
    aBuf += bytesRead;
  }

  return PR_TRUE;
}

PRBool nsRawReader::DecodeVideoFrame(PRBool &aKeyframeSkip,
                                     PRInt64 aTimeThreshold,
                                     PRUint32 &aDroppedFrames)
{
  mozilla::MonitorAutoEnter autoEnter(mMonitor);
  NS_ASSERTION(mDecoder->OnStateMachineThread() || mDecoder->OnDecodeThread(),
               "Should be on state machine thread or decode thread.");

  if (!mFrameSize)
    return PR_FALSE; // Metadata read failed.  We should refuse to play.


      return PR_FALSE;
    }

    if (currentFrameTime >= aTimeThreshold)
      break;

    mCurrentFrame++;
    currentFrameTime += 1000.0 / mFrameRate;
    aDroppedFrames++;
  }

  VideoData::YCbCrBuffer b;
  b.mPlanes[0].mData = buffer;
  b.mPlanes[0].mStride = mMetadata.frameWidth * mMetadata.lumaChannelBpp / 8.0;
  b.mPlanes[0].mHeight = mMetadata.frameHeight;
  b.mPlanes[0].mWidth = mMetadata.frameWidth;



  nsresult rv = stream->Seek(nsISeekableStream::NS_SEEK_SET, offset);
  NS_ENSURE_SUCCESS(rv, rv);

  mVideoQueue.Erase();

  while(mVideoQueue.GetSize() == 0) {
    PRBool keyframeSkip = PR_FALSE;
    PRUint32 droppedFrames = 0;
    if (!DecodeVideoFrame(keyframeSkip, 0, droppedFrames)) {
      mCurrentFrame = frame;
      return NS_ERROR_FAILURE;
    }

    {
      mozilla::MonitorAutoExit autoMonitorExit(mMonitor);
      mozilla::MonitorAutoEnter autoMonitor(mDecoder->GetMonitor());
      if (mDecoder->GetDecodeState() ==





  nsRawReader(nsBuiltinDecoder* aDecoder);
  ~nsRawReader();

  virtual nsresult Init();
  virtual nsresult ResetDecode();
  virtual PRBool DecodeAudioData();

  virtual PRBool DecodeVideoFrame(PRBool &aKeyframeSkip,
                                  PRInt64 aTimeThreshold,
                                  PRUint32 &aDroppedFrames);

  virtual PRBool HasAudio()
  {
    return PR_FALSE;
  }

  virtual PRBool HasVideo()
  {





    mAudioQueue.Finish();
    return PR_FALSE;
  }

  return DecodeAudioPacket(packet);
}

PRBool nsWebMReader::DecodeVideoFrame(PRBool &aKeyframeSkip,
                                      PRInt64 aTimeThreshold,
                                      PRUint32 &aDroppedFrames)
{
  MonitorAutoEnter mon(mMonitor);
  NS_ASSERTION(mDecoder->OnStateMachineThread() || mDecoder->OnDecodeThread(),
               "Should be on state machine or decode thread.");
  int r = 0;
  nestegg_packet* packet = NextPacket(VIDEO);

  if (!packet) {

    }

    vpx_codec_stream_info_t si;
    memset(&si, 0, sizeof(si));
    si.sz = sizeof(si);
    vpx_codec_peek_stream_info(&vpx_codec_vp8_dx_algo, data, length, &si);
    if ((aKeyframeSkip && !si.is_kf) || (aKeyframeSkip && si.is_kf && tstamp_ms < aTimeThreshold)) {
      aKeyframeSkip = PR_TRUE;
      aDroppedFrames++;
      break;
    }

    if (aKeyframeSkip && si.is_kf) {
      aKeyframeSkip = PR_FALSE;
    }

    if(vpx_codec_decode(&mVP8, data, length, NULL, 0)) {
      nestegg_free_packet(packet);
      return PR_FALSE;
    }

    // If the timestamp of the video frame is less than
    // the time threshold required then it is not added
    // to the video queue and won't be displayed.
    if (tstamp_ms < aTimeThreshold) {
      aDroppedFrames++;
      continue;
    }

    vpx_codec_iter_t  iter = NULL;
    vpx_image_t      *img;

    while((img = vpx_codec_get_frame(&mVP8, &iter))) {
      NS_ASSERTION(mInfo.mPicture.width == static_cast<PRInt32>(img->d_w), 





  virtual nsresult Init();
  virtual nsresult ResetDecode();
  virtual PRBool DecodeAudioData();

  // If the Theora granulepos has not been captured, it may read several packets
  // until one with a granulepos has been captured, to ensure that all packets
  // read have valid time info.  
  virtual PRBool DecodeVideoFrame(PRBool &aKeyframeSkip,
                                  PRInt64 aTimeThreshold,
                                  PRUint32 &aDroppedFrames);

  virtual PRBool HasAudio()
  {
    mozilla::MonitorAutoEnter mon(mMonitor);
    return mHasAudio;
  }

  virtual PRBool HasVideo()






// undef the GetCurrentTime macro defined in WinBase.h from the MS Platform SDK
%{C++
#ifdef GetCurrentTime
#undef GetCurrentTime
#endif
%}

[scriptable, uuid(92999d57-ab9f-47f2-9567-723f409dc312)]
interface nsIDOMHTMLMediaElement : nsIDOMHTMLElement
{
  // error state
  readonly attribute nsIDOMHTMLMediaError error;

  // network state
           attribute DOMString src;
  readonly attribute DOMString currentSrc;


  // Mozilla extension: load data from another media element. This is like
  // load() but we don't run the resource selection algorithm; instead
  // we just set our source to other's currentSrc. This is optimized
  // so that this element will get access to all of other's cached/
  // buffered data. In fact any future data downloaded by this element or
  // other will be sharable by both elements.
  void mozLoadFrom(in nsIDOMHTMLMediaElement other);

  // The approximate rate at which the media resource is being downloaded in
  // bytes per second. If the rate is not available (possibly due
  // to not having downloaded enough data to compute a consistent value)
  // this will be NaN.
  readonly attribute float mozDownloadRate;

  // The approximate rate at which the media resource is being decoded in
  // bytes per second. If the rate is not available this will be
  // NaN.
  readonly attribute float mozDecodeRate;
};





 * <video> element.
 *
 * For more information on this interface, please see
 * http://www.whatwg.org/specs/web-apps/current-work/#video
 *
 * @status UNDER_DEVELOPMENT
 */

[scriptable, uuid(32234b6b-e0c5-4539-8b17-cf44035b457a)]
interface nsIDOMHTMLVideoElement : nsIDOMHTMLMediaElement
{
           attribute long width; 
           attribute long height;
  readonly attribute unsigned long videoWidth;
  readonly attribute unsigned long videoHeight;
           attribute DOMString poster;

  // A count of the number of frames that have been decoded and ready
  // to be displayed. This can be used for computing the decoding framerate
  readonly attribute unsigned long mozDecodedFrames;

  // A count of the number of frames that have been dropped for performance
  // reasons during playback.
  readonly attribute unsigned long mozDroppedFrames;
};


Return to bug 580531