Gecko:MediaRecorder

Reference spec:

https://dvcs.w3.org/hg/dap/raw-file/tip/media-stream-capture/RecordingProposal.html
bugs:
bug 803414 Audio Recording - Web API & Implementation
bug 834165 Implement BlobEvent
bug 825110 [meta] Porting WebRTC video_module for B2G
bug 825112 [meta] Porting WebRTC audio_module for B2G
media encoder
audio encoder
video encoder
media writer
ogg writer

MediaEncoder Draft

API draft

class MediaSegmentAdapter {
public:
   enum BufferType {
       AUDIO,
       VIDEO
   }

   /* AppendSegment may run on MediaStreamGraph thread to avoid race condition */
   void AppendSegment(MediaSegment) = 0;
   void SetBufferLength(size_t);
   size_t GetBufferLength();

   MediaSegment DequeueSegment();

protected:
   MediaSegmentAdapter();
   Queue<MediaSegment> mBuffer;

   friend class MediaCodec;
}

class VideoAdapter : MediaSegmentAdapter {
public:
   /* This version deep copy/color convert the input buffer into a local buffer */
   void AppendSegment(MediaSegment);
}

class GrallocAdapter : MediaSegmentAdapter {
public: 
   /* This version |do not| copy frame data, but queue the GraphicBuffer directly
      which can be used with SurfaceMediaSource or other SurfaceFlinger compatible 
      mechanism for hardware supported video encoding */
   void AppendSegment(MediaSegment);
}

class AudioAdapter : MediaSegmentAdapter {
public: 
   /* Copy/resample the data into local buffer */
   void AppendSegment(MediaSegment);
}

class MediaEncoder : MediaStreamListener {
public:
   /* Callback functions for MediaStream */
   void NotifyConsumption(MediaStreamGraph, Consumption);
   void NotifyPull(MediaStreamGraph, StreamTime);
   void NotifyBlockingChanged(MediaStreamGraph, Blocking);
   void NotifyHasCurrentData(MediaStreamGraph, bool);
   void NotifyOutput(MediaStreamGraph);
   void NotifyFinished(MediaStreamGraph);
   /* Queue the MediaSegment into correspond adapters */
   /* XXX: Is it possible to determine Audio related paramenters from this callback? 
           Or we have to query them from MediaStream directly? */
   /* AppendSegment into Adapter need block MediaStreamGraph thread to avoid race condition
      and we should schedule one encoding loop if any track updated */
   void NotifyQueuedTrackChanges(MediaStreamGraph, TrackID, TrackRate, TrackTicks, uint32_t, MediaSegment);

   /* Callback functions to JS */
   void SetDataAvailableCallback()
   void SetErrorCallback()
   void SetMuteTrackCallback()
   void SetPauseCallback()
   void SetPhotoCallback()
   void SetRecordingCallback()
   void SetResumeCallback()
   void SetStopCallback()
   void SetUnmuteTrackCallback()
   void SetWarningCallback()

   /* Option query functions */
   Vector<String> GetSupportMIMEType()
   Pair<int, int> GetSupportWidth() 
   Pair<int, int> GetSupportHeight()

   /* Set requested encoder */
   void SetMIMEType();
   void SetVideoWidth();
   void SetVideoHeight();

   /* JS control functions */
   void MuteTrack(TrackID)
   void Pause()
   void Record()
   void RequestData --?
       - Cause encoder flush data & callback?
   void Resume()
   void SetOptions()
   void Stop()
   void TakePhoto --?
   void UnmuteTrack

   /* initial internal state and codecs */
   void Init()

   /* create MediaEncoder for given MediaStream */
   MediaEncoder(MediaStream);

private: 
   void QueueVideoSegments();
   void QueueAudioSegments();
   void SelectCodec();
   void ConfigCodec();
   void SetVideoQueueSize();
   void SetAudioQueueSize();

   /* data member */
   MediaSegmentAdapter mVideoSegments;
   MediaSegmentAdapter mAudioSegments;
   MediaCodec mVideoCodec;
   MediaCodec mAudioCodec;
   MediaWriter mWriter;
   Thread mEncoderThread;
}

class CodecParams {
   /* key value pair for detailed codec settings */
   Map<Key, Value> mMap;
}

/* base class for general codecs */
class MediaCodec {
public:
   MediaCodec();
   status_t Init() = 0;

   /* Let MediaCodec setup buffer length based on codec characteristic
      e.g. Stagefright video codecs may only use 1 buffer since the buffer maybe shared between hardwares */
    

   /* Mimic android::CameraParameter to collect backend codec related params in general class */
   CodecParams GetParams() = 0;
   status_t SetParams(CodecParams) = 0;
              
   /* Start the encoder, if the encoder got its own thread, create the thread here */
   status_t Encode() = 0;

   /* Read the encoded data from encoder, blocking until buffer available */
   status_t GetEncodedData(MediaSegment& encData) = 0;

   /* codec specific metadata */
   Metadata GetMetadata();
   void Flush() = 0;

private:
   MediaSegmentAdapter mQueue;
}

class AudioCodec : public MediaCodec {
public:
   /* AudioCodec may need collect enough buffers to be encode, return COLLECT as needed */
   EncoderStatus Encode(MediaBuffer in, void* out, size_t length) = 0;

private:
   bool IsDataEnough();
   void* mLocalBuffer[MIN_FRAMES];
}

class VideoCodec : public MediaCodec {
public: 
   EncoderStatus Encode(MediaBuffer in, void* out, size_t length) = 0;
}

class OpusCodec : public AudioCodec {
   // Use libopus to encode audio
private:
   // libopus ctx, etc...
}

class TheoraCodec : public VideoCodec {
   // Use libtheora to encode video
private:
   // libtheora encoder is not blocking, thus we have to loop until frame complete
}

class MediaWriter {
public:
   void AddTrack(MediaCodec);

   /* Block until container packet write done*/
   nsTArray<char> GetPacket(); 
} 

class OggWriter {
   // Use libogg to write container
public:
   // libogg context and others
}

Working flow

1. MediaEncoder create MediaCodecs, MediaWriter based on request
   1. MediaEncoder create MediaSegmentAdapters from MediaCodecs
2. MediaEncoder register callback to MediaStream
3. MediaStreamGraph thread callback to MediaEncoder when stream update
4. MediaEncoder queue new MediaSegments into each MediaSegmentAdapters based on its type
   1. MediaSegmentAdapter copy/enqueue/color convert/etc... the data and queue them up
5. MediaEncoder post a task to encoder thread
6. Encoder thread ask MediaWriter for Packet
   1. MediaWriter ask Codecs for encoded data
   2. MediaWriter write packets
7. MediaEncoder call onRecordingCallback with raw data or nsArray
8. MediaRecord API post encoded data blob to Javascript layer

NOTE: step 8 are not specified in this API