video_frame.h 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277
  1. /*
  2. * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
  3. *
  4. * Use of this source code is governed by a BSD-style license
  5. * that can be found in the LICENSE file in the root of the source
  6. * tree. An additional intellectual property rights grant can be found
  7. * in the file PATENTS. All contributing project authors may
  8. * be found in the AUTHORS file in the root of the source tree.
  9. */
  10. #ifndef API_VIDEO_VIDEO_FRAME_H_
  11. #define API_VIDEO_VIDEO_FRAME_H_
  12. #include <stdint.h>
  13. #include <utility>
  14. #include "absl/types/optional.h"
  15. #include "api/rtp_packet_infos.h"
  16. #include "api/scoped_refptr.h"
  17. #include "api/video/color_space.h"
  18. #include "api/video/hdr_metadata.h"
  19. #include "api/video/video_frame_buffer.h"
  20. #include "api/video/video_rotation.h"
  21. #include "rtc_base/checks.h"
  22. #include "rtc_base/system/rtc_export.h"
  23. namespace webrtc {
  24. class RTC_EXPORT VideoFrame {
  25. public:
  26. struct RTC_EXPORT UpdateRect {
  27. int offset_x;
  28. int offset_y;
  29. int width;
  30. int height;
  31. // Makes this UpdateRect a bounding box of this and other rect.
  32. void Union(const UpdateRect& other);
  33. // Makes this UpdateRect an intersection of this and other rect.
  34. void Intersect(const UpdateRect& other);
  35. // Sets everything to 0, making this UpdateRect a zero-size (empty) update.
  36. void MakeEmptyUpdate();
  37. bool IsEmpty() const;
  38. // Per-member equality check. Empty rectangles with different offsets would
  39. // be considered different.
  40. bool operator==(const UpdateRect& other) const {
  41. return other.offset_x == offset_x && other.offset_y == offset_y &&
  42. other.width == width && other.height == height;
  43. }
  44. bool operator!=(const UpdateRect& other) const { return !(*this == other); }
  45. // Scales update_rect given original frame dimensions.
  46. // Cropping is applied first, then rect is scaled down.
  47. // Update rect is snapped to 2x2 grid due to possible UV subsampling and
  48. // then expanded by additional 2 pixels in each direction to accommodate any
  49. // possible scaling artifacts.
  50. // Note, close but not equal update_rects on original frame may result in
  51. // the same scaled update rects.
  52. UpdateRect ScaleWithFrame(int frame_width,
  53. int frame_height,
  54. int crop_x,
  55. int crop_y,
  56. int crop_width,
  57. int crop_height,
  58. int scaled_width,
  59. int scaled_height) const;
  60. };
  61. struct RTC_EXPORT ProcessingTime {
  62. TimeDelta Elapsed() const { return finish - start; }
  63. Timestamp start;
  64. Timestamp finish;
  65. };
  66. // Preferred way of building VideoFrame objects.
  67. class RTC_EXPORT Builder {
  68. public:
  69. Builder();
  70. ~Builder();
  71. VideoFrame build();
  72. Builder& set_video_frame_buffer(
  73. const rtc::scoped_refptr<VideoFrameBuffer>& buffer);
  74. Builder& set_timestamp_ms(int64_t timestamp_ms);
  75. Builder& set_timestamp_us(int64_t timestamp_us);
  76. Builder& set_timestamp_rtp(uint32_t timestamp_rtp);
  77. Builder& set_ntp_time_ms(int64_t ntp_time_ms);
  78. Builder& set_rotation(VideoRotation rotation);
  79. Builder& set_color_space(const absl::optional<ColorSpace>& color_space);
  80. Builder& set_color_space(const ColorSpace* color_space);
  81. Builder& set_id(uint16_t id);
  82. Builder& set_update_rect(const absl::optional<UpdateRect>& update_rect);
  83. Builder& set_packet_infos(RtpPacketInfos packet_infos);
  84. private:
  85. uint16_t id_ = 0;
  86. rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer_;
  87. int64_t timestamp_us_ = 0;
  88. uint32_t timestamp_rtp_ = 0;
  89. int64_t ntp_time_ms_ = 0;
  90. VideoRotation rotation_ = kVideoRotation_0;
  91. absl::optional<ColorSpace> color_space_;
  92. absl::optional<UpdateRect> update_rect_;
  93. RtpPacketInfos packet_infos_;
  94. };
  95. // To be deprecated. Migrate all use to Builder.
  96. VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
  97. webrtc::VideoRotation rotation,
  98. int64_t timestamp_us);
  99. VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
  100. uint32_t timestamp_rtp,
  101. int64_t render_time_ms,
  102. VideoRotation rotation);
  103. ~VideoFrame();
  104. // Support move and copy.
  105. VideoFrame(const VideoFrame&);
  106. VideoFrame(VideoFrame&&);
  107. VideoFrame& operator=(const VideoFrame&);
  108. VideoFrame& operator=(VideoFrame&&);
  109. // Get frame width.
  110. int width() const;
  111. // Get frame height.
  112. int height() const;
  113. // Get frame size in pixels.
  114. uint32_t size() const;
  115. // Get frame ID. Returns 0 if ID is not set. Not guarantee to be transferred
  116. // from the sender to the receiver, but preserved on single side. The id
  117. // should be propagated between all frame modifications during its lifetime
  118. // from capturing to sending as encoded image. It is intended to be unique
  119. // over a time window of a few minutes for peer connection, to which
  120. // corresponding video stream belongs to.
  121. uint16_t id() const { return id_; }
  122. void set_id(uint16_t id) { id_ = id; }
  123. // System monotonic clock, same timebase as rtc::TimeMicros().
  124. int64_t timestamp_us() const { return timestamp_us_; }
  125. void set_timestamp_us(int64_t timestamp_us) { timestamp_us_ = timestamp_us; }
  126. // TODO(nisse): After the cricket::VideoFrame and webrtc::VideoFrame
  127. // merge, timestamps other than timestamp_us will likely be
  128. // deprecated.
  129. // Set frame timestamp (90kHz).
  130. void set_timestamp(uint32_t timestamp) { timestamp_rtp_ = timestamp; }
  131. // Get frame timestamp (90kHz).
  132. uint32_t timestamp() const { return timestamp_rtp_; }
  133. // For now, transport_frame_id and rtp timestamp are the same.
  134. // TODO(nisse): Must be handled differently for QUIC.
  135. uint32_t transport_frame_id() const { return timestamp(); }
  136. // Set capture ntp time in milliseconds.
  137. void set_ntp_time_ms(int64_t ntp_time_ms) { ntp_time_ms_ = ntp_time_ms; }
  138. // Get capture ntp time in milliseconds.
  139. int64_t ntp_time_ms() const { return ntp_time_ms_; }
  140. // Naming convention for Coordination of Video Orientation. Please see
  141. // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/ts_126114v120700p.pdf
  142. //
  143. // "pending rotation" or "pending" = a frame that has a VideoRotation > 0.
  144. //
  145. // "not pending" = a frame that has a VideoRotation == 0.
  146. //
  147. // "apply rotation" = modify a frame from being "pending" to being "not
  148. // pending" rotation (a no-op for "unrotated").
  149. //
  150. VideoRotation rotation() const { return rotation_; }
  151. void set_rotation(VideoRotation rotation) { rotation_ = rotation; }
  152. // Get color space when available.
  153. const absl::optional<ColorSpace>& color_space() const { return color_space_; }
  154. void set_color_space(const absl::optional<ColorSpace>& color_space) {
  155. color_space_ = color_space;
  156. }
  157. // Get render time in milliseconds.
  158. // TODO(nisse): Deprecated. Migrate all users to timestamp_us().
  159. int64_t render_time_ms() const;
  160. // Return the underlying buffer. Never nullptr for a properly
  161. // initialized VideoFrame.
  162. rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer() const;
  163. void set_video_frame_buffer(
  164. const rtc::scoped_refptr<VideoFrameBuffer>& buffer);
  165. // TODO(nisse): Deprecated.
  166. // Return true if the frame is stored in a texture.
  167. bool is_texture() const {
  168. return video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative;
  169. }
  170. bool has_update_rect() const { return update_rect_.has_value(); }
  171. // Returns update_rect set by the builder or set_update_rect() or whole frame
  172. // rect if no update rect is available.
  173. UpdateRect update_rect() const {
  174. return update_rect_.value_or(UpdateRect{0, 0, width(), height()});
  175. }
  176. // Rectangle must be within the frame dimensions.
  177. void set_update_rect(const VideoFrame::UpdateRect& update_rect) {
  178. RTC_DCHECK_GE(update_rect.offset_x, 0);
  179. RTC_DCHECK_GE(update_rect.offset_y, 0);
  180. RTC_DCHECK_LE(update_rect.offset_x + update_rect.width, width());
  181. RTC_DCHECK_LE(update_rect.offset_y + update_rect.height, height());
  182. update_rect_ = update_rect;
  183. }
  184. void clear_update_rect() { update_rect_ = absl::nullopt; }
  185. // Get information about packets used to assemble this video frame. Might be
  186. // empty if the information isn't available.
  187. const RtpPacketInfos& packet_infos() const { return packet_infos_; }
  188. void set_packet_infos(RtpPacketInfos value) {
  189. packet_infos_ = std::move(value);
  190. }
  191. const absl::optional<ProcessingTime> processing_time() const {
  192. return processing_time_;
  193. }
  194. void set_processing_time(const ProcessingTime& processing_time) {
  195. processing_time_ = processing_time;
  196. }
  197. private:
  198. VideoFrame(uint16_t id,
  199. const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
  200. int64_t timestamp_us,
  201. uint32_t timestamp_rtp,
  202. int64_t ntp_time_ms,
  203. VideoRotation rotation,
  204. const absl::optional<ColorSpace>& color_space,
  205. const absl::optional<UpdateRect>& update_rect,
  206. RtpPacketInfos packet_infos);
  207. uint16_t id_;
  208. // An opaque reference counted handle that stores the pixel data.
  209. rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer_;
  210. uint32_t timestamp_rtp_;
  211. int64_t ntp_time_ms_;
  212. int64_t timestamp_us_;
  213. VideoRotation rotation_;
  214. absl::optional<ColorSpace> color_space_;
  215. // Updated since the last frame area. If present it means that the bounding
  216. // box of all the changes is within the rectangular area and is close to it.
  217. // If absent, it means that there's no information about the change at all and
  218. // update_rect() will return a rectangle corresponding to the entire frame.
  219. absl::optional<UpdateRect> update_rect_;
  220. // Information about packets used to assemble this video frame. This is needed
  221. // by |SourceTracker| when the frame is delivered to the RTCRtpReceiver's
  222. // MediaStreamTrack, in order to implement getContributingSources(). See:
  223. // https://w3c.github.io/webrtc-pc/#dom-rtcrtpreceiver-getcontributingsources
  224. RtpPacketInfos packet_infos_;
  225. // Processing timestamps of the frame. For received video frames these are the
  226. // timestamps when the frame is sent to the decoder and the decoded image
  227. // returned from the decoder.
  228. // Currently, not set for locally captured video frames.
  229. absl::optional<ProcessingTime> processing_time_;
  230. };
  231. } // namespace webrtc
  232. #endif // API_VIDEO_VIDEO_FRAME_H_