1/**********
2This library is free software; you can redistribute it and/or modify it under
3the terms of the GNU Lesser General Public License as published by the
4Free Software Foundation; either version 3 of the License, or (at your
5option) any later version. (See <http://www.gnu.org/copyleft/lesser.html>.)
6
7This library is distributed in the hope that it will be useful, but WITHOUT
8ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
9FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for
10more details.
11
12You should have received a copy of the GNU Lesser General Public License
13along with this library; if not, write to the Free Software Foundation, Inc.,
1451 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
15**********/
16// "liveMedia"
17// Copyright (c) 1996-2020 Live Networks, Inc. All rights reserved.
18// RTP sink for H.264 or H.265 video
19// Implementation
20
21#include "H264or5VideoRTPSink.hh"
22#include "H264or5VideoStreamFramer.hh"
23
24////////// H264or5Fragmenter definition //////////
25
26// Because of the ideosyncracies of the H.264 RTP payload format, we implement
27// "H264or5VideoRTPSink" using a separate "H264or5Fragmenter" class that delivers,
28// to the "H264or5VideoRTPSink", only fragments that will fit within an outgoing
29// RTP packet. I.e., we implement fragmentation in this separate "H264or5Fragmenter"
30// class, rather than in "H264or5VideoRTPSink".
31// (Note: This class should be used only by "H264or5VideoRTPSink", or a subclass.)
32
33class H264or5Fragmenter: public FramedFilter {
34public:
35 H264or5Fragmenter(int hNumber, UsageEnvironment& env, FramedSource* inputSource,
36 unsigned inputBufferMax, unsigned maxOutputPacketSize);
37 virtual ~H264or5Fragmenter();
38
39 Boolean lastFragmentCompletedNALUnit() const { return fLastFragmentCompletedNALUnit; }
40
41private: // redefined virtual functions:
42 virtual void doGetNextFrame();
43 virtual void doStopGettingFrames();
44
45private:
46 static void afterGettingFrame(void* clientData, unsigned frameSize,
47 unsigned numTruncatedBytes,
48 struct timeval presentationTime,
49 unsigned durationInMicroseconds);
50 void afterGettingFrame1(unsigned frameSize,
51 unsigned numTruncatedBytes,
52 struct timeval presentationTime,
53 unsigned durationInMicroseconds);
54 void reset();
55
56private:
57 int fHNumber;
58 unsigned fInputBufferSize;
59 unsigned fMaxOutputPacketSize;
60 unsigned char* fInputBuffer;
61 unsigned fNumValidDataBytes;
62 unsigned fCurDataOffset;
63 unsigned fSaveNumTruncatedBytes;
64 Boolean fLastFragmentCompletedNALUnit;
65};
66
67
68////////// H264or5VideoRTPSink implementation //////////
69
70H264or5VideoRTPSink
71::H264or5VideoRTPSink(int hNumber,
72 UsageEnvironment& env, Groupsock* RTPgs, unsigned char rtpPayloadFormat,
73 u_int8_t const* vps, unsigned vpsSize,
74 u_int8_t const* sps, unsigned spsSize,
75 u_int8_t const* pps, unsigned ppsSize)
76 : VideoRTPSink(env, RTPgs, rtpPayloadFormat, 90000, hNumber == 264 ? "H264" : "H265"),
77 fHNumber(hNumber), fOurFragmenter(NULL), fFmtpSDPLine(NULL) {
78 if (vps != NULL) {
79 fVPSSize = vpsSize;
80 fVPS = new u_int8_t[fVPSSize];
81 memmove(fVPS, vps, fVPSSize);
82 } else {
83 fVPSSize = 0;
84 fVPS = NULL;
85 }
86 if (sps != NULL) {
87 fSPSSize = spsSize;
88 fSPS = new u_int8_t[fSPSSize];
89 memmove(fSPS, sps, fSPSSize);
90 } else {
91 fSPSSize = 0;
92 fSPS = NULL;
93 }
94 if (pps != NULL) {
95 fPPSSize = ppsSize;
96 fPPS = new u_int8_t[fPPSSize];
97 memmove(fPPS, pps, fPPSSize);
98 } else {
99 fPPSSize = 0;
100 fPPS = NULL;
101 }
102}
103
104H264or5VideoRTPSink::~H264or5VideoRTPSink() {
105 fSource = fOurFragmenter; // hack: in case "fSource" had gotten set to NULL before we were called
106 delete[] fFmtpSDPLine;
107 delete[] fVPS; delete[] fSPS; delete[] fPPS;
108 stopPlaying(); // call this now, because we won't have our 'fragmenter' when the base class destructor calls it later.
109
110 // Close our 'fragmenter' as well:
111 Medium::close(fOurFragmenter);
112 fSource = NULL; // for the base class destructor, which gets called next
113}
114
115Boolean H264or5VideoRTPSink::continuePlaying() {
116 // First, check whether we have a 'fragmenter' class set up yet.
117 // If not, create it now:
118 if (fOurFragmenter == NULL) {
119 fOurFragmenter = new H264or5Fragmenter(fHNumber, envir(), fSource, OutPacketBuffer::maxSize,
120 ourMaxPacketSize() - 12/*RTP hdr size*/);
121 } else {
122 fOurFragmenter->reassignInputSource(fSource);
123 }
124 fSource = fOurFragmenter;
125
126 // Then call the parent class's implementation:
127 return MultiFramedRTPSink::continuePlaying();
128}
129
130void H264or5VideoRTPSink::doSpecialFrameHandling(unsigned /*fragmentationOffset*/,
131 unsigned char* /*frameStart*/,
132 unsigned /*numBytesInFrame*/,
133 struct timeval framePresentationTime,
134 unsigned /*numRemainingBytes*/) {
135 // Set the RTP 'M' (marker) bit iff
136 // 1/ The most recently delivered fragment was the end of (or the only fragment of) an NAL unit, and
137 // 2/ This NAL unit was the last NAL unit of an 'access unit' (i.e. video frame).
138 if (fOurFragmenter != NULL) {
139 H264or5VideoStreamFramer* framerSource
140 = (H264or5VideoStreamFramer*)(fOurFragmenter->inputSource());
141 // This relies on our fragmenter's source being a "H264or5VideoStreamFramer".
142 if (((H264or5Fragmenter*)fOurFragmenter)->lastFragmentCompletedNALUnit()
143 && framerSource != NULL && framerSource->pictureEndMarker()) {
144 setMarkerBit();
145 framerSource->pictureEndMarker() = False;
146 }
147 }
148
149 setTimestamp(framePresentationTime);
150}
151
152Boolean H264or5VideoRTPSink
153::frameCanAppearAfterPacketStart(unsigned char const* /*frameStart*/,
154 unsigned /*numBytesInFrame*/) const {
155 return False;
156}
157
158
159////////// H264or5Fragmenter implementation //////////
160
161H264or5Fragmenter::H264or5Fragmenter(int hNumber,
162 UsageEnvironment& env, FramedSource* inputSource,
163 unsigned inputBufferMax, unsigned maxOutputPacketSize)
164 : FramedFilter(env, inputSource),
165 fHNumber(hNumber),
166 fInputBufferSize(inputBufferMax+1), fMaxOutputPacketSize(maxOutputPacketSize) {
167 fInputBuffer = new unsigned char[fInputBufferSize];
168 reset();
169}
170
171H264or5Fragmenter::~H264or5Fragmenter() {
172 delete[] fInputBuffer;
173 detachInputSource(); // so that the subsequent ~FramedFilter() doesn't delete it
174}
175
176void H264or5Fragmenter::doGetNextFrame() {
177 if (fNumValidDataBytes == 1) {
178 // We have no NAL unit data currently in the buffer. Read a new one:
179 fInputSource->getNextFrame(&fInputBuffer[1], fInputBufferSize - 1,
180 afterGettingFrame, this,
181 FramedSource::handleClosure, this);
182 } else {
183 // We have NAL unit data in the buffer. There are three cases to consider:
184 // 1. There is a new NAL unit in the buffer, and it's small enough to deliver
185 // to the RTP sink (as is).
186 // 2. There is a new NAL unit in the buffer, but it's too large to deliver to
187 // the RTP sink in its entirety. Deliver the first fragment of this data,
188 // as a FU packet, with one extra preceding header byte (for the "FU header").
189 // 3. There is a NAL unit in the buffer, and we've already delivered some
190 // fragment(s) of this. Deliver the next fragment of this data,
191 // as a FU packet, with two (H.264) or three (H.265) extra preceding header bytes
192 // (for the "NAL header" and the "FU header").
193
194 if (fMaxSize < fMaxOutputPacketSize) { // shouldn't happen
195 envir() << "H264or5Fragmenter::doGetNextFrame(): fMaxSize ("
196 << fMaxSize << ") is smaller than expected\n";
197 } else {
198 fMaxSize = fMaxOutputPacketSize;
199 }
200
201 fLastFragmentCompletedNALUnit = True; // by default
202 if (fCurDataOffset == 1) { // case 1 or 2
203 if (fNumValidDataBytes - 1 <= fMaxSize) { // case 1
204 memmove(fTo, &fInputBuffer[1], fNumValidDataBytes - 1);
205 fFrameSize = fNumValidDataBytes - 1;
206 fCurDataOffset = fNumValidDataBytes;
207 } else { // case 2
208 // We need to send the NAL unit data as FU packets. Deliver the first
209 // packet now. Note that we add "NAL header" and "FU header" bytes to the front
210 // of the packet (overwriting the existing "NAL header").
211 if (fHNumber == 264) {
212 fInputBuffer[0] = (fInputBuffer[1] & 0xE0) | 28; // FU indicator
213 fInputBuffer[1] = 0x80 | (fInputBuffer[1] & 0x1F); // FU header (with S bit)
214 } else { // 265
215 u_int8_t nal_unit_type = (fInputBuffer[1]&0x7E)>>1;
216 fInputBuffer[0] = (fInputBuffer[1] & 0x81) | (49<<1); // Payload header (1st byte)
217 fInputBuffer[1] = fInputBuffer[2]; // Payload header (2nd byte)
218 fInputBuffer[2] = 0x80 | nal_unit_type; // FU header (with S bit)
219 }
220 memmove(fTo, fInputBuffer, fMaxSize);
221 fFrameSize = fMaxSize;
222 fCurDataOffset += fMaxSize - 1;
223 fLastFragmentCompletedNALUnit = False;
224 }
225 } else { // case 3
226 // We are sending this NAL unit data as FU packets. We've already sent the
227 // first packet (fragment). Now, send the next fragment. Note that we add
228 // "NAL header" and "FU header" bytes to the front. (We reuse these bytes that
229 // we already sent for the first fragment, but clear the S bit, and add the E
230 // bit if this is the last fragment.)
231 unsigned numExtraHeaderBytes;
232 if (fHNumber == 264) {
233 fInputBuffer[fCurDataOffset-2] = fInputBuffer[0]; // FU indicator
234 fInputBuffer[fCurDataOffset-1] = fInputBuffer[1]&~0x80; // FU header (no S bit)
235 numExtraHeaderBytes = 2;
236 } else { // 265
237 fInputBuffer[fCurDataOffset-3] = fInputBuffer[0]; // Payload header (1st byte)
238 fInputBuffer[fCurDataOffset-2] = fInputBuffer[1]; // Payload header (2nd byte)
239 fInputBuffer[fCurDataOffset-1] = fInputBuffer[2]&~0x80; // FU header (no S bit)
240 numExtraHeaderBytes = 3;
241 }
242 unsigned numBytesToSend = numExtraHeaderBytes + (fNumValidDataBytes - fCurDataOffset);
243 if (numBytesToSend > fMaxSize) {
244 // We can't send all of the remaining data this time:
245 numBytesToSend = fMaxSize;
246 fLastFragmentCompletedNALUnit = False;
247 } else {
248 // This is the last fragment:
249 fInputBuffer[fCurDataOffset-1] |= 0x40; // set the E bit in the FU header
250 fNumTruncatedBytes = fSaveNumTruncatedBytes;
251 }
252 memmove(fTo, &fInputBuffer[fCurDataOffset-numExtraHeaderBytes], numBytesToSend);
253 fFrameSize = numBytesToSend;
254 fCurDataOffset += numBytesToSend - numExtraHeaderBytes;
255 }
256
257 if (fCurDataOffset >= fNumValidDataBytes) {
258 // We're done with this data. Reset the pointers for receiving new data:
259 fNumValidDataBytes = fCurDataOffset = 1;
260 }
261
262 // Complete delivery to the client:
263 FramedSource::afterGetting(this);
264 }
265}
266
267void H264or5Fragmenter::doStopGettingFrames() {
268 // Make sure that we don't have any stale data fragments lying around, should we later resume:
269 reset();
270 FramedFilter::doStopGettingFrames();
271}
272
273void H264or5Fragmenter::afterGettingFrame(void* clientData, unsigned frameSize,
274 unsigned numTruncatedBytes,
275 struct timeval presentationTime,
276 unsigned durationInMicroseconds) {
277 H264or5Fragmenter* fragmenter = (H264or5Fragmenter*)clientData;
278 fragmenter->afterGettingFrame1(frameSize, numTruncatedBytes, presentationTime,
279 durationInMicroseconds);
280}
281
282void H264or5Fragmenter::afterGettingFrame1(unsigned frameSize,
283 unsigned numTruncatedBytes,
284 struct timeval presentationTime,
285 unsigned durationInMicroseconds) {
286 fNumValidDataBytes += frameSize;
287 fSaveNumTruncatedBytes = numTruncatedBytes;
288 fPresentationTime = presentationTime;
289 fDurationInMicroseconds = durationInMicroseconds;
290
291 // Deliver data to the client:
292 doGetNextFrame();
293}
294
295void H264or5Fragmenter::reset() {
296 fNumValidDataBytes = fCurDataOffset = 1;
297 fSaveNumTruncatedBytes = 0;
298 fLastFragmentCompletedNALUnit = True;
299}
300