1/**************************************************************************/
2/* video_stream_theora.cpp */
3/**************************************************************************/
4/* This file is part of: */
5/* GODOT ENGINE */
6/* https://godotengine.org */
7/**************************************************************************/
8/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
9/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */
10/* */
11/* Permission is hereby granted, free of charge, to any person obtaining */
12/* a copy of this software and associated documentation files (the */
13/* "Software"), to deal in the Software without restriction, including */
14/* without limitation the rights to use, copy, modify, merge, publish, */
15/* distribute, sublicense, and/or sell copies of the Software, and to */
16/* permit persons to whom the Software is furnished to do so, subject to */
17/* the following conditions: */
18/* */
19/* The above copyright notice and this permission notice shall be */
20/* included in all copies or substantial portions of the Software. */
21/* */
22/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
23/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
24/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
25/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
26/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
27/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
28/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
29/**************************************************************************/
30
31#include "video_stream_theora.h"
32
33#include "core/config/project_settings.h"
34#include "core/os/os.h"
35#include "scene/resources/image_texture.h"
36
37#ifdef _MSC_VER
38#pragma warning(push)
39#pragma warning(disable : 4127)
40#endif
41
42#include "thirdparty/misc/yuv2rgb.h"
43
44#ifdef _MSC_VER
45#pragma warning(pop)
46#endif
47
48int VideoStreamPlaybackTheora::buffer_data() {
49 char *buffer = ogg_sync_buffer(&oy, 4096);
50
51#ifdef THEORA_USE_THREAD_STREAMING
52
53 int read;
54
55 do {
56 thread_sem->post();
57 read = MIN(ring_buffer.data_left(), 4096);
58 if (read) {
59 ring_buffer.read((uint8_t *)buffer, read);
60 ogg_sync_wrote(&oy, read);
61 } else {
62 OS::get_singleton()->delay_usec(100);
63 }
64
65 } while (read == 0);
66
67 return read;
68
69#else
70
71 uint64_t bytes = file->get_buffer((uint8_t *)buffer, 4096);
72 ogg_sync_wrote(&oy, bytes);
73 return (bytes);
74
75#endif
76}
77
78int VideoStreamPlaybackTheora::queue_page(ogg_page *page) {
79 if (theora_p) {
80 ogg_stream_pagein(&to, page);
81 if (to.e_o_s) {
82 theora_eos = true;
83 }
84 }
85 if (vorbis_p) {
86 ogg_stream_pagein(&vo, page);
87 if (vo.e_o_s) {
88 vorbis_eos = true;
89 }
90 }
91 return 0;
92}
93
94void VideoStreamPlaybackTheora::video_write() {
95 th_ycbcr_buffer yuv;
96 th_decode_ycbcr_out(td, yuv);
97
98 int pitch = 4;
99 frame_data.resize(size.x * size.y * pitch);
100 {
101 uint8_t *w = frame_data.ptrw();
102 char *dst = (char *)w;
103
104 if (px_fmt == TH_PF_444) {
105 yuv444_2_rgb8888((uint8_t *)dst, (uint8_t *)yuv[0].data, (uint8_t *)yuv[1].data, (uint8_t *)yuv[2].data, size.x, size.y, yuv[0].stride, yuv[1].stride, size.x << 2);
106
107 } else if (px_fmt == TH_PF_422) {
108 yuv422_2_rgb8888((uint8_t *)dst, (uint8_t *)yuv[0].data, (uint8_t *)yuv[1].data, (uint8_t *)yuv[2].data, size.x, size.y, yuv[0].stride, yuv[1].stride, size.x << 2);
109
110 } else if (px_fmt == TH_PF_420) {
111 yuv420_2_rgb8888((uint8_t *)dst, (uint8_t *)yuv[0].data, (uint8_t *)yuv[1].data, (uint8_t *)yuv[2].data, size.x, size.y, yuv[0].stride, yuv[1].stride, size.x << 2);
112 }
113
114 format = Image::FORMAT_RGBA8;
115 }
116
117 Ref<Image> img = memnew(Image(size.x, size.y, 0, Image::FORMAT_RGBA8, frame_data)); //zero copy image creation
118
119 texture->update(img); //zero copy send to rendering server
120
121 frames_pending = 1;
122}
123
124void VideoStreamPlaybackTheora::clear() {
125 if (file.is_null()) {
126 return;
127 }
128
129 if (vorbis_p) {
130 ogg_stream_clear(&vo);
131 if (vorbis_p >= 3) {
132 vorbis_block_clear(&vb);
133 vorbis_dsp_clear(&vd);
134 }
135 vorbis_comment_clear(&vc);
136 vorbis_info_clear(&vi);
137 vorbis_p = 0;
138 }
139 if (theora_p) {
140 ogg_stream_clear(&to);
141 th_decode_free(td);
142 th_comment_clear(&tc);
143 th_info_clear(&ti);
144 theora_p = 0;
145 }
146 ogg_sync_clear(&oy);
147
148#ifdef THEORA_USE_THREAD_STREAMING
149 thread_exit = true;
150 thread_sem->post(); //just in case
151 thread.wait_to_finish();
152 ring_buffer.clear();
153#endif
154
155 theora_p = 0;
156 vorbis_p = 0;
157 videobuf_ready = 0;
158 frames_pending = 0;
159 videobuf_time = 0;
160 theora_eos = false;
161 vorbis_eos = false;
162
163 file.unref();
164 playing = false;
165}
166
167void VideoStreamPlaybackTheora::set_file(const String &p_file) {
168 ERR_FAIL_COND(playing);
169 ogg_packet op;
170 th_setup_info *ts = nullptr;
171
172 file_name = p_file;
173 file = FileAccess::open(p_file, FileAccess::READ);
174 ERR_FAIL_COND_MSG(file.is_null(), "Cannot open file '" + p_file + "'.");
175
176#ifdef THEORA_USE_THREAD_STREAMING
177 thread_exit = false;
178 thread_eof = false;
179 //pre-fill buffer
180 int to_read = ring_buffer.space_left();
181 uint64_t read = file->get_buffer(read_buffer.ptr(), to_read);
182 ring_buffer.write(read_buffer.ptr(), read);
183
184 thread.start(_streaming_thread, this);
185#endif
186
187 ogg_sync_init(&oy);
188
189 /* init supporting Vorbis structures needed in header parsing */
190 vorbis_info_init(&vi);
191 vorbis_comment_init(&vc);
192
193 /* init supporting Theora structures needed in header parsing */
194 th_comment_init(&tc);
195 th_info_init(&ti);
196
197 theora_eos = false;
198 vorbis_eos = false;
199
200 /* Ogg file open; parse the headers */
201 /* Only interested in Vorbis/Theora streams */
202 int stateflag = 0;
203
204 int audio_track_skip = audio_track;
205
206 while (!stateflag) {
207 int ret = buffer_data();
208 if (ret == 0) {
209 break;
210 }
211 while (ogg_sync_pageout(&oy, &og) > 0) {
212 ogg_stream_state test;
213
214 /* is this a mandated initial header? If not, stop parsing */
215 if (!ogg_page_bos(&og)) {
216 /* don't leak the page; get it into the appropriate stream */
217 queue_page(&og);
218 stateflag = 1;
219 break;
220 }
221
222 ogg_stream_init(&test, ogg_page_serialno(&og));
223 ogg_stream_pagein(&test, &og);
224 ogg_stream_packetout(&test, &op);
225
226 /* identify the codec: try theora */
227 if (!theora_p && th_decode_headerin(&ti, &tc, &ts, &op) >= 0) {
228 /* it is theora */
229 memcpy(&to, &test, sizeof(test));
230 theora_p = 1;
231 } else if (!vorbis_p && vorbis_synthesis_headerin(&vi, &vc, &op) >= 0) {
232 /* it is vorbis */
233 if (audio_track_skip) {
234 vorbis_info_clear(&vi);
235 vorbis_comment_clear(&vc);
236 ogg_stream_clear(&test);
237 vorbis_info_init(&vi);
238 vorbis_comment_init(&vc);
239
240 audio_track_skip--;
241 } else {
242 memcpy(&vo, &test, sizeof(test));
243 vorbis_p = 1;
244 }
245 } else {
246 /* whatever it is, we don't care about it */
247 ogg_stream_clear(&test);
248 }
249 }
250 /* fall through to non-bos page parsing */
251 }
252
253 /* we're expecting more header packets. */
254 while ((theora_p && theora_p < 3) || (vorbis_p && vorbis_p < 3)) {
255 int ret = 0;
256
257 /* look for further theora headers */
258 if (theora_p && theora_p < 3) {
259 ret = ogg_stream_packetout(&to, &op);
260 }
261 while (theora_p && theora_p < 3 && ret) {
262 if (ret < 0) {
263 fprintf(stderr, "Error parsing Theora stream headers; corrupt stream?\n");
264 clear();
265 return;
266 }
267 if (!th_decode_headerin(&ti, &tc, &ts, &op)) {
268 fprintf(stderr, "Error parsing Theora stream headers; corrupt stream?\n");
269 clear();
270 return;
271 }
272 ret = ogg_stream_packetout(&to, &op);
273 theora_p++;
274 }
275
276 /* look for more vorbis header packets */
277 if (vorbis_p && vorbis_p < 3) {
278 ret = ogg_stream_packetout(&vo, &op);
279 }
280 while (vorbis_p && vorbis_p < 3 && ret) {
281 if (ret < 0) {
282 fprintf(stderr, "Error parsing Vorbis stream headers; corrupt stream?\n");
283 clear();
284 return;
285 }
286 ret = vorbis_synthesis_headerin(&vi, &vc, &op);
287 if (ret) {
288 fprintf(stderr, "Error parsing Vorbis stream headers; corrupt stream?\n");
289 clear();
290 return;
291 }
292 vorbis_p++;
293 if (vorbis_p == 3) {
294 break;
295 }
296 ret = ogg_stream_packetout(&vo, &op);
297 }
298
299 /* The header pages/packets will arrive before anything else we
300 care about, or the stream is not obeying spec */
301
302 if (ogg_sync_pageout(&oy, &og) > 0) {
303 queue_page(&og); /* demux into the appropriate stream */
304 } else {
305 int ret2 = buffer_data(); /* someone needs more data */
306 if (ret2 == 0) {
307 fprintf(stderr, "End of file while searching for codec headers.\n");
308 clear();
309 return;
310 }
311 }
312 }
313
314 /* And now we have it all. Initialize decoders. */
315 if (theora_p) {
316 td = th_decode_alloc(&ti, ts);
317 px_fmt = ti.pixel_fmt;
318 switch (ti.pixel_fmt) {
319 case TH_PF_420:
320 //printf(" 4:2:0 video\n");
321 break;
322 case TH_PF_422:
323 //printf(" 4:2:2 video\n");
324 break;
325 case TH_PF_444:
326 //printf(" 4:4:4 video\n");
327 break;
328 case TH_PF_RSVD:
329 default:
330 printf(" video\n (UNKNOWN Chroma sampling!)\n");
331 break;
332 }
333 th_decode_ctl(td, TH_DECCTL_GET_PPLEVEL_MAX, &pp_level_max,
334 sizeof(pp_level_max));
335 pp_level = 0;
336 th_decode_ctl(td, TH_DECCTL_SET_PPLEVEL, &pp_level, sizeof(pp_level));
337 pp_inc = 0;
338
339 int w;
340 int h;
341 w = ((ti.pic_x + ti.frame_width + 1) & ~1) - (ti.pic_x & ~1);
342 h = ((ti.pic_y + ti.frame_height + 1) & ~1) - (ti.pic_y & ~1);
343 size.x = w;
344 size.y = h;
345
346 Ref<Image> img = Image::create_empty(w, h, false, Image::FORMAT_RGBA8);
347 texture->set_image(img);
348
349 } else {
350 /* tear down the partial theora setup */
351 th_info_clear(&ti);
352 th_comment_clear(&tc);
353 }
354
355 th_setup_free(ts);
356
357 if (vorbis_p) {
358 vorbis_synthesis_init(&vd, &vi);
359 vorbis_block_init(&vd, &vb);
360 //_setup(vi.channels, vi.rate);
361 } else {
362 /* tear down the partial vorbis setup */
363 vorbis_info_clear(&vi);
364 vorbis_comment_clear(&vc);
365 }
366
367 playing = false;
368 buffering = true;
369 time = 0;
370 audio_frames_wrote = 0;
371}
372
373double VideoStreamPlaybackTheora::get_time() const {
374 // FIXME: AudioServer output latency was fixed in af9bb0e, previously it used to
375 // systematically return 0. Now that it gives a proper latency, it broke this
376 // code where the delay compensation likely never really worked.
377 return time - /* AudioServer::get_singleton()->get_output_latency() - */ delay_compensation;
378}
379
380Ref<Texture2D> VideoStreamPlaybackTheora::get_texture() const {
381 return texture;
382}
383
384void VideoStreamPlaybackTheora::update(double p_delta) {
385 if (file.is_null()) {
386 return;
387 }
388
389 if (!playing || paused) {
390 //printf("not playing\n");
391 return;
392 }
393
394#ifdef THEORA_USE_THREAD_STREAMING
395 thread_sem->post();
396#endif
397
398 time += p_delta;
399
400 if (videobuf_time > get_time()) {
401 return; //no new frames need to be produced
402 }
403
404 bool frame_done = false;
405 bool audio_done = !vorbis_p;
406
407 while (!frame_done || (!audio_done && !vorbis_eos)) {
408 //a frame needs to be produced
409
410 ogg_packet op;
411 bool no_theora = false;
412 bool buffer_full = false;
413
414 while (vorbis_p && !audio_done && !buffer_full) {
415 int ret;
416 float **pcm;
417
418 /* if there's pending, decoded audio, grab it */
419 ret = vorbis_synthesis_pcmout(&vd, &pcm);
420 if (ret > 0) {
421 const int AUXBUF_LEN = 4096;
422 int to_read = ret;
423 float aux_buffer[AUXBUF_LEN];
424
425 while (to_read) {
426 int m = MIN(AUXBUF_LEN / vi.channels, to_read);
427
428 int count = 0;
429
430 for (int j = 0; j < m; j++) {
431 for (int i = 0; i < vi.channels; i++) {
432 aux_buffer[count++] = pcm[i][j];
433 }
434 }
435
436 if (mix_callback) {
437 int mixed = mix_callback(mix_udata, aux_buffer, m);
438 to_read -= mixed;
439 if (mixed != m) { //could mix no more
440 buffer_full = true;
441 break;
442 }
443 } else {
444 to_read -= m; //just pretend we sent the audio
445 }
446 }
447
448 vorbis_synthesis_read(&vd, ret - to_read);
449
450 audio_frames_wrote += ret - to_read;
451
452 } else {
453 /* no pending audio; is there a pending packet to decode? */
454 if (ogg_stream_packetout(&vo, &op) > 0) {
455 if (vorbis_synthesis(&vb, &op) == 0) { /* test for success! */
456 vorbis_synthesis_blockin(&vd, &vb);
457 }
458 } else { /* we need more data; break out to suck in another page */
459 break;
460 }
461 }
462
463 audio_done = videobuf_time < (audio_frames_wrote / float(vi.rate));
464
465 if (buffer_full) {
466 break;
467 }
468 }
469
470 while (theora_p && !frame_done) {
471 /* theora is one in, one out... */
472 if (ogg_stream_packetout(&to, &op) > 0) {
473 /*HACK: This should be set after a seek or a gap, but we might not have
474 a granulepos for the first packet (we only have them for the last
475 packet on a page), so we just set it as often as we get it.
476 To do this right, we should back-track from the last packet on the
477 page and compute the correct granulepos for the first packet after
478 a seek or a gap.*/
479 if (op.granulepos >= 0) {
480 th_decode_ctl(td, TH_DECCTL_SET_GRANPOS, &op.granulepos,
481 sizeof(op.granulepos));
482 }
483 ogg_int64_t videobuf_granulepos;
484 if (th_decode_packetin(td, &op, &videobuf_granulepos) == 0) {
485 videobuf_time = th_granule_time(td, videobuf_granulepos);
486
487 //printf("frame time %f, play time %f, ready %i\n", (float)videobuf_time, get_time(), videobuf_ready);
488
489 /* is it already too old to be useful? This is only actually
490 useful cosmetically after a SIGSTOP. Note that we have to
491 decode the frame even if we don't show it (for now) due to
492 keyframing. Soon enough libtheora will be able to deal
493 with non-keyframe seeks. */
494
495 if (videobuf_time >= get_time()) {
496 frame_done = true;
497 } else {
498 /*If we are too slow, reduce the pp level.*/
499 pp_inc = pp_level > 0 ? -1 : 0;
500 }
501 }
502
503 } else {
504 no_theora = true;
505 break;
506 }
507 }
508
509#ifdef THEORA_USE_THREAD_STREAMING
510 if (file.is_valid() && thread_eof && no_theora && theora_eos && ring_buffer.data_left() == 0) {
511#else
512 if (file.is_valid() && /*!videobuf_ready && */ no_theora && theora_eos) {
513#endif
514 //printf("video done, stopping\n");
515 stop();
516 return;
517 }
518
519 if (!frame_done || !audio_done) {
520 //what's the point of waiting for audio to grab a page?
521
522 buffer_data();
523 while (ogg_sync_pageout(&oy, &og) > 0) {
524 queue_page(&og);
525 }
526 }
527
528 /* If playback has begun, top audio buffer off immediately. */
529 //if(stateflag) audio_write_nonblocking();
530
531 /* are we at or past time for this video frame? */
532 if (videobuf_ready && videobuf_time <= get_time()) {
533 //video_write();
534 //videobuf_ready=0;
535 } else {
536 //printf("frame at %f not ready (time %f), ready %i\n", (float)videobuf_time, get_time(), videobuf_ready);
537 }
538
539 double tdiff = videobuf_time - get_time();
540 /*If we have lots of extra time, increase the post-processing level.*/
541 if (tdiff > ti.fps_denominator * 0.25 / ti.fps_numerator) {
542 pp_inc = pp_level < pp_level_max ? 1 : 0;
543 } else if (tdiff < ti.fps_denominator * 0.05 / ti.fps_numerator) {
544 pp_inc = pp_level > 0 ? -1 : 0;
545 }
546 }
547
548 video_write();
549}
550
551void VideoStreamPlaybackTheora::play() {
552 if (!playing) {
553 time = 0;
554 } else {
555 stop();
556 }
557
558 playing = true;
559 delay_compensation = GLOBAL_GET("audio/video/video_delay_compensation_ms");
560 delay_compensation /= 1000.0;
561}
562
563void VideoStreamPlaybackTheora::stop() {
564 if (playing) {
565 clear();
566 set_file(file_name); //reset
567 }
568 playing = false;
569 time = 0;
570}
571
572bool VideoStreamPlaybackTheora::is_playing() const {
573 return playing;
574}
575
576void VideoStreamPlaybackTheora::set_paused(bool p_paused) {
577 paused = p_paused;
578}
579
580bool VideoStreamPlaybackTheora::is_paused() const {
581 return paused;
582}
583
584double VideoStreamPlaybackTheora::get_length() const {
585 return 0;
586}
587
588double VideoStreamPlaybackTheora::get_playback_position() const {
589 return get_time();
590}
591
592void VideoStreamPlaybackTheora::seek(double p_time) {
593 WARN_PRINT_ONCE("Seeking in Theora videos is not implemented yet (it's only supported for GDExtension-provided video streams).");
594}
595
596int VideoStreamPlaybackTheora::get_channels() const {
597 return vi.channels;
598}
599
600void VideoStreamPlaybackTheora::set_audio_track(int p_idx) {
601 audio_track = p_idx;
602}
603
604int VideoStreamPlaybackTheora::get_mix_rate() const {
605 return vi.rate;
606}
607
608#ifdef THEORA_USE_THREAD_STREAMING
609
610void VideoStreamPlaybackTheora::_streaming_thread(void *ud) {
611 VideoStreamPlaybackTheora *vs = static_cast<VideoStreamPlaybackTheora *>(ud);
612
613 while (!vs->thread_exit) {
614 //just fill back the buffer
615 if (!vs->thread_eof) {
616 int to_read = vs->ring_buffer.space_left();
617 if (to_read > 0) {
618 uint64_t read = vs->file->get_buffer(vs->read_buffer.ptr(), to_read);
619 vs->ring_buffer.write(vs->read_buffer.ptr(), read);
620 vs->thread_eof = vs->file->eof_reached();
621 }
622 }
623
624 vs->thread_sem->wait();
625 }
626}
627
628#endif
629
630VideoStreamPlaybackTheora::VideoStreamPlaybackTheora() {
631 texture = Ref<ImageTexture>(memnew(ImageTexture));
632
633#ifdef THEORA_USE_THREAD_STREAMING
634 int rb_power = nearest_shift(RB_SIZE_KB * 1024);
635 ring_buffer.resize(rb_power);
636 read_buffer.resize(RB_SIZE_KB * 1024);
637 thread_sem = Semaphore::create();
638
639#endif
640}
641
642VideoStreamPlaybackTheora::~VideoStreamPlaybackTheora() {
643#ifdef THEORA_USE_THREAD_STREAMING
644 memdelete(thread_sem);
645#endif
646 clear();
647};
648
649void VideoStreamTheora::_bind_methods() {}
650
651Ref<Resource> ResourceFormatLoaderTheora::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress, CacheMode p_cache_mode) {
652 Ref<FileAccess> f = FileAccess::open(p_path, FileAccess::READ);
653 if (f.is_null()) {
654 if (r_error) {
655 *r_error = ERR_CANT_OPEN;
656 }
657 return Ref<Resource>();
658 }
659
660 VideoStreamTheora *stream = memnew(VideoStreamTheora);
661 stream->set_file(p_path);
662
663 Ref<VideoStreamTheora> ogv_stream = Ref<VideoStreamTheora>(stream);
664
665 if (r_error) {
666 *r_error = OK;
667 }
668
669 return ogv_stream;
670}
671
672void ResourceFormatLoaderTheora::get_recognized_extensions(List<String> *p_extensions) const {
673 p_extensions->push_back("ogv");
674}
675
676bool ResourceFormatLoaderTheora::handles_type(const String &p_type) const {
677 return ClassDB::is_parent_class(p_type, "VideoStream");
678}
679
680String ResourceFormatLoaderTheora::get_resource_type(const String &p_path) const {
681 String el = p_path.get_extension().to_lower();
682 if (el == "ogv") {
683 return "VideoStreamTheora";
684 }
685 return "";
686}
687