Ring Daemon 16.0.0
Loading...
Searching...
No Matches
video_rtp_session.cpp
Go to the documentation of this file.
1/*
2 * Copyright (C) 2004-2025 Savoir-faire Linux Inc.
3 *
4 * This program is free software: you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation, either version 3 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <https://www.gnu.org/licenses/>.
16 */
17
18#include "client/videomanager.h"
19#include "video_rtp_session.h"
20#include "video_sender.h"
22#include "video_mixer.h"
23#include "socket_pair.h"
24#include "sip/sipvoiplink.h" // for enqueueKeyframeRequest
25#include "manager.h"
26#include "media_const.h"
27#ifdef ENABLE_PLUGIN
28#include "plugin/streamdata.h"
30#endif
31#include "logger.h"
32#include "string_utils.h"
33#include "call.h"
34#include "conference.h"
35#include "congestion_control.h"
36
37#include "account_const.h"
38
39#include <dhtnet/ice_socket.h>
40#include <asio/post.hpp>
41#include <asio/io_context.hpp>
42
43#include <sstream>
44#include <map>
45#include <string>
46#include <thread>
47#include <chrono>
48
49namespace jami {
50namespace video {
51
52using std::string;
53
54static constexpr unsigned MAX_REMB_DEC {1};
55
56constexpr auto DELAY_AFTER_RESTART = std::chrono::milliseconds(1000);
57constexpr auto EXPIRY_TIME_RTCP = std::chrono::seconds(2);
58constexpr auto DELAY_AFTER_REMB_INC = std::chrono::seconds(1);
59constexpr auto DELAY_AFTER_REMB_DEC = std::chrono::milliseconds(500);
60
62 const string& streamId,
64 const std::shared_ptr<MediaRecorder>& rec)
65 : RtpSession(callId, streamId, MediaType::MEDIA_VIDEO)
66 , localVideoParams_(localVideoParams)
67 , videoBitrateInfo_ {}
68 , rtcpCheckerThread_([] { return true; }, [this] { processRtcpChecker(); }, [] {})
69 , cc(std::make_unique<CongestionControl>())
70{
71 recorder_ = rec;
72 setupVideoBitrateInfo(); // reset bitrate
73 JAMI_LOG("[{:p}] Video RTP session created for call {} (recorder {:p})", fmt::ptr(this), callId_, fmt::ptr(recorder_));
74}
75
77{
79 stop();
80 JAMI_LOG("[{:p}] Video RTP session destroyed", fmt::ptr(this));
81}
82
85{
86 return videoBitrateInfo_;
87}
88
91void
93{
95 // adjust send->codec bitrate info for higher video resolutions
96 auto codecVideo = std::static_pointer_cast<jami::SystemVideoCodecInfo>(send_.codec);
97 if (codecVideo) {
98 auto const pixels = localVideoParams_.height * localVideoParams_.width;
99 codecVideo->bitrate = std::max((unsigned int)(pixels * 0.001), SystemCodecInfo::DEFAULT_VIDEO_BITRATE);
100 codecVideo->maxBitrate = std::max((unsigned int)(pixels * 0.0015), SystemCodecInfo::DEFAULT_MAX_BITRATE);
101 }
102 setupVideoBitrateInfo();
103}
104
105void
107{
108 cbKeyFrameRequest_ = std::move(cb);
109}
110
111void
112VideoRtpSession::startSender()
113{
114 std::lock_guard lock(mutex_);
115
116 JAMI_DBG("[%p] Start video RTP sender: input [%s] - muted [%s]",
117 this,
118 conference_ ? "Video Mixer" : input_.c_str(),
119 send_.onHold ? "YES" : "NO");
120
121 if (not socketPair_) {
122 // Ignore if the transport is not set yet
123 JAMI_WARN("[%p] Transport not set yet", this);
124 return;
125 }
126
128 if (sender_) {
129 if (videoLocal_)
130 videoLocal_->detach(sender_.get());
131 if (videoMixer_)
132 videoMixer_->detach(sender_.get());
133 JAMI_WARN("[%p] Restarting video sender", this);
134 }
135
136 if (not conference_) {
137 videoLocal_ = getVideoInput(input_);
138 if (videoLocal_) {
139 videoLocal_->setRecorderCallback(
140 [w=weak_from_this()](const MediaStream& ms) {
141 asio::post(*Manager::instance().ioContext(), [w=std::move(w), ms]() {
142 if (auto shared = w.lock())
143 shared->attachLocalRecorder(ms);
144 });
145 });
146 auto newParams = videoLocal_->getParams();
147 try {
148 if (newParams.valid()
149 && newParams.wait_for(NEWPARAMS_TIMEOUT) == std::future_status::ready) {
150 localVideoParams_ = newParams.get();
151 } else {
152 JAMI_ERR("[%p] No valid new video parameters", this);
153 return;
154 }
155 } catch (const std::exception& e) {
156 JAMI_ERR("Exception during retrieving video parameters: %s", e.what());
157 return;
158 }
159 } else {
160 JAMI_WARN("Unable to lock video input");
161 return;
162 }
163
164#if (defined(__ANDROID__) || (defined(TARGET_OS_IOS) && TARGET_OS_IOS))
165 videoLocal_->setupSink(localVideoParams_.width, localVideoParams_.height);
166#endif
167 }
168
169 // be sure to not send any packets before saving last RTP seq value
170 socketPair_->stopSendOp();
171
172 auto codecVideo = std::static_pointer_cast<jami::SystemVideoCodecInfo>(send_.codec);
173 auto autoQuality = codecVideo->isAutoQualityEnabled;
174
175 send_.linkableHW = conference_ == nullptr;
176 send_.bitrate = videoBitrateInfo_.videoBitrateCurrent;
177 // NOTE:
178 // Current implementation does not handle resolution change
179 // (needed by window sharing feature) with HW codecs, so HW
180 // codecs will be disabled for now.
181 bool allowHwAccel = (localVideoParams_.format != "x11grab" && localVideoParams_.format != "dxgigrab" && localVideoParams_.format != "lavfi");
182
183 if (socketPair_)
184 initSeqVal_ = socketPair_->lastSeqValOut();
185
186 try {
187 sender_.reset();
188 socketPair_->stopSendOp(false);
189 MediaStream ms
190 = !videoMixer_
191 ? MediaStream("video sender",
193 1 / static_cast<rational<int>>(localVideoParams_.framerate),
194 localVideoParams_.width == 0 ? 1080u : localVideoParams_.width,
195 localVideoParams_.height == 0 ? 720u : localVideoParams_.height,
196 send_.bitrate,
197 static_cast<rational<int>>(localVideoParams_.framerate))
198 : videoMixer_->getStream("Video Sender");
199 sender_.reset(new VideoSender(
200 getRemoteRtpUri(), ms, send_, *socketPair_, initSeqVal_ + 1, mtu_, allowHwAccel));
201 if (changeOrientationCallback_)
202 sender_->setChangeOrientationCallback(changeOrientationCallback_);
203 if (socketPair_)
204 socketPair_->setPacketLossCallback([this]() { cbKeyFrameRequest_(); });
205
206 } catch (const MediaEncoderException& e) {
207 JAMI_ERR("%s", e.what());
208 send_.enabled = false;
209 }
210 lastMediaRestart_ = clock::now();
211 last_REMB_inc_ = clock::now();
212 last_REMB_dec_ = clock::now();
213 if (autoQuality and not rtcpCheckerThread_.isRunning())
214 rtcpCheckerThread_.start();
215 else if (not autoQuality and rtcpCheckerThread_.isRunning())
216 rtcpCheckerThread_.join();
217 // Block reads to received feedback packets
218 if(socketPair_)
219 socketPair_->setReadBlockingMode(true);
220 }
221}
222
223void
225{
226 std::lock_guard lock(mutex_);
227
228 // ensure that start has been called before restart
229 if (not socketPair_)
230 return;
231
232 startSender();
233
234 if (conference_)
235 setupConferenceVideoPipeline(*conference_, Direction::SEND);
236 else
237 setupVideoPipeline();
238}
239
240void
241VideoRtpSession::stopSender(bool forceStopSocket)
242{
243 // Concurrency protection must be done by caller.
244
245 JAMI_DBG("[%p] Stop video RTP sender: input [%s] - muted [%s]",
246 this,
247 conference_ ? "Video Mixer" : input_.c_str(),
248 send_.onHold ? "YES" : "NO");
249
250 if (sender_) {
251 if (videoLocal_)
252 videoLocal_->detach(sender_.get());
253 if (videoMixer_)
254 videoMixer_->detach(sender_.get());
255 sender_.reset();
256 }
257
258 if (socketPair_) {
261 socketPair_->stopSendOp();
262 socketPair_->setReadBlockingMode(false);
263 }
264 }
265}
266
267void
268VideoRtpSession::startReceiver()
269{
270 // Concurrency protection must be done by caller.
271
272 JAMI_DBG("[%p] Starting receiver", this);
273
275 if (receiveThread_)
276 JAMI_WARN("[%p] Already has a receiver, restarting", this);
277 receiveThread_.reset(
278 new VideoReceiveThread(callId_, !conference_, receive_.receiving_sdp, mtu_));
279
280 // ensure that start has been called
281 if (not socketPair_)
282 return;
283
284 // XXX keyframe requests can timeout if unanswered
285 receiveThread_->addIOContext(*socketPair_);
286 receiveThread_->setSuccessfulSetupCb(onSuccessfulSetup_);
287 receiveThread_->startLoop();
288 receiveThread_->setRequestKeyFrameCallback([this]() { cbKeyFrameRequest_(); });
289 receiveThread_->setRotation(rotation_.load());
290 if (videoMixer_ and conference_) {
291 // Note, this should be managed differently, this is a bit hacky
292 auto audioId = streamId_;
293 string_replace(audioId, "video", "audio");
294 auto activeStream = videoMixer_->verifyActive(audioId);
295 videoMixer_->removeAudioOnlySource(callId_, audioId);
296 if (activeStream)
297 videoMixer_->setActiveStream(streamId_);
298 }
299 receiveThread_->setRecorderCallback([w=weak_from_this()](const MediaStream& ms) {
300 asio::post(*Manager::instance().ioContext(), [w=std::move(w), ms]() {
301 if (auto shared = w.lock())
302 shared->attachRemoteRecorder(ms);
303 });
304 });
305 } else {
306 JAMI_DBG("[%p] Video receiver disabled", this);
307 if (receiveThread_ and videoMixer_ and conference_) {
308 // Note, this should be managed differently, this is a bit hacky
309 auto audioId_ = streamId_;
310 string_replace(audioId_, "video", "audio");
311 auto activeStream = videoMixer_->verifyActive(streamId_);
312 videoMixer_->addAudioOnlySource(callId_, audioId_);
313 receiveThread_->detach(videoMixer_.get());
314 if (activeStream)
315 videoMixer_->setActiveStream(audioId_);
316 }
317 }
318 if (socketPair_)
319 socketPair_->setReadBlockingMode(true);
320}
321
322void
323VideoRtpSession::stopReceiver(bool forceStopSocket)
324{
325 // Concurrency protection must be done by caller.
326
327 JAMI_DBG("[%p] Stopping receiver", this);
328
329 if (not receiveThread_)
330 return;
331
332 if (videoMixer_) {
333 auto activeStream = videoMixer_->verifyActive(streamId_);
334 auto audioId = streamId_;
335 string_replace(audioId, "video", "audio");
336 videoMixer_->addAudioOnlySource(callId_, audioId);
337 receiveThread_->detach(videoMixer_.get());
338 if (activeStream)
339 videoMixer_->setActiveStream(audioId);
340 }
341
342 // We need to disable the read operation, otherwise the
343 // receiver thread will block since the peer stopped sending
344 // RTP packets.
345 bool const isSendingVideo = send_.enabled && !send_.onHold;
346 if (socketPair_) {
348 socketPair_->setReadBlockingMode(false);
349 socketPair_->stopSendOp();
350 }
351 }
352
353 auto ms = receiveThread_->getInfo();
354 if (auto ob = recorder_->getStream(ms.name)) {
355 receiveThread_->detach(ob);
356 recorder_->removeStream(ms);
357 }
358
360 receiveThread_->stopLoop();
361 receiveThread_->stopSink();
362}
363
364void
365VideoRtpSession::start(std::unique_ptr<dhtnet::IceSocket> rtp_sock, std::unique_ptr<dhtnet::IceSocket> rtcp_sock)
366{
367 std::lock_guard lock(mutex_);
368
370 stop();
371 return;
372 }
373
374 try {
375 if (rtp_sock and rtcp_sock) {
376 if (send_.addr) {
377 rtp_sock->setDefaultRemoteAddress(send_.addr);
378 }
379
381 if (rtcpAddr) {
382 rtcp_sock->setDefaultRemoteAddress(rtcpAddr);
383 }
384 socketPair_.reset(new SocketPair(std::move(rtp_sock), std::move(rtcp_sock)));
385 } else {
386 socketPair_.reset(new SocketPair(getRemoteRtpUri().c_str(), receive_.addr.getPort()));
387 }
388
389 last_REMB_inc_ = clock::now();
390 last_REMB_dec_ = clock::now();
391
392 socketPair_->setRtpDelayCallback(
393 [&](int gradient, int deltaT) { delayMonitor(gradient, deltaT); });
394
396 socketPair_->createSRTP(receive_.crypto.getCryptoSuite().c_str(),
398 send_.crypto.getCryptoSuite().c_str(),
399 send_.crypto.getSrtpKeyInfo().c_str());
400 }
401 } catch (const std::runtime_error& e) {
402 JAMI_ERR("[%p] Socket creation failed: %s", this, e.what());
403 return;
404 }
405
406 startReceiver();
407 startSender();
408
409 if (conference_) {
411 setupConferenceVideoPipeline(*conference_, Direction::SEND);
412 }
414 setupConferenceVideoPipeline(*conference_, Direction::RECV);
415 }
416 } else {
417 setupVideoPipeline();
418 }
419}
420
421void
423{
424 std::lock_guard lock(mutex_);
425
426 stopSender(true);
427 stopReceiver(true);
428
429 if (socketPair_)
430 socketPair_->interrupt();
431
432 rtcpCheckerThread_.join();
433
434 // reset default video quality if exist
437
439 storeVideoBitrateInfo();
440
441 socketPair_.reset();
442 videoLocal_.reset();
443}
444
445void
447{
448 std::lock_guard lock(mutex_);
449
450 // Sender
451 if (dir == Direction::SEND) {
452 if (send_.onHold == mute) {
453 JAMI_DBG("[%p] Local already %s", this, mute ? "muted" : "un-muted");
454 return;
455 }
456
457 if ((send_.onHold = mute)) {
458 if (videoLocal_) {
459 auto ms = videoLocal_->getInfo();
460 if (auto ob = recorder_->getStream(ms.name)) {
461 videoLocal_->detach(ob);
462 recorder_->removeStream(ms);
463 }
464 }
465 stopSender();
466 } else {
468 }
469 return;
470 }
471
472 // Receiver
473 if (receive_.onHold == mute) {
474 JAMI_DBG("[%p] Remote already %s", this, mute ? "muted" : "un-muted");
475 return;
476 }
477
478 if ((receive_.onHold = mute)) {
479 if (receiveThread_) {
480 auto ms = receiveThread_->getInfo();
481 if (auto ob = recorder_->getStream(ms.name)) {
482 receiveThread_->detach(ob);
483 recorder_->removeStream(ms);
484 }
485 }
486 stopReceiver();
487 } else {
488 startReceiver();
489 if (conference_ and not receive_.onHold) {
490 setupConferenceVideoPipeline(*conference_, Direction::RECV);
491 }
492 }
493}
494
495void
497{
498 std::lock_guard lock(mutex_);
499#if __ANDROID__
500 if (videoLocal_)
502#else
503 if (sender_)
504 sender_->forceKeyFrame();
505#endif
506}
507
508void
510{
511 rotation_.store(rotation);
512 if (receiveThread_)
513 receiveThread_->setRotation(rotation);
514}
515
516void
517VideoRtpSession::setupVideoPipeline()
518{
519 if (sender_) {
520 if (videoLocal_) {
521 JAMI_DBG("[%p] Setup video pipeline on local capture device", this);
522 videoLocal_->attach(sender_.get());
523 }
524 } else {
525 videoLocal_.reset();
526 }
527}
528
529void
530VideoRtpSession::setupConferenceVideoPipeline(Conference& conference, Direction dir)
531{
532 if (dir == Direction::SEND) {
533 JAMI_DBG("[%p] Setup video sender pipeline on conference %s for call %s",
534 this,
535 conference.getConfId().c_str(),
536 callId_.c_str());
537 videoMixer_ = conference.getVideoMixer();
538 if (sender_) {
539 // Swap sender from local video to conference video mixer
540 if (videoLocal_)
541 videoLocal_->detach(sender_.get());
542 if (videoMixer_)
543 videoMixer_->attach(sender_.get());
544 } else {
545 JAMI_WARN("[%p] no sender", this);
546 }
547 } else {
548 JAMI_DBG("[%p] Setup video receiver pipeline on conference %s for call %s",
549 this,
550 conference.getConfId().c_str(),
551 callId_.c_str());
552 if (receiveThread_) {
553 receiveThread_->stopSink();
554 if (videoMixer_)
555 videoMixer_->attachVideo(receiveThread_.get(), callId_, streamId_);
556 } else {
557 JAMI_WARN("[%p] no receiver", this);
558 }
559 }
560}
561
562void
564{
565 std::lock_guard lock(mutex_);
566
568
569 conference_ = &conference;
570 videoMixer_ = conference.getVideoMixer();
571 JAMI_DBG("[%p] enterConference (conf: %s)", this, conference.getConfId().c_str());
572
573 if (send_.enabled or receiveThread_) {
574 // Restart encoder with conference parameter ON in order to unlink HW encoder
575 // from HW decoder.
577 if (conference_) {
578 setupConferenceVideoPipeline(conference, Direction::RECV);
579 }
580 }
581}
582
583void
585{
586 std::lock_guard lock(mutex_);
587
588 if (!conference_)
589 return;
590
591 JAMI_DBG("[%p] exitConference (conf: %s)", this, conference_->getConfId().c_str());
592
593 if (videoMixer_) {
594 if (sender_)
595 videoMixer_->detach(sender_.get());
596
597 if (receiveThread_) {
598 auto activeStream = videoMixer_->verifyActive(streamId_);
599 videoMixer_->detachVideo(receiveThread_.get());
600 receiveThread_->startSink();
601 if (activeStream)
602 videoMixer_->setActiveStream(streamId_);
603 }
604
605 videoMixer_.reset();
606 }
607
608 conference_ = nullptr;
609}
610
611bool
612VideoRtpSession::check_RCTP_Info_RR(RTCPInfo& rtcpi)
613{
614 auto rtcpInfoVect = socketPair_->getRtcpRR();
615 unsigned totalLost = 0;
616 unsigned totalJitter = 0;
617 unsigned nbDropNotNull = 0;
618 auto vectSize = rtcpInfoVect.size();
619
620 if (vectSize != 0) {
621 for (const auto& it : rtcpInfoVect) {
622 if (it.fraction_lost != 0) // Exclude null drop
624 totalLost += it.fraction_lost;
625 totalJitter += ntohl(it.jitter);
626 }
627 rtcpi.packetLoss = nbDropNotNull ? (float) (100 * totalLost) / (256.0 * nbDropNotNull) : 0;
628 // Jitter is expressed in timestamp unit -> convert to milliseconds
629 // https://stackoverflow.com/questions/51956520/convert-jitter-from-rtp-timestamp-unit-to-millisseconds
630 rtcpi.jitter = (totalJitter / vectSize / 90000.0f) * 1000;
631 rtcpi.nb_sample = vectSize;
632 rtcpi.latency = socketPair_->getLastLatency();
633 return true;
634 }
635 return false;
636}
637
638bool
639VideoRtpSession::check_RCTP_Info_REMB(uint64_t* br)
640{
641 auto rtcpInfoVect = socketPair_->getRtcpREMB();
642
643 if (!rtcpInfoVect.empty()) {
644 auto pkt = rtcpInfoVect.back();
645 auto temp = cc->parseREMB(pkt);
646 *br = (temp >> 10) | ((temp << 6) & 0xff00) | ((temp << 16) & 0x30000);
647 return true;
648 }
649 return false;
650}
651
652void
653VideoRtpSession::adaptQualityAndBitrate()
654{
655 setupVideoBitrateInfo();
656
657 uint64_t br;
658 if (check_RCTP_Info_REMB(&br)) {
659 delayProcessing(br);
660 }
661
662 RTCPInfo rtcpi {};
663 if (check_RCTP_Info_RR(rtcpi)) {
664 dropProcessing(&rtcpi);
665 }
666}
667
668void
669VideoRtpSession::dropProcessing(RTCPInfo* rtcpi)
670{
671 // If bitrate has changed, let time to receive fresh RTCP packets
672 auto now = clock::now();
673 auto restartTimer = now - lastMediaRestart_;
675 return;
676 }
677#ifndef __ANDROID__
678 // Do nothing if jitter is more than 1 second
679 if (rtcpi->jitter > 1000) {
680 return;
681 }
682#endif
683 auto pondLoss = getPonderateLoss(rtcpi->packetLoss);
684 auto oldBitrate = videoBitrateInfo_.videoBitrateCurrent;
686
687 // Fill histoLoss and histoJitter_ with samples
688 if (restartTimer < DELAY_AFTER_RESTART + std::chrono::seconds(1)) {
689 return;
690 } else {
691 // If ponderate drops are inferior to 10% that mean drop are not from congestion but from
692 // network...
693 // ... we can increase
694 if (pondLoss >= 5.0f && rtcpi->packetLoss > 0.0f) {
695 newBitrate *= 1.0f - rtcpi->packetLoss / 150.0f;
696 histoLoss_.clear();
697 lastMediaRestart_ = now;
698 JAMI_DBG(
699 "[BandwidthAdapt] Detected transmission bandwidth overuse, decrease bitrate from "
700 "%u Kbps to %d Kbps, ratio %f (ponderate loss: %f%%, packet loss rate: %f%%)",
703 (float) newBitrate / oldBitrate,
704 pondLoss,
705 rtcpi->packetLoss);
706 }
707 }
708
709 setNewBitrate(newBitrate);
710}
711
712void
713VideoRtpSession::delayProcessing(int br)
714{
715 int newBitrate = videoBitrateInfo_.videoBitrateCurrent;
716 if (br == 0x6803)
717 newBitrate *= 0.85f;
718 else if (br == 0x7378) {
719 auto now = clock::now();
720 auto msSinceLastDecrease = std::chrono::duration_cast<std::chrono::milliseconds>(
721 now - lastBitrateDecrease);
722 auto increaseCoefficient = std::min(msSinceLastDecrease.count() / 600000.0f + 1.0f, 1.05f);
724 } else
725 return;
726
727 setNewBitrate(newBitrate);
728}
729
730void
731VideoRtpSession::setNewBitrate(unsigned int newBR)
732{
733 newBR = std::max(newBR, videoBitrateInfo_.videoBitrateMin);
734 newBR = std::min(newBR, videoBitrateInfo_.videoBitrateMax);
735
736 if (newBR < videoBitrateInfo_.videoBitrateCurrent)
737 lastBitrateDecrease = clock::now();
738
739 if (videoBitrateInfo_.videoBitrateCurrent != newBR) {
740 videoBitrateInfo_.videoBitrateCurrent = newBR;
741 storeVideoBitrateInfo();
742
743#if __ANDROID__
744 if (auto input_device = std::dynamic_pointer_cast<VideoInput>(videoLocal_))
746#endif
747
748 if (sender_) {
749 auto ret = sender_->setBitrate(newBR);
750 if (ret == -1)
751 JAMI_ERR("Fail to access the encoder");
752 else if (ret == 0)
754 } else {
755 JAMI_ERR("Fail to access the sender");
756 }
757 }
758}
759
760void
761VideoRtpSession::setupVideoBitrateInfo()
762{
763 auto codecVideo = std::static_pointer_cast<jami::SystemVideoCodecInfo>(send_.codec);
764 if (codecVideo) {
765 videoBitrateInfo_ = {
766 codecVideo->bitrate,
767 codecVideo->minBitrate,
768 codecVideo->maxBitrate,
769 codecVideo->quality,
770 codecVideo->minQuality,
771 codecVideo->maxQuality,
772 videoBitrateInfo_.cptBitrateChecking,
773 videoBitrateInfo_.maxBitrateChecking,
774 videoBitrateInfo_.packetLostThreshold,
775 };
776 } else {
777 videoBitrateInfo_
778 = {0, 0, 0, 0, 0, 0, 0, MAX_ADAPTATIVE_BITRATE_ITERATION, PACKET_LOSS_THRESHOLD};
779 }
780}
781
782void
783VideoRtpSession::storeVideoBitrateInfo()
784{
785 if (auto codecVideo = std::static_pointer_cast<jami::SystemVideoCodecInfo>(send_.codec)) {
786 codecVideo->bitrate = videoBitrateInfo_.videoBitrateCurrent;
787 codecVideo->quality = videoBitrateInfo_.videoQualityCurrent;
788 }
789}
790
791void
792VideoRtpSession::processRtcpChecker()
793{
794 adaptQualityAndBitrate();
795 socketPair_->waitForRTCP(std::chrono::seconds(rtcp_checking_interval));
796}
797
798void
799VideoRtpSession::attachRemoteRecorder(const MediaStream& ms)
800{
801 std::lock_guard lock(mutex_);
802 if (!recorder_ || !receiveThread_)
803 return;
804 if (auto ob = recorder_->addStream(ms)) {
805 receiveThread_->attach(ob);
806 }
807}
808
809void
810VideoRtpSession::attachLocalRecorder(const MediaStream& ms)
811{
812 std::lock_guard lock(mutex_);
813 if (!recorder_ || !videoLocal_ || !Manager::instance().videoPreferences.getRecordPreview())
814 return;
815 if (auto ob = recorder_->addStream(ms)) {
816 videoLocal_->attach(ob);
817 }
818}
819
820void
822{
823 if (!recorder_)
824 return;
825 if (receiveThread_) {
826 receiveThread_->setRecorderCallback([w=weak_from_this()](const MediaStream& ms) {
827 asio::post(*Manager::instance().ioContext(), [w=std::move(w), ms]() {
828 if (auto shared = w.lock())
829 shared->attachRemoteRecorder(ms);
830 });
831 });
832 }
833 if (videoLocal_ && !send_.onHold) {
834 videoLocal_->setRecorderCallback([w=weak_from_this()](const MediaStream& ms) {
835 asio::post(*Manager::instance().ioContext(), [w=std::move(w), ms]() {
836 if (auto shared = w.lock())
837 shared->attachLocalRecorder(ms);
838 });
839 });
840 }
841}
842
843void
845{
846 if (!recorder_)
847 return;
848 if (receiveThread_) {
849 auto ms = receiveThread_->getInfo();
850 if (auto ob = recorder_->getStream(ms.name)) {
851 receiveThread_->detach(ob);
852 recorder_->removeStream(ms);
853 }
854 }
855 if (videoLocal_) {
856 auto ms = videoLocal_->getInfo();
857 if (auto ob = recorder_->getStream(ms.name)) {
858 videoLocal_->detach(ob);
859 recorder_->removeStream(ms);
860 }
861 }
862}
863
864void
866{
867 changeOrientationCallback_ = std::move(cb);
868 if (sender_)
869 sender_->setChangeOrientationCallback(changeOrientationCallback_);
870}
871
872float
873VideoRtpSession::getPonderateLoss(float lastLoss)
874{
875 float pond = 0.0f, pondLoss = 0.0f, totalPond = 0.0f;
876 constexpr float coefficient_a = -1 / 100.0f;
877 constexpr float coefficient_b = 100.0f;
878
879 auto now = clock::now();
880
881 histoLoss_.emplace_back(now, lastLoss);
882
883 for (auto it = histoLoss_.begin(); it != histoLoss_.end();) {
884 auto delay = std::chrono::duration_cast<std::chrono::milliseconds>(now - it->first);
885
886 // 1ms -> 100%
887 // 2000ms -> 80%
888 if (delay <= EXPIRY_TIME_RTCP) {
889 if (it->second == 0.0f)
890 pond = 20.0f; // Reduce weight of null drop
891 else
892 pond = std::min(delay.count() * coefficient_a + coefficient_b, 100.0f);
893 totalPond += pond;
894 pondLoss += it->second * pond;
895 ++it;
896 } else
897 it = histoLoss_.erase(it);
898 }
899 if (totalPond == 0)
900 return 0.0f;
901
902 return pondLoss / totalPond;
903}
904
905void
906VideoRtpSession::delayMonitor(int gradient, int deltaT)
907{
908 float estimation = cc->kalmanFilter(gradient);
909 float thresh = cc->get_thresh();
910
911 cc->update_thresh(estimation, deltaT);
912
913 BandwidthUsage bwState = cc->get_bw_state(estimation, thresh);
914 auto now = clock::now();
915
917 auto remb_timer_dec = now - last_REMB_dec_;
918 if ((not remb_dec_cnt_) or (remb_timer_dec > DELAY_AFTER_REMB_DEC)) {
919 last_REMB_dec_ = now;
920 remb_dec_cnt_ = 0;
921 }
922
923 // Limit REMB decrease to MAX_REMB_DEC every DELAY_AFTER_REMB_DEC ms
924 if (remb_dec_cnt_ < MAX_REMB_DEC && remb_timer_dec < DELAY_AFTER_REMB_DEC) {
925 remb_dec_cnt_++;
926 JAMI_WARN("[BandwidthAdapt] Detected reception bandwidth overuse");
927 uint8_t* buf = nullptr;
928 uint64_t br = 0x6803; // Decrease 3
929 auto v = cc->createREMB(br);
930 buf = &v[0];
931 socketPair_->writeData(buf, v.size());
932 last_REMB_inc_ = clock::now();
933 }
934 } else if (bwState == BandwidthUsage::bwNormal) {
935 auto remb_timer_inc = now - last_REMB_inc_;
937 uint8_t* buf = nullptr;
938 uint64_t br = 0x7378; // INcrease
939 auto v = cc->createREMB(br);
940 buf = &v[0];
941 socketPair_->writeData(buf, v.size());
942 last_REMB_inc_ = clock::now();
943 }
944 }
945}
946} // namespace video
947} // namespace jami
std::string getSrtpKeyInfo() const
std::string getCryptoSuite() const
static LIBJAMI_TEST_EXPORT Manager & instance()
Definition manager.cpp:676
std::string input_
Definition rtp_session.h:81
std::string getRemoteRtpUri() const
Definition rtp_session.h:88
std::recursive_mutex mutex_
Definition rtp_session.h:76
const std::string callId_
Definition rtp_session.h:77
virtual void updateMedia(const MediaDescription &send, const MediaDescription &receive)
Definition rtp_session.h:54
MediaDescription receive_
Definition rtp_session.h:83
const std::string streamId_
Definition rtp_session.h:78
MediaDescription send_
Definition rtp_session.h:82
std::function< void(MediaType, bool)> onSuccessfulSetup_
Definition rtp_session.h:86
std::unique_ptr< SocketPair > socketPair_
Definition rtp_session.h:80
std::shared_ptr< MediaRecorder > recorder_
Definition rtp_session.h:85
bool isRunning() const noexcept
void setRotation(int rotation)
Set video orientation.
const VideoBitrateInfo & getVideoBitrateInfo()
void setMuted(bool mute, Direction dir=Direction::SEND) override
void enterConference(Conference &conference)
void updateMedia(const MediaDescription &send, const MediaDescription &receive) override
Setup internal VideoBitrateInfo structure from media descriptors.
void setRequestKeyFrameCallback(std::function< void(void)> cb)
void setChangeOrientationCallback(std::function< void(int)> cb)
VideoRtpSession(const std::string &callId, const std::string &streamId, const DeviceParams &localVideoParams, const std::shared_ptr< MediaRecorder > &rec)
void start(std::unique_ptr< dhtnet::IceSocket > rtp_sock, std::unique_ptr< dhtnet::IceSocket > rtcp_sock) override
#define JAMI_ERR(...)
Definition logger.h:218
#define JAMI_DBG(...)
Definition logger.h:216
#define JAMI_WARN(...)
Definition logger.h:217
#define JAMI_LOG(formatstr,...)
Definition logger.h:225
constexpr auto DELAY_AFTER_REMB_DEC
constexpr auto DELAY_AFTER_REMB_INC
constexpr auto DELAY_AFTER_RESTART
constexpr auto EXPIRY_TIME_RTCP
static constexpr unsigned MAX_REMB_DEC
void emitSignal(Args... args)
Definition ring_signal.h:64
static constexpr auto NEWPARAMS_TIMEOUT
@ MEDIA_VIDEO
Definition media_codec.h:48
void string_replace(std::string &str, const std::string &from, const std::string &to)
DeviceParams Parameters used by MediaDecoder and MediaEncoder to open a LibAV device/stream.
std::string format
rational< double > framerate
MediaDescription Negotiated RTP media slot.
dhtnet::IpAddr addr
Endpoint socket address.
dhtnet::IpAddr rtcp_addr
RTCP socket address.
std::shared_ptr< SystemCodecInfo > codec
RTP.
CryptoAttribute crypto
Crypto parameters.
std::string receiving_sdp
static constexpr unsigned DEFAULT_CODEC_QUALITY
Definition media_codec.h:61
static constexpr unsigned DEFAULT_MAX_BITRATE
Definition media_codec.h:73
static constexpr unsigned DEFAULT_NO_QUALITY
Definition media_codec.h:70
static constexpr unsigned DEFAULT_VIDEO_BITRATE
Definition media_codec.h:74