39#include <dhtnet/ice_socket.h>
40#include <asio/post.hpp>
41#include <asio/io_context.hpp>
62 const string& streamId,
64 const std::shared_ptr<MediaRecorder>&
rec)
67 , videoBitrateInfo_ {}
68 , rtcpCheckerThread_([] {
return true; }, [
this] { processRtcpChecker(); }, [] {})
69 , cc(std::make_unique<CongestionControl>())
72 setupVideoBitrateInfo();
73 JAMI_LOG(
"[{:p}] Video RTP session created for call {} (recorder {:p})", fmt::ptr(
this), callId_, fmt::ptr(recorder_));
80 JAMI_LOG(
"[{:p}] Video RTP session destroyed", fmt::ptr(
this));
86 return videoBitrateInfo_;
102 setupVideoBitrateInfo();
108 cbKeyFrameRequest_ = std::move(
cb);
112VideoRtpSession::startSender()
114 std::lock_guard lock(
mutex_);
116 JAMI_DBG(
"[%p] Start video RTP sender: input [%s] - muted [%s]",
119 send_.onHold ?
"YES" :
"NO");
123 JAMI_WARN(
"[%p] Transport not set yet",
this);
130 videoLocal_->detach(sender_.get());
132 videoMixer_->detach(sender_.get());
133 JAMI_WARN(
"[%p] Restarting video sender",
this);
136 if (
not conference_) {
139 videoLocal_->setRecorderCallback(
142 if (
auto shared = w.lock())
143 shared->attachLocalRecorder(ms);
146 auto newParams = videoLocal_->getParams();
152 JAMI_ERR(
"[%p] No valid new video parameters",
this);
155 }
catch (
const std::exception&
e) {
156 JAMI_ERR(
"Exception during retrieving video parameters: %s",
e.what());
164#if (defined(__ANDROID__) || (defined(TARGET_OS_IOS) && TARGET_OS_IOS))
165 videoLocal_->setupSink(localVideoParams_.
width, localVideoParams_.
height);
191 ? MediaStream(
"video sender",
193 1 /
static_cast<rational<int>
>(localVideoParams_.
framerate),
194 localVideoParams_.
width == 0 ? 1080u : localVideoParams_.width,
195 localVideoParams_.height == 0 ? 720u : localVideoParams_.height,
198 : videoMixer_->getStream(
"Video Sender");
199 sender_.reset(
new VideoSender(
201 if (changeOrientationCallback_)
202 sender_->setChangeOrientationCallback(changeOrientationCallback_);
204 socketPair_->setPacketLossCallback([
this]() { cbKeyFrameRequest_(); });
206 }
catch (
const MediaEncoderException&
e) {
210 lastMediaRestart_ = clock::now();
211 last_REMB_inc_ = clock::now();
212 last_REMB_dec_ = clock::now();
214 rtcpCheckerThread_.
start();
216 rtcpCheckerThread_.
join();
226 std::lock_guard lock(
mutex_);
237 setupVideoPipeline();
245 JAMI_DBG(
"[%p] Stop video RTP sender: input [%s] - muted [%s]",
248 send_.onHold ?
"YES" :
"NO");
252 videoLocal_->detach(sender_.get());
254 videoMixer_->detach(sender_.get());
268VideoRtpSession::startReceiver()
272 JAMI_DBG(
"[%p] Starting receiver",
this);
276 JAMI_WARN(
"[%p] Already has a receiver, restarting",
this);
277 receiveThread_.reset(
287 receiveThread_->startLoop();
288 receiveThread_->setRequestKeyFrameCallback([
this]() { cbKeyFrameRequest_(); });
289 receiveThread_->setRotation(rotation_.load());
290 if (videoMixer_
and conference_) {
299 receiveThread_->setRecorderCallback([w=
weak_from_this()](
const MediaStream& ms) {
301 if (
auto shared = w.lock())
302 shared->attachRemoteRecorder(ms);
306 JAMI_DBG(
"[%p] Video receiver disabled",
this);
307 if (receiveThread_
and videoMixer_
and conference_) {
313 receiveThread_->detach(videoMixer_.get());
315 videoMixer_->setActiveStream(
audioId_);
327 JAMI_DBG(
"[%p] Stopping receiver",
this);
329 if (
not receiveThread_)
337 receiveThread_->detach(videoMixer_.get());
339 videoMixer_->setActiveStream(
audioId);
353 auto ms = receiveThread_->getInfo();
355 receiveThread_->detach(
ob);
360 receiveThread_->stopLoop();
361 receiveThread_->stopSink();
367 std::lock_guard lock(
mutex_);
389 last_REMB_inc_ = clock::now();
390 last_REMB_dec_ = clock::now();
401 }
catch (
const std::runtime_error&
e) {
402 JAMI_ERR(
"[%p] Socket creation failed: %s",
this,
e.what());
417 setupVideoPipeline();
424 std::lock_guard lock(
mutex_);
432 rtcpCheckerThread_.
join();
439 storeVideoBitrateInfo();
448 std::lock_guard lock(
mutex_);
453 JAMI_DBG(
"[%p] Local already %s",
this,
mute ?
"muted" :
"un-muted");
459 auto ms = videoLocal_->getInfo();
461 videoLocal_->detach(
ob);
474 JAMI_DBG(
"[%p] Remote already %s",
this,
mute ?
"muted" :
"un-muted");
479 if (receiveThread_) {
480 auto ms = receiveThread_->getInfo();
482 receiveThread_->detach(
ob);
498 std::lock_guard lock(
mutex_);
504 sender_->forceKeyFrame();
511 rotation_.store(rotation);
513 receiveThread_->setRotation(rotation);
517VideoRtpSession::setupVideoPipeline()
521 JAMI_DBG(
"[%p] Setup video pipeline on local capture device",
this);
522 videoLocal_->attach(sender_.get());
533 JAMI_DBG(
"[%p] Setup video sender pipeline on conference %s for call %s",
541 videoLocal_->detach(sender_.get());
543 videoMixer_->attach(sender_.get());
548 JAMI_DBG(
"[%p] Setup video receiver pipeline on conference %s for call %s",
552 if (receiveThread_) {
553 receiveThread_->stopSink();
565 std::lock_guard lock(
mutex_);
586 std::lock_guard lock(
mutex_);
591 JAMI_DBG(
"[%p] exitConference (conf: %s)",
this, conference_->getConfId().c_str());
595 videoMixer_->detach(sender_.get());
597 if (receiveThread_) {
599 videoMixer_->detachVideo(receiveThread_.get());
600 receiveThread_->startSink();
608 conference_ =
nullptr;
622 if (
it.fraction_lost != 0)
639VideoRtpSession::check_RCTP_Info_REMB(
uint64_t*
br)
645 auto temp = cc->parseREMB(
pkt);
646 *
br = (
temp >> 10) | ((
temp << 6) & 0xff00) | ((
temp << 16) & 0x30000);
653VideoRtpSession::adaptQualityAndBitrate()
655 setupVideoBitrateInfo();
658 if (check_RCTP_Info_REMB(&
br)) {
663 if (check_RCTP_Info_RR(
rtcpi)) {
664 dropProcessing(&
rtcpi);
669VideoRtpSession::dropProcessing(RTCPInfo*
rtcpi)
672 auto now = clock::now();
679 if (
rtcpi->jitter > 1000) {
697 lastMediaRestart_ =
now;
699 "[BandwidthAdapt] Detected transmission bandwidth overuse, decrease bitrate from "
700 "%u Kbps to %d Kbps, ratio %f (ponderate loss: %f%%, packet loss rate: %f%%)",
713VideoRtpSession::delayProcessing(
int br)
718 else if (
br == 0x7378) {
719 auto now = clock::now();
721 now - lastBitrateDecrease);
731VideoRtpSession::setNewBitrate(
unsigned int newBR)
737 lastBitrateDecrease = clock::now();
741 storeVideoBitrateInfo();
744 if (
auto input_device = std::dynamic_pointer_cast<VideoInput>(videoLocal_))
749 auto ret = sender_->setBitrate(
newBR);
751 JAMI_ERR(
"Fail to access the encoder");
755 JAMI_ERR(
"Fail to access the sender");
761VideoRtpSession::setupVideoBitrateInfo()
765 videoBitrateInfo_ = {
778 = {0, 0, 0, 0, 0, 0, 0, MAX_ADAPTATIVE_BITRATE_ITERATION, PACKET_LOSS_THRESHOLD};
783VideoRtpSession::storeVideoBitrateInfo()
792VideoRtpSession::processRtcpChecker()
794 adaptQualityAndBitrate();
795 socketPair_->waitForRTCP(std::chrono::seconds(rtcp_checking_interval));
799VideoRtpSession::attachRemoteRecorder(
const MediaStream& ms)
801 std::lock_guard lock(
mutex_);
805 receiveThread_->attach(
ob);
810VideoRtpSession::attachLocalRecorder(
const MediaStream& ms)
812 std::lock_guard lock(
mutex_);
816 videoLocal_->attach(
ob);
825 if (receiveThread_) {
828 if (
auto shared = w.lock())
829 shared->attachRemoteRecorder(ms);
836 if (
auto shared = w.lock())
837 shared->attachLocalRecorder(ms);
848 if (receiveThread_) {
849 auto ms = receiveThread_->getInfo();
851 receiveThread_->detach(
ob);
856 auto ms = videoLocal_->getInfo();
858 videoLocal_->detach(
ob);
867 changeOrientationCallback_ = std::move(
cb);
869 sender_->setChangeOrientationCallback(changeOrientationCallback_);
873VideoRtpSession::getPonderateLoss(
float lastLoss)
879 auto now = clock::now();
883 for (
auto it = histoLoss_.begin();
it != histoLoss_.end();) {
884 auto delay = std::chrono::duration_cast<std::chrono::milliseconds>(
now -
it->first);
889 if (
it->second == 0.0f)
897 it = histoLoss_.erase(
it);
909 float thresh = cc->get_thresh();
914 auto now = clock::now();
919 last_REMB_dec_ =
now;
926 JAMI_WARN(
"[BandwidthAdapt] Detected reception bandwidth overuse");
929 auto v = cc->createREMB(
br);
932 last_REMB_inc_ = clock::now();
939 auto v = cc->createREMB(
br);
942 last_REMB_inc_ = clock::now();
std::string getSrtpKeyInfo() const
std::string getCryptoSuite() const
static LIBJAMI_TEST_EXPORT Manager & instance()
std::string getRemoteRtpUri() const
std::recursive_mutex mutex_
const std::string callId_
virtual void updateMedia(const MediaDescription &send, const MediaDescription &receive)
MediaDescription receive_
const std::string streamId_
std::function< void(MediaType, bool)> onSuccessfulSetup_
std::unique_ptr< SocketPair > socketPair_
std::shared_ptr< MediaRecorder > recorder_
bool isRunning() const noexcept
void setRotation(int rotation)
Set video orientation.
void initRecorder() override
const VideoBitrateInfo & getVideoBitrateInfo()
void deinitRecorder() override
void setMuted(bool mute, Direction dir=Direction::SEND) override
void enterConference(Conference &conference)
void updateMedia(const MediaDescription &send, const MediaDescription &receive) override
Setup internal VideoBitrateInfo structure from media descriptors.
void setRequestKeyFrameCallback(std::function< void(void)> cb)
void restartSender() override
void setChangeOrientationCallback(std::function< void(int)> cb)
VideoRtpSession(const std::string &callId, const std::string &streamId, const DeviceParams &localVideoParams, const std::shared_ptr< MediaRecorder > &rec)
void start(std::unique_ptr< dhtnet::IceSocket > rtp_sock, std::unique_ptr< dhtnet::IceSocket > rtcp_sock) override
#define JAMI_LOG(formatstr,...)
constexpr auto DELAY_AFTER_REMB_DEC
constexpr auto DELAY_AFTER_REMB_INC
constexpr auto DELAY_AFTER_RESTART
constexpr auto EXPIRY_TIME_RTCP
static constexpr unsigned MAX_REMB_DEC
void emitSignal(Args... args)
static constexpr auto NEWPARAMS_TIMEOUT
void string_replace(std::string &str, const std::string &from, const std::string &to)
Specific VoIPLink for SIP (SIP core for incoming and outgoing events).
DeviceParams Parameters used by MediaDecoder and MediaEncoder to open a LibAV device/stream.
rational< double > framerate
static constexpr unsigned DEFAULT_CODEC_QUALITY
static constexpr unsigned DEFAULT_MAX_BITRATE
static constexpr unsigned DEFAULT_NO_QUALITY
static constexpr unsigned DEFAULT_VIDEO_BITRATE
unsigned cptBitrateChecking
float packetLostThreshold
unsigned videoBitrateCurrent
unsigned maxBitrateChecking
unsigned videoQualityCurrent