audio_input.cpp

Go to the documentation of this file.

/*
 *  Copyright (C) 2004-2026 Savoir-faire Linux Inc.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program. If not, see <https://www.gnu.org/licenses/>.
 */
 
#include "audio_frame_resizer.h"
#include "audio_input.h"
#include "jami/media_const.h"
#include "manager.h"
#include "media_decoder.h"
#include "resampler.h"
#include "logger.h"
#include "ringbufferpool.h"
#include "tracepoint.h"
#include "video/video_device.h"
 
#include <future>
#include <memory>
 
namespace jami {
 
static constexpr auto MS_PER_PACKET = std::chrono::milliseconds(20);
 
AudioInput::AudioInput(const std::string& id)
    : id_(id)
    , format_(Manager::instance().getRingBufferPool().getInternalAudioFormat())
    , frameSize_(static_cast<int>(format_.sample_rate * MS_PER_PACKET.count()) / 1000)
    , resampler_(new Resampler)
    , resizer_(new AudioFrameResizer(format_,
                                     frameSize_,
                                     [this](std::shared_ptr<AudioFrame>&& f) { frameResized(std::move(f)); }))
    , deviceGuard_()
    , loop_([] { return true; }, [this] { process(); }, [] {})
{
    JAMI_DEBUG("Creating audio input with id: {}", id_);
    ringBuf_ = Manager::instance().getRingBufferPool().createRingBuffer(id_);
}
 
AudioInput::AudioInput(const std::string& id, const std::string& resource)
    : AudioInput(id)
{
    switchInput(resource);
}
 
AudioInput::~AudioInput()
{
    if (playingFile_) {
        Manager::instance().getRingBufferPool().unBindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
        Manager::instance().getRingBufferPool().unBindHalfDuplexOut(id_, id_);
    }
    ringBuf_.reset();
    loop_.join();
 
    Manager::instance().getRingBufferPool().flush(id_);
}
 
void
AudioInput::process()
{
    readFromDevice();
}
 
void
AudioInput::updateStartTime(int64_t start)
{
    if (decoder_) {
        decoder_->updateStartTime(start);
    }
}
 
void
AudioInput::frameResized(std::shared_ptr<AudioFrame>&& ptr)
{
    std::shared_ptr<AudioFrame> frame = std::move(ptr);
    frame->pointer()->pts = static_cast<int64_t>(sent_samples);
    sent_samples += frame->pointer()->nb_samples;
 
    notify(std::static_pointer_cast<MediaFrame>(std::move(frame)));
}
 
void
AudioInput::setSeekTime(int64_t time)
{
    if (decoder_) {
        decoder_->setSeekTime(time);
    }
}
 
void
AudioInput::readFromDevice()
{
    {
        std::lock_guard lk(resourceMutex_);
        if (decodingFile_)
            while (ringBuf_ && ringBuf_->isEmpty())
                readFromFile();
        if (playingFile_) {
            while (ringBuf_ && ringBuf_->getLength(id_) == 0)
                readFromQueue();
        }
    }
 
    auto& bufferPool = Manager::instance().getRingBufferPool();
    if (not bufferPool.waitForDataAvailable(id_, wakeUp_))
        std::this_thread::sleep_until(wakeUp_);
    wakeUp_ += MS_PER_PACKET;
 
    auto audioFrame = bufferPool.getData(id_);
    if (not audioFrame)
        return;
 
    if (muteState_) {
        libav_utils::fillWithSilence(audioFrame->pointer());
        audioFrame->has_voice = false; // force no voice activity when muted
    }
 
    std::lock_guard lk(fmtMutex_);
    if (bufferPool.getInternalAudioFormat() != format_)
        audioFrame = resampler_->resample(std::move(audioFrame), format_);
    resizer_->enqueue(std::move(audioFrame));
 
    if (recorderCallback_ && settingMS_.exchange(false)) {
        recorderCallback_(MediaStream("a:local", format_, static_cast<int64_t>(sent_samples)));
    }
 
    jami_tracepoint(audio_input_read_from_device_end, id_.c_str());
}
 
void
AudioInput::readFromQueue()
{
    if (!decoder_)
        return;
    if (paused_ || !decoder_->emitFrame(true)) {
        std::this_thread::sleep_for(MS_PER_PACKET);
    }
}
 
void
AudioInput::readFromFile()
{
    if (!decoder_)
        return;
    const auto ret = decoder_->decode();
    switch (ret) {
    case MediaDemuxer::Status::Success:
        break;
    case MediaDemuxer::Status::EndOfFile:
        createDecoder();
        break;
    case MediaDemuxer::Status::ReadError:
        JAMI_ERR() << "Failed to decode frame";
        break;
    case MediaDemuxer::Status::ReadBufferOverflow:
        JAMI_ERR() << "Read buffer overflow detected";
        break;
    case MediaDemuxer::Status::FallBack:
    case MediaDemuxer::Status::RestartRequired:
        break;
    }
}
 
bool
AudioInput::initCapture(const std::string& device)
{
    std::string targetId = device;
#if defined(_WIN32)
    // There are two possible formats for device:
    // 1. A string containing "window-id:hwnd=XXXX" where XXXX is the HWND of the window to capture
    // 2. A string that does not contain a window handle, in which case we capture desktop audio
    std::string pattern = "window-id:hwnd=";
    size_t winHandlePos = device.find(pattern);
 
    if (winHandlePos != std::string::npos) {
        // Get HWND from device URI
        size_t startPos = winHandlePos + pattern.size();
        size_t endPos = device.find(' ', startPos);
        if (endPos == std::string::npos) {
            endPos = device.size();
        }
        targetId = device.substr(startPos, endPos - startPos);
    } else {
        targetId = video::DEVICE_DESKTOP;
    }
#elif defined(__linux__)
    // On Linux, we always capture desktop audio because window-specific audio capture is not yet implemented
    // Possible to implement window audio capture on X11 specifically in the future, but not Wayland as of Jan 2026
    // See https://github.com/flatpak/xdg-desktop-portal/issues/957
    targetId = video::DEVICE_DESKTOP;
#elif defined(__APPLE__)
    // As of Jan 2026, audio capture has not been implemented for macOS (TODO)
    targetId = video::DEVICE_DESKTOP;
#endif
 
    devOpts_ = {};
    devOpts_.input = targetId;
    devOpts_.channel = format_.nb_channels;
    devOpts_.framerate = format_.sample_rate;
 
    // This will cause the audio layer to create a ring buffer with id=targetId
    // The audio layer will then fill it with the audio from the captured window/desktop
    deviceGuard_ = Manager::instance().startCaptureStream(targetId);
    if (!deviceGuard_) {
        if (!targetId.empty())
            JAMI_ERROR("Failed to start capture stream for window-id: {}", targetId);
        else
            JAMI_ERROR("Failed to start capture stream for desktop audio");
        return false;
    }
 
    // We want the audio input's ring buffer to read the captured audio from the audio layer
    // Then the audio RTP session will handle sending the audio over the network
    Manager::instance().getRingBufferPool().bindHalfDuplexOut(id_, targetId);
 
    sourceRingBufferId_ = targetId;
    playingDevice_ = true;
    return true;
}
 
bool
AudioInput::initDevice(const std::string& device)
{
    devOpts_ = {};
    devOpts_.input = device;
    devOpts_.channel = format_.nb_channels;
    devOpts_.framerate = format_.sample_rate;
    deviceGuard_ = Manager::instance().startAudioStream(AudioDeviceType::CAPTURE);
    playingDevice_ = true;
    return true;
}
 
void
AudioInput::configureFilePlayback(const std::string& path, std::shared_ptr<MediaDemuxer>& demuxer, int index)
{
    decoder_.reset();
    devOpts_ = {};
    devOpts_.input = path;
    devOpts_.name = path;
    auto decoder = std::make_unique<MediaDecoder>(demuxer, index, [this](std::shared_ptr<MediaFrame>&& frame) {
        if (muteState_)
            libav_utils::fillWithSilence(frame->pointer());
        if (ringBuf_)
            ringBuf_->put(std::static_pointer_cast<AudioFrame>(frame));
    });
    decoder->emulateRate();
    decoder->setInterruptCallback([](void* data) -> int { return not static_cast<AudioInput*>(data)->isCapturing(); },
                                  this);
 
    // have file audio mixed into the local buffer so it gets played
    Manager::instance().getRingBufferPool().bindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
    // Bind to itself to be able to read from the ringbuffer
    Manager::instance().getRingBufferPool().bindHalfDuplexOut(id_, id_);
 
    sourceRingBufferId_ = id_;
    deviceGuard_ = Manager::instance().startAudioStream(AudioDeviceType::PLAYBACK);
 
    wakeUp_ = std::chrono::steady_clock::now() + MS_PER_PACKET;
    playingFile_ = true;
    decoder_ = std::move(decoder);
    resource_ = path;
    loop_.start();
}
 
void
AudioInput::setPaused(bool paused)
{
    if (paused) {
        Manager::instance().getRingBufferPool().unBindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
        deviceGuard_.reset();
    } else {
        Manager::instance().getRingBufferPool().bindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
        deviceGuard_ = Manager::instance().startAudioStream(AudioDeviceType::PLAYBACK);
    }
    paused_ = paused;
}
 
void
AudioInput::flushBuffers()
{
    if (decoder_) {
        decoder_->flushBuffers();
    }
}
 
bool
AudioInput::initFile(const std::string& path)
{
    if (access(path.c_str(), R_OK) != 0) {
        JAMI_ERROR("File '{}' not available", path);
        return false;
    }
 
    devOpts_ = {};
    devOpts_.input = path;
    devOpts_.name = path;
    devOpts_.loop = "1";
    // sets devOpts_'s sample rate and number of channels
    if (!createDecoder()) {
        JAMI_WARN() << "Unable to decode audio from file, switching back to default device";
        return initDevice("");
    }
    wakeUp_ = std::chrono::steady_clock::now() + MS_PER_PACKET;
 
    // have file audio mixed into the local buffer so it gets played
    Manager::instance().getRingBufferPool().bindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
    sourceRingBufferId_ = id_;
    decodingFile_ = true;
    deviceGuard_ = Manager::instance().startAudioStream(AudioDeviceType::PLAYBACK);
    return true;
}
 
std::shared_future<DeviceParams>
AudioInput::switchInput(const std::string& resource)
{
    // Always switch inputs, even if it's the same resource, so audio will be in sync with video
    std::unique_lock lk(resourceMutex_);
 
    JAMI_DEBUG("Switching audio source from [{}] to [{}]", resource_, resource);
 
    auto oldGuard = std::move(deviceGuard_);
 
    decoder_.reset();
    if (decodingFile_) {
        decodingFile_ = false;
        Manager::instance().getRingBufferPool().unBindHalfDuplexOut(RingBufferPool::DEFAULT_ID, id_);
    }
 
    playingDevice_ = false;
    resource_ = resource;
    sourceRingBufferId_.clear();
    devOptsFound_ = false;
 
    std::promise<DeviceParams> p;
    foundDevOpts_.swap(p);
 
    if (resource_.empty()) {
        if (initDevice(""))
            foundDevOpts(devOpts_);
    } else {
        static const std::string& sep = libjami::Media::VideoProtocolPrefix::SEPARATOR;
        const auto pos = resource_.find(sep);
        if (pos == std::string::npos)
            return {};
 
        const auto prefix = resource_.substr(0, pos);
        if ((pos + sep.size()) >= resource_.size())
            return {};
 
        const auto suffix = resource_.substr(pos + sep.size());
 
        bool ready = false;
        if (prefix == libjami::Media::VideoProtocolPrefix::FILE)
            ready = initFile(suffix);
        else if (prefix == libjami::Media::VideoProtocolPrefix::DISPLAY)
            ready = initCapture(suffix);
        else
            ready = initDevice(suffix);
 
        if (ready)
            foundDevOpts(devOpts_);
    }
 
    futureDevOpts_ = foundDevOpts_.get_future().share();
    wakeUp_ = std::chrono::steady_clock::now() + MS_PER_PACKET;
    lk.unlock();
    if (not loop_.isRunning())
        loop_.start();
    if (onSuccessfulSetup_)
        onSuccessfulSetup_(MEDIA_AUDIO, 0);
    return futureDevOpts_;
}
 
void
AudioInput::foundDevOpts(const DeviceParams& params)
{
    if (!devOptsFound_) {
        devOptsFound_ = true;
        foundDevOpts_.set_value(params);
    }
}
 
void
AudioInput::setRecorderCallback(const std::function<void(const MediaStream& ms)>& cb)
{
    settingMS_.exchange(true);
    recorderCallback_ = cb;
    if (decoder_)
        decoder_->setContextCallback([this]() {
            if (recorderCallback_)
                recorderCallback_(getInfo());
        });
}
 
bool
AudioInput::createDecoder()
{
    decoder_.reset();
    if (devOpts_.input.empty()) {
        foundDevOpts(devOpts_);
        return false;
    }
 
    auto decoder = std::make_unique<MediaDecoder>([this](std::shared_ptr<MediaFrame>&& frame) {
        if (ringBuf_)
            ringBuf_->put(std::static_pointer_cast<AudioFrame>(frame));
    });
 
    // NOTE don't emulate rate, file is read as frames are needed
 
    decoder->setInterruptCallback([](void* data) -> int { return not static_cast<AudioInput*>(data)->isCapturing(); },
                                  this);
 
    if (decoder->openInput(devOpts_) < 0) {
        JAMI_ERR() << "Unable to open input '" << devOpts_.input << "'";
        foundDevOpts(devOpts_);
        return false;
    }
 
    if (decoder->setupAudio() < 0) {
        JAMI_ERR() << "Unable to setup decoder for '" << devOpts_.input << "'";
        foundDevOpts(devOpts_);
        return false;
    }
 
    auto ms = decoder->getStream(devOpts_.input);
    devOpts_.channel = ms.nbChannels;
    devOpts_.framerate = ms.sampleRate;
    JAMI_DBG() << "Created audio decoder: " << ms;
 
    decoder_ = std::move(decoder);
    foundDevOpts(devOpts_);
    decoder_->setContextCallback([this]() {
        if (recorderCallback_)
            recorderCallback_(getInfo());
    });
    return true;
}
 
void
AudioInput::setFormat(const AudioFormat& fmt)
{
    std::lock_guard lk(fmtMutex_);
    format_ = fmt;
    resizer_->setFormat(format_, static_cast<int>(format_.sample_rate * MS_PER_PACKET.count()) / 1000);
}
 
void
AudioInput::setMuted(bool isMuted)
{
    JAMI_WARN("Audio Input muted [%s]", isMuted ? "YES" : "NO");
    muteState_ = isMuted;
}
 
MediaStream
AudioInput::getInfo() const
{
    std::lock_guard lk(fmtMutex_);
    return MediaStream("a:local", format_, static_cast<int64_t>(sent_samples));
}
 
MediaStream
AudioInput::getInfo(const std::string& name) const
{
    std::lock_guard lk(fmtMutex_);
    auto ms = MediaStream(name, format_, static_cast<int64_t>(sent_samples));
    return ms;
}
 
} // namespace jami