audio_processor.h

Go to the documentation of this file.

/*
 *  Copyright (C) 2004-2025 Savoir-faire Linux Inc.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program. If not, see <https://www.gnu.org/licenses/>.
 */
#pragma once
 
#include "noncopyable.h"
#include "media/audio/audio_frame_resizer.h"
#include "media/audio/resampler.h"
#include "media/audio/audio_format.h"
#include "media/libav_deps.h"
#include "logger.h"
 
#include <atomic>
#include <memory>
 
namespace jami {
 
class AudioProcessor
{
private:
    NON_COPYABLE(AudioProcessor);
 
public:
    AudioProcessor(AudioFormat format, unsigned frameSize)
        : playbackQueue_(format, (int) frameSize)
        , recordQueue_(format, (int) frameSize)
        , resampler_(new Resampler)
        , format_(format)
        , frameSize_(frameSize)
        , frameDurationMs_((unsigned int) (frameSize_ * (1.0 / format_.sample_rate) * 1000))
    {}
    virtual ~AudioProcessor() = default;
 
    virtual void putRecorded(std::shared_ptr<AudioFrame>&& buf)
    {
        recordStarted_ = true;
        if (!playbackStarted_)
            return;
        enqueue(recordQueue_, std::move(buf));
    };
    virtual void putPlayback(const std::shared_ptr<AudioFrame>& buf)
    {
        playbackStarted_ = true;
        if (!recordStarted_)
            return;
        auto copy = buf;
        enqueue(playbackQueue_, std::move(copy));
    };
 
    virtual std::shared_ptr<AudioFrame> getProcessed() = 0;
 
    virtual void enableEchoCancel(bool enabled) = 0;
 
    virtual void enableNoiseSuppression(bool enabled) = 0;
 
    virtual void enableAutomaticGainControl(bool enabled) = 0;
 
    virtual void enableVoiceActivityDetection(bool enabled) = 0;
 
protected:
    AudioFrameResizer playbackQueue_;
    AudioFrameResizer recordQueue_;
    std::unique_ptr<Resampler> resampler_;
    std::atomic_bool playbackStarted_;
    std::atomic_bool recordStarted_;
    AudioFormat format_;
    unsigned int frameSize_;
    unsigned int frameDurationMs_;
 
    // artificially extend voice activity by this long
    unsigned int forceMinimumVoiceActivityMs {1000};
 
    // current number of frames to force the voice activity to be true
    unsigned int forceVoiceActiveFramesLeft {0};
 
    // voice activity must be active for this long _before_ it is considered legitimate
    unsigned int minimumConsequtiveDurationMs {200};
 
    // current number of frames that the voice activity has been true
    unsigned int consecutiveActiveFrames {0};
 
    bool tidyQueues()
    {
        auto recordFrameSize = recordQueue_.frameSize();
        auto playbackFrameSize = playbackQueue_.frameSize();
        while (recordQueue_.samples() > recordFrameSize * 10
            && 2 * playbackQueue_.samples() * recordFrameSize < recordQueue_.samples() * playbackFrameSize) {
            JAMI_LOG("record overflow {:d} / {:d} - playback: {:d}", recordQueue_.samples(), frameSize_, playbackQueue_.samples());
            recordQueue_.dequeue();
        }
        while (playbackQueue_.samples() > playbackFrameSize * 10
            && 2 * recordQueue_.samples() * playbackFrameSize < playbackQueue_.samples() * recordFrameSize) {
            JAMI_LOG("playback overflow {:d} / {:d} - record: {:d}", playbackQueue_.samples(), frameSize_, recordQueue_.samples());
            playbackQueue_.dequeue();
        }
        if (recordQueue_.samples() < recordFrameSize
            || playbackQueue_.samples() < playbackFrameSize) {
            // If there are not enough samples in either queue, we are unable to
            // process anything.
            return true;
        }
        return false;
    }
 
    bool getStabilizedVoiceActivity(bool voiceStatus)
    {
        bool newVoice = false;
 
        if (voiceStatus) {
            // we detected activity
            consecutiveActiveFrames += 1;
 
            // make sure that we have been active for necessary time
            if (consecutiveActiveFrames > minimumConsequtiveDurationMs / frameDurationMs_) {
                newVoice = true;
 
                // set number of frames that will be forced positive
                forceVoiceActiveFramesLeft = (int) forceMinimumVoiceActivityMs / frameDurationMs_;
            }
        } else if (forceVoiceActiveFramesLeft > 0) {
            // if we didn't detect voice, but we haven't elapsed the minimum duration,
            // force voice to be true
            newVoice = true;
            forceVoiceActiveFramesLeft -= 1;
 
            consecutiveActiveFrames += 1;
        } else {
            // else no voice and no need to force
            newVoice = false;
            consecutiveActiveFrames = 0;
        }
 
        return newVoice;
    }
 
private:
    void enqueue(AudioFrameResizer& frameResizer, std::shared_ptr<AudioFrame>&& buf)
    {
        if (buf->getFormat() != format_) {
            frameResizer.enqueue(resampler_->resample(std::move(buf), format_));
        } else
            frameResizer.enqueue(std::move(buf));
    };
};
 
} // namespace jami