From f1fc9df1527237c0449585bcaa620851583a271e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20Smoczy=C5=84ski?= Date: Tue, 12 Jan 2021 13:31:39 +0100 Subject: [PATCH] [EGD-4977] Reduce audio lag during voice call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reduce audio delay by reducing audio buffer size in router operation. Audio streams are now created directly in the operations, not in the audio service, which gives more flexibility. Audio Buffer size is calculated based on endpoints (source, sink) and operation capabilities. This commit also enables allocations in a non-cacheable region of OCRAM for endpoints that use DMA for data transport. Introduce power-of-two operations that use built-in functions and possibly dedicated hardware instructions of an MCU. These operations are required by the audio stream buffer size calculation algorithm. Signed-off-by: Marcin Smoczyński --- module-audio/Audio/Audio.cpp | 4 +- module-audio/Audio/Audio.hpp | 28 +++------- module-audio/Audio/BluetoothProxyAudio.cpp | 4 +- module-audio/Audio/Endpoint.cpp | 11 +++- module-audio/Audio/Endpoint.hpp | 18 +++++- module-audio/Audio/Operation/Operation.cpp | 13 ++--- module-audio/Audio/Operation/Operation.hpp | 29 +++------- .../Audio/Operation/PlaybackOperation.cpp | 14 ++++- .../Audio/Operation/PlaybackOperation.hpp | 10 +++- .../Audio/Operation/RecorderOperation.cpp | 10 ++-- .../Audio/Operation/RouterOperation.cpp | 18 ++++-- .../Audio/Operation/RouterOperation.hpp | 9 ++- module-audio/Audio/StreamFactory.cpp | 55 +++++++++++++++++++ module-audio/Audio/StreamFactory.hpp | 29 ++++++++++ module-audio/Audio/decoder/Decoder.cpp | 5 +- module-audio/Audio/decoder/Decoder.hpp | 5 +- module-audio/CMakeLists.txt | 1 + .../rt1051/bsp/audio/RT1051Audiocodec.cpp | 6 +- .../rt1051/bsp/audio/RT1051Audiocodec.hpp | 4 +- .../rt1051/bsp/audio/RT1051CellularAudio.cpp | 7 +-- .../rt1051/bsp/audio/RT1051CellularAudio.hpp | 4 +- .../board/rt1051/bsp/audio/SAIAudioDevice.cpp | 7 +-- .../board/rt1051/bsp/audio/SAIAudioDevice.hpp | 7 +-- module-bsp/bsp/audio/bsp_audio.cpp | 32 +++-------- module-bsp/bsp/audio/bsp_audio.hpp | 40 ++------------ module-bsp/targets/Target_Linux.cmake | 3 - module-utils/math/Math.hpp | 40 +++++++++++++- module-utils/test/test_math.cpp | 40 +++++++++++++- 28 files changed, 290 insertions(+), 163 deletions(-) create mode 100644 module-audio/Audio/StreamFactory.cpp create mode 100644 module-audio/Audio/StreamFactory.hpp diff --git a/module-audio/Audio/Audio.cpp b/module-audio/Audio/Audio.cpp index 887c3471dcea1874a98b8d80a1ba232e5bb014fd..7acc778e597f54b0573d83a47a5f6a4d51547f4f 100644 --- a/module-audio/Audio/Audio.cpp +++ b/module-audio/Audio/Audio.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020, Mudita Sp. z.o.o. All rights reserved. +// Copyright (c) 2017-2021, Mudita Sp. z.o.o. All rights reserved. // For licensing, see https://github.com/mudita/MuditaOS/LICENSE.md #include "Audio.hpp" @@ -90,8 +90,6 @@ namespace audio break; } currentOperation = std::move(ret); - currentOperation->SetDataStreams(&dataStreamOut, &dataStreamIn); - UpdateProfiles(); } catch (const AudioInitException &audioException) { diff --git a/module-audio/Audio/Audio.hpp b/module-audio/Audio/Audio.hpp index 2a42fb9a426a5d8ac9759232d92d263ba356990c..a110c6ab03f683e0bf3c6adc676baeb2bf9f4f6d 100644 --- a/module-audio/Audio/Audio.hpp +++ b/module-audio/Audio/Audio.hpp @@ -1,19 +1,18 @@ -// Copyright (c) 2017-2020, Mudita Sp. z.o.o. All rights reserved. +// Copyright (c) 2017-2021, Mudita Sp. z.o.o. All rights reserved. // For licensing, see https://github.com/mudita/MuditaOS/LICENSE.md #pragma once -#include -#include -#include -#include - #include #include "AudioCommon.hpp" -#include "Stream.hpp" -#include "Operation/Operation.hpp" #include "decoder/Decoder.hpp" +#include "Operation/Operation.hpp" + +#include +#include +#include +#include namespace audio { @@ -96,17 +95,14 @@ namespace audio const audio::PlaybackType &playbackType = audio::PlaybackType::None); virtual audio::RetCode Start(); - virtual audio::RetCode Stop(); - virtual audio::RetCode Pause(); - virtual audio::RetCode Resume(); - virtual audio::RetCode Mute(); private: void UpdateProfiles(); + AudioSinkState audioSinkState; std::shared_ptr btData; @@ -115,14 +111,6 @@ namespace audio std::unique_ptr currentOperation; AudioServiceMessage::Callback serviceCallback; - - // for efficiency multiple of 24 and 32 (max audio samples size) - static constexpr auto defaultAudioStreamBlockSize = 2048; - StandardStreamAllocator allocatorOut; - Stream dataStreamOut{allocatorOut, defaultAudioStreamBlockSize}; - - StandardStreamAllocator allocatorIn; - Stream dataStreamIn{allocatorIn, defaultAudioStreamBlockSize}; }; } // namespace audio diff --git a/module-audio/Audio/BluetoothProxyAudio.cpp b/module-audio/Audio/BluetoothProxyAudio.cpp index a7084eddfde7aceb5be97d9c987215d780d03c4e..f7e68d81ecf437f87c39665dca218c5e3221155d 100644 --- a/module-audio/Audio/BluetoothProxyAudio.cpp +++ b/module-audio/Audio/BluetoothProxyAudio.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020, Mudita Sp. z.o.o. All rights reserved. +// Copyright (c) 2017-2021, Mudita Sp. z.o.o. All rights reserved. // For licensing, see https://github.com/mudita/MuditaOS/LICENSE.md #include "BluetoothProxyAudio.hpp" @@ -10,7 +10,7 @@ namespace bsp audio::Stream &dataStreamOut, audio::Stream &dataStreamIn, AudioDevice::Format &format) - : AudioDevice(nullptr), dataStreamOut(dataStreamOut), dataStreamIn(dataStreamIn), serviceCallback(callback), + : dataStreamOut(dataStreamOut), dataStreamIn(dataStreamIn), serviceCallback(std::move(callback)), audioFormat(format) { LOG_DEBUG("BluetoothProxyAudio created."); diff --git a/module-audio/Audio/Endpoint.cpp b/module-audio/Audio/Endpoint.cpp index 9f6c6d2a8924310fce67bccd52b22c74d672e503..1ec4d3f42dbaa86ab49590c5a4d59988a3389312 100644 --- a/module-audio/Audio/Endpoint.cpp +++ b/module-audio/Audio/Endpoint.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020, Mudita Sp. z.o.o. All rights reserved. +// Copyright (c) 2017-2021, Mudita Sp. z.o.o. All rights reserved. // For licensing, see https://github.com/mudita/MuditaOS/LICENSE.md #include "Endpoint.hpp" @@ -32,6 +32,15 @@ bool Endpoint::isConnected() const noexcept return _stream != nullptr; } +Sink::Sink(const Capabilities &caps) : Endpoint(caps) +{} + +Source::Source(const Capabilities &caps) : Endpoint(caps) +{} + +IOProxy::IOProxy(const Capabilities &sourceCaps, const Capabilities &sinkCaps) : Source(sourceCaps), Sink(sinkCaps) +{} + StreamConnection::StreamConnection(Source *source, Sink *sink, Stream *stream) : _sink(sink), _source(source), _stream(stream) { diff --git a/module-audio/Audio/Endpoint.hpp b/module-audio/Audio/Endpoint.hpp index b291e83b0815db026a83e6cd8fb2d3faacb04d49..89ac7c85eb748fd0ae28fcd450a8e2466035a631 100644 --- a/module-audio/Audio/Endpoint.hpp +++ b/module-audio/Audio/Endpoint.hpp @@ -1,10 +1,12 @@ -// Copyright (c) 2017-2020, Mudita Sp. z.o.o. All rights reserved. +// Copyright (c) 2017-2021, Mudita Sp. z.o.o. All rights reserved. // For licensing, see https://github.com/mudita/MuditaOS/LICENSE.md #pragma once #include "Stream.hpp" +#include + namespace audio { class Endpoint @@ -13,7 +15,8 @@ namespace audio struct Capabilities { bool usesDMA = false; - std::size_t maxBlockSize = 0; + std::size_t minBlockSize = 1; + std::size_t maxBlockSize = SIZE_MAX; }; Endpoint() = default; @@ -33,6 +36,9 @@ namespace audio class Sink : public Endpoint { public: + Sink() = default; + explicit Sink(const Capabilities &caps); + virtual void onDataSend() = 0; virtual void enableOutput() = 0; virtual void disableOutput() = 0; @@ -41,14 +47,20 @@ namespace audio class Source : public Endpoint { public: + Source() = default; + explicit Source(const Capabilities &caps); + virtual void onDataReceive() = 0; virtual void enableInput() = 0; virtual void disableInput() = 0; }; - class IOProxy : public Sink, public Source + class IOProxy : public Source, public Sink { public: + IOProxy() = default; + IOProxy(const Capabilities &sourceCaps, const Capabilities &sinkCaps); + inline bool isSinkConnected() const noexcept { return Sink::isConnected(); diff --git a/module-audio/Audio/Operation/Operation.cpp b/module-audio/Audio/Operation/Operation.cpp index 3284ddc5b44fddd648f31a769c30b8d007f3b3b3..bae1785cb64afb7401df36018913d5566dfa05c7 100644 --- a/module-audio/Audio/Operation/Operation.cpp +++ b/module-audio/Audio/Operation/Operation.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020, Mudita Sp. z.o.o. All rights reserved. +// Copyright (c) 2017-2021, Mudita Sp. z.o.o. All rights reserved. // For licensing, see https://github.com/mudita/MuditaOS/LICENSE.md #include "Operation.hpp" @@ -93,14 +93,9 @@ namespace audio supportedProfiles.emplace_back(Profile::Create(profile, nullptr, volume, gain), isAvailable); } - std::optional> Operation::CreateDevice(bsp::AudioDevice::Type type, - bsp::AudioDevice::audioCallback_t callback) + + std::optional> Operation::CreateDevice(bsp::AudioDevice::Type type) { - if (type == bsp::AudioDevice::Type::Bluetooth) { - auto audioFormat = currentProfile->GetAudioFormat(); - return std::make_unique( - serviceCallback, *dataStreamOut, *dataStreamIn, audioFormat); - } - return bsp::AudioDevice::Create(type, callback).value_or(nullptr); + return bsp::AudioDevice::Create(type).value_or(nullptr); } } // namespace audio diff --git a/module-audio/Audio/Operation/Operation.hpp b/module-audio/Audio/Operation/Operation.hpp index 21416f595842cbe1e0ca5e21e8d59bdcd1b81e24..126ca4fb8c3d278f45ae9ef89bd54d46738c77e6 100644 --- a/module-audio/Audio/Operation/Operation.hpp +++ b/module-audio/Audio/Operation/Operation.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020, Mudita Sp. z.o.o. All rights reserved. +// Copyright (c) 2017-2021, Mudita Sp. z.o.o. All rights reserved. // For licensing, see https://github.com/mudita/MuditaOS/LICENSE.md #pragma once @@ -8,7 +8,6 @@ #include #include