From 14901580347673ce91e308950a2b73b70ba9dcf3 Mon Sep 17 00:00:00 2001 From: Alek Rudnik Date: Wed, 6 Oct 2021 14:18:35 +0200 Subject: [PATCH] [EGD-7725] Tune up file system io access Increasd io stream buffers: - 64kB for big files with short lifetime scope - 16kB for big files with long lifetime scope - 1kB - dont changes default forr the rest --- module-audio/Audio/decoder/Decoder.cpp | 4 +++ module-audio/Audio/decoder/Decoder.hpp | 1 + module-audio/Audio/encoder/Encoder.cpp | 5 +++- module-audio/Audio/encoder/Encoder.hpp | 8 +++--- module-db/Database/sqlite3vfs.cpp | 7 +++++- .../endpoints/backup/BackupRestore.cpp | 25 +++++++++++++++---- .../endpoints/filesystem/FileContext.cpp | 2 +- .../endpoints/filesystem/FileContext.hpp | 2 +- module-utils/log/Logger.cpp | 8 ++++-- 9 files changed, 47 insertions(+), 15 deletions(-) diff --git a/module-audio/Audio/decoder/Decoder.cpp b/module-audio/Audio/decoder/Decoder.cpp index a417917c51e757266944a8af5c1b1ca85e93ee40..767863df2a34204f2386a853ce7e491fb961d43a 100644 --- a/module-audio/Audio/decoder/Decoder.cpp +++ b/module-audio/Audio/decoder/Decoder.cpp @@ -23,6 +23,10 @@ namespace audio return; } + constexpr size_t streamBufferSize = 16 * 1024; + streamBuffer = std::make_unique(streamBufferSize); + setvbuf(fd, streamBuffer.get(), _IOFBF, streamBufferSize); + std::fseek(fd, 0, SEEK_END); fileSize = std::ftell(fd); std::rewind(fd); diff --git a/module-audio/Audio/decoder/Decoder.hpp b/module-audio/Audio/decoder/Decoder.hpp index 2fb41231da841552b0a3c17231b674b6df996d16..a0c88e4e9975ec17f6107734481dfebe88a012e6 100644 --- a/module-audio/Audio/decoder/Decoder.hpp +++ b/module-audio/Audio/decoder/Decoder.hpp @@ -83,6 +83,7 @@ namespace audio uint32_t chanNumber = 0; float position = 0; std::FILE *fd = nullptr; + std::unique_ptr streamBuffer; uint32_t fileSize = 0; std::string filePath; diff --git a/module-audio/Audio/encoder/Encoder.cpp b/module-audio/Audio/encoder/Encoder.cpp index a5efc58bda3aafe5da297ddffcaac1b66749d9d0..4f825c187154b76efda4131386d2778daf09d800 100644 --- a/module-audio/Audio/encoder/Encoder.cpp +++ b/module-audio/Audio/encoder/Encoder.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020, Mudita Sp. z.o.o. All rights reserved. +// Copyright (c) 2017-2021, Mudita Sp. z.o.o. All rights reserved. // For licensing, see https://github.com/mudita/MuditaOS/LICENSE.md #include "Encoder.hpp" @@ -19,6 +19,9 @@ namespace audio if (fd == NULL) { return; } + constexpr size_t streamBufferSize = 6; + streamBuffer = std::make_unique(streamBufferSize); + setvbuf(fd, streamBuffer.get(), _IOFBF, streamBufferSize); isInitialized = true; } diff --git a/module-audio/Audio/encoder/Encoder.hpp b/module-audio/Audio/encoder/Encoder.hpp index 134ba3d585029db96b94d6a1129c7d0197dad21f..e69acb77ede546e11d62c2c9b23ca60ae279eae8 100644 --- a/module-audio/Audio/encoder/Encoder.hpp +++ b/module-audio/Audio/encoder/Encoder.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020, Mudita Sp. z.o.o. All rights reserved. +// Copyright (c) 2017-2021, Mudita Sp. z.o.o. All rights reserved. // For licensing, see https://github.com/mudita/MuditaOS/LICENSE.md #pragma once @@ -40,8 +40,9 @@ namespace audio const Format format; protected: - float position = 0; - std::FILE *fd = nullptr; + float position = 0; + std::FILE *fd = nullptr; + std::unique_ptr streamBuffer; uint32_t fileSize = 0; std::string filePath; @@ -49,4 +50,3 @@ namespace audio }; } // namespace audio - diff --git a/module-db/Database/sqlite3vfs.cpp b/module-db/Database/sqlite3vfs.cpp index 79e172b717eb511689774dc90b977519e0940456..393419f0fb11dbdaac3946a87d0bcd98f1e60f04 100644 --- a/module-db/Database/sqlite3vfs.cpp +++ b/module-db/Database/sqlite3vfs.cpp @@ -128,7 +128,6 @@ #include #include - /* ** Size of the write buffer used by journal files in bytes. */ @@ -152,6 +151,7 @@ struct EcophoneFile { sqlite3_file base; /* Base class. Must be first. */ std::FILE *fd; /* File descriptor */ + std::unique_ptr streamBuffer; char *aBuffer; /* Pointer to malloc'd buffer */ int nBuffer; /* Valid bytes of data in zBuffer */ @@ -317,6 +317,7 @@ static int ecophoneClose(sqlite3_file *pFile) EcophoneFile *p = (EcophoneFile *)pFile; rc = ecophoneFlushBuffer(p); sqlite3_free(p->aBuffer); + p->streamBuffer.reset(); std::fclose(p->fd); return rc; @@ -616,6 +617,10 @@ static int ecophoneOpen(sqlite3_vfs *pVfs, /* VFS */ sqlite3_free(aBuf); return SQLITE_CANTOPEN; } + // set as 16 kB instead 64kB as it is allocated for each open db file + constexpr size_t streamBufferSize = 16 * 1024; + p->streamBuffer = std::make_unique(streamBufferSize); + setvbuf(p->fd, p->streamBuffer.get(), _IOFBF, streamBufferSize); p->aBuffer = aBuf; if (pOutFlags) { diff --git a/module-services/service-desktop/endpoints/backup/BackupRestore.cpp b/module-services/service-desktop/endpoints/backup/BackupRestore.cpp index e5bac998d4d08bb1f2a9de603e9877447cc7efc0..190ce004936a1ec6b3001e23c9f5c5779d4a5910 100644 --- a/module-services/service-desktop/endpoints/backup/BackupRestore.cpp +++ b/module-services/service-desktop/endpoints/backup/BackupRestore.cpp @@ -56,7 +56,13 @@ static bool copyFile(const std::filesystem::path &from, const std::filesystem::p std::unique_ptr buffer = std::make_unique(purefs::buffer::tar_buf); size_t bytes; - while ((bytes = std::fread(buffer.get(), 1, purefs::buffer::tar_buf, fromFp)) != 0) { + constexpr size_t streamBufferSize = 64 * 1024; + auto streamBufferFrom = std::make_unique(streamBufferSize); + auto streamBufferTo = std::make_unique(streamBufferSize); + setvbuf(fromFp, streamBufferFrom.get(), _IOFBF, streamBufferSize); + setvbuf(toFp, streamBufferTo.get(), _IOFBF, streamBufferSize); + + while ((bytes = std::fread(buffer.get(), purefs::buffer::tar_buf, 1, fromFp)) != 0) { if (std::fwrite(buffer.get(), 1, bytes, toFp) != bytes) { return false; } @@ -220,14 +226,16 @@ bool BackupRestore::PackUserFiles(std::filesystem::path &path) LOG_INFO("opening tar file..."); int ret = mtar_open(&tarFile, tarFilePath.c_str(), "w"); - if (ret != MTAR_ESUCCESS) { LOG_ERROR("opening tar file failed, quitting..."); BackupRestore::RemoveBackupDir(path); return false; } - auto buffer = std::make_unique(purefs::buffer::tar_buf); + auto buffer = std::make_unique(purefs::buffer::tar_buf); + constexpr size_t streamBufferSize = 64 * 1024; + auto streamBuffer = std::make_unique(streamBufferSize); + setvbuf(tarFile.stream, streamBuffer.get(), _IOFBF, streamBufferSize); std::error_code e; for (auto &direntry : std::filesystem::directory_iterator(path)) { @@ -356,7 +364,10 @@ bool BackupRestore::UnpackBackupFile(const std::filesystem::path &tarFilePath) return false; } - auto buffer = std::make_unique(purefs::buffer::tar_buf); + auto buffer = std::make_unique(purefs::buffer::tar_buf); + constexpr size_t streamBufferSize = 64 * 1024; + auto streamBuffer = std::make_unique(streamBufferSize); + setvbuf(tarFile.stream, streamBuffer.get(), _IOFBF, streamBufferSize); do { ret = mtar_read_header(&tarFile, &tarHeader); @@ -374,6 +385,10 @@ bool BackupRestore::UnpackBackupFile(const std::filesystem::path &tarFilePath) return false; } + constexpr size_t streamBufferSize = 16 * 1024; + auto streamBuffer = std::make_unique(streamBufferSize); + setvbuf(file, streamBuffer.get(), _IOFBF, streamBufferSize); + uint32_t loopcount = (tarHeader.size / purefs::buffer::tar_buf) + 1u; uint32_t readsize = 0u; @@ -394,7 +409,7 @@ bool BackupRestore::UnpackBackupFile(const std::filesystem::path &tarFilePath) return false; } - if (std::fwrite(buffer.get(), 1, readsize, file) != readsize) { + if (std::fwrite(buffer.get(), readsize, 1, file) != readsize) { LOG_ERROR("writting file failed, quitting..."); mtar_close(&tarFile); std::fclose(file); diff --git a/module-services/service-desktop/endpoints/filesystem/FileContext.cpp b/module-services/service-desktop/endpoints/filesystem/FileContext.cpp index b58ec6e124f59f79a55b842b1bdca5967cb84fc4..0e71d44d350a4c27785e6a66f6b8b45cbef52460 100644 --- a/module-services/service-desktop/endpoints/filesystem/FileContext.cpp +++ b/module-services/service-desktop/endpoints/filesystem/FileContext.cpp @@ -17,7 +17,7 @@ FileContext::FileContext( throw std::runtime_error("File open error"); } - constexpr size_t streamBufferSize = 16384; + constexpr size_t streamBufferSize = 64 * 1024; streamBuffer = std::make_unique(streamBufferSize); setvbuf(file, streamBuffer.get(), _IOFBF, streamBufferSize); diff --git a/module-services/service-desktop/endpoints/include/endpoints/filesystem/FileContext.hpp b/module-services/service-desktop/endpoints/include/endpoints/filesystem/FileContext.hpp index fecbd18dd49205e19e888b3ab2d33a1eaffd8dc3..e2e65d03ea2e7349176cce7972db489a2ced4095 100644 --- a/module-services/service-desktop/endpoints/include/endpoints/filesystem/FileContext.hpp +++ b/module-services/service-desktop/endpoints/include/endpoints/filesystem/FileContext.hpp @@ -39,11 +39,11 @@ class FileContext protected: std::filesystem::path path{}; std::FILE *file{}; + std::unique_ptr streamBuffer; std::size_t size{}; std::size_t offset{}; std::size_t chunkSize{}; CRC32 runningCrc32Digest; - std::unique_ptr streamBuffer; }; class FileReadContext : public FileContext diff --git a/module-utils/log/Logger.cpp b/module-utils/log/Logger.cpp index b3d4047f64d40ba881bd2f5f51d211a0a3831888..ae1306033f3548afbbd823209e2d983adcf85ee1 100644 --- a/module-utils/log/Logger.cpp +++ b/module-utils/log/Logger.cpp @@ -68,8 +68,8 @@ namespace Log void Logger::init(Application app, size_t fileSize) { - application = std::move(app); - maxFileSize = fileSize; + application = std::move(app); + maxFileSize = fileSize; #if LOG_USE_COLOR == 1 enableColors(true); #else @@ -157,6 +157,10 @@ namespace Log status = -EIO; } + constexpr size_t streamBufferSize = 64 * 1024; + auto streamBuffer = std::make_unique(streamBufferSize); + logFile.rdbuf()->pubsetbuf(streamBuffer.get(), streamBufferSize); + if (firstDump) { addFileHeader(logFile); }