From cdf5dbf557b29f31c9755feac594d2d2f4013351 Mon Sep 17 00:00:00 2001 From: Lefucjusz Date: Mon, 26 Feb 2024 18:27:10 +0100 Subject: [PATCH] [BH-1887] Speed up update process * Added stream buffering for reading data from tar archive with update. * Increased size of the buffer used for unpacking files from tar archive. * Changed buffers data type from vectors to raw heap-allocated arrays. --- harmony_changelog.md | 1 + .../endpoints/update/UpdateHelper.cpp | 15 +++---- module-utils/tar/include/tar/tar.hpp | 7 +++- module-utils/tar/tar.cpp | 41 ++++++++++--------- .../drivers/src/purefs/fs/filesystem_ext4.cpp | 4 +- pure_changelog.md | 1 + 6 files changed, 37 insertions(+), 32 deletions(-) diff --git a/harmony_changelog.md b/harmony_changelog.md index efd870989f69463f6af24277361edfd8f8c6a07d..63b6d77849ebe2ec704e3a0b629becfeafb71e2d 100644 --- a/harmony_changelog.md +++ b/harmony_changelog.md @@ -16,6 +16,7 @@ ### Changed / Improved * Updated FSL drivers from NXP * Increased battery charger stack size +* Increased speed of update process ## [2.5.0 2024-02-09] diff --git a/module-services/service-desktop/endpoints/update/UpdateHelper.cpp b/module-services/service-desktop/endpoints/update/UpdateHelper.cpp index 259ec0aebabb561f9a3f287ec2cd1c9a205f81d4..913dfb55fefc9e638290deb3d196963d42495aee 100644 --- a/module-services/service-desktop/endpoints/update/UpdateHelper.cpp +++ b/module-services/service-desktop/endpoints/update/UpdateHelper.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2023, Mudita Sp. z.o.o. All rights reserved. +// Copyright (c) 2017-2024, Mudita Sp. z.o.o. All rights reserved. // For licensing, see https://github.com/mudita/MuditaOS/LICENSE.md #include @@ -20,11 +20,6 @@ namespace sdesktop::endpoints { - namespace - { - constexpr auto chunkSize = 1024 * 128; - } - struct UpdatePackageEntries { explicit UpdatePackageEntries(const json11::Json &json) @@ -80,11 +75,13 @@ namespace sdesktop::endpoints return false; } + constexpr auto chunkSize = 128 * 1024; + auto rawDataBuffer = std::make_unique(chunkSize); + MD5 md5; - std::vector raw_data(chunkSize); std::size_t bytesRead; - while ((bytesRead = std::fread(raw_data.data(), 1, chunkSize, fd)) > 0) { - md5.add(raw_data.data(), bytesRead); + while ((bytesRead = std::fread(rawDataBuffer.get(), sizeof(*rawDataBuffer.get()), chunkSize, fd)) > 0) { + md5.add(rawDataBuffer.get(), bytesRead); } std::fclose(fd); diff --git a/module-utils/tar/include/tar/tar.hpp b/module-utils/tar/include/tar/tar.hpp index 4a324909885c1a4e28a9d81e763365da4beb0762..63401420677a73e6bc75d4ec89ea148410843726 100644 --- a/module-utils/tar/include/tar/tar.hpp +++ b/module-utils/tar/include/tar/tar.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, Mudita Sp. z.o.o. All rights reserved. +// Copyright (c) 2017-2024, Mudita Sp. z.o.o. All rights reserved. // For licensing, see https://github.com/mudita/MuditaOS/LICENSE.md #pragma once @@ -21,13 +21,16 @@ namespace tar bool is_file() const; bool is_directory() const; - void read(const std::byte *data, std::size_t size) const; + void read(std::uint8_t *data, std::size_t size) const; private: explicit entry(const std::filesystem::path &path); friend class iterator; mutable mtar_t handle{}; mtar_header_t tar_header; + + static constexpr auto streamBufferSize = 64 * 1024; + std::unique_ptr streamBuffer; }; class iterator diff --git a/module-utils/tar/tar.cpp b/module-utils/tar/tar.cpp index 0c3f119fdeffa09bcf887a846470a1c5a5efa613..d6797d0bf3c933f93b705082e6b3f9d7d6b36ba2 100644 --- a/module-utils/tar/tar.cpp +++ b/module-utils/tar/tar.cpp @@ -1,9 +1,8 @@ -// Copyright (c) 2017-2022, Mudita Sp. z.o.o. All rights reserved. +// Copyright (c) 2017-2024, Mudita Sp. z.o.o. All rights reserved. // For licensing, see https://github.com/mudita/MuditaOS/LICENSE.md #include "tar/tar.hpp" -#include #include #include @@ -14,23 +13,25 @@ namespace return entry.name() != "." and entry.name() != ".." and entry.name() != "..."; } - void write_to_file(const tar::entry &entry, const std::filesystem::path &path) + void writeToFile(const tar::entry &entry, const std::filesystem::path &path) { - constexpr std::size_t chunk_size = 1024 * 128; - std::ofstream out_file{path, std::ios::binary}; - if (not out_file.is_open()) { + auto fd = std::fopen(path.c_str(), "wb"); + if (fd == nullptr) { throw std::filesystem::filesystem_error("During opening " + path.string(), std::error_code{}); } - std::vector raw_data(chunk_size); - auto bytes_left = entry.size(); - while (bytes_left > 0) { - const std::size_t to_read = bytes_left > chunk_size ? chunk_size : bytes_left; - entry.read(raw_data.data(), to_read); + constexpr std::size_t chunkSize = 256 * 1024; + auto rawDataBuffer = std::make_unique(chunkSize); - out_file.write(reinterpret_cast(raw_data.data()), to_read); - bytes_left -= to_read; + auto bytesLeft = static_cast(entry.size()); + while (bytesLeft > 0) { + const auto bytesToRead = std::min(bytesLeft, chunkSize); + entry.read(rawDataBuffer.get(), bytesToRead); + std::fwrite(rawDataBuffer.get(), sizeof(*rawDataBuffer.get()), bytesToRead, fd); + bytesLeft -= bytesToRead; } + + std::fclose(fd); } } // namespace @@ -41,6 +42,9 @@ namespace tar if (mtar_open(&handle, path.c_str(), "rb") != MTAR_ESUCCESS) { throw std::filesystem::filesystem_error("During opening tar file " + path.string(), std::error_code{}); } + streamBuffer = std::make_unique(streamBufferSize); + setvbuf(handle.stream, streamBuffer.get(), _IOFBF, streamBufferSize); + if (mtar_read_header(&handle, &tar_header) != MTAR_ESUCCESS) { throw std::filesystem::filesystem_error("During reading from tar file " + path.string(), std::error_code{}); } @@ -65,10 +69,10 @@ namespace tar { return tar_header.name; } - void entry::read(const std::byte *data, const std::size_t size) const + void entry::read(std::uint8_t *data, const std::size_t size) const { - const std::size_t to_read = size > tar_header.size ? tar_header.size : size; - if (mtar_read_data(&handle, (void *)(data), to_read) != MTAR_ESUCCESS) { + const auto bytesToRead = std::min(size, static_cast(tar_header.size)); + if (mtar_read_data(&handle, reinterpret_cast(data), bytesToRead) != MTAR_ESUCCESS) { throw std::filesystem::filesystem_error("During reading from tar file", std::error_code{}); } } @@ -89,7 +93,6 @@ namespace tar mtar_read_header(&entry_->handle, &entry_->tar_header) != MTAR_ESUCCESS) { entry_ = {}; } - return *this; } iterator iterator::operator++(int) @@ -115,8 +118,8 @@ namespace tar std::filesystem::create_directories(full_path); } else if (entry.is_file()) { - write_to_file(entry, full_path); + writeToFile(entry, full_path); } } } -} // namespace tar \ No newline at end of file +} // namespace tar diff --git a/module-vfs/drivers/src/purefs/fs/filesystem_ext4.cpp b/module-vfs/drivers/src/purefs/fs/filesystem_ext4.cpp index f4108f4ea7d9634a6ef7f1ba76f5cc07747e674a..4e03bf00ad08fc1d2dcad50059bc2846e96c920b 100644 --- a/module-vfs/drivers/src/purefs/fs/filesystem_ext4.cpp +++ b/module-vfs/drivers/src/purefs/fs/filesystem_ext4.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022, Mudita Sp. z.o.o. All rights reserved. +// Copyright (c) 2017-2024, Mudita Sp. z.o.o. All rights reserved. // For licensing, see https://github.com/mudita/MuditaOS/LICENSE.md #include @@ -394,7 +394,7 @@ namespace purefs::fs::drivers mnt, [](const char *path) { if (ext4_inode_exist(path, EXT4_DE_DIR) == 0) { - LOG_WARN("rmdir syscall instead of unlink is recommended for remove directory"); + LOG_WARN("rmdir syscall instead of unlink is recommended to remove directory"); return -ext4_dir_rm(path); } else { diff --git a/pure_changelog.md b/pure_changelog.md index 43e89163863517564b09b37502113f031f9c25bc..7219773cce4d1bf9902bf12359f56098a953008a 100644 --- a/pure_changelog.md +++ b/pure_changelog.md @@ -12,6 +12,7 @@ ### Changed / Improved * Updated share capital in EULA. * Updated copy on final onboarding screen. +* Increased speed of update process. ## [1.11.0 2023-12-15]