From 9d7e8b1dd096fbc6c595aa43053bc1d75dc23401 Mon Sep 17 00:00:00 2001 From: Marin Wang Date: Fri, 13 Nov 2026 17:44:12 +0800 Subject: [PATCH 1/2] feat: compress rotated logs with gzip and bound retention --- include/MaaUtils/Logger.h | 7 +- source/CMakeLists.txt | 2 +- source/Logger/LogMaintenance.cpp | 186 +++++++++++++++++++++++++++++++ source/Logger/LogMaintenance.h | 11 ++ source/Logger/Logger.cpp | 76 ++++++------- 5 files changed, 242 insertions(+), 40 deletions(-) create mode 100644 source/Logger/LogMaintenance.cpp create mode 100644 source/Logger/LogMaintenance.h diff --git a/include/MaaUtils/Logger.h b/include/MaaUtils/Logger.h index 9efe6ff..e1d5b99 100644 --- a/include/MaaUtils/Logger.h +++ b/include/MaaUtils/Logger.h @@ -1,5 +1,7 @@ #pragma once +#include + #include "MaaUtils/LoggerUtils.h" #include "MaaUtils/ScopeLeave.hpp" @@ -60,6 +62,7 @@ class MAA_UTILS_API Logger void start_logging(std::filesystem::path dir); void set_stdout_level(level lv); void flush(); + void cleanup(bool force = false); private: template @@ -75,7 +78,6 @@ class MAA_UTILS_API Logger Logger() = default; void reinit(); - void cleanup(); bool rotate(); void open(bool append = true); void close(); @@ -97,6 +99,9 @@ class MAA_UTILS_API Logger std::mutex trace_mutex_; size_t log_count_ = 0; + std::mutex cleanup_mutex_; + std::chrono::steady_clock::time_point last_cleanup_ {}; + std::future cleanup_task_; }; class LogScopeEnterHelper diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 98987bf..cf6af18 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -3,7 +3,7 @@ file(GLOB_RECURSE maa_utils_header ${MAAUTILS_DIR}/include/MaaUtils/*) add_library(MaaUtils SHARED ${maa_utils_src} ${maa_utils_header}) target_include_directories(MaaUtils PRIVATE ${MAAUTILS_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(MaaUtils PUBLIC Boost::regex PRIVATE Boost::system ${OpenCV_LIBS}) +target_link_libraries(MaaUtils PUBLIC Boost::regex PRIVATE Boost::system ${OpenCV_LIBS} ZLIB::ZLIB) if(WIN32) target_link_libraries(MaaUtils PRIVATE d3d12 dxgi Cfgmgr32) diff --git a/source/Logger/LogMaintenance.cpp b/source/Logger/LogMaintenance.cpp new file mode 100644 index 0000000..94c3f96 --- /dev/null +++ b/source/Logger/LogMaintenance.cpp @@ -0,0 +1,186 @@ +#include "LogMaintenance.h" + +#include +#include +#include +#include + +#include + +#include "MaaUtils/Platform.h" +#include "MaaUtils/ScopeLeave.hpp" +#include "MaaUtils/Time.hpp" + +MAA_LOG_NS_BEGIN + +namespace +{ + +enum class ArtifactKind +{ + Vision, + Log, + Error, +}; + +struct LogArtifact +{ + std::filesystem::path path; + std::filesystem::file_time_type modified; + uintmax_t size = 0; + ArtifactKind kind = ArtifactKind::Log; +}; + +bool compress_log(LogArtifact& artifact) +{ + auto archive = artifact.path; + archive += ".gz"; + auto temporary = archive; + temporary += "." + format_now_for_filename() + ".tmp"; + + std::error_code ec; + if (std::filesystem::exists(archive, ec) || ec) { + return false; + } + std::ifstream input(artifact.path, std::ios::binary); + if (!input) { + return false; + } +#ifdef _WIN32 + gzFile output = gzopen_w(temporary.c_str(), "wb1x"); +#else + gzFile output = gzopen(temporary.c_str(), "wb1x"); +#endif + if (!output) { + return false; + } + OnScopeLeave([&] { + if (output) { + gzclose(output); + } + std::filesystem::remove(temporary, ec); + }); + + std::array buffer; + while (input) { + input.read(buffer.data(), buffer.size()); + auto size = static_cast(input.gcount()); + if (size && gzwrite(output, buffer.data(), size) != static_cast(size)) { + return false; + } + } + if (!input.eof()) { + return false; + } + input.close(); + int result = gzclose(output); + output = nullptr; + if (result != Z_OK) { + return false; + } + + // Do not replace a backup which changed while it was being read. + if (std::filesystem::file_size(artifact.path, ec) != artifact.size || ec + || std::filesystem::last_write_time(artifact.path, ec) != artifact.modified || ec) { + return false; + } + auto compressed_size = std::filesystem::file_size(temporary, ec); + if (ec) { + return false; + } + // Retention is based on the original log's age, not the compression time. + std::filesystem::last_write_time(temporary, artifact.modified, ec); + if (ec) { + return false; + } + std::filesystem::rename(temporary, archive, ec); + if (ec) { + return false; + } + if (!std::filesystem::remove(artifact.path, ec) || ec) { + return false; + } + artifact.path = std::move(archive); + artifact.size = compressed_size; + return true; +} + +} // namespace + +void maintain_log_files(const std::filesystem::path& dir) +{ + constexpr auto kMaxAge = std::chrono::hours(24 * 7); + constexpr auto kVisionMaxAge = std::chrono::hours(24 * 3); + constexpr uintmax_t kMaxBytes = 1024ULL * 1024 * 1024; + const auto now = std::filesystem::file_time_type::clock::now(); + std::vector artifacts; + + std::error_code ec; + auto iter = std::filesystem::recursive_directory_iterator(dir, std::filesystem::directory_options::skip_permission_denied, ec); + const auto end = std::filesystem::recursive_directory_iterator(); + for (; !ec && iter != end; iter.increment(ec)) { + const auto& entry = *iter; + if (!std::filesystem::is_regular_file(entry.symlink_status(ec)) || ec) { + ec.clear(); + continue; + } + auto last_write = entry.last_write_time(ec); + if (ec) { + ec.clear(); + continue; + } + auto relative = entry.path().lexically_relative(dir); + const auto name = path_to_utf8_string(relative.filename()); + const auto ext = relative.extension(); + const bool backup = relative.parent_path().empty() && name.starts_with("maafw.bak."); + ArtifactKind kind; + if (backup && (name.ends_with(".log") || name.ends_with(".log.gz"))) { + kind = ArtifactKind::Log; + } + else if ((ext == ".png" || ext == ".jpg") && *relative.begin() == "vision") { + kind = ArtifactKind::Vision; + } + else if ((ext == ".png" || ext == ".jpg") && *relative.begin() == "on_error") { + kind = ArtifactKind::Error; + } + else { + if (backup && name.ends_with(".tmp") && now - last_write > kMaxAge) { + std::filesystem::remove(entry.path(), ec); + } + ec.clear(); + continue; + } + auto max_age = kind == ArtifactKind::Vision ? kVisionMaxAge : kMaxAge; + if (now - last_write > max_age && std::filesystem::remove(entry.path(), ec)) { + continue; + } + ec.clear(); + auto size = entry.file_size(ec); + if (!ec) { + artifacts.push_back({ entry.path(), last_write, size, kind }); + } + ec.clear(); + } + + uintmax_t total_size = 0; + for (auto& artifact : artifacts) { + if (artifact.kind == ArtifactKind::Log && artifact.path.extension() == ".log") { + compress_log(artifact); + } + total_size += artifact.size; + } + // Discard vision drawings first; retain failure screenshots for as long as possible. + std::ranges::sort(artifacts, [](const LogArtifact& lhs, const LogArtifact& rhs) { + return lhs.kind != rhs.kind ? lhs.kind < rhs.kind : lhs.modified < rhs.modified; + }); + for (const auto& artifact : artifacts) { + if (total_size <= kMaxBytes) { + break; + } + if (std::filesystem::remove(artifact.path, ec)) { + total_size -= artifact.size; + } + } +} + +MAA_LOG_NS_END diff --git a/source/Logger/LogMaintenance.h b/source/Logger/LogMaintenance.h new file mode 100644 index 0000000..a75ed43 --- /dev/null +++ b/source/Logger/LogMaintenance.h @@ -0,0 +1,11 @@ +#pragma once + +#include + +#include "MaaUtils/Conf.h" + +MAA_LOG_NS_BEGIN + +void maintain_log_files(const std::filesystem::path& dir); + +MAA_LOG_NS_END diff --git a/source/Logger/Logger.cpp b/source/Logger/Logger.cpp index 93cba6e..798c337 100644 --- a/source/Logger/Logger.cpp +++ b/source/Logger/Logger.cpp @@ -15,6 +15,7 @@ #include #endif +#include "LogMaintenance.h" #include "MaaUtils/Encoding.h" #include "MaaUtils/ImageIo.h" #include "MaaUtils/Platform.h" @@ -109,54 +110,41 @@ void Logger::flush() if (rotated) { log_proc_info(); } + cleanup(rotated); } void Logger::reinit() { - cleanup(); bool rotated = rotate(); open(!rotated); log_proc_info(); + cleanup(true); } -static void remove_old_files(const std::filesystem::path& dir) +void Logger::cleanup(bool force) { - constexpr auto kMaxAge = std::chrono::hours(24 * 7); // 一周 - const auto now = std::filesystem::file_time_type::clock::now(); - - std::error_code ec; - for (const auto& entry : std::filesystem::recursive_directory_iterator(dir, ec)) { - if (ec) { - break; - } - if (!entry.is_regular_file(ec) || ec) { - continue; - } - - const auto ext = path_to_utf8_string(entry.path().extension()); - if (ext != ".log" && ext != ".jpg" && ext != ".png") { - continue; - } - - auto last_write = entry.last_write_time(ec); - if (ec) { - continue; - } - - auto age = now - last_write; - if (age > kMaxAge) { - std::filesystem::remove(entry.path(), ec); - } + if (log_dir_.empty()) { + return; } -} - -void Logger::cleanup() -{ - if (log_dir_.empty() || !std::filesystem::exists(log_dir_)) { + std::unique_lock lock(cleanup_mutex_); + const auto now = std::chrono::steady_clock::now(); + if (cleanup_task_.valid() && cleanup_task_.wait_for(std::chrono::seconds(0)) != std::future_status::ready) { return; } - - std::thread(remove_old_files, log_dir_).detach(); + if (!force && now - last_cleanup_ < std::chrono::minutes(5)) { + return; + } + try { + if (cleanup_task_.valid()) { + cleanup_task_.get(); + } + // The future joins on destruction, so the worker cannot outlive MaaUtils. + cleanup_task_ = std::async(std::launch::async, maintain_log_files, log_dir_); + last_cleanup_ = now; + } + catch (const std::exception& e) { + std::cerr << "Log maintenance failed: " << e.what() << std::endl; + } } bool Logger::rotate() @@ -170,7 +158,7 @@ bool Logger::rotate() ofs_.close(); } - constexpr uintmax_t MaxLogSize = 16ULL * 1024 * 1024; + constexpr uintmax_t MaxLogSize = 128ULL * 1024 * 1024; const uintmax_t log_size = std::filesystem::file_size(log_path_); if (log_size < MaxLogSize) { return false; @@ -179,8 +167,20 @@ bool Logger::rotate() std::error_code ec; const std::filesystem::path bak_path = log_dir_ / std::format(kLogbakFilename, format_now_for_filename()); - std::filesystem::copy(log_path_, bak_path, std::filesystem::copy_options::overwrite_existing, ec); - + auto temporary = bak_path; + temporary += ".tmp"; + // Publish only complete backups. A failed backup must not truncate the active log. + if (!std::filesystem::copy_file(log_path_, temporary, std::filesystem::copy_options::none, ec)) { + if (ec != std::errc::file_exists) { + std::filesystem::remove(temporary, ec); + } + return false; + } + std::filesystem::rename(temporary, bak_path, ec); + if (ec) { + std::filesystem::remove(temporary, ec); + return false; + } return true; } From 8fe8cbe6ddec3701b60278c8c64a8097c1c8ae20 Mon Sep 17 00:00:00 2001 From: Marin Wang Date: Fri, 13 Nov 2026 18:18:24 +0800 Subject: [PATCH 2/2] fix: handle Windows shutdown during gzip log maintenance --- source/Logger/LogMaintenance.cpp | 48 ++++++++++++++++++++++++++++++-- source/Logger/LogMaintenance.h | 2 ++ source/Logger/Logger.cpp | 10 +++++-- 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/source/Logger/LogMaintenance.cpp b/source/Logger/LogMaintenance.cpp index 94c3f96..8526f28 100644 --- a/source/Logger/LogMaintenance.cpp +++ b/source/Logger/LogMaintenance.cpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include #include @@ -11,6 +13,10 @@ #include "MaaUtils/ScopeLeave.hpp" #include "MaaUtils/Time.hpp" +#ifdef _WIN32 +#include "MaaUtils/SafeWindows.hpp" +#endif + MAA_LOG_NS_BEGIN namespace @@ -31,6 +37,21 @@ struct LogArtifact ArtifactKind kind = ArtifactKind::Log; }; +#ifdef _WIN32 +struct MaintenanceTask +{ + std::packaged_task run; + HMODULE library = nullptr; +}; + +void CALLBACK run_maintenance(PTP_CALLBACK_INSTANCE instance, void* context) +{ + std::unique_ptr task(static_cast(context)); + FreeLibraryWhenCallbackReturns(instance, task->library); + task->run(); +} +#endif + bool compress_log(LogArtifact& artifact) { auto archive = artifact.path; @@ -47,9 +68,9 @@ bool compress_log(LogArtifact& artifact) return false; } #ifdef _WIN32 - gzFile output = gzopen_w(temporary.c_str(), "wb1x"); + gzFile output = gzopen_w(temporary.c_str(), "wb6x"); #else - gzFile output = gzopen(temporary.c_str(), "wb1x"); + gzFile output = gzopen(temporary.c_str(), "wb6x"); #endif if (!output) { return false; @@ -183,4 +204,27 @@ void maintain_log_files(const std::filesystem::path& dir) } } +std::future start_log_maintenance(const std::filesystem::path& dir) +{ +#ifdef _WIN32 + // DLL teardown must not wait for workers that Windows terminates at process exit. + // Keep MaaUtils loaded until the callback returns during normal library unloading. + auto task = std::make_unique(); + task->run = std::packaged_task([dir] { maintain_log_files(dir); }); + auto future = task->run.get_future(); + if (!GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS, reinterpret_cast(&run_maintenance), &task->library)) { + throw std::system_error(GetLastError(), std::system_category(), "Hold MaaUtils for log maintenance"); + } + if (!TrySubmitThreadpoolCallback(run_maintenance, task.get(), nullptr)) { + auto error = GetLastError(); + FreeLibrary(task->library); + throw std::system_error(error, std::system_category(), "Start log maintenance"); + } + task.release(); + return future; +#else + return std::async(std::launch::async, maintain_log_files, dir); +#endif +} + MAA_LOG_NS_END diff --git a/source/Logger/LogMaintenance.h b/source/Logger/LogMaintenance.h index a75ed43..d50cef1 100644 --- a/source/Logger/LogMaintenance.h +++ b/source/Logger/LogMaintenance.h @@ -1,11 +1,13 @@ #pragma once #include +#include #include "MaaUtils/Conf.h" MAA_LOG_NS_BEGIN void maintain_log_files(const std::filesystem::path& dir); +std::future start_log_maintenance(const std::filesystem::path& dir); MAA_LOG_NS_END diff --git a/source/Logger/Logger.cpp b/source/Logger/Logger.cpp index 798c337..8804e1a 100644 --- a/source/Logger/Logger.cpp +++ b/source/Logger/Logger.cpp @@ -87,6 +87,13 @@ Logger& Logger::get_instance() void Logger::start_logging(std::filesystem::path dir) { + { + std::unique_lock lock(cleanup_mutex_); + // Finish work in the previous directory before switching or disabling logging. + if (cleanup_task_.valid()) { + cleanup_task_.wait(); + } + } log_dir_ = std::move(dir); if (log_dir_.empty()) { log_path_.clear(); @@ -138,8 +145,7 @@ void Logger::cleanup(bool force) if (cleanup_task_.valid()) { cleanup_task_.get(); } - // The future joins on destruction, so the worker cannot outlive MaaUtils. - cleanup_task_ = std::async(std::launch::async, maintain_log_files, log_dir_); + cleanup_task_ = start_log_maintenance(log_dir_); last_cleanup_ = now; } catch (const std::exception& e) {