commit 53efc0fd91ac7f277c61e24fb7de7cfa5a83af69
parent dc88a6998363dffce5c11f92bc80d3485fd06234
Author: cfillion <cfillion@users.noreply.github.com>
Date: Mon, 27 Feb 2017 15:12:39 -0500
archive: compress files in worker thread when exporting
Diffstat:
3 files changed, 118 insertions(+), 49 deletions(-)
diff --git a/src/archive.cpp b/src/archive.cpp
@@ -157,47 +157,6 @@ void ImportArchive::importPackage(const string &data)
m_tx->install(ver, pinned, m_reader);
}
-size_t Archive::create(const auto_string &path, ReaPack *reapack)
-{
- size_t count = 0;
-
- stringstream toc;
- Registry reg(Path::prefixRoot(Path::REGISTRY));
-
- ArchiveWriter writer(path);
-
- for(const Remote &remote : reapack->config()->remotes.getEnabled()) {
- bool addedRemote = false;
-
- for(const Registry::Entry &entry : reg.getEntries(remote.name())) {
- ++count;
-
- if(!addedRemote) {
- toc << "REPO " << remote.toString() << '\n';
- if(writer.addFile(Index::pathFor(remote.name())))
- continue; // TODO: report error
- addedRemote = true;
- }
-
- toc << "PACK "
- << quoted(entry.category) << '\x20'
- << quoted(entry.package) << '\x20'
- << quoted(entry.version.toString()) << '\x20'
- << entry.pinned << '\n'
- ;
-
- for(const Registry::File &file : reg.getFiles(entry)) {
- if(writer.addFile(file.path))
- break; // TODO: report error
- }
- }
- }
-
- writer.addFile(ARCHIVE_TOC, toc);
-
- return count;
-}
-
ArchiveReader::ArchiveReader(const auto_string &path)
{
zlib_filefunc64_def filefunc;
@@ -285,6 +244,49 @@ void FileExtractor::run(DownloadContext *)
finish(Success);
}
+size_t Archive::create(const auto_string &path, ThreadPool *pool, ReaPack *reapack)
+{
+ size_t count = 0;
+ vector<ThreadTask *> jobs;
+
+ stringstream toc;
+ Registry reg(Path::prefixRoot(Path::REGISTRY));
+
+ ArchiveWriterPtr writer = make_shared<ArchiveWriter>(path);
+
+ for(const Remote &remote : reapack->config()->remotes.getEnabled()) {
+ bool addedRemote = false;
+
+ for(const Registry::Entry &entry : reg.getEntries(remote.name())) {
+ ++count;
+
+ if(!addedRemote) {
+ toc << "REPO " << remote.toString() << '\n';
+ jobs.push_back(new FileCompressor(Index::pathFor(remote.name()), writer));
+ addedRemote = true;
+ }
+
+ toc << "PACK "
+ << quoted(entry.category) << '\x20'
+ << quoted(entry.package) << '\x20'
+ << quoted(entry.version.toString()) << '\x20'
+ << entry.pinned << '\n'
+ ;
+
+ for(const Registry::File &file : reg.getFiles(entry))
+ jobs.push_back(new FileCompressor(file.path, writer));
+ }
+ }
+
+ writer->addFile(ARCHIVE_TOC, toc);
+
+ // start after we've written the table of contents in the main thread
+ for(ThreadTask *job : jobs)
+ pool->push(job);
+
+ return count;
+}
+
ArchiveWriter::ArchiveWriter(const auto_string &path)
{
zlib_filefunc64_def filefunc;
@@ -339,3 +341,35 @@ int ArchiveWriter::addFile(const Path &path, istream &stream) noexcept
return zipCloseFileInZip(m_zip);
}
+
+FileCompressor::FileCompressor(const Path &target, const ArchiveWriterPtr &writer)
+ : m_path(target), m_writer(writer)
+{
+ setSummary("Compressing %s: " + target.join());
+}
+
+void FileCompressor::run(DownloadContext *)
+{
+ if(aborted()) {
+ finish(Aborted, {"cancelled", m_path.join()});
+ return;
+ }
+
+ ThreadNotifier::get()->notify({this, Running});
+
+ ifstream stream;
+ if(!FS::open(stream, m_path)) {
+ finish(Failure, {FS::lastError(), m_path.join()});
+ return;
+ }
+
+ const int error = m_writer->addFile(m_path, stream);
+ stream.close();
+
+ if(error) {
+ const format &msg = format("Failed to compress file (%d)") % error;
+ finish(Failure, {msg.str(), m_path.join()});
+ }
+ else
+ finish(Success);
+}
diff --git a/src/archive.hpp b/src/archive.hpp
@@ -23,12 +23,13 @@
#include "thread.hpp"
class ReaPack;
+class ThreadPool;
typedef void *zipFile;
namespace Archive {
void import(const auto_string &path, ReaPack *);
- size_t create(const auto_string &path, ReaPack *);
+ size_t create(const auto_string &path, ThreadPool *pool, ReaPack *);
};
class ArchiveReader {
@@ -42,6 +43,8 @@ private:
zipFile m_zip;
};
+typedef std::shared_ptr<ArchiveReader> ArchiveReaderPtr;
+
class ArchiveWriter {
public:
ArchiveWriter(const auto_string &path);
@@ -53,7 +56,7 @@ private:
zipFile m_zip;
};
-typedef std::shared_ptr<ArchiveReader> ArchiveReaderPtr;
+typedef std::shared_ptr<ArchiveWriter> ArchiveWriterPtr;
class FileExtractor : public ThreadTask {
public:
@@ -68,4 +71,16 @@ private:
ArchiveReaderPtr m_reader;
};
+class FileCompressor : public ThreadTask {
+public:
+ FileCompressor(const Path &target, const ArchiveWriterPtr &);
+
+ bool concurrent() const override { return false; }
+ void run(DownloadContext *) override;
+
+private:
+ Path m_path;
+ ArchiveWriterPtr m_writer;
+};
+
#endif
diff --git a/src/manager.cpp b/src/manager.cpp
@@ -25,6 +25,7 @@
#include "filedialog.hpp"
#include "import.hpp"
#include "menu.hpp"
+#include "progress.hpp"
#include "reapack.hpp"
#include "remote.hpp"
#include "resource.hpp"
@@ -524,24 +525,43 @@ void Manager::exportArchive()
if(path.empty())
return;
- auto_char msg[512];
+ ThreadPool *pool = new ThreadPool;
+ Dialog *progress = Dialog::Create<Progress>(instance(), parent(), pool);
try {
- const size_t count = Archive::create(path, m_reapack);
+ const size_t count = Archive::create(path, pool, m_reapack);
- auto_snprintf(msg, auto_size(msg),
- AUTO_STR("Done! %zu packages were exported in the archive."), count);
+ const auto finish = [=] {
+ Dialog::Destroy(progress);
+
+ auto_char msg[255];
+ auto_snprintf(msg, auto_size(msg),
+ AUTO_STR("Done! %zu package%s were exported in the archive."),
+ count, count == 1 ? AUTO_STR("") : AUTO_STR("s"));
+ MessageBox(handle(), msg, title, MB_OK);
+
+ delete pool;
+ };
+
+ pool->onDone(finish);
+
+ if(pool->idle())
+ finish();
}
catch(const reapack_error &e) {
+ Dialog::Destroy(progress);
+ delete pool;
+
const auto_string &desc = make_autostring(e.what());
+ auto_char msg[512];
auto_snprintf(msg, auto_size(msg),
- AUTO_STR("ReaPack could not write into %s (%s)."),
+ AUTO_STR("An error occured while writing into %s.\r\n\r\n%s."),
path.c_str(), desc.c_str()
);
+ MessageBox(handle(), msg, title, MB_OK);
}
- MessageBox(handle(), msg, title, MB_OK);
}
void Manager::launchBrowser()
