commit abb5bb0d19a031b1bbb37967a1eb4855a12893a1
parent 6febcf28d85c49d24cf8469a99fbc5e559445b1f
Author: cfillion <cfillion@users.noreply.github.com>
Date: Sun, 26 Feb 2017 23:34:46 -0500
download: write data directly to disk when applicable
Diffstat:
16 files changed, 186 insertions(+), 89 deletions(-)
diff --git a/src/download.cpp b/src/download.cpp
@@ -17,6 +17,7 @@
#include "download.hpp"
+#include "filesystem.hpp"
#include "reapack.hpp"
#include <boost/format.hpp>
@@ -91,7 +92,7 @@ size_t Download::WriteData(char *data, size_t rawsize, size_t nmemb, void *ptr)
{
const size_t size = rawsize * nmemb;
- static_cast<string *>(ptr)->append(data, size);
+ static_cast<ostream *>(ptr)->write(data, size);
return size;
}
@@ -102,17 +103,11 @@ int Download::UpdateProgress(void *ptr, const double, const double,
return static_cast<Download *>(ptr)->aborted();
}
-Download::Download(const string &name, const string &url,
- const NetworkOpts &opts, const int flags)
- : m_name(name), m_url(url), m_opts(opts), m_flags(flags)
+Download::Download(const string &url, const NetworkOpts &opts, const int flags)
+ : m_url(url), m_opts(opts), m_flags(flags)
{
}
-string Download::summary() const
-{
- return "Downloading %s: " + m_name;
-}
-
void Download::start()
{
WorkerThread *thread = new WorkerThread;
@@ -129,6 +124,10 @@ void Download::run(DownloadContext *ctx)
ThreadNotifier::get()->notify({this, Running});
+ ostream *stream = openStream();
+ if(!stream)
+ return;
+
curl_easy_setopt(ctx->m_curl, CURLOPT_URL, m_url.c_str());
curl_easy_setopt(ctx->m_curl, CURLOPT_PROXY, m_opts.proxy.c_str());
curl_easy_setopt(ctx->m_curl, CURLOPT_SSL_VERIFYPEER, m_opts.verifyPeer);
@@ -137,7 +136,7 @@ void Download::run(DownloadContext *ctx)
curl_easy_setopt(ctx->m_curl, CURLOPT_PROGRESSDATA, this);
curl_easy_setopt(ctx->m_curl, CURLOPT_WRITEFUNCTION, WriteData);
- curl_easy_setopt(ctx->m_curl, CURLOPT_WRITEDATA, &m_contents);
+ curl_easy_setopt(ctx->m_curl, CURLOPT_WRITEDATA, stream);
curl_slist *headers = nullptr;
if(has(Download::NoCacheFlag))
@@ -148,6 +147,7 @@ void Download::run(DownloadContext *ctx)
curl_easy_setopt(ctx->m_curl, CURLOPT_ERRORBUFFER, errbuf);
const CURLcode res = curl_easy_perform(ctx->m_curl);
+ closeStream();
if(aborted())
finish(Aborted, {"aborted", m_url});
@@ -160,3 +160,39 @@ void Download::run(DownloadContext *ctx)
curl_slist_free_all(headers);
}
+
+MemoryDownload::MemoryDownload(const string &name, const string &url,
+ const NetworkOpts &opts, int flags)
+ : Download(url, opts, flags), m_name(name)
+{
+}
+
+string MemoryDownload::summary() const
+{
+ return "Downloading %s: " + m_name;
+}
+
+FileDownload::FileDownload(const Path &target, const string &url,
+ const NetworkOpts &opts, int flags)
+ : Download(url, opts, flags), m_path(target)
+{
+}
+
+string FileDownload::summary() const
+{
+ return "Downloading %s: " + m_path.target().join();
+}
+
+ostream *FileDownload::openStream()
+{
+ if(FS::open(m_stream, m_path.temp()))
+ return &m_stream;
+
+ finish(Failure, {FS::lastError(), m_path.temp().join()});
+ return nullptr;
+}
+
+void FileDownload::closeStream()
+{
+ m_stream.close();
+}
diff --git a/src/download.hpp b/src/download.hpp
@@ -19,9 +19,11 @@
#define REAPACK_DOWNLOAD_HPP
#include "config.hpp"
+#include "path.hpp"
#include "thread.hpp"
-#include <string>
+#include <fstream>
+#include <sstream>
#include <curl/curl.h>
@@ -41,28 +43,59 @@ public:
NoCacheFlag = 1<<0,
};
- Download(const std::string &name, const std::string &url,
- const NetworkOpts &, int flags = 0);
-
- std::string summary() const override;
+ Download(const std::string &url, const NetworkOpts &, int flags = 0);
const std::string &url() const { return m_url; }
- const NetworkOpts &options() const { return m_opts; }
- bool has(Flag f) const { return (m_flags & f) != 0; }
-
- const std::string &contents() { return m_contents; }
void start();
void run(DownloadContext *) override;
private:
+ virtual std::ostream *openStream() = 0;
+ virtual void closeStream() {}
+
+private:
+ bool has(Flag f) const { return (m_flags & f) != 0; }
static size_t WriteData(char *, size_t, size_t, void *);
static int UpdateProgress(void *, double, double, double, double);
- std::string m_name;
std::string m_url;
NetworkOpts m_opts;
int m_flags;
- std::string m_contents;
+};
+
+class MemoryDownload : public Download {
+public:
+ MemoryDownload(const std::string &name, const std::string &url,
+ const NetworkOpts &, int flags = 0);
+
+ std::string contents() const { return m_stream.str(); }
+
+protected:
+ std::ostream *openStream() override { return &m_stream; }
+
+private:
+ std::string summary() const override;
+
+ std::string m_name;
+ std::stringstream m_stream;
+};
+
+class FileDownload : public Download {
+public:
+ FileDownload(const Path &target, const std::string &url,
+ const NetworkOpts &, int flags = 0);
+
+ const TempPath &path() const { return m_path; }
+
+protected:
+ std::ostream *openStream() override;
+ void closeStream() override;
+
+private:
+ std::string summary() const override;
+
+ TempPath m_path;
+ std::ofstream m_stream;
};
#endif
diff --git a/src/filesystem.cpp b/src/filesystem.cpp
@@ -46,14 +46,19 @@ FILE *FS::open(const Path &path)
#endif
}
-bool FS::write(const Path &path, const string &contents)
+bool FS::open(ofstream &stream, const Path &path)
{
mkdir(path.dirname());
const Path &fullPath = Path::prefixRoot(path);
- ofstream file(make_autostring(fullPath.join()), ios_base::binary);
+ stream.open(make_autostring(fullPath.join()), ios_base::binary);
+ return stream.good();
+}
- if(!file)
+bool FS::write(const Path &path, const string &contents)
+{
+ ofstream file;
+ if(!open(file, path))
return false;
file << contents;
@@ -62,6 +67,15 @@ bool FS::write(const Path &path, const string &contents)
return true;
}
+bool FS::rename(const TempPath &path)
+{
+#ifdef _WIN32
+ remove(path.target());
+#endif
+
+ return rename(path.temp(), path.target());
+}
+
bool FS::rename(const Path &from, const Path &to)
{
const string &fullFrom = Path::prefixRoot(from).join();
diff --git a/src/filesystem.hpp b/src/filesystem.hpp
@@ -21,10 +21,13 @@
#include <string>
class Path;
+class TempPath;
namespace FS {
FILE *open(const Path &);
+ bool open(std::ofstream &, const Path &);
bool write(const Path &, const std::string &);
+ bool rename(const TempPath &);
bool rename(const Path &, const Path &);
bool remove(const Path &);
bool removeRecursive(const Path &);
diff --git a/src/import.cpp b/src/import.cpp
@@ -93,7 +93,8 @@ void Import::fetch()
setWaiting(true);
- Download *dl = m_download = new Download({}, url, m_reapack->config()->network);
+ const auto &opts = m_reapack->config()->network;
+ MemoryDownload *dl = m_download = new MemoryDownload({}, url, opts);
dl->onFinish([=] {
const ThreadTask::State state = dl->state();
diff --git a/src/import.hpp b/src/import.hpp
@@ -24,7 +24,7 @@
#include <string>
-class Download;
+class MemoryDownload;
class ReaPack;
class Remote;
@@ -45,7 +45,7 @@ private:
void setWaiting(bool);
ReaPack *m_reapack;
- Download *m_download;
+ MemoryDownload *m_download;
short m_fakePos;
HWND m_url;
diff --git a/src/index.cpp b/src/index.cpp
@@ -82,7 +82,7 @@ IndexPtr Index::load(const string &name, const char *data)
return IndexPtr(ri);
}
-Download *Index::fetch(const Remote &remote,
+FileDownload *Index::fetch(const Remote &remote,
const bool stale, const NetworkOpts &opts)
{
time_t mtime = 0, now = time(nullptr);
@@ -94,7 +94,8 @@ Download *Index::fetch(const Remote &remote,
return nullptr;
}
- return new Download(remote.name(), remote.url(), opts, Download::NoCacheFlag);
+ const Path &path = pathFor(remote.name());
+ return new FileDownload(path, remote.url(), opts, Download::NoCacheFlag);
}
Index::Index(const string &name)
diff --git a/src/index.hpp b/src/index.hpp
@@ -28,8 +28,8 @@
#include "package.hpp"
#include "source.hpp"
-class Download;
class Index;
+class FileDownload;
class Path;
class Remote;
class TiXmlElement;
@@ -41,7 +41,7 @@ class Index : public std::enable_shared_from_this<const Index> {
public:
static Path pathFor(const std::string &name);
static IndexPtr load(const std::string &name, const char *data = nullptr);
- static Download *fetch(const Remote &, bool stale, const NetworkOpts &);
+ static FileDownload *fetch(const Remote &, bool stale, const NetworkOpts &);
Index(const std::string &name);
~Index();
diff --git a/src/path.cpp b/src/path.cpp
@@ -234,3 +234,9 @@ UseRootPath::~UseRootPath()
{
Path::s_root = move(m_backup);
}
+
+TempPath::TempPath(const Path &target)
+ : m_target(target), m_temp(target)
+{
+ m_temp[m_temp.size() - 1] += ".part";
+}
diff --git a/src/path.hpp b/src/path.hpp
@@ -79,4 +79,16 @@ private:
Path m_backup;
};
+class TempPath {
+public:
+ TempPath(const Path &target);
+
+ const Path &target() const { return m_target; }
+ const Path &temp() const { return m_temp; }
+
+private:
+ Path m_target;
+ Path m_temp;
+};
+
#endif
diff --git a/src/reapack.cpp b/src/reapack.cpp
@@ -322,7 +322,7 @@ void ReaPack::fetchIndexes(const vector<Remote> &remotes,
void ReaPack::doFetchIndex(const Remote &remote, ThreadPool *pool,
HWND parent, const bool stale)
{
- Download *dl = Index::fetch(remote, stale, m_config->network);
+ FileDownload *dl = Index::fetch(remote, stale, m_config->network);
if(!dl)
return;
@@ -343,16 +343,14 @@ void ReaPack::doFetchIndex(const Remote &remote, ThreadPool *pool,
};
dl->onFinish([=] {
- const Path &path = Index::pathFor(remote.name());
-
switch(dl->state()) {
case ThreadTask::Success:
- if(!FS::write(path, dl->contents()))
+ if(!FS::rename(dl->path()))
warn(FS::lastError(), AUTO_STR("Write Failed"));
break;
case ThreadTask::Failure:
- if(stale || !FS::exists(path))
- warn(dl->contents(), AUTO_STR("Download Failed"));
+ if(stale || !FS::exists(dl->path().target()))
+ warn(dl->error().message, AUTO_STR("Download Failed"));
break;
default:
break;
diff --git a/src/task.cpp b/src/task.cpp
@@ -62,36 +62,38 @@ bool InstallTask::start()
}
for(const Source *src : m_version->sources()) {
- const NetworkOpts &opts = tx()->config()->network;
- Download *dl = new Download(src->fullName(), src->url(), opts);
- dl->onFinish(bind(&InstallTask::saveSource, this, dl, src));
+ const Path &targetPath = src->targetPath();
- m_waiting.insert(dl);
- tx()->threadPool()->push(dl);
+ const auto old = find_if(m_oldFiles.begin(), m_oldFiles.end(),
+ [&](const Registry::File &f) { return f.path == targetPath; });
+
+ if(old != m_oldFiles.end())
+ m_oldFiles.erase(old);
+
+ // if(m_reader) {
+ // }
+ // else {
+ const NetworkOpts &opts = tx()->config()->network;
+ FileDownload *dl = new FileDownload(targetPath, src->url(), opts);
+ push(dl, dl->path());
+ // }
}
return true;
}
-void InstallTask::saveSource(Download *dl, const Source *src)
+void InstallTask::push(ThreadTask *job, const TempPath &path)
{
- m_waiting.erase(dl);
-
- const Path &targetPath = src->targetPath();
-
- const auto old = find_if(m_oldFiles.begin(), m_oldFiles.end(),
- [&](const Registry::File &f) { return f.path == targetPath; });
+ job->onStart([=] { m_newFiles.push_back(path); });
+ job->onFinish([=] {
+ m_waiting.erase(job);
- if(old != m_oldFiles.end())
- m_oldFiles.erase(old);
-
- Path tmpPath(targetPath);
- tmpPath[tmpPath.size() - 1] += ".new";
+ if(job->state() != ThreadTask::Success)
+ rollback();
+ });
- if(tx()->saveFile(dl, tmpPath))
- m_newFiles.push_back({targetPath, tmpPath});
- else
- rollback();
+ m_waiting.insert(job);
+ tx()->threadPool()->push(job);
}
void InstallTask::commit()
@@ -99,15 +101,10 @@ void InstallTask::commit()
if(m_fail)
return;
- for(const PathGroup &paths : m_newFiles) {
-#ifdef _WIN32
- // TODO: rename to .old
- FS::remove(paths.target);
-#endif
-
- if(!FS::rename(paths.temp, paths.target)) {
+ for(const TempPath &paths : m_newFiles) {
+ if(!FS::rename(paths)) {
tx()->receipt()->addError({"Cannot rename to target: " + FS::lastError(),
- paths.target.join()});
+ paths.target().join()});
// it's a bit late to rollback here as some files might already have been
// overwritten. at least we can delete the temporary files
@@ -145,8 +142,8 @@ void InstallTask::commit()
void InstallTask::rollback()
{
- for(const PathGroup &paths : m_newFiles)
- FS::removeRecursive(paths.temp);
+ for(const TempPath &paths : m_newFiles)
+ FS::removeRecursive(paths.temp());
for(ThreadTask *job : m_waiting)
job->abort();
diff --git a/src/task.hpp b/src/task.hpp
@@ -25,7 +25,6 @@
#include <unordered_set>
#include <vector>
-class Download;
class Index;
class Source;
class ThreadTask;
@@ -62,18 +61,16 @@ public:
void rollback() override;
private:
- struct PathGroup { Path target; Path temp; };
-
- void saveSource(Download *, const Source *);
+ void push(ThreadTask *, const TempPath &);
const Version *m_version;
bool m_pin;
Registry::Entry m_oldEntry;
bool m_fail;
IndexPtr m_index; // keep in memory
- std::unordered_set<ThreadTask *> m_waiting;
std::vector<Registry::File> m_oldFiles;
- std::vector<PathGroup> m_newFiles;
+ std::vector<TempPath> m_newFiles;
+ std::unordered_set<ThreadTask *> m_waiting;
};
class UninstallTask : public Task {
diff --git a/src/transaction.cpp b/src/transaction.cpp
@@ -118,7 +118,7 @@ void Transaction::synchronize(const Package *pkg, const InstallOpts &opts)
void Transaction::fetchIndex(const Remote &remote, const function<void()> &cb)
{
- Download *dl = Index::fetch(remote, true, m_config->network);
+ FileDownload *dl = Index::fetch(remote, true, m_config->network);
if(!dl) {
// the index was last downloaded less than a few seconds ago
@@ -127,8 +127,13 @@ void Transaction::fetchIndex(const Remote &remote, const function<void()> &cb)
}
dl->onFinish([=] {
- if(saveFile(dl, Index::pathFor(remote.name())))
+ if(dl->state() != ThreadTask::Success)
+ return;
+
+ if(FS::rename(dl->path()))
cb();
+ else
+ m_receipt.addError({FS::lastError(), dl->path().target().join()});
});
m_threadPool.push(dl);
@@ -176,18 +181,6 @@ void Transaction::uninstall(const Registry::Entry &entry)
m_nextQueue.push(make_shared<UninstallTask>(entry, this));
}
-bool Transaction::saveFile(Download *dl, const Path &path)
-{
- if(dl->state() == ThreadTask::Success) {
- if(FS::write(path, dl->contents()))
- return true;
- else
- m_receipt.addError({FS::lastError(), path.join()});
- }
-
- return false;
-}
-
bool Transaction::runTasks()
{
if(!m_nextQueue.empty()) {
diff --git a/src/transaction.hpp b/src/transaction.hpp
@@ -70,7 +70,6 @@ public:
void registerAll(bool add, const Registry::Entry &);
void registerFile(const HostTicket &t) { m_regQueue.push(t); }
- bool saveFile(Download *, const Path &);
private:
class CompareTask {
diff --git a/test/path.cpp b/test/path.cpp
@@ -236,3 +236,10 @@ TEST_CASE("append full paths") {
REQUIRE(a == Path("a/b/c/d/e/f"));
}
+
+TEST_CASE("temporary path") {
+ TempPath a(Path("hello/world"));
+
+ REQUIRE(a.target() == Path("hello/world"));
+ REQUIRE(a.temp() == Path("hello/world.part"));
+}
