From 9bfc40f5aafb20e167c6451f2f6ab8bf7a42e179 Mon Sep 17 00:00:00 2001 From: Jonathan Kunstwald Date: Tue, 21 May 2019 14:45:37 +0200 Subject: [PATCH 1/2] Add FileWatcher to replace timestamp polling --- src/glow/common/file_watcher.cc | 444 ++++++++++++++++++++++++++++++++ src/glow/common/file_watcher.hh | 99 +++++++ src/glow/objects/Shader.cc | 31 ++- src/glow/objects/Shader.hh | 15 +- 4 files changed, 570 insertions(+), 19 deletions(-) create mode 100644 src/glow/common/file_watcher.cc create mode 100644 src/glow/common/file_watcher.hh diff --git a/src/glow/common/file_watcher.cc b/src/glow/common/file_watcher.cc new file mode 100644 index 0000000..f1d3319 --- /dev/null +++ b/src/glow/common/file_watcher.cc @@ -0,0 +1,444 @@ +// MIT License +// +// Copyright(c) 2017 Thomas Monkman +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files(the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions : +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +#include "file_watcher.hh" + +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include + +#include +#include +#include + +#include +#include +#endif // WIN32 + +#ifdef __unix__ +#include +#include +#include +#include +#include +#include +#include +#endif // __unix__ + +#if !defined(_WIN32) && !defined(__unix__) +#error "Unimplemented platform!" +#endif + +namespace +{ +static std::size_t constexpr sBufferSize = 1024 * 256; + +struct PathParts +{ + std::string directory; + std::string filename; + + PathParts(std::string const& directory, std::string const& filename) : directory(directory), filename(filename) {} +}; + +const PathParts splitDirectoryAndFile(const std::string& path) +{ + const auto predict = [](char character) { +#ifdef _WIN32 + return character == _T('\\') || character == _T('/'); +#elif __unix__ + return character == '/'; +#endif // __unix__ + }; + + const auto pivot = std::find_if(path.rbegin(), path.rend(), predict).base(); + // if the path is something like "test.txt" there will be no directoy part, however we still need one, so insert './' + const std::string directory = [&]() { + const auto extracted_directory = std::string(path.begin(), pivot); + return (extracted_directory.size() > 0) ? extracted_directory : "./"; + }(); + const std::string filename = std::string(pivot, path.end()); + return PathParts(directory, filename); +} + +#ifdef _WIN32 +DWORD const sListenFilters = FILE_NOTIFY_CHANGE_SECURITY | FILE_NOTIFY_CHANGE_CREATION | FILE_NOTIFY_CHANGE_LAST_ACCESS | FILE_NOTIFY_CHANGE_LAST_WRITE + | FILE_NOTIFY_CHANGE_SIZE | FILE_NOTIFY_CHANGE_ATTRIBUTES | FILE_NOTIFY_CHANGE_DIR_NAME | FILE_NOTIFY_CHANGE_FILE_NAME; + +std::map const sEventTypeMapping = {{FILE_ACTION_ADDED, glow::FileWatcher::Event::Added}, + {FILE_ACTION_REMOVED, glow::FileWatcher::Event::Removed}, + {FILE_ACTION_MODIFIED, glow::FileWatcher::Event::Modified}, + {FILE_ACTION_RENAMED_OLD_NAME, glow::FileWatcher::Event::RenamedOld}, + {FILE_ACTION_RENAMED_NEW_NAME, glow::FileWatcher::Event::RenamedNew}}; + +inline HANDLE handleToNative(glow::FileWatcher::WinHandle const& wrap) +{ + HANDLE res; + memcpy(&res, &wrap.data, sizeof(HANDLE)); + return res; +} + +inline glow::FileWatcher::WinHandle handleWrap(HANDLE native) +{ + glow::FileWatcher::WinHandle res; + memcpy(&res.data, &native, sizeof(HANDLE)); + return res; +} + +#elif defined __unix__ +std::size_t constexpr sEventSize = (sizeof(struct inotify_event)); + +bool is_file(const std::string& path) const +{ + struct stat statbuf = {}; + if (stat(path.c_str(), &statbuf) != 0) + { + throw std::system_error(errno, std::system_category()); + } + return S_ISREG(statbuf.st_mode); +} +#endif +} + +glow::FileWatcher::FileWatcher(const std::string& path, std::regex pattern, callback_t callback) + : mPath(path), mPattern(pattern), mCallback(callback), mDirectory(getDirectory(path)) +{ + init(); +} + +glow::FileWatcher::FileWatcher(const std::string& path, callback_t callback) : FileWatcher(path, std::regex(".*"), callback) {} + +glow::FileWatcher::~FileWatcher() +{ + mDestroy.store(true); + mRunning = std::promise(); + +#ifdef _WIN32 + SetEvent(handleToNative(mCloseEvent)); +#elif __unix__ + inotify_rm_watch(mDirectory.folder, mDirectory.watch); +#endif // __unix__ + + mCv.notify_all(); + mWatchThread.join(); + mCallbackThread.join(); + +#ifdef _WIN32 + CloseHandle(handleToNative(mDirectory)); +#elif __unix__ + close(mDirectory.folder); +#endif // __unix__ +} + +void glow::FileWatcher::init() +{ +#ifdef _WIN32 + auto closeEvent = CreateEvent(nullptr, TRUE, FALSE, nullptr); + if (!closeEvent) + throw std::system_error(static_cast(GetLastError()), std::system_category()); + else + mCloseEvent = handleWrap(closeEvent); +#endif // WIN32 + mCallbackThread = std::thread([this]() { + try + { + callbackThread(); + } + catch (...) + { + try + { + mRunning.set_exception(std::current_exception()); + } + catch (...) + { + } // set_exception() may throw too + } + }); + mWatchThread = std::thread([this]() { + try + { + monitorDirectory(); + } + catch (...) + { + try + { + mRunning.set_exception(std::current_exception()); + } + catch (...) + { + } // set_exception() may throw too + } + }); + + std::future future = mRunning.get_future(); + future.get(); // block until the monitor_directory is up and running +} + +bool glow::FileWatcher::passFilter(const std::string& file_path) +{ + if (mWatchingSingleFile) + { + const std::string extracted_filename = {splitDirectoryAndFile(file_path).filename}; + // if we are watching a single file, only that file should trigger action + return extracted_filename == mFilename; + } + return std::regex_match(file_path, mPattern); +} + +#ifdef _WIN32 +glow::FileWatcher::WinHandle glow::FileWatcher::getDirectory(const std::string& path) +{ + auto file_info = GetFileAttributes(path.c_str()); + + if (file_info == INVALID_FILE_ATTRIBUTES) + { + throw std::system_error(static_cast(GetLastError()), std::system_category()); + } + mWatchingSingleFile = (file_info & FILE_ATTRIBUTE_DIRECTORY) == false; + + const std::string watch_path = [this, &path]() { + if (mWatchingSingleFile) + { + const auto parsed_path = splitDirectoryAndFile(path); + mFilename = parsed_path.filename; + return parsed_path.directory; + } + else + { + return path; + } + }(); + + HANDLE directory = ::CreateFile(watch_path.c_str(), // pointer to the file name + FILE_LIST_DIRECTORY, // access (read/write) mode + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, // share mode + nullptr, // security descriptor + OPEN_EXISTING, // how to create + FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OVERLAPPED, // file attributes + nullptr); // file with attributes to copy + + if (directory == INVALID_HANDLE_VALUE) + { + throw std::system_error(static_cast(GetLastError()), std::system_category()); + } + return handleWrap(directory); +} +#elif defined __unix__ +FolderInfo getDirectory(const std::string& path) +{ + const auto folder = inotify_init(); + if (folder < 0) + { + throw std::system_error(errno, std::system_category()); + } + mWatchingSingleFile = is_file(path); + + const std::string watch_path = [this, &path]() { + if (mWatchingSingleFile) + { + const auto parsed_path = splitDirectoryAndFile(path); + mFilename = parsed_path.filename; + return parsed_path.directory; + } + else + { + return path; + } + }(); + + const auto watch = inotify_add_watch(folder, watch_path.c_str(), IN_MODIFY | IN_CREATE | IN_DELETE); + if (watch < 0) + { + throw std::system_error(errno, std::system_category()); + } + return {folder, watch}; +} +#endif + +void glow::FileWatcher::monitorDirectory() +{ +#ifdef _WIN32 + std::vector buffer(sBufferSize); + DWORD bytes_returned = 0; + OVERLAPPED overlapped_buffer{0}; + + overlapped_buffer.hEvent = CreateEvent(nullptr, TRUE, FALSE, nullptr); + if (!overlapped_buffer.hEvent) + { + std::cerr << "Error creating monitor event" << std::endl; + } + + std::array handles{overlapped_buffer.hEvent, handleToNative(mCloseEvent)}; + + auto async_pending = false; + mRunning.set_value(); + do + { + std::vector> parsed_information; + ReadDirectoryChangesW(handleToNative(mDirectory), buffer.data(), static_cast(buffer.size()), TRUE, sListenFilters, &bytes_returned, + &overlapped_buffer, nullptr); + + async_pending = true; + + switch (WaitForMultipleObjects(2, handles.data(), FALSE, INFINITE)) + { + case WAIT_OBJECT_0: + { + if (!GetOverlappedResult(handleToNative(mDirectory), &overlapped_buffer, &bytes_returned, TRUE)) + { + throw std::system_error(static_cast(GetLastError()), std::system_category()); + } + async_pending = false; + + if (bytes_returned == 0) + { + break; + } + + FILE_NOTIFY_INFORMATION* file_information = reinterpret_cast(&buffer[0]); + do + { + std::wstring wideString = std::wstring(file_information->FileName, file_information->FileNameLength / 2); + std::wstring_convert> converter; + std::string changed_file = converter.to_bytes(wideString); + if (passFilter(changed_file)) + { + parsed_information.emplace_back(changed_file, sEventTypeMapping.at(file_information->Action)); + } + + if (file_information->NextEntryOffset == 0) + { + break; + } + + file_information = reinterpret_cast(reinterpret_cast(file_information) + file_information->NextEntryOffset); + } while (true); + break; + } + case WAIT_OBJECT_0 + 1: + // quit + break; + case WAIT_FAILED: + break; + } + // dispatch callbacks + { + std::lock_guard lock(mCallbackMutex); + mCallbackInformation.insert(mCallbackInformation.end(), parsed_information.begin(), parsed_information.end()); + } + mCv.notify_all(); + } while (!mDestroy.load()); + + if (async_pending) + { + // clean up running async io + CancelIo(handleToNative(mDirectory)); + GetOverlappedResult(handleToNative(mDirectory), &overlapped_buffer, &bytes_returned, TRUE); + } +#elif defined __unix__ + std::vector buffer(sBufferSize); + + _running.set_value(); + while (!mDestroy.load()) + { + const auto length = read(mDirectory.folder, static_cast(buffer.data()), buffer.size()); + if (length > 0) + { + int i = 0; + std::vector> parsed_information; + while (i < length) + { + struct inotify_event* event = reinterpret_cast(&buffer[i]); + if (event->len) + { + const std::string changed_file{event->name}; + if (pass_filter(changed_file)) + { + if (event->mask & IN_CREATE) + { + parsed_information.emplace_back(std::string{changed_file}, Event::Added); + } + else if (event->mask & IN_DELETE) + { + parsed_information.emplace_back(std::string{changed_file}, Event::Removed); + } + else if (event->mask & IN_MODIFY) + { + parsed_information.emplace_back(std::string{changed_file}, Event::Modified); + } + } + } + i += sEventSize + event->len; + } + // dispatch callbacks + { + std::lock_guard lock(mCallbackMutex); + mCallbackInformation.insert(mCallbackInformation.end(), parsed_information.begin(), parsed_information.end()); + } + mCv.notify_all(); + } + } +#endif +} + +void glow::FileWatcher::callbackThread() +{ + while (!mDestroy.load()) + { + std::unique_lock lock(mCallbackMutex); + if (mCallbackInformation.empty() && mDestroy == false) + { + mCv.wait(lock, [this] { return mCallbackInformation.size() > 0 || mDestroy.load(); }); + } + decltype(mCallbackInformation) callback_information = {}; + std::swap(callback_information, mCallbackInformation); + lock.unlock(); + + for (const auto& file : callback_information) + { + if (mCallback) + { + try + { + mCallback(file.first, file.second); + } + catch (const std::exception&) + { + } + } + } + } +} diff --git a/src/glow/common/file_watcher.hh b/src/glow/common/file_watcher.hh new file mode 100644 index 0000000..a7cd5a6 --- /dev/null +++ b/src/glow/common/file_watcher.hh @@ -0,0 +1,99 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "non_copyable.hh" + +namespace glow +{ +/** + * File watching utility, based on + * https://github.com/ThomasMonkman/filewatch + * + * Usage example: + * auto watch = FileWatcher("shaders/geometry.fsh", [&](auto const& path, auto const event) + * { + * if (event == FileWatcher::Event::Modified) + * reload(); + * }); + */ +class FileWatcher +{ +public: + struct WinHandle + { + char data[sizeof(void*)]; + }; + + enum class Event + { + Added, + Removed, + Modified, + RenamedOld, + RenamedNew + }; + + using callback_t = std::function; + + FileWatcher(std::string const& path, std::regex pattern, callback_t callback); + FileWatcher(std::string const& path, callback_t callback); + + ~FileWatcher(); + + GLOW_NON_COPYABLE(FileWatcher); + +private: + std::string const mPath; + std::regex mPattern; + + bool mWatchingSingleFile = false; + std::string mFilename; + + std::atomic_bool mDestroy = {false}; + callback_t mCallback; + + std::thread mWatchThread; + + std::condition_variable mCv; + std::mutex mCallbackMutex; + std::vector> mCallbackInformation; + std::thread mCallbackThread; + + std::promise mRunning; +#ifdef _WIN32 + WinHandle mDirectory; + WinHandle mCloseEvent; +#elif defined __unix__ + struct FolderInfo + { + int folder; + int watch; + }; + + FolderInfo mDirectory; +#endif + + void init(); + + bool passFilter(const std::string& file_path); + +#ifdef _WIN32 + WinHandle getDirectory(const std::string& path); +#elif defined __unix__ + FolderInfo getDirectory(const std::string& path); +#endif + + void monitorDirectory(); + void callbackThread(); +}; +} diff --git a/src/glow/objects/Shader.cc b/src/glow/objects/Shader.cc index 0a8b496..ee7bc4a 100644 --- a/src/glow/objects/Shader.cc +++ b/src/glow/objects/Shader.cc @@ -9,7 +9,6 @@ #include #include -#include #include #include #include @@ -105,7 +104,7 @@ void Shader::reload() return; } setSource(util::readall(file)); - mLastModification = util::fileModificationTime(mFileName); + mChangedOnDisk = false; } // compile @@ -156,6 +155,10 @@ SharedShader Shader::createFromSource(GLenum shaderType, const unsigned char sou return createFromSource(shaderType, ss.str()); } +#include +#include +#include + SharedShader Shader::createFromFile(GLenum shaderType, const std::string& filename) { GLOW_ACTION(); @@ -170,7 +173,10 @@ SharedShader Shader::createFromFile(GLenum shaderType, const std::string& filena auto shader = std::make_shared(shaderType); shader->setObjectLabel(filename); shader->mFileName = filename; - shader->mLastModification = util::fileModificationTime(filename); + shader->mWatcher = std::make_unique(filename, [shader](auto const&, FileWatcher::Event const type) { + if (type == FileWatcher::Event::Modified) + shader->onDiskChange(); + }); shader->setSource(util::readall(shaderFile)); shader->compile(); return shader; @@ -178,17 +184,9 @@ SharedShader Shader::createFromFile(GLenum shaderType, const std::string& filena bool Shader::newerVersionAvailable() { - // check if dependency changed - for (auto const& kvp : mFileDependencies) - if (util::fileModificationTime(kvp.first) > kvp.second) - return true; - // see if file-backed - if (!mFileName.empty() && std::ifstream(mFileName).good()) - { - if (util::fileModificationTime(mFileName) > mLastModification) - return true; - } + if (mWatcher || !mFileDependencies.empty()) + return mChangedOnDisk; return false; } @@ -237,5 +235,10 @@ void Shader::addDependency(const std::string& filename) if (kvp.first == filename) return; - mFileDependencies.push_back(make_pair(filename, util::fileModificationTime(filename))); + mFileDependencies.push_back(make_pair(filename, std::make_unique(filename, [this](auto const&, FileWatcher::Event const type) { + if (type == FileWatcher::Event::Modified) + this->onDiskChange(); + }))); } + +void Shader::onDiskChange() { mChangedOnDisk = true; } diff --git a/src/glow/objects/Shader.hh b/src/glow/objects/Shader.hh index ae92358..04c7f0f 100644 --- a/src/glow/objects/Shader.hh +++ b/src/glow/objects/Shader.hh @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -32,13 +33,15 @@ class Shader final : public NamedObject /// True iff shader has a compiled version bool mCompiled = false; - /// Filepath of this shader (if applicable) + /// Filepath of this shader (if on-disk) std::string mFileName; - /// Last modification of the file (if applicable) - int64_t mLastModification = 0; + /// Watcher (if on-disk) + std::unique_ptr mWatcher; + /// Whether the file on disk has changed since the last call to newerVersionAvailable() + bool mChangedOnDisk = false; - /// Filenames with modification times (dependencies) - std::vector> mFileDependencies; + /// Dependency watchers + std::vector>> mFileDependencies; /// Primary source of the shader std::vector mSources; @@ -83,6 +86,8 @@ public: private: /// Adds a file as dependency void addDependency(std::string const& filename); + /// Signals a file change on disk + void onDiskChange(); public: // static construction /// Creates and compiles (!) a shader from a given source string -- GitLab From 6416f5348876a9dac1e943e3698d478f79b7f95c Mon Sep 17 00:00:00 2001 From: Jonathan Kunstwald Date: Wed, 22 May 2019 13:07:08 +0200 Subject: [PATCH 2/2] Fix unix permutation for file watcher --- src/glow/common/file_watcher.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/glow/common/file_watcher.cc b/src/glow/common/file_watcher.cc index f1d3319..88faa55 100644 --- a/src/glow/common/file_watcher.cc +++ b/src/glow/common/file_watcher.cc @@ -119,7 +119,7 @@ inline glow::FileWatcher::WinHandle handleWrap(HANDLE native) #elif defined __unix__ std::size_t constexpr sEventSize = (sizeof(struct inotify_event)); -bool is_file(const std::string& path) const +bool is_file(const std::string& path) { struct stat statbuf = {}; if (stat(path.c_str(), &statbuf) != 0) @@ -257,7 +257,7 @@ glow::FileWatcher::WinHandle glow::FileWatcher::getDirectory(const std::string& return handleWrap(directory); } #elif defined __unix__ -FolderInfo getDirectory(const std::string& path) +glow::FileWatcher::FolderInfo glow::FileWatcher::getDirectory(const std::string& path) { const auto folder = inotify_init(); if (folder < 0) @@ -371,7 +371,7 @@ void glow::FileWatcher::monitorDirectory() #elif defined __unix__ std::vector buffer(sBufferSize); - _running.set_value(); + mRunning.set_value(); while (!mDestroy.load()) { const auto length = read(mDirectory.folder, static_cast(buffer.data()), buffer.size()); @@ -385,11 +385,11 @@ void glow::FileWatcher::monitorDirectory() if (event->len) { const std::string changed_file{event->name}; - if (pass_filter(changed_file)) + if (passFilter(changed_file)) { if (event->mask & IN_CREATE) { - parsed_information.emplace_back(std::string{changed_file}, Event::Added); + parsed_information.emplace_back(std::string{changed_file}, Event::Modified); } else if (event->mask & IN_DELETE) { -- GitLab