Commit 114e1c5d authored by Marco Amagliani's avatar Marco Amagliani
Browse files

Added a cache for the DoCloud optimization results.

The cache is active by default and it is in folder \\camfs1\General_access\Martin_Marinov\ReForm\Cache .
The cache directory can be changed using the environment variable "ReFormCacheDir" for example 
set ReFormCacheDir \\camfs1\ASM\Users\Marco_Amagliani\reform\
Details:
 - the data defining the optimization problem (problem_data) to be sent to DoCloud are now stored in memory, not in a file.
 - these data are used to create a key_name. The cache system looks for a file key_name.lp . If it exists and the content of key_name.lp is the same as problem_data , the result of the optimization is retrieved in a file key_name.dat.
  - if the cache does not have an entry for prblem_data, the data are sent to DoCloud to be solved and then we store in key_name.lp as and key_name.dat in the cache to be reused the next time.
It is expected more than 50% improvement on test execution (when the cache data are found).

[git-p4: depot-paths = "//ReForm/ReForm/main/CoMISo/": change = 12123]
parent 08902a05
......@@ -5,6 +5,7 @@ SET(my_headers
${CMAKE_CURRENT_SOURCE_DIR}/CBCSolver.hh
${CMAKE_CURRENT_SOURCE_DIR}/CPLEXSolver.hh
${CMAKE_CURRENT_SOURCE_DIR}/DOCloudSolver.hh
${CMAKE_CURRENT_SOURCE_DIR}/DOCloudCache.hh
${CMAKE_CURRENT_SOURCE_DIR}/GurobiHelper.hh
${CMAKE_CURRENT_SOURCE_DIR}/GUROBISolver.hh
${CMAKE_CURRENT_SOURCE_DIR}/IPOPTSolver.hh
......@@ -45,6 +46,7 @@ SET(my_sources
${CMAKE_CURRENT_SOURCE_DIR}/CBCSolver.cc
${CMAKE_CURRENT_SOURCE_DIR}/CPLEXSolver.cc
${CMAKE_CURRENT_SOURCE_DIR}/DOCloudSolver.cc
${CMAKE_CURRENT_SOURCE_DIR}/DOCloudCache.cc
${CMAKE_CURRENT_SOURCE_DIR}/GurobiHelper.cc
${CMAKE_CURRENT_SOURCE_DIR}/GUROBISolver.cc
${CMAKE_CURRENT_SOURCE_DIR}/IPOPTSolver.cc
......
//== INCLUDES =================================================================
//== COMPILE-TIME PACKAGE REQUIREMENTS ========================================
#include <CoMISo/Config/config.hh>
#if COMISO_DOCLOUD_AVAILABLE
//=============================================================================
#include "DOCloudCache.hh"
#include <Base/Utils/OutcomeUtils.hh>
#include <Base/Debug/DebOut.hh>
#include <fstream>
#include <iomanip>
#include <cctype>
#include <functional>
#include <boost/filesystem.hpp>
#include <windows.h>
DEB_module("DOCloudCache")
//== NAMESPACES ===============================================================
namespace COMISO {
namespace DOcloud {
namespace {
const std::string& cache_directory()
{
static std::string cache_dir =
"\\\\camfs1\\General_access\\Martin_Marinov\\ReForm\\Cache\\";
static bool already_read = false;
if (!already_read)
{
already_read = true;
const char* env_cache_dir = getenv("ReFormCacheDir");
if (env_cache_dir != nullptr && env_cache_dir[0] != 0)
{
cache_dir = env_cache_dir;
if (cache_dir.back() != '\\')
cache_dir += '\\'; // Eventually add '\' to the directory string.
}
}
return cache_dir;
}
//
std::string full_cache_filename(const std::string& _filename)
{
return cache_directory() + _filename;
}
// Create a new temporary exclusive file without extension that is used to
// prevent write or read operation on files with the same name and extension
// .lp or .dat. while the cache is being written. This is the only class that
// uses windows specific APIs.
class FileLock
{
public:
FileLock(const std::string& _filename)
{
file_hnd_ = CreateFile(_filename.c_str(),
GENERIC_WRITE,
0, // ShareMode - 0 prevents any sharing
nullptr, // SecurityAttributes
CREATE_NEW, // Fails if file already exists.
FILE_ATTRIBUTE_NORMAL | FILE_FLAG_DELETE_ON_CLOSE, // File attributes.
NULL);
}
// We can write the DoCloud results only id the lock is successful.
bool sucess() const { return file_hnd_ != INVALID_HANDLE_VALUE; }
// If there is an active lock we can not read the related data because they
// are being written.
static bool active(const std::string& _filename)
{
return GetFileAttributes(_filename.c_str()) != INVALID_FILE_ATTRIBUTES;
}
// The destructor removes the lock. from this moment the data can be freely
// read and there should not be anyone who tries to rewrite them.
~FileLock()
{
if (sucess())
CloseHandle(file_hnd_); // This will delete the file.
}
private:
HANDLE file_hnd_;
};
bool load_file(const std::string& _filename, std::string& _file_cnts)
{
std::ifstream in_file_strm(_filename, std::ios::ate);
if (!in_file_strm.is_open())
return false;
_file_cnts.reserve(in_file_strm.tellg());
in_file_strm.seekg(0, std::ios::beg);
_file_cnts.assign(std::istreambuf_iterator<char>(in_file_strm),
std::istreambuf_iterator<char>());
return true;
}
bool save_file(const std::string& _filename, const std::string& _file_cnts)
{
std::ofstream out_file_strm(_filename);
if (!out_file_strm.is_open())
return false;
out_file_strm << _file_cnts;
return true;
}
// Finds a key string from the file name. This string will be used as file name
// where to store the related cached data.
void string_to_key(const std::string& _str, std::string& _key)
{
// 1. Writes the file length.
const std::hash<std::string> hash_fn;
_key = std::to_string(hash_fn(_str));
}
// Load variables and objective values from a file.
bool load_data(const std::string& _filename,
std::vector<double>& _x, double& _obj_val)
{
std::ifstream in_file_strm(_filename);
if (!in_file_strm.is_open())
return false;
size_t dim = std::numeric_limits<size_t>::max();
in_file_strm >> dim;
if (dim != _x.size())
return false;
for (auto& xi : _x)
in_file_strm >> xi;
in_file_strm >> _obj_val;
return !in_file_strm.bad();
}
// Store variables and objective values in a file.
bool save_data(const std::string& _filename,
const std::vector<double>& _x, const double& _obj_val)
{
std::ofstream out_file_strm(_filename);
out_file_strm << std::setprecision(std::numeric_limits<double>::digits10 + 2);
if (!out_file_strm.is_open())
return false;
out_file_strm << _x.size() << std::endl;
for (const auto& xi : _x)
out_file_strm << xi << std::endl;;
out_file_strm << _obj_val;
return !out_file_strm.bad();
}
} // namespace
bool Cache::restore_result(
std::string& _lp_prbl_str, // string containing the .lp problem
std::vector<double>& _x, // result.
double& _obj_val) // objective function value.
{
found_ = false;
lp_file_cnts_ = std::move(_lp_prbl_str);
if (cache_directory().empty())
return false;
string_to_key(lp_file_cnts_, key_);
key_ += '_';
for (size_t iter_nmbr = 0; iter_nmbr < 10; ++iter_nmbr)
{
last_filename_ = full_cache_filename(key_) + std::to_string(iter_nmbr);
std::string dat_filename(last_filename_ + ".dat");
boost::system::error_code err_cod;
if (!boost::filesystem::exists(
boost::filesystem::path(dat_filename.c_str()), err_cod) ||
err_cod.value() != boost::system::errc::success)
{
// If the .dat file does not exist it is not safe to check the lock because
// it is possible that this process finds no lock, another process sets the
// lock and start writing data, this process reads not fully written data.
break;
}
if (FileLock::active(last_filename_))
break;
std::string cache_cnts;
if (!load_file(last_filename_ + ".lp", cache_cnts))
break;
if (cache_cnts == lp_file_cnts_)
{
found_ = load_data(last_filename_ + ".dat", _x, _obj_val);
return found_;
}
}
return false;
}
namespace {
// Save the couple of files fname.lp and fname.dat . They are an element of a
// sort of map from file.lp to file.dat. So if there is an error saving the .dat
// file, the .lp file must also e deleted.
class CacheSaver
{
public:
CacheSaver() : success_(false) {}
~CacheSaver()
{
if (success_)
return;
// Removes files eventually written if there has been any kind of failure.
for (const auto& f_name : used_files_)
{
if (!f_name.empty())
std::remove(f_name.c_str());
}
}
void save(const std::string& _filename, const std::vector<double>& _x,
const double& _obj_val, const std::string& _lp_cnts)
{
DEB_enter_func;
FileLock file_lock(_filename);
if (file_lock.sucess())
{
used_files_[0] = _filename + ".lp";
success_ = save_file(used_files_[0], _lp_cnts);
if (success_)
{
used_files_[1] = _filename + ".dat";
success_ = save_data(used_files_[1], _x, _obj_val);
}
}
}
private:
bool success_;
std::string used_files_[2];
};
} // namespace
void
Cache::store_result(const std::vector<double>& _x, const double& _obj_val)
{
DEB_enter_func;
THROW_OUTCOME_if(found_, TODO); /* Multiple store of Docloud cache value. */
if (!last_filename_.empty())
{
CacheSaver saver;
saver.save(last_filename_, _x, _obj_val, lp_file_cnts_);
}
}
} // namespace DOcloud
} // namespace COMISO
#endif // COMISO_DOCLOUD_AVAILABLE
//=============================================================================
#ifndef COMISO_DOCloudCache_HH
#define COMISO_DOCloudCache_HH
//== COMPILE-TIME PACKAGE REQUIREMENTS ========================================
#include <CoMISo/Config/config.hh>
#if COMISO_DOCLOUD_AVAILABLE
//== INCLUDES =================================================================
#include <string>
#include <vector>
//== NAMESPACES ===============================================================
namespace COMISO {
namespace DOcloud {
// given a .lp file, checks if we have saved the result for the same problem.
// If so they are returned, so the calling function can avoid to compute them.
//
class Cache
{
public:
Cache() : found_(false) {}
bool restore_result(
std::string& _file_name, // .lp file defining the optimization problem
std::vector<double>& _x, // result.
double& _obj_val); // objective function value.
// We can store the result for the given .lp file. This makes sense only we have
// not found cache data for the given .lp file, and in order to avoid data
// corruption this function fails if the data have been found.
void store_result(const std::vector<double>& _x, const double& _obj_val);
const std::string& get_lp_content() { return lp_file_cnts_; }
private:
std::string key_; // String generated from .lp file content data.
// This is a sort of hash key generate form the .lp
// file content.
std::string last_filename_;// Last name we have tried to get cached data.
std::string lp_file_cnts_; // Content of the input .lp file.
bool found_; // Remembers if we have found a cache for the input
// .lp file.
};
} // namespace DOcloud
} // namespace COMISO
//=============================================================================
#endif // COMISO_DOCLOUD_AVAILABLE
//=============================================================================
#endif // COMISO_DOCloudCache_HH
//=============================================================================
......@@ -15,6 +15,7 @@
//=============================================================================
#include "DOCloudSolver.hh"
#include "DOCloudCache.hh"
#include <Base/Debug/DebTime.hh>
#include <Base/Debug/DebUtils.hh>
......@@ -43,8 +44,6 @@ namespace COMISO {
//== IMPLEMENTATION ==========================================================
namespace {
namespace cURLpp { // some classes to wrap around the libcurl C data
struct Session
......@@ -201,22 +200,11 @@ private:
class Upload : public Request
{
public:
Upload(const char* _filename) : filename_(_filename), file_(nullptr) {}
Upload(const std::string& _filename) : filename_(_filename), file_(nullptr) {}
virtual ~Upload()
{
if (file_ != nullptr)
std::fclose(file_);
}
protected:
virtual void prepare();
virtual void finalize();
private:
std::string filename_;
FILE* file_;
virtual size_t send_data() = 0;
};
void Upload::prepare()
......@@ -224,13 +212,9 @@ void Upload::prepare()
/* tell it to "upload" to the URL */
curl_easy_setopt(hnd_, CURLOPT_UPLOAD, 1L);
/* set where to read from (on Windows you need to use READFUNCTION too) */
file_ = std::fopen(filename_.data(), "rb");
curl_easy_setopt(hnd_, CURLOPT_READDATA, file_);
size_t data_len = send_data();
/* and give the size of the upload (optional) */
const auto filelen = _filelength(fileno(file_));
curl_easy_setopt(hnd_, CURLOPT_INFILESIZE_LARGE, (curl_off_t)filelen);
curl_easy_setopt(hnd_, CURLOPT_INFILESIZE_LARGE, (curl_off_t)data_len);
/* we want to use our own read function */
//curl_easy_setopt(hnd_, CURLOPT_READFUNCTION, read_func);
......@@ -254,6 +238,77 @@ void Upload::finalize()
"Upload speed: " << rate / 1024. << "Kbps; Time: " << time << "s.");
}
class UploadFile : public Upload
{
public:
UploadFile(const char* _filename) : filename_(_filename) {}
UploadFile(const std::string& _filename) : filename_(_filename) {}
virtual ~UploadFile()
{
if (file_ != nullptr)
std::fclose(file_);
}
protected:
virtual size_t send_data()
{
/* set where to read from (on Windows you need to use READFUNCTION too) */
file_ = std::fopen(filename_.data(), "rb");
curl_easy_setopt(hnd_, CURLOPT_READDATA, file_);
return _filelength(fileno(file_));
}
private:
std::string filename_;
FILE* file_;
};
class UploadData : public Upload
{
public:
UploadData(const std::string& _data) : file_dat_(_data) { }
protected:
virtual size_t send_data()
{
curl_easy_setopt(hnd_, CURLOPT_READFUNCTION, Buffer::copy);
curl_easy_setopt(hnd_, CURLOPT_READDATA, &file_dat_);
return file_dat_.len_;
}
private:
class Buffer
{
public:
Buffer(const std::string& _data)
: ptr_(_data.c_str()), len_(_data.size()), pos_(0) {}
static size_t copy(void* _target, const size_t _elem_size,
const size_t _n_elem, void* _from_buf);
const char* ptr_;
size_t len_;
private:
size_t pos_;
};
Buffer file_dat_;
};
size_t UploadData::Buffer::copy(void* _target, const size_t _elem_size,
const size_t _n_elem, void* _from_buf)
{
auto dat = reinterpret_cast<Buffer *>(_from_buf);
size_t char_to_write = dat->len_ - dat->pos_;
size_t buffer_len = _elem_size * _n_elem;
if (char_to_write > buffer_len)
char_to_write = buffer_len;
std::copy(dat->ptr_ + dat->pos_, dat->ptr_ + dat->pos_ + char_to_write,
(char*)_target);
dat->pos_ += char_to_write;
return char_to_write;
}
class Get : public Request
{
protected:
......@@ -269,7 +324,8 @@ protected:
}
};
}//cURLpp
} // namespace cURLpp
namespace DOcloud {
......@@ -451,10 +507,33 @@ private:
int sol_sec_nmbr_; // number of seconds at the last new solution
int stld_sec_nmbr_; // number of seconds since the last new solution
private:
protected:
void make();
void upload();
void start();
void upload_internal(cURLpp::Upload& _upload);
virtual void upload()
{
cURLpp::UploadFile upl(filename_);
upload_internal(upl);
}
};
class JobWithDat : public Job
{
public:
JobWithDat(const char* _filename, const std::string& _file_dat)
: Job(_filename), file_dat_(_file_dat) {}
private:
const std::string& file_dat_; // data to be sent as data attachment.
protected:
virtual void upload()
{
cURLpp::UploadData upl_dat(file_dat_);
upload_internal(upl_dat);
}
};
Job::~Job()
......@@ -498,17 +577,16 @@ void Job::make()
THROW_OUTCOME_if(!hdr_tkns.find_value("Location:", url_), TODO);
}
void Job::upload()
void Job::upload_internal(cURLpp::Upload& _upload)
{
cURLpp::Upload upload(filename_);
THROW_OUTCOME_if(!upload.valid(), TODO); //Failed to initialize the request
THROW_OUTCOME_if(!_upload.valid(), TODO); //Failed to initialize the request
auto url = url_ + "/attachments/" + filename_ + "/blob";
upload.set_url(url.data());
upload.add_http_header(api_key__.c_str());
upload.perform();
HeaderTokens hdr_tkns(upload.header());
check_http_error(upload, hdr_tkns, 204);
_upload.set_url(url.data());
_upload.add_http_header(api_key__.c_str());
_upload.perform();
HeaderTokens hdr_tkns(_upload.header());
check_http_error(_upload, hdr_tkns, 204);
}
void Job::start()
......@@ -728,8 +806,7 @@ double Job::solution(std::vector<double>& _x) const
return obj_val;
}
} // namespace DOcloud
namespace {
std::string lp_file_name()
{
......@@ -751,14 +828,14 @@ class WriteExpression
enum { LINE_TRESHOLD_LEN = 100 };
public:
WriteExpression(std::ofstream& _out_str) : out_str_(_out_str)
WriteExpression(std::ostringstream& _out_str) : out_str_stream(_out_str)
{
start();
}
void start()
{
f_size_ = out_str_.tellp();
f_size_ = out_str_stream.tellp();
at_start_ = true;
}
......@@ -778,9 +855,9 @@ public:
return;
add_monomial_internal(_coeff, _i_var);
if (_j_var == _i_var)
out_str_ << "^2";
out_str_stream << "^2";
else
out_str_ << " * " << XVAR(_j_var);
out_str_stream << " * " << XVAR(_j_var);
wrap_long_line();
}
......@@ -788,10 +865,10 @@ private:
void wrap_long_line()
{
const auto new_f_size = out_str_.tellp();
const auto new_f_size = out_str_stream.tellp();
if (new_f_size - f_size_ > LINE_TRESHOLD_LEN)
{
out_str_ << std::endl;
out_str_stream << std::endl;
f_size_ = new_f_size;
}
}
......@@ -801,27 +878,27 @@ private:
if (_coeff == 1)
{
if (!at_start_)
out_str_ << " + ";
out_str_stream << " + ";
}
else if (_coeff == -1)
out_str_ << " - ";
out_str_stream << " - ";
else
{
if (!at_start_)
{
if (_coeff > 0)
out_str_ << " + ";
out_str_stream << " + ";
else
out_str_ << ' ';
out_str_stream << ' ';
}
out_str_ << _coeff << ' ';
out_str_stream << _coeff << ' ';
}
out_str_ << XVAR(_i_var);
out_str_stream << XVAR(_i_var);
at_start_ = false;
}
private:
std::ofstream& out_str_;
std::ostringstream& out_str_stream;
std::fstream::pos_type f_size_;