Commit bdf6ed30 authored by Martin Marinov's avatar Martin Marinov
Browse files

ReForm, DOcloud: Added DOCloudConfig, and some minor fixes.

[git-p4: depot-paths = "//ReForm/ReForm/main/CoMISo/": change = 12200]
parent c6a14a95
...@@ -6,6 +6,7 @@ SET(my_headers ...@@ -6,6 +6,7 @@ SET(my_headers
${CMAKE_CURRENT_SOURCE_DIR}/CPLEXSolver.hh ${CMAKE_CURRENT_SOURCE_DIR}/CPLEXSolver.hh
${CMAKE_CURRENT_SOURCE_DIR}/cURLpp.hh ${CMAKE_CURRENT_SOURCE_DIR}/cURLpp.hh
${CMAKE_CURRENT_SOURCE_DIR}/DOCloudCache.hh ${CMAKE_CURRENT_SOURCE_DIR}/DOCloudCache.hh
${CMAKE_CURRENT_SOURCE_DIR}/DOCloudConfig.hh
${CMAKE_CURRENT_SOURCE_DIR}/DOCloudJob.hh ${CMAKE_CURRENT_SOURCE_DIR}/DOCloudJob.hh
${CMAKE_CURRENT_SOURCE_DIR}/DOCloudSolver.hh ${CMAKE_CURRENT_SOURCE_DIR}/DOCloudSolver.hh
${CMAKE_CURRENT_SOURCE_DIR}/GurobiHelper.hh ${CMAKE_CURRENT_SOURCE_DIR}/GurobiHelper.hh
......
...@@ -6,6 +6,8 @@ ...@@ -6,6 +6,8 @@
//============================================================================= //=============================================================================
#include "DOCloudCache.hh" #include "DOCloudCache.hh"
#include "DOCloudConfig.hh"
#include <Base/Utils/OutcomeUtils.hh> #include <Base/Utils/OutcomeUtils.hh>
#include <Base/Debug/DebOut.hh> #include <Base/Debug/DebOut.hh>
...@@ -13,6 +15,7 @@ ...@@ -13,6 +15,7 @@
#include <iomanip> #include <iomanip>
#include <cctype> #include <cctype>
#include <functional> #include <functional>
#include <sstream>
#include <boost/filesystem.hpp> #include <boost/filesystem.hpp>
...@@ -27,31 +30,6 @@ namespace DOcloud { ...@@ -27,31 +30,6 @@ namespace DOcloud {
namespace { namespace {
const std::string& cache_directory()
{
static std::string cache_dir =
"\\\\camfs1\\General_access\\Martin_Marinov\\ReForm\\Cache\\";
static bool already_read = false;
if (!already_read)
{
already_read = true;
const char* env_cache_dir = getenv("ReFormCacheDir");
if (env_cache_dir != nullptr && env_cache_dir[0] != 0)
{
cache_dir = env_cache_dir;
if (cache_dir.back() != '\\')
cache_dir += '\\'; // Eventually add '\' to the directory string.
}
}
return cache_dir;
}
//
std::string full_cache_filename(const std::string& _filename)
{
return cache_directory() + _filename;
}
// Create a new temporary exclusive file without extension that is used to // Create a new temporary exclusive file without extension that is used to
// prevent write or read operation on files with the same name and extension // prevent write or read operation on files with the same name and extension
// .lp or .dat. while the cache is being written. This is the only class that // .lp or .dat. while the cache is being written. This is the only class that
...@@ -116,11 +94,12 @@ bool save_file(const std::string& _filename, const std::string& _file_cnts) ...@@ -116,11 +94,12 @@ bool save_file(const std::string& _filename, const std::string& _file_cnts)
// Finds a key string from the file name. This string will be used as file name // Finds a key string from the file name. This string will be used as file name
// where to store the related cached data. // where to store the related cached data.
void string_to_key(const std::string& _str, std::string& _key) std::string string_to_hash(const std::string& _str)
{ {
// 1. Writes the file length.
const std::hash<std::string> hash_fn; const std::hash<std::string> hash_fn;
_key = std::to_string(hash_fn(_str)); std::stringstream strm;
strm << std::hex << hash_fn(_str);
return strm.str();
} }
const size_t NO_SOLUTION_CODE = UINT_MAX; const size_t NO_SOLUTION_CODE = UINT_MAX;
...@@ -173,25 +152,25 @@ bool save_data(const std::string& _filename, ...@@ -173,25 +152,25 @@ bool save_data(const std::string& _filename,
} // namespace } // namespace
Cache::Cache(const std::string& _mip_lp)
: mip_lp_(_mip_lp), hash_(string_to_hash(mip_lp_)), found_(false)
{
DEB_enter_func;
DEB_line(2, "Cache hash: " << hash_);
}
bool Cache::restore_result( bool Cache::restore_result(std::vector<double>& _x, double& _obj_val)
std::string& _lp_prbl_str, // string containing the .lp problem
std::vector<double>& _x, // result.
double& _obj_val) // objective function value.
{ {
DEB_enter_func; DEB_enter_func;
found_ = false; const auto* cache_loc = Config::query().cache_location();
lp_file_cnts_ = std::move(_lp_prbl_str); if (cache_loc == nullptr) // cache location not provided, disabale the cache
if (cache_directory().empty())
return false; return false;
string_to_key(lp_file_cnts_, key_);
DEB_line(2, "Key cache name: " << key_);
key_ += '_';
for (size_t iter_nmbr = 0; iter_nmbr < 10; ++iter_nmbr) for (size_t iter_nmbr = 0; iter_nmbr < 10; ++iter_nmbr)
{ {
last_filename_ = full_cache_filename(key_) + std::to_string(iter_nmbr); filename_ = cache_loc + hash_ + '_' + std::to_string(iter_nmbr);
std::string dat_filename(last_filename_ + ".dat"); std::string dat_filename(filename_ + ".dat");
boost::system::error_code err_cod; boost::system::error_code err_cod;
if (!boost::filesystem::exists( if (!boost::filesystem::exists(
boost::filesystem::path(dat_filename.c_str()), err_cod) || boost::filesystem::path(dat_filename.c_str()), err_cod) ||
...@@ -203,16 +182,16 @@ bool Cache::restore_result( ...@@ -203,16 +182,16 @@ bool Cache::restore_result(
break; break;
} }
if (FileLock::active(last_filename_)) if (FileLock::active(filename_))
break; break;
std::string cache_cnts; std::string cache_cnts;
if (!load_file(last_filename_ + ".lp", cache_cnts)) if (!load_file(filename_ + ".lp", cache_cnts))
break; break;
if (cache_cnts == lp_file_cnts_) if (cache_cnts == mip_lp_)
{ {
found_ = load_data(last_filename_ + ".dat", _x, _obj_val); found_ = load_data(filename_ + ".dat", _x, _obj_val);
return found_; return found_;
} }
} }
...@@ -233,10 +212,10 @@ public: ...@@ -233,10 +212,10 @@ public:
if (success_) if (success_)
return; return;
// Removes files eventually written if there has been any kind of failure. // Removes files eventually written if there has been any kind of failure.
for (const auto& f_name : used_files_) for (const auto& filename : used_files_)
{ {
if (!f_name.empty()) if (!filename.empty())
std::remove(f_name.c_str()); std::remove(filename.c_str());
} }
} }
...@@ -269,11 +248,11 @@ void ...@@ -269,11 +248,11 @@ void
Cache::store_result(const std::vector<double>& _x, const double& _obj_val) Cache::store_result(const std::vector<double>& _x, const double& _obj_val)
{ {
DEB_enter_func; DEB_enter_func;
THROW_OUTCOME_if(found_, TODO); /* Multiple store of Docloud cache value. */ THROW_OUTCOME_if(found_, TODO); /* Multiple store of DOcloud cache value. */
if (!last_filename_.empty()) if (!filename_.empty())
{ {
CacheSaver saver; CacheSaver saver;
saver.save(last_filename_, _x, _obj_val, lp_file_cnts_); saver.save(filename_, _x, _obj_val, mip_lp_);
} }
} }
......
...@@ -21,27 +21,27 @@ namespace DOcloud { ...@@ -21,27 +21,27 @@ namespace DOcloud {
class Cache class Cache
{ {
public: public:
Cache() : found_(false) {} // Manage the cache for an optimization problem stored in .lp format
Cache(const std::string& _mip_lp);
const std::string& hash() const { return hash_; }
bool restore_result( bool restore_result(
std::string& _file_name, // .lp file defining the optimization problem
std::vector<double>& _x, // result. std::vector<double>& _x, // result.
double& _obj_val); // objective function value. double& _obj_val // objective function value.
);
// We can store the result for the given .lp file. This makes sense only we have // We can store the result for the given .lp file. This makes sense only we have
// not found cache data for the given .lp file, and in order to avoid data // not found cache data for the given .lp file, and in order to avoid data
// corruption this function fails if the data have been found. // corruption this function fails if the data have been found.
void store_result(const std::vector<double>& _x, const double& _obj_val); void store_result(const std::vector<double>& _x, const double& _obj_val);
const std::string& get_lp_content() { return lp_file_cnts_; }
private: private:
std::string key_; // String generated from .lp file content data. const std::string& mip_lp_; // The MIP represented in the .lp format
// This is a sort of hash key generate form the .lp const std::string hash_; // hask for the lp_ problem
// file content. bool found_; // Remembers if we have found a cache for the input problem
std::string last_filename_;// Last name we have tried to get cached data. std::string filename_; // File name to access the cached data (no extension)
std::string lp_file_cnts_; // Content of the input .lp file.
bool found_; // Remembers if we have found a cache for the input
// .lp file.
}; };
} // namespace DOcloud } // namespace DOcloud
......
//=============================================================================
//
// CLASS DOCloudSolver
//
//=============================================================================
#ifndef COMISO_DOCLOUDCONFIG_HH
#define COMISO_DOCLOUDCONFIG_HH
//== COMPILE-TIME PACKAGE REQUIREMENTS ========================================
#include <CoMISo/Config/config.hh>
#if COMISO_DOCLOUD_AVAILABLE
#include <string>
//== NAMESPACES ===============================================================
namespace COMISO {
namespace DOcloud {
//== CLASS DEFINITION =========================================================
/**
Configuration options for the IBM Decision Optimization Cloud.
*/
class Config
{
public:
// access the configuration singleton
static const Config& query();
static Config& modify();
void set_root_url(const char* const _root_url);
const char* root_url() const { return root_url_.data(); }
void set_api_key(const char* _api_key);
const char* api_key() const { return api_key_.data(); }
void set_infeasible_timeout(const int _infs_time);
int infeasible_timeout() const { return infs_time_; }
void set_feasible_timeout(const int _fsbl_time);
int feasible_timeout() const { return fsbl_time_; }
void set_cache_location(const char* const _cache_loc);
const char* cache_location() const { return cache_loc_.data(); }
private:
std::string root_url_;
std::string api_key_;
int infs_time_;
int fsbl_time_;
std::string cache_loc_;
private:
Config();
static Config& object();
};
//=============================================================================
} // namespace DOcloud
} // namespace COMISO
//=============================================================================
#endif // COMISO_DOCLOUD_AVAILABLE
//=============================================================================
#endif // COMISO_DOCLOUDCONFIG_HH
//=============================================================================
...@@ -7,6 +7,7 @@ ...@@ -7,6 +7,7 @@
#include "DOcloudJob.hh" #include "DOcloudJob.hh"
#if COMISO_DOCLOUD_AVAILABLE #if COMISO_DOCLOUD_AVAILABLE
#include "DOCloudConfig.hh"
#include <Base/Utils/OutcomeUtils.hh> #include <Base/Utils/OutcomeUtils.hh>
#include <Base/Debug/DebUtils.hh> #include <Base/Debug/DebUtils.hh>
...@@ -21,11 +22,36 @@ DEB_module("DOcloud") ...@@ -21,11 +22,36 @@ DEB_module("DOcloud")
namespace COMISO { namespace COMISO {
namespace DOcloud { namespace DOcloud {
static char* root_url__ = //////////////////////////////////////////////////////////////////////////
"https://api-oaas.docloud.ibmcloud.com/job_manager/rest/v1/jobs"; // Config
static std::string api_key__ = static const char* app_type__ = "Content-Type: application/json";
"X-IBM-Client-Id: api_0821c92f-0f2b-4ea5-be24-ecc9cd7695dd";
static char* app_type__ = "Content-Type: application/json"; Config::Config()
: root_url_("https://api-oaas.docloud.ibmcloud.com/job_manager/rest/v1/jobs"),
api_key_("X-IBM-Client-Id: api_0821c92f-0f2b-4ea5-be24-ecc9cd7695dd"),
infs_time_(300), fsbl_time_(15),
cache_loc_("\\\\camfs1\\General_access\\Martin_Marinov\\ReForm\\Cache\\")
{
const char* env_cache_dir = getenv("ReFormCacheDir");
if (env_cache_dir != nullptr && env_cache_dir[0] != 0)
{
cache_loc_ = env_cache_dir;
if (cache_loc_.back() != '\\')
cache_loc_ += '\\'; // Eventually add '\' to the directory string.
}
}
Config& Config::object()
{
// TODO: implement MT-lock
static Config config;
return config;
}
//////////////////////////////////////////////////////////////////////////
// Config
const Config& Config::query() { return object(); }
Config& Config::modify() { return object(); }
class HeaderTokens class HeaderTokens
{ {
...@@ -59,8 +85,8 @@ public: ...@@ -59,8 +85,8 @@ public:
typedef std::vector<std::string>::const_iterator const_iterator; typedef std::vector<std::string>::const_iterator const_iterator;
const_iterator begin() const { return tkns_.begin();} const_iterator begin() const { return tkns_.begin(); }
const_iterator end() const { return tkns_.end();} const_iterator end() const { return tkns_.end(); }
private: private:
std::vector<std::string> tkns_; std::vector<std::string> tkns_;
...@@ -185,7 +211,7 @@ Job::~Job() ...@@ -185,7 +211,7 @@ Job::~Job()
} }
del.set_url(url_.data()); del.set_url(url_.data());
del.add_http_header(api_key__.c_str()); del.add_http_header(Config::query().api_key());
del.perform(); del.perform();
delete stts_; delete stts_;
...@@ -201,8 +227,8 @@ void Job::make() ...@@ -201,8 +227,8 @@ void Job::make()
cURLpp::Post post(post_loc); cURLpp::Post post(post_loc);
THROW_OUTCOME_if(!post.valid(), TODO); //Failed to initialize the request THROW_OUTCOME_if(!post.valid(), TODO); //Failed to initialize the request
post.set_url(root_url__); post.set_url(Config::query().root_url());
post.add_http_header(api_key__.c_str()); post.add_http_header(Config::query().api_key());
post.add_http_header(app_type__); post.add_http_header(app_type__);
post.perform(); post.perform();
...@@ -216,21 +242,21 @@ void Job::make() ...@@ -216,21 +242,21 @@ void Job::make()
stts_ = new JsonTokens; stts_ = new JsonTokens;
} }
void Job::upload(cURLpp::Upload& _upload) void Job::upload(cURLpp::Upload& _upld)
{ {
THROW_OUTCOME_if(!_upload.valid(), TODO); //Failed to initialize the request THROW_OUTCOME_if(!_upld.valid(), TODO); //Failed to initialize the request
auto url = url_ + "/attachments/" + filename_ + "/blob"; auto url = url_ + "/attachments/" + filename_ + "/blob";
_upload.set_url(url.data()); _upld.set_url(url.data());
_upload.add_http_header(api_key__.c_str()); _upld.add_http_header(Config::query().api_key());
_upload.perform(); _upld.perform();
HttpStatus http_stat(_upload); HttpStatus http_stat(_upld);
http_stat.check(204); http_stat.check(204);
} }
void Job::upload() void Job::upload()
{ {
if (file_buf_ == nullptr) if (file_buf_.empty())
{// file is not buffered into memory {// file is not buffered into memory
cURLpp::UploadFile upld(filename_); cURLpp::UploadFile upld(filename_);
upload(upld); upload(upld);
...@@ -249,7 +275,7 @@ void Job::start() ...@@ -249,7 +275,7 @@ void Job::start()
auto url = url_ + "/execute"; auto url = url_ + "/execute";
post.set_url(url.data()); post.set_url(url.data());
post.add_http_header(api_key__.c_str()); post.add_http_header(Config::query().api_key());
post.add_http_header(app_type__); post.add_http_header(app_type__);
post.perform(); post.perform();
HttpStatus http_stat(post); HttpStatus http_stat(post);
...@@ -266,7 +292,7 @@ void Job::sync_status() ...@@ -266,7 +292,7 @@ void Job::sync_status()
cURLpp::Get get; cURLpp::Get get;
THROW_OUTCOME_if(!get.valid(), TODO); //Failed to initialize the request THROW_OUTCOME_if(!get.valid(), TODO); //Failed to initialize the request
get.set_url(url_.data()); get.set_url(url_.data());
get.add_http_header(api_key__.c_str()); get.add_http_header(Config::query().api_key());
get.perform(); get.perform();
HttpStatus http_stat(get); HttpStatus http_stat(get);
http_stat.check(200); http_stat.check(200);
...@@ -305,7 +331,7 @@ void Job::sync_log() ...@@ -305,7 +331,7 @@ void Job::sync_log()
const std::string url = url_ + "/log/items?start=" + const std::string url = url_ + "/log/items?start=" +
std::to_string(log_seq_idx_) + "&continuous=true"; std::to_string(log_seq_idx_) + "&continuous=true";
get.set_url(url.data()); get.set_url(url.data());
get.add_http_header(api_key__.c_str()); get.add_http_header(Config::query().api_key());
get.perform(); get.perform();
HttpStatus http_stat(get); HttpStatus http_stat(get);
http_stat.check(200); http_stat.check(200);
...@@ -401,7 +427,7 @@ void Job::abort() ...@@ -401,7 +427,7 @@ void Job::abort()
THROW_OUTCOME_if(!del.valid(), TODO); //Failed to initialize the request THROW_OUTCOME_if(!del.valid(), TODO); //Failed to initialize the request
const std::string url = url_ + "/execute"; const std::string url = url_ + "/execute";
del.set_url(url.data()); del.set_url(url.data());
del.add_http_header(api_key__.c_str()); del.add_http_header(Config::query().api_key());
del.perform(); del.perform();
HttpStatus http_stat(del); HttpStatus http_stat(del);
...@@ -437,7 +463,7 @@ double Job::solution(std::vector<double>& _x) const ...@@ -437,7 +463,7 @@ double Job::solution(std::vector<double>& _x) const
auto url = url_ + "/attachments/solution.json/blob"; auto url = url_ + "/attachments/solution.json/blob";
get.set_url(url.data()); get.set_url(url.data());
get.add_http_header(api_key__.c_str()); get.add_http_header(Config::query().api_key());
get.perform(); get.perform();
HttpStatus http_stat(get); HttpStatus http_stat(get);
......
...@@ -25,7 +25,7 @@ class JsonTokens; ...@@ -25,7 +25,7 @@ class JsonTokens;
class Job : public cURLpp::Session class Job : public cURLpp::Session
{ {
public: public:
Job(const char* _filename, const char* _file_buf = nullptr) Job(const std::string& _filename, const std::string& _file_buf)
: filename_(_filename), file_buf_(_file_buf), stts_(nullptr) : filename_(_filename), file_buf_(_file_buf), stts_(nullptr)
{} {}
~Job(); ~Job();
...@@ -54,12 +54,12 @@ public: ...@@ -54,12 +54,12 @@ public:
protected: protected:
void make(); void make();
void start(); void start();
void upload(cURLpp::Upload& _upload); void upload(cURLpp::Upload& _upld);
void upload(); void upload();
private: private:
const char* filename_; const std::string filename_;
const char* file_buf_; const std::string file_buf_;
std::string url_; std::string url_;
JsonTokens* stts_; JsonTokens* stts_;
// these variables are initialized in start() // these variables are initialized in start()
......
...@@ -278,21 +278,22 @@ void DOCloudSolver::solve( ...@@ -278,21 +278,22 @@ void DOCloudSolver::solve(
"DOCloudSolver received a problem with non-constant gradient!"); "DOCloudSolver received a problem with non-constant gradient!");
std::vector<double> x(_problem->n_unknowns(), 0.0); // solution std::vector<double> x(_problem->n_unknowns(), 0.0); // solution
std::string lp_prbl_str = const std::string mip_lp = DOcloud::create_lp_string(_problem, _constraints,
DOcloud::create_lp_string(_problem, _constraints, _discrete_constraints, x); _discrete_constraints, x);
double obj_val; double obj_val;
DOcloud::Cache cache; DOcloud::Cache cache(mip_lp);
if (cache.restore_result(lp_prbl_str, x, obj_val)) if (cache.restore_result(x, obj_val))
DEB_line(3, "MIP cached.") DEB_line(3, "MIP cached.")
else else
{ {
DEB_line(1, "MIP not cached, computing optimization."); DEB_line(1, "MIP not cached, computing optimization.");
DOcloud::Job job("DoClouProblem.lp", cache.get_lp_content().data()); const std::string lp_hash = cache.hash() + ".lp";
DOcloud::Job job(lp_hash, mip_lp);
job.setup(); job.setup();
job.wait(); job.wait();
obj_val = job.solution(x); obj_val = job.solution(x);
//cache.store_result(x, obj_val); cache.store_result(x, obj_val);
} }
THROW_OUTCOME_if(x.empty(), MIPS_NO_SOLUTION); THROW_OUTCOME_if(x.empty(), MIPS_NO_SOLUTION);
......
...@@ -106,7 +106,7 @@ private: ...@@ -106,7 +106,7 @@ private:
class UploadData : public