ReForm, DOcloud: Added DOCloudConfig, and some minor fixes.

[git-p4: depot-paths = "//ReForm/ReForm/main/CoMISo/": change = 12200]

NSolver/CMakeLists.txt

@@ -6,6 +6,7 @@ SET(my_headers
    ${CMAKE_CURRENT_SOURCE_DIR}/CPLEXSolver.hh
    ${CMAKE_CURRENT_SOURCE_DIR}/cURLpp.hh
    ${CMAKE_CURRENT_SOURCE_DIR}/DOCloudCache.hh
+   ${CMAKE_CURRENT_SOURCE_DIR}/DOCloudConfig.hh
    ${CMAKE_CURRENT_SOURCE_DIR}/DOCloudJob.hh
    ${CMAKE_CURRENT_SOURCE_DIR}/DOCloudSolver.hh
    ${CMAKE_CURRENT_SOURCE_DIR}/GurobiHelper.hh

NSolver/DOCloudCache.cc

@@ -6,6 +6,8 @@
 
 //=============================================================================
 #include "DOCloudCache.hh"
+#include "DOCloudConfig.hh"
+
 #include <Base/Utils/OutcomeUtils.hh>
 #include <Base/Debug/DebOut.hh>
 
@@ -13,6 +15,7 @@
 #include <iomanip>
 #include <cctype>
 #include <functional>
+#include <sstream>
 
 #include <boost/filesystem.hpp>
 
@@ -27,31 +30,6 @@ namespace DOcloud {
 
 namespace {
 
-const std::string& cache_directory()
-{
-  static std::string cache_dir = 
-    "\\\\camfs1\\General_access\\Martin_Marinov\\ReForm\\Cache\\";
-  static bool already_read = false;
-  if (!already_read)
-  {
-    already_read = true;
-    const char* env_cache_dir = getenv("ReFormCacheDir");
-    if (env_cache_dir != nullptr && env_cache_dir[0] != 0)
-    {
-      cache_dir = env_cache_dir;
-      if (cache_dir.back() != '\\')
-        cache_dir += '\\'; // Eventually add '\' to the directory string.
-    }
-  }
-  return cache_dir;
-}
-
-// 
-std::string full_cache_filename(const std::string& _filename)
-{
-  return cache_directory() + _filename;
-}
-
 // Create a new temporary exclusive file without extension that is used to 
 // prevent write or read operation on files with the same name and extension
 // .lp or .dat. while the cache is being written. This is the only class that
@@ -116,11 +94,12 @@ bool save_file(const std::string& _filename, const std::string& _file_cnts)
 
 // Finds a key string from the file name. This string will be used as file name
 // where to store the related cached data.
-void string_to_key(const std::string& _str, std::string& _key)
+std::string string_to_hash(const std::string& _str)
 {
-  // 1. Writes the file length.
   const std::hash<std::string> hash_fn;
-  _key = std::to_string(hash_fn(_str));
+  std::stringstream strm;
+  strm << std::hex << hash_fn(_str);
+  return strm.str();
 }
 
 const size_t NO_SOLUTION_CODE = UINT_MAX;
@@ -173,25 +152,25 @@ bool save_data(const std::string& _filename,
 
 } // namespace
 
+Cache::Cache(const std::string& _mip_lp) 
+  : mip_lp_(_mip_lp), hash_(string_to_hash(mip_lp_)), found_(false) 
+{
+  DEB_enter_func;
+  DEB_line(2, "Cache hash: " << hash_);
+}
 
-bool Cache::restore_result(
-  std::string& _lp_prbl_str, // string containing the .lp problem
-  std::vector<double>& _x,   // result.
-  double& _obj_val)          // objective function value.
+bool Cache::restore_result(std::vector<double>& _x, double& _obj_val) 
 {
   DEB_enter_func;
-  found_ = false;
-  lp_file_cnts_ = std::move(_lp_prbl_str);
-  if (cache_directory().empty())
+  const auto* cache_loc = Config::query().cache_location();
+  if (cache_loc == nullptr) // cache location not provided, disabale the cache
     return false;
-  string_to_key(lp_file_cnts_, key_);
-  DEB_line(2, "Key cache name: " << key_);
-  key_ += '_';
+
   for (size_t iter_nmbr = 0; iter_nmbr < 10; ++iter_nmbr)
   {
-    last_filename_ = full_cache_filename(key_) + std::to_string(iter_nmbr);
+    filename_ = cache_loc + hash_ + '_' + std::to_string(iter_nmbr);
 
-    std::string dat_filename(last_filename_ + ".dat");
+    std::string dat_filename(filename_ + ".dat");
     boost::system::error_code err_cod;
     if (!boost::filesystem::exists(
           boost::filesystem::path(dat_filename.c_str()), err_cod) ||
@@ -203,16 +182,16 @@ bool Cache::restore_result(
       break;
     }
 
-    if (FileLock::active(last_filename_))
+    if (FileLock::active(filename_))
       break;
 
     std::string cache_cnts;
-    if (!load_file(last_filename_ + ".lp", cache_cnts))
+    if (!load_file(filename_ + ".lp", cache_cnts))
       break;
 
-    if (cache_cnts == lp_file_cnts_)
+    if (cache_cnts == mip_lp_)
     {
-      found_ = load_data(last_filename_ + ".dat", _x, _obj_val);
+      found_ = load_data(filename_ + ".dat", _x, _obj_val);
       return found_;
     }
   }
@@ -233,10 +212,10 @@ public:
     if (success_)
       return;
     // Removes files eventually written if there has been any kind of failure.
-    for (const auto& f_name : used_files_)
+    for (const auto& filename : used_files_)
     {
-      if (!f_name.empty())
-        std::remove(f_name.c_str());
+      if (!filename.empty())
+        std::remove(filename.c_str());
     }
   }
 
@@ -269,11 +248,11 @@ void
 Cache::store_result(const std::vector<double>& _x, const double& _obj_val)
 {
   DEB_enter_func;
-  THROW_OUTCOME_if(found_, TODO); /* Multiple store of Docloud cache value. */
-  if (!last_filename_.empty())
+  THROW_OUTCOME_if(found_, TODO); /* Multiple store of DOcloud cache value. */
+  if (!filename_.empty())
   {
     CacheSaver saver;
-    saver.save(last_filename_, _x, _obj_val, lp_file_cnts_);
+    saver.save(filename_, _x, _obj_val, mip_lp_);
   }
 }
 

NSolver/DOCloudCache.hh

@@ -21,27 +21,27 @@ namespace DOcloud {
 class Cache
 {
 public:
-  Cache() : found_(false) {}
+  // Manage the cache for an optimization problem stored in .lp format
+  Cache(const std::string& _mip_lp);
+
+  const std::string& hash() const { return hash_; }
 
   bool restore_result(
-    std::string& _file_name,  // .lp file defining the optimization problem
     std::vector<double>& _x,  // result.
-    double& _obj_val);        // objective function value.
+    double& _obj_val // objective function value.
+    ); 
 
   // We can store the result for the given .lp file. This makes sense only we have
   // not found cache data for the given .lp file, and in order to avoid data
   // corruption this function fails if the data have been found.
   void store_result(const std::vector<double>& _x, const double& _obj_val);
 
-  const std::string& get_lp_content() { return lp_file_cnts_; }
+
 private:
-  std::string key_;          // String generated from .lp file content data. 
-                             // This is a sort of hash key generate form the .lp
-                             // file content.
-  std::string last_filename_;// Last name we have tried to get cached data. 
-  std::string lp_file_cnts_; // Content of the input .lp file.
-  bool        found_;        // Remembers if we have found a cache for the input
-                             // .lp file.
+  const std::string& mip_lp_; // The MIP represented in the .lp format 
+  const std::string hash_; // hask for the lp_ problem
+  bool found_; // Remembers if we have found a cache for the input problem
+  std::string filename_; // File name to access the cached data (no extension)
 };
 
 } // namespace DOcloud

NSolver/DOCloudConfig.hh

+//=============================================================================
+//
+//  CLASS DOCloudSolver
+//
+//=============================================================================
+#ifndef COMISO_DOCLOUDCONFIG_HH
+#define COMISO_DOCLOUDCONFIG_HH
+
+//== COMPILE-TIME PACKAGE REQUIREMENTS ========================================
+#include <CoMISo/Config/config.hh>
+#if COMISO_DOCLOUD_AVAILABLE
+
+#include <string>
+
+//== NAMESPACES ===============================================================
+namespace COMISO {
+namespace DOcloud {
+
+//== CLASS DEFINITION =========================================================
+
+/**
+    Configuration options for the IBM Decision Optimization Cloud.
+*/
+class Config
+{
+public:
+  // access the configuration singleton
+  static const Config& query();
+  static Config& modify();
+
+  void set_root_url(const char* const _root_url);
+  const char* root_url() const { return root_url_.data(); }
+
+  void set_api_key(const char* _api_key);
+  const char* api_key() const { return api_key_.data(); }
+
+  void set_infeasible_timeout(const int _infs_time);
+  int infeasible_timeout() const { return infs_time_; }
+
+  void set_feasible_timeout(const int _fsbl_time); 
+  int feasible_timeout() const { return fsbl_time_; }
+
+  void set_cache_location(const char* const _cache_loc);
+  const char* cache_location() const { return cache_loc_.data(); }
+
+private:
+  std::string root_url_;
+  std::string api_key_;
+  int infs_time_;
+  int fsbl_time_;
+  std::string cache_loc_;
+
+private:
+  Config();
+  static Config& object();
+};
+
+//=============================================================================
+} // namespace DOcloud
+} // namespace COMISO
+
+//=============================================================================
+#endif // COMISO_DOCLOUD_AVAILABLE
+//=============================================================================
+#endif // COMISO_DOCLOUDCONFIG_HH
+//=============================================================================
+

NSolver/DOCloudJob.cc

@@ -7,6 +7,7 @@
 #include "DOcloudJob.hh"
 #if COMISO_DOCLOUD_AVAILABLE
 
+#include "DOCloudConfig.hh"
 #include <Base/Utils/OutcomeUtils.hh>
 #include <Base/Debug/DebUtils.hh>
 
@@ -21,11 +22,36 @@ DEB_module("DOcloud")
 namespace COMISO {
 namespace DOcloud { 
 
-static char* root_url__ = 
-  "https://api-oaas.docloud.ibmcloud.com/job_manager/rest/v1/jobs";
-static std::string api_key__ = 
-  "X-IBM-Client-Id: api_0821c92f-0f2b-4ea5-be24-ecc9cd7695dd";
-static char* app_type__ = "Content-Type: application/json";
+//////////////////////////////////////////////////////////////////////////
+// Config
+static const char* app_type__ = "Content-Type: application/json";
+
+Config::Config()
+  : root_url_("https://api-oaas.docloud.ibmcloud.com/job_manager/rest/v1/jobs"),
+    api_key_("X-IBM-Client-Id: api_0821c92f-0f2b-4ea5-be24-ecc9cd7695dd"),
+    infs_time_(300), fsbl_time_(15), 
+    cache_loc_("\\\\camfs1\\General_access\\Martin_Marinov\\ReForm\\Cache\\")
+{
+  const char* env_cache_dir = getenv("ReFormCacheDir");
+  if (env_cache_dir != nullptr && env_cache_dir[0] != 0)
+  {
+    cache_loc_ = env_cache_dir;
+    if (cache_loc_.back() != '\\')
+      cache_loc_ += '\\'; // Eventually add '\' to the directory string.
+  }
+}
+
+Config& Config::object()
+{
+  // TODO: implement MT-lock
+  static Config config;
+  return config;
+}
+
+//////////////////////////////////////////////////////////////////////////
+// Config
+const Config& Config::query() { return object(); } 
+Config& Config::modify() { return object(); } 
 
 class HeaderTokens
 {
@@ -59,8 +85,8 @@ public:
 
   typedef std::vector<std::string>::const_iterator const_iterator;
 
-  const_iterator begin() const { return tkns_.begin();}
-  const_iterator end() const { return tkns_.end();}
+  const_iterator begin() const { return tkns_.begin(); }
+  const_iterator end() const { return tkns_.end(); }
 
 private:
   std::vector<std::string> tkns_;
@@ -185,7 +211,7 @@ Job::~Job()
   }
 
   del.set_url(url_.data());
-  del.add_http_header(api_key__.c_str());
+  del.add_http_header(Config::query().api_key());
   del.perform();
 
   delete stts_;
@@ -201,8 +227,8 @@ void Job::make()
   cURLpp::Post post(post_loc);
   THROW_OUTCOME_if(!post.valid(), TODO); //Failed to initialize the request
 
-  post.set_url(root_url__);
-  post.add_http_header(api_key__.c_str());
+  post.set_url(Config::query().root_url());
+  post.add_http_header(Config::query().api_key());
   post.add_http_header(app_type__);
   post.perform();
 
@@ -216,21 +242,21 @@ void Job::make()
     stts_ = new JsonTokens;
 }
 
-void Job::upload(cURLpp::Upload& _upload)
+void Job::upload(cURLpp::Upload& _upld)
 {
-  THROW_OUTCOME_if(!_upload.valid(), TODO); //Failed to initialize the request
+  THROW_OUTCOME_if(!_upld.valid(), TODO); //Failed to initialize the request
 
   auto url = url_ + "/attachments/" + filename_ + "/blob";
-  _upload.set_url(url.data());
-  _upload.add_http_header(api_key__.c_str());
-  _upload.perform();
-  HttpStatus http_stat(_upload);
+  _upld.set_url(url.data());
+  _upld.add_http_header(Config::query().api_key());
+  _upld.perform();
+  HttpStatus http_stat(_upld);
   http_stat.check(204);
 }
 
 void Job::upload()
 {
-  if (file_buf_ == nullptr)
+  if (file_buf_.empty())
   {// file is not buffered into memory
     cURLpp::UploadFile upld(filename_);
     upload(upld);
@@ -249,7 +275,7 @@ void Job::start()
 
   auto url = url_ + "/execute";
   post.set_url(url.data());
-  post.add_http_header(api_key__.c_str());
+  post.add_http_header(Config::query().api_key());
   post.add_http_header(app_type__);
   post.perform();
   HttpStatus http_stat(post);
@@ -266,7 +292,7 @@ void Job::sync_status()
   cURLpp::Get get;
   THROW_OUTCOME_if(!get.valid(), TODO); //Failed to initialize the request
   get.set_url(url_.data());
-  get.add_http_header(api_key__.c_str());
+  get.add_http_header(Config::query().api_key());
   get.perform();
   HttpStatus http_stat(get);
   http_stat.check(200);
@@ -305,7 +331,7 @@ void Job::sync_log()
   const std::string url = url_ + "/log/items?start=" + 
     std::to_string(log_seq_idx_) + "&continuous=true";
   get.set_url(url.data());
-  get.add_http_header(api_key__.c_str());
+  get.add_http_header(Config::query().api_key());
   get.perform();
   HttpStatus http_stat(get);
   http_stat.check(200);
@@ -401,7 +427,7 @@ void Job::abort()
   THROW_OUTCOME_if(!del.valid(), TODO); //Failed to initialize the request
   const std::string url = url_ + "/execute";
   del.set_url(url.data());
-  del.add_http_header(api_key__.c_str());
+  del.add_http_header(Config::query().api_key());
   del.perform();
 
   HttpStatus http_stat(del);
@@ -437,7 +463,7 @@ double Job::solution(std::vector<double>& _x) const
 
   auto url = url_ + "/attachments/solution.json/blob";
   get.set_url(url.data());
-  get.add_http_header(api_key__.c_str());
+  get.add_http_header(Config::query().api_key());
   get.perform();
 
   HttpStatus http_stat(get);

NSolver/DOCloudJob.hh

@@ -25,7 +25,7 @@ class JsonTokens;
 class Job : public cURLpp::Session
 {
 public:
-  Job(const char* _filename, const char* _file_buf = nullptr) 
+  Job(const std::string& _filename, const std::string& _file_buf) 
     : filename_(_filename), file_buf_(_file_buf), stts_(nullptr) 
   {}
   ~Job();
@@ -54,12 +54,12 @@ public:
 protected:
   void make();
   void start();
-  void upload(cURLpp::Upload& _upload);
+  void upload(cURLpp::Upload& _upld);
   void upload();
 
 private:
-  const char* filename_;
-  const char* file_buf_;
+  const std::string filename_;
+  const std::string file_buf_;
   std::string url_;
   JsonTokens* stts_;
   // these variables are initialized in start() 

NSolver/DOCloudSolver.cc

@@ -278,21 +278,22 @@ void DOCloudSolver::solve(
     "DOCloudSolver received a problem with non-constant gradient!");
 
   std::vector<double> x(_problem->n_unknowns(), 0.0); // solution
-  std::string lp_prbl_str =
-    DOcloud::create_lp_string(_problem, _constraints,  _discrete_constraints, x);
+  const std::string mip_lp = DOcloud::create_lp_string(_problem, _constraints,  
+    _discrete_constraints, x);
 
   double obj_val;
-  DOcloud::Cache cache;
-  if (cache.restore_result(lp_prbl_str, x, obj_val))
+  DOcloud::Cache cache(mip_lp);
+  if (cache.restore_result(x, obj_val))
     DEB_line(3, "MIP cached.")
   else
   {
     DEB_line(1, "MIP not cached, computing optimization.");
-    DOcloud::Job job("DoClouProblem.lp", cache.get_lp_content().data());
+    const std::string lp_hash = cache.hash() + ".lp";
+    DOcloud::Job job(lp_hash, mip_lp);
     job.setup();
     job.wait();
     obj_val = job.solution(x);
-    //cache.store_result(x, obj_val);
+    cache.store_result(x, obj_val);
   }
   THROW_OUTCOME_if(x.empty(), MIPS_NO_SOLUTION);
 

NSolver/cURLpp.hh

@@ -106,7 +106,7 @@ private:
 class UploadData : public Upload
 {
 public:
-  UploadData(const std::string& _buf) : buf_(_buf)  {}
+  explicit UploadData(const std::string& _buf) : buf_(_buf)  {}
 
 protected:
   virtual size_t send_data();
@@ -115,7 +115,7 @@ private:
   class Buffer
   {
   public:
-    Buffer(const std::string& _data)
+    explicit Buffer(const std::string& _data)
       : ptr_(_data.c_str()), len_(_data.size()), pos_(0) 
     {}
 
@@ -123,13 +123,13 @@ private:
       const size_t _n_elem, void* _from_buf);
 
     size_t length() const { return len_; }
+
   private:
     const char* ptr_;
     size_t len_;
     size_t pos_;
   };
 
-  class Buffer;
   Buffer buf_;
 };