Merge branch 'unify_with_ReForm' into 'master'

Unify with ReForm



See merge request !5

.gitmodules

+[submodule "Base"]
+	path = Base
+	url = https://www.graphics.rwth-aachen.de:9000/CoMISo/Base.git

Base @ d6a5c15d

+Subproject commit d6a5c15da072a05eb4f389a12d3247488a7668ad

CI/ci-linux.sh

@@ -29,7 +29,12 @@ elif [ "$LANGUAGE" == "C++11" ]; then
 fi  
 
 #########################################
+# Base Submodule init
+git submodule init
+git submodule update
 
+
+#########################################
 # Make release build folder
 if [ ! -d build-release-$BUILDPATH ]; then
   mkdir build-release-$BUILDPATH

CI/ci-mac.sh

@@ -31,6 +31,10 @@ echo "============================="
 echo "Starting Build: "
 echo "============================="
 
+#########################################
+# Base Submodule init
+git submodule init
+git submodule update
 
 
 #########################################

CMakeLists.txt

@@ -350,9 +350,16 @@ link_directories (
   ${COMISO_LINK_DIRECTORIES}
 )
 
+# enable debug stuff
+add_definitions(-DDEB_ON)
+
 # source code directories
 set (directories 
   .
+  Base/Code
+  Base/Debug
+  Base/Security
+  Base/Utils
   Solver
   NSolver
   EigenSolver

Config/GmmTypes.hh

+// (C) Copyright 2014 by Autodesk, Inc.
+//
+// The information contained herein is confidential, proprietary
+// to Autodesk,  Inc.,  and considered a trade secret as defined
+// in section 499C of the penal code of the State of California.
+// Use of  this information  by  anyone  other  than  authorized
+// employees of Autodesk, Inc.  is granted  only under a written
+// non-disclosure agreement,  expressly  prescribing  the  scope
+// and manner of such use.
+
+#ifndef GMMTYPES_HH_INCLUDED
+#define GMMTYPES_HH_INCLUDED
+
+#include <gmm/gmm_matrix.h>
+
+namespace COMISO_GMM
+{  
+
+// Matrix Types
+typedef gmm::col_matrix< gmm::wsvector<double> > WSColMatrix;
+typedef gmm::row_matrix< gmm::wsvector<double> > WSRowMatrix;
+typedef gmm::col_matrix< gmm::rsvector<double> > RSColMatrix;
+typedef gmm::row_matrix< gmm::rsvector<double> > RSRowMatrix;
+typedef gmm::csc_matrix<double> CSCMatrix;
+
+}//namespace COMISO_GMM
+
+#endif//GMMTYPES_HH_INCLUDED

Config/StdTypes.hh

+// (C) Copyright 2014 by Autodesk, Inc.
+//
+// The information contained herein is confidential, proprietary
+// to Autodesk,  Inc.,  and considered a trade secret as defined
+// in section 499C of the penal code of the State of California.
+// Use of  this information  by  anyone  other  than  authorized
+// employees of Autodesk, Inc.  is granted  only under a written
+// non-disclosure agreement,  expressly  prescribing  the  scope
+// and manner of such use.
+
+#ifndef STDTYPES_HH_INCLUDED
+#define STDTYPES_HH_INCLUDED
+
+#include <vector>
+
+namespace COMISO_STD
+{  
+
+// Vector Types
+typedef std::vector<double> DoubleVector;
+typedef std::vector<int> IntVector;
+typedef std::vector<unsigned int> UIntVector;
+
+}//namespace COMISO_GMM
+
+#endif//STDTYPES_HH_INCLUDED

Examples/small_cplex_soc/main.cc

@@ -31,6 +31,7 @@
 #include <CoMISo/NSolver/NPDerivativeChecker.hh>
 #include <CoMISo/NSolver/CPLEXSolver.hh>
 #include <CoMISo/NSolver/IPOPTSolver.hh>
+#include <CoMISo/NSolver/ConeConstraint.hh>
 
 // solve least squares problem for x=1, y=2 and x-2y+z = 1
 // with hard constraints x =-3, z>=3, z^2 >= x^2+y^2

NSolver/CBCSolver.cc

@@ -12,6 +12,9 @@
 
 //=============================================================================
 #include "CBCSolver.hh"
+#include <CoMISo/Utils/CoMISoError.hh>
+
+#include <Base/Debug/DebTime.hh>
 
 // For Branch and bound
 #include "OsiSolverInterface.hpp"
@@ -50,6 +53,8 @@ namespace COMISO {
 
 //== IMPLEMENTATION ==========================================================
 
+namespace {
+
 // These are some "tweaks" for the CbcModel, copied from some sample Cbc code
 // I have no idea what any of these do!
 void model_tweak(CbcModel& model)
@@ -127,6 +132,7 @@ void model_tweak(CbcModel& model)
   //model.addHeuristic(&heuristic2);
 }
 
+#define TRACE_CBT(DESCR, EXPR) DEB_line(7, DESCR << ": " << EXPR)
 #define P(X) ((X).data())
 
 bool solve_impl(
@@ -136,10 +142,11 @@ bool solve_impl(
   const double                              _time_limit
 )
 {
-  if(!_problem->constant_hessian())
-    std::cerr << "Warning: CBCSolver received a problem with non-constant hessian!" << std::endl;
-  if(!_problem->constant_gradient())
-    std::cerr <<  "Warning: CBCSolver received a problem with non-constant gradient!" << std::endl;
+  DEB_enter_func;
+  DEB_warning_if(!_problem->constant_hessian(), 1,
+                 "CBCSolver received a problem with non-constant hessian!");
+  DEB_warning_if(!_problem->constant_gradient(), 1,
+                 "CBCSolver received a problem with non-constant gradient!");
 
   const int n_rows = _constraints.size(); // Constraints #
   const int n_cols = _problem->n_unknowns(); // Unknowns #
@@ -162,8 +169,8 @@ bool solve_impl(
 
   for (size_t i = 0; i < _constraints.size();  ++i)
   {
-    if(!_constraints[i]->is_linear())
-      std::cerr << "Warning: constraint " << i << " is non-linear" << std::endl;
+    DEB_error_if(!_constraints[i]->is_linear(), "constraint " << i <<
+                 " is non-linear");
 
     NConstraintInterface::SVectorNC gc;
     _constraints[i]->eval_gradient(P(x), gc);
@@ -189,8 +196,8 @@ bool solve_impl(
       break;
     }
 
-//    TRACE_CBT("Constraint " << i << " is of type " <<
-//              _constraints[i]->constraint_type() << "; RHS val", b);
+    TRACE_CBT("Constraint " << i << " is of type " <<
+              _constraints[i]->constraint_type() << "; RHS val", b);
   }
 
   //----------------------------------------------
@@ -199,15 +206,15 @@ bool solve_impl(
 
   std::vector<double> objective(n_cols);
   _problem->eval_gradient(P(x), P(objective));
-//  TRACE_CBT("Objective linear term", objective);
+  TRACE_CBT("Objective linear term", objective);
 
   const double c = _problem->eval_f(P(x));
   // ICGB: Where does objective constant term go in CBC?
   // MCM: I could not find this either: It is possible that the entire model
   // can be translated to accomodate the constant (if != 0). Ask DB!
-//  DEB_error_if(c > FLT_EPSILON, "Ignoring a non-zero constant objective term: "
-//               << c);
-//  TRACE_CBT("Objective constant term", c);
+  DEB_error_if(c > FLT_EPSILON, "Ignoring a non-zero constant objective term: "
+               << c);
+  TRACE_CBT("Objective constant term", c);
 
   // CBC Problem initialize
   OsiClpSolverInterface si;
@@ -257,7 +264,7 @@ bool solve_impl(
   model.branchAndBound();
 
   const double* solution = model.bestSolution();
-//  THROW_OUTCOME_if(solution == nullptr, UNSPECIFIED_CBC_EXCEPTION);
+  //COMISO_THROW_if(solution == nullptr, UNSPECIFIED_CBC_EXCEPTION);
 
   // store if solution available
   if(solution != 0)
@@ -271,6 +278,8 @@ bool solve_impl(
 
 #undef P
 
+}//namespace 
+
 bool CBCSolver::solve(
   NProblemInterface*                        _problem,
   const std::vector<NConstraintInterface*>& _constraints,
@@ -278,6 +287,7 @@ bool CBCSolver::solve(
   const double                              _time_limit
 )
 {
+  DEB_enter_func;
   bool valid_solution = false;
   try
   {
@@ -285,8 +295,8 @@ bool CBCSolver::solve(
   }
   catch (CoinError& ce)
   {
-    std::cerr << "CoinError code = " << ce.message() << std::endl;
-//    THROW_OUTCOME(UNSPECIFIED_CBC_EXCEPTION);
+    DEB_warning(1, "CoinError code = " << ce.message() << "]\n");
+    //COMISO_THROW(UNSPECIFIED_CBC_EXCEPTION);
   }
   return valid_solution;
 }

NSolver/CPLEXSolver.hh

@@ -53,14 +53,14 @@ public:
 
   // ********** SOLVE **************** //
   // this function has to be inline due to static linking issues
-  inline bool solve(NProblemInterface*                  _problem,                // problem instance
+  bool solve(NProblemInterface*                  _problem,                // problem instance
                     std::vector<NConstraintInterface*>& _constraints,            // linear constraints
                     std::vector<PairIndexVtype>&        _discrete_constraints,   // discrete constraints
                     const double                        _time_limit = 60,
                     const bool                          _silent = false);  // time limit in seconds
 
   // same as above but without discrete constraints (for convenience)
-  inline bool solve(NProblemInterface*                  _problem,                // problem instance
+  bool solve(NProblemInterface*                  _problem,                // problem instance
                     std::vector<NConstraintInterface*>& _constraints,            // linear constraints
                     const double                        _time_limit = 60,
                     const bool                          _silent = false)  // time limit in seconds
@@ -82,7 +82,7 @@ public:
 
 
   // optimization with additional lazy constraints that are only added iteratively to the problem if not satisfied
-  inline bool solve(NProblemInterface*                        _problem,
+  bool solve(NProblemInterface*                        _problem,
                     const std::vector<NConstraintInterface*>& _constraints,
                     const std::vector<NConstraintInterface*>& _lazy_constraints,
                     std::vector<PairIndexVtype>&              _discrete_constraints,   // discrete constraints
@@ -93,7 +93,7 @@ public:
 
 
   // same as above with additional lazy constraints that are only added iteratively to the problem if not satisfied
-  inline bool solve(NProblemInterface*                        _problem,
+  bool solve(NProblemInterface*                        _problem,
                     const std::vector<NConstraintInterface*>& _constraints,
                     const std::vector<NConstraintInterface*>& _lazy_constraints,
                     const double                              _almost_infeasible = 0.5,
@@ -106,7 +106,7 @@ public:
 
 
   // with handling of cone constrints
-  inline bool solve2(NProblemInterface*                  _problem,                // problem instance
+  bool solve2(NProblemInterface*                  _problem,                // problem instance
                     std::vector<NConstraintInterface*>& _constraints,            // linear constraints
                     std::vector<PairIndexVtype>&        _discrete_constraints,   // discrete constraints
                     const double                        _time_limit = 60,
@@ -117,10 +117,10 @@ public:
 //  void set_problem_env_output_path( const std::string &_problem_env_output_path);
 //  void set_solution_input_path    ( const std::string &_solution_input_path);
 
-    inline void test();
+    void test();
 
 protected:
-  double* P(std::vector<double>& _v)
+  inline double* P(std::vector<double>& _v)
   {
     if( !_v.empty())
       return ((double*)&_v[0]);
@@ -150,12 +150,6 @@ private:
 
 //=============================================================================
 #endif // COMISO_CPLEX_AVAILABLE
-//=============================================================================
-#if defined(INCLUDE_TEMPLATES) && !defined(COMISO_CPLEXSOLVER_C)
-#define COMISO_CPLEXSOLVER_TEMPLATES
-#include "CPLEXSolverT.cc"
-#endif
-//=============================================================================
 #endif // ACG_CPLEXSOLVER_HH defined
 //=============================================================================
 

NSolver/DOCloudCache.cc

+//== INCLUDES =================================================================
+
+//== COMPILE-TIME PACKAGE REQUIREMENTS ========================================
+#include <CoMISo/Config/config.hh>
+#if COMISO_DOCLOUD_AVAILABLE
+
+//=============================================================================
+#include "DOCloudCache.hh"
+#include "DOCloudConfig.hh"
+
+#include <Base/Debug/DebOut.hh>
+
+#include <fstream>
+#include <iomanip>
+#include <cctype>
+#include <functional>
+#include <sstream>
+
+#include <boost/filesystem.hpp>
+
+#include <windows.h>
+
+DEB_module("DOCloudCache")
+
+//== NAMESPACES ===============================================================
+namespace COMISO {
+
+namespace DOcloud {
+
+namespace {
+
+// Create a new temporary exclusive file without extension that is used to 
+// prevent write or read operation on files with the same name and extension
+// .lp or .dat. while the cache is being written. This is the only class that
+// uses windows specific APIs.
+class FileLock
+{
+public:
+  FileLock(const std::string& _filename)
+  {
+    file_hnd_ = CreateFile(_filename.c_str(),
+      GENERIC_WRITE,
+      0,            // ShareMode - 0 prevents any sharing
+      nullptr,      // SecurityAttributes
+      CREATE_NEW,   // Fails if file already exists.
+      FILE_ATTRIBUTE_NORMAL | FILE_FLAG_DELETE_ON_CLOSE, // File attributes.
+      NULL);
+  }
+
+  // We can write the DoCloud results only id the lock is successful.
+  bool sucess() const { return file_hnd_ != INVALID_HANDLE_VALUE; }
+
+  // If there is an active lock we can not read the related data because they 
+  // are being written.
+  static bool active(const std::string& _filename)
+  {
+    return GetFileAttributes(_filename.c_str()) != INVALID_FILE_ATTRIBUTES;
+  }
+
+  // The destructor removes the lock. from this moment the data can be freely
+  // read and there should not be anyone who tries to rewrite them.
+  ~FileLock()
+  {
+    if (sucess())
+      CloseHandle(file_hnd_); // This will delete the file.
+  }
+
+private:
+  HANDLE file_hnd_;
+};
+
+
+bool load_file(const std::string& _filename, std::string& _file_cnts)
+{
+  std::ifstream in_file_strm(_filename, std::ios::ate);
+  if (!in_file_strm.is_open())
+    return false;
+  _file_cnts.reserve(in_file_strm.tellg());
+  in_file_strm.seekg(0, std::ios::beg);
+  _file_cnts.assign(std::istreambuf_iterator<char>(in_file_strm),
+                    std::istreambuf_iterator<char>());
+  return true;
+}
+
+bool save_file(const std::string& _filename, const std::string& _file_cnts)
+{
+  std::ofstream out_file_strm(_filename);
+  if (!out_file_strm.is_open())
+    return false;
+  out_file_strm << _file_cnts;
+  return true;
+}
+
+// Finds a key string from the file name. This string will be used as file name
+// where to store the related cached data.
+std::string string_to_hash(const std::string& _str)
+{
+  const std::hash<std::string> hash_fn;
+  std::stringstream strm;
+  strm << std::hex << hash_fn(_str);
+  return strm.str();
+}
+
+const size_t NO_SOLUTION_CODE = UINT_MAX;
+
+// Load variables and objective values from a file.
+bool load_data(const std::string& _filename, 
+               std::vector<double>& _x, double& _obj_val)
+{
+  std::ifstream in_file_strm(_filename);
+  if (!in_file_strm.is_open())
+    return false;
+
+  size_t dim = std::numeric_limits<size_t>::max();
+  in_file_strm >> dim;
+  if (dim == NO_SOLUTION_CODE)
+  {
+    _x.clear();
+    return true;
+  }
+
+  if (dim != _x.size())
+    return false;
+
+  for (auto& xi : _x)
+    in_file_strm >> xi;
+  in_file_strm >> _obj_val;
+  return !in_file_strm.bad();
+}
+
+// Store variables and objective values in a file.
+bool save_data(const std::string& _filename, 
+               const std::vector<double>& _x, const double& _obj_val)
+{
+  std::ofstream out_file_strm(_filename);
+  out_file_strm << std::setprecision(std::numeric_limits<double>::digits10 + 2);
+  if (!out_file_strm.is_open())
+    return false;
+
+  if (_x.empty())
+  {
+    out_file_strm << NO_SOLUTION_CODE;
+    return true;
+  }
+  out_file_strm << _x.size() << std::endl;
+  for (const auto& xi : _x)
+    out_file_strm << xi << std::endl;;
+  out_file_strm << _obj_val;
+  return !out_file_strm.bad();
+}
+
+} // namespace
+
+Cache::Cache(const std::string& _mip_lp) 
+  : mip_lp_(_mip_lp), hash_(string_to_hash(mip_lp_)), found_(false) 
+{
+  DEB_enter_func;
+  DEB_line(2, "Cache hash: " << hash_);
+}
+
+bool Cache::restore_result(std::vector<double>& _x, double& _obj_val) 
+{
+  DEB_enter_func;
+  const auto* cache_loc = Config::query().cache_location();
+  if (cache_loc == nullptr) // cache location not provided, disabale the cache
+    return false;
+
+  for (size_t iter_nmbr = 0; iter_nmbr < 10; ++iter_nmbr)
+  {
+    filename_ = cache_loc + hash_ + '_' + std::to_string(iter_nmbr);
+
+    std::string dat_filename(filename_ + ".dat");
+    boost::system::error_code err_cod;
+    if (!boost::filesystem::exists(
+          boost::filesystem::path(dat_filename.c_str()), err_cod) ||
+         err_cod.value() != boost::system::errc::success)
+    {
+      // If the .dat file does not exist it is not safe to check the lock because
+      // it is possible that this process finds no lock, another process sets the
+      // lock and start writing data, this process reads not fully written data.
+      break;
+    }
+
+    if (FileLock::active(filename_))
+      break;
+
+    std::string cache_cnts;
+    if (!load_file(filename_ + ".lp", cache_cnts))
+      break;
+
+    if (cache_cnts == mip_lp_)
+    {
+      found_ = load_data(filename_ + ".dat", _x, _obj_val);
+      return found_;
+    }
+  }
+  return false;
+}
+
+namespace {
+// Save the couple of files fname.lp and fname.dat . They are an element of a 
+// sort of map from file.lp to file.dat. So if there is an error saving the .dat
+// file, the .lp file must also e deleted.
+class CacheSaver
+{
+public:
+  CacheSaver() : success_(false) {}
+
+  ~CacheSaver()
+  {
+    if (success_)
+      return;
+    // Removes files eventually written if there has been any kind of failure.
+    for (const auto& filename : used_files_)
+    {
+      if (!filename.empty())
+        std::remove(filename.c_str());
+    }
+  }
+
+  void save(const std::string& _filename, const std::vector<double>& _x,
+    const double& _obj_val, const std::string& _lp_cnts)
+  {
+    DEB_enter_func;
+
+    FileLock file_lock(_filename);
+    if (file_lock.sucess())
+    {
+      used_files_[0] = _filename + ".lp";
+      success_ = save_file(used_files_[0], _lp_cnts);
+      if (success_)
+      {
+        used_files_[1] = _filename + ".dat";
+        success_ = save_data(used_files_[1], _x, _obj_val);
+      }
+    }
+  }
+
+private:
+  bool success_;
+  std::string used_files_[2];
+};
+
+} // namespace
+
+void Cache::store_result(const std::vector<double>& _x, const double& _obj_val)
+{
+  DEB_enter_func;
+  if (filename_.empty() || found_)
+  {// restore_result() either not called at all, or hit the cache
+    DEB_error("store_result() called incorrectly");
+    return;
+  }
+  CacheSaver saver;
+  saver.save(filename_, _x, _obj_val, mip_lp_);
+}
+
+} // namespace DOcloud
+} // namespace COMISO
+
+#endif // COMISO_DOCLOUD_AVAILABLE
+//=============================================================================
+

NSolver/DOCloudCache.hh

+#ifndef COMISO_DOCloudCache_HH
+#define COMISO_DOCloudCache_HH
+
+//== COMPILE-TIME PACKAGE REQUIREMENTS ========================================
+#include <CoMISo/Config/config.hh>
+#if COMISO_DOCLOUD_AVAILABLE
+
+//== INCLUDES =================================================================
+
+#include <string>
+#include <vector>
+
+//== NAMESPACES ===============================================================
+namespace COMISO {
+namespace DOcloud {
+
+// given a .lp file, checks if we have saved the result for the same problem.
+// If so they are returned, so the calling function can avoid to compute them.
+//
+
+class Cache
+{
+public:
+  // Manage the cache for an optimization problem stored in .lp format
+  Cache(const std::string& _mip_lp);
+
+  const std::string& hash() const { return hash_; }
+
+  bool restore_result(
+    std::vector<double>& _x,  // result.
+    double& _obj_val // objective function value.
+    ); 
+
+  // We can store the result for the given .lp file. This makes sense only we have
+  // not found cache data for the given .lp file, and in order to avoid data
+  // corruption this function fails if the data have been found.
+  void store_result(const std::vector<double>& _x, const double& _obj_val);
+
+
+private:
+  const std::string& mip_lp_; // The MIP represented in the .lp format 
+  const std::string hash_; // hask for the lp_ problem
+  bool found_; // Remembers if we have found a cache for the input problem
+  std::string filename_; // File name to access the cached data (no extension)
+};
+
+} // namespace DOcloud
+} // namespace COMISO
+//=============================================================================
+
+#endif // COMISO_DOCLOUD_AVAILABLE
+//=============================================================================
+#endif // COMISO_DOCloudCache_HH
+//=============================================================================