/////////////////////////////////////////////////////////////////////////
// SymmetricExtensionGraphMasterMaster.hh
// produced: 22/03/2020 jr
/////////////////////////////////////////////////////////////////////////
#ifndef SYMMETRICEXTENSIONGRAPHMASTER_HH
#define SYMMETRICEXTENSIONGRAPHMASTER_HH

#include <iostream>
#include <iomanip>
#include <fstream>
#include <string>
#include <sstream>
#include <filesystem>

#include <assert.h>
#include <deque>
#include <vector>
#include <set>
#include <map>

#include <thread>
#include <chrono>
#include <atomic>
#include <condition_variable>

#include "Signal.hh"
#include "Global.hh"
#include "ContainerIO.hh"
#include "CommandlineOptions.hh"
#include "Options.hh"
#include "Graphics.hh"
#include "Cache.hh"

#include "LabelSet.hh"
#include "SimplicialComplex.hh"

#include "ConfData.hh"
#include "PointConfiguration.hh"
#include "Chirotope.hh"
#include "Symmetry.hh"
#include "CheckSymmetries.hh"
#include "ClassifiedExtensionSymmetries.hh"
#include "SwitchTable.hh"
#include "Admissibles.hh"
#include "Incidences.hh"
#include "Volumes.hh"
#include "PartialTriang.hh"
#include "SymmetricExtensionGraph.hh"
#include "RegularityCheck.hh"
#include "ScreenReport.hh"
#include "CheckPoint.hh"

namespace topcom {

  class SymmetricExtensionGraphMaster;

  class SymmetricExtensionGraphMaster {
  public:
    typedef typename SymmetricExtensionGraph::node_type node_type;
  private:
    class Worker {
    private:
      
      // context information:
      const int                            _workerID;
      const SymmetricExtensionGraphMaster* _callerptr;
      const node_type*                     _root_nodeptr;
      
      // results:
      size_type                            _totalcount;
      size_type                            _symcount;
      size_type                            _nodecount;
      size_type                            _deadendcount;
      size_type                            _earlydeadendcount;
      Cache<SimplicialComplex>             _local_triangcache;
      size_type&                           _mincard;
      SimplicialComplex&                   _mintriang;
      
      // statistics:
      size_type                            _seecount;
      int                                  _maxiter_coversimptighten; 
      size_type                            _no_of_runs;
      
      // multi-threading:
      std::deque<node_type>                _open_nodes;
      enum State { idle, hired, busy, done, interrupted, stopped };
      State                                _state;
    public:
      mutable std::condition_variable      worker_condition;
    private:
      Worker();
    public:
      Worker(const int,
	     const SymmetricExtensionGraphMaster&);
      Worker(const Worker&);
    private:
      const Worker& operator=(const Worker&);
    public:
      
      // accessors:
      inline int       workerID()                                const { return _workerID; }
      inline size_type totalcount()                              const { return _totalcount; }
      inline size_type symcount()                                const { return _symcount; }
      inline size_type nodecount()                               const { return _nodecount; }
      inline size_type deadendcount()                            const { return _deadendcount; }
      inline size_type earlydeadendcount()                       const { return _earlydeadendcount; }
      inline const Cache<SimplicialComplex>& local_triangcache() const { return _local_triangcache; }
      inline size_type mincard()                                 const { return _mincard; }
      inline const SimplicialComplex& mintriang()                const { return _mintriang; }
      inline size_type maxiter_coversimptighten()                const { return _maxiter_coversimptighten; }
      
      // accessor to statistics:
      inline size_type seecount()                                const { return _seecount; }
      inline size_type no_of_runs()                              const { return _no_of_runs; }
      
      // accessor to the non-const open nodes structure because we want to move elements from it to the caller:
      inline       std::deque<node_type>& open_nodes()        { return _open_nodes; }
      
      // accessors for multi-threading control:
      inline bool check_state(const State state)  const {
	std::lock_guard<std::mutex> main_lock(_callerptr->_main_waitmutex);
	return (_state == state);
      }
      inline bool is_idle()                       const { return (check_state(State::idle)); }
      inline bool is_hired()                      const { return (check_state(State::hired)); }
      inline bool is_busy()                       const { return (check_state(State::busy)); }
      inline bool is_done()                       const { return (check_state(State::done)); }
      inline bool is_interrupted()                const { return (check_state(State::interrupted)); }
      inline bool is_stopped()                    const { return (check_state(State::stopped)); }
      
      // modifiers for multi-threading control:
      inline void set_state(const State state) {
	std::lock_guard<std::mutex> main_lock(_callerptr->_main_waitmutex);
	_state = state;
      }
      inline void set_idle()                            { set_state(State::idle); }
      inline void set_hired()                           { set_state(State::hired); }
      inline void set_busy()                            { set_state(State::busy); }
      inline void set_done()                            { set_state(State::done); }
      inline void set_interrupted()                     { set_state(State::interrupted); }
      inline void set_stopped()                         { set_state(State::stopped); }
      inline bool wake_up()                       const;
      inline void interrupt_worker();
      inline void stop_worker();
      inline void pass_work(const node_type*);
      inline void clear_results();
      inline void run();
      void        operator()();
    };
  private:
    static const std::string              _dump_startline;
    static const std::string              _dump_endline;
  private:

    // the following are input data preprocessed from input:
    const ConfData&                       _confdata;
    const Options&                        _options;
  private:

    // root node of the exploration (empty in the beginning, otherwise from _open_nodes):
    node_type*                            _rootptr;
  private:
    
    // members holding results, updated lock-free by master after a worker has finished:
    size_type                             _totalcount;
    size_type                             _symcount;
    size_type                             _nodecount;
    size_type                             _deadendcount;
    size_type                             _earlydeadendcount;
    Cache<SimplicialComplex>              _triangcache;

    // members holding results, updated directly by workers using a lock:
    mutable size_type                     _mincard;
    mutable SimplicialComplex             _mintriang;
  private:

    // structures to report intermediate results:
    ScreenReport                          _screenreport;
    CheckPoint                            _checkpoint;
    int                                   _dump_no;
  private:

    // multi-threading parameters:
    int                                   _no_of_threads;
    size_type                             _node_budget;

    // multi-threading structures:
    std::vector<std::thread>              _threads;
    std::vector<Worker>                   _workers;
    std::deque<node_type>                 _open_nodes;
    size_type                             _current_workbuffersize;
    bool                                  _interrupt;
    mutable int                           _no_of_idle_threads;
    mutable int                           _no_of_pending_results;
    mutable std::mutex                    _main_waitmutex;
    mutable std::condition_variable       _main_condition;
    friend class SymmetricExtensionGraphMaster::Worker;

  private:

    // statistics:
    size_type                             _seecount; // no. of single-element evaluations
    std::string                           _seemstname; // label for modified-switch-table method
    std::string                           _seecetname; // label for critical-element method
    int                                   _maxiter_coversimptighten; // max. no. of non-trivial strong-pruning rounds
  private:
    SymmetricExtensionGraphMaster();
    SymmetricExtensionGraphMaster(const SymmetricExtensionGraphMaster&);
    SymmetricExtensionGraphMaster& operator=(const SymmetricExtensionGraphMaster&);
  public:
    // constructor:
    inline SymmetricExtensionGraphMaster(const ConfData&,
					 const Options&);
    // destructor:
    inline ~SymmetricExtensionGraphMaster();

    // accessors for multi-threading control:
    inline int get_no_of_idle_threads() const {
      std::lock_guard<std::mutex> main_lock(_main_waitmutex);
      return _no_of_idle_threads;
    }
    inline int get_no_of_pending_results() const {
      std::lock_guard<std::mutex> main_lock(_main_waitmutex);
      return _no_of_pending_results;
    }
    
    // modifiers for multi-threading control:
    inline void set_no_of_idle_threads(const int i) const {
      std::lock_guard<std::mutex> main_lock(_main_waitmutex);
      _no_of_idle_threads = i;
    }
    inline void set_no_of_pending_results(const int i) const {
      std::lock_guard<std::mutex> main_lock(_main_waitmutex);
      _no_of_pending_results = i;
    }
    inline void increase_no_of_idle_threads() const {
      std::lock_guard<std::mutex> main_lock(_main_waitmutex);
      ++_no_of_idle_threads;
    }
    inline void increase_no_of_pending_results() const {
      std::lock_guard<std::mutex> main_lock(_main_waitmutex);
      ++_no_of_pending_results;
    }
    inline void decrease_no_of_idle_threads() const {
      std::lock_guard<std::mutex> main_lock(_main_waitmutex);
      --_no_of_idle_threads;
    }
    inline void decrease_no_of_pending_results() const {
      std::lock_guard<std::mutex> main_lock(_main_waitmutex);
      --_no_of_pending_results;
    }

    // adapting to workload:
    inline void update_node_budget();

    // reporting:
    inline Message& report_progress(Message&, bool force = false);

    // checkpointing:
    inline void trigger_checkpoint(const bool force = false);
    inline void interrupt_workers();
    inline void uninterrupt_workers();
    inline void report_checkpoint();
    
    // results:
    inline size_type totalcount()               const { return _totalcount; }
    inline size_type symcount()                 const { return _symcount; }
    inline size_type nodecount()                const { return _nodecount; }
    inline size_type deadendcount()             const { return _deadendcount; }
    inline size_type earlydeadendcount()        const { return _earlydeadendcount; }
    inline size_type mincard()                  const { return _mincard; }
    inline const SimplicialComplex& mintriang() const { return _mintriang; }
    inline size_type workbuffersize()           const { return _open_nodes.size(); }

    // statistics:
    inline size_type maxiter_coversimptighten() const { return _maxiter_coversimptighten; }
    inline size_type seecount()                 const { return _seecount; }
    inline std::string seemstname()             const { return _seemstname; }
    inline std::string seecetname()             const { return _seecetname; }
    
    // multi-threading control:
    inline bool wake_up() const;

    // internal methods:
    inline bool _exists_idle_thread() const;
    inline bool _exists_pending_result() const;
    inline bool _all_threads_idle() const;
    void        _collect_results();
    void _run();

    // collect init/term steps:
    void _init();
    void _init_workers();
    void _term();
    void _term_workers();

    // stream output/input:
    std::ostream& write(std::ostream& ost) const;
    inline friend std::ostream& operator<<(std::ostream& ost, const SymmetricExtensionGraphMaster& segm) {
      return segm.write(ost);
    }
    std::istream& read(std::istream& ist);
    inline friend std::istream& operator>>(std::istream& ist, SymmetricExtensionGraphMaster& segm) {
      return segm.read(ist);
    }
  };

  inline SymmetricExtensionGraphMaster::SymmetricExtensionGraphMaster(const ConfData& confdata,
								      const Options& options) :
    _confdata(confdata),
    _options(options),
    _rootptr(0),
    _totalcount(0),
    _symcount(0),
    _nodecount(0),
    _deadendcount(0),
    _earlydeadendcount(0),
    _triangcache(),
    _mincard(std::numeric_limits<size_type>::max()),
    _mintriang(),
    _screenreport(),
    _checkpoint(),
    _dump_no(0),
    _no_of_threads(1),
    _node_budget(1),
    _open_nodes(),
    _current_workbuffersize(0UL),
    _interrupt(false),
    _seecount(0),
    _seemstname("no. of single-element evaluations (modified switch-table method)"),
    _seecetname("no. of single-element evaluations (critical-element method)"),
    _maxiter_coversimptighten(0) {

    // initialize remaining data:
    _init();

    // initialize signal handling:
    Signal::init();

    // explore:
    _run();
  }

  inline SymmetricExtensionGraphMaster::~SymmetricExtensionGraphMaster() {
    _term();
  }

  // budgeting:
  inline void SymmetricExtensionGraphMaster::update_node_budget() {
    if (CommandlineOptions::workbuffercontrol()) {
      if (CommandlineOptions::dump_status()) {
	_node_budget = CommandlineOptions::dump_frequency();
      }
      else {
	_node_budget = std::numeric_limits<size_type>::max();
      }
    }
    else {
      _node_budget = std::max<size_type>(1, _open_nodes.size() * _open_nodes.size() * CommandlineOptions::scale_nodebudget() / 100);
      if (_node_budget < CommandlineOptions::min_nodebudget()) {
	_node_budget = CommandlineOptions::min_nodebudget();
      }
      else if (_node_budget > CommandlineOptions::max_nodebudget()) {
	_node_budget = CommandlineOptions::max_nodebudget();
      }
      if (CommandlineOptions::dump_status() && (_node_budget > CommandlineOptions::dump_frequency())) {
	_node_budget = CommandlineOptions::dump_frequency();
      }
    }
  }

  // reporting:
  inline Message& SymmetricExtensionGraphMaster::report_progress(Message& msg, bool force) {
    if (force || _screenreport.decide_report(this->_nodecount)) {
      _screenreport.execute_ext_master_report(msg,
					      this->get_no_of_idle_threads(),
					      this->_nodecount,
					      this->_symcount,
					      this->_totalcount,
					      this->_deadendcount,
					      this->_earlydeadendcount,
					      this->_open_nodes.size(),
					      CommandlineOptions::min_workbuffersize(),
					      this->_node_budget);
    }
    return msg;
  }
  
  // checkpointing:
  inline void SymmetricExtensionGraphMaster::trigger_checkpoint(bool force) {
    if (force || _checkpoint.decide_dump(this->_nodecount)) {
      interrupt_workers();
      report_checkpoint();
      uninterrupt_workers();
    }
  }
  
  inline void SymmetricExtensionGraphMaster::interrupt_workers() {
    _interrupt = true;      
    if (CommandlineOptions::parallel_enumeration()) {
      
      // parallel enumeration requires to sync all workers before reporting the checkpoint;
      // the following will cause the workers to stop at the next possible occasion and
      // the master will wait for all workers to get idle:
      while (_no_of_idle_threads < _no_of_threads) {
	{
	  std::unique_lock<std::mutex> main_lock(_main_waitmutex);
	  _main_condition.wait(main_lock, [this] { return wake_up(); } );
	}
	
	// collection of results is important after
	// any worker is done in order to change its state to idle:
	_collect_results();
      }
    } 
  }

  inline void SymmetricExtensionGraphMaster::uninterrupt_workers() {
    _interrupt = false;
  }

  inline void SymmetricExtensionGraphMaster::report_checkpoint() {
    if (!CommandlineOptions::dump_status()) {
      
      // no dump requested -- no checkpoint:
      return;
    }
    _checkpoint.open_dump();
    {
      std::unique_lock<std::mutex> main_lock(_main_waitmutex);
      _checkpoint.dump_stream() << _dump_startline << std::endl;
      Message(_checkpoint.dump_stream()).print_timestamp();
      write(_checkpoint.dump_stream());
      _checkpoint.dump_stream() << _dump_endline << std::endl;
    }
    _checkpoint.close_dump();
  }

  inline bool SymmetricExtensionGraphMaster::wake_up() const {

    // use shared data directly since lock is organized by the wait of the condition variable:
    return (Signal::signal_received()) // need to handle signal
      || (_no_of_pending_results > 0) // need to collect results
      || (!_open_nodes.empty() && (_no_of_idle_threads > 0)) // can distribute work
      || (_open_nodes.empty() && (_no_of_idle_threads == _no_of_threads)); // need to terminate
  }

  inline bool SymmetricExtensionGraphMaster::_exists_idle_thread() const {
    return (get_no_of_idle_threads() > 0);
  }

  inline bool SymmetricExtensionGraphMaster::_exists_pending_result() const {
    return (get_no_of_pending_results() > 0);
  }

  inline bool SymmetricExtensionGraphMaster::_all_threads_idle() const {
    return (get_no_of_idle_threads() >= _no_of_threads);
  }

  // inlines for worker subclass:
  inline bool SymmetricExtensionGraphMaster::Worker::wake_up() const {

    // no lock here because the lock is held by the condition variable already:
    return ((_state == State::hired) || (_state == State::stopped));
  }

  inline void SymmetricExtensionGraphMaster::Worker::pass_work(const node_type* nodeptr) {
    MessageStreams::debug() << message::lock
			    << message::tab << "worker " << _workerID << " is assigned to work on "
			    << *nodeptr << " ..." << std::endl
			    << message::unlock;
    _callerptr->decrease_no_of_idle_threads();
    _root_nodeptr               = nodeptr;
    set_hired();
    MessageStreams::debug() << message::tab << "... done" << std::endl;
  }
  
  inline void SymmetricExtensionGraphMaster::Worker::run() {
    MessageStreams::debug() << message::lock
			    << message::tab << "constructing new enumerator ..." << std::endl
			    << message::unlock;
    set_busy();
    SymmetricExtensionGraph seg(_workerID,
				_no_of_runs,
				_callerptr->_symcount,
				_callerptr->_confdata,
				_callerptr->_options,
				_root_nodeptr,
				_callerptr->_triangcache,
				_mincard,
				_mintriang,
				_open_nodes,
				_callerptr->_node_budget,
				&_callerptr->_current_workbuffersize,
				&_callerptr->_interrupt);
    _nodecount                = seg.nodecount();
    _totalcount               = seg.totalcount();
    _symcount                 = seg.symcount();
    _deadendcount             = seg.deadendcount();
    _earlydeadendcount        = seg.earlydeadendcount();
    _local_triangcache.add(seg.local_triangcache());
    _seecount                 = seg.seecount();
    if (seg.maxiter_coversimptighten() > _maxiter_coversimptighten) {
      _maxiter_coversimptighten = seg.maxiter_coversimptighten();
    }
    ++_no_of_runs;
    delete(_root_nodeptr);
    set_done();
    _callerptr->increase_no_of_pending_results();
    MessageStreams::debug() << message::lock
			    << message::tab << "... enumerator of worker " << _workerID << " going out of scope" << std::endl
			    << message::unlock;
  }

  inline void SymmetricExtensionGraphMaster::Worker::clear_results() {
    _root_nodeptr           = 0;
    _symcount               = 0;
    _totalcount             = 0;
    _nodecount              = 0;
    _deadendcount           = 0;
    _earlydeadendcount      = 0;
    _local_triangcache.clear();
    _seecount               = 0;
    _open_nodes.clear();
    _callerptr->decrease_no_of_pending_results();
    set_idle();
    _callerptr->increase_no_of_idle_threads();
  }

  inline void SymmetricExtensionGraphMaster::Worker::interrupt_worker() {
    set_interrupted();
  }

  inline void SymmetricExtensionGraphMaster::Worker::stop_worker() {
    set_stopped();
  }

}; // namespace topcom

#endif

// eof SymmetricExtensionGraphMaster.hh
