diff --git a/Config/sequencerd.cfg.in b/Config/sequencerd.cfg.in index 61d81707..5a9cbbce 100644 --- a/Config/sequencerd.cfg.in +++ b/Config/sequencerd.cfg.in @@ -190,17 +190,6 @@ CAL_TARGET=(SCIENCE close open on on on on off off off off off off # UPDATE_TIME_REMAINING=10 # block updates to active target with less than this exptime (sec) remaining -# ----------------------------------------------------------------------------- -# TELEM_PROVIDER=( ) -# -# This is a list of telemetry providers where is the daemon name, -# and is the port on which to send the telemetry request. -# Provide one per line. -# -TELEM_PROVIDER=(slitd @SLITD_NB_PORT@) -TELEM_PROVIDER=(tcsd @TCSD_NB_PORT@) -TELEM_PROVIDER=(camerad @CAMERAD_NB_PORT@) -# SUBSCRIBE_TO=(slitd "tcp://127.0.0.1:@SLITD_PUB_PORT@") SUBSCRIBE_TO=(tcsd "tcp://127.0.0.1:@TCSD_PUB_PORT@") SUBSCRIBE_TO=(camerad "tcp://127.0.0.1:@CAMERAD_PUB_PORT@") diff --git a/acamd/acam_interface.cpp b/acamd/acam_interface.cpp index f19fbfcf..f616b332 100644 --- a/acamd/acam_interface.cpp +++ b/acamd/acam_interface.cpp @@ -1507,7 +1507,7 @@ namespace Acam { /***** Acam::Interface::request_snapshot ************************************/ /** - * @brief publises request for snapshot + * @brief [obsolete] publises request for snapshot * @details publishing Topic::SNAPSHOT induces subscribers to publish a * snapshot of their telemetry * @@ -1534,7 +1534,7 @@ namespace Acam { /***** Acam::Interface::wait_for_snapshots **********************************/ /** - * @brief wait for everyone to publish their snaphots + * @brief [obsolete] wait for everyone to publish their snaphots * @details When forcing subscribers to publish their telemetry, * this waits until they have done so. * @@ -1599,33 +1599,32 @@ namespace Acam { * */ void Interface::handletopic_tcsd( const nlohmann::json &jmessage ) { - { - std::lock_guard lock(snapshot_mtx); - snapshot_status[Topic::TCSD]=true; - } + + std::lock_guard lock(tcsdata_mtx); + // extract and store values in the class // - Common::extract_telemetry_value( jmessage, "TCSNAME", telem.tcsname ); - Common::extract_telemetry_value( jmessage, "ISOPEN", telem.is_tcs_open ); - Common::extract_telemetry_value( jmessage, "CASANGLE", telem.angle_scope ); - Common::extract_telemetry_value( jmessage, "TELRA", telem.ra_scope_hms ); - Common::extract_telemetry_value( jmessage, "TELDEC", telem.dec_scope_dms ); - Common::extract_telemetry_value( jmessage, "RA", telem.ra_scope_h ); - Common::extract_telemetry_value( jmessage, "DEC", telem.dec_scope_d ); - Common::extract_telemetry_value( jmessage, "RAOFFSET", telem.offsetra ); - Common::extract_telemetry_value( jmessage, "DECLOFFS", telem.offsetdec ); - Common::extract_telemetry_value( jmessage, "AZ", telem.az ); - Common::extract_telemetry_value( jmessage, "TELFOCUS", telem.telfocus ); - Common::extract_telemetry_value( jmessage, "AIRMASS", telem.airmass ); + Common::extract_telemetry_value( jmessage, "TCSNAME", tcsdata.tcsname ); + Common::extract_telemetry_value( jmessage, "ISOPEN", tcsdata.is_tcs_open ); + Common::extract_telemetry_value( jmessage, Key::Tcsd::CASANGLE, tcsdata.angle_scope ); + Common::extract_telemetry_value( jmessage, Key::Tcsd::TELRA, tcsdata.ra_scope_hms ); + Common::extract_telemetry_value( jmessage, Key::Tcsd::TELDEC, tcsdata.dec_scope_dms ); + Common::extract_telemetry_value( jmessage, "RA", tcsdata.ra_scope_h ); + Common::extract_telemetry_value( jmessage, "DEC", tcsdata.dec_scope_d ); + Common::extract_telemetry_value( jmessage, "RAOFFSET", tcsdata.offsetra ); + Common::extract_telemetry_value( jmessage, "DECLOFFS", tcsdata.offsetdec ); + Common::extract_telemetry_value( jmessage, Key::Tcsd::AZ, tcsdata.az ); + Common::extract_telemetry_value( jmessage, "TELFOCUS", tcsdata.telfocus ); + Common::extract_telemetry_value( jmessage, Key::Tcsd::AIRMASS, tcsdata.airmass ); // save them to the database // - this->database.add_key_val( "CASANGLE", telem.angle_scope ); - this->database.add_key_val( "RAtel", telem.ra_scope_h ); - this->database.add_key_val( "DECLtel", telem.dec_scope_d ); - this->database.add_key_val( "AZ", telem.az ); - this->database.add_key_val( "focus", telem.telfocus ); - this->database.add_key_val( "AIRMASS", telem.airmass ); + this->database.add_key_val( "CASANGLE", tcsdata.angle_scope ); + this->database.add_key_val( "RAtel", tcsdata.ra_scope_h ); + this->database.add_key_val( "DECLtel", tcsdata.dec_scope_d ); + this->database.add_key_val( "AZ", tcsdata.az ); + this->database.add_key_val( "focus", tcsdata.telfocus ); + this->database.add_key_val( "AIRMASS", tcsdata.airmass ); } /***** Acam::Interface::handletopic_tcsd ************************************/ @@ -1638,10 +1637,6 @@ namespace Acam { * */ void Interface::handletopic_targetinfo( const nlohmann::json &jmessage ) { - { - std::lock_guard lock(snapshot_mtx); - snapshot_status[Topic::TARGETINFO]=true; - } this->database.add_from_json( jmessage, "OBS_ID" ); this->database.add_from_json( jmessage, "NAME" ); this->database.add_from_json( jmessage, "POINTMODE" ); @@ -1659,10 +1654,6 @@ namespace Acam { * */ void Interface::handletopic_slitd( const nlohmann::json &jmessage ) { - { - std::lock_guard lock(snapshot_mtx); - snapshot_status[Topic::SLITD]=true; - } this->telemkeys.add_json_key(jmessage, "SLITO", "SLITO", "slit offset in arcsec", "FLOAT", false); this->telemkeys.add_json_key(jmessage, "SLITW", "SLITW", "slit width in arcsec", "FLOAT", false); } @@ -2706,7 +2697,7 @@ namespace Acam { do { if ( iface.camera.andor.camera_info.exptime == 0 ) continue; // wait for non-zero exposure time - if ( iface.collect_header_info() == ERROR ) { // collect header information + if ( iface.assemble_header_info() == ERROR ) { // assemble header keyword information logwrite(function,"ERROR collecting header info"); continue; } @@ -4083,6 +4074,10 @@ logwrite( function, message.str() ); // if ( this->motion.is_open() ) error |= this->motion.cover( "close", dontcare ); + // diable target acquisition + // + error |= this->acquire( "stop", dontcare); + // stop the framegrab thread // error |= this->framegrab( "stop", dontcare ); @@ -4375,7 +4370,7 @@ logwrite( function, message.str() ); do { logwrite( function, std::to_string(nacquires) ); error |= this->camera.andor.acquire_one(); // acquire a single image - error |= this->collect_header_info(); // collect header information + error |= this->assemble_header_info(); // assemble header keyword information error |= this->camera.write_frame( "", this->imagename, this->tcs_online.load() ); // write to FITS file @@ -4420,7 +4415,7 @@ logwrite( function, message.str() ); do { logwrite( function, std::to_string(nacquires) ); error |= this->camera.andor.get_recent(3000); // - error |= this->collect_header_info(); // collect header information + error |= this->assemble_header_info(); // assemble header keyword information error |= this->camera.write_frame( "", this->imagename, this->tcs_online.load() ); // write to FITS file @@ -4454,7 +4449,7 @@ logwrite( function, message.str() ); // if ( tokens[1]=="getrecent" ) { error = this->camera.andor.get_recent(3000); - error |= this->collect_header_info(); // collect header information + error |= this->assemble_header_info(); // assemble header keyword information error |= this->camera.write_frame( "", this->imagename, this->tcs_online.load() ); // write to FITS file @@ -4583,7 +4578,7 @@ logwrite( function, message.str() ); retstring.append( " Gather information and add it to the internal keyword database.\n" ); return HELP; } - else error = this->collect_header_info(); // collect header information + else error = this->assemble_header_info(); // assemble header keyword information } else // -------------------------------- @@ -5005,7 +5000,7 @@ logwrite( function, message.str() ); /***** Acam::Interface::solve ***********************************************/ - /***** Acam::Interface::collect_header_info *********************************/ + /***** Acam::Interface::assemble_header_info ********************************/ /** * @brief gather information and add it to the internal keyword database * @details Some of the keys are fixed, some come from the Andor::Information @@ -5016,24 +5011,24 @@ logwrite( function, message.str() ); * @return ERROR or NO_ERROR * */ - long Interface::collect_header_info() { - // force subscribers to publish now, then wait - // esults in struct telem. - this->request_snapshot(); - this->wait_for_snapshots(); - - bool _tcs = telem.is_tcs_open; - std::string tcsname = ( _tcs ? telem.tcsname : "offline" ); + long Interface::assemble_header_info() { double angle_acam=NAN, ra_acam=NAN, dec_acam=NAN; // outputs from fpoffsets - if ( _tcs ) this->target.save_casangle( telem.angle_scope ); // store in the Target class, required for acquisition + // ---------- scope lock tcsdata -------------- + { + std::lock_guard lock(tcsdata_mtx); + + bool _tcs = tcsdata.is_tcs_open; + std::string tcsname = ( _tcs ? tcsdata.tcsname : "offline" ); + + if ( _tcs ) this->target.save_casangle( tcsdata.angle_scope ); // store in the Target class, required for acquisition // Compute FP offsets from TCS coordinates (SCOPE) to ACAM coodinates. // compute_offset() always wants degrees and get_coords() returns RA hours. // Results in degrees. // - if ( _tcs ) this->fpoffsets.compute_offset( "SCOPE", "ACAM", (telem.ra_scope_h*TO_DEGREES), telem.dec_scope_d, telem.angle_scope, + if ( _tcs ) this->fpoffsets.compute_offset( "SCOPE", "ACAM", (tcsdata.ra_scope_h*TO_DEGREES), tcsdata.dec_scope_d, tcsdata.angle_scope, ra_acam, dec_acam, angle_acam ); // Get some info from the Andor::Information class, @@ -5054,11 +5049,21 @@ logwrite( function, message.str() ); this->camera.fitsinfo.fitskeys.addkey( "TCS", tcsname, "" ); + this->camera.fitsinfo.fitskeys.addkey( "TELFOCUS", tcsdata.telfocus, "telescope focus (mm)" ); + this->camera.fitsinfo.fitskeys.addkey( "AIRMASS", tcsdata.airmass, "" ); + this->camera.fitsinfo.fitskeys.addkey( "RA", tcsdata.ra_scope_hms, "Telecscope Right Ascension" ); + this->camera.fitsinfo.fitskeys.addkey( "DEC", tcsdata.dec_scope_dms, "Telescope Declination" ); + this->camera.fitsinfo.fitskeys.addkey( "TELRA", tcsdata.ra_scope_h, "Telecscope Right Ascension hours" ); + this->camera.fitsinfo.fitskeys.addkey( "TELDEC", tcsdata.dec_scope_d, "Telescope Declination degrees" ); + this->camera.fitsinfo.fitskeys.addkey( "RAOFFS", tcsdata.offsetra, "Telescope RA offset" ); + this->camera.fitsinfo.fitskeys.addkey( "DECLOFFS", tcsdata.offsetdec, "Telescope DEC offset" ); + this->camera.fitsinfo.fitskeys.addkey( "CASANGLE", tcsdata.angle_scope, "Cassegrain ring angle" ); + } + // ---------- end scope lock tcsdata ---------- + this->camera.fitsinfo.fitskeys.addkey( "CREATOR", "acamd", "file creator" ); this->camera.fitsinfo.fitskeys.addkey( "INSTRUME", "NGPS", "name of instrument" ); this->camera.fitsinfo.fitskeys.addkey( "TELESCOP", "P200", "name of telescope" ); - this->camera.fitsinfo.fitskeys.addkey( "TELFOCUS", telem.telfocus, "telescope focus (mm)" ); - this->camera.fitsinfo.fitskeys.addkey( "AIRMASS", telem.airmass, "" ); // get parameters from FPOffsets, results are stored in the class // @@ -5109,13 +5114,6 @@ logwrite( function, message.str() ); this->camera.fitsinfo.fitskeys.addkey( "POSANG", angle_acam, "" ); this->camera.fitsinfo.fitskeys.addkey( "TARGET", this->target.get_name(), "target name" ); - this->camera.fitsinfo.fitskeys.addkey( "RA", telem.ra_scope_hms, "Telecscope Right Ascension" ); - this->camera.fitsinfo.fitskeys.addkey( "DEC", telem.dec_scope_dms, "Telescope Declination" ); - this->camera.fitsinfo.fitskeys.addkey( "TELRA", telem.ra_scope_h, "Telecscope Right Ascension hours" ); - this->camera.fitsinfo.fitskeys.addkey( "TELDEC", telem.dec_scope_d, "Telescope Declination degrees" ); - this->camera.fitsinfo.fitskeys.addkey( "RAOFFS", telem.offsetra, "Telescope RA offset" ); - this->camera.fitsinfo.fitskeys.addkey( "DECLOFFS", telem.offsetdec, "Telescope DEC offset" ); - this->camera.fitsinfo.fitskeys.addkey( "CASANGLE", telem.angle_scope, "Cassegrain ring angle" ); this->camera.fitsinfo.fitskeys.addkey( "WCSAXES", 2, "" ); this->camera.fitsinfo.fitskeys.addkey( "RADESYSA", "ICRS", "" ); this->camera.fitsinfo.fitskeys.addkey( "CTYPE1", "RA---TAN", "" ); @@ -5137,7 +5135,7 @@ logwrite( function, message.str() ); return NO_ERROR; } - /***** Acam::Interface::collect_header_info *********************************/ + /***** Acam::Interface::assemble_header_info ********************************/ /***** Acam::Interface::target_coords ***************************************/ diff --git a/acamd/acam_interface.h b/acamd/acam_interface.h index 8c4b317e..04e29eb8 100644 --- a/acamd/acam_interface.h +++ b/acamd/acam_interface.h @@ -555,7 +555,9 @@ namespace Acam { double az; double telfocus; double airmass; - } telem; + } tcsdata; + + std::mutex tcsdata_mtx; std::mutex snapshot_mtx; std::unordered_map snapshot_status; @@ -613,8 +615,14 @@ namespace Acam { inline std::string get_imagename() { return this->imagename; } inline std::string get_wcsname() { return this->wcsname; } - inline void set_imagename( std::string name_in ) { this->imagename = ( name_in.empty() ? DEFAULT_IMAGENAME : name_in ); return; } - inline void set_wcsname( std::string name_in ) { this->wcsname = name_in; return; } + inline void set_imagename( std::string name_in ) { + this->imagename = ( name_in.empty() ? DEFAULT_IMAGENAME : std::move(name_in) ); + return; + } + inline void set_wcsname( std::string name_in ) { + this->wcsname = std::move(name_in); + return; + } GuideManager guide_manager; @@ -688,7 +696,7 @@ namespace Acam { long exptime( const std::string args, std::string &retstring ); long fan_mode( std::string args, std::string &retstring ); - long collect_header_info(); + long assemble_header_info(); inline void init_names() { imagename=""; wcsname=""; return; } // TODO still needed? diff --git a/camerad/astrocam.cpp b/camerad/astrocam.cpp index 23d4db37..64b8d4b3 100644 --- a/camerad/astrocam.cpp +++ b/camerad/astrocam.cpp @@ -31,6 +31,7 @@ namespace AstroCam { // build JSON message with my telemetry jmessage_out[Key::SOURCE] = "camerad"; jmessage_out[Key::Camerad::READY] = this->can_expose.load(); + jmessage_out[Key::Camerad::SHUTTERTIME] = this->camera.shutter.get_duration(); // publish JSON message try { diff --git a/camerad/simulator-arc.cpp b/camerad/simulator-arc.cpp index a85f4a6c..dece6c3b 100644 --- a/camerad/simulator-arc.cpp +++ b/camerad/simulator-arc.cpp @@ -43,7 +43,7 @@ namespace AstroCam { // If no string is given then use vector of configured devices // if ( devices_in.empty() ) { - this->devnums = this->configured_devnums; + this->connected_devnums = this->configured_devnums; } else { // Otherwise, tokenize the device list string and build devnums from the tokens @@ -53,8 +53,8 @@ namespace AstroCam { for ( const auto &n : tokens ) { // For each token in the devices_in string, try { int dev = std::stoi( n ); // convert to int - if ( std::find( this->devnums.begin(), this->devnums.end(), dev ) == this->devnums.end() ) { // If it's not already in the vector, - this->devnums.push_back( dev ); // then push into devnums vector. + if ( std::find( this->connected_devnums.begin(), this->connected_devnums.end(), dev ) == this->connected_devnums.end() ) { // If it's not already in the vector, + this->connected_devnums.push_back( dev ); // then push into devnums vector. } } catch (std::invalid_argument &) { @@ -76,7 +76,7 @@ namespace AstroCam { // For each requested dev in devnums, if there is a matching controller in the config file, // then get the devname and store it in the controller map. // - for ( const auto &dev : this->devnums ) { + for ( const auto &dev : this->connected_devnums ) { if ( this->controller.find( dev ) != this->controller.end() ) { this->controller[ dev ].devname = "sim"+std::to_string(dev); } @@ -84,7 +84,7 @@ namespace AstroCam { // set the controller connected state true // - for ( const auto &dev : this->devnums ) { + for ( const auto &dev : this->connected_devnums ) { this->controller[dev].connected = true; } @@ -110,7 +110,7 @@ namespace AstroCam { // clear the controller connected state // - for ( const auto &dev : this->devnums ) { + for ( const auto &dev : this->connected_devnums ) { this->controller[dev].connected = false; } @@ -150,7 +150,7 @@ namespace AstroCam { std::stringstream lodfilestream; // But only use it if the device is open // - if ( std::find( this->devnums.begin(), this->devnums.end(), fw->first ) != this->devnums.end() ) { + if ( std::find( this->connected_devnums.begin(), this->connected_devnums.end(), fw->first ) != this->connected_devnums.end() ) { lodfilestream << fw->first << " " << fw->second; // Call do_load_firmware with the built up string. @@ -452,14 +452,14 @@ namespace AstroCam { } /***** AstroCam::Interface::native ******************************************/ - - long Interface::_image_size( std::string args, std::string &retstring, const bool save_as_default ) { - std::string function = "AstroCam::Interface::_image_size"; - std::stringstream message; - logwrite( function, "NOP" ); - return( NO_ERROR ); - } - +/* + *long Interface::_image_size( std::string args, std::string &retstring, const bool save_as_default ) { + * std::string function = "AstroCam::Interface::_image_size"; + * std::stringstream message; + * logwrite( function, "NOP" ); + * return( NO_ERROR ); + *} + */ /***** AstroCam::Simulator::dothread_expose *********************************/ /** diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 3fc96925..cdbdba1d 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -11,22 +11,33 @@ set(PROJECT_UTILS_DIR ${PROJECT_BASE_DIR}/common) set( CMAKE_CXX_STANDARD 17 ) set( CMAKE_CXX_STANDARD_REQUIRED ON ) -include_directories( ${PROJECT_BASE_DIR}/common ) -include_directories( ${PROJECT_BASE_DIR}/utils ) # needed for logentry - find_path( PYTHON_DEV "Python.h" PATHS /usr/include/python3.9 ) find_library( PYTHON_LIB python3.9 NAMES libpython3.9 PATHS /usr/lib64 ) find_library( ZMQPP_LIB zmqpp NAMES libzmqpp PATHS /usr/local/lib ) find_library( ZMQ_LIB zmq NAMES libzmq PATHS /usr/local/lib ) +if(NOT ZMQPP_LIB) + message(FATAL_ERROR "libzmqpp not found") +endif() +if(NOT ZMQ_LIB) + message(FATAL_ERROR "libzmq not found") +endif() + +include_directories( ${PROJECT_BASE_DIR}/common ) +include_directories( ${PROJECT_BASE_DIR}/utils ) # needed for logentry include_directories( ${PYTHON_DEV} ) add_library(common STATIC ${PROJECT_UTILS_DIR}/common.cpp + ) +target_link_libraries(common PUBLIC ${ZMQPP_LIB} ${ZMQ_LIB} ) + add_library(skyinfo STATIC ${PROJECT_UTILS_DIR}/skyinfo.cpp + ) +target_link_libraries(skyinfo ${PYTHON_LIB} ) diff --git a/common/common.cpp b/common/common.cpp index 3fc176eb..bfd417e8 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -9,6 +9,38 @@ namespace Common { + /***** Common::Broadcaster::emit ********************************************/ + /** + * @brief logs a narrative message and publishes it on Topic::BROADCAST + * @param[in] function name of caller (used for log line) + * @param[in] severity one of Severity::NOTICE, Severity::WARNING, Severity::ERROR + * @param[in] message operator-facing narrative text + * @details Logs message via logwrite, then publishes a JSON payload + * on Topic::BROADCAST if the publisher has been initialized. + * + */ + void Broadcaster::emit( const std::string &function, + const std::string &severity, + const std::string &message ) { + logwrite( function, severity+": "+message ); + + if ( ! this->publisher ) return; + + nlohmann::json jmessage; + jmessage[Key::SOURCE] = this->source; + jmessage[Key::Broadcast::SEVERITY] = severity; + jmessage[Key::Broadcast::MESSAGE] = message; + + try { + this->publisher->publish( jmessage, Topic::BROADCAST ); + } + catch ( const std::exception &e ) { + logwrite( function, "ERROR publishing broadcast: "+std::string(e.what()) ); + } + } + /***** Common::Broadcaster::emit ********************************************/ + + /***** Common::collect_telemetry ********************************************/ /** * @brief send the TELEMREQUEST command to daemon to get telemetry diff --git a/common/common.h b/common/common.h index d540dbd3..b8c768f9 100644 --- a/common/common.h +++ b/common/common.h @@ -22,6 +22,7 @@ #include "logentry.h" #include "network.h" +#include "message_keys.h" const long NOTHING = -1; const long NO_ERROR = 0; @@ -369,6 +370,56 @@ namespace Common { }; + /**************** Common::Broadcaster ***************************************/ + /** + * @class Broadcaster + * @brief logs a narrative message and publishes it on Topic::BROADCAST + * @details Captures a reference to a publisher and the source daemon name + * at construction time so that call sites need only supply the + * caller function name and message (and severity, for emit). + * The publisher reference is to the daemon's Common::PubSub, which + * may be null at the time this Broadcaster is constructed and gets + * populated later by init_pubsub. + * + */ + class Broadcaster { + private: + const std::unique_ptr &publisher; ///< reference to owner's publisher + std::string source; ///< source daemon name + + public: + Broadcaster( const std::unique_ptr &publisher, + std::string source ) + : publisher(publisher), source(std::move(source)) { } + + /** + * @brief publish a NOTICE severity broadcast + */ + inline void notice( const std::string &function, const std::string &message ) { + this->emit( function, Severity::NOTICE, message ); + } + + /** + * @brief publish a WARNING severity broadcast + */ + inline void warning( const std::string &function, const std::string &message ) { + this->emit( function, Severity::WARNING, message ); + } + + /** + * @brief publish an ERROR severity broadcast + */ + inline void error( const std::string &function, const std::string &message ) { + this->emit( function, Severity::ERROR, message ); + } + + void emit( const std::string &function, + const std::string &severity, + const std::string &message ); + }; + /**************** Common::Broadcaster ***************************************/ + + void collect_telemetry(const std::pair &provider, std::string &retstring); /***** Common::extract_telemetry_value **************************************/ diff --git a/common/message_keys.h b/common/message_keys.h index ab7d8a46..ef738a7e 100644 --- a/common/message_keys.h +++ b/common/message_keys.h @@ -8,13 +8,39 @@ #include +namespace Daemon { + inline const std::string ACAMD = "acamd"; + inline const std::string CALIBD = "calibd"; + inline const std::string CAMERAD = "camerad"; + inline const std::string FLEXURED = "flexured"; + inline const std::string FOCUSD = "focusd"; + inline const std::string POWERD = "powerd"; + inline const std::string SEQUENCER = "sequencerd"; + inline const std::string SLICECAMD = "slicecamd"; + inline const std::string SLITD = "slitd"; + inline const std::string TCSD = "tcsd"; + inline const std::string THERMALD = "thermald"; +} + +namespace Severity { + inline const std::string NOTICE = "NOTICE"; + inline const std::string WARNING = "WARNING"; + inline const std::string ERROR = "ERROR"; +} + namespace Topic { inline const std::string SNAPSHOT = "_snapshot"; inline const std::string TARGETINFO = "targetinfo"; + inline const std::string BROADCAST = "broadcast"; inline const std::string TCSD = "tcsd"; inline const std::string SLITD = "slitd"; inline const std::string CAMERAD = "camerad"; inline const std::string ACAMD = "acamd"; + inline const std::string CALIBD = "calibd"; + inline const std::string FLEXURED = "flexured"; + inline const std::string FOCUSD = "focusd"; + inline const std::string POWERD = "powerd"; + inline const std::string THERMALD = "thermald"; inline const std::string SEQ_DAEMONSTATE = "seq_daemonstate"; inline const std::string SEQ_SEQSTATE = "seq_seqstate"; inline const std::string SEQ_THREADSTATE = "seq_threadstate"; @@ -26,12 +52,23 @@ namespace Key { inline const std::string SOURCE = "source"; + namespace Broadcast { + inline const std::string SEVERITY = "severity"; + inline const std::string MESSAGE = "message"; + } + namespace Sequencer { inline const std::string SEQSTATE = "seqstate"; } namespace Camerad { - inline const std::string READY = "ready"; + inline const std::string READY = "ready"; + inline const std::string SHUTTERTIME = "shuttime_sec"; + inline const std::string EXPTIME = "exptime"; + inline const std::string IMNUM = "imnum"; + inline const std::string IMNAME = "imname"; + inline const std::string FRAMECOUNT = "framecount"; + inline const std::string FRAMETRANSFER = "frametransfer"; } namespace Acamd { @@ -50,4 +87,23 @@ namespace Key { inline const std::string FINEACQUIRE_LOCKED = "fineacquire_locked"; inline const std::string FINEACQUIRE_RUNNING = "fineacquire_running"; } + + namespace Slitd { + inline const std::string SLITPOSA = "slitposa"; + inline const std::string SLITPOSB = "slitposb"; + inline const std::string SLITW = "slitw"; + inline const std::string SLITO = "slito"; + inline const std::string ISOPEN = "isopen"; + inline const std::string ISHOME = "ishome"; + } + + namespace Tcsd { + inline const std::string TELRA = "TELRA"; + inline const std::string TELDEC = "TELDEC"; + inline const std::string ALT = "ALT"; + inline const std::string AZ = "AZ"; + inline const std::string AIRMASS = "AIRMASS"; + inline const std::string CASANGLE = "CASANGLE"; + } + } diff --git a/common/tcsd_commands.h b/common/tcsd_commands.h index e8cffebc..01fa1f85 100644 --- a/common/tcsd_commands.h +++ b/common/tcsd_commands.h @@ -26,6 +26,7 @@ const std::string TCSD_NATIVE = "native"; const std::string TCSD_OFFSETRATE = "offsetrate"; const std::string TCSD_OPEN = "open"; const std::string TCSD_PTOFFSET = "offset"; +const std::string TCSD_PUBLISHSTATE = "publishstate"; const std::string TCSD_RETOFFSETS = "retoffsets"; const std::string TCSD_RINGGO = "ringgo"; const std::string TCSD_SET_FOCUS = "setfocus"; @@ -51,6 +52,7 @@ const std::vector TCSD_SYNTAX = { TCSD_OFFSETRATE+" [ ? | ]", TCSD_OPEN+" ? | ", TCSD_PTOFFSET+" ? | ", + TCSD_PUBLISHSTATE+" ? | on | off", TCSD_RETOFFSETS+" [ ? ]", TCSD_RINGGO+" ? | ", TCSD_SET_FOCUS+" ? | ", diff --git a/flexured/flexure_interface.cpp b/flexured/flexure_interface.cpp index 55288f71..626eaa9c 100644 --- a/flexured/flexure_interface.cpp +++ b/flexured/flexure_interface.cpp @@ -520,8 +520,8 @@ namespace Flexure { else if ( messagetype == "tcsinfo" ) { double casangle=NAN, alt=NAN; - Common::extract_telemetry_value( message_in, "CASANGLE", casangle ); - Common::extract_telemetry_value( message_in, "ALT", alt ); + Common::extract_telemetry_value( message_in, Key::Tcsd::CASANGLE, casangle ); + Common::extract_telemetry_value( message_in, Key::Tcsd::ALT, alt ); message.str(""); message << "casangle=" << casangle << " alt=" << alt; logwrite( function, message.str() ); } diff --git a/sequencerd/sequence.cpp b/sequencerd/sequence.cpp index 5db8147d..61110c60 100644 --- a/sequencerd/sequence.cpp +++ b/sequencerd/sequence.cpp @@ -48,16 +48,37 @@ namespace Sequencer { * */ void Sequence::handletopic_camerad(const nlohmann::json &jmessage) { + // when I write to the completed table I will write the actual EXPTIME + this->target.column_from_json( "EXPTIME", Key::Camerad::SHUTTERTIME, jmessage ); + + // updates my internal state whether the camera allows an exposure if (jmessage.contains(Key::Camerad::READY)) { int isready = jmessage[Key::Camerad::READY].get(); this->can_expose.store(isready, std::memory_order_relaxed); - std::lock_guard lock(camerad_mtx); - this->camerad_cv.notify_all(); } + + std::lock_guard lock(camerad_mtx); + this->camerad_cv.notify_all(); } /***** Sequencer::Sequence::handletopic_camerad ****************************/ + /***** Sequencer::Sequence::handletopic_slitd ******************************/ + /** + * @brief handles Topic::SLITD telemetry + * @param[in] jmessage subscribed-received JSON message + * + */ + void Sequence::handletopic_slitd(const nlohmann::json &jmessage) { + this->target.column_from_json( "SLITWIDTH", Key::Slitd::SLITW, jmessage ); + this->target.column_from_json( "SLITOFFSET", Key::Slitd::SLITO, jmessage ); + + std::lock_guard lock(slitd_mtx); + this->slitd_cv.notify_all(); + } + /***** Sequencer::Sequence::handletopic_slitd ******************************/ + + /***** Sequencer::Sequence::handletopic_slicecamd **************************/ /** * @brief handles Topic::SLICECAMD telemetry @@ -65,7 +86,7 @@ namespace Sequencer { * */ void Sequence::handletopic_slicecamd(const nlohmann::json &jmessage) { - // set is_fineacquire_locked flag + // updates my internal state whether fineacquire is locked bool fineacquirelocked; Common::extract_telemetry_value( jmessage, Key::Slicecamd::FINEACQUIRE_LOCKED, fineacquirelocked ); this->is_fineacquire_locked.store(fineacquirelocked, std::memory_order_relaxed); @@ -75,6 +96,27 @@ namespace Sequencer { /***** Sequencer::Sequence::handletopic_slicecamd **************************/ + /***** Sequencer::Sequence::handletopic_tcsd *******************************/ + /** + * @brief handles Topic::TCSD telemetry + * @param[in] jmessage subscribed-received JSON message + * + */ + void Sequence::handletopic_tcsd(const nlohmann::json &jmessage) { + // the completed target table contains TCS data now + this->target.column_from_json( "TELRA", Key::Tcsd::TELRA, jmessage ); + this->target.column_from_json( "TELDECL", Key::Tcsd::TELDEC, jmessage ); + this->target.column_from_json( "ALT", Key::Tcsd::ALT, jmessage ); + this->target.column_from_json( "AZ", Key::Tcsd::AZ, jmessage ); + this->target.column_from_json( "AIRMASS", Key::Tcsd::AIRMASS, jmessage ); + this->target.column_from_json( "CASANGLE", Key::Tcsd::CASANGLE, jmessage ); + + std::lock_guard lock(tcsd_mtx); + this->tcsd_cv.notify_all(); + } + /***** Sequencer::Sequence::handletopic_tcsd *******************************/ + + /***** Sequencer::Sequence::handletopic_acamd ******************************/ /** * @brief handles Topic::ACAMD telemetry @@ -82,7 +124,7 @@ namespace Sequencer { * */ void Sequence::handletopic_acamd(const nlohmann::json &jmessage) { - // set is_acam_guiding flag + // updates my internal state whether acam is guiding bool acquired; Common::extract_telemetry_value( jmessage, Key::Acamd::IS_ACQUIRED, acquired ); this->is_acam_guiding.store(acquired, std::memory_order_relaxed); @@ -94,16 +136,12 @@ namespace Sequencer { /***** Sequencer::Sequence::publish_snapshot *******************************/ /** - * @brief publishes snapshot of my telemetry + * @brief publishes snapshot of all of my telemetry * @details This publishes a JSON message containing a snapshot of my * telemetry. * */ void Sequence::publish_snapshot() { - std::string dontcare; - this->publish_snapshot(dontcare); - } - void Sequence::publish_snapshot(std::string &retstring) { this->publish_seqstate(); this->publish_waitstate(); this->publish_daemonstate(); @@ -232,23 +270,25 @@ namespace Sequencer { /***** Sequencer::Sequence::broadcast_daemonstate ***************************/ /** * @brief publishes daemonstate and can control seqstate - * @details If not STARTING or STOPPING and not all daemons ready then - * this ensures that the seqstate drops into NOTREADY. + * @details Daemon-readiness changes may only force the sequencer into + * NOTREADY when the current seqstate is itself READY or + * NOTREADY. Lifecycle states (STARTING, STOPPING, RUNNING, + * PAUSED, ABORTING, FAILED) are owned by the lifecycle + * functions and are never overridden here. The one-hot + * seqstate contract is preserved. * */ void Sequence::broadcast_daemonstate() { // always publish daemonstate when called this->publish_daemonstate(); - // If any daemon isn't ready then the sequencer can't be ready, - // but don't override STARTING or STOPPING, unless none are ready. - if ( daemon_manager.are_all_clear() ) { - seq_state_manager.set_only( {Sequencer::SEQ_NOTREADY} ); - } - else - if ( ! seq_state_manager.is_set(SEQ_STARTING) && - ! seq_state_manager.is_set(SEQ_STOPPING) && - ! daemon_manager.are_all_set() ) { + // Only degrade seqstate to NOTREADY when the sequencer is currently + // READY or NOTREADY. Never override an active lifecycle transition + // (STARTING, STOPPING, RUNNING, PAUSED, ABORTING) or FAILED. + // + if ( ! daemon_manager.are_all_set() && + seq_state_manager.are_any_set( Sequencer::SEQ_READY, + Sequencer::SEQ_NOTREADY ) ) { seq_state_manager.set_only( {Sequencer::SEQ_NOTREADY} ); } } @@ -257,31 +297,42 @@ namespace Sequencer { /***** Sequencer::Sequence::broadcast_seqstate ******************************/ /** - * @brief writes string of seq_state to the async port - * @details This broadcasts the seqstate as a string with the "SEQSTATE:" - * message tag. + * @brief publishes seq_state on the SEQ_SEQSTATE topic + * @details Legacy UDP "SEQSTATE:" async strings have been removed. + * Seqstate is now broadcast only via PUB-SUB. * */ void Sequence::broadcast_seqstate() { + const std::string function("Sequencer::Sequence::broadcast_seqstate"); + + // publish the structured seqstate topic + // this->publish_seqstate(); - this->async.enqueue_and_log( "Sequencer::Sequence::broadcast_seqstate", - "SEQSTATE: "+seq_state_manager.get_set_states() ); this->cv.notify_all(); + + // emit a NOTICE on Topic::BROADCAST only when the lifecycle state has + // actually changed, so operators (and logs) get a breadcrumb trail of + // state transitions without noise from repeated identical callbacks. + // + std::string current( this->seq_state_manager.get_set_states() ); + rtrim( current ); + if ( current != this->last_seqstate_str ) { + this->last_seqstate_str = current; + this->broadcast.notice( function, "sequencer state: "+current ); + } } /***** Sequencer::Sequence::broadcast_seqstate ******************************/ /***** Sequencer::Sequence::broadcast_waitstate *****************************/ /** - * @brief writes string of all set wait_state bits to the asyn port - * @details This broadcasts the seqstate as a string with the "WAITSTATE:" - * message tag. + * @brief publishes wait_state on the SEQ_WAITSTATE topic + * @details Legacy UDP "WAITSTATE:" async strings have been removed. + * Waitstate is now broadcast only via PUB-SUB. * */ void Sequence::broadcast_waitstate() { this->publish_waitstate(); - this->async.enqueue_and_log( "Sequencer::Sequence::broadcast_waitstate", - "WAITSTATE: "+wait_state_manager.get_set_states() ); this->cv.notify_all(); } /***** Sequencer::Sequence::broadcast_waitstate *****************************/ @@ -420,26 +471,26 @@ namespace Sequencer { { ScopedState wait_state( wait_state_manager, Sequencer::SEQ_WAIT_USER ); - this->async.enqueue_and_log( function, "NOTICE: waiting for USER to send \"continue\" signal" ); + this->broadcast.notice( function, "waiting for USER to send \"continue\" signal" ); while ( !this->cancel_flag.load() && !this->is_usercontinue.load() ) { std::unique_lock lock(cv_mutex); this->cv.wait( lock, [this]() { return( this->is_usercontinue.load() || this->cancel_flag.load() ); } ); } - this->async.enqueue_and_log( function, "NOTICE: received " - +(this->cancel_flag.load() ? std::string("cancel") : std::string("continue")) - +" signal!" ); + this->broadcast.notice( function, "received " + +(this->cancel_flag.load() ? std::string("cancel") : std::string("continue")) + +" signal!" ); } // end scope for wait_state = WAIT_USER if ( this->cancel_flag.load() ) { - this->async.enqueue_and_log( function, "NOTICE: sequence cancelled" ); + this->broadcast.notice( function, "sequence cancelled" ); return ABORT; } this->is_usercontinue.store(false); - this->async.enqueue_and_log( function, "NOTICE: received USER continue signal!" ); + this->broadcast.notice( function, "received USER continue signal!" ); return NO_ERROR; } @@ -471,14 +522,14 @@ namespace Sequencer { // The Sequencer can only be started once // if ( thread_state_manager.is_set( Sequencer::THR_SEQUENCE_START ) ) { - this->async.enqueue_and_log( function, "ERROR sequencer already running" ); + this->broadcast.error( function, "sequencer already running" ); return; } // The Sequencer can only be started when state is READY // if ( ! seq_state_manager.is_set( Sequencer::SEQ_READY ) ) { - this->async.enqueue_and_log( function, "ERROR cannot start: system not ready" ); + this->broadcast.error( function, "cannot start: system not ready" ); return; } @@ -527,7 +578,7 @@ namespace Sequencer { } message.str(""); message << "NOTICE: " << targetstatus; - this->async.enqueue( message.str() ); // broadcast target status + this->broadcast.notice(function, message.str()); if ( targetstate == TargetInfo::TARGET_FOUND ) { // target found, get the threads going @@ -536,9 +587,9 @@ namespace Sequencer { // if ( ! this->daemon_manager.is_set( Sequencer::DAEMON_TCS ) ) { if ( ! this->target.ra_hms.empty() || ! this->target.dec_dms.empty() ) { - message.str(""); message << "ERROR cannot move to target " << this->target.name + message.str(""); message << "cannot move to target " << this->target.name << " because TCS is not connected"; - this->async.enqueue_and_log( function, message.str() ); + this->broadcast.error( function, message.str() ); this->thread_error_manager.set( THR_SEQUENCE_START ); // report error break; } @@ -549,14 +600,6 @@ namespace Sequencer { this->thread_error_manager.set( THR_SEQUENCE_START ); // report any error break; } - - // let the world know of the state change - // - message.str(""); message << "TARGETSTATE:" << this->target.state << " TARGET:" << this->target.name << " OBSID:" << this->target.obsid; - this->async.enqueue( message.str() ); -#ifdef LOGLEVEL_DEBUG - logwrite( function, "[DEBUG] target found, starting threads" ); -#endif } else // targetstate not TARGET_FOUND if ( targetstate == TargetInfo::TARGET_NOT_FOUND ) { // no target found is an automatic stop @@ -565,7 +608,7 @@ namespace Sequencer { } else if ( targetstate == TargetInfo::TARGET_ERROR ) { // request stop on error - this->async.enqueue_and_log( function, "ERROR getting next target. stopping" ); + this->broadcast.error( function, "getting next target. stopping" ); break; } @@ -632,14 +675,14 @@ namespace Sequencer { logwrite(function, "DONE waiting on threads"); if ( this->cancel_flag.load() ) { - this->async.enqueue_and_log( function, "NOTICE: sequence cancelled" ); + this->broadcast.notice( function, "sequence cancelled" ); return; } // For pointmode ACAM, there is nothing to be done so get out // if ( this->target.pointmode == Acam::POINTMODE_ACAM ) { - this->async.enqueue_and_log( function, "NOTICE: target list processing has stopped" ); + this->broadcast.notice( function, "target list processing has stopped" ); break; } @@ -649,16 +692,16 @@ namespace Sequencer { // start ACAM acquisition. If it fails then wait for user to continue or cancel. if ( this->do_acam_acquire() != NO_ERROR ) { - this->async.enqueue_and_log( function, "WARNING acam acquisition failed" ); + this->broadcast.warning( function, "acam acquisition failed" ); if (this->wait_for_user()==ABORT) { - this->async.enqueue_and_log( function, "NOTICE: cancelled" ); + this->broadcast.notice( function, "cancelled" ); return; } } else // start SLICECAM fine acquisition if ( this->do_slicecam_fineacquire() != NO_ERROR ) { - this->async.enqueue_and_log( function, "WARNING slicecam fine acquisition failed" ); + this->broadcast.warning( function, "slicecam fine acquisition failed" ); } } @@ -666,7 +709,7 @@ namespace Sequencer { // send offsets. wait for user if that fails to continue or cancel. if ( this->target_offset() == ERROR ) { if (this->wait_for_user()==ABORT) { - this->async.enqueue_and_log( function, "NOTICE: cancelled" ); + this->broadcast.notice( function, "cancelled" ); return; } } @@ -707,7 +750,7 @@ namespace Sequencer { // When an exposure is aborted then it will be marked as UNASSIGNED // if ( this->cancel_flag.load() ) { - this->async.enqueue_and_log( function, "NOTICE: exposure cancelled" ); + this->broadcast.notice( function, "exposure cancelled" ); error = this->target.update_state( Sequencer::TARGET_UNASSIGNED ); message.str(""); message << ( error==NO_ERROR ? "" : "ERROR " ) << "marking target " << this->target.name << " id " << this->target.obsid << " order " << this->target.obsorder @@ -716,7 +759,7 @@ namespace Sequencer { return; } - this->async.enqueue_and_log( function, "NOTICE: done waiting for expose" ); + this->broadcast.notice( function, "done waiting for expose" ); message.str(""); message << "exposure complete for target " << this->target.name << " id " << this->target.obsid << " order " << this->target.obsorder; logwrite( function, message.str() ); @@ -740,22 +783,12 @@ namespace Sequencer { break; } - // before writing to the completed database table, get current - // telemetry from other daemons. - // - this->get_external_telemetry(); - // Update this target's state in the database // error = this->target.update_state( Sequencer::TARGET_COMPLETE ); // update the active target table if (error==NO_ERROR) error = this->target.insert_completed(); // insert into the completed table if (error!=NO_ERROR) this->thread_error_manager.set( THR_SEQUENCE_START ); // report any error - // let the world know of the state change - // - message.str(""); message << "TARGETSTATE:" << this->target.state << " TARGET:" << this->target.name << " OBSID:" << this->target.obsid; - this->async.enqueue( message.str() ); - // Check the "dotype" -- // If this was "do one" then do_once is set and get out now. // @@ -801,7 +834,7 @@ namespace Sequencer { std::unique_lock lock(this->camerad_mtx); if (!this->can_expose.load()) { - this->async.enqueue_and_log(function, "NOTICE: waiting for camera to be ready to expose"); + this->broadcast.notice( function, "waiting for camera to be ready to expose"); this->camerad_cv.wait( lock, [this]() { return( this->can_expose.load() || this->cancel_flag.load() ); @@ -837,14 +870,14 @@ namespace Sequencer { if (!activechans.str().empty()) { std::string cmd = CAMERAD_ACTIVATE + activechans.str(); if (this->camerad.send(cmd, reply)!=NO_ERROR) { - this->async.enqueue_and_log(function, "ERROR sending \""+cmd+"\": "+reply); + this->broadcast.error( function, "sending \""+cmd+"\": "+reply); throw std::runtime_error("camera returned "+reply); } } if (!deactivechans.str().empty()) { std::string cmd = CAMERAD_DEACTIVATE + deactivechans.str(); if (this->camerad.send(cmd, reply)!=NO_ERROR) { - this->async.enqueue_and_log(function, "ERROR sending \""+cmd+"\": "+reply); + this->broadcast.error( function, "sending \""+cmd+"\": "+reply); throw std::runtime_error("camera returned "+reply); } } @@ -857,7 +890,7 @@ namespace Sequencer { long exptime_msec = (long)( this->target.exptime_req * 1000 ); camcmd.str(""); camcmd << CAMERAD_EXPTIME << " " << exptime_msec; if (error==NO_ERROR && (error=this->camerad.send( camcmd.str(), reply ))!=NO_ERROR) { - this->async.enqueue_and_log( function, "ERROR sending \""+camcmd.str()+"\": "+reply ); + this->broadcast.error( function, "sending \""+camcmd.str()+"\": "+reply ); throw std::runtime_error( "camera returned "+reply ); } @@ -865,12 +898,12 @@ namespace Sequencer { // camcmd.str(""); camcmd << CAMERAD_BIN << " spat " << this->target.binspat; if (error==NO_ERROR && (error=this->camerad.send( camcmd.str(), reply ))!=NO_ERROR) { - this->async.enqueue_and_log( function, "ERROR sending \""+camcmd.str()+"\": "+reply ); + this->broadcast.error( function, "sending \""+camcmd.str()+"\": "+reply ); throw std::runtime_error( "camera returned "+reply ); } camcmd.str(""); camcmd << CAMERAD_BIN << " spec " << this->target.binspect; if (error==NO_ERROR && (error=this->camerad.send( camcmd.str(), reply ))!=NO_ERROR) { - this->async.enqueue_and_log( function, "ERROR sending \""+camcmd.str()+"\": "+reply ); + this->broadcast.error( function, "sending \""+camcmd.str()+"\": "+reply ); throw std::runtime_error( "camera returned "+reply ); } @@ -926,12 +959,10 @@ namespace Sequencer { break; } - this->async.enqueue( "NOTICE: moving slit to "+modestr+" position" ); - - logwrite( function, "moving slit to "+slitcmd.str()+" for "+modestr+"position" ); + this->broadcast.notice(function, "moving slit to "+modestr+" position"); if ( this->slitd.command_timeout( slitcmd.str(), reply, SLITD_SET_TIMEOUT ) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR setting slit" ); + this->broadcast.error( function, "setting slit" ); this->thread_error_manager.set( THR_SLIT_SET ); throw std::runtime_error("slit returned: "+reply); } @@ -957,12 +988,11 @@ namespace Sequencer { this->daemon_manager.clear( Sequencer::DAEMON_POWER ); // powerd not ready if ( this->reopen_hardware(this->powerd, POWERD_REOPEN, 10000 ) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR initializing power control" ); + this->broadcast.error( function, "initializing power control" ); throw std::runtime_error("could not initialize power control"); } this->daemon_manager.set( Sequencer::DAEMON_POWER ); // powerd ready - return NO_ERROR; } /***** Sequencer::Sequence::power_init **************************************/ @@ -1009,13 +1039,13 @@ namespace Sequencer { this->thread_error_manager.set( THR_SLIT_INIT ); // assume the worst, clear on success if ( this->set_power_switch(ON, POWER_SLIT, std::chrono::seconds(5)) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR powering slit hardware" ); + this->broadcast.error( function, "powering slit hardware" ); throw std::runtime_error("could not power slit hardware"); } bool was_opened=false; if ( this->open_hardware(this->slitd, was_opened) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR connecting to slit" ); + this->broadcast.error( function, "connecting to slit" ); throw std::runtime_error("could not open connection to slit hardware"); } @@ -1024,7 +1054,7 @@ namespace Sequencer { bool ishomed=false; std::string reply; if ( this->slitd.command( SLITD_ISHOME, reply ) ) { - this->async.enqueue_and_log( function, "ERROR communicating with slit hardware" ); + this->broadcast.error( function, "communicating with slit hardware" ); throw std::runtime_error("could not communicate with slit hardware: "+reply); } this->parse_state( function, reply, ishomed ); @@ -1034,7 +1064,7 @@ namespace Sequencer { if ( !ishomed ) { logwrite( function, "sending home command" ); if ( this->slitd.command_timeout( SLITD_HOME, reply, SLITD_HOME_TIMEOUT ) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR communicating with slit hardware" ); + this->broadcast.error( function, "communicating with slit hardware" ); throw std::runtime_error("could not home slit hardware: "+reply); } } @@ -1044,7 +1074,7 @@ namespace Sequencer { if ( was_opened && !this->config_init["SLIT"].empty() ) { std::string cmd = SLITD_SET+" "+this->config_init["SLIT"]; if ( this->slitd.command_timeout( cmd, reply, SLITD_SET_TIMEOUT ) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR sending \""+cmd+"\" to slit" ); + this->broadcast.error( function, "sending \""+cmd+"\" to slit" ); throw std::runtime_error("slit "+cmd+" returned: "+reply); } } @@ -1098,7 +1128,7 @@ namespace Sequencer { if (error==NO_ERROR && !this->config_shutdown["SLIT"].empty() ) { std::string cmd = SLITD_SET+" "+this->config_shutdown["SLIT"]; if ( this->slitd.command_timeout( cmd, reply, SLITD_SET_TIMEOUT ) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR sending \""+cmd+"\" to slit" ); + this->broadcast.error( function, "sending \""+cmd+"\" to slit" ); throw std::runtime_error(cmd+" returned: "+reply); } } @@ -1108,7 +1138,7 @@ namespace Sequencer { logwrite( function, "closing slit hardware" ); error = this->slitd.command( SLITD_CLOSE, reply ); if ( error != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR closing connection to slit hardware" ); + this->broadcast.error( function, "closing connection to slit hardware" ); throw std::runtime_error("closing slit connection returned: "+reply); } @@ -1145,14 +1175,14 @@ namespace Sequencer { // make sure hardware is powered on // if ( this->set_power_switch(ON, POWER_SLICECAM, std::chrono::seconds(10)) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR initializing slicecam control" ); + this->broadcast.error( function, "initializing slicecam control" ); throw std::runtime_error("could not power slicecam hardware"); } // open connection is all that is needed, slicecamd takes care of everything // if ( this->open_hardware(this->slicecamd, SLICECAMD_OPEN, SLICECAMD_OPEN_TIMEOUT) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR starting slicecam" ); + this->broadcast.error( function, "starting slicecam" ); throw SlicecamException("could not start slicecam"); } @@ -1185,7 +1215,7 @@ namespace Sequencer { // make sure hardware is powered on // if ( this->set_power_switch(ON, POWER_ACAM, std::chrono::seconds(10)) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR powering acam hardware" ); + this->broadcast.error( function, "powering acam hardware" ); throw std::runtime_error("could not power acam hardware"); } @@ -1193,7 +1223,7 @@ namespace Sequencer { // bool was_opened=false; if ( this->open_hardware(this->acamd, ACAMD_OPEN, ACAMD_OPEN_TIMEOUT, was_opened) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR opening acam camera" ); + this->broadcast.error( function, "opening acam camera" ); throw AcamException(ErrorCode::ERROR_ACAM_CAMERA, "could not open acam camera"); } @@ -1204,14 +1234,14 @@ namespace Sequencer { if ( ! this->config_init["ACAM_FILTER"].empty() ) { cmd = ACAMD_FILTER+" "+this->config_init["ACAM_FILTER"]; if ( this->acamd.command_timeout( cmd, reply, ACAMD_MOVE_TIMEOUT ) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR sending \""+cmd+"\" to acamd: "+reply ); + this->broadcast.error( function, "sending \""+cmd+"\" to acamd: "+reply ); throw std::runtime_error("acam "+cmd+" returned: "+reply); } } if ( ! this->config_init["ACAM_COVER"].empty() ) { cmd = ACAMD_COVER+" "+this->config_init["ACAM_COVER"]; if ( this->acamd.command_timeout( cmd, reply, ACAMD_MOVE_TIMEOUT ) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR sending \""+cmd+"\" to acamd: "+reply ); + this->broadcast.error( function, "sending \""+cmd+"\" to acamd: "+reply ); throw std::runtime_error("acam "+cmd+" returned: "+reply); } } @@ -1260,14 +1290,14 @@ namespace Sequencer { } if ( (error=this->connect_to_daemon(this->slicecamd)) != NO_ERROR ) { - this->async.enqueue_and_log(function, "ERROR connecting to slicecamd"); + this->broadcast.error( function, "connecting to slicecamd"); } // close connections between slicecamd and the hardware with which it communicates // logwrite( function, "closing slicecam hardware" ); if ( (error=this->slicecamd.command_timeout( SLICECAMD_SHUTDOWN, reply, SLICECAMD_SHUTDOWN_TIMEOUT )) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR closing connection to slicecam hardware" ); + this->broadcast.error( function, "closing connection to slicecam hardware" ); } // disconnect me from slicecamd, irrespective of any previous error @@ -1278,7 +1308,7 @@ namespace Sequencer { // Turn off power to slicecam hardware. // if ( this->set_power_switch(OFF, POWER_SLICECAM, std::chrono::seconds(0)) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR switching off slicecam" ); + this->broadcast.error( function, "switching off slicecam" ); throw std::runtime_error("could not power off slicecam hardware"); } @@ -1318,14 +1348,14 @@ namespace Sequencer { if ( ! this->config_shutdown["ACAM_FILTER"].empty() ) { cmd = ACAMD_FILTER+" "+this->config_shutdown["ACAM_FILTER"]; if ( this->acamd.command_timeout( cmd, reply, ACAMD_MOVE_TIMEOUT ) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR sending \""+cmd+"\" to acamd: "+reply ); + this->broadcast.error( function, "sending \""+cmd+"\" to acamd: "+reply ); throw std::runtime_error("acam "+cmd+" returned: "+reply); } } if ( ! this->config_shutdown["ACAM_COVER"].empty() ) { cmd = ACAMD_COVER+" "+this->config_shutdown["ACAM_COVER"]; if ( this->acamd.command_timeout( cmd, reply, ACAMD_MOVE_TIMEOUT ) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR sending \""+cmd+"\" to acamd: "+reply ); + this->broadcast.error( function, "sending \""+cmd+"\" to acamd: "+reply ); throw std::runtime_error("acam "+cmd+" returned: "+reply); } } @@ -1338,7 +1368,7 @@ namespace Sequencer { if ( error==NO_ERROR ) { logwrite( function, "closing acam hardware" ); error = this->acamd.command_timeout( ACAMD_SHUTDOWN, ACAMD_SHUTDOWN_TIMEOUT ); - if ( error != NO_ERROR ) this->async.enqueue_and_log( function, "ERROR shutting down acam" ); + if ( error != NO_ERROR ) this->broadcast.error( function, "shutting down acam" ); } // disconnect me from acamd, irrespective of any previous error @@ -1349,7 +1379,7 @@ namespace Sequencer { // Turn off power to acam hardware. // if ( this->set_power_switch(OFF, POWER_ACAM, std::chrono::seconds(0)) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR switching off acam" ); + this->broadcast.error( function, "switching off acam" ); throw std::runtime_error("could not switch off acam"); } @@ -1379,14 +1409,14 @@ namespace Sequencer { // make sure calib hardware is powered if ( this->set_power_switch(ON, POWER_CALIB, std::chrono::seconds(5)) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR powering focus control" ); + this->broadcast.error( function, "powering focus control" ); throw std::runtime_error("could not power focus control"); } // connect to calibd bool was_opened=false; if ( this->open_hardware(this->calibd, was_opened) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR initializing calib control" ); + this->broadcast.error( function, "initializing calib control" ); throw std::runtime_error("could not power calib control"); } @@ -1398,14 +1428,14 @@ namespace Sequencer { std::string reply; long error = this->calibd.command( CALIBD_ISHOME, reply ); if ( error!=NO_ERROR || this->parse_state( function, reply, ishomed ) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR communicating with calib hardware" ); + this->broadcast.error( function, "communicating with calib hardware" ); throw std::runtime_error("could not communicate with calib hardware: "+reply); } // home calib actuators if not already homed if ( !ishomed ) { logwrite( function, "sending home command" ); if ( this->calibd.command_timeout( CALIBD_HOME, reply, CALIBD_HOME_TIMEOUT ) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR communicating with calib hardware" ); + this->broadcast.error( function, "communicating with calib hardware" ); throw std::runtime_error("could not communicate with calib hardware: "+reply); } } @@ -1418,7 +1448,7 @@ namespace Sequencer { if ( !this->config_init["CALIB_DOOR"].empty() ) cmd << " door=" << this->config_init["CALIB_DOOR"]; logwrite( function, "calib default: "+cmd.str() ); if ( this->calibd.command_timeout( cmd.str(), CALIBD_SET_TIMEOUT ) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR moving calib door and/or cover" ); + this->broadcast.error( function, "moving calib door and/or cover" ); throw std::runtime_error("could not move calib door and/or cover"); } } @@ -1455,7 +1485,7 @@ namespace Sequencer { // bool poweron=false; if ( check_power_switch(ON, POWER_CALIB, poweron ) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR checking calib power switch" ); + this->broadcast.error( function, "checking calib power switch" ); throw std::runtime_error("checking calib power switch"); } @@ -1473,7 +1503,7 @@ namespace Sequencer { if ( !this->config_shutdown["CALIB_DOOR"].empty() ) cmd << " door=" << this->config_shutdown["CALIB_DOOR"]; logwrite( function, "calib default: "+cmd.str() ); if ( this->calibd.command_timeout( cmd.str(), CALIBD_SET_TIMEOUT ) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR moving calib door and/or cover" ); + this->broadcast.error( function, "moving calib door and/or cover" ); throw std::runtime_error("moving calib door and/or cover"); } } @@ -1486,7 +1516,7 @@ namespace Sequencer { std::string reply; logwrite( function, "closing calib hardware" ); error = this->calibd.send( CALIBD_CLOSE, reply ); - if ( error != NO_ERROR ) this->async.enqueue_and_log( function, "ERROR closing connection to calib hardware" ); + if ( error != NO_ERROR ) this->broadcast.error( function, "closing connection to calib hardware" ); } // disconnect me from calibd, irrespective of any previous error @@ -1497,14 +1527,14 @@ namespace Sequencer { // Turn off power to calib hardware. // if ( this->set_power_switch(OFF, POWER_CALIB, std::chrono::seconds(0)) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR switching off calib hardware" ); + this->broadcast.error( function, "switching off calib hardware" ); error=ERROR; } // always turn off power to lamps // if ( this->set_power_switch(OFF, POWER_LAMP, std::chrono::seconds(5)) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR powering off lamps" ); + this->broadcast.error( function, "powering off lamps" ); error=ERROR; } @@ -1538,7 +1568,7 @@ namespace Sequencer { this->daemon_manager.clear( Sequencer::DAEMON_TCS ); // tcsd not ready if ( this->open_hardware(this->tcsd) != NO_ERROR ) { - this->async.enqueue_and_log( "Sequencer::Sequence::tcs_init", "ERROR initializing TCS" ); + this->broadcast.error( "Sequencer::Sequence::tcs_init", "initializing TCS" ); this->thread_error_manager.set( THR_TCS_INIT ); throw std::runtime_error("could not initialize TCS"); } @@ -1582,7 +1612,7 @@ namespace Sequencer { std::string reply; error = this->tcsd.send( TCSD_CLOSE, reply ); if ( error != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR: closing connection to TCS" ); + this->broadcast.error( function, "closing connection to TCS" ); throw std::runtime_error("closing TCS connection: "+reply); } } @@ -1616,13 +1646,13 @@ namespace Sequencer { // make sure hardware is powered on // if ( this->set_power_switch(ON, POWER_FLEXURE, std::chrono::seconds(21)) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR powering flexure control" ); + this->broadcast.error( function, "powering flexure control" ); this->thread_error_manager.set( THR_FLEXURE_INIT ); throw std::runtime_error("could not power flexure control"); } if ( this->open_hardware(this->flexured) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR initializing flexure control" ); + this->broadcast.error( function, "initializing flexure control" ); this->thread_error_manager.set( THR_FLEXURE_INIT ); throw std::runtime_error("could not initialize flexure control"); } @@ -1669,7 +1699,7 @@ namespace Sequencer { } if ( this->connect_to_daemon(this->flexured) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR connecting to flexure hardware" ); + this->broadcast.error( function, "connecting to flexure hardware" ); error=ERROR; } @@ -1678,7 +1708,7 @@ namespace Sequencer { // logwrite( function, "closing flexure hardware" ); if (error==NO_ERROR && (error=this->flexured.command( FLEXURED_CLOSE, reply )) != NO_ERROR) { - this->async.enqueue_and_log( function, "ERROR closing connection to flexure hardware" ); + this->broadcast.error( function, "closing connection to flexure hardware" ); } // disconnect me from flexured, irrespective of any previous error @@ -1689,7 +1719,7 @@ namespace Sequencer { // Turn off power to flexure hardware. // if ( this->set_power_switch(OFF, POWER_FLEXURE, std::chrono::seconds(0)) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR switching off flexure" ); + this->broadcast.error( function, "switching off flexure" ); throw std::runtime_error("switching off flexure hardware"); } @@ -1716,14 +1746,14 @@ namespace Sequencer { this->thread_error_manager.set( THR_FOCUS_INIT ); // assume failure, clear on success if ( this->set_power_switch(ON, POWER_FOCUS, std::chrono::seconds(5)) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR powering focus control" ); + this->broadcast.error( function, "powering focus control" ); throw std::runtime_error("could not power focus control"); } // connect to focusd bool was_opened=false; if ( this->open_hardware(this->focusd, was_opened) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR initializing focus control" ); + this->broadcast.error( function, "initializing focus control" ); throw std::runtime_error("could not open focus hardware"); } @@ -1735,14 +1765,14 @@ namespace Sequencer { std::string reply; long error = this->focusd.command( FOCUSD_ISHOME, reply ); if ( error!=NO_ERROR || this->parse_state( function, reply, ishomed ) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR communicating with focus hardware" ); + this->broadcast.error( function, "communicating with focus hardware" ); throw std::runtime_error("focus "+FOCUSD_ISHOME+" returned: "+reply); } // home focus actuators if not already homed if ( !ishomed ) { logwrite( function, "sending home command" ); if ( this->focusd.command_timeout( FOCUSD_HOME, reply, FOCUSD_HOME_TIMEOUT ) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR communicating with focus hardware" ); + this->broadcast.error( function, "communicating with focus hardware" ); throw std::runtime_error("focus "+FOCUSD_HOME+" returned: "+reply); } } @@ -1752,7 +1782,7 @@ namespace Sequencer { for ( const auto &chan : chans ) { std::string command = "set " + chan + " nominal"; if ( this->focusd.command_timeout( command, reply, FOCUSD_SET_TIMEOUT ) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR setting focus "+chan ); + this->broadcast.error( function, "setting focus "+chan ); throw std::runtime_error("focus "+command+" returned: "+reply); } } @@ -1798,7 +1828,7 @@ namespace Sequencer { } if ( this->connect_to_daemon(this->focusd) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR connecting to focus hardware" ); + this->broadcast.error( function, "connecting to focus hardware" ); error=ERROR; } @@ -1807,7 +1837,7 @@ namespace Sequencer { // logwrite( function, "closing focus hardware" ); if (error==NO_ERROR && (error=this->focusd.command( FOCUSD_CLOSE, reply )) != NO_ERROR) { - this->async.enqueue_and_log( function, "ERROR closing connection to focus hardware" ); + this->broadcast.error( function, "closing connection to focus hardware" ); } // disconnect me from focusd, irrespective of any previous error @@ -1818,7 +1848,7 @@ namespace Sequencer { // Turn off power to focus hardware. // if ( this->set_power_switch(OFF, POWER_FOCUS, std::chrono::seconds(0)) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR switching off focus" ); + this->broadcast.error( function, "switching off focus" ); throw std::runtime_error("switching off focus hardware"); } @@ -1846,13 +1876,13 @@ namespace Sequencer { // make sure hardware is powered on // if ( this->set_power_switch(ON, POWER_CAMERA, std::chrono::seconds(5)) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR powering camera" ); + this->broadcast.error( function, "powering camera" ); throw std::runtime_error("switching on camera"); } bool was_opened=false; if ( this->open_hardware(this->camerad, "open", 12000, was_opened) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR initializing camera" ); + this->broadcast.error( function, "initializing camera" ); throw std::runtime_error("initializing camera"); } @@ -1862,7 +1892,7 @@ namespace Sequencer { if ( was_opened) { for ( const auto &cmd : this->camera_prologue ) { if ( this->camerad.command_timeout( cmd, reply, CAMERA_PROLOG_TIMEOUT ) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR sending \""+cmd+"\" to camera" ); + this->broadcast.error( function, "sending \""+cmd+"\" to camera" ); throw std::runtime_error("sending \""+cmd+"\" to camera"); } } @@ -1933,7 +1963,7 @@ namespace Sequencer { // turn off power to camera hardware // if ( this->set_power_switch(OFF, POWER_CAMERA, std::chrono::seconds(5)) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR powering off camera" ); + this->broadcast.error( function, "powering off camera" ); throw std::runtime_error("switching off camera"); } @@ -1960,6 +1990,7 @@ namespace Sequencer { ScopedState thr_state( thread_state_manager, Sequencer::THR_MOVE_TO_TARGET ); ScopedState wait_state( wait_state_manager, Sequencer::SEQ_WAIT_TCS ); + ScopedState wait_moveto( wait_state_manager, Sequencer::SEQ_WAIT_MOVETO ); // If RA and DEC fields are both empty then no telescope move // @@ -1972,7 +2003,7 @@ namespace Sequencer { // if ( this->target.ra_hms == this->last_ra_hms && this->target.dec_dms == this->last_dec_dms ) { - this->async.enqueue_and_log( function, "NOTICE: no move required for repeat target" ); + this->broadcast.notice( function, "no move required for repeat target" ); return NO_ERROR; } @@ -1996,7 +2027,7 @@ namespace Sequencer { if ( ra_isnan ) { message << " RA=\"" << this->target.ra_hms << "\""; } if ( dec_isnan ) { message << " DEC=\"" << this->target.dec_dms << "\""; } message << " to decimal"; - this->async.enqueue_and_log( function, "ERROR "+message.str() ); + this->broadcast.error( function, message.str() ); this->thread_error_manager.set( THR_MOVE_TO_TARGET ); throw std::runtime_error(message.str()); } @@ -2014,9 +2045,9 @@ namespace Sequencer { double _solved_angle = ( angle_out < 0 ? angle_out + 360.0 : angle_out ); if ( std::abs(_solved_angle) - std::abs(this->target.casangle) > 0.01 ) { - message.str(""); message << "NOTICE: Calculated angle " << angle_out + message.str(""); message << "Calculated angle " << angle_out << " is not equivalent to casangle " << this->target.casangle; - this->async.enqueue_and_log( function, message.str() ); + this->broadcast.notice( function, message.str() ); } // Send coordinates using TCS-native COORDS command. @@ -2046,8 +2077,8 @@ namespace Sequencer { error = this->tcsd.send( coords_cmd.str(), coords_reply ); // send to the TCS // second failure return error if ( error != NO_ERROR || coords_reply.compare( 0, strlen(TCS_SUCCESS_STR), TCS_SUCCESS_STR ) != 0 ) { - message.str(""); message << "ERROR sending COORDS command. TCS reply: " << coords_reply; - this->async.enqueue_and_log( function, message.str() ); + message.str(""); message << "sending COORDS command. TCS reply: " << coords_reply; + this->broadcast.error( function, message.str() ); this->thread_error_manager.set( THR_MOVE_TO_TARGET ); throw std::runtime_error("sending COORDS to TCS: "+coords_reply); } @@ -2060,7 +2091,7 @@ namespace Sequencer { std::stringstream ringgo_cmd; std::string noreply("DONTWAIT"); // indicates don't wait for reply ringgo_cmd << TCSD_RINGGO << " " << angle_out; // this is calculated cass angle - this->async.enqueue_and_log( function, "sending "+ringgo_cmd.str()+" to TCS" ); + this->broadcast.notice( function, "sending "+ringgo_cmd.str()+" to TCS" ); error = this->tcsd.send( ringgo_cmd.str(), noreply ); } @@ -2068,16 +2099,16 @@ namespace Sequencer { { ScopedState wait_state( wait_state_manager, Sequencer::SEQ_WAIT_TCSOP ); - this->async.enqueue_and_log( function, "NOTICE: waiting for TCS operator to send \"ontarget\" signal" ); + this->broadcast.notice( function, "waiting for TCS operator to send \"ontarget\" signal" ); while ( !this->cancel_flag.load() && !this->is_ontarget.load() ) { std::unique_lock lock(cv_mutex); this->cv.wait( lock, [this]() { return( this->is_ontarget.load() || this->cancel_flag.load() ); } ); } - this->async.enqueue_and_log( function, "NOTICE: received " - +(this->cancel_flag.load() ? std::string("cancel") : std::string("ontarget")) - +" signal!" ); + this->broadcast.notice( function, "received " + +(this->cancel_flag.load() ? std::string("cancel") : std::string("ontarget")) + +" signal!" ); } // If waiting for TCS operator was cancelled then don't continue @@ -2233,7 +2264,7 @@ namespace Sequencer { // const auto &calinfo = this->caltarget.get_info(calname); - this->async.enqueue_and_log(function, "NOTICE: configuring calibrator for "+calname); + this->broadcast.notice( function, "configuring calibrator for "+calname); // set the calib door and cover // @@ -2245,7 +2276,7 @@ namespace Sequencer { logwrite( function, "calib: "+cmd.str() ); if ( !this->cancel_flag.load() && this->calibd.command_timeout( cmd.str(), CALIBD_SET_TIMEOUT ) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR moving calib door and/or cover" ); + this->broadcast.error( function, "moving calib door and/or cover" ); throw std::runtime_error("moving calib door and/or cover"); } @@ -2258,7 +2289,7 @@ namespace Sequencer { logwrite( function, message.str() ); std::string reply; if ( this->powerd.send( cmd.str(), reply ) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR "+message.str() ); + this->broadcast.error( function, message.str() ); throw std::runtime_error("setting lamp "+message.str()); } } @@ -2282,13 +2313,13 @@ namespace Sequencer { if ( this->cancel_flag.load() ) break; cmd.str(""); cmd << CALIBD_LAMPMOD << " " << mod << " " << (state?1:0) << " 1000"; if ( this->calibd.command( cmd.str() ) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR "+cmd.str() ); + this->broadcast.error( function, cmd.str() ); throw std::runtime_error("setting lamp modulator "+cmd.str()); } } if ( this->cancel_flag.load() ) { - this->async.enqueue_and_log( function, "NOTICE: abort may have left calib system partially set" ); + this->broadcast.notice( function, "abort may have left calib system partially set" ); } this->thread_error_manager.clear( THR_CALIBRATOR_SET ); // success @@ -2300,6 +2331,13 @@ namespace Sequencer { /***** Sequencer::Sequence::abort_process *********************************/ /** * @brief tries to abort everything happening + * @details Sets SEQ_ABORTING via RAII for the duration of the abort, + * then on exit: + * - if aborting during RUNNING or PAUSED, restores SEQ_READY + * - if aborting during STARTING or STOPPING, sets SEQ_FAILED + * (indeterminate lifecycle state; requires startup/shutdown + * to clear) + * - otherwise leaves seqstate unchanged on exit * */ void Sequence::abort_process() { @@ -2307,6 +2345,21 @@ namespace Sequencer { ScopedState thr_state( this->thread_state_manager, Sequencer::THR_ABORT_PROCESS ); + // Decide post-abort seqstate before entering SEQ_ABORTING. These snapshots + // must be taken before any seqstate mutation below, because set_only() + // clears all other lifecycle bits. + // + const bool abort_during_run = this->seq_state_manager.are_any_set( + Sequencer::SEQ_RUNNING, + Sequencer::SEQ_PAUSED ); + const bool abort_during_lifecycle = this->seq_state_manager.are_any_set( + Sequencer::SEQ_STARTING, + Sequencer::SEQ_STOPPING ); + + // Enter SEQ_ABORTING as a strict one-hot state. + // + this->seq_state_manager.set_only( {Sequencer::SEQ_ABORTING} ); + this->cancel_flag.store(false); // stop any exposure that may be in progress @@ -2328,8 +2381,25 @@ namespace Sequencer { // this->do_once.store(true); - this->async.enqueue_and_log( function, "NOTICE: cancel signal sent" ); + this->broadcast.notice( function, "cancel signal sent" ); + + // Exit SEQ_ABORTING to a strict one-hot terminal state chosen from the + // snapshot taken at entry. If neither condition applies (e.g. abort + // invoked while READY/NOTREADY/FAILED) we leave the state at NOTREADY + // so callers never see SEQ_ABORTING linger and no prior bit is retained. + // + if ( abort_during_run ) { + this->seq_state_manager.set_only( {Sequencer::SEQ_READY} ); + } + else if ( abort_during_lifecycle ) { + this->seq_state_manager.set_only( {Sequencer::SEQ_FAILED} ); + } + else { + this->seq_state_manager.set_only( {Sequencer::SEQ_NOTREADY} ); + } } + /***** Sequencer::Sequence::abort_process *********************************/ + /***** Sequencer::Sequence::stop_exposure *********************************/ /** @@ -2345,7 +2415,7 @@ namespace Sequencer { // This function is only used while exposing // if ( ! this->wait_state_manager.is_set( Sequencer::SEQ_WAIT_EXPOSE ) ) { - this->async.enqueue_and_log( function, "NOTICE: not currently exposing" ); + this->broadcast.notice(function, "not currently exposing"); return; } @@ -2362,12 +2432,12 @@ namespace Sequencer { else if ( error == NOTHING ) { // if not exposing, this is a way to ensure WAIT_EXPOSE bit can be cleared - this->async.enqueue_and_log( function, "NOTICE: not exposing" ); + this->broadcast.notice(function, "not currently exposing"); this->wait_state_manager.clear( Sequencer::SEQ_WAIT_EXPOSE ); } else if ( error == BUSY ) { - this->async.enqueue_and_log( function, "NOTICE: too late to stop exposure" ); + this->broadcast.warning(function, "too late to stop exposure"); // can't stop in the last 5 sec so wait that long and it should stop on its own std::this_thread::sleep_for(std::chrono::seconds(5)); } @@ -2393,7 +2463,7 @@ namespace Sequencer { // can only repeat when state is READY // if ( ! seq_state_manager.is_set( Sequencer::SEQ_READY ) ) { - this->async.enqueue_and_log( function, "ERROR cannot repeat: system not ready" ); + this->broadcast.error(function, "cannot repeat: system not ready"); return ERROR; } @@ -2440,7 +2510,7 @@ namespace Sequencer { } if ( this->cancel_flag.load() ) { - this->async.enqueue_and_log( function, "NOTICE: cancelled repeat exposure" ); + this->broadcast.notice(function, "cancelled repeat exposure"); return NO_ERROR; } @@ -2503,7 +2573,7 @@ namespace Sequencer { // if ( this->camerad.async( message.str() ) != NO_ERROR ) { // if ( this->camerad.send( message.str(), reply ) != NO_ERROR ) { if ( this->camerad.command_timeout( message.str(), reply, 30000 ) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR sending camera "+message.str() ); + this->broadcast.error( function, "sending camera "+message.str() ); this->thread_error_manager.set( THR_TRIGGER_EXPOSURE ); // tell the world this thread had an error this->target.update_state( Sequencer::TARGET_PENDING ); // return the target state to pending this->wait_state_manager.clear( Sequencer::SEQ_WAIT_EXPOSE ); // clear EXPOSE bit @@ -2541,7 +2611,7 @@ namespace Sequencer { // This function is only used while exposing // if ( ! this->wait_state_manager.is_set( Sequencer::SEQ_WAIT_EXPOSE ) ) { - this->async.enqueue_and_log( function, "ERROR cannot update exposure time when not currently exposing" ); + this->broadcast.error( function, "cannot update exposure time when not currently exposing" ); error = ERROR; } @@ -2563,14 +2633,11 @@ namespace Sequencer { if ( error==NO_ERROR ) { this->target.exptime_req = updated_exptime; - message.str(""); message << "NOTICE: updated exptime to " << updated_exptime << " sec"; - this->async.enqueue_and_log( function, message.str() ); + message.str(""); message << "updated exptime to " << updated_exptime << " sec"; + this->broadcast.notice( function, message.str() ); } - // announce the success or failure in an asynchronous broadcast message - // - message.str(""); message << "MODIFY_EXPTIME: " << this->target.exptime_req << ( error==NO_ERROR ? " DONE" : " ERROR" ); - this->async.enqueue( message.str() ); + // @TODO publish new exptime return; } @@ -2588,10 +2655,12 @@ namespace Sequencer { std::stringstream message; long error=NO_ERROR; - if ( ! seq_state_manager.are_any_set( Sequencer::SEQ_READY, Sequencer::SEQ_NOTREADY ) ) { - message << "ERROR cannot perform system startup while " + if ( ! seq_state_manager.are_any_set( Sequencer::SEQ_READY, + Sequencer::SEQ_NOTREADY, + Sequencer::SEQ_FAILED ) ) { + message << "cannot perform system startup while " << seq_state_manager.get_set_states(); - this->async.enqueue_and_log( function, message.str() ); + this->broadcast.error( function, message.str() ); return ERROR; } @@ -2612,10 +2681,12 @@ namespace Sequencer { // so initialize the power control first. // auto start_power = std::async(std::launch::async, &Sequence::power_init, this); - error = start_power.get(); - - if ( error != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR starting power control. Will try to continue (but don't hold your breath)" ); + try { + start_power.get(); + } + catch (const std::exception& e) { + logwrite( function, "ERROR intializing power control: "+std::string(e.what()) ); + return ERROR; } // threads to start, pair their ThreadStatusBit with the function to call @@ -2681,7 +2752,7 @@ namespace Sequencer { // restart the slicecam daemon, then loop to try again. if (attempt < maxattempts) { if ( set_power_switch(OFF, POWER_SLICECAM, std::chrono::seconds(5)) != NO_ERROR ) { - async.enqueue_and_log( function, "ERROR switching off slicecams" ); + this->broadcast.error( function, "switching off slicecams" ); __error=ERROR; break; } @@ -2695,7 +2766,7 @@ namespace Sequencer { continue; } else { - async.enqueue_and_log( function, "ERROR exceeded max attempts starting slicecam" ); + this->broadcast.error( function, "exceeded max attempts starting slicecam" ); __error=ERROR; } } @@ -2711,7 +2782,7 @@ namespace Sequencer { } } // end while if (__error == ERROR) { - async.enqueue_and_log( function, "ERROR slicecam not initialized" ); + this->broadcast.error( function, "slicecam not initialized" ); error=ERROR; } } @@ -2735,7 +2806,7 @@ namespace Sequencer { if (e.code == ErrorCode::ERROR_ACAM_CAMERA) { if (attempt < maxattempts) { if ( set_power_switch(OFF, POWER_ACAM_CAM, std::chrono::seconds(5)) != NO_ERROR ) { - async.enqueue_and_log( function, "ERROR switching off acam camera" ); + this->broadcast.error( function, "switching off acam camera" ); __error=ERROR; } logwrite(function, "acam camera powered off"); @@ -2748,7 +2819,7 @@ namespace Sequencer { continue; } else { - async.enqueue_and_log( function, "ERROR exceeded max attempts starting acam" ); + this->broadcast.error( function, "exceeded max attempts starting acam" ); __error=ERROR; } } @@ -2766,7 +2837,7 @@ namespace Sequencer { } } // end while if (__error == ERROR) { - async.enqueue_and_log( function, "ERROR acam not initialized" ); + this->broadcast.error( function, "acam not initialized" ); error=ERROR; } } @@ -2776,7 +2847,7 @@ namespace Sequencer { seq_state_manager.set_only( {Sequencer::SEQ_READY} ); } else { - seq_state_manager.set_only( {Sequencer::SEQ_NOTREADY} ); + seq_state_manager.set_only( {Sequencer::SEQ_FAILED} ); } return error; @@ -2796,16 +2867,34 @@ namespace Sequencer { const std::string function("Sequencer::Sequence::shutdown"); long error=ERROR; - ScopedState thr_state( this->thread_state_manager, Sequencer::THR_SHUTDOWN ); // this thread is running + // Reject if a conflicting lifecycle transition is already in progress. + // All other states (READY, NOTREADY, FAILED, RUNNING, PAUSED) are valid + // starting points for a shutdown. + // + if ( seq_state_manager.are_any_set( Sequencer::SEQ_STOPPING, + Sequencer::SEQ_STARTING, + Sequencer::SEQ_ABORTING ) ) { + std::stringstream message; + message << "cannot perform system shutdown while " + << seq_state_manager.get_set_states(); + this->broadcast.error( function, message.str() ); + return ERROR; + } - // set only STOPPING (and clear everything else) - ScopedState seq_state( seq_state_manager, Sequencer::SEQ_STOPPING, true ); // state=STOPPING (only) + // stop everything first + // + this->abort_process(); - seq_state.destruct_set( Sequencer::SEQ_NOTREADY ); // set state=NOTREADY on exit + ScopedState thr_state( this->thread_state_manager, Sequencer::THR_SHUTDOWN ); // this thread is running - // stop everything + // Enter SEQ_STOPPING as a strict one-hot state. Explicit management (not + // ScopedState RAII) is used here because abort_process() below independently + // transitions seqstate, and an RAII destructor using set_and_clear would + // re-add NOTREADY on top of any FAILED bit left by abort_process, producing + // a non-one-hot state. The terminal transition is made explicitly before + // every return from this function. // - this->abort_process(); + seq_state_manager.set_only( {Sequencer::SEQ_STOPPING} ); // clear stop flags // @@ -2821,8 +2910,12 @@ namespace Sequencer { // so make sure power control is initialized before continuing. // auto start_power = std::async(std::launch::async, &Sequence::power_init, this); - if ( start_power.get() != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR from power control. Will try to continue (but don't hold your breath)" ); + try { + start_power.get(); + } + catch (const std::exception& e) { + logwrite( function, "ERROR intializing power control: "+std::string(e.what()) ); + return ERROR; } // container of shutdown threads to launch, @@ -2860,15 +2953,19 @@ namespace Sequencer { } } - std::stringstream message; if (error==NO_ERROR) { - message << "NOTICE: instrument is shut down"; + this->broadcast.notice(function, "instrument is shut down"); } else { - message << "ERROR occurred during shutdown and may not have completed"; + this->broadcast.error(function, "shut down may not be complete"); } - this->async.enqueue_and_log( function, message.str() ); + // Always end in NOTREADY regardless of worker errors. SEQ_FAILED is + // reserved for startup failures and aborted lifecycle transitions. + // Worker errors during shutdown are logged above but do not prevent + // the instrument from being considered shut down (not ready). + // + seq_state_manager.set_only( {Sequencer::SEQ_NOTREADY} ); return error; } @@ -3125,10 +3222,7 @@ namespace Sequencer { retstring = ( this->do_once.load() ? "ONE" : "ALL" ); - // send an async message with the current type - // - message.str(""); message << "DOTYPE: " << retstring; - this->async.enqueue( message.str() ); + // @TODO publish dotype return( error ); } @@ -3433,120 +3527,6 @@ namespace Sequencer { /***** Sequencer::Sequence::make_telemetry_message **************************/ - /***** Sequencer::Sequence::get_external_telemetry **************************/ - /** - * @brief collect telemetry from other daemon(s) - * @details This is used for any telemetry that I need to collect from - * another daemon. Common::collect_telemetry() sends a command - * to the daemon, which will respond with a JSON message. The - * daemon(s) to contact are configured with the TELEM_PROVIDER - * key in the config file. - * - */ - void Sequence::get_external_telemetry() { - // Loop through each configured telemetry provider. This requests - // their telemetry which is returned as a serialized json string - // held in retstring. - // - // handle_json_message() will parse the serialized json string. - // - std::string retstring; - for ( const auto &provider : this->telemetry_providers ) { - Common::collect_telemetry( provider, retstring ); - handle_json_message(retstring); - } - return; - } - /***** Sequencer::Sequence::get_external_telemetry **************************/ - - - /***** Sequencer::Sequence::handle_json_message *****************************/ - /** - * @brief parses incoming telemetry messages - * @details Requesting telemetry from another daemon returns a serialized - * JSON message which needs to be passed in here to parse it. - * @param[in] message_in incoming serialized JSON message (as a string) - * @return ERROR | NO_ERROR - * - */ - long Sequence::handle_json_message( const std::string message_in ) { - const std::string function("Sequencer::Sequence::handle_json_message"); - std::stringstream message; - - if ( message_in.empty() ) { - logwrite( function, "ERROR empty JSON message" ); - return ERROR; - } - - try { - nlohmann::json jmessage = nlohmann::json::parse( message_in ); - std::string messagetype; - - // jmessage must not contain key "error" and must contain key "messagetype" - // - if ( !jmessage.contains("error") ) { - if ( jmessage.contains("messagetype") && jmessage["messagetype"].is_string() ) { - messagetype = jmessage["messagetype"]; - } - else { - logwrite( function, "ERROR received JSON message with missing or invalid messagetype" ); - return ERROR; - } - } - else { - logwrite( function, "ERROR in JSON message" ); - return ERROR; - } - - // No errors, so disseminate the message contents based on the message type. - // - // column_from_json( colname, jkey, jmessage ) will extract the value of - // expected type with key jkey from json string jmessage, and assign it - // to this->target.external_telemetry[colname] map. It is expected that - // "colname" is the column name in the database. - // - if ( messagetype == "camerainfo" ) { - this->target.column_from_json( "EXPTIME", "SHUTTIME_SEC", jmessage ); - } - else - if ( messagetype == "slitinfo" ) { - this->target.column_from_json( "SLITWIDTH", "SLITW", jmessage ); - this->target.column_from_json( "SLITOFFSET", "SLITO", jmessage ); - } - else - if ( messagetype == "tcsinfo" ) { - this->target.column_from_json( "TELRA", "TELRA", jmessage ); - this->target.column_from_json( "TELDECL", "TELDEC", jmessage ); - this->target.column_from_json( "ALT", "ALT", jmessage ); - this->target.column_from_json( "AZ", "AZ", jmessage ); - this->target.column_from_json( "AIRMASS", "AIRMASS", jmessage ); - this->target.column_from_json( "CASANGLE", "CASANGLE", jmessage ); - } - else - if ( messagetype == "test" ) { - } - else { - message.str(""); message << "ERROR received unhandled JSON message type \"" << messagetype << "\""; - logwrite( function, message.str() ); - return ERROR; - } - } - catch ( const nlohmann::json::parse_error &e ) { - message.str(""); message << "ERROR json exception parsing message: " << e.what(); - logwrite( function, message.str() ); - return ERROR; - } - catch ( const std::exception &e ) { - message.str(""); message << "ERROR parsing message: " << e.what(); - logwrite( function, message.str() ); - return ERROR; - } - - return NO_ERROR; - } - /***** Sequencer::Sequence::handle_json_message *****************************/ - - /***** Sequencer::Sequence::dothread_test_fpoffset **************************/ /** * @brief for testing, calls a Python function from a thread @@ -3713,7 +3693,7 @@ namespace Sequencer { } // connection failed too many times if (attempt > maxattempts) { - async.enqueue_and_log(function, "ERROR exceeded max attempts connecting to " + daemon.name); + this->broadcast.error( function, "exceeded max attempts connecting to " + daemon.name); return ERROR; } @@ -3722,7 +3702,7 @@ namespace Sequencer { error |= daemon.send( "isopen", reply ); error |= this->parse_state( function, reply, isopen ); if ( error != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR opening "+daemon.name+" hardware" ); + this->broadcast.error( function, "opening "+daemon.name+" hardware" ); return ERROR; } @@ -3732,7 +3712,7 @@ namespace Sequencer { logwrite( function, "opening "+daemon.name+" hardware connections with " +std::to_string(opentimeout)+" ms timeout" ); if ( daemon.command_timeout( opencmd, reply, opentimeout ) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR opening connection to "+daemon.name+" hardware" ); + this->broadcast.error( function, "opening connection to "+daemon.name+" hardware" ); return ERROR; } was_opened=true; @@ -3761,7 +3741,7 @@ namespace Sequencer { if ( !daemon.socket.isconnected() ) { logwrite( function, "connecting to "+daemon.name+" daemon" ); if ( daemon.connect() != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR connecting to "+daemon.name ); + this->broadcast.error( function, "connecting to "+daemon.name ); return ERROR; } } @@ -3872,7 +3852,6 @@ namespace Sequencer { retstring.append( " fpoffset ? | \n" ); retstring.append( " getnext [ ? ]\n" ); retstring.append( " getobsid [ ? ]\n" ); - retstring.append( " gettelem [ ? ]\n" ); retstring.append( " isready [ ? ]\n" ); retstring.append( " moveto [ ? | ]\n" ); retstring.append( " notify [ ? ]\n" ); @@ -3894,33 +3873,35 @@ namespace Sequencer { } else - // ---------------------------------------------------- - // async -- queue an asynchronous message - // ---------------------------------------------------- - // - if ( testname == "async" ) { - if ( tokens.size() > 1 && tokens[1] == "?" ) { - retstring = "test async [ ]\n"; - retstring.append( " Queue and broadcast optional . If not supplied\n" ); - retstring.append( " then broadcast \"test\".\n" ); - return HELP; - } - if ( tokens.size() > 1 ) { - bool first=true; - message.str(""); - for ( const auto &word : tokens ) { - if ( first ) { first=false; continue; } // skip the testname - message << word << " "; - } - logwrite( function, message.str() ); - this->async.enqueue( message.str() ); - } - else { - logwrite( function, "test" ); - this->async.enqueue( "test" ); - } - } - else +/*** + * // ---------------------------------------------------- + * // async -- queue an asynchronous message + * // ---------------------------------------------------- + * // + * if ( testname == "async" ) { + * if ( tokens.size() > 1 && tokens[1] == "?" ) { + * retstring = "test async [ ]\n"; + * retstring.append( " Queue and broadcast optional . If not supplied\n" ); + * retstring.append( " then broadcast \"test\".\n" ); + * return HELP; + * } + * if ( tokens.size() > 1 ) { + * bool first=true; + * message.str(""); + * for ( const auto &word : tokens ) { + * if ( first ) { first=false; continue; } // skip the testname + * message << word << " "; + * } + * logwrite( function, message.str() ); + * this->async.enqueue( message.str() ); + * } + * else { + * logwrite( function, "test" ); + * this->async.enqueue( "test" ); + * } + * } + * else + ***/ // ---------------------------------------------------- // prologue -- show the camera prologue commands @@ -3955,16 +3936,16 @@ namespace Sequencer { // write to the log (textually) which bits are set // retstring.clear(); - message.str(""); message << "NOTICE: daemons ready: " << this->daemon_manager.get_set_states(); - this->async.enqueue_and_log( function, message.str() ); + message.str(""); message << "daemons ready: " << this->daemon_manager.get_set_states(); + this->broadcast.notice( function, message.str() ); retstring.append( message.str() ); retstring.append( "\n" ); - message.str(""); message << "NOTICE: daemons not ready: " << this->daemon_manager.get_cleared_states(); - this->async.enqueue_and_log( function, message.str() ); + message.str(""); message << "daemons not ready: " << this->daemon_manager.get_cleared_states(); + this->broadcast.notice( function, message.str() ); retstring.append( message.str() ); retstring.append( "\n" ); - message.str(""); message << "NOTICE: camera ready to expose: " << (this->can_expose.load() ? "yes" : "no"); - this->async.enqueue_and_log( function, message.str() ); + message.str(""); message << "camera ready to expose: " << (this->can_expose.load() ? "yes" : "no"); + this->broadcast.notice( function, message.str() ); retstring.append( message.str() ); error = NO_ERROR; @@ -4044,7 +4025,7 @@ namespace Sequencer { } message.str(""); message << "STATES: " << this->seq_state_manager.get_set_states(); - this->async.enqueue( message.str() ); + this->broadcast.notice( function, message.str() ); logwrite( function, message.str() ); message.str(""); message << "THREADS: " << this->thread_state_manager.get_set_states(); @@ -4151,8 +4132,7 @@ namespace Sequencer { } error = NO_ERROR; - message.str(""); message << "NOTICE: " << targetstatus; - this->async.enqueue( message.str() ); // broadcast target status + this->broadcast.notice(function, targetstatus); if ( ret == TargetInfo::TargetState::TARGET_FOUND ) { rts << "name obsid order ra dec casangle slitangle airmasslim\n"; @@ -4255,24 +4235,6 @@ namespace Sequencer { retstring = rts.str(); } else - // ---------------------------------------------------- - // gettelem -- get external telemetry - // ---------------------------------------------------- - // - if ( testname == "gettelem" ) { - if ( tokens.size() > 1 && tokens[1] == "?" ) { - retstring = "test gettelem\n"; - retstring.append( " Get external telemetry from other daemons.\n" ); - return HELP; - } - this->get_external_telemetry(); - message.str(""); - for ( const auto &[name,data] : this->target.external_telemetry ) { - message << "name=" << name << " valid=" << (data.valid?"T":"F") << " value=" << data.value << "\n"; - } - retstring = message.str(); - } - else // ---------------------------------------------------- // addrow -- insert a (fixed, hard-coded) row into the database @@ -4328,10 +4290,10 @@ namespace Sequencer { // let the world know of the state change // - message.str(""); message << "TARGETSTATE:" << this->target.state - << " TARGET:" << this->target.name - << " OBSID:" << this->target.obsid; - this->async.enqueue( message.str() ); + message.str(""); message << this->target.state + << " TARGET:" << this->target.name + << " (" << this->target.obsid << ")"; + this->broadcast.notice( function, message.str() ); } else @@ -4359,10 +4321,10 @@ namespace Sequencer { // let the world know of the state change // - message.str(""); message << "TARGETSTATE:" << this->target.state - << " TARGET:" << this->target.name - << " OBSID:" << this->target.obsid; - this->async.enqueue( message.str() ); + message.str(""); message << this->target.state + << " TARGET:" << this->target.name + << " (" << this->target.obsid << ")"; + this->broadcast.notice( function, message.str() ); } } else @@ -4469,9 +4431,9 @@ namespace Sequencer { else this->test_solver_args.clear(); // clear previous solver args if not specified if ( !this->test_solver_args.empty() ) { - message.str(""); message << "NOTICE: test solver args: " << this->test_solver_args; + message.str(""); message << "test solver args: " << this->test_solver_args; } - this->async.enqueue_and_log( function, message.str() ); + this->broadcast.notice( function, message.str() ); // clear stop flags // @@ -4525,12 +4487,12 @@ namespace Sequencer { bool cas_isnan = std::isnan( angle_in ); if ( ra_isnan || dec_isnan || cas_isnan ) { - message.str(""); message << "ERROR: converting"; + message.str(""); message << "converting"; if ( ra_isnan ) { message << " RA=\"" << this->target.ra_hms << "\""; } if ( dec_isnan ) { message << " DEC=\"" << this->target.dec_dms << "\""; } if ( cas_isnan ) { message << " CASS=\"" << cass_now << "\""; } message << " to decimal"; - this->async.enqueue_and_log( function, message.str() ); + this->broadcast.error( function, message.str() ); return ERROR; } diff --git a/sequencerd/sequence.h b/sequencerd/sequence.h index ab86cc45..6e59b57a 100644 --- a/sequencerd/sequence.h +++ b/sequencerd/sequence.h @@ -125,6 +125,8 @@ namespace Sequencer { SEQ_STOPPING, ///< set when sequencer is shutting down SEQ_PAUSED, ///< set when sequencer is paused SEQ_STARTING, ///< set when sequencer is starting up + SEQ_FAILED, ///< set on a fatal/indeterminate failure; cleared only by startup or shutdown + SEQ_ABORTING, ///< transitory; set/cleared via RAII in abort_process() NUM_SEQ_STATES }; @@ -134,7 +136,9 @@ namespace Sequencer { {SEQ_RUNNING, "RUNNING"}, {SEQ_STOPPING, "STOPPING"}, {SEQ_PAUSED, "PAUSED"}, - {SEQ_STARTING, "STARTING"} + {SEQ_STARTING, "STARTING"}, + {SEQ_FAILED, "FAILED"}, + {SEQ_ABORTING, "ABORTING"} }; /** @@ -153,7 +157,9 @@ namespace Sequencer { SEQ_WAIT_SLIT, ///< set when waiting for slit SEQ_WAIT_TCS, ///< set when waiting for tcs // states - SEQ_WAIT_ACQUIRE, ///< set when waiting for acquire + SEQ_WAIT_MOVETO, ///< set when waiting for move-to-target + SEQ_WAIT_ACAM_ACQUIRE, ///< set when waiting for ACAM acquire + SEQ_WAIT_FINEACQUIRE, ///< set when waiting for slicecam fineacquire SEQ_WAIT_EXPOSE, ///< set when waiting for camera exposure SEQ_WAIT_READOUT, ///< set when waiting for camera readout SEQ_WAIT_TCSOP, ///< set when waiting specifically for tcs operator @@ -163,21 +169,23 @@ namespace Sequencer { const std::map wait_state_names = { // daemons - {SEQ_WAIT_ACAM, "ACAM"}, - {SEQ_WAIT_CALIB, "CALIB"}, - {SEQ_WAIT_CAMERA, "CAMERA"}, - {SEQ_WAIT_FLEXURE, "FLEXURE"}, - {SEQ_WAIT_FOCUS, "FOCUS"}, - {SEQ_WAIT_POWER, "POWER"}, - {SEQ_WAIT_SLICECAM, "SLICECAM"}, - {SEQ_WAIT_SLIT, "SLIT"}, - {SEQ_WAIT_TCS, "TCS"}, + {SEQ_WAIT_ACAM, "ACAM"}, + {SEQ_WAIT_CALIB, "CALIB"}, + {SEQ_WAIT_CAMERA, "CAMERA"}, + {SEQ_WAIT_FLEXURE, "FLEXURE"}, + {SEQ_WAIT_FOCUS, "FOCUS"}, + {SEQ_WAIT_POWER, "POWER"}, + {SEQ_WAIT_SLICECAM, "SLICECAM"}, + {SEQ_WAIT_SLIT, "SLIT"}, + {SEQ_WAIT_TCS, "TCS"}, // states - {SEQ_WAIT_ACQUIRE, "ACQUIRE"}, - {SEQ_WAIT_EXPOSE, "EXPOSE"}, - {SEQ_WAIT_READOUT, "READOUT"}, - {SEQ_WAIT_TCSOP, "TCSOP"}, - {SEQ_WAIT_USER, "USER"} + {SEQ_WAIT_MOVETO, "MOVETO"}, + {SEQ_WAIT_ACAM_ACQUIRE, "ACAM_ACQUIRE"}, + {SEQ_WAIT_FINEACQUIRE, "FINEACQUIRE"}, + {SEQ_WAIT_EXPOSE, "EXPOSE"}, + {SEQ_WAIT_READOUT, "READOUT"}, + {SEQ_WAIT_TCSOP, "TCSOP"}, + {SEQ_WAIT_USER, "USER"} }; /** @@ -345,6 +353,10 @@ namespace Sequencer { [this](const nlohmann::json &msg) { handletopic_acamd(msg); } ) }, { Topic::SLICECAMD, std::function( [this](const nlohmann::json &msg) { handletopic_slicecamd(msg); } ) }, + { Topic::SLITD, std::function( + [this](const nlohmann::json &msg) { handletopic_slitd(msg); } ) }, + { Topic::TCSD, std::function( + [this](const nlohmann::json &msg) { handletopic_tcsd(msg); } ) }, { Topic::CAMERAD, std::function( [this](const nlohmann::json &msg) { handletopic_camerad(msg); } ) } }; @@ -376,8 +388,6 @@ namespace Sequencer { inline void reset_cancel_flag() { this->cancel_flag.store(false); } - std::map telemetry_providers; ///< map of port[daemon_name] for external telemetry providers - double acquisition_timeout; ///< timeout for target acquisition (in sec) set by configuration parameter ACAM_ACQUIRE_TIMEOUT int acquisition_max_retrys; ///< max number of acquisition loop attempts double tcs_offsetrate_ra; ///< TCS offset rate RA ("MRATE") in arcsec per second @@ -396,6 +406,10 @@ namespace Sequencer { std::condition_variable acam_cv; std::mutex camerad_mtx; std::condition_variable camerad_cv; + std::mutex slitd_mtx; + std::condition_variable slitd_cv; + std::mutex tcsd_mtx; + std::condition_variable tcsd_cv; std::mutex wait_mtx; std::condition_variable cv; std::mutex cv_mutex; @@ -406,9 +420,6 @@ namespace Sequencer { std::atomic do_once; ///< set if "do one" selected, clear if "do all" selected - std::mutex seqstate_mtx; - std::condition_variable seqstate_cv; - ImprovedStateManager(Sequencer::NUM_THREAD_STATES)> thread_error_manager{ Sequencer::thread_names }; ImprovedStateManager(Sequencer::NUM_SEQ_STATES)> seq_state_manager{Sequencer::seq_state_names}; ImprovedStateManager(Sequencer::NUM_WAIT_STATES)> wait_state_manager{Sequencer::wait_state_names}; @@ -435,8 +446,6 @@ namespace Sequencer { std::string daemon_control; ///< daemon control script - Common::Queue async; ///< asynchronous message queue - // Here are all the daemon client objects that the Sequencer connects to. // Common::DaemonClient acamd { "acamd" }; @@ -468,8 +477,9 @@ namespace Sequencer { void handletopic_camerad( const nlohmann::json &jmessage ); void handletopic_acamd( const nlohmann::json &jmessage ); void handletopic_slicecamd( const nlohmann::json &jmessage ); + void handletopic_slitd( const nlohmann::json &jmessage ); + void handletopic_tcsd( const nlohmann::json &jmessage ); void publish_snapshot(); - void publish_snapshot(std::string &retstring); void publish_seqstate(); void publish_waitstate(); void publish_daemonstate(); @@ -499,8 +509,12 @@ namespace Sequencer { /// void set_seqstate_bit( uint32_t mb ); ///< set the specified masked bit in the seqstate word void broadcast_daemonstate(); ///< void broadcast_threadstate(); ///< - void broadcast_seqstate(); ///< writes the seqstate string to the async port - void broadcast_waitstate(); ///< writes the waitstate string to the async port + void broadcast_seqstate(); ///< publishes the seqstate on the seq_seqstate topic + void broadcast_waitstate(); ///< publishes the waitstate on the seq_waitstate topic + + Common::Broadcaster broadcast { this->publisher, Sequencer::DAEMON_NAME }; ///< logs and publishes a narrative message on Topic::BROADCAST + + std::string last_seqstate_str; ///< last seqstate string announced via broadcast_seqstate() (for change detection) uint32_t get_reqstate(); ///< get the reqstate word @@ -538,8 +552,6 @@ namespace Sequencer { long target_offset(); void make_telemetry_message( std::string &retstring ); ///< assembles my telemetry message - void get_external_telemetry(); ///< collect telemetry from another daemon - long handle_json_message( const std::string message_in ); ///< parses incoming telemetry messages long set_power_switch( PowerState state, const std::string which, std::chrono::seconds delay ); long check_power_switch( PowerState checkstate, const std::string which, bool &is_set ); diff --git a/sequencerd/sequence_acquisition.cpp b/sequencerd/sequence_acquisition.cpp index 7beab093..dbbdbbc2 100644 --- a/sequencerd/sequence_acquisition.cpp +++ b/sequencerd/sequence_acquisition.cpp @@ -20,7 +20,7 @@ namespace Sequencer { std::string reply; ScopedState thr_state( thread_state_manager, Sequencer::THR_ACQUISITION ); - ScopedState wait_state( wait_state_manager, Sequencer::SEQ_WAIT_ACQUIRE ); + ScopedState wait_state( wait_state_manager, Sequencer::SEQ_WAIT_ACAM_ACQUIRE ); // form and send the ACQUIRE command to ACAM // @@ -29,17 +29,17 @@ namespace Sequencer { double angle_in = this->target.slitangle; if ( std::isnan(ra_in) || std::isnan(dec_in) ) { - this->async.enqueue_and_log( function, "ERROR converting target coordinates to decimal" ); + this->broadcast.error( function, "converting target coordinates to decimal" ); return ERROR; } std::ostringstream cmd; cmd << ACAMD_ACQUIRE << " " << ra_in << " " << dec_in << " " << angle_in; - this->async.enqueue_and_log( function, "NOTICE: starting ACAM acquisition" ); + this->broadcast.notice( function, "starting ACAM acquisition" ); if ( this->acamd.command( cmd.str(), reply ) != NO_ERROR ) { - this->async.enqueue_and_log( function, "ERROR sending acquire command to acamd" ); + this->broadcast.error( function, "sending acquire command to acamd" ); return ERROR; } @@ -58,11 +58,11 @@ namespace Sequencer { if (this->cancel_flag.load()) return ABORT; if (use_timeout && !this->is_acam_guiding.load()) { - this->async.enqueue_and_log(function, "ERROR ACAM acquisition timed out!"); + this->broadcast.error( function, "ACAM acquisition timed out!" ); return TIMEOUT; } - this->async.enqueue_and_log(function, "ACAM target acquired"); + this->broadcast.notice( function, "ACAM target acquired" ); return NO_ERROR; } /***** Sequencer::Sequence::do_acam_acquire **********************************/ @@ -77,21 +77,21 @@ namespace Sequencer { long Sequence::do_slicecam_fineacquire() { const std::string function("Sequencer::Sequence::do_slicecam_fineacquire"); - ScopedState wait_state(wait_state_manager, Sequencer::SEQ_WAIT_ACQUIRE); + ScopedState wait_state(wait_state_manager, Sequencer::SEQ_WAIT_FINEACQUIRE); // TODO don't hard-code the arguments here: std::string reply; if (this->slicecamd.command( SLICECAMD_FINEACQUIRE+" start L", reply ) != NO_ERROR) { - this->async.enqueue_and_log(function, "ERROR starting slicecam fine acquisition"); + this->broadcast.error( function, "starting slicecam fine acquisition" ); return ERROR; } if ( reply.find("ERROR") != std::string::npos ) { - this->async.enqueue_and_log(function, "slicecam fine acquisition mode: "+reply); + this->broadcast.error( function, "slicecam fine acquisition mode: "+reply ); return ERROR; } - this->async.enqueue_and_log(function, "NOTICE: slicecam fine acquisition started"); + this->broadcast.notice( function, "slicecam fine acquisition started" ); const bool use_timeout = ( this->acquisition_timeout > 0 ); const auto timeout_time = std::chrono::steady_clock::now() @@ -108,11 +108,11 @@ namespace Sequencer { if (this->cancel_flag.load()) return ABORT; if (use_timeout && !this->is_fineacquire_locked.load()) { - this->async.enqueue_and_log(function, "ERROR slicecam fine acquisition timed out!"); + this->broadcast.error( function, "slicecam fine acquisition timed out!" ); return TIMEOUT; } - this->async.enqueue_and_log(function, "slicecam fine acquisition target acquired"); + this->broadcast.notice( function, "slicecam fine acquisition target acquired" ); return NO_ERROR; } /***** Sequencer::Sequence::do_slicecam_fineacquire **************************/ diff --git a/sequencerd/sequencer_server.cpp b/sequencerd/sequencer_server.cpp index 06514cef..bb907ceb 100644 --- a/sequencerd/sequencer_server.cpp +++ b/sequencerd/sequencer_server.cpp @@ -27,13 +27,11 @@ namespace Sequencer { case SIGTERM: case SIGINT: logwrite(function, "received termination signal"); - message << "NOTICE:" << Sequencer::DAEMON_NAME << " exit"; - Server::instance->sequence.async.enqueue( message.str() ); + Server::instance->sequence.broadcast.notice(function, message.str()); Server::instance->exit_cleanly(); // shutdown the daemon break; case SIGHUP: // TODO reconfigure? - Server::instance->sequence.async.enqueue_and_log( function, - "ERROR: caught unhandled HUP signal" ); + logwrite( function, "ERROR: caught unhandled HUP signal" ); break; case SIGPIPE: logwrite(function, "ignored SIGPIPE"); @@ -41,8 +39,6 @@ namespace Sequencer { default: message << "received unknown signal " << strsignal(signo); logwrite( function, message.str() ); - message.str(""); message << "NOTICE:" << Sequencer::DAEMON_NAME << " exit"; - Server::instance->sequence.async.enqueue( message.str() ); break; } return; @@ -90,7 +86,7 @@ namespace Sequencer { catch (const std::exception &e) { // should be impossible message.str(""); message << "ERROR parsing entry " << entry << " of " << this->config.n_entries << ": " << e.what(); - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); return ERROR; } @@ -106,11 +102,11 @@ namespace Sequencer { } catch (const std::exception &e) { message.str(""); message << "ERROR parsing NBPORT: " << e.what(); - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); return ERROR; } message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -121,11 +117,11 @@ namespace Sequencer { } catch (const std::exception &e) { message.str(""); message << "ERROR parsing BLKPORT: " << e.what(); - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); return ERROR; } message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -136,11 +132,11 @@ namespace Sequencer { } catch (const std::exception &e) { message.str(""); message << "ERROR parsing ASYNCPORT: " << e.what(); - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); return ERROR; } message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -151,11 +147,11 @@ namespace Sequencer { } catch (const std::exception &e) { message.str(""); message << "ERROR parsing MESSAGEPORT: " << e.what(); - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); return ERROR; } message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -163,7 +159,7 @@ namespace Sequencer { if (config.param[entry] == "MESSAGEGROUP") { this->messagegroup = config.arg[entry]; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -172,7 +168,7 @@ namespace Sequencer { if ( config.param[entry] == "PUB_ENDPOINT" ) { this->sequence.publisher_address = config.arg[entry]; this->sequence.publisher_topic = DAEMON_NAME; - this->sequence.async.enqueue_and_log(function, "SEQUENCERD:config:"+config.param[entry]+"="+config.arg[entry]); + logwrite(function, "SEQUENCERD:config:"+config.param[entry]+"="+config.arg[entry]); applied++; } @@ -180,14 +176,14 @@ namespace Sequencer { // if ( config.param[entry] == "SUB_ENDPOINT" ) { this->sequence.subscriber_address = config.arg[entry]; - this->sequence.async.enqueue_and_log(function, "SEQUENCERD:config:"+config.param[entry]+"="+config.arg[entry]); + logwrite(function, "SEQUENCERD:config:"+config.param[entry]+"="+config.arg[entry]); applied++; } // DAEMON_CONTROL_SCRIPT if (config.param[entry] == "DAEMON_CONTROL_SCRIPT") { this->sequence.daemon_control = config.arg[entry]; - this->sequence.async.enqueue_and_log(function, "SEQUENCERD:config:"+config.param[entry]+"="+config.arg[entry]); + logwrite(function, "SEQUENCERD:config:"+config.param[entry]+"="+config.arg[entry]); applied++; } @@ -198,11 +194,11 @@ namespace Sequencer { } catch (const std::exception &e) { message.str(""); message << "ERROR parsing ACAMD_PORT: " << e.what(); - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); return ERROR; } message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -213,11 +209,11 @@ namespace Sequencer { } catch (const std::invalid_argument &e) { message.str(""); message << "ERROR parsing CAMERAD_PORT: " << e.what(); - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); return ERROR; } message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -228,11 +224,11 @@ namespace Sequencer { } catch (const std::exception &e) { message.str(""); message << "ERROR parsing CAMERAD_NBPORT: " << e.what(); - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); return ERROR; } message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -243,11 +239,11 @@ namespace Sequencer { } catch (const std::exception &e) { message.str(""); message << "ERROR parsing FLEXURED_PORT: " << e.what(); - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); return ERROR; } message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -258,11 +254,11 @@ namespace Sequencer { } catch (const std::exception &e) { message.str(""); message << "ERROR parsing POWERD_PORT: " << e.what(); - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); return ERROR; } message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -273,11 +269,11 @@ namespace Sequencer { } catch (const std::exception &e) { message.str(""); message << "ERROR parsing SLICECAMD_PORT: " << e.what(); - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); return ERROR; } message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -288,11 +284,11 @@ namespace Sequencer { } catch (const std::exception &e) { message.str(""); message << "ERROR parsing SLITD_PORT: " << e.what(); - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); return ERROR; } message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -303,11 +299,11 @@ namespace Sequencer { } catch (const std::exception &e) { message.str(""); message << "ERROR parsing TCSD_PORT: " << e.what(); - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); return ERROR; } message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -318,11 +314,11 @@ namespace Sequencer { } catch (const std::exception &e) { message.str(""); message << "ERROR parsing CALIBD_PORT: " << e.what(); - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); return ERROR; } message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -333,11 +329,11 @@ namespace Sequencer { } catch (const std::exception &e) { message.str(""); message << "ERROR parsing FILTERD_PORT: " << e.what(); - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); return ERROR; } message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -348,11 +344,11 @@ namespace Sequencer { } catch (const std::exception &e) { message.str(""); message << "ERROR parsing FOCUSD_PORT: " << e.what(); - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); return ERROR; } message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -363,11 +359,11 @@ namespace Sequencer { } catch (const std::exception &e) { message.str(""); message << "ERROR parsing ACQUIRE_TIMEOUT: " << e.what(); - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); return ERROR; } message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -379,27 +375,27 @@ namespace Sequencer { } catch (std::invalid_argument &) { message.str(""); message << "ACQUIRE_RETRYS: unable to convert " << config.arg[entry] << " to integer. retry limit disabled."; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); this->sequence.acquisition_max_retrys = -1; } catch (std::out_of_range &) { - this->sequence.async.enqueue_and_log( function, "ACQUIRE_RETRYS number out of integer range. retry limit disabled." ); + logwrite( function, "ACQUIRE_RETRYS number out of integer range. retry limit disabled." ); this->sequence.acquisition_max_retrys = -1; } this->sequence.acquisition_max_retrys = rt; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } // TCS_WHICH -- which TCS to connect to, defults to real if not specified if ( config.param[entry] == "TCS_WHICH" ) { if ( config.arg[entry] != "sim" && config.arg[entry] != "real" ) { - this->sequence.async.enqueue_and_log( function, "ERROR TCS_WHICH expected { sim real }" ); + logwrite( function, "ERROR TCS_WHICH expected { sim real }" ); return ERROR; } this->sequence.tcs_which = config.arg[entry]; - this->sequence.async.enqueue_and_log( function, "SEQUENCERD:config:"+config.param[entry]+"="+config.arg[entry] ); + logwrite( function, "SEQUENCERD:config:"+config.param[entry]+"="+config.arg[entry] ); applied++; } @@ -410,22 +406,22 @@ namespace Sequencer { mrate = std::stod( config.arg[entry] ); if ( mrate < 0 || mrate > 60 ) { message.str(""); message << "ERROR: TCS_OFFSET_RATE_RA " << mrate << " out of range {0:60}"; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); return( ERROR ); } } catch (std::invalid_argument &) { message.str(""); message << "ERROR: bad TCS_OFFSET_RATE_RA: unable to convert " << config.arg[entry] << " to double"; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); return(ERROR); } catch (std::out_of_range &) { - this->sequence.async.enqueue_and_log( function, "ERROR: TCS_OFFSET_RATE_RA number out of double range" ); + logwrite( function, "ERROR: TCS_OFFSET_RATE_RA number out of double range" ); return(ERROR); } this->sequence.tcs_offsetrate_ra = mrate; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -436,22 +432,22 @@ namespace Sequencer { mrate = std::stod( config.arg[entry] ); if ( mrate < 0 || mrate > 60 ) { message.str(""); message << "ERROR: TCS_OFFSET_RATE_DEC " << mrate << " out of range {0:60}"; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); return( ERROR ); } } catch (std::invalid_argument &) { message.str(""); message << "ERROR: bad TCS_OFFSET_RATE_DEC: unable to convert " << config.arg[entry] << " to double"; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); return(ERROR); } catch (std::out_of_range &) { - this->sequence.async.enqueue_and_log( function, "ERROR: TCS_OFFSET_RATE_DEC number out of double range" ); + logwrite( function, "ERROR: TCS_OFFSET_RATE_DEC number out of double range" ); return(ERROR); } this->sequence.tcs_offsetrate_dec = mrate; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -463,16 +459,16 @@ namespace Sequencer { } catch (std::invalid_argument &) { message.str(""); message << "ERROR: bad TCS_SETTLE_TIMEOUT: unable to convert " << config.arg[entry] << " to double"; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); return(ERROR); } catch (std::out_of_range &) { - this->sequence.async.enqueue_and_log( function, "ERROR: TCS_SETTLE_TIMEOUT number out of double range" ); + logwrite( function, "ERROR: TCS_SETTLE_TIMEOUT number out of double range" ); return(ERROR); } this->sequence.tcs_settle_timeout = to; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -484,16 +480,16 @@ namespace Sequencer { } catch (std::invalid_argument &) { message.str(""); message << "ERROR: bad TCS_SETTLE_STABLE: unable to convert " << config.arg[entry] << " to double"; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); return(ERROR); } catch (std::out_of_range &) { - this->sequence.async.enqueue_and_log( function, "ERROR: TCS_SETTLE_STABLE number out of double range" ); + logwrite( function, "ERROR: TCS_SETTLE_STABLE number out of double range" ); return(ERROR); } this->sequence.tcs_settle_stable = stablet; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -505,16 +501,16 @@ namespace Sequencer { } catch (std::invalid_argument &) { message.str(""); message << "ERROR: bad TCS_DOMEAZI_READY: unable to convert " << config.arg[entry] << " to double"; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); return(ERROR); } catch (std::out_of_range &) { - this->sequence.async.enqueue_and_log( function, "ERROR: TCS_DOMEAZI_READY number out of double range" ); + logwrite( function, "ERROR: TCS_DOMEAZI_READY number out of double range" ); return(ERROR); } this->sequence.tcs_domeazi_ready = domeazi; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -525,16 +521,16 @@ namespace Sequencer { to = std::stod( config.arg[entry] ); } catch (std::invalid_argument &) { - this->sequence.async.enqueue_and_log( function, "ERROR: bad TCS_PREAUTH_TIME: unable to convert to double" ); + logwrite( function, "ERROR: bad TCS_PREAUTH_TIME: unable to convert to double" ); return(ERROR); } catch (std::out_of_range &) { - this->sequence.async.enqueue_and_log( function, "ERROR: TCS_PREAUTH_TIME number out of double range" ); + logwrite( function, "ERROR: TCS_PREAUTH_TIME number out of double range" ); return(ERROR); } this->sequence.tcs_preauth_time = to; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -545,16 +541,16 @@ namespace Sequencer { offset = std::stod( config.arg[entry] ); } catch (std::invalid_argument &) { - this->sequence.async.enqueue_and_log( function, "ERROR: bad ACQUIRE_OFFSET_THRESHOLD: unable to convert to double" ); + logwrite( function, "ERROR: bad ACQUIRE_OFFSET_THRESHOLD: unable to convert to double" ); return(ERROR); } catch (std::out_of_range &) { - this->sequence.async.enqueue_and_log( function, "ERROR: ACQUIRE_OFFSET_THRESHOLD number out of double range" ); + logwrite( function, "ERROR: ACQUIRE_OFFSET_THRESHOLD number out of double range" ); return(ERROR); } this->sequence.target.offset_threshold = offset; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -565,16 +561,16 @@ namespace Sequencer { repeat = std::stoi( config.arg[entry] ); } catch (std::invalid_argument &) { - this->sequence.async.enqueue_and_log( function, "ERROR: bad ACQUIRE_MIN_REPEAT: unable to convert to int" ); + logwrite( function, "ERROR: bad ACQUIRE_MIN_REPEAT: unable to convert to int" ); return(ERROR); } catch (std::out_of_range &) { - this->sequence.async.enqueue_and_log( function, "ERROR: ACQUIRE_MIN_REPEAT number out of int range" ); + logwrite( function, "ERROR: ACQUIRE_MIN_REPEAT number out of int range" ); return(ERROR); } this->sequence.target.min_repeat = repeat; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -585,16 +581,16 @@ namespace Sequencer { offset = std::stod( config.arg[entry] ); } catch (std::invalid_argument &) { - this->sequence.async.enqueue_and_log( function, "ERROR: bad ACQUIRE_TCS_MAX_OFFSET: unable to convert to double" ); + logwrite( function, "ERROR: bad ACQUIRE_TCS_MAX_OFFSET: unable to convert to double" ); return(ERROR); } catch (std::out_of_range &) { - this->sequence.async.enqueue_and_log( function, "ERROR: ACQUIRE_TCS_MAX_OFFSET number out of double range" ); + logwrite( function, "ERROR: ACQUIRE_TCS_MAX_OFFSET number out of double range" ); return(ERROR); } this->sequence.target.max_tcs_offset = offset; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); applied++; } @@ -609,7 +605,7 @@ namespace Sequencer { if ( this->sequence.target.configure_db( DB_HOST, config.arg[entry] ) == NO_ERROR ) { applied++; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } } @@ -618,7 +614,7 @@ namespace Sequencer { if ( this->sequence.target.configure_db( DB_PORT, config.arg[entry] ) == NO_ERROR ) { applied++; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } } @@ -627,7 +623,7 @@ namespace Sequencer { if ( this->sequence.target.configure_db( DB_USER, config.arg[entry] ) == NO_ERROR ) { applied++; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } } @@ -636,7 +632,7 @@ namespace Sequencer { if ( this->sequence.target.configure_db( DB_PASS, config.arg[entry] ) == NO_ERROR ) { applied++; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } } @@ -645,7 +641,7 @@ namespace Sequencer { if ( this->sequence.target.configure_db( DB_SCHEMA, config.arg[entry] ) == NO_ERROR ) { applied++; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } } @@ -654,7 +650,7 @@ namespace Sequencer { if ( this->sequence.target.configure_db( DB_ACTIVE, config.arg[entry] ) == NO_ERROR ) { applied++; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } } @@ -663,7 +659,7 @@ namespace Sequencer { if ( this->sequence.target.configure_db( DB_COMPLETED, config.arg[entry] ) == NO_ERROR ) { applied++; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } } @@ -672,7 +668,7 @@ namespace Sequencer { if ( this->sequence.target.configure_db( DB_SETS, config.arg[entry] ) == NO_ERROR ) { applied++; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } } @@ -681,7 +677,7 @@ namespace Sequencer { this->sequence.camera_prologue.push_back( this->config.arg[entry] ); applied++; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } // CAMERA_EPILOGUE @@ -689,7 +685,7 @@ namespace Sequencer { this->sequence.camera_epilogue.push_back( this->config.arg[entry] ); applied++; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } // *__INIT @@ -701,7 +697,7 @@ namespace Sequencer { this->sequence.config_init[key] = this->config.arg[entry]; applied++; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } // *__SHUTDOWN @@ -713,7 +709,7 @@ namespace Sequencer { this->sequence.config_shutdown[key] = this->config.arg[entry]; applied++; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } // VIRTUAL_SLITW_ACQUIRE @@ -755,7 +751,7 @@ namespace Sequencer { if ( this->sequence.power_switch[POWER_LAMP].configure( this->config.arg[entry] ) == NO_ERROR ) { applied++; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } } @@ -764,7 +760,7 @@ namespace Sequencer { if ( this->sequence.power_switch[POWER_SLIT].configure( this->config.arg[entry] ) == NO_ERROR ) { applied++; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } } @@ -773,7 +769,7 @@ namespace Sequencer { if ( this->sequence.power_switch[POWER_CAMERA].configure( this->config.arg[entry] ) == NO_ERROR ) { applied++; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } } @@ -782,7 +778,7 @@ namespace Sequencer { if ( this->sequence.power_switch[POWER_CALIB].configure( this->config.arg[entry] ) == NO_ERROR ) { applied++; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } } @@ -791,7 +787,7 @@ namespace Sequencer { if ( this->sequence.power_switch[POWER_FLEXURE].configure( this->config.arg[entry] ) == NO_ERROR ) { applied++; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } } @@ -800,7 +796,7 @@ namespace Sequencer { if ( this->sequence.power_switch[POWER_FILTER].configure( this->config.arg[entry] ) == NO_ERROR ) { applied++; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } } @@ -809,7 +805,7 @@ namespace Sequencer { if ( this->sequence.power_switch[POWER_FOCUS].configure( this->config.arg[entry] ) == NO_ERROR ) { applied++; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } } @@ -818,7 +814,7 @@ namespace Sequencer { if ( this->sequence.power_switch[POWER_TELEM].configure( this->config.arg[entry] ) == NO_ERROR ) { applied++; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } } @@ -827,7 +823,7 @@ namespace Sequencer { if ( this->sequence.power_switch[POWER_THERMAL].configure( this->config.arg[entry] ) == NO_ERROR ) { applied++; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } } @@ -836,7 +832,7 @@ namespace Sequencer { if ( this->sequence.power_switch[POWER_ACAM].configure( this->config.arg[entry] ) == NO_ERROR ) { applied++; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } } @@ -845,7 +841,7 @@ namespace Sequencer { if ( this->sequence.power_switch[POWER_ACAM_CAM].configure( this->config.arg[entry] ) == NO_ERROR ) { applied++; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } } @@ -854,7 +850,7 @@ namespace Sequencer { if ( this->sequence.power_switch[POWER_SLICECAM].configure( this->config.arg[entry] ) == NO_ERROR ) { applied++; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } } @@ -863,33 +859,8 @@ namespace Sequencer { if ( this->sequence.caltarget.configure( config.arg[entry] ) == NO_ERROR ) { applied++; message.str(""); message << "SEQUENCERD:config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( function, message.str() ); - } - } - - // TELEM_PROVIDER : contains daemon name and port to contact for header telemetry info - // - if ( config.param[entry] == "TELEM_PROVIDER" ) { - std::vector tokens; - Tokenize( config.arg[entry], tokens, " " ); - try { - if ( tokens.size() == 2 ) { - this->sequence.telemetry_providers[tokens.at(0)] = std::stod(tokens.at(1)); - } - else { - message.str(""); message << "ERROR bad format TELEM_PROVIDER=\"" << config.arg[entry] << "\": expected "; - logwrite( function, message.str() ); - return ERROR; - } - } - catch ( const std::exception &e ) { - message.str(""); message << "ERROR parsing TELEM_PROVIDER from " << config.arg[entry] << ": " << e.what(); logwrite( function, message.str() ); - return ERROR; } - message.str(""); message << "config:" << config.param[entry] << "=" << config.arg[entry]; - this->sequence.async.enqueue_and_log( "SEQUENCERD", function, message.str() ); - applied++; } } // end loop through the entries in the configuration file @@ -1020,7 +991,7 @@ namespace Sequencer { /***** Server::async_main ***************************************************/ /** - * @brief asynchronous message sending thread + * @brief [obsolete] asynchronous message sending thread * @param[in] seq reference to Sequencer::Server object * @param[in] sock Network::udpSocket socket object * @@ -1028,36 +999,38 @@ namespace Sequencer { * sent out via multi-cast UDP datagram. * */ - void Server::async_main( Sequencer::Server &seq, Network::UdpSocket sock ) { - std::string function = "Sequencer::Server::async_main"; - std::stringstream message; - int retval; - - retval = sock.Create(); // create the UDP socket - if (retval < 0) { - logwrite(function, "error creating UDP multicast socket for asynchronous messages"); - seq.exit_cleanly(); // do not continue on error - } - if (retval==1) { // exit this thread but continue with daemon - logwrite(function, "asyncrhonous message port disabled by request"); - } - - while (1) { - std::string message = seq.sequence.async.dequeue(); // get the latest message from the queue (blocks) - retval = sock.Send(message); // transmit the message - if (retval < 0) { - std::stringstream errstm; - errstm << "error sending UDP message: " << message; - logwrite(function, errstm.str()); - } - if (message=="exit") { // terminate this thread - sock.Close(); - return; - } - } - - return; - } +/*** + *void Server::async_main( Sequencer::Server &seq, Network::UdpSocket sock ) { + * std::string function = "Sequencer::Server::async_main"; + * std::stringstream message; + * int retval; + * + * retval = sock.Create(); // create the UDP socket + * if (retval < 0) { + * logwrite(function, "error creating UDP multicast socket for asynchronous messages"); + * seq.exit_cleanly(); // do not continue on error + * } + * if (retval==1) { // exit this thread but continue with daemon + * logwrite(function, "asyncrhonous message port disabled by request"); + * } + * + * while (1) { + * std::string message = seq.sequence.async.dequeue(); // get the latest message from the queue (blocks) + * retval = sock.Send(message); // transmit the message + * if (retval < 0) { + * std::stringstream errstm; + * errstm << "error sending UDP message: " << message; + * logwrite(function, errstm.str()); + * } + * if (message=="exit") { // terminate this thread + * sock.Close(); + * return; + * } + * } + * + * return; + *} + ***/ /***** Server::async_main ***************************************************/ @@ -1096,11 +1069,11 @@ namespace Sequencer { if ( ( pollret=sock.Poll() ) <= 0 ) { if (pollret==0) { message.str(""); message << "ERROR: Poll timeout on fd " << sock.getfd() << " thread " << sock.id; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } if ( pollret <0 && errno ) { message.str(""); message << "ERROR: Poll error on fd " << sock.getfd() << " thread " << sock.id << ": " << strerror(errno); - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } break; // this will close the connection } @@ -1116,11 +1089,11 @@ namespace Sequencer { #endif if ( ret<0 && errno ) { // could be an actual read error message.str(""); message << "ERROR: Read error on fd " << sock.getfd() << ": " << strerror(errno); - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } if (ret==-2) { // or a timeout message.str(""); message << "ERROR: timeout reading from fd " << sock.getfd(); - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); } break; // Breaking out of the while loop will close the connection. // This probably means that the client has terminated abruptly, @@ -1159,12 +1132,12 @@ namespace Sequencer { catch ( std::runtime_error &e ) { std::stringstream errstream; errstream << e.what(); message.str(""); message << "ERROR: parsing arguments: " << errstream.str(); - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); ret = -1; } catch ( ... ) { message.str(""); message << "ERROR: unknown error parsing arguments: " << args; - this->sequence.async.enqueue_and_log( function, message.str() ); + logwrite( function, message.str() ); ret = -1; } @@ -1397,7 +1370,7 @@ namespace Sequencer { // if ( cmd == SEQUENCERD_REPEAT ) { if ( !this->sequence.seq_state_manager.is_set( Sequencer::SEQ_READY ) ) { - this->sequence.async.enqueue_and_log( function, "ERROR cannot start exposure: not ready" ); + this->sequence.broadcast.error( function, "cannot start exposure: not ready" ); ret = ERROR; } else { @@ -1493,7 +1466,7 @@ namespace Sequencer { if ( cmd == SEQUENCERD_TARGETSET ) { ret= this->sequence.target.targetset( args, retstring ); message.str(""); message << "TARGETSET: " << retstring; - this->sequence.async.enqueue( message.str() ); + logwrite(function, message.str()); retstring.append( " " ); } else @@ -1504,14 +1477,14 @@ namespace Sequencer { // Can only pause during an exposure // if ( ! this->sequence.seq_state_manager.is_set( Sequencer::SEQ_WAIT_EXPOSE ) ) { - this->sequence.async.enqueue_and_log( function, "ERROR: can only pause during an active exposure" ); + this->sequence.broadcast.error( function, "can only pause during an active exposure" ); ret = ERROR; } else // Can't already be paused // if ( this->sequence.seq_state_manager.is_set( Sequencer::SEQ_PAUSED ) ) { - this->sequence.async.enqueue_and_log( function, "ERROR: already paused" ); + this->sequence.broadcast.error( function, "already paused" ); ret = ERROR; } else { @@ -1530,7 +1503,7 @@ namespace Sequencer { // Can only resume when paused // if ( ! this->sequence.seq_state_manager.is_set( Sequencer::SEQ_PAUSED ) ) { - this->sequence.async.enqueue_and_log( function, "ERROR: can only resume when paused" ); + this->sequence.broadcast.error( function, "can only resume when paused" ); ret = ERROR; } else { @@ -1557,7 +1530,7 @@ namespace Sequencer { if ( cmd.compare( SEQUENCERD_MODEXPTIME ) == 0 ) { Tokenize( args, tokens, " " ); if ( tokens.size() != 1 ) { - this->sequence.async.enqueue_and_log( function, "ERROR: expected MODEXPTIME " ); + this->sequence.broadcast.error( function, "expected MODEXPTIME " ); ret = ERROR; } else { @@ -1566,13 +1539,13 @@ namespace Sequencer { double exptime_req=0; try { exptime_req = std::stod( tokens.at(0) ); } catch( std::out_of_range &e ) { - message.str(""); message << "ERROR: out of range parsing args " << args << ": " << e.what(); - this->sequence.async.enqueue_and_log( function, message.str() ); + message.str(""); message << "out of range parsing args " << args << ": " << e.what(); + this->sequence.broadcast.error( function, message.str() ); ret = ERROR; } catch( std::invalid_argument &e ) { - message.str(""); message << "ERROR: invalid argument parsing args " << args << ": " << e.what(); - this->sequence.async.enqueue_and_log( function, message.str() ); + message.str(""); message << "invalid argument parsing args " << args << ": " << e.what(); + this->sequence.broadcast.error( function, message.str() ); ret = ERROR; } @@ -1612,8 +1585,8 @@ namespace Sequencer { // Unknown commands generate an error // else { - message.str(""); message << "ERROR: unknown command: " << cmd; - this->sequence.async.enqueue_and_log( function, message.str() ); + message.str(""); message << "unknown command: " << cmd; + this->sequence.broadcast.error( function, message.str() ); ret = ERROR; } diff --git a/sequencerd/sequencer_server.h b/sequencerd/sequencer_server.h index f785d42d..7fff66a5 100644 --- a/sequencerd/sequencer_server.h +++ b/sequencerd/sequencer_server.h @@ -145,7 +145,7 @@ namespace Sequencer { static void block_main( Sequencer::Server &server, std::shared_ptr ); ///< main function for blocking connection thread static void thread_main( Sequencer::Server &server, std::shared_ptr sock); ///< main function for all non-blocked threads static void gui_main( Sequencer::Server &server, std::shared_ptr sock ); ///< main function for gui threads - static void async_main( Sequencer::Server &server, Network::UdpSocket sock ); ///< asynchronous message sending thread +// static void async_main( Sequencer::Server &server, Network::UdpSocket sock ); ///< asynchronous message sending thread void handle_signal( int signo ); diff --git a/sequencerd/sequencerd.cpp b/sequencerd/sequencerd.cpp index 3b9ba63f..ac6bef1c 100644 --- a/sequencerd/sequencerd.cpp +++ b/sequencerd/sequencerd.cpp @@ -131,11 +131,13 @@ int main(int argc, char **argv) { // if ( sequencerd.sequence.init_pubsub( { Topic::CAMERAD, Topic::ACAMD, - Topic::SLICECAMD } ) == ERROR ) { + Topic::SLICECAMD, + Topic::SLITD, + Topic::TCSD } ) == ERROR ) { logwrite(function, "ERROR initializing publisher-subscriber handler"); sequencerd.exit_cleanly(); } - sequencerd.sequence.seq_state_manager.set(Sequencer::SEQ_NOTREADY); + sequencerd.sequence.seq_state_manager.set_only({Sequencer::SEQ_NOTREADY}); std::this_thread::sleep_for( std::chrono::milliseconds(100) ); sequencerd.sequence.publish_snapshot(); @@ -222,14 +224,17 @@ int main(int argc, char **argv) { // Instantiate a multicast UDP object and spawn a thread to send asynchronous messages // - Network::UdpSocket msg(sequencerd.messageport, sequencerd.messagegroup); - std::thread( std::ref(Sequencer::Server::async_main), - std::ref(sequencerd), - msg ).detach(); +/*** + *Network::UdpSocket msg(sequencerd.messageport, sequencerd.messagegroup); + *std::thread( std::ref(Sequencer::Server::async_main), + * std::ref(sequencerd), + * msg ).detach(); + ***/ // Create my own asynchronous listener thread. // This thread allows the sequencer to listen for asynchronous messages. // + Network::UdpSocket msg(sequencerd.messageport, sequencerd.messagegroup); std::thread( std::ref( Sequencer::Sequence::dothread_sequencer_async_listener ), std::ref( sequencerd.sequence), msg diff --git a/slicecamd/slicecam_interface.cpp b/slicecamd/slicecam_interface.cpp index 7f26c795..e6e86bd6 100644 --- a/slicecamd/slicecam_interface.cpp +++ b/slicecamd/slicecam_interface.cpp @@ -462,16 +462,16 @@ namespace Slicecam { // Common::extract_telemetry_value( jmessage, "TCSNAME", telem.tcsname ); Common::extract_telemetry_value( jmessage, "ISOPEN", telem.is_tcs_open ); - Common::extract_telemetry_value( jmessage, "CASANGLE", telem.angle_scope ); - Common::extract_telemetry_value( jmessage, "TELRA", telem.ra_scope_hms ); - Common::extract_telemetry_value( jmessage, "TELDEC", telem.dec_scope_dms ); + Common::extract_telemetry_value( jmessage, Key::Tcsd::CASANGLE, telem.angle_scope ); + Common::extract_telemetry_value( jmessage, Key::Tcsd::TELRA, telem.ra_scope_hms ); + Common::extract_telemetry_value( jmessage, Key::Tcsd::TELDEC, telem.dec_scope_dms ); Common::extract_telemetry_value( jmessage, "RA", telem.ra_scope_h ); Common::extract_telemetry_value( jmessage, "DEC", telem.dec_scope_d ); Common::extract_telemetry_value( jmessage, "RAOFFSET", telem.offsetra ); Common::extract_telemetry_value( jmessage, "DECLOFFS", telem.offsetdec ); - Common::extract_telemetry_value( jmessage, "AZ", telem.az ); + Common::extract_telemetry_value( jmessage, Key::Tcsd::AZ, telem.az ); Common::extract_telemetry_value( jmessage, "TELFOCUS", telem.telfocus ); - Common::extract_telemetry_value( jmessage, "AIRMASS", telem.airmass ); + Common::extract_telemetry_value( jmessage, Key::Tcsd::AIRMASS, telem.airmass ); } /***** Slicecam::Interface::handletopic_tcsd ********************************/ diff --git a/slitd/slit_interface.cpp b/slitd/slit_interface.cpp index 2569478c..96edc275 100644 --- a/slitd/slit_interface.cpp +++ b/slitd/slit_interface.cpp @@ -104,10 +104,10 @@ namespace Slit { std::string retstring; this->is_open( "", retstring ); - snapshot.isopen = ( retstring=="true" ? true : false ); - if ( snapshot.isopen ) { + status.isopen = ( retstring=="true" ? true : false ); + if ( status.isopen ) { this->is_home( "", retstring ); - snapshot.ishome = ( retstring=="true" ? true : false ); + status.ishome = ( retstring=="true" ? true : false ); } this->get( retstring ); @@ -285,14 +285,14 @@ namespace Slit { return HELP; } - if ( std::isnan(snapshot.width.arcsec()) ) { + if ( std::isnan(status.width.arcsec()) ) { logwrite( "Slit::Interface::offset", "ERROR width not previously set" ); retstring="undefined_width"; return ERROR; } std::stringstream cmd; - cmd << snapshot.width.arcsec() << " " << args; + cmd << status.width.arcsec() << " " << args; return this->set( cmd.str(), retstring ); } @@ -374,7 +374,7 @@ namespace Slit { else fval = std::round( fval * 10.0 ) / 10.0; // round to nearest tenth } reqwidth = SlitDimension( fval, unit ); - reqoffset = snapshot.offset; + reqoffset = status.offset; } if ( tokens.size() == 2 ) { if ( tokens.at(1).find("mm") != std::string::npos ) unit=Unit::MM; else unit=Unit::ARCSEC; @@ -502,30 +502,27 @@ namespace Slit { // this call reads the controller and returns the numeric values // - error = this->read_positions( poswidth, posoffset, snapshot.posA, snapshot.posB ); + error = this->read_positions( poswidth, posoffset, status.posA, status.posB ); // store the current readings in the class // - snapshot.width = SlitDimension( poswidth, Unit::MM ); - snapshot.offset = SlitDimension( posoffset, Unit::MM ); + status.width = SlitDimension( poswidth, Unit::MM ); + status.offset = SlitDimension( posoffset, Unit::MM ); // form the return value // std::stringstream s; if ( args=="mm" ) { - s << std::setprecision(2) << std::fixed << snapshot.width.mm() << " " - << std::setprecision(3) << snapshot.offset.mm() << " mm"; + s << std::setprecision(2) << std::fixed << status.width.mm() << " " + << std::setprecision(3) << status.offset.mm() << " mm"; } else { - s << std::setprecision(2) << std::fixed << snapshot.width.arcsec() << " " - << std::setprecision(3) << snapshot.offset.arcsec(); + s << std::setprecision(2) << std::fixed << status.width.arcsec() << " " + << std::setprecision(3) << status.offset.arcsec(); } retstring = s.str(); - message.str(""); message << "NOTICE:" << Slit::DAEMON_NAME << " " << retstring; - this->async.enqueue( message.str() ); - - this->publish_snapshot(); + this->publish_status(); return error; } @@ -713,55 +710,42 @@ namespace Slit { * @param[in] jmessage_in subscribed-received JSON message * */ - void Interface::handletopic_snapshot( const nlohmann::json &jmessage_in ) { - // If my name is in the jmessage then publish my snapshot - // - if ( jmessage_in.contains( Slit::DAEMON_NAME ) ) { - this->publish_snapshot(); - } - else - if ( jmessage_in.contains( "test" ) ) { - logwrite( "Slit::Interface::handletopic_snapshot", jmessage_in.dump() ); - } + void Interface::handletopic_snapshot( const nlohmann::json &jmessage ) { + if ( jmessage.contains(Topic::SLITD) ) this->publish_status(); } /***** Slit::Interface::handletopic_snapshot ********************************/ - /***** Slit::Interface::publish_snapshot ************************************/ + /***** Slit::Interface::publish_status **************************************/ /** - * @brief publishes snapshot of my telemetry - * @details This publishes a JSON message containing a snapshot of my - * telemetry. + * @brief publishes my status on change + * @param[in] force optional (default=false) force publish irrespective of change * */ - void Interface::publish_snapshot() { - std::string dontcare; - this->publish_snapshot(dontcare); - } - void Interface::publish_snapshot(std::string &retstring) { + void Interface::publish_status(bool force) { + + // unless forced, only publish if there was a change + if ( !force && this->status == this->last_published_status ) return; + nlohmann::json jmessage_out; - jmessage_out["source"] = "slitd"; - jmessage_out["ISOPEN"] = snapshot.isopen; - jmessage_out["ISHOME"] = snapshot.ishome; - jmessage_out["SLITW"] = snapshot.width.arcsec(); - jmessage_out["SLITO"] = snapshot.offset.arcsec(); - jmessage_out["SLITPOSA"] = snapshot.posA; - jmessage_out["SLITPOSB"] = snapshot.posB; - - // for backwards compatibility - jmessage_out["messagetype"] = "slitinfo"; - retstring=jmessage_out.dump(); - retstring.append(JEOF); + jmessage_out[Key::SOURCE] = Topic::SLITD; + jmessage_out[Key::Slitd::ISOPEN] = this->status.isopen; + jmessage_out[Key::Slitd::ISHOME] = this->status.ishome; + jmessage_out[Key::Slitd::SLITW] = this->status.width.arcsec(); + jmessage_out[Key::Slitd::SLITO] = this->status.offset.arcsec(); + jmessage_out[Key::Slitd::SLITPOSA] = this->status.posA; + jmessage_out[Key::Slitd::SLITPOSB] = this->status.posB; + + this->last_published_status = this->status; try { this->publisher->publish( jmessage_out ); } catch ( const std::exception &e ) { - logwrite( "Slit::Interface::publish_snapshot", - "ERROR publishing message: "+std::string(e.what()) ); - return; + logwrite( "Slit::Interface::publish_status", + "ERROR publishing status: "+std::string(e.what()) ); } } - /***** Slit::Interface::publish_snapshot ************************************/ + /***** Slit::Interface::publish_status **************************************/ } diff --git a/slitd/slit_interface.h b/slitd/slit_interface.h index 02111f81..235c2455 100644 --- a/slitd/slit_interface.h +++ b/slitd/slit_interface.h @@ -8,6 +8,7 @@ #pragma once +#include "message_keys.h" #include "network.h" #include "pi.h" #include "logentry.h" @@ -207,16 +208,24 @@ namespace Slit { SlitDimension minwidth; ///< set by config file SlitDimension center; ///< position of center in actuator units - typedef struct { + struct Status { SlitDimension width; SlitDimension offset; float posA=NAN; float posB=NAN; bool ishome=false; bool isopen=false; - } snapshot_t; - snapshot_t snapshot; + bool operator==(const Status& other) const { + return std::tie(width, offset, posA, posB, ishome, isopen) == + std::tie(other.width, other.offset, other.posA, other.posB, other.ishome, other.isopen); + } + + bool operator!=(const Status& other) const { return !(*this == other); } + }; + + Status status; + Status last_published_status; Common::Queue async; @@ -233,8 +242,7 @@ namespace Slit { void stop_subscriber_thread() { Common::PubSubHandler::stop_subscriber_thread(*this); } void handletopic_snapshot( const nlohmann::json &jmessage ); - void publish_snapshot(); - void publish_snapshot(std::string &retstring); + void publish_status(bool force=false); long initialize_class(); long open(); ///< opens the PI socket connection diff --git a/slitd/slit_server.cpp b/slitd/slit_server.cpp index 3a298599..1c39d8f4 100644 --- a/slitd/slit_server.cpp +++ b/slitd/slit_server.cpp @@ -597,22 +597,6 @@ namespace Slit { if ( cmd == SLITD_NATIVE ) { ret = this->interface.send_command( args, retstring ); } - else - - // send telemetry on request - // - if ( cmd == TELEMREQUEST ) { - if ( args=="?" || args=="help" ) { - retstring=TELEMREQUEST+"\n"; - retstring.append( " Returns a serialized JSON message containing telemetry\n" ); - retstring.append( " information, terminated with \"EOF\\n\".\n" ); - ret=HELP; - } - else { - this->interface.publish_snapshot(retstring); - ret = JSON; - } - } // unknown commands generate an error // diff --git a/slitd/slitd.cpp b/slitd/slitd.cpp index 23daaa3e..4fd6a1fb 100644 --- a/slitd/slitd.cpp +++ b/slitd/slitd.cpp @@ -127,7 +127,7 @@ int main(int argc, char **argv) { } std::this_thread::sleep_for( std::chrono::milliseconds(100) ); - slitd.interface.publish_snapshot(); + slitd.interface.publish_status(true); // This will pre-thread N_THREADS threads. // The 0th thread is reserved for the blocking port, and the rest are for the non-blocking port. diff --git a/tcsd/tcs_interface.cpp b/tcsd/tcs_interface.cpp index ed59ae1f..1519fc51 100644 --- a/tcsd/tcs_interface.cpp +++ b/tcsd/tcs_interface.cpp @@ -42,28 +42,32 @@ namespace TCS { this->get_tcs_info(); nlohmann::json jmessage_out; - jmessage_out["source"] = "tcsd"; - - jmessage_out["ISOPEN"] = this->tcs_info.isopen; - jmessage_out["TCSNAME"] = this->tcs_info.tcsname; - - jmessage_out["PA"] = this->tcs_info.pa; // double - jmessage_out["CASANGLE"] = this->tcs_info.cassangle; // double - jmessage_out["HA"] = this->tcs_info.ha; // string - jmessage_out["RAOFFSET"] = this->tcs_info.offsetra; // double - jmessage_out["DECLOFFS"] = this->tcs_info.offsetdec; // double - jmessage_out["TELRA"] = this->tcs_info.ra_hms; // string "hh:mm:ss.s" - jmessage_out["TELDEC"] = this->tcs_info.dec_dms; // string "dd:mm:ss.s" - jmessage_out["RA"] = radec_to_decimal( this->tcs_info.ra_hms ); - jmessage_out["DEC"] = radec_to_decimal( this->tcs_info.dec_dms ); - jmessage_out["AZ"] = this->tcs_info.azimuth; - jmessage_out["ALT"] = 90. - this->tcs_info.zenithangle; - jmessage_out["ZENANGLE"] = this->tcs_info.zenithangle; - jmessage_out["DOMEAZ"] = this->tcs_info.domeazimuth; - jmessage_out["DOMESHUT"] = this->tcs_info.domeshutters==1?"open":"closed"; - jmessage_out["TELFOCUS"] = this->tcs_info.focus; - jmessage_out["AIRMASS"] = this->tcs_info.airmass; - jmessage_out["MOTION"] = this->tcs_info.motion; + jmessage_out[Key::SOURCE] = Daemon::TCSD; + + { + std::lock_guard lock(tcs_info_mtx); + + jmessage_out["ISOPEN"] = this->tcs_info.isopen; + jmessage_out["TCSNAME"] = this->tcs_info.tcsname; + + jmessage_out["PA"] = this->tcs_info.pa; // double + jmessage_out[Key::Tcsd::CASANGLE] = this->tcs_info.cassangle; // double + jmessage_out["HA"] = this->tcs_info.ha; // string + jmessage_out["RAOFFSET"] = this->tcs_info.offsetra; // double + jmessage_out["DECLOFFS"] = this->tcs_info.offsetdec; // double + jmessage_out[Key::Tcsd::TELRA] = this->tcs_info.ra_hms; // string "hh:mm:ss.s" + jmessage_out[Key::Tcsd::TELDEC] = this->tcs_info.dec_dms; // string "dd:mm:ss.s" + jmessage_out["RA"] = radec_to_decimal( this->tcs_info.ra_hms ); + jmessage_out["DEC"] = radec_to_decimal( this->tcs_info.dec_dms ); + jmessage_out[Key::Tcsd::AZ] = this->tcs_info.azimuth; + jmessage_out[Key::Tcsd::ALT] = 90. - this->tcs_info.zenithangle; + jmessage_out["ZENANGLE"] = this->tcs_info.zenithangle; + jmessage_out["DOMEAZ"] = this->tcs_info.domeazimuth; + jmessage_out["DOMESHUT"] = this->tcs_info.domeshutters==1?"open":"closed"; + jmessage_out["TELFOCUS"] = this->tcs_info.focus; + jmessage_out[Key::Tcsd::AIRMASS] = this->tcs_info.airmass; + jmessage_out["MOTION"] = this->tcs_info.motion; + } // for backwards compatibility jmessage_out["messagetype"] = "tcsinfo"; @@ -82,6 +86,58 @@ namespace TCS { /***** TCS::Interface::publish_snapshot *************************************/ + /***** TCS::Interface::do_continuous_snapshot *******************************/ + /** + * @brief publish snapshot at 1 Hz when connected + * + */ + void Interface::do_continuous_snapshot() { + auto next = std::chrono::steady_clock::now(); + while (should_publish.load()) { + bool isopen = false; + { + std::lock_guard lock(tcs_info_mtx); + isopen = this->tcs_info.isopen; + } + if (isopen) publish_snapshot(); + next += std::chrono::seconds(1); + std::this_thread::sleep_until(next); + } + } + /***** TCS::Interface::do_continuous_snapshot *******************************/ + + + /***** TCS::Interface::publish_state ****************************************/ + /** + * @brief set | get snapshot publish state + * @param[in] arg on|off + * @param[out] retstring reference to string to contain the state + * @return NO_ERROR | HELP + * + */ + long Interface::publish_state( const std::string &arg, std::string &retstring ) { + const std::string function = "TCS::Interface::publish_state"; + + // help + if ( arg == "?" || arg == "help" ) { + retstring = TCSD_PUBLISHSTATE; + retstring.append( " on | off\n" ); + retstring.append( " set | get continuous snapshot publish state\n" ); + return HELP; + } + // on + else if ( arg == "on" ) should_publish.store(true); + + // off + else if ( arg == "off" ) should_publish.store(false); + + retstring = should_publish.load() ? "on" : "off"; + + return NO_ERROR; + } + /***** TCS::Interface::publish_state ****************************************/ + + /***** TCS::Interface::get_tcs_info *****************************************/ /** * @brief fills the tcs_info class @@ -92,6 +148,8 @@ namespace TCS { long error = NO_ERROR; std::string retstring; + std::lock_guard lock(tcs_info_mtx); + // erase the class because it's all or nothing. If something fails partway // through, we don't want to mix values from a command now with values from // an earlier command. E.G. if reqpos fails here but reqstat and weather @@ -371,10 +429,12 @@ namespace TCS { } } + { + std::lock_guard lock(tcs_info_mtx); this->tcs_info.isopen = ( ! _name.empty() ? true : false ); this->tcs_info.tcsname = _name; - retstring = ( this->tcs_info.isopen ? "true" : "false" ); // return string is the state + } asyncmsg.str(""); asyncmsg << "TCSD:open:" << retstring; this->async.enqueue( asyncmsg.str() ); // broadcast the state @@ -1137,10 +1197,12 @@ namespace TCS { // parse the reply which stores it in the TcsInfo class // - this->tcs_info.parse_pa(tcsreply); - std::ostringstream oss; + this->tcs_info.parse_pa(tcsreply); + { + std::lock_guard lock(tcs_info_mtx); oss << this->tcs_info.pa; + } retstring = oss.str(); if ( !retstring.empty() && !silent ) logwrite( function, retstring ); diff --git a/tcsd/tcs_interface.h b/tcsd/tcs_interface.h index 9c76b03f..62776446 100644 --- a/tcsd/tcs_interface.h +++ b/tcsd/tcs_interface.h @@ -13,6 +13,7 @@ #include "common.h" #include "tcs_constants.h" #include "tcsd_commands.h" +#include "message_keys.h" #include #include #include @@ -427,6 +428,7 @@ logwrite(function,message.str()); private: zmqpp::context context; std::string default_tcs; ///< default TCS to use specified in .cfg + std::mutex tcs_info_mtx; ///< protects tcs_info public: inline void set_default_tcs(const std::string &which) { this->default_tcs=which; } @@ -445,13 +447,13 @@ logwrite(function,message.str()); std::condition_variable publish_condition; std::condition_variable collect_condition; - std::atomic publish_enable; + std::atomic should_publish; std::atomic collect_enable; Interface() : context(), offsetrate(0), - publish_enable(false), + should_publish(true), collect_enable(false), subscriber(std::make_unique(context, Common::PubSub::Mode::SUB)), is_subscriber_thread_running(false), @@ -487,6 +489,7 @@ logwrite(function,message.str()); void publish_snapshot(); void publish_snapshot(std::string &retstring); + void do_continuous_snapshot(); /** * These are the functions for communicating with the TCS @@ -499,6 +502,7 @@ logwrite(function,message.str()); long isopen( std::string &retstring ); long isopen( const std::string &arg, std::string &retstring ); long close(); + long publish_state( const std::string &arg, std::string &retstring ); long get_name( const std::string &arg, std::string &retstring ); long get_weather_coords( const std::string &arg, std::string &retstring ); long get_coords( const std::string &arg, std::string &retstring ); diff --git a/tcsd/tcs_server.cpp b/tcsd/tcs_server.cpp index 6139981e..96882f93 100644 --- a/tcsd/tcs_server.cpp +++ b/tcsd/tcs_server.cpp @@ -668,6 +668,13 @@ void doit(TcsIO &tcs_io, const std::string &client_cmd, bool is_slow_command) { } else + // publishstate + // + if ( cmd.compare( TCSD_PUBLISHSTATE ) == 0 ) { + ret = this->interface.publish_state( args, retstring ); + } + else + // llist // if ( cmd.compare( TCSD_LLIST ) == 0 ) { diff --git a/tcsd/tcsd.cpp b/tcsd/tcsd.cpp index e7d19000..c21c20b5 100644 --- a/tcsd/tcsd.cpp +++ b/tcsd/tcsd.cpp @@ -132,6 +132,10 @@ int main(int argc, char **argv) { // publish my snapshot so the world knows I'm online tcsd.interface.publish_snapshot(); + // thread to publish snapshot when connected + std::thread( &TCS::Interface::do_continuous_snapshot, + std::ref(tcsd.interface) ).detach(); + // This will pre-thread N_THREADS threads, a little differently from other // daemons. There will be N_THREADS-1 non-blocking threads as before then // loop forever on Accept to dynamically spawn a new thread for each blocking diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index baa453ba..d8992d58 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -66,3 +66,15 @@ target_link_libraries( listener PRIVATE utilities ) find_library( ZMQPP_LIB zmqpp NAMES libzmqpp PATHS /usr/local/lib ) find_library( ZMQ_LIB zmq NAMES libzmq PATHS /usr/local/lib ) +add_executable( seqmon + ${PROJECT_UTILS_DIR}/seqmon.cpp + ) +target_include_directories( seqmon PRIVATE + ${PROJECT_BASE_DIR}/utils + ${PROJECT_BASE_DIR}/common + ) +target_link_libraries( seqmon PRIVATE + ${ZMQPP_LIB} + ${ZMQ_LIB} + ) + diff --git a/utils/seqmon.cpp b/utils/seqmon.cpp new file mode 100644 index 00000000..2d0e053e --- /dev/null +++ b/utils/seqmon.cpp @@ -0,0 +1,491 @@ +/** + * @file seqmon.cpp + * @brief terminal-based subscriber that mimics the sequencer operator display + * @author David Hale + * + * Subscribes to sequencer topics on the ZMQ broker and renders a simple + * color-coded terminal display: + * + * seq_seqstate lifecycle state (top line, color by value) + * seq_waitstate active wait bits (second line, list of set labels) + * seq_daemonstate daemon online/offline status + * acamd acquisition mode, acquired flag, seeing + * slicecamd fine-acquisition locked/running flags + * broadcast operator messages (scrolling log, color by severity) + * + * On startup a Topic::SNAPSHOT request is published so that sequencerd, + * acamd, and slicecamd immediately re-publish their current telemetry, + * avoiding a stale display until the next natural update. + * + * This is a standalone utility intended as a stand-in for the GUI message + * window during testing. It has no dependencies on the sequencer sources; + * it only shares the message_keys.h contract. + * + */ + +#include +#include +#include "message_keys.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + + constexpr const char *BROKER_ENDPOINT = "tcp://localhost:5556"; // subscribers connect here + constexpr const char *BROKER_PUB_ENDPOINT = "tcp://localhost:5555"; // publishers connect here + + // ANSI escape codes + // + constexpr const char *ANSI_RESET = "\033[0m"; + constexpr const char *ANSI_GRAY = "\033[90m"; + constexpr const char *ANSI_RED = "\033[31m"; + constexpr const char *ANSI_GREEN = "\033[32m"; + constexpr const char *ANSI_YELLOW = "\033[33m"; + constexpr const char *ANSI_BLUE = "\033[34m"; + constexpr const char *ANSI_CYAN = "\033[36m"; + constexpr const char *ANSI_BOLDYEL = "\033[1;33m"; + + // cursor control: save/restore, move up 3 lines, clear entire line + // + constexpr const char *CURSOR_SAVE = "\033[s"; + constexpr const char *CURSOR_RESTORE = "\033[u"; + constexpr const char *CURSOR_HOME = "\033[H"; + constexpr const char *CLEAR_SCREEN = "\033[2J"; + constexpr const char *CLEAR_LINE = "\033[2K"; + + // DECSTBM scrolling-region control. The header occupies HEADER_ROWS fixed + // rows at the top of the terminal; broadcast messages scroll within the + // region starting at row HEADER_ROWS+1 and extending to the bottom. Using + // 999 as the bottom bound lets the terminal clamp to its actual height. + // + // NOTE: the "6" in SCROLL_REGION_SET and CURSOR_LOG_START is HEADER_ROWS+1. + // If HEADER_ROWS changes, update these literals to match. + // + constexpr int HEADER_ROWS = 5; + constexpr const char *SCROLL_REGION_SET = "\033[6;999r"; // top = HEADER_ROWS+1 + constexpr const char *SCROLL_REGION_RESET = "\033[r"; // full-screen scrolling + constexpr const char *CURSOR_LOG_START = "\033[6;1H"; // row HEADER_ROWS+1, col 1 + constexpr const char *CURSOR_LOG_END = "\033[999;1H"; // bottom of scroll region + + std::atomic running{true}; + + // Cached last-known state so a redraw after a broadcast message can + // reprint the fixed top lines. + // + std::string last_seqstate = "(unknown)"; + std::string last_waitstate = "(none)"; + std::string last_daemonstate = "(none)"; + std::string last_acq_mode = "?"; // "stopped" | "guiding" | "acquiring" + bool last_is_acquired = false; // Key::Acamd::IS_ACQUIRED + bool last_fa_locked = false; // Key::Slicecamd::FINEACQUIRE_LOCKED + bool last_fa_running = false; // Key::Slicecamd::FINEACQUIRE_RUNNING + double last_seeing = 0.0; // Key::Acamd::SEEING + + /***** color_for_state ******************************************************/ + /** + * @brief map a lifecycle state label to an ANSI color + * @param state state string (e.g. "NOTREADY", "RUNNING") + * @return ANSI escape sequence + */ + const char *color_for_state( const std::string &state ) { + if ( state.find("FAILED") != std::string::npos ) return ANSI_RED; + if ( state.find("ABORTING") != std::string::npos ) return ANSI_BOLDYEL; + if ( state.find("RUNNING") != std::string::npos ) return ANSI_GREEN; + if ( state.find("READY") != std::string::npos ) return ANSI_YELLOW; + if ( state.find("STARTING") != std::string::npos ) return ANSI_BLUE; + if ( state.find("STOPPING") != std::string::npos ) return ANSI_BLUE; + if ( state.find("PAUSED") != std::string::npos ) return ANSI_CYAN; + if ( state.find("NOTREADY") != std::string::npos ) return ANSI_GRAY; + return ANSI_RESET; + } + /***** color_for_state ******************************************************/ + + + /***** color_for_severity ***************************************************/ + /** + * @brief map a broadcast severity label to an ANSI color + * @param severity severity string (NOTICE|WARNING|ERROR) + * @return ANSI escape sequence + */ + const char *color_for_severity( const std::string &severity ) { + if ( severity == Severity::ERROR ) return ANSI_RED; + if ( severity == Severity::WARNING ) return ANSI_YELLOW; + return ANSI_RESET; + } + /***** color_for_severity ***************************************************/ + + + /***** color_for_daemonstate ************************************************/ + /** + * @brief map a broadcast severity label to an ANSI color + * @param severity severity string (NOTICE|WARNING|ERROR) + * @return ANSI escape sequence + */ + const char *color_for_daemonstate( bool state) { + return ( state ? ANSI_GREEN : ANSI_RED ); + } + /***** color_for_daemonstate ************************************************/ + + + /***** color_for_acqmode ****************************************************/ + /** + * @brief map an acquisition mode string to an ANSI color + * @param mode "acquiring" | "guiding" | "stopped" + * @return ANSI escape sequence + */ + const char *color_for_acqmode( const std::string &mode ) { + if ( mode == "acquiring" ) return ANSI_GREEN; + if ( mode == "guiding" ) return ANSI_CYAN; + return ANSI_GRAY; // "stopped" or unknown + } + /***** color_for_acqmode ****************************************************/ + + + /***** timestamp ************************************************************/ + /** + * @brief local time as HH:MM:SS + * @return formatted string + */ + std::string timestamp() { + std::time_t now = std::time(nullptr); + std::tm tm_local{}; + localtime_r( &now, &tm_local ); + char buf[16]; + std::strftime( buf, sizeof(buf), "%H:%M:%S", &tm_local ); + return std::string(buf); + } + /***** timestamp ************************************************************/ + + + /***** redraw_status ********************************************************/ + /** + * @brief reprint the fixed status header (lines 1 and 2) in place + * @details Uses ANSI save/restore so the current scroll position for + * broadcast messages is not disturbed. + */ + void redraw_status() { + std::cout << CURSOR_SAVE + << CURSOR_HOME + << CLEAR_LINE + << "STATE: " << color_for_state(last_seqstate) + << last_seqstate << ANSI_RESET << "\n" + << CLEAR_LINE + << "WAITING: " << last_waitstate << ANSI_RESET << "\n" + << CLEAR_LINE + << "SUBSYSTEMS: " << last_daemonstate << "\n" + << CLEAR_LINE + << "ACQUISITION: " + << "ACAM: " << color_for_acqmode(last_acq_mode) + << last_acq_mode << ANSI_RESET + << " " << ( last_is_acquired ? ANSI_GREEN : ANSI_GRAY ) + << "[acquired]" << ANSI_RESET + << " SLICECAM: " + << ( last_fa_locked ? ANSI_GREEN : ANSI_GRAY ) << "locked" << ANSI_RESET + << " " + << ( last_fa_running ? ANSI_GREEN : ANSI_GRAY ) << "running" << ANSI_RESET + << " SEEING: " << std::fixed << std::setprecision(2) + << ANSI_BLUE << last_seeing << ANSI_RESET + << "\n" + << std::string(60, '-') << "\n" + << CURSOR_RESTORE + << std::flush; + } + /***** redraw_status ********************************************************/ + + + /***** handle_seqstate ******************************************************/ + /** + * @brief parse a seq_seqstate payload and update the top status line + * @param payload JSON payload as string + */ + void handle_seqstate( const std::string &payload ) { + try { + auto j = nlohmann::json::parse( payload ); + if ( j.contains(Key::Sequencer::SEQSTATE) ) { + last_seqstate = j[Key::Sequencer::SEQSTATE].get(); + if ( last_seqstate.empty() ) last_seqstate = "(none)"; + redraw_status(); + } + } + catch ( const std::exception &e ) { + std::cerr << "seqstate parse error: " << e.what() << "\n"; + } + } + /***** handle_seqstate ******************************************************/ + + + /***** handle_waitstate *****************************************************/ + /** + * @brief parse a seq_waitstate payload and update the wait-bit line + * @param payload JSON payload as string + * @details Payload contains one boolean per wait-bit label; collect the + * labels whose value is true. + */ + void handle_waitstate( const std::string &payload ) { + try { + auto j = nlohmann::json::parse( payload ); + std::ostringstream oss; + bool first = true; + for ( auto it = j.begin(); it != j.end(); ++it ) { + if ( ! it.value().is_boolean() ) continue; + if ( it.value().get() ) { + if ( !first ) oss << " "; + oss << it.key(); + first = false; + } + } + last_waitstate = oss.str(); + if ( last_waitstate.empty() ) last_waitstate = "(idle)"; + redraw_status(); + } + catch ( const std::exception &e ) { + std::cerr << "waitstate parse error: " << e.what() << "\n"; + } + } + /***** handle_waitstate *****************************************************/ + + + void handle_daemonstate( const std::string &payload ) { + try { + auto j = nlohmann::json::parse(payload); + std::ostringstream oss; + for (auto it = j.begin(); it != j.end(); ++it) { + if (!it.value().is_boolean()) continue; + bool daemonstate = it.value().get(); + oss << color_for_daemonstate(daemonstate) + << it.key() << ANSI_RESET << " "; + } + last_daemonstate = oss.str(); + redraw_status(); + } + catch (const std::exception &e) { + std::cerr << "daemonstate parse error: " << e.what() << "\n"; + } + } + + + /***** handle_acamd *********************************************************/ + /** + * @brief parse an acamd payload and update acquisition mode/status + * @param payload JSON payload as string + * @details Handles partial payloads; only keys present are updated. + */ + void handle_acamd( const std::string &payload ) { + try { + auto j = nlohmann::json::parse( payload ); + bool changed = false; + if ( j.contains( Key::Acamd::ACQUIRE_MODE ) ) { + last_acq_mode = j[Key::Acamd::ACQUIRE_MODE].get(); + changed = true; + } + if ( j.contains( Key::Acamd::IS_ACQUIRED ) ) { + last_is_acquired = j[Key::Acamd::IS_ACQUIRED].get(); + changed = true; + } + if ( j.contains( Key::Acamd::SEEING ) && !j[Key::Acamd::SEEING].is_null() ) { + last_seeing = j[Key::Acamd::SEEING].get(); + changed = true; + } + if ( changed ) redraw_status(); + } + catch ( const std::exception &e ) { + std::cerr << "acamd parse error: " << e.what() << "\n"; + } + } + /***** handle_acamd *********************************************************/ + + + /***** handle_slicecamd *****************************************************/ + /** + * @brief parse a slicecamd payload and update fine-acquisition status + * @param payload JSON payload as string + * @details Handles partial payloads; only keys present are updated. + */ + void handle_slicecamd( const std::string &payload ) { + try { + auto j = nlohmann::json::parse( payload ); + bool changed = false; + if ( j.contains( Key::Slicecamd::FINEACQUIRE_LOCKED ) ) { + last_fa_locked = j[Key::Slicecamd::FINEACQUIRE_LOCKED].get(); + changed = true; + } + if ( j.contains( Key::Slicecamd::FINEACQUIRE_RUNNING ) ) { + last_fa_running = j[Key::Slicecamd::FINEACQUIRE_RUNNING].get(); + changed = true; + } + if ( changed ) redraw_status(); + } + catch ( const std::exception &e ) { + std::cerr << "slicecamd parse error: " << e.what() << "\n"; + } + } + /***** handle_slicecamd *****************************************************/ + + + /***** handle_broadcast *****************************************************/ + /** + * @brief parse a broadcast payload and append a colored line to the log + * @param payload JSON payload as string + */ + void handle_broadcast( const std::string &payload ) { + try { + auto j = nlohmann::json::parse( payload ); + std::string severity = j.value( Key::Broadcast::SEVERITY, std::string("NOTICE") ); + std::string message = j.value( Key::Broadcast::MESSAGE, std::string("") ); + std::string source = j.value( Key::SOURCE, std::string("?") ); + // Park the cursor at the bottom of the scroll region before emitting the + // newline so the message lands inside the region and the header above + // it is not scrolled. + // + std::cout << CURSOR_LOG_END + << color_for_severity(severity) + << "[" << timestamp() << "] " + << "[" << source << "] " + << severity << ": " << message + << ANSI_RESET << "\n" + << std::flush; + } + catch ( const std::exception &e ) { + std::cerr << "broadcast parse error: " << e.what() << "\n"; + } + } + /***** handle_broadcast *****************************************************/ + + + /***** request_snapshot *****************************************************/ + /** + * @brief publish a Topic::SNAPSHOT request to named daemons + * @details Each daemon whose name appears as a key in the payload will + * respond by re-publishing its current telemetry snapshot, allowing + * seqmon to populate the status header immediately on startup rather + * than waiting for the next natural update from each daemon. + * @param publisher connected ZMQ publish socket + */ + void request_snapshot( zmqpp::socket &publisher ) { + nlohmann::json j; + j[Daemon::SEQUENCER] = true; + j[Daemon::ACAMD] = true; + j[Daemon::SLICECAMD] = true; + zmqpp::message zmsg; + zmsg.add( Topic::SNAPSHOT ); + zmsg.add( j.dump() ); + publisher.send( zmsg ); + } + /***** request_snapshot *****************************************************/ + + + /***** signal_handler *******************************************************/ + /** + * @brief SIGINT handler; triggers clean exit of the main loop + */ + void signal_handler( int /*signum*/ ) { + running.store(false); + } + /***** signal_handler *******************************************************/ + +} // anonymous namespace + + +void usage( const char *exe ) { + std::cout << "usage: " << exe << " [endpoint]\n" + << " endpoint ZMQ broker address (default " << BROKER_ENDPOINT << ")\n"; +} + + +int main( int argc, char *argv[] ) { + + std::string endpoint = BROKER_ENDPOINT; + + if ( argc > 1 ) { + std::string arg = argv[1]; + if ( arg == "-h" || arg == "--help" ) { + usage(argv[0]); + return 0; + } + endpoint = arg; + } + + std::signal( SIGINT, signal_handler ); + std::signal( SIGTERM, signal_handler ); + + // set up ZMQ subscriber and publisher + // + zmqpp::context context; + zmqpp::socket subscriber( context, zmqpp::socket_type::subscribe ); + zmqpp::socket publisher ( context, zmqpp::socket_type::publish ); + + try { + subscriber.connect( endpoint ); + subscriber.subscribe( Topic::BROADCAST ); + subscriber.subscribe( Topic::SEQ_SEQSTATE ); + subscriber.subscribe( Topic::SEQ_WAITSTATE ); + subscriber.subscribe( Topic::SEQ_DAEMONSTATE ); + subscriber.subscribe( Topic::ACAMD ); + subscriber.subscribe( Topic::SLICECAMD ); + + publisher.connect( BROKER_PUB_ENDPOINT ); + } + catch ( const std::exception &e ) { + std::cerr << "ERROR connecting to " << endpoint << ": " << e.what() << "\n"; + return 1; + } + + // initial screen: clear, draw status header, leave room for scrolling log + // + std::cout << CLEAR_SCREEN << CURSOR_HOME + << "STATE: " << last_seqstate << "\n" + << "WAITING: " << last_waitstate << "\n" + << "SUBSYSTEMS: " << last_daemonstate << "\n" + << "ACQUISITION: ACAM: " << last_acq_mode + << " SLICECAM: locked running" + << " SEEING: " << std::fixed << std::setprecision(2) << last_seeing << "\n" + << std::string(60, '-') << "\n" + << SCROLL_REGION_SET + << CURSOR_LOG_START + << "seqmon subscribed to " << endpoint + << " (Ctrl-C to exit)\n" + << std::flush; + + // ZMQ PUB sockets have a slow-joiner problem: messages sent immediately + // after connect are dropped before the broker registers the connection. + // A brief sleep ensures the publisher is ready before we send the snapshot + // request. + // + std::this_thread::sleep_for( std::chrono::milliseconds(200) ); + request_snapshot( publisher ); + + // poll so Ctrl-C can break out promptly + // + zmqpp::poller poller; + poller.add( subscriber, zmqpp::poller::poll_in ); + + while ( running.load() ) { + if ( poller.poll(500) == 0 ) continue; + if ( ! poller.has_input(subscriber) ) continue; + + zmqpp::message zmsg; + subscriber.receive( zmsg ); + if ( zmsg.parts() < 2 ) continue; // guard against malformed/partial messages + std::string topic, payload; + zmsg >> topic >> payload; + + if ( topic == Topic::SEQ_SEQSTATE ) handle_seqstate( payload ); + else if ( topic == Topic::SEQ_WAITSTATE ) handle_waitstate( payload ); + else if ( topic == Topic::SEQ_DAEMONSTATE ) handle_daemonstate( payload ); + else if ( topic == Topic::ACAMD ) handle_acamd( payload ); + else if ( topic == Topic::SLICECAMD ) handle_slicecamd( payload ); + else if ( topic == Topic::BROADCAST ) handle_broadcast( payload ); + // unrecognized topics silently ignored + } + + std::cout << SCROLL_REGION_RESET << "\nseqmon exiting\n" << std::flush; + return 0; +}