slicecamd: anchor the fine-acquire settle to the executed TCS move, not a timer

Config/slicecamd.cfg.in

@@ -98,6 +98,13 @@ FINE_ACQUIRE_SETTLE_FRAMES=2
 #
 FINE_ACQUIRE_SETTLE_SEC=3.0
 
+# FINE_ACQUIRE_MOVE_TIMEOUT=<sec>
+# Max time <sec> to wait for acamd to report a requested goal change applied
+# before the settle begins. On timeout the settle proceeds anyway. Set to 0
+# to disable the wait.
+#
+FINE_ACQUIRE_MOVE_TIMEOUT=60.0
+
 # FINE_ACQUIRE_GAIN=<g>
 # Proportional gain <g> = {0..1} applied to the commanded offset when the residual
 # is at or below FINE_ACQUIRE_GAIN_THRESHOLD arcsec.

acamd/acam_interface.cpp

@@ -1443,6 +1443,7 @@ namespace Acam {
     const int         attempts     = this->target.attempts;
     const std::string filter       = this->motion.get_current_filtername();
     const std::string cover        = this->motion.get_current_coverpos();
+    const bool        goalshift_pending = this->target.goalshift_pending.load();
 
     // unless forced, only publish if there was a change in any one of these
     //
@@ -1452,14 +1453,16 @@ namespace Acam {
          nacquired    == this->last_status.nacquired    &&
          attempts     == this->last_status.attempts     &&
          filter       == this->last_status.filter       &&
-         cover        == this->last_status.cover ) return;
+         cover        == this->last_status.cover        &&
+         goalshift_pending == this->last_status.goalshift_pending ) return;
 
     this->last_status.acquire_mode = acquire_mode;
     this->last_status.is_acquired  = is_acquired;
     this->last_status.nacquired    = nacquired;
     this->last_status.attempts     = attempts;
     this->last_status.filter       = filter;
     this->last_status.cover        = cover;
+    this->last_status.goalshift_pending = goalshift_pending;
 
     // assemble the telemetry into a json message
     //
@@ -1473,6 +1476,7 @@ namespace Acam {
     jmessage_out[Key::Acamd::BACKGROUND]   = this->astrometry.get_background();
     jmessage_out[Key::Acamd::FILTER]       = filter;
     jmessage_out[Key::Acamd::COVER]        = cover;
+    jmessage_out[Key::Acamd::GOALSHIFT_PENDING] = goalshift_pending;
     jmessage_out[Key::PUBTIME] = get_time_us();
 
     try {
@@ -3609,10 +3613,13 @@ logwrite( function, message.str() );
         offset = angular_separation( acam_goal.ra, acam_goal.dec, acam_ra, acam_dec );
       }
 *****/
+      const bool goal_change_at_compute = this->goalshift_pending.load(std::memory_order_acquire);
+      const double goal_ra_used  = this->acam_goal.ra;
+      const double goal_dec_used = this->acam_goal.dec;
       if ( iface->fpoffsets.solve_offset( acam_ra, acam_dec,
-                                          this->acam_goal.ra, this->acam_goal.dec,
+                                          goal_ra_used, goal_dec_used,
                                           ra_off, dec_off ) == ERROR ) break;
-      offset = angular_separation( this->acam_goal.ra, this->acam_goal.dec, acam_ra, acam_dec );
+      offset = angular_separation( goal_ra_used, goal_dec_used, acam_ra, acam_dec );
 
         message.str(""); message << "[DEBUG] acam_ra=" << acam_ra << " acam_dec=" << acam_dec << " acam_goal.ra=" 
                                  << acam_goal.ra << "  .dec=" << this->acam_goal.dec << " .ang=" << this->acam_goal.angle;
@@ -3692,6 +3699,11 @@ logwrite( function, message.str() );
           // send offset to TCS here (returns when offset is complete)
           if ( iface->tcsd.pt_offset( ra_off*3600., dec_off*3600., OFFSETRATE )==ERROR) break;
           this->allow_large_offset.store(false);  // deliberate-offset allowance consumed
+          if ( goal_change_at_compute &&
+               goal_ra_used  == this->acam_goal.ra &&
+               goal_dec_used == this->acam_goal.dec ) {
+            this->goalshift_pending.store( false, std::memory_order_release );
+          }
           std::this_thread::sleep_for( std::chrono::seconds(1) );
         }
 
@@ -5530,6 +5542,8 @@ logwrite( function, message.str() );
     //
     this->fpoffsets.apply_offset( this->target.acam_goal.ra,  this->target.dRA,
                                   this->target.acam_goal.dec, this->target.dDEC );
+    this->target.goalshift_pending.store( true, std::memory_order_release );
+    this->publish_status(true);
 
 
     message.str(""); message << this->target.dRA << " " << this->target.dDEC;

acamd/acam_interface.h

@@ -349,6 +349,7 @@ namespace Acam {
 
       std::atomic<bool> is_acquired;       ///< set if target acquired successfully
       std::atomic<bool> stop_acquisition;  ///< set if the acquisition sequence should stop
+      std::atomic<bool> goalshift_pending{false};  ///< a requested goal change awaits execution by the guide loop
 
       double tcs_max_offset;
       double tcs_max_putonslit_offset{300.};  ///< max offset (arcsec) for a deliberate goal offset (put-on-slit etc.) applied while guiding; defaults 300 if ACQUIRE_TCS_MAX_PUTONSLIT_OFFSET absent
@@ -530,6 +531,7 @@ namespace Acam {
         int         attempts     = 0;
         std::string filter       = "";
         std::string cover        = "";
+        bool        goalshift_pending = false;
       } last_status;
 
     public:

common/message_keys.h

@@ -106,6 +106,7 @@ namespace Key {
     inline const std::string ATTEMPTS     = "attempts";
     inline const std::string SEEING       = "seeing";
     inline const std::string BACKGROUND   = "background";
+    inline const std::string GOALSHIFT_PENDING = "goalshift_pending";  ///< a requested goal change awaits execution by the guide loop
   }
 
   namespace Slicecamd {

slicecamd/slicecam_interface.cpp

@@ -60,6 +60,10 @@ namespace Slicecam {
       const bool was_running = this->is_fineacquire_running.load(std::memory_order_acquire);
       this->is_fineacquire_locked.store(false, std::memory_order_release);
       this->is_fineacquire_running.store(false, std::memory_order_release);
+      {
+      std::lock_guard<std::mutex> lock(this->acam_mtx);
+      this->acam_cv.notify_all();
+      }
       this->publish_status();
       logwrite(function, was_running ? "stop requested" : "stopped");
       retstring=this->is_fineacquire_running.load(std::memory_order_acquire)?"running":"stopped";
@@ -562,13 +566,46 @@ namespace Slicecam {
     const double cmd_dra  = effective_gain * med_dra;
     const double cmd_ddec = effective_gain * med_ddec;
 
+    const int64_t send_time = get_time_us();
+    const bool was_guiding = this->is_acam_guiding.load();
+
     if ( this->offset_acam_goal( { cmd_dra, cmd_ddec }, true ) != NO_ERROR ) {
       logwrite( function, "ERROR failed to send offset to ACAM" );
       this->is_fineacquire_running.store( false, std::memory_order_release );
       this->publish_status();
       return;
     }
 
+    // while guiding the correction only re-points acamd's goal; wait until
+    // acamd reports the goal change applied before settling
+    if ( was_guiding && this->fineacquire_state.move_timeout_sec > 0.0 ) {
+      std::unique_lock<std::mutex> lock(this->acam_mtx);
+      // completion requires seeing THIS request registered (pending true on a
+      // status after the send) and then cleared; statuses arrive in publish
+      // order, so the false cannot predate the request
+      bool seen_pending = false;
+      const bool applied = this->acam_cv.wait_for( lock,
+          std::chrono::duration<double>( this->fineacquire_state.move_timeout_sec ),
+          [this, send_time, &seen_pending]() {
+            if ( !this->is_fineacquire_running.load(std::memory_order_acquire) ||
+                 !this->should_framegrab_run.load(std::memory_order_acquire) ) return true;
+            if ( this->last_acam_pubtime.load(std::memory_order_acquire) <= send_time ) return false;
+            if ( this->acam_goalshift_pending.load(std::memory_order_acquire) ) {
+              seen_pending = true;
+              return false;
+            }
+            return seen_pending;
+          });
+      if ( !this->should_framegrab_run.load(std::memory_order_acquire) ) return;
+      if ( !this->is_fineacquire_running.load(std::memory_order_acquire) ) return;
+      if ( !applied ) {
+        std::ostringstream w;
+        w << "WARNING goal change not applied within "
+          << this->fineacquire_state.move_timeout_sec << " s; proceeding";
+        logwrite( function, w.str() );
+      }
+    }
+
     // time-based settle: wait for the TCS to physically finish the move before
     // sampling resumes. The frame-count settle (settle_frames) is too short in
     // wall-clock when autoexpose shortens the exposure (e.g. bright targets), so
@@ -924,6 +961,10 @@ namespace Slicecam {
     Common::extract_telemetry_value( jmessage, Key::PUBTIME, pubtime );
     this->last_acam_pubtime.store( pubtime, std::memory_order_relaxed );
 
+    bool goalshift_pending=false;
+    Common::extract_telemetry_value( jmessage, Key::Acamd::GOALSHIFT_PENDING, goalshift_pending );
+    this->acam_goalshift_pending.store( goalshift_pending, std::memory_order_relaxed );
+
     // wake any thread waiting on ACAM state (e.g. fineacquire)
     std::lock_guard<std::mutex> lock(this->acam_mtx);
     this->acam_cv.notify_all();
@@ -1418,6 +1459,19 @@ namespace Slicecam {
         applied++;
       }
       else
+      if ( config.param[entry] == "FINE_ACQUIRE_MOVE_TIMEOUT" ) {
+        try { this->fineacquire_state.move_timeout_sec = std::stod( config.arg[entry] ); }
+        catch ( const std::exception &e ) {
+          message.str(""); message << "ERROR invalid FINE_ACQUIRE_MOVE_TIMEOUT "
+                                   << config.arg[entry] << ": " << e.what();
+          logwrite( function, message.str() );
+          return ERROR;
+        }
+        message.str(""); message << "SLICECAMD:config:" << config.param[entry] << "=" << config.arg[entry];
+        logwrite( function, message.str() );
+        applied++;
+      }
+      else
       if ( config.param[entry] == "FINE_ACQUIRE_GAIN" ) {
         try { this->fineacquire_state.gain = std::stod( config.arg[entry] ); }
         catch ( const std::exception &e ) {
@@ -2101,6 +2155,10 @@ namespace Slicecam {
     //
     if ( whattodo == "stop" ) {
       this->should_framegrab_run.store( false, std::memory_order_release );  // tells framegrab loop to stop
+      {
+      std::lock_guard<std::mutex> lock(this->acam_mtx);
+      this->acam_cv.notify_all();
+      }
       if ( this->is_framegrab_running.load(std::memory_order_acquire) ) {    // wait for it to stop
         int wait_ms = std::max( static_cast<int>(3000*(this->camera.andor.begin()->second->camera_info.exptime+1)), 5000 );
         // alert user that framegrabbing has stopped
@@ -2180,6 +2238,10 @@ namespace Slicecam {
       // frame will be grabbed.
       //
       this->should_framegrab_run.store( false, std::memory_order_release );
+      {
+      std::lock_guard<std::mutex> lock(this->acam_mtx);
+      this->acam_cv.notify_all();
+      }
       if ( this->is_framegrab_running.load(std::memory_order_acquire) ) return;
     }
     else
@@ -2596,6 +2658,10 @@ namespace Slicecam {
     //
     bool is_guiding = this->is_acam_guiding.load();
 
+    // arcsec for the log; the guiding path sends degrees
+    const double ra_off_arcsec  = ra_off  * 3600.;
+    const double dec_off_arcsec = dec_off * 3600.;
+
     // send the offsets now
     //
     if ( is_guiding ) {
@@ -2632,7 +2698,7 @@ namespace Slicecam {
     }
 
     std::ostringstream message;
-    message << "requested offsets dRA=" << ra_off << " dDEC=" << dec_off << " arcsec";
+    message << "requested offsets dRA=" << ra_off_arcsec << " dDEC=" << dec_off_arcsec << " arcsec";
     logwrite(function, message.str());
 
     return NO_ERROR;

slicecamd/slicecam_interface.h

@@ -95,6 +95,7 @@ namespace Slicecam {
     int    settle_frames = 0;       ///< countdown of frames to discard while telescope settles
     int    settle_count  = 2;       ///< configured: frames to discard after each move
     double settle_sec    = 3.0;     ///< time-based settle (sec) after each move; 0 disables (needed when autoexpose shortens frames so the frame-count settle is too brief in wall-clock)
+    double move_timeout_sec = 60.0; ///< max wait (sec) for acamd to report the goal change applied; 0 disables the wait
     int    consecutive_centroid_failures = 0; ///< counts consecutive centroid failures
     // exposure compensation (shared by the reactive trim and, later, autoexpose)
     double exptime_min     = 0.1;   ///< clamp: minimum auto-adjusted exposure (sec)
@@ -193,6 +194,7 @@ namespace Slicecam {
       std::atomic<bool> is_acam_guiding;         ///< is acam guiding?
 
       std::atomic<int64_t> last_acam_pubtime{0};   ///< pubtime (us) of latest received acamd status
+      std::atomic<bool> acam_goalshift_pending{false};  ///< acamd has an unexecuted goal change
 
       // Latest target (goal) coords published on Topic::TARGETINFO
       // NAN until a TARGETINFO arrives, so manual runs with no sequencer target log nan.