i40e/base: catch NVM write semaphore timeout and retry

In some circumstances, a multi-write transaction takes longer than
the default 3 minutes timeout on the write semaphore. If the write
failed with an EBUSY status, this is likely the problem. So here
it tries to reacquire the semaphore and then retry the write.

Test report: http://www.dpdk.org/ml/archives/dev/2015-May/017384.html

Signed-off-by: Helin Zhang <helin.zhang@intel.com>
Acked-by: Jijiang Liu <jijiang.liu@intel.com>
Tested-by: Min Cao <min.cao@intel.com>

diff --git a/lib/librte_pmd_i40e/i40e/i40e_nvm.c b/lib/librte_pmd_i40e/i40e/i40e_nvm.c
index c63f5ba..e82f2a4 100644 (file)
--- a/lib/librte_pmd_i40e/i40e/i40e_nvm.c
+++ b/lib/librte_pmd_i40e/i40e/i40e_nvm.c
@@ -895,11 +895,13 @@ STATIC enum i40e_status_code i40e_nvmupd_state_writing(struct i40e_hw *hw,
 {
        enum i40e_status_code status;
        enum i40e_nvmupd_cmd upd_cmd;
+       bool retry_attempt = false;
 
        DEBUGFUNC("i40e_nvmupd_state_writing");
 
        upd_cmd = i40e_nvmupd_validate_command(hw, cmd, perrno);
 
+retry:
        switch (upd_cmd) {
        case I40E_NVMUPD_WRITE_CON:
                status = i40e_nvmupd_nvm_write(hw, cmd, bytes, perrno);
@@ -938,6 +940,39 @@ STATIC enum i40e_status_code i40e_nvmupd_state_writing(struct i40e_hw *hw,
                *perrno = -ESRCH;
                break;
        }
+
+       /* In some circumstances, a multi-write transaction takes longer
+        * than the default 3 minute timeout on the write semaphore.  If
+        * the write failed with an EBUSY status, this is likely the problem,
+        * so here we try to reacquire the semaphore then retry the write.
+        * We only do one retry, then give up.
+        */
+       if (status && (hw->aq.asq_last_status == I40E_AQ_RC_EBUSY) &&
+           !retry_attempt) {
+               enum i40e_status_code old_status = status;
+               u32 old_asq_status = hw->aq.asq_last_status;
+               u32 gtime;
+
+               gtime = rd32(hw, I40E_GLVFGEN_TIMER);
+               if (gtime >= hw->nvm.hw_semaphore_timeout) {
+                       i40e_debug(hw, I40E_DEBUG_ALL,
+                                  "NVMUPD: write semaphore expired (%d >= %lld), retrying\n",
+                                  gtime, hw->nvm.hw_semaphore_timeout);
+                       i40e_release_nvm(hw);
+                       status = i40e_acquire_nvm(hw, I40E_RESOURCE_WRITE);
+                       if (status) {
+                               i40e_debug(hw, I40E_DEBUG_ALL,
+                                          "NVMUPD: write semaphore reacquire failed aq_err = %d\n",
+                                          hw->aq.asq_last_status);
+                               status = old_status;
+                               hw->aq.asq_last_status = old_asq_status;
+                       } else {
+                               retry_attempt = true;
+                               goto retry;
+                       }
+               }
+       }
+
        return status;
 }