base/services/nfsd/recovery.c

   1 /* NFSv4.1 client for Windows
   2  * Copyright © 2012 The Regents of the University of Michigan
   3  *
   4  * Olga Kornievskaia <aglo@umich.edu>
   5  * Casey Bodley <cbodley@umich.edu>
   6  *
   7  * This library is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU Lesser General Public License as published by
   9  * the Free Software Foundation; either version 2.1 of the License, or (at
  10  * your option) any later version.
  11  *
  12  * This library is distributed in the hope that it will be useful, but
  13  * without any warranty; without even the implied warranty of merchantability
  14  * or fitness for a particular purpose.  See the GNU Lesser General Public
  15  * License for more details.
  16  *
  17  * You should have received a copy of the GNU Lesser General Public License
  18  * along with this library; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  20  */
  21
  22 #include <time.h>
  23
  24 #include "recovery.h"
  25 #include "delegation.h"
  26 #include "nfs41_callback.h"
  27 #include "nfs41_compound.h"
  28 #include "nfs41_ops.h"
  29 #include "daemon_debug.h"
  30
  31
  32 /* session/client recovery uses a lock and condition variable in nfs41_client
  33  * to prevent multiple threads from attempting to recover at the same time */
  34 bool_t nfs41_recovery_start_or_wait(
  35     IN nfs41_client *client)
  36 {
  37     bool_t status = TRUE;
  38
  39     EnterCriticalSection(&client->recovery.lock);
  40
  41     if (!client->recovery.in_recovery) {
  42         dprintf(1, "Entering recovery mode for client %llu\n", client->clnt_id);
  43         client->recovery.in_recovery = TRUE;
  44     } else {
  45         status = FALSE;
  46         dprintf(1, "Waiting for recovery of client %llu\n", client->clnt_id);
  47         while (client->recovery.in_recovery)
  48             SleepConditionVariableCS(&client->recovery.cond,
  49                 &client->recovery.lock, INFINITE);
  50         dprintf(1, "Woke up after recovery of client %llu\n", client->clnt_id);
  51     }
  52
  53     LeaveCriticalSection(&client->recovery.lock);
  54     return status;
  55 }
  56
  57 void nfs41_recovery_finish(
  58     IN nfs41_client *client)
  59 {
  60     EnterCriticalSection(&client->recovery.lock);
  61     dprintf(1, "Finished recovery for client %llu\n", client->clnt_id);
  62     client->recovery.in_recovery = FALSE;
  63     WakeAllConditionVariable(&client->recovery.cond);
  64     LeaveCriticalSection(&client->recovery.lock);
  65 }
  66
  67
  68 /* session/client/state recovery */
  69 int nfs41_recover_session(
  70     IN nfs41_session *session,
  71     IN bool_t client_state_lost)
  72 {
  73     enum nfsstat4 status = NFS4_OK;
  74
  75 restart_recovery:
  76     /* recover the session */
  77     status = nfs41_session_renew(session);
  78
  79     if (status == NFS4ERR_STALE_CLIENTID) {
  80         /* recover the client */
  81         client_state_lost = TRUE;
  82         status = nfs41_client_renew(session->client);
  83         if (status == NFS4_OK)
  84             goto restart_recovery; /* resume session recovery */
  85
  86         eprintf("nfs41_client_renew() failed with %d\n", status);
  87     } else if (status) {
  88         eprintf("nfs41_session_renew() failed with %d\n", status);
  89     } else if (client_state_lost) {
  90         /* recover the client's state */
  91         status = nfs41_recover_client_state(session, session->client);
  92         if (status == NFS4ERR_BADSESSION)
  93             goto restart_recovery;
  94     }
  95     return status;
  96 }
  97
  98 void nfs41_recover_sequence_flags(
  99     IN nfs41_session *session,
 100     IN uint32_t flags)
 101 {
 102     const uint32_t revoked = flags &
 103         (SEQ4_STATUS_EXPIRED_ALL_STATE_REVOKED
 104         | SEQ4_STATUS_EXPIRED_SOME_STATE_REVOKED
 105         | SEQ4_STATUS_ADMIN_STATE_REVOKED
 106         | SEQ4_STATUS_RECALLABLE_STATE_REVOKED);
 107     const uint32_t restarted = flags &
 108         SEQ4_STATUS_RESTART_RECLAIM_NEEDED;
 109
 110     /* no state recovery needed */
 111     if (revoked == 0 && restarted == 0)
 112         return;
 113
 114     if (!nfs41_recovery_start_or_wait(session->client))
 115         return;
 116
 117     if (revoked) {
 118         /* free stateids and attempt to recover them */
 119         nfs41_client_state_revoked(session, session->client, revoked);
 120
 121         /* if RESTART_RECLAIM_NEEDED is also set, just do RECLAIM_COMPLETE */
 122         if (restarted) nfs41_reclaim_complete(session);
 123
 124     } else if (restarted) {
 125         /* do server reboot state recovery */
 126         uint32_t status = nfs41_recover_client_state(session, session->client);
 127         if (status == NFS4ERR_BADSESSION) {
 128             /* recover the session and finish state recovery */
 129             nfs41_recover_session(session, TRUE);
 130         }
 131     }
 132
 133     nfs41_recovery_finish(session->client);
 134 }
 135
 136
 137 /* client state recovery for server reboot or lease expiration */
 138 static int recover_open_grace(
 139     IN nfs41_session *session,
 140     IN nfs41_path_fh *parent,
 141     IN nfs41_path_fh *file,
 142     IN state_owner4 *owner,
 143     IN uint32_t access,
 144     IN uint32_t deny,
 145     OUT stateid4 *stateid,
 146     OUT open_delegation4 *delegation)
 147 {
 148     /* reclaim the open stateid with CLAIM_PREVIOUS */
 149     open_claim4 claim;
 150     claim.claim = CLAIM_PREVIOUS;
 151     claim.u.prev.delegate_type = delegation->type;
 152
 153     return nfs41_open(session, parent, file, owner, &claim, access, deny,
 154         OPEN4_NOCREATE, 0, NULL, FALSE, stateid, delegation, NULL);
 155 }
 156
 157 static int recover_open_no_grace(
 158     IN nfs41_session *session,
 159     IN nfs41_path_fh *parent,
 160     IN nfs41_path_fh *file,
 161     IN state_owner4 *owner,
 162     IN uint32_t access,
 163     IN uint32_t deny,
 164     OUT stateid4 *stateid,
 165     OUT open_delegation4 *delegation)
 166 {
 167     open_claim4 claim;
 168     int status;
 169
 170     if (delegation->type != OPEN_DELEGATE_NONE) {
 171         /* attempt out-of-grace recovery with CLAIM_DELEGATE_PREV */
 172         claim.claim = CLAIM_DELEGATE_PREV;
 173         claim.u.deleg_prev.filename = &file->name;
 174
 175         status = nfs41_open(session, parent, file, owner,
 176             &claim, access, deny, OPEN4_NOCREATE, 0, NULL, FALSE,
 177             stateid, delegation, NULL);
 178         if (status == NFS4_OK || status == NFS4ERR_BADSESSION)
 179             goto out;
 180
 181         /* server support for CLAIM_DELEGATE_PREV is optional;
 182          * fall back to CLAIM_NULL on errors */
 183     }
 184
 185     /* attempt out-of-grace recovery with CLAIM_NULL */
 186     claim.claim = CLAIM_NULL;
 187     claim.u.null.filename = &file->name;
 188
 189     /* ask nicely for the delegation we had */
 190     if (delegation->type == OPEN_DELEGATE_READ)
 191         access |= OPEN4_SHARE_ACCESS_WANT_READ_DELEG;
 192     else if (delegation->type == OPEN_DELEGATE_WRITE)
 193         access |= OPEN4_SHARE_ACCESS_WANT_WRITE_DELEG;
 194
 195     status = nfs41_open(session, parent, file, owner,
 196         &claim, access, deny, OPEN4_NOCREATE, 0, NULL, FALSE,
 197         stateid, delegation, NULL);
 198 out:
 199     return status;
 200 }
 201
 202 static int recover_open(
 203     IN nfs41_session *session,
 204     IN nfs41_open_state *open,
 205     IN OUT bool_t *grace)
 206 {
 207     open_delegation4 delegation = { 0 };
 208     stateid4 stateid = { 0 };
 209     int status = NFS4ERR_BADHANDLE;
 210
 211     /* check for an associated delegation */
 212     AcquireSRWLockExclusive(&open->lock);
 213     if (open->delegation.state) {
 214         nfs41_delegation_state *deleg = open->delegation.state;
 215         if (deleg->revoked) {
 216             /* reclaim the delegation along with the open */
 217             AcquireSRWLockShared(&deleg->lock);
 218             delegation.type = deleg->state.type;
 219             ReleaseSRWLockShared(&deleg->lock);
 220         } else if (deleg->state.recalled) {
 221             /* we'll need an open stateid regardless */
 222         } else if (list_empty(&open->locks.list)) {
 223             /* if there are locks, we need an open stateid to
 224              * reclaim them; otherwise, the open can be delegated */
 225             open->do_close = FALSE;
 226             status = NFS4_OK;
 227         }
 228     }
 229     ReleaseSRWLockExclusive(&open->lock);
 230
 231     if (status == NFS4_OK) /* use existing delegation */
 232         goto out;
 233
 234     if (*grace) {
 235         status = recover_open_grace(session, &open->parent, &open->file,
 236             &open->owner, open->share_access, open->share_deny,
 237             &stateid, &delegation);
 238         if (status == NFS4ERR_NO_GRACE) {
 239             *grace = FALSE;
 240             /* send RECLAIM_COMPLETE before any out-of-grace recovery */
 241             nfs41_reclaim_complete(session);
 242         }
 243     }
 244     if (!*grace) {
 245         status = recover_open_no_grace(session, &open->parent, &open->file,
 246             &open->owner, open->share_access, open->share_deny,
 247             &stateid, &delegation);
 248     }
 249
 250     if (status)
 251         goto out;
 252
 253     AcquireSRWLockExclusive(&open->lock);
 254     /* update the open stateid */
 255     memcpy(&open->stateid, &stateid, sizeof(stateid4));
 256     open->do_close = TRUE;
 257
 258     if (open->delegation.state) {
 259         nfs41_delegation_state *deleg = open->delegation.state;
 260         if (deleg->revoked) {
 261             /* update delegation state */
 262             AcquireSRWLockExclusive(&deleg->lock);
 263             if (delegation.type != OPEN_DELEGATE_READ &&
 264                 delegation.type != OPEN_DELEGATE_WRITE) {
 265                 eprintf("recover_open() got delegation type %u, "
 266                     "expected %u\n", delegation.type, deleg->state.type);
 267             } else {
 268                 memcpy(&deleg->state, &delegation, sizeof(open_delegation4));
 269                 deleg->revoked = FALSE;
 270             }
 271             ReleaseSRWLockExclusive(&deleg->lock);
 272         }
 273     } else /* granted a new delegation? */
 274         nfs41_delegation_granted(session, &open->parent, &open->file,
 275             &delegation, FALSE, &open->delegation.state);
 276     ReleaseSRWLockExclusive(&open->lock);
 277 out:
 278     return status;
 279 }
 280
 281 static int recover_locks(
 282     IN nfs41_session *session,
 283     IN nfs41_open_state *open,
 284     IN OUT bool_t *grace)
 285 {
 286     stateid_arg stateid;
 287     struct list_entry *entry;
 288     nfs41_lock_state *lock;
 289     int status = NFS4_OK;
 290
 291     AcquireSRWLockExclusive(&open->lock);
 292
 293     /* initialize the open stateid for the first lock request */
 294     memcpy(&stateid.stateid, &open->stateid, sizeof(stateid4));
 295     stateid.type = STATEID_OPEN;
 296     stateid.open = open;
 297     stateid.delegation = NULL;
 298
 299     /* recover any locks for this open */
 300     list_for_each(entry, &open->locks.list) {
 301         lock = list_container(entry, nfs41_lock_state, open_entry);
 302         if (lock->delegated)
 303             continue;
 304
 305         if (*grace) {
 306             status = nfs41_lock(session, &open->file, &open->owner,
 307                 lock->exclusive ? WRITE_LT : READ_LT, lock->offset,
 308                 lock->length, TRUE, FALSE, &stateid);
 309             if (status == NFS4ERR_NO_GRACE) {
 310                 *grace = FALSE;
 311                 /* send RECLAIM_COMPLETE before any out-of-grace recovery */
 312                 nfs41_reclaim_complete(session);
 313             }
 314         }
 315         if (!*grace) {
 316             /* attempt out-of-grace recovery with a normal LOCK */
 317             status = nfs41_lock(session, &open->file, &open->owner,
 318                 lock->exclusive ? WRITE_LT : READ_LT, lock->offset,
 319                 lock->length, FALSE, FALSE, &stateid);
 320         }
 321         if (status == NFS4ERR_BADSESSION)
 322             break;
 323     }
 324
 325     if (status != NFS4ERR_BADSESSION) {
 326         /* if we got a lock stateid back, save the lock with the open */
 327         if (stateid.type == STATEID_LOCK)
 328             memcpy(&open->locks.stateid, &stateid.stateid, sizeof(stateid4));
 329         else
 330             open->locks.stateid.seqid = 0;
 331     }
 332
 333     ReleaseSRWLockExclusive(&open->lock);
 334     return status;
 335 }
 336
 337 /* delegation recovery via WANT_DELEGATION */
 338 static int recover_delegation_want(
 339     IN nfs41_session *session,
 340     IN nfs41_delegation_state *deleg,
 341     IN OUT bool_t *grace)
 342 {
 343     deleg_claim4 claim;
 344     open_delegation4 delegation = { 0 };
 345     uint32_t want_flags = 0;
 346     int status = NFS4_OK;
 347
 348     AcquireSRWLockShared(&deleg->lock);
 349     delegation.type = deleg->state.type;
 350     ReleaseSRWLockShared(&deleg->lock);
 351
 352     if (delegation.type == OPEN_DELEGATE_READ)
 353         want_flags |= OPEN4_SHARE_ACCESS_WANT_READ_DELEG;
 354     else
 355         want_flags |= OPEN4_SHARE_ACCESS_WANT_WRITE_DELEG;
 356
 357     if (*grace) {
 358         /* recover the delegation with WANT_DELEGATION/CLAIM_PREVIOUS */
 359         claim.claim = CLAIM_PREVIOUS;
 360         claim.prev_delegate_type = delegation.type;
 361
 362         status = nfs41_want_delegation(session, &deleg->file, &claim,
 363             want_flags, FALSE, &delegation);
 364         if (status == NFS4ERR_NO_GRACE) {
 365             *grace = FALSE;
 366             /* send RECLAIM_COMPLETE before any out-of-grace recovery */
 367             nfs41_reclaim_complete(session);
 368         }
 369     }
 370     if (!*grace) {
 371         /* attempt out-of-grace recovery with with CLAIM_DELEG_PREV_FH */
 372         claim.claim = CLAIM_DELEG_PREV_FH;
 373
 374         status = nfs41_want_delegation(session, &deleg->file, &claim,
 375             want_flags, FALSE, &delegation);
 376     }
 377     if (status)
 378         goto out;
 379
 380     /* update delegation state */
 381     AcquireSRWLockExclusive(&deleg->lock);
 382     if (delegation.type != OPEN_DELEGATE_READ &&
 383             delegation.type != OPEN_DELEGATE_WRITE) {
 384         eprintf("recover_delegation_want() got delegation type %u, "
 385             "expected %u\n", delegation.type, deleg->state.type);
 386     } else {
 387         memcpy(&deleg->state, &delegation, sizeof(open_delegation4));
 388         deleg->revoked = FALSE;
 389     }
 390     ReleaseSRWLockExclusive(&deleg->lock);
 391 out:
 392     return status;
 393 }
 394
 395 /* delegation recovery via OPEN (requires corresponding CLOSE) */
 396 static int recover_delegation_open(
 397     IN nfs41_session *session,
 398     IN nfs41_delegation_state *deleg,
 399     IN OUT bool_t *grace)
 400 {
 401     state_owner4 owner;
 402     open_delegation4 delegation = { 0 };
 403     stateid_arg stateid;
 404     uint32_t access = OPEN4_SHARE_ACCESS_READ;
 405     uint32_t deny = OPEN4_SHARE_DENY_NONE;
 406     int status = NFS4_OK;
 407
 408     /* choose the desired access mode based on delegation type */
 409     AcquireSRWLockShared(&deleg->lock);
 410     delegation.type = deleg->state.type;
 411     if (delegation.type == OPEN_DELEGATE_WRITE)
 412         access |= OPEN4_SHARE_ACCESS_WRITE | OPEN4_SHARE_ACCESS_WANT_WRITE_DELEG;
 413     else
 414         access |= OPEN4_SHARE_ACCESS_WANT_READ_DELEG;
 415     ReleaseSRWLockShared(&deleg->lock);
 416
 417     /* construct a temporary open owner by concatenating the time
 418      * in seconds with the delegation pointer */
 419     time((time_t*)owner.owner);
 420     memcpy(owner.owner + sizeof(time_t), deleg, sizeof(deleg));
 421     owner.owner_len = sizeof(time_t) + sizeof(deleg);
 422
 423     if (*grace) {
 424         status = recover_open_grace(session, &deleg->parent, &deleg->file,
 425             &owner, access, deny, &stateid.stateid, &delegation);
 426         if (status == NFS4ERR_NO_GRACE) {
 427             *grace = FALSE;
 428             /* send RECLAIM_COMPLETE before any out-of-grace recovery */
 429             nfs41_reclaim_complete(session);
 430         }
 431     }
 432     if (!*grace) {
 433         status = recover_open_no_grace(session, &deleg->parent, &deleg->file,
 434             &owner, access, deny, &stateid.stateid, &delegation);
 435     }
 436     if (status)
 437         goto out;
 438
 439     /* update delegation state */
 440     AcquireSRWLockExclusive(&deleg->lock);
 441     if (delegation.type != OPEN_DELEGATE_READ &&
 442             delegation.type != OPEN_DELEGATE_WRITE) {
 443         eprintf("recover_delegation_open() got delegation type %u, "
 444             "expected %u\n", delegation.type, deleg->state.type);
 445     } else {
 446         memcpy(&deleg->state, &delegation, sizeof(open_delegation4));
 447         deleg->revoked = FALSE;
 448     }
 449     ReleaseSRWLockExclusive(&deleg->lock);
 450
 451     /* send CLOSE to free the open stateid */
 452     stateid.open = NULL;
 453     stateid.delegation = NULL;
 454     stateid.type = STATEID_OPEN;
 455     nfs41_close(session, &deleg->file, &stateid);
 456 out:
 457     return status;
 458 }
 459
 460 static int recover_delegation(
 461     IN nfs41_session *session,
 462     IN nfs41_delegation_state *deleg,
 463     IN OUT bool_t *grace,
 464     IN OUT bool_t *want_supported)
 465 {
 466     int status;
 467
 468     /* 10.2.1. Delegation Recovery
 469      * When a client needs to reclaim a delegation and there is no
 470      * associated open, the client may use the CLAIM_PREVIOUS variant
 471      * of the WANT_DELEGATION operation.  However, since the server is
 472      * not required to support this operation, an alternative is to
 473      * reclaim via a dummy OPEN together with the delegation using an
 474      * OPEN of type CLAIM_PREVIOUS. */
 475     if (*want_supported)
 476         status = recover_delegation_want(session, deleg, grace);
 477     else
 478         status = NFS4ERR_NOTSUPP;
 479
 480     if (status == NFS4ERR_NOTSUPP) {
 481         *want_supported = FALSE;
 482         status = recover_delegation_open(session, deleg, grace);
 483     }
 484     return status;
 485 }
 486
 487 int nfs41_recover_client_state(
 488     IN nfs41_session *session,
 489     IN nfs41_client *client)
 490 {
 491     const struct cb_layoutrecall_args recall = { PNFS_LAYOUTTYPE_FILE,
 492         PNFS_IOMODE_ANY, TRUE, { PNFS_RETURN_ALL } };
 493     struct client_state *state = &session->client->state;
 494     struct list_entry *entry;
 495     nfs41_open_state *open;
 496     nfs41_delegation_state *deleg;
 497     bool_t grace = TRUE;
 498     bool_t want_supported = TRUE;
 499     int status = NFS4_OK;
 500
 501     EnterCriticalSection(&state->lock);
 502
 503     /* flag all delegations as revoked until successful recovery;
 504      * recover_open() and recover_delegation_open() will only ask
 505      * for delegations when revoked = TRUE */
 506     list_for_each(entry, &state->delegations) {
 507         deleg = list_container(entry, nfs41_delegation_state, client_entry);
 508         deleg->revoked = TRUE;
 509     }
 510
 511     /* recover each of the client's opens and associated delegations */
 512     list_for_each(entry, &state->opens) {
 513         open = list_container(entry, nfs41_open_state, client_entry);
 514         status = recover_open(session, open, &grace);
 515         if (status == NFS4_OK)
 516             status = recover_locks(session, open, &grace);
 517         if (status == NFS4ERR_BADSESSION)
 518             goto unlock;
 519     }
 520
 521     /* recover delegations that weren't associated with any opens */
 522     list_for_each(entry, &state->delegations) {
 523         deleg = list_container(entry, nfs41_delegation_state, client_entry);
 524         if (deleg->revoked) {
 525             status = recover_delegation(session,
 526                 deleg, &grace, &want_supported);
 527             if (status == NFS4ERR_BADSESSION)
 528                 goto unlock;
 529         }
 530     }
 531
 532     /* return any delegations that were reclaimed as 'recalled' */
 533     status = nfs41_client_delegation_recovery(client);
 534 unlock:
 535     LeaveCriticalSection(&state->lock);
 536
 537     /* revoke all of the client's layouts */
 538     pnfs_file_layout_recall(client, &recall);
 539
 540     if (grace && status != NFS4ERR_BADSESSION) {
 541         /* send reclaim_complete, but don't fail on errors */
 542         nfs41_reclaim_complete(session);
 543     }
 544     return status;
 545 }
 546
 547 static uint32_t stateid_array(
 548     IN struct list_entry *delegations,
 549     IN struct list_entry *opens,
 550     OUT stateid_arg **stateids_out,
 551     OUT uint32_t **statuses_out)
 552 {
 553     struct list_entry *entry;
 554     nfs41_open_state *open;
 555     nfs41_delegation_state *deleg;
 556     stateid_arg *stateids = NULL;
 557     uint32_t *statuses = NULL;
 558     uint32_t i = 0, count = 0;
 559
 560     /* count how many stateids the client needs to test */
 561     list_for_each(entry, delegations)
 562         count++;
 563     list_for_each(entry, opens)
 564         count += 3; /* open and potentially lock and layout */
 565
 566     if (count == 0)
 567         goto out;
 568
 569     /* allocate the stateid and status arrays */
 570     stateids = calloc(count, sizeof(stateid_arg));
 571     if (stateids == NULL)
 572         goto out_err;
 573     statuses = calloc(count, sizeof(uint32_t));
 574     if (statuses == NULL)
 575         goto out_err;
 576     memset(statuses, NFS4ERR_BAD_STATEID, count * sizeof(uint32_t));
 577
 578     /* copy stateids into the array */
 579     list_for_each(entry, delegations) {
 580         deleg = list_container(entry, nfs41_delegation_state, client_entry);
 581         AcquireSRWLockShared(&deleg->lock);
 582         /* delegation stateid */
 583         memcpy(&stateids[i].stateid, &deleg->state.stateid, sizeof(stateid4));
 584         stateids[i].type = STATEID_DELEG_FILE;
 585         stateids[i].delegation = deleg;
 586         i++;
 587         ReleaseSRWLockShared(&deleg->lock);
 588     }
 589
 590     list_for_each(entry, opens) {
 591         open = list_container(entry, nfs41_open_state, client_entry);
 592
 593         AcquireSRWLockShared(&open->lock);
 594         /* open stateid */
 595         memcpy(&stateids[i].stateid, &open->stateid, sizeof(stateid4));
 596         stateids[i].type = STATEID_OPEN;
 597         stateids[i].open = open;
 598         i++;
 599
 600         if (open->locks.stateid.seqid) { /* lock stateid? */
 601             memcpy(&stateids[i].stateid, &open->locks.stateid, sizeof(stateid4));
 602             stateids[i].type = STATEID_LOCK;
 603             stateids[i].open = open;
 604             i++;
 605         }
 606
 607         if (open->layout) { /* layout stateid? */
 608             AcquireSRWLockShared(&open->layout->lock);
 609             if (open->layout->stateid.seqid) {
 610                 memcpy(&stateids[i].stateid, &open->layout->stateid, sizeof(stateid4));
 611                 stateids[i].type = STATEID_LAYOUT;
 612                 stateids[i].open = open;
 613                 i++;
 614             }
 615             ReleaseSRWLockShared(&open->layout->lock);
 616         }
 617         ReleaseSRWLockShared(&open->lock);
 618     }
 619
 620     count = i;
 621     *stateids_out = stateids;
 622     *statuses_out = statuses;
 623 out:
 624     return count;
 625
 626 out_err:
 627     free(stateids);
 628     free(statuses);
 629     count = 0;
 630     goto out;
 631 }
 632
 633 void nfs41_client_state_revoked(
 634     IN nfs41_session *session,
 635     IN nfs41_client *client,
 636     IN uint32_t revoked)
 637 {
 638     const struct cb_layoutrecall_args recall = { PNFS_LAYOUTTYPE_FILE,
 639         PNFS_IOMODE_ANY, TRUE, { PNFS_RETURN_ALL } };
 640     struct list_entry empty, *opens;
 641     struct client_state *clientstate = &session->client->state;
 642     stateid_arg *stateids = NULL;
 643     uint32_t *statuses = NULL;
 644     uint32_t i, count;
 645     bool_t grace = TRUE;
 646     bool_t want_supported = TRUE;
 647
 648     EnterCriticalSection(&clientstate->lock);
 649
 650     if (revoked == SEQ4_STATUS_RECALLABLE_STATE_REVOKED) {
 651         /* only delegations were revoked. use an empty list for opens */
 652         list_init(&empty);
 653         opens = &empty;
 654     } else {
 655         opens = &clientstate->opens;
 656     }
 657
 658     /* get an array of the client's stateids */
 659     count = stateid_array(&clientstate->delegations,
 660         opens, &stateids, &statuses);
 661     if (count == 0)
 662         goto out;
 663
 664     /* determine which stateids were revoked with TEST_STATEID */
 665     if ((revoked & SEQ4_STATUS_EXPIRED_ALL_STATE_REVOKED) == 0)
 666         nfs41_test_stateid(session, stateids, count, statuses);
 667
 668     /* free all revoked stateids with FREE_STATEID */
 669     for (i = 0; i < count; i++)
 670         if (statuses[i])
 671             nfs41_free_stateid(session, &stateids[i].stateid);
 672
 673     /* revoke all of the client's layouts */
 674     pnfs_file_layout_recall(client, &recall);
 675
 676     /* recover the revoked stateids */
 677     for (i = 0; i < count; i++) {
 678         if (statuses[i]) {
 679             if (stateids[i].type == STATEID_DELEG_FILE)
 680                 stateids[i].delegation->revoked = TRUE;
 681             else if (stateids[i].type == STATEID_OPEN)
 682                 recover_open(session, stateids[i].open, &grace);
 683             else if (stateids[i].type == STATEID_LOCK)
 684                 recover_locks(session, stateids[i].open, &grace);
 685         }
 686     }
 687     for (i = 0; i < count; i++) {
 688         /* delegations that weren't recovered by recover_open() */
 689         if (statuses[i] && stateids[i].type == STATEID_DELEG_FILE
 690             && stateids[i].delegation->revoked)
 691             recover_delegation(session, stateids[i].delegation,
 692                 &grace, &want_supported);
 693     }
 694
 695     nfs41_client_delegation_recovery(client);
 696 out:
 697     LeaveCriticalSection(&clientstate->lock);
 698     free(stateids);
 699     free(statuses);
 700 }
 701
 702
 703 static bool_t recover_stateid_open(
 704     IN nfs_argop4 *argop,
 705     IN stateid_arg *stateid)
 706 {
 707     bool_t retry = FALSE;
 708
 709     if (stateid->open) {
 710         stateid4 *source = &stateid->open->stateid;
 711
 712         /* if the source stateid is different, update and retry */
 713         AcquireSRWLockShared(&stateid->open->lock);
 714         if (memcmp(&stateid->stateid, source, sizeof(stateid4))) {
 715             memcpy(&stateid->stateid, source, sizeof(stateid4));
 716             retry = TRUE;
 717         }
 718         ReleaseSRWLockShared(&stateid->open->lock);
 719     }
 720     return retry;
 721 }
 722
 723 static bool_t recover_stateid_lock(
 724     IN nfs_argop4 *argop,
 725     IN stateid_arg *stateid)
 726 {
 727     bool_t retry = FALSE;
 728
 729     if (stateid->open) {
 730         stateid4 *source = &stateid->open->locks.stateid;
 731
 732         /* if the source stateid is different, update and retry */
 733         AcquireSRWLockShared(&stateid->open->lock);
 734         if (memcmp(&stateid->stateid, source, sizeof(stateid4))) {
 735             if (argop->op == OP_LOCK && source->seqid == 0) {
 736                 /* resend LOCK with an open stateid */
 737                 nfs41_lock_args *lock = (nfs41_lock_args*)argop->arg;
 738                 lock->locker.new_lock_owner = 1;
 739                 lock->locker.u.open_owner.open_stateid = stateid;
 740                 lock->locker.u.open_owner.lock_owner = &stateid->open->owner;
 741                 source = &stateid->open->stateid;
 742             }
 743
 744             memcpy(&stateid->stateid, source, sizeof(stateid4));
 745             retry = TRUE;
 746         }
 747         ReleaseSRWLockShared(&stateid->open->lock);
 748     }
 749     return retry;
 750 }
 751
 752 static bool_t recover_stateid_delegation(
 753     IN nfs_argop4 *argop,
 754     IN stateid_arg *stateid)
 755 {
 756     bool_t retry = FALSE;
 757
 758     if (stateid->open) {
 759         /* if the source stateid is different, update and retry */
 760         AcquireSRWLockShared(&stateid->open->lock);
 761         if (argop->op == OP_OPEN && stateid->open->do_close) {
 762             /* for nfs41_delegation_to_open(); if we've already reclaimed
 763              * an open stateid, just fail this OPEN with BAD_STATEID */
 764         } else if (stateid->open->delegation.state) {
 765             nfs41_delegation_state *deleg = stateid->open->delegation.state;
 766             stateid4 *source = &deleg->state.stateid;
 767             AcquireSRWLockShared(&deleg->lock);
 768             if (memcmp(&stateid->stateid, source, sizeof(stateid4))) {
 769                 memcpy(&stateid->stateid, source, sizeof(stateid4));
 770                 retry = TRUE;
 771             }
 772             ReleaseSRWLockShared(&deleg->lock);
 773         }
 774         ReleaseSRWLockShared(&stateid->open->lock);
 775     } else if (stateid->delegation) {
 776         nfs41_delegation_state *deleg = stateid->delegation;
 777         stateid4 *source = &deleg->state.stateid;
 778         AcquireSRWLockShared(&deleg->lock);
 779         if (memcmp(&stateid->stateid, source, sizeof(stateid4))) {
 780             memcpy(&stateid->stateid, source, sizeof(stateid4));
 781             retry = TRUE;
 782         }
 783         ReleaseSRWLockShared(&deleg->lock);
 784     }
 785     return retry;
 786 }
 787
 788 bool_t nfs41_recover_stateid(
 789     IN nfs41_session *session,
 790     IN nfs_argop4 *argop)
 791 {
 792     stateid_arg *stateid = NULL;
 793
 794     /* get the stateid_arg from the operation's arguments */
 795     if (argop->op == OP_OPEN) {
 796         nfs41_op_open_args *open = (nfs41_op_open_args*)argop->arg;
 797         if (open->claim->claim == CLAIM_DELEGATE_CUR)
 798             stateid = open->claim->u.deleg_cur.delegate_stateid;
 799         else if (open->claim->claim == CLAIM_DELEG_CUR_FH)
 800             stateid = open->claim->u.deleg_cur_fh.delegate_stateid;
 801     } else if (argop->op == OP_CLOSE) {
 802         nfs41_op_close_args *close = (nfs41_op_close_args*)argop->arg;
 803         stateid = close->stateid;
 804     } else if (argop->op == OP_READ) {
 805         nfs41_read_args *read = (nfs41_read_args*)argop->arg;
 806         stateid = read->stateid;
 807     } else if (argop->op == OP_WRITE) {
 808         nfs41_write_args *write = (nfs41_write_args*)argop->arg;
 809         stateid = write->stateid;
 810     } else if (argop->op == OP_LOCK) {
 811         nfs41_lock_args *lock = (nfs41_lock_args*)argop->arg;
 812         if (lock->locker.new_lock_owner)
 813             stateid = lock->locker.u.open_owner.open_stateid;
 814         else
 815             stateid = lock->locker.u.lock_owner.lock_stateid;
 816     } else if (argop->op == OP_LOCKU) {
 817         nfs41_locku_args *locku = (nfs41_locku_args*)argop->arg;
 818         stateid = locku->lock_stateid;
 819     } else if (argop->op == OP_SETATTR) {
 820         nfs41_setattr_args *setattr = (nfs41_setattr_args*)argop->arg;
 821         stateid = setattr->stateid;
 822     } else if (argop->op == OP_LAYOUTGET) {
 823         pnfs_layoutget_args *lget = (pnfs_layoutget_args*)argop->arg;
 824         stateid = lget->stateid;
 825     } else if (argop->op == OP_DELEGRETURN) {
 826         nfs41_delegreturn_args *dr = (nfs41_delegreturn_args*)argop->arg;
 827         stateid = dr->stateid;
 828     }
 829     if (stateid == NULL)
 830         return FALSE;
 831
 832     /* if there's recovery in progress, wait for it to finish */
 833     EnterCriticalSection(&session->client->recovery.lock);
 834     while (session->client->recovery.in_recovery)
 835         SleepConditionVariableCS(&session->client->recovery.cond,
 836             &session->client->recovery.lock, INFINITE);
 837     LeaveCriticalSection(&session->client->recovery.lock);
 838
 839     switch (stateid->type) {
 840     case STATEID_OPEN:
 841         return recover_stateid_open(argop, stateid);
 842
 843     case STATEID_LOCK:
 844         return recover_stateid_lock(argop, stateid);
 845
 846     case STATEID_DELEG_FILE:
 847         return recover_stateid_delegation(argop, stateid);
 848
 849     default:
 850         eprintf("%s can't recover stateid type %u\n",
 851             nfs_opnum_to_string(argop->op), stateid->type);
 852         break;
 853     }
 854     return FALSE;
 855 }