1/* 2 * pNFS functions to call and manage layout drivers. 3 * 4 * Copyright (c) 2002 [year of first publication] 5 * The Regents of the University of Michigan 6 * All Rights Reserved 7 * 8 * Dean Hildebrand <dhildebz@umich.edu> 9 * 10 * Permission is granted to use, copy, create derivative works, and 11 * redistribute this software and such derivative works for any purpose, 12 * so long as the name of the University of Michigan is not used in 13 * any advertising or publicity pertaining to the use or distribution 14 * of this software without specific, written prior authorization. If 15 * the above copyright notice or any other identification of the 16 * University of Michigan is included in any copy of any portion of 17 * this software, then the disclaimer below must also be included. 18 * 19 * This software is provided as is, without representation or warranty 20 * of any kind either express or implied, including without limitation 21 * the implied warranties of merchantability, fitness for a particular 22 * purpose, or noninfringement. The Regents of the University of 23 * Michigan shall not be liable for any damages, including special, 24 * indirect, incidental, or consequential damages, with respect to any 25 * claim arising out of or in connection with the use of the software, 26 * even if it has been or is hereafter advised of the possibility of 27 * such damages. 28 */ 29 30#include <linux/nfs_fs.h> 31#include <linux/nfs_page.h> 32#include <linux/module.h> 33#include "internal.h" 34#include "pnfs.h" 35#include "iostat.h" 36#include "nfs4trace.h" 37#include "delegation.h" 38#include "nfs42.h" 39 40#define NFSDBG_FACILITY NFSDBG_PNFS 41#define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ) 42 43/* Locking: 44 * 45 * pnfs_spinlock: 46 * protects pnfs_modules_tbl. 47 */ 48static DEFINE_SPINLOCK(pnfs_spinlock); 49 50/* 51 * pnfs_modules_tbl holds all pnfs modules 52 */ 53static LIST_HEAD(pnfs_modules_tbl); 54 55static int 56pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, nfs4_stateid stateid, 57 enum pnfs_iomode iomode, bool sync); 58 59/* Return the registered pnfs layout driver module matching given id */ 60static struct pnfs_layoutdriver_type * 61find_pnfs_driver_locked(u32 id) 62{ 63 struct pnfs_layoutdriver_type *local; 64 65 list_for_each_entry(local, &pnfs_modules_tbl, pnfs_tblid) 66 if (local->id == id) 67 goto out; 68 local = NULL; 69out: 70 dprintk("%s: Searching for id %u, found %p\n", __func__, id, local); 71 return local; 72} 73 74static struct pnfs_layoutdriver_type * 75find_pnfs_driver(u32 id) 76{ 77 struct pnfs_layoutdriver_type *local; 78 79 spin_lock(&pnfs_spinlock); 80 local = find_pnfs_driver_locked(id); 81 if (local != NULL && !try_module_get(local->owner)) { 82 dprintk("%s: Could not grab reference on module\n", __func__); 83 local = NULL; 84 } 85 spin_unlock(&pnfs_spinlock); 86 return local; 87} 88 89void 90unset_pnfs_layoutdriver(struct nfs_server *nfss) 91{ 92 if (nfss->pnfs_curr_ld) { 93 if (nfss->pnfs_curr_ld->clear_layoutdriver) 94 nfss->pnfs_curr_ld->clear_layoutdriver(nfss); 95 /* Decrement the MDS count. Purge the deviceid cache if zero */ 96 if (atomic_dec_and_test(&nfss->nfs_client->cl_mds_count)) 97 nfs4_deviceid_purge_client(nfss->nfs_client); 98 module_put(nfss->pnfs_curr_ld->owner); 99 } 100 nfss->pnfs_curr_ld = NULL; 101} 102 103/* 104 * Try to set the server's pnfs module to the pnfs layout type specified by id. 105 * Currently only one pNFS layout driver per filesystem is supported. 106 * 107 * @id layout type. Zero (illegal layout type) indicates pNFS not in use. 108 */ 109void 110set_pnfs_layoutdriver(struct nfs_server *server, const struct nfs_fh *mntfh, 111 u32 id) 112{ 113 struct pnfs_layoutdriver_type *ld_type = NULL; 114 115 if (id == 0) 116 goto out_no_driver; 117 if (!(server->nfs_client->cl_exchange_flags & 118 (EXCHGID4_FLAG_USE_NON_PNFS | EXCHGID4_FLAG_USE_PNFS_MDS))) { 119 printk(KERN_ERR "NFS: %s: id %u cl_exchange_flags 0x%x\n", 120 __func__, id, server->nfs_client->cl_exchange_flags); 121 goto out_no_driver; 122 } 123 ld_type = find_pnfs_driver(id); 124 if (!ld_type) { 125 request_module("%s-%u", LAYOUT_NFSV4_1_MODULE_PREFIX, id); 126 ld_type = find_pnfs_driver(id); 127 if (!ld_type) { 128 dprintk("%s: No pNFS module found for %u.\n", 129 __func__, id); 130 goto out_no_driver; 131 } 132 } 133 server->pnfs_curr_ld = ld_type; 134 if (ld_type->set_layoutdriver 135 && ld_type->set_layoutdriver(server, mntfh)) { 136 printk(KERN_ERR "NFS: %s: Error initializing pNFS layout " 137 "driver %u.\n", __func__, id); 138 module_put(ld_type->owner); 139 goto out_no_driver; 140 } 141 /* Bump the MDS count */ 142 atomic_inc(&server->nfs_client->cl_mds_count); 143 144 dprintk("%s: pNFS module for %u set\n", __func__, id); 145 return; 146 147out_no_driver: 148 dprintk("%s: Using NFSv4 I/O\n", __func__); 149 server->pnfs_curr_ld = NULL; 150} 151 152int 153pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *ld_type) 154{ 155 int status = -EINVAL; 156 struct pnfs_layoutdriver_type *tmp; 157 158 if (ld_type->id == 0) { 159 printk(KERN_ERR "NFS: %s id 0 is reserved\n", __func__); 160 return status; 161 } 162 if (!ld_type->alloc_lseg || !ld_type->free_lseg) { 163 printk(KERN_ERR "NFS: %s Layout driver must provide " 164 "alloc_lseg and free_lseg.\n", __func__); 165 return status; 166 } 167 168 spin_lock(&pnfs_spinlock); 169 tmp = find_pnfs_driver_locked(ld_type->id); 170 if (!tmp) { 171 list_add(&ld_type->pnfs_tblid, &pnfs_modules_tbl); 172 status = 0; 173 dprintk("%s Registering id:%u name:%s\n", __func__, ld_type->id, 174 ld_type->name); 175 } else { 176 printk(KERN_ERR "NFS: %s Module with id %d already loaded!\n", 177 __func__, ld_type->id); 178 } 179 spin_unlock(&pnfs_spinlock); 180 181 return status; 182} 183EXPORT_SYMBOL_GPL(pnfs_register_layoutdriver); 184 185void 186pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *ld_type) 187{ 188 dprintk("%s Deregistering id:%u\n", __func__, ld_type->id); 189 spin_lock(&pnfs_spinlock); 190 list_del(&ld_type->pnfs_tblid); 191 spin_unlock(&pnfs_spinlock); 192} 193EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver); 194 195/* 196 * pNFS client layout cache 197 */ 198 199/* Need to hold i_lock if caller does not already hold reference */ 200void 201pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo) 202{ 203 atomic_inc(&lo->plh_refcount); 204} 205 206static struct pnfs_layout_hdr * 207pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags) 208{ 209 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; 210 return ld->alloc_layout_hdr(ino, gfp_flags); 211} 212 213static void 214pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo) 215{ 216 struct nfs_server *server = NFS_SERVER(lo->plh_inode); 217 struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld; 218 219 if (!list_empty(&lo->plh_layouts)) { 220 struct nfs_client *clp = server->nfs_client; 221 222 spin_lock(&clp->cl_lock); 223 list_del_init(&lo->plh_layouts); 224 spin_unlock(&clp->cl_lock); 225 } 226 put_rpccred(lo->plh_lc_cred); 227 return ld->free_layout_hdr(lo); 228} 229 230static void 231pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo) 232{ 233 struct nfs_inode *nfsi = NFS_I(lo->plh_inode); 234 dprintk("%s: freeing layout cache %p\n", __func__, lo); 235 nfsi->layout = NULL; 236 /* Reset MDS Threshold I/O counters */ 237 nfsi->write_io = 0; 238 nfsi->read_io = 0; 239} 240 241void 242pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo) 243{ 244 struct inode *inode = lo->plh_inode; 245 246 if (atomic_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) { 247 if (!list_empty(&lo->plh_segs)) 248 WARN_ONCE(1, "NFS: BUG unfreed layout segments.\n"); 249 pnfs_detach_layout_hdr(lo); 250 spin_unlock(&inode->i_lock); 251 pnfs_free_layout_hdr(lo); 252 } 253} 254 255static int 256pnfs_iomode_to_fail_bit(u32 iomode) 257{ 258 return iomode == IOMODE_RW ? 259 NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED; 260} 261 262static void 263pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit) 264{ 265 lo->plh_retry_timestamp = jiffies; 266 if (!test_and_set_bit(fail_bit, &lo->plh_flags)) 267 atomic_inc(&lo->plh_refcount); 268} 269 270static void 271pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit) 272{ 273 if (test_and_clear_bit(fail_bit, &lo->plh_flags)) 274 atomic_dec(&lo->plh_refcount); 275} 276 277static void 278pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode) 279{ 280 struct inode *inode = lo->plh_inode; 281 struct pnfs_layout_range range = { 282 .iomode = iomode, 283 .offset = 0, 284 .length = NFS4_MAX_UINT64, 285 }; 286 LIST_HEAD(head); 287 288 spin_lock(&inode->i_lock); 289 pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); 290 pnfs_mark_matching_lsegs_invalid(lo, &head, &range); 291 spin_unlock(&inode->i_lock); 292 pnfs_free_lseg_list(&head); 293 dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__, 294 iomode == IOMODE_RW ? "RW" : "READ"); 295} 296 297static bool 298pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode) 299{ 300 unsigned long start, end; 301 int fail_bit = pnfs_iomode_to_fail_bit(iomode); 302 303 if (test_bit(fail_bit, &lo->plh_flags) == 0) 304 return false; 305 end = jiffies; 306 start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT; 307 if (!time_in_range(lo->plh_retry_timestamp, start, end)) { 308 /* It is time to retry the failed layoutgets */ 309 pnfs_layout_clear_fail_bit(lo, fail_bit); 310 return false; 311 } 312 return true; 313} 314 315static void 316init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg) 317{ 318 INIT_LIST_HEAD(&lseg->pls_list); 319 INIT_LIST_HEAD(&lseg->pls_lc_list); 320 atomic_set(&lseg->pls_refcount, 1); 321 smp_mb(); 322 set_bit(NFS_LSEG_VALID, &lseg->pls_flags); 323 lseg->pls_layout = lo; 324} 325 326static void pnfs_free_lseg(struct pnfs_layout_segment *lseg) 327{ 328 struct inode *ino = lseg->pls_layout->plh_inode; 329 330 NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg); 331} 332 333static void 334pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo, 335 struct pnfs_layout_segment *lseg) 336{ 337 struct inode *inode = lo->plh_inode; 338 339 WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); 340 list_del_init(&lseg->pls_list); 341 /* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */ 342 atomic_dec(&lo->plh_refcount); 343 if (list_empty(&lo->plh_segs)) 344 clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 345 rpc_wake_up(&NFS_SERVER(inode)->roc_rpcwaitq); 346} 347 348/* Return true if layoutreturn is needed */ 349static bool 350pnfs_layout_need_return(struct pnfs_layout_hdr *lo, 351 struct pnfs_layout_segment *lseg) 352{ 353 struct pnfs_layout_segment *s; 354 355 if (!test_and_clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 356 return false; 357 358 list_for_each_entry(s, &lo->plh_segs, pls_list) 359 if (s != lseg && test_bit(NFS_LSEG_LAYOUTRETURN, &s->pls_flags)) 360 return false; 361 362 return true; 363} 364 365static bool 366pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo) 367{ 368 if (test_and_set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) 369 return false; 370 lo->plh_return_iomode = 0; 371 pnfs_get_layout_hdr(lo); 372 clear_bit(NFS_LAYOUT_RETURN_BEFORE_CLOSE, &lo->plh_flags); 373 return true; 374} 375 376static void pnfs_layoutreturn_before_put_lseg(struct pnfs_layout_segment *lseg, 377 struct pnfs_layout_hdr *lo, struct inode *inode) 378{ 379 lo = lseg->pls_layout; 380 inode = lo->plh_inode; 381 382 spin_lock(&inode->i_lock); 383 if (pnfs_layout_need_return(lo, lseg)) { 384 nfs4_stateid stateid; 385 enum pnfs_iomode iomode; 386 bool send; 387 388 stateid = lo->plh_stateid; 389 iomode = lo->plh_return_iomode; 390 send = pnfs_prepare_layoutreturn(lo); 391 spin_unlock(&inode->i_lock); 392 if (send) { 393 /* Send an async layoutreturn so we dont deadlock */ 394 pnfs_send_layoutreturn(lo, stateid, iomode, false); 395 } 396 } else 397 spin_unlock(&inode->i_lock); 398} 399 400void 401pnfs_put_lseg(struct pnfs_layout_segment *lseg) 402{ 403 struct pnfs_layout_hdr *lo; 404 struct inode *inode; 405 406 if (!lseg) 407 return; 408 409 dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg, 410 atomic_read(&lseg->pls_refcount), 411 test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); 412 413 /* Handle the case where refcount != 1 */ 414 if (atomic_add_unless(&lseg->pls_refcount, -1, 1)) 415 return; 416 417 lo = lseg->pls_layout; 418 inode = lo->plh_inode; 419 /* Do we need a layoutreturn? */ 420 if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) 421 pnfs_layoutreturn_before_put_lseg(lseg, lo, inode); 422 423 if (atomic_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) { 424 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags)) { 425 spin_unlock(&inode->i_lock); 426 return; 427 } 428 pnfs_get_layout_hdr(lo); 429 pnfs_layout_remove_lseg(lo, lseg); 430 spin_unlock(&inode->i_lock); 431 pnfs_free_lseg(lseg); 432 pnfs_put_layout_hdr(lo); 433 } 434} 435EXPORT_SYMBOL_GPL(pnfs_put_lseg); 436 437static void pnfs_free_lseg_async_work(struct work_struct *work) 438{ 439 struct pnfs_layout_segment *lseg; 440 struct pnfs_layout_hdr *lo; 441 442 lseg = container_of(work, struct pnfs_layout_segment, pls_work); 443 lo = lseg->pls_layout; 444 445 pnfs_free_lseg(lseg); 446 pnfs_put_layout_hdr(lo); 447} 448 449static void pnfs_free_lseg_async(struct pnfs_layout_segment *lseg) 450{ 451 INIT_WORK(&lseg->pls_work, pnfs_free_lseg_async_work); 452 schedule_work(&lseg->pls_work); 453} 454 455void 456pnfs_put_lseg_locked(struct pnfs_layout_segment *lseg) 457{ 458 if (!lseg) 459 return; 460 461 assert_spin_locked(&lseg->pls_layout->plh_inode->i_lock); 462 463 dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg, 464 atomic_read(&lseg->pls_refcount), 465 test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); 466 if (atomic_dec_and_test(&lseg->pls_refcount)) { 467 struct pnfs_layout_hdr *lo = lseg->pls_layout; 468 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags)) 469 return; 470 pnfs_get_layout_hdr(lo); 471 pnfs_layout_remove_lseg(lo, lseg); 472 pnfs_free_lseg_async(lseg); 473 } 474} 475EXPORT_SYMBOL_GPL(pnfs_put_lseg_locked); 476 477static u64 478end_offset(u64 start, u64 len) 479{ 480 u64 end; 481 482 end = start + len; 483 return end >= start ? end : NFS4_MAX_UINT64; 484} 485 486/* 487 * is l2 fully contained in l1? 488 * start1 end1 489 * [----------------------------------) 490 * start2 end2 491 * [----------------) 492 */ 493static bool 494pnfs_lseg_range_contained(const struct pnfs_layout_range *l1, 495 const struct pnfs_layout_range *l2) 496{ 497 u64 start1 = l1->offset; 498 u64 end1 = end_offset(start1, l1->length); 499 u64 start2 = l2->offset; 500 u64 end2 = end_offset(start2, l2->length); 501 502 return (start1 <= start2) && (end1 >= end2); 503} 504 505/* 506 * is l1 and l2 intersecting? 507 * start1 end1 508 * [----------------------------------) 509 * start2 end2 510 * [----------------) 511 */ 512static bool 513pnfs_lseg_range_intersecting(const struct pnfs_layout_range *l1, 514 const struct pnfs_layout_range *l2) 515{ 516 u64 start1 = l1->offset; 517 u64 end1 = end_offset(start1, l1->length); 518 u64 start2 = l2->offset; 519 u64 end2 = end_offset(start2, l2->length); 520 521 return (end1 == NFS4_MAX_UINT64 || end1 > start2) && 522 (end2 == NFS4_MAX_UINT64 || end2 > start1); 523} 524 525static bool 526should_free_lseg(const struct pnfs_layout_range *lseg_range, 527 const struct pnfs_layout_range *recall_range) 528{ 529 return (recall_range->iomode == IOMODE_ANY || 530 lseg_range->iomode == recall_range->iomode) && 531 pnfs_lseg_range_intersecting(lseg_range, recall_range); 532} 533 534static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg, 535 struct list_head *tmp_list) 536{ 537 if (!atomic_dec_and_test(&lseg->pls_refcount)) 538 return false; 539 pnfs_layout_remove_lseg(lseg->pls_layout, lseg); 540 list_add(&lseg->pls_list, tmp_list); 541 return true; 542} 543 544/* Returns 1 if lseg is removed from list, 0 otherwise */ 545static int mark_lseg_invalid(struct pnfs_layout_segment *lseg, 546 struct list_head *tmp_list) 547{ 548 int rv = 0; 549 550 if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) { 551 /* Remove the reference keeping the lseg in the 552 * list. It will now be removed when all 553 * outstanding io is finished. 554 */ 555 dprintk("%s: lseg %p ref %d\n", __func__, lseg, 556 atomic_read(&lseg->pls_refcount)); 557 if (pnfs_lseg_dec_and_remove_zero(lseg, tmp_list)) 558 rv = 1; 559 } 560 return rv; 561} 562 563/* Returns count of number of matching invalid lsegs remaining in list 564 * after call. 565 */ 566int 567pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo, 568 struct list_head *tmp_list, 569 struct pnfs_layout_range *recall_range) 570{ 571 struct pnfs_layout_segment *lseg, *next; 572 int invalid = 0, removed = 0; 573 574 dprintk("%s:Begin lo %p\n", __func__, lo); 575 576 if (list_empty(&lo->plh_segs)) 577 return 0; 578 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 579 if (!recall_range || 580 should_free_lseg(&lseg->pls_range, recall_range)) { 581 dprintk("%s: freeing lseg %p iomode %d " 582 "offset %llu length %llu\n", __func__, 583 lseg, lseg->pls_range.iomode, lseg->pls_range.offset, 584 lseg->pls_range.length); 585 invalid++; 586 removed += mark_lseg_invalid(lseg, tmp_list); 587 } 588 dprintk("%s:Return %i\n", __func__, invalid - removed); 589 return invalid - removed; 590} 591 592/* note free_me must contain lsegs from a single layout_hdr */ 593void 594pnfs_free_lseg_list(struct list_head *free_me) 595{ 596 struct pnfs_layout_segment *lseg, *tmp; 597 598 if (list_empty(free_me)) 599 return; 600 601 list_for_each_entry_safe(lseg, tmp, free_me, pls_list) { 602 list_del(&lseg->pls_list); 603 pnfs_free_lseg(lseg); 604 } 605} 606 607void 608pnfs_destroy_layout(struct nfs_inode *nfsi) 609{ 610 struct pnfs_layout_hdr *lo; 611 LIST_HEAD(tmp_list); 612 613 spin_lock(&nfsi->vfs_inode.i_lock); 614 lo = nfsi->layout; 615 if (lo) { 616 lo->plh_block_lgets++; /* permanently block new LAYOUTGETs */ 617 pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL); 618 pnfs_get_layout_hdr(lo); 619 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED); 620 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED); 621 pnfs_clear_retry_layoutget(lo); 622 spin_unlock(&nfsi->vfs_inode.i_lock); 623 pnfs_free_lseg_list(&tmp_list); 624 pnfs_put_layout_hdr(lo); 625 } else 626 spin_unlock(&nfsi->vfs_inode.i_lock); 627} 628EXPORT_SYMBOL_GPL(pnfs_destroy_layout); 629 630static bool 631pnfs_layout_add_bulk_destroy_list(struct inode *inode, 632 struct list_head *layout_list) 633{ 634 struct pnfs_layout_hdr *lo; 635 bool ret = false; 636 637 spin_lock(&inode->i_lock); 638 lo = NFS_I(inode)->layout; 639 if (lo != NULL && list_empty(&lo->plh_bulk_destroy)) { 640 pnfs_get_layout_hdr(lo); 641 list_add(&lo->plh_bulk_destroy, layout_list); 642 ret = true; 643 } 644 spin_unlock(&inode->i_lock); 645 return ret; 646} 647 648/* Caller must hold rcu_read_lock and clp->cl_lock */ 649static int 650pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp, 651 struct nfs_server *server, 652 struct list_head *layout_list) 653{ 654 struct pnfs_layout_hdr *lo, *next; 655 struct inode *inode; 656 657 list_for_each_entry_safe(lo, next, &server->layouts, plh_layouts) { 658 inode = igrab(lo->plh_inode); 659 if (inode == NULL) 660 continue; 661 list_del_init(&lo->plh_layouts); 662 if (pnfs_layout_add_bulk_destroy_list(inode, layout_list)) 663 continue; 664 rcu_read_unlock(); 665 spin_unlock(&clp->cl_lock); 666 iput(inode); 667 spin_lock(&clp->cl_lock); 668 rcu_read_lock(); 669 return -EAGAIN; 670 } 671 return 0; 672} 673 674static int 675pnfs_layout_free_bulk_destroy_list(struct list_head *layout_list, 676 bool is_bulk_recall) 677{ 678 struct pnfs_layout_hdr *lo; 679 struct inode *inode; 680 struct pnfs_layout_range range = { 681 .iomode = IOMODE_ANY, 682 .offset = 0, 683 .length = NFS4_MAX_UINT64, 684 }; 685 LIST_HEAD(lseg_list); 686 int ret = 0; 687 688 while (!list_empty(layout_list)) { 689 lo = list_entry(layout_list->next, struct pnfs_layout_hdr, 690 plh_bulk_destroy); 691 dprintk("%s freeing layout for inode %lu\n", __func__, 692 lo->plh_inode->i_ino); 693 inode = lo->plh_inode; 694 695 pnfs_layoutcommit_inode(inode, false); 696 697 spin_lock(&inode->i_lock); 698 list_del_init(&lo->plh_bulk_destroy); 699 lo->plh_block_lgets++; /* permanently block new LAYOUTGETs */ 700 if (is_bulk_recall) 701 set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 702 if (pnfs_mark_matching_lsegs_invalid(lo, &lseg_list, &range)) 703 ret = -EAGAIN; 704 spin_unlock(&inode->i_lock); 705 pnfs_free_lseg_list(&lseg_list); 706 pnfs_put_layout_hdr(lo); 707 iput(inode); 708 } 709 return ret; 710} 711 712int 713pnfs_destroy_layouts_byfsid(struct nfs_client *clp, 714 struct nfs_fsid *fsid, 715 bool is_recall) 716{ 717 struct nfs_server *server; 718 LIST_HEAD(layout_list); 719 720 spin_lock(&clp->cl_lock); 721 rcu_read_lock(); 722restart: 723 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 724 if (memcmp(&server->fsid, fsid, sizeof(*fsid)) != 0) 725 continue; 726 if (pnfs_layout_bulk_destroy_byserver_locked(clp, 727 server, 728 &layout_list) != 0) 729 goto restart; 730 } 731 rcu_read_unlock(); 732 spin_unlock(&clp->cl_lock); 733 734 if (list_empty(&layout_list)) 735 return 0; 736 return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall); 737} 738 739int 740pnfs_destroy_layouts_byclid(struct nfs_client *clp, 741 bool is_recall) 742{ 743 struct nfs_server *server; 744 LIST_HEAD(layout_list); 745 746 spin_lock(&clp->cl_lock); 747 rcu_read_lock(); 748restart: 749 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { 750 if (pnfs_layout_bulk_destroy_byserver_locked(clp, 751 server, 752 &layout_list) != 0) 753 goto restart; 754 } 755 rcu_read_unlock(); 756 spin_unlock(&clp->cl_lock); 757 758 if (list_empty(&layout_list)) 759 return 0; 760 return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall); 761} 762 763/* 764 * Called by the state manger to remove all layouts established under an 765 * expired lease. 766 */ 767void 768pnfs_destroy_all_layouts(struct nfs_client *clp) 769{ 770 nfs4_deviceid_mark_client_invalid(clp); 771 nfs4_deviceid_purge_client(clp); 772 773 pnfs_destroy_layouts_byclid(clp, false); 774} 775 776/* 777 * Compare 2 layout stateid sequence ids, to see which is newer, 778 * taking into account wraparound issues. 779 */ 780static bool pnfs_seqid_is_newer(u32 s1, u32 s2) 781{ 782 return (s32)(s1 - s2) > 0; 783} 784 785/* update lo->plh_stateid with new if is more recent */ 786void 787pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new, 788 bool update_barrier) 789{ 790 u32 oldseq, newseq, new_barrier; 791 int empty = list_empty(&lo->plh_segs); 792 793 oldseq = be32_to_cpu(lo->plh_stateid.seqid); 794 newseq = be32_to_cpu(new->seqid); 795 if (empty || pnfs_seqid_is_newer(newseq, oldseq)) { 796 nfs4_stateid_copy(&lo->plh_stateid, new); 797 if (update_barrier) { 798 new_barrier = be32_to_cpu(new->seqid); 799 } else { 800 /* Because of wraparound, we want to keep the barrier 801 * "close" to the current seqids. 802 */ 803 new_barrier = newseq - atomic_read(&lo->plh_outstanding); 804 } 805 if (empty || pnfs_seqid_is_newer(new_barrier, lo->plh_barrier)) 806 lo->plh_barrier = new_barrier; 807 } 808} 809 810static bool 811pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo, 812 const nfs4_stateid *stateid) 813{ 814 u32 seqid = be32_to_cpu(stateid->seqid); 815 816 return !pnfs_seqid_is_newer(seqid, lo->plh_barrier); 817} 818 819/* lget is set to 1 if called from inside send_layoutget call chain */ 820static bool 821pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo) 822{ 823 return lo->plh_block_lgets || 824 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); 825} 826 827int 828pnfs_choose_layoutget_stateid(nfs4_stateid *dst, struct pnfs_layout_hdr *lo, 829 struct pnfs_layout_range *range, 830 struct nfs4_state *open_state) 831{ 832 int status = 0; 833 834 dprintk("--> %s\n", __func__); 835 spin_lock(&lo->plh_inode->i_lock); 836 if (pnfs_layoutgets_blocked(lo)) { 837 status = -EAGAIN; 838 } else if (!nfs4_valid_open_stateid(open_state)) { 839 status = -EBADF; 840 } else if (list_empty(&lo->plh_segs) || 841 test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) { 842 int seq; 843 844 do { 845 seq = read_seqbegin(&open_state->seqlock); 846 nfs4_stateid_copy(dst, &open_state->stateid); 847 } while (read_seqretry(&open_state->seqlock, seq)); 848 } else 849 nfs4_stateid_copy(dst, &lo->plh_stateid); 850 spin_unlock(&lo->plh_inode->i_lock); 851 dprintk("<-- %s\n", __func__); 852 return status; 853} 854 855/* 856* Get layout from server. 857* for now, assume that whole file layouts are requested. 858* arg->offset: 0 859* arg->length: all ones 860*/ 861static struct pnfs_layout_segment * 862send_layoutget(struct pnfs_layout_hdr *lo, 863 struct nfs_open_context *ctx, 864 struct pnfs_layout_range *range, 865 gfp_t gfp_flags) 866{ 867 struct inode *ino = lo->plh_inode; 868 struct nfs_server *server = NFS_SERVER(ino); 869 struct nfs4_layoutget *lgp; 870 struct pnfs_layout_segment *lseg; 871 loff_t i_size; 872 873 dprintk("--> %s\n", __func__); 874 875 /* 876 * Synchronously retrieve layout information from server and 877 * store in lseg. If we race with a concurrent seqid morphing 878 * op, then re-send the LAYOUTGET. 879 */ 880 do { 881 lgp = kzalloc(sizeof(*lgp), gfp_flags); 882 if (lgp == NULL) 883 return NULL; 884 885 i_size = i_size_read(ino); 886 887 lgp->args.minlength = PAGE_CACHE_SIZE; 888 if (lgp->args.minlength > range->length) 889 lgp->args.minlength = range->length; 890 if (range->iomode == IOMODE_READ) { 891 if (range->offset >= i_size) 892 lgp->args.minlength = 0; 893 else if (i_size - range->offset < lgp->args.minlength) 894 lgp->args.minlength = i_size - range->offset; 895 } 896 lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE; 897 lgp->args.range = *range; 898 lgp->args.type = server->pnfs_curr_ld->id; 899 lgp->args.inode = ino; 900 lgp->args.ctx = get_nfs_open_context(ctx); 901 lgp->gfp_flags = gfp_flags; 902 lgp->cred = lo->plh_lc_cred; 903 904 lseg = nfs4_proc_layoutget(lgp, gfp_flags); 905 } while (lseg == ERR_PTR(-EAGAIN)); 906 907 if (IS_ERR(lseg)) { 908 switch (PTR_ERR(lseg)) { 909 case -ENOMEM: 910 case -ERESTARTSYS: 911 break; 912 default: 913 /* remember that LAYOUTGET failed and suspend trying */ 914 pnfs_layout_io_set_failed(lo, range->iomode); 915 } 916 return NULL; 917 } else 918 pnfs_layout_clear_fail_bit(lo, 919 pnfs_iomode_to_fail_bit(range->iomode)); 920 921 return lseg; 922} 923 924static void pnfs_clear_layoutcommit(struct inode *inode, 925 struct list_head *head) 926{ 927 struct nfs_inode *nfsi = NFS_I(inode); 928 struct pnfs_layout_segment *lseg, *tmp; 929 930 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) 931 return; 932 list_for_each_entry_safe(lseg, tmp, &nfsi->layout->plh_segs, pls_list) { 933 if (!test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 934 continue; 935 pnfs_lseg_dec_and_remove_zero(lseg, head); 936 } 937} 938 939void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo) 940{ 941 clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags); 942 smp_mb__after_atomic(); 943 wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN); 944 rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq); 945} 946 947static int 948pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, nfs4_stateid stateid, 949 enum pnfs_iomode iomode, bool sync) 950{ 951 struct inode *ino = lo->plh_inode; 952 struct nfs4_layoutreturn *lrp; 953 int status = 0; 954 955 lrp = kzalloc(sizeof(*lrp), GFP_NOFS); 956 if (unlikely(lrp == NULL)) { 957 status = -ENOMEM; 958 spin_lock(&ino->i_lock); 959 pnfs_clear_layoutreturn_waitbit(lo); 960 spin_unlock(&ino->i_lock); 961 pnfs_put_layout_hdr(lo); 962 goto out; 963 } 964 965 lrp->args.stateid = stateid; 966 lrp->args.layout_type = NFS_SERVER(ino)->pnfs_curr_ld->id; 967 lrp->args.inode = ino; 968 lrp->args.range.iomode = iomode; 969 lrp->args.range.offset = 0; 970 lrp->args.range.length = NFS4_MAX_UINT64; 971 lrp->args.layout = lo; 972 lrp->clp = NFS_SERVER(ino)->nfs_client; 973 lrp->cred = lo->plh_lc_cred; 974 975 status = nfs4_proc_layoutreturn(lrp, sync); 976out: 977 dprintk("<-- %s status: %d\n", __func__, status); 978 return status; 979} 980 981/* 982 * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr 983 * when the layout segment list is empty. 984 * 985 * Note that a pnfs_layout_hdr can exist with an empty layout segment 986 * list when LAYOUTGET has failed, or when LAYOUTGET succeeded, but the 987 * deviceid is marked invalid. 988 */ 989int 990_pnfs_return_layout(struct inode *ino) 991{ 992 struct pnfs_layout_hdr *lo = NULL; 993 struct nfs_inode *nfsi = NFS_I(ino); 994 LIST_HEAD(tmp_list); 995 nfs4_stateid stateid; 996 int status = 0, empty; 997 bool send; 998 999 dprintk("NFS: %s for inode %lu\n", __func__, ino->i_ino); 1000 1001 spin_lock(&ino->i_lock); 1002 lo = nfsi->layout; 1003 if (!lo) { 1004 spin_unlock(&ino->i_lock); 1005 dprintk("NFS: %s no layout to return\n", __func__); 1006 goto out; 1007 } 1008 stateid = nfsi->layout->plh_stateid; 1009 /* Reference matched in nfs4_layoutreturn_release */ 1010 pnfs_get_layout_hdr(lo); 1011 empty = list_empty(&lo->plh_segs); 1012 pnfs_clear_layoutcommit(ino, &tmp_list); 1013 pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL); 1014 1015 if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) { 1016 struct pnfs_layout_range range = { 1017 .iomode = IOMODE_ANY, 1018 .offset = 0, 1019 .length = NFS4_MAX_UINT64, 1020 }; 1021 NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range); 1022 } 1023 1024 /* Don't send a LAYOUTRETURN if list was initially empty */ 1025 if (empty) { 1026 spin_unlock(&ino->i_lock); 1027 dprintk("NFS: %s no layout segments to return\n", __func__); 1028 goto out_put_layout_hdr; 1029 } 1030 1031 set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); 1032 send = pnfs_prepare_layoutreturn(lo); 1033 spin_unlock(&ino->i_lock); 1034 pnfs_free_lseg_list(&tmp_list); 1035 if (send) 1036 status = pnfs_send_layoutreturn(lo, stateid, IOMODE_ANY, true); 1037out_put_layout_hdr: 1038 pnfs_put_layout_hdr(lo); 1039out: 1040 dprintk("<-- %s status: %d\n", __func__, status); 1041 return status; 1042} 1043EXPORT_SYMBOL_GPL(_pnfs_return_layout); 1044 1045int 1046pnfs_commit_and_return_layout(struct inode *inode) 1047{ 1048 struct pnfs_layout_hdr *lo; 1049 int ret; 1050 1051 spin_lock(&inode->i_lock); 1052 lo = NFS_I(inode)->layout; 1053 if (lo == NULL) { 1054 spin_unlock(&inode->i_lock); 1055 return 0; 1056 } 1057 pnfs_get_layout_hdr(lo); 1058 /* Block new layoutgets and read/write to ds */ 1059 lo->plh_block_lgets++; 1060 spin_unlock(&inode->i_lock); 1061 filemap_fdatawait(inode->i_mapping); 1062 ret = pnfs_layoutcommit_inode(inode, true); 1063 if (ret == 0) 1064 ret = _pnfs_return_layout(inode); 1065 spin_lock(&inode->i_lock); 1066 lo->plh_block_lgets--; 1067 spin_unlock(&inode->i_lock); 1068 pnfs_put_layout_hdr(lo); 1069 return ret; 1070} 1071 1072bool pnfs_roc(struct inode *ino) 1073{ 1074 struct nfs_inode *nfsi = NFS_I(ino); 1075 struct nfs_open_context *ctx; 1076 struct nfs4_state *state; 1077 struct pnfs_layout_hdr *lo; 1078 struct pnfs_layout_segment *lseg, *tmp; 1079 nfs4_stateid stateid; 1080 LIST_HEAD(tmp_list); 1081 bool found = false, layoutreturn = false, roc = false; 1082 1083 spin_lock(&ino->i_lock); 1084 lo = nfsi->layout; 1085 if (!lo || test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) 1086 goto out_noroc; 1087 1088 /* no roc if we hold a delegation */ 1089 if (nfs4_check_delegation(ino, FMODE_READ)) 1090 goto out_noroc; 1091 1092 list_for_each_entry(ctx, &nfsi->open_files, list) { 1093 state = ctx->state; 1094 /* Don't return layout if there is open file state */ 1095 if (state != NULL && state->state != 0) 1096 goto out_noroc; 1097 } 1098 1099 stateid = lo->plh_stateid; 1100 /* always send layoutreturn if being marked so */ 1101 if (test_and_clear_bit(NFS_LAYOUT_RETURN_BEFORE_CLOSE, 1102 &lo->plh_flags)) 1103 layoutreturn = pnfs_prepare_layoutreturn(lo); 1104 1105 pnfs_clear_retry_layoutget(lo); 1106 list_for_each_entry_safe(lseg, tmp, &lo->plh_segs, pls_list) 1107 /* If we are sending layoutreturn, invalidate all valid lsegs */ 1108 if (layoutreturn || test_bit(NFS_LSEG_ROC, &lseg->pls_flags)) { 1109 mark_lseg_invalid(lseg, &tmp_list); 1110 found = true; 1111 } 1112 /* ROC in two conditions: 1113 * 1. there are ROC lsegs 1114 * 2. we don't send layoutreturn 1115 */ 1116 if (found && !layoutreturn) { 1117 /* lo ref dropped in pnfs_roc_release() */ 1118 pnfs_get_layout_hdr(lo); 1119 roc = true; 1120 } 1121 1122out_noroc: 1123 spin_unlock(&ino->i_lock); 1124 pnfs_free_lseg_list(&tmp_list); 1125 pnfs_layoutcommit_inode(ino, true); 1126 if (layoutreturn) 1127 pnfs_send_layoutreturn(lo, stateid, IOMODE_ANY, true); 1128 return roc; 1129} 1130 1131void pnfs_roc_release(struct inode *ino) 1132{ 1133 struct pnfs_layout_hdr *lo; 1134 1135 spin_lock(&ino->i_lock); 1136 lo = NFS_I(ino)->layout; 1137 pnfs_clear_layoutreturn_waitbit(lo); 1138 if (atomic_dec_and_test(&lo->plh_refcount)) { 1139 pnfs_detach_layout_hdr(lo); 1140 spin_unlock(&ino->i_lock); 1141 pnfs_free_layout_hdr(lo); 1142 } else 1143 spin_unlock(&ino->i_lock); 1144} 1145 1146void pnfs_roc_set_barrier(struct inode *ino, u32 barrier) 1147{ 1148 struct pnfs_layout_hdr *lo; 1149 1150 spin_lock(&ino->i_lock); 1151 lo = NFS_I(ino)->layout; 1152 if (pnfs_seqid_is_newer(barrier, lo->plh_barrier)) 1153 lo->plh_barrier = barrier; 1154 spin_unlock(&ino->i_lock); 1155 trace_nfs4_layoutreturn_on_close(ino, 0); 1156} 1157 1158void pnfs_roc_get_barrier(struct inode *ino, u32 *barrier) 1159{ 1160 struct nfs_inode *nfsi = NFS_I(ino); 1161 struct pnfs_layout_hdr *lo; 1162 u32 current_seqid; 1163 1164 spin_lock(&ino->i_lock); 1165 lo = nfsi->layout; 1166 current_seqid = be32_to_cpu(lo->plh_stateid.seqid); 1167 1168 /* Since close does not return a layout stateid for use as 1169 * a barrier, we choose the worst-case barrier. 1170 */ 1171 *barrier = current_seqid + atomic_read(&lo->plh_outstanding); 1172 spin_unlock(&ino->i_lock); 1173} 1174 1175bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task) 1176{ 1177 struct nfs_inode *nfsi = NFS_I(ino); 1178 struct pnfs_layout_hdr *lo; 1179 bool sleep = false; 1180 1181 /* we might not have grabbed lo reference. so need to check under 1182 * i_lock */ 1183 spin_lock(&ino->i_lock); 1184 lo = nfsi->layout; 1185 if (lo && test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) 1186 sleep = true; 1187 spin_unlock(&ino->i_lock); 1188 1189 if (sleep) 1190 rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL); 1191 1192 return sleep; 1193} 1194 1195/* 1196 * Compare two layout segments for sorting into layout cache. 1197 * We want to preferentially return RW over RO layouts, so ensure those 1198 * are seen first. 1199 */ 1200static s64 1201pnfs_lseg_range_cmp(const struct pnfs_layout_range *l1, 1202 const struct pnfs_layout_range *l2) 1203{ 1204 s64 d; 1205 1206 /* high offset > low offset */ 1207 d = l1->offset - l2->offset; 1208 if (d) 1209 return d; 1210 1211 /* short length > long length */ 1212 d = l2->length - l1->length; 1213 if (d) 1214 return d; 1215 1216 /* read > read/write */ 1217 return (int)(l1->iomode == IOMODE_READ) - (int)(l2->iomode == IOMODE_READ); 1218} 1219 1220static bool 1221pnfs_lseg_range_is_after(const struct pnfs_layout_range *l1, 1222 const struct pnfs_layout_range *l2) 1223{ 1224 return pnfs_lseg_range_cmp(l1, l2) > 0; 1225} 1226 1227static bool 1228pnfs_lseg_no_merge(struct pnfs_layout_segment *lseg, 1229 struct pnfs_layout_segment *old) 1230{ 1231 return false; 1232} 1233 1234void 1235pnfs_generic_layout_insert_lseg(struct pnfs_layout_hdr *lo, 1236 struct pnfs_layout_segment *lseg, 1237 bool (*is_after)(const struct pnfs_layout_range *, 1238 const struct pnfs_layout_range *), 1239 bool (*do_merge)(struct pnfs_layout_segment *, 1240 struct pnfs_layout_segment *), 1241 struct list_head *free_me) 1242{ 1243 struct pnfs_layout_segment *lp, *tmp; 1244 1245 dprintk("%s:Begin\n", __func__); 1246 1247 list_for_each_entry_safe(lp, tmp, &lo->plh_segs, pls_list) { 1248 if (test_bit(NFS_LSEG_VALID, &lp->pls_flags) == 0) 1249 continue; 1250 if (do_merge(lseg, lp)) { 1251 mark_lseg_invalid(lp, free_me); 1252 continue; 1253 } 1254 if (is_after(&lseg->pls_range, &lp->pls_range)) 1255 continue; 1256 list_add_tail(&lseg->pls_list, &lp->pls_list); 1257 dprintk("%s: inserted lseg %p " 1258 "iomode %d offset %llu length %llu before " 1259 "lp %p iomode %d offset %llu length %llu\n", 1260 __func__, lseg, lseg->pls_range.iomode, 1261 lseg->pls_range.offset, lseg->pls_range.length, 1262 lp, lp->pls_range.iomode, lp->pls_range.offset, 1263 lp->pls_range.length); 1264 goto out; 1265 } 1266 list_add_tail(&lseg->pls_list, &lo->plh_segs); 1267 dprintk("%s: inserted lseg %p " 1268 "iomode %d offset %llu length %llu at tail\n", 1269 __func__, lseg, lseg->pls_range.iomode, 1270 lseg->pls_range.offset, lseg->pls_range.length); 1271out: 1272 pnfs_get_layout_hdr(lo); 1273 1274 dprintk("%s:Return\n", __func__); 1275} 1276EXPORT_SYMBOL_GPL(pnfs_generic_layout_insert_lseg); 1277 1278static void 1279pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo, 1280 struct pnfs_layout_segment *lseg, 1281 struct list_head *free_me) 1282{ 1283 struct inode *inode = lo->plh_inode; 1284 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 1285 1286 if (ld->add_lseg != NULL) 1287 ld->add_lseg(lo, lseg, free_me); 1288 else 1289 pnfs_generic_layout_insert_lseg(lo, lseg, 1290 pnfs_lseg_range_is_after, 1291 pnfs_lseg_no_merge, 1292 free_me); 1293} 1294 1295static struct pnfs_layout_hdr * 1296alloc_init_layout_hdr(struct inode *ino, 1297 struct nfs_open_context *ctx, 1298 gfp_t gfp_flags) 1299{ 1300 struct pnfs_layout_hdr *lo; 1301 1302 lo = pnfs_alloc_layout_hdr(ino, gfp_flags); 1303 if (!lo) 1304 return NULL; 1305 atomic_set(&lo->plh_refcount, 1); 1306 INIT_LIST_HEAD(&lo->plh_layouts); 1307 INIT_LIST_HEAD(&lo->plh_segs); 1308 INIT_LIST_HEAD(&lo->plh_bulk_destroy); 1309 lo->plh_inode = ino; 1310 lo->plh_lc_cred = get_rpccred(ctx->cred); 1311 return lo; 1312} 1313 1314static struct pnfs_layout_hdr * 1315pnfs_find_alloc_layout(struct inode *ino, 1316 struct nfs_open_context *ctx, 1317 gfp_t gfp_flags) 1318{ 1319 struct nfs_inode *nfsi = NFS_I(ino); 1320 struct pnfs_layout_hdr *new = NULL; 1321 1322 dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout); 1323 1324 if (nfsi->layout != NULL) 1325 goto out_existing; 1326 spin_unlock(&ino->i_lock); 1327 new = alloc_init_layout_hdr(ino, ctx, gfp_flags); 1328 spin_lock(&ino->i_lock); 1329 1330 if (likely(nfsi->layout == NULL)) { /* Won the race? */ 1331 nfsi->layout = new; 1332 return new; 1333 } else if (new != NULL) 1334 pnfs_free_layout_hdr(new); 1335out_existing: 1336 pnfs_get_layout_hdr(nfsi->layout); 1337 return nfsi->layout; 1338} 1339 1340/* 1341 * iomode matching rules: 1342 * iomode lseg match 1343 * ----- ----- ----- 1344 * ANY READ true 1345 * ANY RW true 1346 * RW READ false 1347 * RW RW true 1348 * READ READ true 1349 * READ RW true 1350 */ 1351static bool 1352pnfs_lseg_range_match(const struct pnfs_layout_range *ls_range, 1353 const struct pnfs_layout_range *range) 1354{ 1355 struct pnfs_layout_range range1; 1356 1357 if ((range->iomode == IOMODE_RW && 1358 ls_range->iomode != IOMODE_RW) || 1359 !pnfs_lseg_range_intersecting(ls_range, range)) 1360 return 0; 1361 1362 /* range1 covers only the first byte in the range */ 1363 range1 = *range; 1364 range1.length = 1; 1365 return pnfs_lseg_range_contained(ls_range, &range1); 1366} 1367 1368/* 1369 * lookup range in layout 1370 */ 1371static struct pnfs_layout_segment * 1372pnfs_find_lseg(struct pnfs_layout_hdr *lo, 1373 struct pnfs_layout_range *range) 1374{ 1375 struct pnfs_layout_segment *lseg, *ret = NULL; 1376 1377 dprintk("%s:Begin\n", __func__); 1378 1379 list_for_each_entry(lseg, &lo->plh_segs, pls_list) { 1380 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) && 1381 !test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) && 1382 pnfs_lseg_range_match(&lseg->pls_range, range)) { 1383 ret = pnfs_get_lseg(lseg); 1384 break; 1385 } 1386 } 1387 1388 dprintk("%s:Return lseg %p ref %d\n", 1389 __func__, ret, ret ? atomic_read(&ret->pls_refcount) : 0); 1390 return ret; 1391} 1392 1393/* 1394 * Use mdsthreshold hints set at each OPEN to determine if I/O should go 1395 * to the MDS or over pNFS 1396 * 1397 * The nfs_inode read_io and write_io fields are cumulative counters reset 1398 * when there are no layout segments. Note that in pnfs_update_layout iomode 1399 * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a 1400 * WRITE request. 1401 * 1402 * A return of true means use MDS I/O. 1403 * 1404 * From rfc 5661: 1405 * If a file's size is smaller than the file size threshold, data accesses 1406 * SHOULD be sent to the metadata server. If an I/O request has a length that 1407 * is below the I/O size threshold, the I/O SHOULD be sent to the metadata 1408 * server. If both file size and I/O size are provided, the client SHOULD 1409 * reach or exceed both thresholds before sending its read or write 1410 * requests to the data server. 1411 */ 1412static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx, 1413 struct inode *ino, int iomode) 1414{ 1415 struct nfs4_threshold *t = ctx->mdsthreshold; 1416 struct nfs_inode *nfsi = NFS_I(ino); 1417 loff_t fsize = i_size_read(ino); 1418 bool size = false, size_set = false, io = false, io_set = false, ret = false; 1419 1420 if (t == NULL) 1421 return ret; 1422 1423 dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n", 1424 __func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz); 1425 1426 switch (iomode) { 1427 case IOMODE_READ: 1428 if (t->bm & THRESHOLD_RD) { 1429 dprintk("%s fsize %llu\n", __func__, fsize); 1430 size_set = true; 1431 if (fsize < t->rd_sz) 1432 size = true; 1433 } 1434 if (t->bm & THRESHOLD_RD_IO) { 1435 dprintk("%s nfsi->read_io %llu\n", __func__, 1436 nfsi->read_io); 1437 io_set = true; 1438 if (nfsi->read_io < t->rd_io_sz) 1439 io = true; 1440 } 1441 break; 1442 case IOMODE_RW: 1443 if (t->bm & THRESHOLD_WR) { 1444 dprintk("%s fsize %llu\n", __func__, fsize); 1445 size_set = true; 1446 if (fsize < t->wr_sz) 1447 size = true; 1448 } 1449 if (t->bm & THRESHOLD_WR_IO) { 1450 dprintk("%s nfsi->write_io %llu\n", __func__, 1451 nfsi->write_io); 1452 io_set = true; 1453 if (nfsi->write_io < t->wr_io_sz) 1454 io = true; 1455 } 1456 break; 1457 } 1458 if (size_set && io_set) { 1459 if (size && io) 1460 ret = true; 1461 } else if (size || io) 1462 ret = true; 1463 1464 dprintk("<-- %s size %d io %d ret %d\n", __func__, size, io, ret); 1465 return ret; 1466} 1467 1468/* stop waiting if someone clears NFS_LAYOUT_RETRY_LAYOUTGET bit. */ 1469static int pnfs_layoutget_retry_bit_wait(struct wait_bit_key *key, int mode) 1470{ 1471 if (!test_bit(NFS_LAYOUT_RETRY_LAYOUTGET, key->flags)) 1472 return 1; 1473 return nfs_wait_bit_killable(key, mode); 1474} 1475 1476static bool pnfs_prepare_to_retry_layoutget(struct pnfs_layout_hdr *lo) 1477{ 1478 if (!pnfs_should_retry_layoutget(lo)) 1479 return false; 1480 /* 1481 * send layoutcommit as it can hold up layoutreturn due to lseg 1482 * reference 1483 */ 1484 pnfs_layoutcommit_inode(lo->plh_inode, false); 1485 return !wait_on_bit_action(&lo->plh_flags, NFS_LAYOUT_RETURN, 1486 pnfs_layoutget_retry_bit_wait, 1487 TASK_UNINTERRUPTIBLE); 1488} 1489 1490static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo) 1491{ 1492 unsigned long *bitlock = &lo->plh_flags; 1493 1494 clear_bit_unlock(NFS_LAYOUT_FIRST_LAYOUTGET, bitlock); 1495 smp_mb__after_atomic(); 1496 wake_up_bit(bitlock, NFS_LAYOUT_FIRST_LAYOUTGET); 1497} 1498 1499/* 1500 * Layout segment is retreived from the server if not cached. 1501 * The appropriate layout segment is referenced and returned to the caller. 1502 */ 1503struct pnfs_layout_segment * 1504pnfs_update_layout(struct inode *ino, 1505 struct nfs_open_context *ctx, 1506 loff_t pos, 1507 u64 count, 1508 enum pnfs_iomode iomode, 1509 gfp_t gfp_flags) 1510{ 1511 struct pnfs_layout_range arg = { 1512 .iomode = iomode, 1513 .offset = pos, 1514 .length = count, 1515 }; 1516 unsigned pg_offset; 1517 struct nfs_server *server = NFS_SERVER(ino); 1518 struct nfs_client *clp = server->nfs_client; 1519 struct pnfs_layout_hdr *lo; 1520 struct pnfs_layout_segment *lseg = NULL; 1521 bool first; 1522 1523 if (!pnfs_enabled_sb(NFS_SERVER(ino))) 1524 goto out; 1525 1526 if (iomode == IOMODE_READ && i_size_read(ino) == 0) 1527 goto out; 1528 1529 if (pnfs_within_mdsthreshold(ctx, ino, iomode)) 1530 goto out; 1531 1532lookup_again: 1533 first = false; 1534 spin_lock(&ino->i_lock); 1535 lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags); 1536 if (lo == NULL) { 1537 spin_unlock(&ino->i_lock); 1538 goto out; 1539 } 1540 1541 /* Do we even need to bother with this? */ 1542 if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) { 1543 dprintk("%s matches recall, use MDS\n", __func__); 1544 goto out_unlock; 1545 } 1546 1547 /* if LAYOUTGET already failed once we don't try again */ 1548 if (pnfs_layout_io_test_failed(lo, iomode) && 1549 !pnfs_should_retry_layoutget(lo)) 1550 goto out_unlock; 1551 1552 first = list_empty(&lo->plh_segs); 1553 if (first) { 1554 /* The first layoutget for the file. Need to serialize per 1555 * RFC 5661 Errata 3208. 1556 */ 1557 if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET, 1558 &lo->plh_flags)) { 1559 spin_unlock(&ino->i_lock); 1560 wait_on_bit(&lo->plh_flags, NFS_LAYOUT_FIRST_LAYOUTGET, 1561 TASK_UNINTERRUPTIBLE); 1562 pnfs_put_layout_hdr(lo); 1563 goto lookup_again; 1564 } 1565 } else { 1566 /* Check to see if the layout for the given range 1567 * already exists 1568 */ 1569 lseg = pnfs_find_lseg(lo, &arg); 1570 if (lseg) 1571 goto out_unlock; 1572 } 1573 1574 /* 1575 * Because we free lsegs before sending LAYOUTRETURN, we need to wait 1576 * for LAYOUTRETURN even if first is true. 1577 */ 1578 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { 1579 spin_unlock(&ino->i_lock); 1580 dprintk("%s wait for layoutreturn\n", __func__); 1581 if (pnfs_prepare_to_retry_layoutget(lo)) { 1582 if (first) 1583 pnfs_clear_first_layoutget(lo); 1584 pnfs_put_layout_hdr(lo); 1585 dprintk("%s retrying\n", __func__); 1586 goto lookup_again; 1587 } 1588 goto out_put_layout_hdr; 1589 } 1590 1591 if (pnfs_layoutgets_blocked(lo)) 1592 goto out_unlock; 1593 atomic_inc(&lo->plh_outstanding); 1594 spin_unlock(&ino->i_lock); 1595 1596 if (list_empty(&lo->plh_layouts)) { 1597 /* The lo must be on the clp list if there is any 1598 * chance of a CB_LAYOUTRECALL(FILE) coming in. 1599 */ 1600 spin_lock(&clp->cl_lock); 1601 if (list_empty(&lo->plh_layouts)) 1602 list_add_tail(&lo->plh_layouts, &server->layouts); 1603 spin_unlock(&clp->cl_lock); 1604 } 1605 1606 pg_offset = arg.offset & ~PAGE_CACHE_MASK; 1607 if (pg_offset) { 1608 arg.offset -= pg_offset; 1609 arg.length += pg_offset; 1610 } 1611 if (arg.length != NFS4_MAX_UINT64) 1612 arg.length = PAGE_CACHE_ALIGN(arg.length); 1613 1614 lseg = send_layoutget(lo, ctx, &arg, gfp_flags); 1615 pnfs_clear_retry_layoutget(lo); 1616 atomic_dec(&lo->plh_outstanding); 1617out_put_layout_hdr: 1618 if (first) 1619 pnfs_clear_first_layoutget(lo); 1620 pnfs_put_layout_hdr(lo); 1621out: 1622 dprintk("%s: inode %s/%llu pNFS layout segment %s for " 1623 "(%s, offset: %llu, length: %llu)\n", 1624 __func__, ino->i_sb->s_id, 1625 (unsigned long long)NFS_FILEID(ino), 1626 lseg == NULL ? "not found" : "found", 1627 iomode==IOMODE_RW ? "read/write" : "read-only", 1628 (unsigned long long)pos, 1629 (unsigned long long)count); 1630 return lseg; 1631out_unlock: 1632 spin_unlock(&ino->i_lock); 1633 goto out_put_layout_hdr; 1634} 1635EXPORT_SYMBOL_GPL(pnfs_update_layout); 1636 1637static bool 1638pnfs_sanity_check_layout_range(struct pnfs_layout_range *range) 1639{ 1640 switch (range->iomode) { 1641 case IOMODE_READ: 1642 case IOMODE_RW: 1643 break; 1644 default: 1645 return false; 1646 } 1647 if (range->offset == NFS4_MAX_UINT64) 1648 return false; 1649 if (range->length == 0) 1650 return false; 1651 if (range->length != NFS4_MAX_UINT64 && 1652 range->length > NFS4_MAX_UINT64 - range->offset) 1653 return false; 1654 return true; 1655} 1656 1657struct pnfs_layout_segment * 1658pnfs_layout_process(struct nfs4_layoutget *lgp) 1659{ 1660 struct pnfs_layout_hdr *lo = NFS_I(lgp->args.inode)->layout; 1661 struct nfs4_layoutget_res *res = &lgp->res; 1662 struct pnfs_layout_segment *lseg; 1663 struct inode *ino = lo->plh_inode; 1664 LIST_HEAD(free_me); 1665 int status = -EINVAL; 1666 1667 if (!pnfs_sanity_check_layout_range(&res->range)) 1668 goto out; 1669 1670 /* Inject layout blob into I/O device driver */ 1671 lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags); 1672 if (!lseg || IS_ERR(lseg)) { 1673 if (!lseg) 1674 status = -ENOMEM; 1675 else 1676 status = PTR_ERR(lseg); 1677 dprintk("%s: Could not allocate layout: error %d\n", 1678 __func__, status); 1679 goto out; 1680 } 1681 1682 init_lseg(lo, lseg); 1683 lseg->pls_range = res->range; 1684 1685 spin_lock(&ino->i_lock); 1686 if (pnfs_layoutgets_blocked(lo)) { 1687 dprintk("%s forget reply due to state\n", __func__); 1688 goto out_forget_reply; 1689 } 1690 1691 if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) { 1692 /* existing state ID, make sure the sequence number matches. */ 1693 if (pnfs_layout_stateid_blocked(lo, &res->stateid)) { 1694 dprintk("%s forget reply due to sequence\n", __func__); 1695 status = -EAGAIN; 1696 goto out_forget_reply; 1697 } 1698 pnfs_set_layout_stateid(lo, &res->stateid, false); 1699 } else { 1700 /* 1701 * We got an entirely new state ID. Mark all segments for the 1702 * inode invalid, and don't bother validating the stateid 1703 * sequence number. 1704 */ 1705 pnfs_mark_matching_lsegs_invalid(lo, &free_me, NULL); 1706 1707 nfs4_stateid_copy(&lo->plh_stateid, &res->stateid); 1708 lo->plh_barrier = be32_to_cpu(res->stateid.seqid); 1709 } 1710 1711 clear_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); 1712 1713 pnfs_get_lseg(lseg); 1714 pnfs_layout_insert_lseg(lo, lseg, &free_me); 1715 1716 if (res->return_on_close) 1717 set_bit(NFS_LSEG_ROC, &lseg->pls_flags); 1718 1719 spin_unlock(&ino->i_lock); 1720 pnfs_free_lseg_list(&free_me); 1721 return lseg; 1722out: 1723 return ERR_PTR(status); 1724 1725out_forget_reply: 1726 spin_unlock(&ino->i_lock); 1727 lseg->pls_layout = lo; 1728 NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg); 1729 goto out; 1730} 1731 1732static void 1733pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo, 1734 struct list_head *tmp_list, 1735 struct pnfs_layout_range *return_range) 1736{ 1737 struct pnfs_layout_segment *lseg, *next; 1738 1739 dprintk("%s:Begin lo %p\n", __func__, lo); 1740 1741 if (list_empty(&lo->plh_segs)) 1742 return; 1743 1744 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) 1745 if (should_free_lseg(&lseg->pls_range, return_range)) { 1746 dprintk("%s: marking lseg %p iomode %d " 1747 "offset %llu length %llu\n", __func__, 1748 lseg, lseg->pls_range.iomode, 1749 lseg->pls_range.offset, 1750 lseg->pls_range.length); 1751 set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); 1752 mark_lseg_invalid(lseg, tmp_list); 1753 set_bit(NFS_LAYOUT_RETURN_BEFORE_CLOSE, 1754 &lo->plh_flags); 1755 } 1756} 1757 1758void pnfs_error_mark_layout_for_return(struct inode *inode, 1759 struct pnfs_layout_segment *lseg) 1760{ 1761 struct pnfs_layout_hdr *lo = NFS_I(inode)->layout; 1762 int iomode = pnfs_iomode_to_fail_bit(lseg->pls_range.iomode); 1763 struct pnfs_layout_range range = { 1764 .iomode = lseg->pls_range.iomode, 1765 .offset = 0, 1766 .length = NFS4_MAX_UINT64, 1767 }; 1768 LIST_HEAD(free_me); 1769 1770 spin_lock(&inode->i_lock); 1771 /* set failure bit so that pnfs path will be retried later */ 1772 pnfs_layout_set_fail_bit(lo, iomode); 1773 if (lo->plh_return_iomode == 0) 1774 lo->plh_return_iomode = range.iomode; 1775 else if (lo->plh_return_iomode != range.iomode) 1776 lo->plh_return_iomode = IOMODE_ANY; 1777 /* 1778 * mark all matching lsegs so that we are sure to have no live 1779 * segments at hand when sending layoutreturn. See pnfs_put_lseg() 1780 * for how it works. 1781 */ 1782 pnfs_mark_matching_lsegs_return(lo, &free_me, &range); 1783 spin_unlock(&inode->i_lock); 1784 pnfs_free_lseg_list(&free_me); 1785} 1786EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return); 1787 1788void 1789pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req) 1790{ 1791 u64 rd_size = req->wb_bytes; 1792 1793 if (pgio->pg_lseg == NULL) { 1794 if (pgio->pg_dreq == NULL) 1795 rd_size = i_size_read(pgio->pg_inode) - req_offset(req); 1796 else 1797 rd_size = nfs_dreq_bytes_left(pgio->pg_dreq); 1798 1799 pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode, 1800 req->wb_context, 1801 req_offset(req), 1802 rd_size, 1803 IOMODE_READ, 1804 GFP_KERNEL); 1805 } 1806 /* If no lseg, fall back to read through mds */ 1807 if (pgio->pg_lseg == NULL) 1808 nfs_pageio_reset_read_mds(pgio); 1809 1810} 1811EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_read); 1812 1813void 1814pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio, 1815 struct nfs_page *req, u64 wb_size) 1816{ 1817 if (pgio->pg_lseg == NULL) 1818 pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode, 1819 req->wb_context, 1820 req_offset(req), 1821 wb_size, 1822 IOMODE_RW, 1823 GFP_NOFS); 1824 /* If no lseg, fall back to write through mds */ 1825 if (pgio->pg_lseg == NULL) 1826 nfs_pageio_reset_write_mds(pgio); 1827} 1828EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write); 1829 1830void 1831pnfs_generic_pg_cleanup(struct nfs_pageio_descriptor *desc) 1832{ 1833 if (desc->pg_lseg) { 1834 pnfs_put_lseg(desc->pg_lseg); 1835 desc->pg_lseg = NULL; 1836 } 1837} 1838EXPORT_SYMBOL_GPL(pnfs_generic_pg_cleanup); 1839 1840/* 1841 * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number 1842 * of bytes (maximum @req->wb_bytes) that can be coalesced. 1843 */ 1844size_t 1845pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio, 1846 struct nfs_page *prev, struct nfs_page *req) 1847{ 1848 unsigned int size; 1849 u64 seg_end, req_start, seg_left; 1850 1851 size = nfs_generic_pg_test(pgio, prev, req); 1852 if (!size) 1853 return 0; 1854 1855 /* 1856 * 'size' contains the number of bytes left in the current page (up 1857 * to the original size asked for in @req->wb_bytes). 1858 * 1859 * Calculate how many bytes are left in the layout segment 1860 * and if there are less bytes than 'size', return that instead. 1861 * 1862 * Please also note that 'end_offset' is actually the offset of the 1863 * first byte that lies outside the pnfs_layout_range. FIXME? 1864 * 1865 */ 1866 if (pgio->pg_lseg) { 1867 seg_end = end_offset(pgio->pg_lseg->pls_range.offset, 1868 pgio->pg_lseg->pls_range.length); 1869 req_start = req_offset(req); 1870 WARN_ON_ONCE(req_start >= seg_end); 1871 /* start of request is past the last byte of this segment */ 1872 if (req_start >= seg_end) { 1873 /* reference the new lseg */ 1874 if (pgio->pg_ops->pg_cleanup) 1875 pgio->pg_ops->pg_cleanup(pgio); 1876 if (pgio->pg_ops->pg_init) 1877 pgio->pg_ops->pg_init(pgio, req); 1878 return 0; 1879 } 1880 1881 /* adjust 'size' iff there are fewer bytes left in the 1882 * segment than what nfs_generic_pg_test returned */ 1883 seg_left = seg_end - req_start; 1884 if (seg_left < size) 1885 size = (unsigned int)seg_left; 1886 } 1887 1888 return size; 1889} 1890EXPORT_SYMBOL_GPL(pnfs_generic_pg_test); 1891 1892int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *hdr) 1893{ 1894 struct nfs_pageio_descriptor pgio; 1895 1896 /* Resend all requests through the MDS */ 1897 nfs_pageio_init_write(&pgio, hdr->inode, FLUSH_STABLE, true, 1898 hdr->completion_ops); 1899 set_bit(NFS_CONTEXT_RESEND_WRITES, &hdr->args.context->flags); 1900 return nfs_pageio_resend(&pgio, hdr); 1901} 1902EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds); 1903 1904static void pnfs_ld_handle_write_error(struct nfs_pgio_header *hdr) 1905{ 1906 1907 dprintk("pnfs write error = %d\n", hdr->pnfs_error); 1908 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags & 1909 PNFS_LAYOUTRET_ON_ERROR) { 1910 pnfs_return_layout(hdr->inode); 1911 } 1912 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) 1913 hdr->task.tk_status = pnfs_write_done_resend_to_mds(hdr); 1914} 1915 1916/* 1917 * Called by non rpc-based layout drivers 1918 */ 1919void pnfs_ld_write_done(struct nfs_pgio_header *hdr) 1920{ 1921 if (likely(!hdr->pnfs_error)) { 1922 pnfs_set_layoutcommit(hdr->inode, hdr->lseg, 1923 hdr->mds_offset + hdr->res.count); 1924 hdr->mds_ops->rpc_call_done(&hdr->task, hdr); 1925 } 1926 trace_nfs4_pnfs_write(hdr, hdr->pnfs_error); 1927 if (unlikely(hdr->pnfs_error)) 1928 pnfs_ld_handle_write_error(hdr); 1929 hdr->mds_ops->rpc_release(hdr); 1930} 1931EXPORT_SYMBOL_GPL(pnfs_ld_write_done); 1932 1933static void 1934pnfs_write_through_mds(struct nfs_pageio_descriptor *desc, 1935 struct nfs_pgio_header *hdr) 1936{ 1937 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 1938 1939 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 1940 list_splice_tail_init(&hdr->pages, &mirror->pg_list); 1941 nfs_pageio_reset_write_mds(desc); 1942 mirror->pg_recoalesce = 1; 1943 } 1944 nfs_pgio_data_destroy(hdr); 1945 hdr->release(hdr); 1946} 1947 1948static enum pnfs_try_status 1949pnfs_try_to_write_data(struct nfs_pgio_header *hdr, 1950 const struct rpc_call_ops *call_ops, 1951 struct pnfs_layout_segment *lseg, 1952 int how) 1953{ 1954 struct inode *inode = hdr->inode; 1955 enum pnfs_try_status trypnfs; 1956 struct nfs_server *nfss = NFS_SERVER(inode); 1957 1958 hdr->mds_ops = call_ops; 1959 1960 dprintk("%s: Writing ino:%lu %u@%llu (how %d)\n", __func__, 1961 inode->i_ino, hdr->args.count, hdr->args.offset, how); 1962 trypnfs = nfss->pnfs_curr_ld->write_pagelist(hdr, how); 1963 if (trypnfs != PNFS_NOT_ATTEMPTED) 1964 nfs_inc_stats(inode, NFSIOS_PNFS_WRITE); 1965 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs); 1966 return trypnfs; 1967} 1968 1969static void 1970pnfs_do_write(struct nfs_pageio_descriptor *desc, 1971 struct nfs_pgio_header *hdr, int how) 1972{ 1973 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops; 1974 struct pnfs_layout_segment *lseg = desc->pg_lseg; 1975 enum pnfs_try_status trypnfs; 1976 1977 trypnfs = pnfs_try_to_write_data(hdr, call_ops, lseg, how); 1978 if (trypnfs == PNFS_NOT_ATTEMPTED) 1979 pnfs_write_through_mds(desc, hdr); 1980} 1981 1982static void pnfs_writehdr_free(struct nfs_pgio_header *hdr) 1983{ 1984 pnfs_put_lseg(hdr->lseg); 1985 nfs_pgio_header_free(hdr); 1986} 1987 1988int 1989pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc) 1990{ 1991 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 1992 1993 struct nfs_pgio_header *hdr; 1994 int ret; 1995 1996 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); 1997 if (!hdr) { 1998 desc->pg_completion_ops->error_cleanup(&mirror->pg_list); 1999 return -ENOMEM; 2000 } 2001 nfs_pgheader_init(desc, hdr, pnfs_writehdr_free); 2002 2003 hdr->lseg = pnfs_get_lseg(desc->pg_lseg); 2004 ret = nfs_generic_pgio(desc, hdr); 2005 if (!ret) 2006 pnfs_do_write(desc, hdr, desc->pg_ioflags); 2007 2008 return ret; 2009} 2010EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages); 2011 2012int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *hdr) 2013{ 2014 struct nfs_pageio_descriptor pgio; 2015 2016 /* Resend all requests through the MDS */ 2017 nfs_pageio_init_read(&pgio, hdr->inode, true, hdr->completion_ops); 2018 return nfs_pageio_resend(&pgio, hdr); 2019} 2020EXPORT_SYMBOL_GPL(pnfs_read_done_resend_to_mds); 2021 2022static void pnfs_ld_handle_read_error(struct nfs_pgio_header *hdr) 2023{ 2024 dprintk("pnfs read error = %d\n", hdr->pnfs_error); 2025 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags & 2026 PNFS_LAYOUTRET_ON_ERROR) { 2027 pnfs_return_layout(hdr->inode); 2028 } 2029 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) 2030 hdr->task.tk_status = pnfs_read_done_resend_to_mds(hdr); 2031} 2032 2033/* 2034 * Called by non rpc-based layout drivers 2035 */ 2036void pnfs_ld_read_done(struct nfs_pgio_header *hdr) 2037{ 2038 if (likely(!hdr->pnfs_error)) { 2039 __nfs4_read_done_cb(hdr); 2040 hdr->mds_ops->rpc_call_done(&hdr->task, hdr); 2041 } 2042 trace_nfs4_pnfs_read(hdr, hdr->pnfs_error); 2043 if (unlikely(hdr->pnfs_error)) 2044 pnfs_ld_handle_read_error(hdr); 2045 hdr->mds_ops->rpc_release(hdr); 2046} 2047EXPORT_SYMBOL_GPL(pnfs_ld_read_done); 2048 2049static void 2050pnfs_read_through_mds(struct nfs_pageio_descriptor *desc, 2051 struct nfs_pgio_header *hdr) 2052{ 2053 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 2054 2055 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { 2056 list_splice_tail_init(&hdr->pages, &mirror->pg_list); 2057 nfs_pageio_reset_read_mds(desc); 2058 mirror->pg_recoalesce = 1; 2059 } 2060 nfs_pgio_data_destroy(hdr); 2061 hdr->release(hdr); 2062} 2063 2064/* 2065 * Call the appropriate parallel I/O subsystem read function. 2066 */ 2067static enum pnfs_try_status 2068pnfs_try_to_read_data(struct nfs_pgio_header *hdr, 2069 const struct rpc_call_ops *call_ops, 2070 struct pnfs_layout_segment *lseg) 2071{ 2072 struct inode *inode = hdr->inode; 2073 struct nfs_server *nfss = NFS_SERVER(inode); 2074 enum pnfs_try_status trypnfs; 2075 2076 hdr->mds_ops = call_ops; 2077 2078 dprintk("%s: Reading ino:%lu %u@%llu\n", 2079 __func__, inode->i_ino, hdr->args.count, hdr->args.offset); 2080 2081 trypnfs = nfss->pnfs_curr_ld->read_pagelist(hdr); 2082 if (trypnfs != PNFS_NOT_ATTEMPTED) 2083 nfs_inc_stats(inode, NFSIOS_PNFS_READ); 2084 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs); 2085 return trypnfs; 2086} 2087 2088/* Resend all requests through pnfs. */ 2089int pnfs_read_resend_pnfs(struct nfs_pgio_header *hdr) 2090{ 2091 struct nfs_pageio_descriptor pgio; 2092 2093 nfs_pageio_init_read(&pgio, hdr->inode, false, hdr->completion_ops); 2094 return nfs_pageio_resend(&pgio, hdr); 2095} 2096EXPORT_SYMBOL_GPL(pnfs_read_resend_pnfs); 2097 2098static void 2099pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr) 2100{ 2101 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops; 2102 struct pnfs_layout_segment *lseg = desc->pg_lseg; 2103 enum pnfs_try_status trypnfs; 2104 int err = 0; 2105 2106 trypnfs = pnfs_try_to_read_data(hdr, call_ops, lseg); 2107 if (trypnfs == PNFS_TRY_AGAIN) 2108 err = pnfs_read_resend_pnfs(hdr); 2109 if (trypnfs == PNFS_NOT_ATTEMPTED || err) 2110 pnfs_read_through_mds(desc, hdr); 2111} 2112 2113static void pnfs_readhdr_free(struct nfs_pgio_header *hdr) 2114{ 2115 pnfs_put_lseg(hdr->lseg); 2116 nfs_pgio_header_free(hdr); 2117} 2118 2119int 2120pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc) 2121{ 2122 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); 2123 2124 struct nfs_pgio_header *hdr; 2125 int ret; 2126 2127 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); 2128 if (!hdr) { 2129 desc->pg_completion_ops->error_cleanup(&mirror->pg_list); 2130 return -ENOMEM; 2131 } 2132 nfs_pgheader_init(desc, hdr, pnfs_readhdr_free); 2133 hdr->lseg = pnfs_get_lseg(desc->pg_lseg); 2134 ret = nfs_generic_pgio(desc, hdr); 2135 if (!ret) 2136 pnfs_do_read(desc, hdr); 2137 return ret; 2138} 2139EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages); 2140 2141static void pnfs_clear_layoutcommitting(struct inode *inode) 2142{ 2143 unsigned long *bitlock = &NFS_I(inode)->flags; 2144 2145 clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock); 2146 smp_mb__after_atomic(); 2147 wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING); 2148} 2149 2150/* 2151 * There can be multiple RW segments. 2152 */ 2153static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp) 2154{ 2155 struct pnfs_layout_segment *lseg; 2156 2157 list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) { 2158 if (lseg->pls_range.iomode == IOMODE_RW && 2159 test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) 2160 list_add(&lseg->pls_lc_list, listp); 2161 } 2162} 2163 2164static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *listp) 2165{ 2166 struct pnfs_layout_segment *lseg, *tmp; 2167 2168 /* Matched by references in pnfs_set_layoutcommit */ 2169 list_for_each_entry_safe(lseg, tmp, listp, pls_lc_list) { 2170 list_del_init(&lseg->pls_lc_list); 2171 pnfs_put_lseg(lseg); 2172 } 2173 2174 pnfs_clear_layoutcommitting(inode); 2175} 2176 2177void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg) 2178{ 2179 pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode); 2180} 2181EXPORT_SYMBOL_GPL(pnfs_set_lo_fail); 2182 2183void 2184pnfs_set_layoutcommit(struct inode *inode, struct pnfs_layout_segment *lseg, 2185 loff_t end_pos) 2186{ 2187 struct nfs_inode *nfsi = NFS_I(inode); 2188 bool mark_as_dirty = false; 2189 2190 spin_lock(&inode->i_lock); 2191 if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) { 2192 nfsi->layout->plh_lwb = end_pos; 2193 mark_as_dirty = true; 2194 dprintk("%s: Set layoutcommit for inode %lu ", 2195 __func__, inode->i_ino); 2196 } else if (end_pos > nfsi->layout->plh_lwb) 2197 nfsi->layout->plh_lwb = end_pos; 2198 if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) { 2199 /* references matched in nfs4_layoutcommit_release */ 2200 pnfs_get_lseg(lseg); 2201 } 2202 spin_unlock(&inode->i_lock); 2203 dprintk("%s: lseg %p end_pos %llu\n", 2204 __func__, lseg, nfsi->layout->plh_lwb); 2205 2206 /* if pnfs_layoutcommit_inode() runs between inode locks, the next one 2207 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */ 2208 if (mark_as_dirty) 2209 mark_inode_dirty_sync(inode); 2210} 2211EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit); 2212 2213void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data) 2214{ 2215 struct nfs_server *nfss = NFS_SERVER(data->args.inode); 2216 2217 if (nfss->pnfs_curr_ld->cleanup_layoutcommit) 2218 nfss->pnfs_curr_ld->cleanup_layoutcommit(data); 2219 pnfs_list_write_lseg_done(data->args.inode, &data->lseg_list); 2220} 2221 2222/* 2223 * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and 2224 * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough 2225 * data to disk to allow the server to recover the data if it crashes. 2226 * LAYOUTCOMMIT is only needed when the NFL4_UFLG_COMMIT_THRU_MDS flag 2227 * is off, and a COMMIT is sent to a data server, or 2228 * if WRITEs to a data server return NFS_DATA_SYNC. 2229 */ 2230int 2231pnfs_layoutcommit_inode(struct inode *inode, bool sync) 2232{ 2233 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 2234 struct nfs4_layoutcommit_data *data; 2235 struct nfs_inode *nfsi = NFS_I(inode); 2236 loff_t end_pos; 2237 int status; 2238 2239 if (!pnfs_layoutcommit_outstanding(inode)) 2240 return 0; 2241 2242 dprintk("--> %s inode %lu\n", __func__, inode->i_ino); 2243 2244 status = -EAGAIN; 2245 if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) { 2246 if (!sync) 2247 goto out; 2248 status = wait_on_bit_lock_action(&nfsi->flags, 2249 NFS_INO_LAYOUTCOMMITTING, 2250 nfs_wait_bit_killable, 2251 TASK_KILLABLE); 2252 if (status) 2253 goto out; 2254 } 2255 2256 status = -ENOMEM; 2257 /* Note kzalloc ensures data->res.seq_res.sr_slot == NULL */ 2258 data = kzalloc(sizeof(*data), GFP_NOFS); 2259 if (!data) 2260 goto clear_layoutcommitting; 2261 2262 status = 0; 2263 spin_lock(&inode->i_lock); 2264 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) 2265 goto out_unlock; 2266 2267 INIT_LIST_HEAD(&data->lseg_list); 2268 pnfs_list_write_lseg(inode, &data->lseg_list); 2269 2270 end_pos = nfsi->layout->plh_lwb; 2271 2272 nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid); 2273 spin_unlock(&inode->i_lock); 2274 2275 data->args.inode = inode; 2276 data->cred = get_rpccred(nfsi->layout->plh_lc_cred); 2277 nfs_fattr_init(&data->fattr); 2278 data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask; 2279 data->res.fattr = &data->fattr; 2280 data->args.lastbytewritten = end_pos - 1; 2281 data->res.server = NFS_SERVER(inode); 2282 2283 if (ld->prepare_layoutcommit) { 2284 status = ld->prepare_layoutcommit(&data->args); 2285 if (status) { 2286 put_rpccred(data->cred); 2287 spin_lock(&inode->i_lock); 2288 set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags); 2289 if (end_pos > nfsi->layout->plh_lwb) 2290 nfsi->layout->plh_lwb = end_pos; 2291 goto out_unlock; 2292 } 2293 } 2294 2295 2296 status = nfs4_proc_layoutcommit(data, sync); 2297out: 2298 if (status) 2299 mark_inode_dirty_sync(inode); 2300 dprintk("<-- %s status %d\n", __func__, status); 2301 return status; 2302out_unlock: 2303 spin_unlock(&inode->i_lock); 2304 kfree(data); 2305clear_layoutcommitting: 2306 pnfs_clear_layoutcommitting(inode); 2307 goto out; 2308} 2309EXPORT_SYMBOL_GPL(pnfs_layoutcommit_inode); 2310 2311int 2312pnfs_generic_sync(struct inode *inode, bool datasync) 2313{ 2314 return pnfs_layoutcommit_inode(inode, true); 2315} 2316EXPORT_SYMBOL_GPL(pnfs_generic_sync); 2317 2318struct nfs4_threshold *pnfs_mdsthreshold_alloc(void) 2319{ 2320 struct nfs4_threshold *thp; 2321 2322 thp = kzalloc(sizeof(*thp), GFP_NOFS); 2323 if (!thp) { 2324 dprintk("%s mdsthreshold allocation failed\n", __func__); 2325 return NULL; 2326 } 2327 return thp; 2328} 2329 2330#if IS_ENABLED(CONFIG_NFS_V4_2) 2331int 2332pnfs_report_layoutstat(struct inode *inode, gfp_t gfp_flags) 2333{ 2334 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; 2335 struct nfs_server *server = NFS_SERVER(inode); 2336 struct nfs_inode *nfsi = NFS_I(inode); 2337 struct nfs42_layoutstat_data *data; 2338 struct pnfs_layout_hdr *hdr; 2339 int status = 0; 2340 2341 if (!pnfs_enabled_sb(server) || !ld->prepare_layoutstats) 2342 goto out; 2343 2344 if (!nfs_server_capable(inode, NFS_CAP_LAYOUTSTATS)) 2345 goto out; 2346 2347 if (test_and_set_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags)) 2348 goto out; 2349 2350 spin_lock(&inode->i_lock); 2351 if (!NFS_I(inode)->layout) { 2352 spin_unlock(&inode->i_lock); 2353 goto out; 2354 } 2355 hdr = NFS_I(inode)->layout; 2356 pnfs_get_layout_hdr(hdr); 2357 spin_unlock(&inode->i_lock); 2358 2359 data = kzalloc(sizeof(*data), gfp_flags); 2360 if (!data) { 2361 status = -ENOMEM; 2362 goto out_put; 2363 } 2364 2365 data->args.fh = NFS_FH(inode); 2366 data->args.inode = inode; 2367 nfs4_stateid_copy(&data->args.stateid, &hdr->plh_stateid); 2368 status = ld->prepare_layoutstats(&data->args); 2369 if (status) 2370 goto out_free; 2371 2372 status = nfs42_proc_layoutstats_generic(NFS_SERVER(inode), data); 2373 2374out: 2375 dprintk("%s returns %d\n", __func__, status); 2376 return status; 2377 2378out_free: 2379 kfree(data); 2380out_put: 2381 pnfs_put_layout_hdr(hdr); 2382 smp_mb__before_atomic(); 2383 clear_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags); 2384 smp_mb__after_atomic(); 2385 goto out; 2386} 2387EXPORT_SYMBOL_GPL(pnfs_report_layoutstat); 2388#endif 2389 2390unsigned int layoutstats_timer; 2391module_param(layoutstats_timer, uint, 0644); 2392EXPORT_SYMBOL_GPL(layoutstats_timer); 2393