dio 170 drivers/block/loop.c static void __loop_update_dio(struct loop_device *lo, bool dio) dio 194 drivers/block/loop.c if (dio) { dio 821 drivers/block/loop.c int dio = (lo->lo_flags & LO_FLAGS_DIRECT_IO); dio 823 drivers/block/loop.c return sprintf(buf, "%s\n", dio ? "1" : "0"); dio 831 drivers/block/loop.c LOOP_ATTR_RO(dio); dio 367 drivers/md/dm-integrity.c static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map); dio 1379 drivers/md/dm-integrity.c static void submit_flush_bio(struct dm_integrity_c *ic, struct dm_integrity_io *dio) dio 1385 drivers/md/dm-integrity.c bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); dio 1408 drivers/md/dm-integrity.c static void do_endio_flush(struct dm_integrity_c *ic, struct dm_integrity_io *dio) dio 1410 drivers/md/dm-integrity.c struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); dio 1412 drivers/md/dm-integrity.c if (unlikely(dio->fua) && likely(!bio->bi_status) && likely(!dm_integrity_failed(ic))) dio 1413 drivers/md/dm-integrity.c submit_flush_bio(ic, dio); dio 1418 drivers/md/dm-integrity.c static void dec_in_flight(struct dm_integrity_io *dio) dio 1420 drivers/md/dm-integrity.c if (atomic_dec_and_test(&dio->in_flight)) { dio 1421 drivers/md/dm-integrity.c struct dm_integrity_c *ic = dio->ic; dio 1424 drivers/md/dm-integrity.c remove_range(ic, &dio->range); dio 1426 drivers/md/dm-integrity.c if (unlikely(dio->write)) dio 1429 drivers/md/dm-integrity.c bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); dio 1431 drivers/md/dm-integrity.c if (unlikely(dio->bi_status) && !bio->bi_status) dio 1432 drivers/md/dm-integrity.c bio->bi_status = dio->bi_status; dio 1433 drivers/md/dm-integrity.c if (likely(!bio->bi_status) && unlikely(bio_sectors(bio) != dio->range.n_sectors)) { dio 1434 drivers/md/dm-integrity.c dio->range.logical_sector += dio->range.n_sectors; dio 1435 drivers/md/dm-integrity.c bio_advance(bio, dio->range.n_sectors << SECTOR_SHIFT); dio 1436 drivers/md/dm-integrity.c INIT_WORK(&dio->work, integrity_bio_wait); dio 1437 drivers/md/dm-integrity.c queue_work(ic->offload_wq, &dio->work); dio 1440 drivers/md/dm-integrity.c do_endio_flush(ic, dio); dio 1446 drivers/md/dm-integrity.c struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); dio 1448 drivers/md/dm-integrity.c dm_bio_restore(&dio->bio_details, bio); dio 1452 drivers/md/dm-integrity.c if (dio->completion) dio 1453 drivers/md/dm-integrity.c complete(dio->completion); dio 1455 drivers/md/dm-integrity.c dec_in_flight(dio); dio 1505 drivers/md/dm-integrity.c struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work); dio 1506 drivers/md/dm-integrity.c struct dm_integrity_c *ic = dio->ic; dio 1514 drivers/md/dm-integrity.c struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); dio 1518 drivers/md/dm-integrity.c unsigned sectors_to_process = dio->range.n_sectors; dio 1519 drivers/md/dm-integrity.c sector_t sector = dio->range.logical_sector; dio 1535 drivers/md/dm-integrity.c __bio_for_each_segment(bv, bio, iter, dio->bio_details.bi_iter) { dio 1552 drivers/md/dm-integrity.c r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset, dio 1553 drivers/md/dm-integrity.c checksums_ptr - checksums, !dio->write ? TAG_CMP : TAG_WRITE); dio 1579 drivers/md/dm-integrity.c struct bio_integrity_payload *bip = dio->bio_details.bi_integrity; dio 1584 drivers/md/dm-integrity.c unsigned data_to_process = dio->range.n_sectors; dio 1595 drivers/md/dm-integrity.c r = dm_integrity_rw_tag(ic, tag, &dio->metadata_block, &dio->metadata_offset, dio 1596 drivers/md/dm-integrity.c this_len, !dio->write ? TAG_READ : TAG_WRITE); dio 1606 drivers/md/dm-integrity.c dec_in_flight(dio); dio 1609 drivers/md/dm-integrity.c dio->bi_status = errno_to_blk_status(r); dio 1610 drivers/md/dm-integrity.c dec_in_flight(dio); dio 1616 drivers/md/dm-integrity.c struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); dio 1621 drivers/md/dm-integrity.c dio->ic = ic; dio 1622 drivers/md/dm-integrity.c dio->bi_status = 0; dio 1625 drivers/md/dm-integrity.c submit_flush_bio(ic, dio); dio 1629 drivers/md/dm-integrity.c dio->range.logical_sector = dm_target_offset(ti, bio->bi_iter.bi_sector); dio 1630 drivers/md/dm-integrity.c dio->write = bio_op(bio) == REQ_OP_WRITE; dio 1631 drivers/md/dm-integrity.c dio->fua = dio->write && bio->bi_opf & REQ_FUA; dio 1632 drivers/md/dm-integrity.c if (unlikely(dio->fua)) { dio 1639 drivers/md/dm-integrity.c if (unlikely(dio->range.logical_sector + bio_sectors(bio) > ic->provided_data_sectors)) { dio 1641 drivers/md/dm-integrity.c (unsigned long long)dio->range.logical_sector, bio_sectors(bio), dio 1645 drivers/md/dm-integrity.c if (unlikely((dio->range.logical_sector | bio_sectors(bio)) & (unsigned)(ic->sectors_per_block - 1))) { dio 1648 drivers/md/dm-integrity.c (unsigned long long)dio->range.logical_sector, bio_sectors(bio)); dio 1685 drivers/md/dm-integrity.c if (unlikely(ic->mode == 'R') && unlikely(dio->write)) dio 1688 drivers/md/dm-integrity.c get_area_and_offset(ic, dio->range.logical_sector, &area, &offset); dio 1689 drivers/md/dm-integrity.c dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset); dio 1692 drivers/md/dm-integrity.c dm_integrity_map_continue(dio, true); dio 1696 drivers/md/dm-integrity.c static bool __journal_read_write(struct dm_integrity_io *dio, struct bio *bio, dio 1699 drivers/md/dm-integrity.c struct dm_integrity_c *ic = dio->ic; dio 1703 drivers/md/dm-integrity.c logical_sector = dio->range.logical_sector; dio 1704 drivers/md/dm-integrity.c n_sectors = dio->range.n_sectors; dio 1715 drivers/md/dm-integrity.c if (likely(dio->write)) dio 1721 drivers/md/dm-integrity.c if (unlikely(!dio->write)) { dio 1768 drivers/md/dm-integrity.c if (likely(dio->write)) dio 1776 drivers/md/dm-integrity.c if (likely(dio->write)) dio 1781 drivers/md/dm-integrity.c if (likely(dio->write)) { dio 1817 drivers/md/dm-integrity.c if (unlikely(!dio->write)) dio 1822 drivers/md/dm-integrity.c if (likely(dio->write)) { dio 1832 drivers/md/dm-integrity.c remove_range(ic, &dio->range); dio 1838 drivers/md/dm-integrity.c dio->range.logical_sector = logical_sector; dio 1839 drivers/md/dm-integrity.c get_area_and_offset(ic, dio->range.logical_sector, &area, &offset); dio 1840 drivers/md/dm-integrity.c dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset); dio 1847 drivers/md/dm-integrity.c static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map) dio 1849 drivers/md/dm-integrity.c struct dm_integrity_c *ic = dio->ic; dio 1850 drivers/md/dm-integrity.c struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); dio 1854 drivers/md/dm-integrity.c bool need_sync_io = ic->internal_hash && !dio->write; dio 1857 drivers/md/dm-integrity.c INIT_WORK(&dio->work, integrity_bio_wait); dio 1858 drivers/md/dm-integrity.c queue_work(ic->offload_wq, &dio->work); dio 1870 drivers/md/dm-integrity.c dio->range.n_sectors = bio_sectors(bio); dio 1873 drivers/md/dm-integrity.c if (dio->write) { dio 1877 drivers/md/dm-integrity.c dio->range.n_sectors = min(dio->range.n_sectors, dio 1879 drivers/md/dm-integrity.c if (unlikely(!dio->range.n_sectors)) { dio 1885 drivers/md/dm-integrity.c range_sectors = dio->range.n_sectors >> ic->sb->log2_sectors_per_block; dio 1903 drivers/md/dm-integrity.c add_journal_node(ic, &ic->journal_tree[pos], dio->range.logical_sector + i); dio 1917 drivers/md/dm-integrity.c } while ((i += ic->sectors_per_block) < dio->range.n_sectors); dio 1923 drivers/md/dm-integrity.c journal_read_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); dio 1925 drivers/md/dm-integrity.c if (unlikely(dio->range.n_sectors > next_sector - dio->range.logical_sector)) dio 1926 drivers/md/dm-integrity.c dio->range.n_sectors = next_sector - dio->range.logical_sector; dio 1930 drivers/md/dm-integrity.c for (i = ic->sectors_per_block; i < dio->range.n_sectors; i += ic->sectors_per_block, jp++) { dio 1931 drivers/md/dm-integrity.c if (!test_journal_node(ic, jp, dio->range.logical_sector + i)) dio 1934 drivers/md/dm-integrity.c dio->range.n_sectors = i; dio 1938 drivers/md/dm-integrity.c if (unlikely(!add_new_range(ic, &dio->range, true))) { dio 1947 drivers/md/dm-integrity.c INIT_WORK(&dio->work, integrity_bio_wait); dio 1948 drivers/md/dm-integrity.c queue_work(ic->wait_wq, &dio->work); dio 1952 drivers/md/dm-integrity.c dio->range.n_sectors = ic->sectors_per_block; dio 1953 drivers/md/dm-integrity.c wait_and_add_new_range(ic, &dio->range); dio 1961 drivers/md/dm-integrity.c unsigned new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); dio 1963 drivers/md/dm-integrity.c remove_range_unlocked(ic, &dio->range); dio 1976 drivers/md/dm-integrity.c if (ic->mode == 'B' && dio->write) { dio 1977 drivers/md/dm-integrity.c if (!block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector, dio 1978 drivers/md/dm-integrity.c dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) { dio 1981 drivers/md/dm-integrity.c bbs = sector_to_bitmap_block(ic, dio->range.logical_sector); dio 1990 drivers/md/dm-integrity.c dio->in_flight = (atomic_t)ATOMIC_INIT(2); dio 1994 drivers/md/dm-integrity.c dio->completion = &read_comp; dio 1996 drivers/md/dm-integrity.c dio->completion = NULL; dio 1998 drivers/md/dm-integrity.c dm_bio_record(&dio->bio_details, bio); dio 2003 drivers/md/dm-integrity.c bio->bi_iter.bi_size = dio->range.n_sectors << SECTOR_SHIFT; dio 2010 drivers/md/dm-integrity.c dio->range.logical_sector + dio->range.n_sectors > le64_to_cpu(ic->sb->recalc_sector)) dio 2013 drivers/md/dm-integrity.c if (!block_bitmap_op(ic, ic->recalc_bitmap, dio->range.logical_sector, dio 2014 drivers/md/dm-integrity.c dio->range.n_sectors, BITMAP_OP_TEST_ALL_CLEAR)) dio 2019 drivers/md/dm-integrity.c integrity_metadata(&dio->work); dio 2022 drivers/md/dm-integrity.c dec_in_flight(dio); dio 2025 drivers/md/dm-integrity.c INIT_WORK(&dio->work, integrity_metadata); dio 2026 drivers/md/dm-integrity.c queue_work(ic->metadata_wq, &dio->work); dio 2032 drivers/md/dm-integrity.c if (unlikely(__journal_read_write(dio, bio, journal_section, journal_entry))) dio 2035 drivers/md/dm-integrity.c do_endio_flush(ic, dio); dio 2041 drivers/md/dm-integrity.c struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work); dio 2043 drivers/md/dm-integrity.c dm_integrity_map_continue(dio, false); dio 2479 drivers/md/dm-integrity.c struct dm_integrity_io *dio; dio 2481 drivers/md/dm-integrity.c dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); dio 2483 drivers/md/dm-integrity.c if (block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector, dio 2484 drivers/md/dm-integrity.c dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) { dio 2485 drivers/md/dm-integrity.c remove_range(ic, &dio->range); dio 2486 drivers/md/dm-integrity.c INIT_WORK(&dio->work, integrity_bio_wait); dio 2487 drivers/md/dm-integrity.c queue_work(ic->offload_wq, &dio->work); dio 2489 drivers/md/dm-integrity.c block_bitmap_op(ic, ic->journal, dio->range.logical_sector, dio 2490 drivers/md/dm-integrity.c dio->range.n_sectors, BITMAP_OP_SET); dio 2503 drivers/md/dm-integrity.c struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); dio 2505 drivers/md/dm-integrity.c block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector, dio 2506 drivers/md/dm-integrity.c dio->range.n_sectors, BITMAP_OP_SET); dio 2508 drivers/md/dm-integrity.c remove_range(ic, &dio->range); dio 2509 drivers/md/dm-integrity.c INIT_WORK(&dio->work, integrity_bio_wait); dio 2510 drivers/md/dm-integrity.c queue_work(ic->offload_wq, &dio->work); dio 447 drivers/scsi/bfa/bfa_fcpim.c idx = bfa_ioim_get_index(scsi_bufflen((struct scsi_cmnd *)ioim->dio)); dio 2127 drivers/scsi/bfa/bfa_fcpim.c bfa_cb_ioim_good_comp(ioim->bfa->bfad, ioim->dio); dio 2170 drivers/scsi/bfa/bfa_fcpim.c bfa_cb_ioim_done(ioim->bfa->bfad, ioim->dio, m->io_status, dio 2402 drivers/scsi/bfa/bfa_fcpim.c bfa_cb_ioim_done(ioim->bfa->bfad, ioim->dio, BFI_IOIM_STS_ABORTED, dio 2417 drivers/scsi/bfa/bfa_fcpim.c bfa_cb_ioim_done(ioim->bfa->bfad, ioim->dio, BFI_IOIM_STS_PATHTOV, dio 2431 drivers/scsi/bfa/bfa_fcpim.c bfa_cb_ioim_abort(ioim->bfa->bfad, ioim->dio); dio 2460 drivers/scsi/bfa/bfa_fcpim.c struct scsi_cmnd *cmnd = (struct scsi_cmnd *) ioim->dio; dio 2936 drivers/scsi/bfa/bfa_fcpim.c bfa_ioim_alloc(struct bfa_s *bfa, struct bfad_ioim_s *dio, dio 2954 drivers/scsi/bfa/bfa_fcpim.c ioim->dio = dio; dio 3324 drivers/scsi/bfa/bfa_fcpim.c cmnd = (struct scsi_cmnd *) ioim->dio; dio 3337 drivers/scsi/bfa/bfa_fcpim.c cmnd = (struct scsi_cmnd *) ioim->dio; dio 166 drivers/scsi/bfa/bfa_fcpim.h struct bfad_ioim_s *dio; /* driver IO handle */ dio 359 drivers/scsi/bfa/bfa_fcpim.h struct bfad_ioim_s *dio, dio 380 drivers/scsi/bfa/bfa_fcpim.h void bfa_cb_ioim_done(void *bfad, struct bfad_ioim_s *dio, dio 388 drivers/scsi/bfa/bfa_fcpim.h void bfa_cb_ioim_good_comp(void *bfad, struct bfad_ioim_s *dio); dio 393 drivers/scsi/bfa/bfa_fcpim.h void bfa_cb_ioim_abort(void *bfad, struct bfad_ioim_s *dio); dio 33 drivers/scsi/bfa/bfad_im.c bfa_cb_ioim_done(void *drv, struct bfad_ioim_s *dio, dio 37 drivers/scsi/bfa/bfad_im.c struct scsi_cmnd *cmnd = (struct scsi_cmnd *)dio; dio 103 drivers/scsi/bfa/bfad_im.c bfa_cb_ioim_good_comp(void *drv, struct bfad_ioim_s *dio) dio 105 drivers/scsi/bfa/bfad_im.c struct scsi_cmnd *cmnd = (struct scsi_cmnd *)dio; dio 131 drivers/scsi/bfa/bfad_im.c bfa_cb_ioim_abort(void *drv, struct bfad_ioim_s *dio) dio 133 drivers/scsi/bfa/bfad_im.c struct scsi_cmnd *cmnd = (struct scsi_cmnd *)dio; dio 208 drivers/scsi/bfa/bfad_im.c if (hal_io->dio != (struct bfad_ioim_s *) cmnd) { dio 69 drivers/staging/comedi/drivers/ni_670x.c int dio; dio 287 drivers/staging/comedi/drivers/ni_pcidio.c int dio; dio 301 fs/block_dev.c struct blkdev_dio *dio = bio->bi_private; dio 302 fs/block_dev.c bool should_dirty = dio->should_dirty; dio 304 fs/block_dev.c if (bio->bi_status && !dio->bio.bi_status) dio 305 fs/block_dev.c dio->bio.bi_status = bio->bi_status; dio 307 fs/block_dev.c if (!dio->multi_bio || atomic_dec_and_test(&dio->ref)) { dio 308 fs/block_dev.c if (!dio->is_sync) { dio 309 fs/block_dev.c struct kiocb *iocb = dio->iocb; dio 312 fs/block_dev.c if (likely(!dio->bio.bi_status)) { dio 313 fs/block_dev.c ret = dio->size; dio 316 fs/block_dev.c ret = blk_status_to_errno(dio->bio.bi_status); dio 319 fs/block_dev.c dio->iocb->ki_complete(iocb, ret, 0); dio 320 fs/block_dev.c if (dio->multi_bio) dio 321 fs/block_dev.c bio_put(&dio->bio); dio 323 fs/block_dev.c struct task_struct *waiter = dio->waiter; dio 325 fs/block_dev.c WRITE_ONCE(dio->waiter, NULL); dio 345 fs/block_dev.c struct blkdev_dio *dio; dio 359 fs/block_dev.c dio = container_of(bio, struct blkdev_dio, bio); dio 360 fs/block_dev.c dio->is_sync = is_sync = is_sync_kiocb(iocb); dio 361 fs/block_dev.c if (dio->is_sync) { dio 362 fs/block_dev.c dio->waiter = current; dio 365 fs/block_dev.c dio->iocb = iocb; dio 368 fs/block_dev.c dio->size = 0; dio 369 fs/block_dev.c dio->multi_bio = false; dio 370 fs/block_dev.c dio->should_dirty = is_read && iter_is_iovec(iter); dio 383 fs/block_dev.c bio->bi_private = dio; dio 396 fs/block_dev.c if (dio->should_dirty) dio 403 fs/block_dev.c dio->size += bio->bi_iter.bi_size; dio 422 fs/block_dev.c if (!dio->multi_bio) { dio 430 fs/block_dev.c dio->multi_bio = true; dio 431 fs/block_dev.c atomic_set(&dio->ref, 2); dio 433 fs/block_dev.c atomic_inc(&dio->ref); dio 448 fs/block_dev.c if (!READ_ONCE(dio->waiter)) dio 458 fs/block_dev.c ret = blk_status_to_errno(dio->bio.bi_status); dio 460 fs/block_dev.c ret = dio->size; dio 462 fs/block_dev.c bio_put(&dio->bio); dio 152 fs/btrfs/file-item.c u64 logical_offset, u8 *dst, int dio) dio 208 fs/btrfs/file-item.c if (dio) dio 216 fs/btrfs/file-item.c if (!dio) dio 176 fs/btrfs/ordered-data.c int type, int dio, int compress_type) dio 200 fs/btrfs/ordered-data.c if (dio) { dio 167 fs/direct-io.c static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio) dio 171 fs/direct-io.c ret = iov_iter_get_pages(sdio->iter, dio->pages, LONG_MAX, DIO_PAGES, dio 174 fs/direct-io.c if (ret < 0 && sdio->blocks_available && (dio->op == REQ_OP_WRITE)) { dio 181 fs/direct-io.c if (dio->page_errors == 0) dio 182 fs/direct-io.c dio->page_errors = ret; dio 184 fs/direct-io.c dio->pages[0] = page; dio 209 fs/direct-io.c static inline struct page *dio_get_page(struct dio *dio, dio 215 fs/direct-io.c ret = dio_refill_pages(dio, sdio); dio 220 fs/direct-io.c return dio->pages[sdio->head]; dio 255 fs/direct-io.c static ssize_t dio_complete(struct dio *dio, ssize_t ret, unsigned int flags) dio 257 fs/direct-io.c loff_t offset = dio->iocb->ki_pos; dio 270 fs/direct-io.c if (dio->result) { dio 271 fs/direct-io.c transferred = dio->result; dio 274 fs/direct-io.c if ((dio->op == REQ_OP_READ) && dio 275 fs/direct-io.c ((offset + transferred) > dio->i_size)) dio 276 fs/direct-io.c transferred = dio->i_size - offset; dio 283 fs/direct-io.c ret = dio->page_errors; dio 285 fs/direct-io.c ret = dio->io_error; dio 289 fs/direct-io.c if (dio->end_io) { dio 291 fs/direct-io.c err = dio->end_io(dio->iocb, offset, ret, dio->private); dio 309 fs/direct-io.c ret > 0 && dio->op == REQ_OP_WRITE && dio 310 fs/direct-io.c dio->inode->i_mapping->nrpages) { dio 311 fs/direct-io.c err = invalidate_inode_pages2_range(dio->inode->i_mapping, dio 315 fs/direct-io.c dio_warn_stale_pagecache(dio->iocb->ki_filp); dio 318 fs/direct-io.c inode_dio_end(dio->inode); dio 326 fs/direct-io.c dio->iocb->ki_pos += transferred; dio 328 fs/direct-io.c if (ret > 0 && dio->op == REQ_OP_WRITE) dio 329 fs/direct-io.c ret = generic_write_sync(dio->iocb, ret); dio 330 fs/direct-io.c dio->iocb->ki_complete(dio->iocb, ret, 0); dio 333 fs/direct-io.c kmem_cache_free(dio_cache, dio); dio 339 fs/direct-io.c struct dio *dio = container_of(work, struct dio, complete_work); dio 341 fs/direct-io.c dio_complete(dio, 0, DIO_COMPLETE_ASYNC | DIO_COMPLETE_INVALIDATE); dio 344 fs/direct-io.c static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio); dio 351 fs/direct-io.c struct dio *dio = bio->bi_private; dio 357 fs/direct-io.c dio_bio_complete(dio, bio); dio 359 fs/direct-io.c spin_lock_irqsave(&dio->bio_lock, flags); dio 360 fs/direct-io.c remaining = --dio->refcount; dio 361 fs/direct-io.c if (remaining == 1 && dio->waiter) dio 362 fs/direct-io.c wake_up_process(dio->waiter); dio 363 fs/direct-io.c spin_unlock_irqrestore(&dio->bio_lock, flags); dio 374 fs/direct-io.c if (dio->result) dio 375 fs/direct-io.c defer_completion = dio->defer_completion || dio 376 fs/direct-io.c (dio->op == REQ_OP_WRITE && dio 377 fs/direct-io.c dio->inode->i_mapping->nrpages); dio 379 fs/direct-io.c INIT_WORK(&dio->complete_work, dio_aio_complete_work); dio 380 fs/direct-io.c queue_work(dio->inode->i_sb->s_dio_done_wq, dio 381 fs/direct-io.c &dio->complete_work); dio 383 fs/direct-io.c dio_complete(dio, 0, DIO_COMPLETE_ASYNC); dio 397 fs/direct-io.c struct dio *dio = bio->bi_private; dio 400 fs/direct-io.c spin_lock_irqsave(&dio->bio_lock, flags); dio 401 fs/direct-io.c bio->bi_private = dio->bio_list; dio 402 fs/direct-io.c dio->bio_list = bio; dio 403 fs/direct-io.c if (--dio->refcount == 1 && dio->waiter) dio 404 fs/direct-io.c wake_up_process(dio->waiter); dio 405 fs/direct-io.c spin_unlock_irqrestore(&dio->bio_lock, flags); dio 418 fs/direct-io.c struct dio *dio = bio->bi_private; dio 420 fs/direct-io.c if (dio->is_async) dio 428 fs/direct-io.c dio_bio_alloc(struct dio *dio, struct dio_submit *sdio, dio 442 fs/direct-io.c bio_set_op_attrs(bio, dio->op, dio->op_flags); dio 443 fs/direct-io.c if (dio->is_async) dio 448 fs/direct-io.c bio->bi_write_hint = dio->iocb->ki_hint; dio 461 fs/direct-io.c static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio) dio 466 fs/direct-io.c bio->bi_private = dio; dio 468 fs/direct-io.c spin_lock_irqsave(&dio->bio_lock, flags); dio 469 fs/direct-io.c dio->refcount++; dio 470 fs/direct-io.c spin_unlock_irqrestore(&dio->bio_lock, flags); dio 472 fs/direct-io.c if (dio->is_async && dio->op == REQ_OP_READ && dio->should_dirty) dio 475 fs/direct-io.c dio->bio_disk = bio->bi_disk; dio 478 fs/direct-io.c sdio->submit_io(bio, dio->inode, sdio->logical_offset_in_bio); dio 479 fs/direct-io.c dio->bio_cookie = BLK_QC_T_NONE; dio 481 fs/direct-io.c dio->bio_cookie = submit_bio(bio); dio 491 fs/direct-io.c static inline void dio_cleanup(struct dio *dio, struct dio_submit *sdio) dio 494 fs/direct-io.c put_page(dio->pages[sdio->head++]); dio 503 fs/direct-io.c static struct bio *dio_await_one(struct dio *dio) dio 508 fs/direct-io.c spin_lock_irqsave(&dio->bio_lock, flags); dio 516 fs/direct-io.c while (dio->refcount > 1 && dio->bio_list == NULL) { dio 518 fs/direct-io.c dio->waiter = current; dio 519 fs/direct-io.c spin_unlock_irqrestore(&dio->bio_lock, flags); dio 520 fs/direct-io.c if (!(dio->iocb->ki_flags & IOCB_HIPRI) || dio 521 fs/direct-io.c !blk_poll(dio->bio_disk->queue, dio->bio_cookie, true)) dio 524 fs/direct-io.c spin_lock_irqsave(&dio->bio_lock, flags); dio 525 fs/direct-io.c dio->waiter = NULL; dio 527 fs/direct-io.c if (dio->bio_list) { dio 528 fs/direct-io.c bio = dio->bio_list; dio 529 fs/direct-io.c dio->bio_list = bio->bi_private; dio 531 fs/direct-io.c spin_unlock_irqrestore(&dio->bio_lock, flags); dio 538 fs/direct-io.c static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio) dio 541 fs/direct-io.c bool should_dirty = dio->op == REQ_OP_READ && dio->should_dirty; dio 545 fs/direct-io.c dio->io_error = -EAGAIN; dio 547 fs/direct-io.c dio->io_error = -EIO; dio 550 fs/direct-io.c if (dio->is_async && should_dirty) { dio 566 fs/direct-io.c static void dio_await_completion(struct dio *dio) dio 570 fs/direct-io.c bio = dio_await_one(dio); dio 572 fs/direct-io.c dio_bio_complete(dio, bio); dio 583 fs/direct-io.c static inline int dio_bio_reap(struct dio *dio, struct dio_submit *sdio) dio 588 fs/direct-io.c while (dio->bio_list) { dio 593 fs/direct-io.c spin_lock_irqsave(&dio->bio_lock, flags); dio 594 fs/direct-io.c bio = dio->bio_list; dio 595 fs/direct-io.c dio->bio_list = bio->bi_private; dio 596 fs/direct-io.c spin_unlock_irqrestore(&dio->bio_lock, flags); dio 597 fs/direct-io.c ret2 = blk_status_to_errno(dio_bio_complete(dio, bio)); dio 630 fs/direct-io.c static int dio_set_defer_completion(struct dio *dio) dio 632 fs/direct-io.c struct super_block *sb = dio->inode->i_sb; dio 634 fs/direct-io.c if (dio->defer_completion) dio 636 fs/direct-io.c dio->defer_completion = true; dio 665 fs/direct-io.c static int get_more_blocks(struct dio *dio, struct dio_submit *sdio, dio 680 fs/direct-io.c ret = dio->page_errors; dio 702 fs/direct-io.c create = dio->op == REQ_OP_WRITE; dio 703 fs/direct-io.c if (dio->flags & DIO_SKIP_HOLES) { dio 704 fs/direct-io.c i_size = i_size_read(dio->inode); dio 709 fs/direct-io.c ret = (*sdio->get_block)(dio->inode, fs_startblk, dio 713 fs/direct-io.c dio->private = map_bh->b_private; dio 716 fs/direct-io.c ret = dio_set_defer_completion(dio); dio 724 fs/direct-io.c static inline int dio_new_bio(struct dio *dio, struct dio_submit *sdio, dio 730 fs/direct-io.c ret = dio_bio_reap(dio, sdio); dio 736 fs/direct-io.c dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages); dio 781 fs/direct-io.c static inline int dio_send_cur_page(struct dio *dio, struct dio_submit *sdio, dio 807 fs/direct-io.c dio_bio_submit(dio, sdio); dio 811 fs/direct-io.c ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh); dio 817 fs/direct-io.c dio_bio_submit(dio, sdio); dio 818 fs/direct-io.c ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh); dio 846 fs/direct-io.c submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page, dio 852 fs/direct-io.c if (dio->op == REQ_OP_WRITE) { dio 874 fs/direct-io.c ret = dio_send_cur_page(dio, sdio, map_bh); dio 893 fs/direct-io.c ret = dio_send_cur_page(dio, sdio, map_bh); dio 895 fs/direct-io.c dio_bio_submit(dio, sdio); dio 911 fs/direct-io.c static inline void dio_zero_block(struct dio *dio, struct dio_submit *sdio, dio 939 fs/direct-io.c if (submit_page_section(dio, sdio, page, 0, this_chunk_bytes, dio 962 fs/direct-io.c static int do_direct_IO(struct dio *dio, struct dio_submit *sdio, dio 973 fs/direct-io.c page = dio_get_page(dio, sdio); dio 994 fs/direct-io.c ret = get_more_blocks(dio, sdio, map_bh); dio 1040 fs/direct-io.c if (dio->op == REQ_OP_WRITE) { dio 1049 fs/direct-io.c i_size_aligned = ALIGN(i_size_read(dio->inode), dio 1060 fs/direct-io.c dio->result += 1 << blkbits; dio 1070 fs/direct-io.c dio_zero_block(dio, sdio, 0, map_bh); dio 1088 fs/direct-io.c ret = submit_page_section(dio, sdio, page, dio 1101 fs/direct-io.c dio->result += this_chunk_bytes; dio 1116 fs/direct-io.c static inline int drop_refcount(struct dio *dio) dio 1132 fs/direct-io.c spin_lock_irqsave(&dio->bio_lock, flags); dio 1133 fs/direct-io.c ret2 = --dio->refcount; dio 1134 fs/direct-io.c spin_unlock_irqrestore(&dio->bio_lock, flags); dio 1176 fs/direct-io.c struct dio *dio; dio 1199 fs/direct-io.c dio = kmem_cache_alloc(dio_cache, GFP_KERNEL); dio 1201 fs/direct-io.c if (!dio) dio 1208 fs/direct-io.c memset(dio, 0, offsetof(struct dio, pages)); dio 1210 fs/direct-io.c dio->flags = flags; dio 1211 fs/direct-io.c if (dio->flags & DIO_LOCKING) { dio 1223 fs/direct-io.c kmem_cache_free(dio_cache, dio); dio 1230 fs/direct-io.c dio->i_size = i_size_read(inode); dio 1231 fs/direct-io.c if (iov_iter_rw(iter) == READ && offset >= dio->i_size) { dio 1232 fs/direct-io.c if (dio->flags & DIO_LOCKING) dio 1234 fs/direct-io.c kmem_cache_free(dio_cache, dio); dio 1246 fs/direct-io.c dio->is_async = false; dio 1248 fs/direct-io.c dio->is_async = false; dio 1250 fs/direct-io.c dio->is_async = true; dio 1252 fs/direct-io.c dio->inode = inode; dio 1254 fs/direct-io.c dio->op = REQ_OP_WRITE; dio 1255 fs/direct-io.c dio->op_flags = REQ_SYNC | REQ_IDLE; dio 1257 fs/direct-io.c dio->op_flags |= REQ_NOWAIT; dio 1259 fs/direct-io.c dio->op = REQ_OP_READ; dio 1262 fs/direct-io.c dio->op_flags |= REQ_HIPRI; dio 1268 fs/direct-io.c if (dio->is_async && iov_iter_rw(iter) == WRITE) { dio 1271 fs/direct-io.c retval = dio_set_defer_completion(dio); dio 1272 fs/direct-io.c else if (!dio->inode->i_sb->s_dio_done_wq) { dio 1278 fs/direct-io.c retval = sb_init_dio_done_wq(dio->inode->i_sb); dio 1285 fs/direct-io.c kmem_cache_free(dio_cache, dio); dio 1301 fs/direct-io.c dio->end_io = end_io; dio 1306 fs/direct-io.c dio->iocb = iocb; dio 1308 fs/direct-io.c spin_lock_init(&dio->bio_lock); dio 1309 fs/direct-io.c dio->refcount = 1; dio 1311 fs/direct-io.c dio->should_dirty = iter_is_iovec(iter) && iov_iter_rw(iter) == READ; dio 1326 fs/direct-io.c retval = do_direct_IO(dio, &sdio, &map_bh); dio 1328 fs/direct-io.c dio_cleanup(dio, &sdio); dio 1341 fs/direct-io.c dio_zero_block(dio, &sdio, 1, &map_bh); dio 1346 fs/direct-io.c ret2 = dio_send_cur_page(dio, &sdio, &map_bh); dio 1353 fs/direct-io.c dio_bio_submit(dio, &sdio); dio 1361 fs/direct-io.c dio_cleanup(dio, &sdio); dio 1368 fs/direct-io.c if (iov_iter_rw(iter) == READ && (dio->flags & DIO_LOCKING)) dio 1369 fs/direct-io.c inode_unlock(dio->inode); dio 1379 fs/direct-io.c if (dio->is_async && retval == 0 && dio->result && dio 1380 fs/direct-io.c (iov_iter_rw(iter) == READ || dio->result == count)) dio 1383 fs/direct-io.c dio_await_completion(dio); dio 1385 fs/direct-io.c if (drop_refcount(dio) == 0) { dio 1386 fs/direct-io.c retval = dio_complete(dio, retval, DIO_COMPLETE_INVALIDATE); dio 1420 fs/direct-io.c dio_cache = KMEM_CACHE(dio, SLAB_PANIC); dio 2769 fs/f2fs/data.c struct f2fs_private_dio *dio = bio->bi_private; dio 2771 fs/f2fs/data.c dec_page_count(F2FS_I_SB(dio->inode), dio 2772 fs/f2fs/data.c dio->write ? F2FS_DIO_WRITE : F2FS_DIO_READ); dio 2774 fs/f2fs/data.c bio->bi_private = dio->orig_private; dio 2775 fs/f2fs/data.c bio->bi_end_io = dio->orig_end_io; dio 2777 fs/f2fs/data.c kvfree(dio); dio 2785 fs/f2fs/data.c struct f2fs_private_dio *dio; dio 2788 fs/f2fs/data.c dio = f2fs_kzalloc(F2FS_I_SB(inode), dio 2790 fs/f2fs/data.c if (!dio) dio 2793 fs/f2fs/data.c dio->inode = inode; dio 2794 fs/f2fs/data.c dio->orig_end_io = bio->bi_end_io; dio 2795 fs/f2fs/data.c dio->orig_private = bio->bi_private; dio 2796 fs/f2fs/data.c dio->write = write; dio 2799 fs/f2fs/data.c bio->bi_private = dio; dio 61 fs/iomap/direct-io.c static void iomap_dio_submit_bio(struct iomap_dio *dio, struct iomap *iomap, dio 64 fs/iomap/direct-io.c atomic_inc(&dio->ref); dio 66 fs/iomap/direct-io.c if (dio->iocb->ki_flags & IOCB_HIPRI) dio 67 fs/iomap/direct-io.c bio_set_polled(bio, dio->iocb); dio 69 fs/iomap/direct-io.c dio->submit.last_queue = bdev_get_queue(iomap->bdev); dio 70 fs/iomap/direct-io.c dio->submit.cookie = submit_bio(bio); dio 73 fs/iomap/direct-io.c static ssize_t iomap_dio_complete(struct iomap_dio *dio) dio 75 fs/iomap/direct-io.c const struct iomap_dio_ops *dops = dio->dops; dio 76 fs/iomap/direct-io.c struct kiocb *iocb = dio->iocb; dio 79 fs/iomap/direct-io.c ssize_t ret = dio->error; dio 82 fs/iomap/direct-io.c ret = dops->end_io(iocb, dio->size, ret, dio->flags); dio 85 fs/iomap/direct-io.c ret = dio->size; dio 87 fs/iomap/direct-io.c if (offset + ret > dio->i_size && dio 88 fs/iomap/direct-io.c !(dio->flags & IOMAP_DIO_WRITE)) dio 89 fs/iomap/direct-io.c ret = dio->i_size - offset; dio 105 fs/iomap/direct-io.c if (!dio->error && dio 106 fs/iomap/direct-io.c (dio->flags & IOMAP_DIO_WRITE) && inode->i_mapping->nrpages) { dio 110 fs/iomap/direct-io.c (offset + dio->size - 1) >> PAGE_SHIFT); dio 119 fs/iomap/direct-io.c if (ret > 0 && (dio->flags & IOMAP_DIO_NEED_SYNC)) dio 123 fs/iomap/direct-io.c kfree(dio); dio 130 fs/iomap/direct-io.c struct iomap_dio *dio = container_of(work, struct iomap_dio, aio.work); dio 131 fs/iomap/direct-io.c struct kiocb *iocb = dio->iocb; dio 133 fs/iomap/direct-io.c iocb->ki_complete(iocb, iomap_dio_complete(dio), 0); dio 141 fs/iomap/direct-io.c static inline void iomap_dio_set_error(struct iomap_dio *dio, int ret) dio 143 fs/iomap/direct-io.c cmpxchg(&dio->error, 0, ret); dio 148 fs/iomap/direct-io.c struct iomap_dio *dio = bio->bi_private; dio 149 fs/iomap/direct-io.c bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY); dio 152 fs/iomap/direct-io.c iomap_dio_set_error(dio, blk_status_to_errno(bio->bi_status)); dio 154 fs/iomap/direct-io.c if (atomic_dec_and_test(&dio->ref)) { dio 155 fs/iomap/direct-io.c if (dio->wait_for_completion) { dio 156 fs/iomap/direct-io.c struct task_struct *waiter = dio->submit.waiter; dio 157 fs/iomap/direct-io.c WRITE_ONCE(dio->submit.waiter, NULL); dio 159 fs/iomap/direct-io.c } else if (dio->flags & IOMAP_DIO_WRITE) { dio 160 fs/iomap/direct-io.c struct inode *inode = file_inode(dio->iocb->ki_filp); dio 162 fs/iomap/direct-io.c INIT_WORK(&dio->aio.work, iomap_dio_complete_work); dio 163 fs/iomap/direct-io.c queue_work(inode->i_sb->s_dio_done_wq, &dio->aio.work); dio 165 fs/iomap/direct-io.c iomap_dio_complete_work(&dio->aio.work); dio 178 fs/iomap/direct-io.c iomap_dio_zero(struct iomap_dio *dio, struct iomap *iomap, loff_t pos, dio 188 fs/iomap/direct-io.c bio->bi_private = dio; dio 194 fs/iomap/direct-io.c iomap_dio_submit_bio(dio, iomap, bio); dio 199 fs/iomap/direct-io.c struct iomap_dio *dio, struct iomap *iomap) dio 203 fs/iomap/direct-io.c unsigned int align = iov_iter_alignment(dio->submit.iter); dio 215 fs/iomap/direct-io.c dio->flags |= IOMAP_DIO_UNWRITTEN; dio 220 fs/iomap/direct-io.c dio->flags |= IOMAP_DIO_COW; dio 233 fs/iomap/direct-io.c (dio->flags & IOMAP_DIO_WRITE_FUA) && dio 242 fs/iomap/direct-io.c iter = *dio->submit.iter; dio 253 fs/iomap/direct-io.c iomap_dio_zero(dio, iomap, pos - pad, pad); dio 258 fs/iomap/direct-io.c if (dio->error) { dio 259 fs/iomap/direct-io.c iov_iter_revert(dio->submit.iter, copied); dio 266 fs/iomap/direct-io.c bio->bi_write_hint = dio->iocb->ki_hint; dio 267 fs/iomap/direct-io.c bio->bi_ioprio = dio->iocb->ki_ioprio; dio 268 fs/iomap/direct-io.c bio->bi_private = dio; dio 284 fs/iomap/direct-io.c if (dio->flags & IOMAP_DIO_WRITE) { dio 289 fs/iomap/direct-io.c dio->flags &= ~IOMAP_DIO_WRITE_FUA; dio 293 fs/iomap/direct-io.c if (dio->flags & IOMAP_DIO_DIRTY) dio 297 fs/iomap/direct-io.c iov_iter_advance(dio->submit.iter, n); dio 299 fs/iomap/direct-io.c dio->size += n; dio 304 fs/iomap/direct-io.c iomap_dio_submit_bio(dio, iomap, bio); dio 315 fs/iomap/direct-io.c ((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode))) { dio 319 fs/iomap/direct-io.c iomap_dio_zero(dio, iomap, pos, fs_block_size - pad); dio 327 fs/iomap/direct-io.c iomap_dio_hole_actor(loff_t length, struct iomap_dio *dio) dio 329 fs/iomap/direct-io.c length = iov_iter_zero(length, dio->submit.iter); dio 330 fs/iomap/direct-io.c dio->size += length; dio 336 fs/iomap/direct-io.c struct iomap_dio *dio, struct iomap *iomap) dio 338 fs/iomap/direct-io.c struct iov_iter *iter = dio->submit.iter; dio 343 fs/iomap/direct-io.c if (dio->flags & IOMAP_DIO_WRITE) { dio 357 fs/iomap/direct-io.c dio->size += copied; dio 365 fs/iomap/direct-io.c struct iomap_dio *dio = data; dio 369 fs/iomap/direct-io.c if (WARN_ON_ONCE(dio->flags & IOMAP_DIO_WRITE)) dio 371 fs/iomap/direct-io.c return iomap_dio_hole_actor(length, dio); dio 373 fs/iomap/direct-io.c if (!(dio->flags & IOMAP_DIO_WRITE)) dio 374 fs/iomap/direct-io.c return iomap_dio_hole_actor(length, dio); dio 375 fs/iomap/direct-io.c return iomap_dio_bio_actor(inode, pos, length, dio, iomap); dio 377 fs/iomap/direct-io.c return iomap_dio_bio_actor(inode, pos, length, dio, iomap); dio 379 fs/iomap/direct-io.c return iomap_dio_inline_actor(inode, pos, length, dio, iomap); dio 407 fs/iomap/direct-io.c struct iomap_dio *dio; dio 414 fs/iomap/direct-io.c dio = kmalloc(sizeof(*dio), GFP_KERNEL); dio 415 fs/iomap/direct-io.c if (!dio) dio 418 fs/iomap/direct-io.c dio->iocb = iocb; dio 419 fs/iomap/direct-io.c atomic_set(&dio->ref, 1); dio 420 fs/iomap/direct-io.c dio->size = 0; dio 421 fs/iomap/direct-io.c dio->i_size = i_size_read(inode); dio 422 fs/iomap/direct-io.c dio->dops = dops; dio 423 fs/iomap/direct-io.c dio->error = 0; dio 424 fs/iomap/direct-io.c dio->flags = 0; dio 426 fs/iomap/direct-io.c dio->submit.iter = iter; dio 427 fs/iomap/direct-io.c dio->submit.waiter = current; dio 428 fs/iomap/direct-io.c dio->submit.cookie = BLK_QC_T_NONE; dio 429 fs/iomap/direct-io.c dio->submit.last_queue = NULL; dio 432 fs/iomap/direct-io.c if (pos >= dio->i_size) dio 436 fs/iomap/direct-io.c dio->flags |= IOMAP_DIO_DIRTY; dio 439 fs/iomap/direct-io.c dio->flags |= IOMAP_DIO_WRITE; dio 443 fs/iomap/direct-io.c dio->flags |= IOMAP_DIO_NEED_SYNC; dio 452 fs/iomap/direct-io.c dio->flags |= IOMAP_DIO_WRITE_FUA; dio 490 fs/iomap/direct-io.c ret = iomap_apply(inode, pos, count, flags, ops, dio, dio 502 fs/iomap/direct-io.c if (iov_iter_rw(iter) == READ && pos >= dio->i_size) { dio 508 fs/iomap/direct-io.c iov_iter_revert(iter, pos - dio->i_size); dio 515 fs/iomap/direct-io.c iomap_dio_set_error(dio, ret); dio 521 fs/iomap/direct-io.c if (dio->flags & IOMAP_DIO_WRITE_FUA) dio 522 fs/iomap/direct-io.c dio->flags &= ~IOMAP_DIO_NEED_SYNC; dio 524 fs/iomap/direct-io.c WRITE_ONCE(iocb->ki_cookie, dio->submit.cookie); dio 525 fs/iomap/direct-io.c WRITE_ONCE(iocb->private, dio->submit.last_queue); dio 542 fs/iomap/direct-io.c dio->wait_for_completion = wait_for_completion; dio 543 fs/iomap/direct-io.c if (!atomic_dec_and_test(&dio->ref)) { dio 549 fs/iomap/direct-io.c if (!READ_ONCE(dio->submit.waiter)) dio 553 fs/iomap/direct-io.c !dio->submit.last_queue || dio 554 fs/iomap/direct-io.c !blk_poll(dio->submit.last_queue, dio 555 fs/iomap/direct-io.c dio->submit.cookie, true)) dio 561 fs/iomap/direct-io.c return iomap_dio_complete(dio); dio 564 fs/iomap/direct-io.c kfree(dio); dio 70 fs/ocfs2/namei.c bool dio); dio 79 fs/ocfs2/namei.c bool dio); dio 2120 fs/ocfs2/namei.c bool dio) dio 2124 fs/ocfs2/namei.c int namelen = dio ? dio 2128 fs/ocfs2/namei.c if (dio) { dio 2176 fs/ocfs2/namei.c bool dio) dio 2190 fs/ocfs2/namei.c blkno, name, lookup, dio); dio 2219 fs/ocfs2/namei.c bool dio) dio 2225 fs/ocfs2/namei.c int namelen = dio ? dio 2279 fs/ocfs2/namei.c if (dio) { dio 2319 fs/ocfs2/namei.c bool dio) dio 2326 fs/ocfs2/namei.c if (dio) { dio 27 fs/ocfs2/namei.h bool dio);