nullb 39 drivers/block/null_blk.h struct nullb *nullb; nullb 198 drivers/block/null_blk_main.c static void null_del_dev(struct nullb *nullb); nullb 333 drivers/block/null_blk_main.c null_del_dev(dev->nullb); nullb 461 drivers/block/null_blk_main.c null_del_dev(dev->nullb); nullb 500 drivers/block/null_blk_main.c static inline int null_cache_active(struct nullb *nullb) nullb 502 drivers/block/null_blk_main.c return test_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags); nullb 689 drivers/block/null_blk_main.c static void null_free_sector(struct nullb *nullb, sector_t sector, nullb 697 drivers/block/null_blk_main.c root = is_cache ? &nullb->dev->cache : &nullb->dev->data; nullb 710 drivers/block/null_blk_main.c nullb->dev->curr_cache -= PAGE_SIZE; nullb 715 drivers/block/null_blk_main.c static struct nullb_page *null_radix_tree_insert(struct nullb *nullb, u64 idx, nullb 720 drivers/block/null_blk_main.c root = is_cache ? &nullb->dev->cache : &nullb->dev->data; nullb 727 drivers/block/null_blk_main.c nullb->dev->curr_cache += PAGE_SIZE; nullb 761 drivers/block/null_blk_main.c static struct nullb_page *__null_lookup_page(struct nullb *nullb, nullb 772 drivers/block/null_blk_main.c root = is_cache ? &nullb->dev->cache : &nullb->dev->data; nullb 782 drivers/block/null_blk_main.c static struct nullb_page *null_lookup_page(struct nullb *nullb, nullb 788 drivers/block/null_blk_main.c page = __null_lookup_page(nullb, sector, for_write, true); nullb 791 drivers/block/null_blk_main.c return __null_lookup_page(nullb, sector, for_write, false); nullb 794 drivers/block/null_blk_main.c static struct nullb_page *null_insert_page(struct nullb *nullb, nullb 796 drivers/block/null_blk_main.c __releases(&nullb->lock) nullb 797 drivers/block/null_blk_main.c __acquires(&nullb->lock) nullb 802 drivers/block/null_blk_main.c t_page = null_lookup_page(nullb, sector, true, ignore_cache); nullb 806 drivers/block/null_blk_main.c spin_unlock_irq(&nullb->lock); nullb 815 drivers/block/null_blk_main.c spin_lock_irq(&nullb->lock); nullb 818 drivers/block/null_blk_main.c t_page = null_radix_tree_insert(nullb, idx, t_page, !ignore_cache); nullb 825 drivers/block/null_blk_main.c spin_lock_irq(&nullb->lock); nullb 826 drivers/block/null_blk_main.c return null_lookup_page(nullb, sector, true, ignore_cache); nullb 829 drivers/block/null_blk_main.c static int null_flush_cache_page(struct nullb *nullb, struct nullb_page *c_page) nullb 839 drivers/block/null_blk_main.c t_page = null_insert_page(nullb, idx << PAGE_SECTORS_SHIFT, true); nullb 845 drivers/block/null_blk_main.c ret = radix_tree_delete_item(&nullb->dev->data, nullb 859 drivers/block/null_blk_main.c i += (nullb->dev->blocksize >> SECTOR_SHIFT)) { nullb 863 drivers/block/null_blk_main.c nullb->dev->blocksize); nullb 871 drivers/block/null_blk_main.c ret = radix_tree_delete_item(&nullb->dev->cache, idx, c_page); nullb 873 drivers/block/null_blk_main.c nullb->dev->curr_cache -= PAGE_SIZE; nullb 878 drivers/block/null_blk_main.c static int null_make_cache_space(struct nullb *nullb, unsigned long n) nullb 885 drivers/block/null_blk_main.c if ((nullb->dev->cache_size * 1024 * 1024) > nullb 886 drivers/block/null_blk_main.c nullb->dev->curr_cache + n || nullb->dev->curr_cache == 0) nullb 889 drivers/block/null_blk_main.c nr_pages = radix_tree_gang_lookup(&nullb->dev->cache, nullb 890 drivers/block/null_blk_main.c (void **)c_pages, nullb->cache_flush_pos, FREE_BATCH); nullb 896 drivers/block/null_blk_main.c nullb->cache_flush_pos = c_pages[i]->page->index; nullb 911 drivers/block/null_blk_main.c err = null_flush_cache_page(nullb, c_pages[i]); nullb 920 drivers/block/null_blk_main.c nullb->cache_flush_pos = 0; nullb 923 drivers/block/null_blk_main.c spin_unlock_irq(&nullb->lock); nullb 924 drivers/block/null_blk_main.c spin_lock_irq(&nullb->lock); nullb 931 drivers/block/null_blk_main.c static int copy_to_nullb(struct nullb *nullb, struct page *source, nullb 940 drivers/block/null_blk_main.c temp = min_t(size_t, nullb->dev->blocksize, n - count); nullb 942 drivers/block/null_blk_main.c if (null_cache_active(nullb) && !is_fua) nullb 943 drivers/block/null_blk_main.c null_make_cache_space(nullb, PAGE_SIZE); nullb 946 drivers/block/null_blk_main.c t_page = null_insert_page(nullb, sector, nullb 947 drivers/block/null_blk_main.c !null_cache_active(nullb) || is_fua); nullb 960 drivers/block/null_blk_main.c null_free_sector(nullb, sector, true); nullb 968 drivers/block/null_blk_main.c static int copy_from_nullb(struct nullb *nullb, struct page *dest, nullb 977 drivers/block/null_blk_main.c temp = min_t(size_t, nullb->dev->blocksize, n - count); nullb 980 drivers/block/null_blk_main.c t_page = null_lookup_page(nullb, sector, false, nullb 981 drivers/block/null_blk_main.c !null_cache_active(nullb)); nullb 1000 drivers/block/null_blk_main.c static void null_handle_discard(struct nullb *nullb, sector_t sector, size_t n) nullb 1004 drivers/block/null_blk_main.c spin_lock_irq(&nullb->lock); nullb 1006 drivers/block/null_blk_main.c temp = min_t(size_t, n, nullb->dev->blocksize); nullb 1007 drivers/block/null_blk_main.c null_free_sector(nullb, sector, false); nullb 1008 drivers/block/null_blk_main.c if (null_cache_active(nullb)) nullb 1009 drivers/block/null_blk_main.c null_free_sector(nullb, sector, true); nullb 1013 drivers/block/null_blk_main.c spin_unlock_irq(&nullb->lock); nullb 1016 drivers/block/null_blk_main.c static int null_handle_flush(struct nullb *nullb) nullb 1020 drivers/block/null_blk_main.c if (!null_cache_active(nullb)) nullb 1023 drivers/block/null_blk_main.c spin_lock_irq(&nullb->lock); nullb 1025 drivers/block/null_blk_main.c err = null_make_cache_space(nullb, nullb 1026 drivers/block/null_blk_main.c nullb->dev->cache_size * 1024 * 1024); nullb 1027 drivers/block/null_blk_main.c if (err || nullb->dev->curr_cache == 0) nullb 1031 drivers/block/null_blk_main.c WARN_ON(!radix_tree_empty(&nullb->dev->cache)); nullb 1032 drivers/block/null_blk_main.c spin_unlock_irq(&nullb->lock); nullb 1036 drivers/block/null_blk_main.c static int null_transfer(struct nullb *nullb, struct page *page, nullb 1043 drivers/block/null_blk_main.c err = copy_from_nullb(nullb, page, off, sector, len); nullb 1047 drivers/block/null_blk_main.c err = copy_to_nullb(nullb, page, off, sector, len, is_fua); nullb 1056 drivers/block/null_blk_main.c struct nullb *nullb = cmd->nq->dev->nullb; nullb 1066 drivers/block/null_blk_main.c null_handle_discard(nullb, sector, blk_rq_bytes(rq)); nullb 1070 drivers/block/null_blk_main.c spin_lock_irq(&nullb->lock); nullb 1073 drivers/block/null_blk_main.c err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset, nullb 1077 drivers/block/null_blk_main.c spin_unlock_irq(&nullb->lock); nullb 1082 drivers/block/null_blk_main.c spin_unlock_irq(&nullb->lock); nullb 1090 drivers/block/null_blk_main.c struct nullb *nullb = cmd->nq->dev->nullb; nullb 1100 drivers/block/null_blk_main.c null_handle_discard(nullb, sector, nullb 1105 drivers/block/null_blk_main.c spin_lock_irq(&nullb->lock); nullb 1108 drivers/block/null_blk_main.c err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset, nullb 1112 drivers/block/null_blk_main.c spin_unlock_irq(&nullb->lock); nullb 1117 drivers/block/null_blk_main.c spin_unlock_irq(&nullb->lock); nullb 1121 drivers/block/null_blk_main.c static void null_stop_queue(struct nullb *nullb) nullb 1123 drivers/block/null_blk_main.c struct request_queue *q = nullb->q; nullb 1125 drivers/block/null_blk_main.c if (nullb->dev->queue_mode == NULL_Q_MQ) nullb 1129 drivers/block/null_blk_main.c static void null_restart_queue_async(struct nullb *nullb) nullb 1131 drivers/block/null_blk_main.c struct request_queue *q = nullb->q; nullb 1133 drivers/block/null_blk_main.c if (nullb->dev->queue_mode == NULL_Q_MQ) nullb 1140 drivers/block/null_blk_main.c struct nullb *nullb = dev->nullb; nullb 1144 drivers/block/null_blk_main.c if (!hrtimer_active(&nullb->bw_timer)) nullb 1145 drivers/block/null_blk_main.c hrtimer_restart(&nullb->bw_timer); nullb 1147 drivers/block/null_blk_main.c if (atomic_long_sub_return(blk_rq_bytes(rq), &nullb->cur_bytes) < 0) { nullb 1148 drivers/block/null_blk_main.c null_stop_queue(nullb); nullb 1150 drivers/block/null_blk_main.c if (atomic_long_read(&nullb->cur_bytes) > 0) nullb 1151 drivers/block/null_blk_main.c null_restart_queue_async(nullb); nullb 1216 drivers/block/null_blk_main.c struct nullb *nullb = dev->nullb; nullb 1226 drivers/block/null_blk_main.c cmd->error = errno_to_blk_status(null_handle_flush(nullb)); nullb 1230 drivers/block/null_blk_main.c if (nullb->dev->badblocks.shift != -1) { nullb 1249 drivers/block/null_blk_main.c struct nullb *nullb = container_of(timer, struct nullb, bw_timer); nullb 1251 drivers/block/null_blk_main.c unsigned int mbps = nullb->dev->mbps; nullb 1253 drivers/block/null_blk_main.c if (atomic_long_read(&nullb->cur_bytes) == mb_per_tick(mbps)) nullb 1256 drivers/block/null_blk_main.c atomic_long_set(&nullb->cur_bytes, mb_per_tick(mbps)); nullb 1257 drivers/block/null_blk_main.c null_restart_queue_async(nullb); nullb 1259 drivers/block/null_blk_main.c hrtimer_forward_now(&nullb->bw_timer, timer_interval); nullb 1264 drivers/block/null_blk_main.c static void nullb_setup_bwtimer(struct nullb *nullb) nullb 1268 drivers/block/null_blk_main.c hrtimer_init(&nullb->bw_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); nullb 1269 drivers/block/null_blk_main.c nullb->bw_timer.function = nullb_bwtimer_fn; nullb 1270 drivers/block/null_blk_main.c atomic_long_set(&nullb->cur_bytes, mb_per_tick(nullb->dev->mbps)); nullb 1271 drivers/block/null_blk_main.c hrtimer_start(&nullb->bw_timer, timer_interval, HRTIMER_MODE_REL); nullb 1274 drivers/block/null_blk_main.c static struct nullb_queue *nullb_to_queue(struct nullb *nullb) nullb 1278 drivers/block/null_blk_main.c if (nullb->nr_queues != 1) nullb 1279 drivers/block/null_blk_main.c index = raw_smp_processor_id() / ((nr_cpu_ids + nullb->nr_queues - 1) / nullb->nr_queues); nullb 1281 drivers/block/null_blk_main.c return &nullb->queues[index]; nullb 1288 drivers/block/null_blk_main.c struct nullb *nullb = q->queuedata; nullb 1289 drivers/block/null_blk_main.c struct nullb_queue *nq = nullb_to_queue(nullb); nullb 1375 drivers/block/null_blk_main.c static void cleanup_queues(struct nullb *nullb) nullb 1379 drivers/block/null_blk_main.c for (i = 0; i < nullb->nr_queues; i++) nullb 1380 drivers/block/null_blk_main.c cleanup_queue(&nullb->queues[i]); nullb 1382 drivers/block/null_blk_main.c kfree(nullb->queues); nullb 1385 drivers/block/null_blk_main.c static void null_del_dev(struct nullb *nullb) nullb 1389 drivers/block/null_blk_main.c if (!nullb) nullb 1392 drivers/block/null_blk_main.c dev = nullb->dev; nullb 1394 drivers/block/null_blk_main.c ida_simple_remove(&nullb_indexes, nullb->index); nullb 1396 drivers/block/null_blk_main.c list_del_init(&nullb->list); nullb 1398 drivers/block/null_blk_main.c del_gendisk(nullb->disk); nullb 1400 drivers/block/null_blk_main.c if (test_bit(NULLB_DEV_FL_THROTTLED, &nullb->dev->flags)) { nullb 1401 drivers/block/null_blk_main.c hrtimer_cancel(&nullb->bw_timer); nullb 1402 drivers/block/null_blk_main.c atomic_long_set(&nullb->cur_bytes, LONG_MAX); nullb 1403 drivers/block/null_blk_main.c null_restart_queue_async(nullb); nullb 1406 drivers/block/null_blk_main.c blk_cleanup_queue(nullb->q); nullb 1408 drivers/block/null_blk_main.c nullb->tag_set == &nullb->__tag_set) nullb 1409 drivers/block/null_blk_main.c blk_mq_free_tag_set(nullb->tag_set); nullb 1410 drivers/block/null_blk_main.c put_disk(nullb->disk); nullb 1411 drivers/block/null_blk_main.c cleanup_queues(nullb); nullb 1412 drivers/block/null_blk_main.c if (null_cache_active(nullb)) nullb 1413 drivers/block/null_blk_main.c null_free_device_storage(nullb->dev, true); nullb 1414 drivers/block/null_blk_main.c kfree(nullb); nullb 1415 drivers/block/null_blk_main.c dev->nullb = NULL; nullb 1418 drivers/block/null_blk_main.c static void null_config_discard(struct nullb *nullb) nullb 1420 drivers/block/null_blk_main.c if (nullb->dev->discard == false) nullb 1422 drivers/block/null_blk_main.c nullb->q->limits.discard_granularity = nullb->dev->blocksize; nullb 1423 drivers/block/null_blk_main.c nullb->q->limits.discard_alignment = nullb->dev->blocksize; nullb 1424 drivers/block/null_blk_main.c blk_queue_max_discard_sectors(nullb->q, UINT_MAX >> 9); nullb 1425 drivers/block/null_blk_main.c blk_queue_flag_set(QUEUE_FLAG_DISCARD, nullb->q); nullb 1444 drivers/block/null_blk_main.c static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq) nullb 1446 drivers/block/null_blk_main.c BUG_ON(!nullb); nullb 1450 drivers/block/null_blk_main.c nq->queue_depth = nullb->queue_depth; nullb 1451 drivers/block/null_blk_main.c nq->dev = nullb->dev; nullb 1454 drivers/block/null_blk_main.c static void null_init_queues(struct nullb *nullb) nullb 1456 drivers/block/null_blk_main.c struct request_queue *q = nullb->q; nullb 1464 drivers/block/null_blk_main.c nq = &nullb->queues[i]; nullb 1466 drivers/block/null_blk_main.c null_init_queue(nullb, nq); nullb 1467 drivers/block/null_blk_main.c nullb->nr_queues++; nullb 1497 drivers/block/null_blk_main.c static int setup_queues(struct nullb *nullb) nullb 1499 drivers/block/null_blk_main.c nullb->queues = kcalloc(nullb->dev->submit_queues, nullb 1502 drivers/block/null_blk_main.c if (!nullb->queues) nullb 1505 drivers/block/null_blk_main.c nullb->queue_depth = nullb->dev->hw_queue_depth; nullb 1510 drivers/block/null_blk_main.c static int init_driver_queues(struct nullb *nullb) nullb 1515 drivers/block/null_blk_main.c for (i = 0; i < nullb->dev->submit_queues; i++) { nullb 1516 drivers/block/null_blk_main.c nq = &nullb->queues[i]; nullb 1518 drivers/block/null_blk_main.c null_init_queue(nullb, nq); nullb 1523 drivers/block/null_blk_main.c nullb->nr_queues++; nullb 1528 drivers/block/null_blk_main.c static int null_gendisk_register(struct nullb *nullb) nullb 1533 drivers/block/null_blk_main.c disk = nullb->disk = alloc_disk_node(1, nullb->dev->home_node); nullb 1536 drivers/block/null_blk_main.c size = (sector_t)nullb->dev->size * 1024 * 1024ULL; nullb 1541 drivers/block/null_blk_main.c disk->first_minor = nullb->index; nullb 1543 drivers/block/null_blk_main.c disk->private_data = nullb; nullb 1544 drivers/block/null_blk_main.c disk->queue = nullb->q; nullb 1545 drivers/block/null_blk_main.c strncpy(disk->disk_name, nullb->disk_name, DISK_NAME_LEN); nullb 1547 drivers/block/null_blk_main.c if (nullb->dev->zoned) { nullb 1558 drivers/block/null_blk_main.c static int null_init_tag_set(struct nullb *nullb, struct blk_mq_tag_set *set) nullb 1561 drivers/block/null_blk_main.c set->nr_hw_queues = nullb ? nullb->dev->submit_queues : nullb 1563 drivers/block/null_blk_main.c set->queue_depth = nullb ? nullb->dev->hw_queue_depth : nullb 1565 drivers/block/null_blk_main.c set->numa_node = nullb ? nullb->dev->home_node : g_home_node; nullb 1572 drivers/block/null_blk_main.c if ((nullb && nullb->dev->blocking) || g_blocking) nullb 1634 drivers/block/null_blk_main.c struct nullb *nullb; nullb 1639 drivers/block/null_blk_main.c nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, dev->home_node); nullb 1640 drivers/block/null_blk_main.c if (!nullb) { nullb 1644 drivers/block/null_blk_main.c nullb->dev = dev; nullb 1645 drivers/block/null_blk_main.c dev->nullb = nullb; nullb 1647 drivers/block/null_blk_main.c spin_lock_init(&nullb->lock); nullb 1649 drivers/block/null_blk_main.c rv = setup_queues(nullb); nullb 1655 drivers/block/null_blk_main.c nullb->tag_set = &tag_set; nullb 1658 drivers/block/null_blk_main.c nullb->tag_set = &nullb->__tag_set; nullb 1659 drivers/block/null_blk_main.c rv = null_init_tag_set(nullb, nullb->tag_set); nullb 1668 drivers/block/null_blk_main.c nullb->tag_set->timeout = 5 * HZ; nullb 1669 drivers/block/null_blk_main.c nullb->q = blk_mq_init_queue(nullb->tag_set); nullb 1670 drivers/block/null_blk_main.c if (IS_ERR(nullb->q)) { nullb 1674 drivers/block/null_blk_main.c null_init_queues(nullb); nullb 1676 drivers/block/null_blk_main.c nullb->q = blk_alloc_queue_node(GFP_KERNEL, dev->home_node); nullb 1677 drivers/block/null_blk_main.c if (!nullb->q) { nullb 1681 drivers/block/null_blk_main.c blk_queue_make_request(nullb->q, null_queue_bio); nullb 1682 drivers/block/null_blk_main.c rv = init_driver_queues(nullb); nullb 1689 drivers/block/null_blk_main.c nullb_setup_bwtimer(nullb); nullb 1693 drivers/block/null_blk_main.c set_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags); nullb 1694 drivers/block/null_blk_main.c blk_queue_write_cache(nullb->q, true, true); nullb 1702 drivers/block/null_blk_main.c blk_queue_chunk_sectors(nullb->q, dev->zone_size_sects); nullb 1703 drivers/block/null_blk_main.c nullb->q->limits.zoned = BLK_ZONED_HM; nullb 1704 drivers/block/null_blk_main.c blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, nullb->q); nullb 1705 drivers/block/null_blk_main.c blk_queue_required_elevator_features(nullb->q, nullb 1709 drivers/block/null_blk_main.c nullb->q->queuedata = nullb; nullb 1710 drivers/block/null_blk_main.c blk_queue_flag_set(QUEUE_FLAG_NONROT, nullb->q); nullb 1711 drivers/block/null_blk_main.c blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, nullb->q); nullb 1714 drivers/block/null_blk_main.c nullb->index = ida_simple_get(&nullb_indexes, 0, 0, GFP_KERNEL); nullb 1715 drivers/block/null_blk_main.c dev->index = nullb->index; nullb 1718 drivers/block/null_blk_main.c blk_queue_logical_block_size(nullb->q, dev->blocksize); nullb 1719 drivers/block/null_blk_main.c blk_queue_physical_block_size(nullb->q, dev->blocksize); nullb 1721 drivers/block/null_blk_main.c null_config_discard(nullb); nullb 1723 drivers/block/null_blk_main.c sprintf(nullb->disk_name, "nullb%d", nullb->index); nullb 1725 drivers/block/null_blk_main.c rv = null_gendisk_register(nullb); nullb 1730 drivers/block/null_blk_main.c list_add_tail(&nullb->list, &nullb_list); nullb 1738 drivers/block/null_blk_main.c blk_cleanup_queue(nullb->q); nullb 1740 drivers/block/null_blk_main.c if (dev->queue_mode == NULL_Q_MQ && nullb->tag_set == &nullb->__tag_set) nullb 1741 drivers/block/null_blk_main.c blk_mq_free_tag_set(nullb->tag_set); nullb 1743 drivers/block/null_blk_main.c cleanup_queues(nullb); nullb 1745 drivers/block/null_blk_main.c kfree(nullb); nullb 1746 drivers/block/null_blk_main.c dev->nullb = NULL; nullb 1755 drivers/block/null_blk_main.c struct nullb *nullb; nullb 1828 drivers/block/null_blk_main.c nullb = list_entry(nullb_list.next, struct nullb, list); nullb 1829 drivers/block/null_blk_main.c dev = nullb->dev; nullb 1830 drivers/block/null_blk_main.c null_del_dev(nullb); nullb 1844 drivers/block/null_blk_main.c struct nullb *nullb; nullb 1854 drivers/block/null_blk_main.c nullb = list_entry(nullb_list.next, struct nullb, list); nullb 1855 drivers/block/null_blk_main.c dev = nullb->dev; nullb 1856 drivers/block/null_blk_main.c null_del_dev(nullb); nullb 76 drivers/block/null_blk_zoned.c struct nullb *nullb = disk->private_data; nullb 77 drivers/block/null_blk_zoned.c struct nullb_device *dev = nullb->dev;