48 static void pmem_do_bvec(struct pmem_device *pmem, struct page *page,  in pmem_do_bvec()  argument
53 	phys_addr_t pmem_off = sector * 512 + pmem->data_offset;  in pmem_do_bvec()
54 	void __pmem *pmem_addr = pmem->virt_addr + pmem_off;  in pmem_do_bvec()
74 	struct pmem_device *pmem = bdev->bd_disk->private_data;  in pmem_make_request()  local
78 		pmem_do_bvec(pmem, bvec.bv_page, bvec.bv_len, bvec.bv_offset,  in pmem_make_request()
93 	struct pmem_device *pmem = bdev->bd_disk->private_data;  in pmem_rw_page()  local
95 	pmem_do_bvec(pmem, page, PAGE_CACHE_SIZE, 0, rw, sector);  in pmem_rw_page()
106 	struct pmem_device *pmem = bdev->bd_disk->private_data;  in pmem_direct_access()  local
107 	resource_size_t offset = sector * 512 + pmem->data_offset;  in pmem_direct_access()
109 	*kaddr = pmem->virt_addr + offset;  in pmem_direct_access()
110 	*pfn = (pmem->phys_addr + offset) >> PAGE_SHIFT;  in pmem_direct_access()
112 	return pmem->size - offset;  in pmem_direct_access()
125 	struct pmem_device *pmem;  in pmem_alloc()  local
127 	pmem = devm_kzalloc(dev, sizeof(*pmem), GFP_KERNEL);  in pmem_alloc()
128 	if (!pmem)  in pmem_alloc()
131 	pmem->phys_addr = res->start;  in pmem_alloc()
132 	pmem->size = resource_size(res);  in pmem_alloc()
136 	if (!devm_request_mem_region(dev, pmem->phys_addr, pmem->size,  in pmem_alloc()
139 				&pmem->phys_addr, pmem->size);  in pmem_alloc()
144 		pmem->virt_addr = (void __pmem *) devm_memremap_pages(dev, res);  in pmem_alloc()
146 		pmem->virt_addr = (void __pmem *) devm_memremap(dev,  in pmem_alloc()
147 				pmem->phys_addr, pmem->size,  in pmem_alloc()
150 	if (IS_ERR(pmem->virt_addr))  in pmem_alloc()
151 		return (void __force *) pmem->virt_addr;  in pmem_alloc()
153 	return pmem;  in pmem_alloc()
156 static void pmem_detach_disk(struct pmem_device *pmem)  in pmem_detach_disk()  argument
158 	if (!pmem->pmem_disk)  in pmem_detach_disk()
161 	del_gendisk(pmem->pmem_disk);  in pmem_detach_disk()
162 	put_disk(pmem->pmem_disk);  in pmem_detach_disk()
163 	blk_cleanup_queue(pmem->pmem_queue);  in pmem_detach_disk()
167 		struct nd_namespace_common *ndns, struct pmem_device *pmem)  in pmem_attach_disk()  argument
172 	pmem->pmem_queue = blk_alloc_queue_node(GFP_KERNEL, nid);  in pmem_attach_disk()
173 	if (!pmem->pmem_queue)  in pmem_attach_disk()
176 	blk_queue_make_request(pmem->pmem_queue, pmem_make_request);  in pmem_attach_disk()
177 	blk_queue_physical_block_size(pmem->pmem_queue, PAGE_SIZE);  in pmem_attach_disk()
178 	blk_queue_max_hw_sectors(pmem->pmem_queue, UINT_MAX);  in pmem_attach_disk()
179 	blk_queue_bounce_limit(pmem->pmem_queue, BLK_BOUNCE_ANY);  in pmem_attach_disk()
180 	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, pmem->pmem_queue);  in pmem_attach_disk()
184 		blk_cleanup_queue(pmem->pmem_queue);  in pmem_attach_disk()
191 	disk->private_data	= pmem;  in pmem_attach_disk()
192 	disk->queue		= pmem->pmem_queue;  in pmem_attach_disk()
196 	set_capacity(disk, (pmem->size - pmem->data_offset) / 512);  in pmem_attach_disk()
197 	pmem->pmem_disk = disk;  in pmem_attach_disk()
208 	struct pmem_device *pmem = dev_get_drvdata(ndns->claim);  in pmem_rw_bytes()  local
210 	if (unlikely(offset + size > pmem->size)) {  in pmem_rw_bytes()
216 		memcpy_from_pmem(buf, pmem->virt_addr + offset, size);  in pmem_rw_bytes()
218 		memcpy_to_pmem(pmem->virt_addr + offset, buf, size);  in pmem_rw_bytes()
228 	struct pmem_device *pmem = dev_get_drvdata(&nd_pfn->dev);  in nd_pfn_init()  local
246 			|| pmem->phys_addr & ND_PFN_MASK)  in nd_pfn_init()
258 	npfns = (pmem->size - SZ_8K) / SZ_4K;  in nd_pfn_init()
272 	npfns = (pmem->size - offset) / SZ_4K;  in nd_pfn_init()
296 	struct pmem_device *pmem;  in nvdimm_namespace_detach_pfn()  local
299 	pmem = dev_get_drvdata(&nd_pfn->dev);  in nvdimm_namespace_detach_pfn()
300 	pmem_detach_disk(pmem);  in nvdimm_namespace_detach_pfn()
317 	struct pmem_device *pmem;  in nvdimm_namespace_attach_pfn()  local
353 	pmem = dev_get_drvdata(dev);  in nvdimm_namespace_attach_pfn()
354 	devm_memunmap(dev, (void __force *) pmem->virt_addr);  in nvdimm_namespace_attach_pfn()
355 	pmem->virt_addr = (void __pmem *) devm_memremap_pages(dev, &nsio->res);  in nvdimm_namespace_attach_pfn()
356 	if (IS_ERR(pmem->virt_addr)) {  in nvdimm_namespace_attach_pfn()
357 		rc = PTR_ERR(pmem->virt_addr);  in nvdimm_namespace_attach_pfn()
362 	pmem->data_offset = offset;  in nvdimm_namespace_attach_pfn()
363 	rc = pmem_attach_disk(dev, ndns, pmem);  in nvdimm_namespace_attach_pfn()
378 	struct pmem_device *pmem;  in nd_pmem_probe()  local
385 	pmem = pmem_alloc(dev, &nsio->res, nd_region->id);  in nd_pmem_probe()
386 	if (IS_ERR(pmem))  in nd_pmem_probe()
387 		return PTR_ERR(pmem);  in nd_pmem_probe()
389 	pmem->ndns = ndns;  in nd_pmem_probe()
390 	dev_set_drvdata(dev, pmem);  in nd_pmem_probe()
399 	if (nd_btt_probe(ndns, pmem) == 0) {  in nd_pmem_probe()
404 	if (nd_pfn_probe(ndns, pmem) == 0) {  in nd_pmem_probe()
409 	return pmem_attach_disk(dev, ndns, pmem);  in nd_pmem_probe()
414 	struct pmem_device *pmem = dev_get_drvdata(dev);  in nd_pmem_remove()  local
417 		nvdimm_namespace_detach_btt(pmem->ndns);  in nd_pmem_remove()
419 		nvdimm_namespace_detach_pfn(pmem->ndns);  in nd_pmem_remove()
421 		pmem_detach_disk(pmem);  in nd_pmem_remove()