root/arch/x86/mm/ioremap.c

DEFINITIONS

1. ioremap_change_attr
2. __ioremap_check_ram
3. __ioremap_check_encrypted
4. __ioremap_check_other
5. __ioremap_collect_map_flags
6. __ioremap_check_mem
7. __ioremap_caller
8. ioremap_nocache
9. ioremap_uc
10. ioremap_wc
11. ioremap_wt
12. ioremap_encrypted
13. ioremap_cache
14. ioremap_prot
15. iounmap
16. arch_ioremap_p4d_supported
17. arch_ioremap_pud_supported
18. arch_ioremap_pmd_supported
19. xlate_dev_mem_ptr
20. unxlate_dev_mem_ptr
21. memremap_should_map_decrypted
22. memremap_is_efi_data
23. memremap_is_setup_data
24. early_memremap_is_setup_data
25. arch_memremap_can_ram_remap
26. early_memremap_pgprot_adjust
27. phys_mem_access_encrypted
28. early_memremap_encrypted
29. early_memremap_encrypted_wp
30. early_memremap_decrypted
31. early_memremap_decrypted_wp
32. early_ioremap_pmd
33. early_ioremap_pte
34. is_early_ioremap_ptep
35. early_ioremap_init
36. __early_set_fixmap

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Re-map IO memory to kernel address space so that we can access it.
   4  * This is needed for high PCI addresses that aren't mapped in the
   5  * 640k-1MB IO memory area on PC's
   6  *
   7  * (C) Copyright 1995 1996 Linus Torvalds
   8  */
   9 
  10 #include <linux/memblock.h>
  11 #include <linux/init.h>
  12 #include <linux/io.h>
  13 #include <linux/ioport.h>
  14 #include <linux/slab.h>
  15 #include <linux/vmalloc.h>
  16 #include <linux/mmiotrace.h>
  17 #include <linux/mem_encrypt.h>
  18 #include <linux/efi.h>
  19 
  20 #include <asm/set_memory.h>
  21 #include <asm/e820/api.h>
  22 #include <asm/efi.h>
  23 #include <asm/fixmap.h>
  24 #include <asm/pgtable.h>
  25 #include <asm/tlbflush.h>
  26 #include <asm/pgalloc.h>
  27 #include <asm/pat.h>
  28 #include <asm/setup.h>
  29 
  30 #include "physaddr.h"
  31 
  32 /*
  33  * Descriptor controlling ioremap() behavior.
  34  */
  35 struct ioremap_desc {
  36         unsigned int flags;
  37 };
  38 
  39 /*
  40  * Fix up the linear direct mapping of the kernel to avoid cache attribute
  41  * conflicts.
  42  */
  43 int ioremap_change_attr(unsigned long vaddr, unsigned long size,
  44                         enum page_cache_mode pcm)
  45 {
  46         unsigned long nrpages = size >> PAGE_SHIFT;
  47         int err;
  48 
  49         switch (pcm) {
  50         case _PAGE_CACHE_MODE_UC:
  51         default:
  52                 err = _set_memory_uc(vaddr, nrpages);
  53                 break;
  54         case _PAGE_CACHE_MODE_WC:
  55                 err = _set_memory_wc(vaddr, nrpages);
  56                 break;
  57         case _PAGE_CACHE_MODE_WT:
  58                 err = _set_memory_wt(vaddr, nrpages);
  59                 break;
  60         case _PAGE_CACHE_MODE_WB:
  61                 err = _set_memory_wb(vaddr, nrpages);
  62                 break;
  63         }
  64 
  65         return err;
  66 }
  67 
  68 /* Does the range (or a subset of) contain normal RAM? */
  69 static unsigned int __ioremap_check_ram(struct resource *res)
  70 {
  71         unsigned long start_pfn, stop_pfn;
  72         unsigned long i;
  73 
  74         if ((res->flags & IORESOURCE_SYSTEM_RAM) != IORESOURCE_SYSTEM_RAM)
  75                 return 0;
  76 
  77         start_pfn = (res->start + PAGE_SIZE - 1) >> PAGE_SHIFT;
  78         stop_pfn = (res->end + 1) >> PAGE_SHIFT;
  79         if (stop_pfn > start_pfn) {
  80                 for (i = 0; i < (stop_pfn - start_pfn); ++i)
  81                         if (pfn_valid(start_pfn + i) &&
  82                             !PageReserved(pfn_to_page(start_pfn + i)))
  83                                 return IORES_MAP_SYSTEM_RAM;
  84         }
  85 
  86         return 0;
  87 }
  88 
  89 /*
  90  * In a SEV guest, NONE and RESERVED should not be mapped encrypted because
  91  * there the whole memory is already encrypted.
  92  */
  93 static unsigned int __ioremap_check_encrypted(struct resource *res)
  94 {
  95         if (!sev_active())
  96                 return 0;
  97 
  98         switch (res->desc) {
  99         case IORES_DESC_NONE:
 100         case IORES_DESC_RESERVED:
 101                 break;
 102         default:
 103                 return IORES_MAP_ENCRYPTED;
 104         }
 105 
 106         return 0;
 107 }
 108 
 109 /*
 110  * The EFI runtime services data area is not covered by walk_mem_res(), but must
 111  * be mapped encrypted when SEV is active.
 112  */
 113 static void __ioremap_check_other(resource_size_t addr, struct ioremap_desc *desc)
 114 {
 115         if (!sev_active())
 116                 return;
 117 
 118         if (!IS_ENABLED(CONFIG_EFI))
 119                 return;
 120 
 121         if (efi_mem_type(addr) == EFI_RUNTIME_SERVICES_DATA)
 122                 desc->flags |= IORES_MAP_ENCRYPTED;
 123 }
 124 
 125 static int __ioremap_collect_map_flags(struct resource *res, void *arg)
 126 {
 127         struct ioremap_desc *desc = arg;
 128 
 129         if (!(desc->flags & IORES_MAP_SYSTEM_RAM))
 130                 desc->flags |= __ioremap_check_ram(res);
 131 
 132         if (!(desc->flags & IORES_MAP_ENCRYPTED))
 133                 desc->flags |= __ioremap_check_encrypted(res);
 134 
 135         return ((desc->flags & (IORES_MAP_SYSTEM_RAM | IORES_MAP_ENCRYPTED)) ==
 136                                (IORES_MAP_SYSTEM_RAM | IORES_MAP_ENCRYPTED));
 137 }
 138 
 139 /*
 140  * To avoid multiple resource walks, this function walks resources marked as
 141  * IORESOURCE_MEM and IORESOURCE_BUSY and looking for system RAM and/or a
 142  * resource described not as IORES_DESC_NONE (e.g. IORES_DESC_ACPI_TABLES).
 143  *
 144  * After that, deal with misc other ranges in __ioremap_check_other() which do
 145  * not fall into the above category.
 146  */
 147 static void __ioremap_check_mem(resource_size_t addr, unsigned long size,
 148                                 struct ioremap_desc *desc)
 149 {
 150         u64 start, end;
 151 
 152         start = (u64)addr;
 153         end = start + size - 1;
 154         memset(desc, 0, sizeof(struct ioremap_desc));
 155 
 156         walk_mem_res(start, end, desc, __ioremap_collect_map_flags);
 157 
 158         __ioremap_check_other(addr, desc);
 159 }
 160 
 161 /*
 162  * Remap an arbitrary physical address space into the kernel virtual
 163  * address space. It transparently creates kernel huge I/O mapping when
 164  * the physical address is aligned by a huge page size (1GB or 2MB) and
 165  * the requested size is at least the huge page size.
 166  *
 167  * NOTE: MTRRs can override PAT memory types with a 4KB granularity.
 168  * Therefore, the mapping code falls back to use a smaller page toward 4KB
 169  * when a mapping range is covered by non-WB type of MTRRs.
 170  *
 171  * NOTE! We need to allow non-page-aligned mappings too: we will obviously
 172  * have to convert them into an offset in a page-aligned mapping, but the
 173  * caller shouldn't need to know that small detail.
 174  */
 175 static void __iomem *
 176 __ioremap_caller(resource_size_t phys_addr, unsigned long size,
 177                  enum page_cache_mode pcm, void *caller, bool encrypted)
 178 {
 179         unsigned long offset, vaddr;
 180         resource_size_t last_addr;
 181         const resource_size_t unaligned_phys_addr = phys_addr;
 182         const unsigned long unaligned_size = size;
 183         struct ioremap_desc io_desc;
 184         struct vm_struct *area;
 185         enum page_cache_mode new_pcm;
 186         pgprot_t prot;
 187         int retval;
 188         void __iomem *ret_addr;
 189 
 190         /* Don't allow wraparound or zero size */
 191         last_addr = phys_addr + size - 1;
 192         if (!size || last_addr < phys_addr)
 193                 return NULL;
 194 
 195         if (!phys_addr_valid(phys_addr)) {
 196                 printk(KERN_WARNING "ioremap: invalid physical address %llx\n",
 197                        (unsigned long long)phys_addr);
 198                 WARN_ON_ONCE(1);
 199                 return NULL;
 200         }
 201 
 202         __ioremap_check_mem(phys_addr, size, &io_desc);
 203 
 204         /*
 205          * Don't allow anybody to remap normal RAM that we're using..
 206          */
 207         if (io_desc.flags & IORES_MAP_SYSTEM_RAM) {
 208                 WARN_ONCE(1, "ioremap on RAM at %pa - %pa\n",
 209                           &phys_addr, &last_addr);
 210                 return NULL;
 211         }
 212 
 213         /*
 214          * Mappings have to be page-aligned
 215          */
 216         offset = phys_addr & ~PAGE_MASK;
 217         phys_addr &= PHYSICAL_PAGE_MASK;
 218         size = PAGE_ALIGN(last_addr+1) - phys_addr;
 219 
 220         retval = reserve_memtype(phys_addr, (u64)phys_addr + size,
 221                                                 pcm, &new_pcm);
 222         if (retval) {
 223                 printk(KERN_ERR "ioremap reserve_memtype failed %d\n", retval);
 224                 return NULL;
 225         }
 226 
 227         if (pcm != new_pcm) {
 228                 if (!is_new_memtype_allowed(phys_addr, size, pcm, new_pcm)) {
 229                         printk(KERN_ERR
 230                 "ioremap error for 0x%llx-0x%llx, requested 0x%x, got 0x%x\n",
 231                                 (unsigned long long)phys_addr,
 232                                 (unsigned long long)(phys_addr + size),
 233                                 pcm, new_pcm);
 234                         goto err_free_memtype;
 235                 }
 236                 pcm = new_pcm;
 237         }
 238 
 239         /*
 240          * If the page being mapped is in memory and SEV is active then
 241          * make sure the memory encryption attribute is enabled in the
 242          * resulting mapping.
 243          */
 244         prot = PAGE_KERNEL_IO;
 245         if ((io_desc.flags & IORES_MAP_ENCRYPTED) || encrypted)
 246                 prot = pgprot_encrypted(prot);
 247 
 248         switch (pcm) {
 249         case _PAGE_CACHE_MODE_UC:
 250         default:
 251                 prot = __pgprot(pgprot_val(prot) |
 252                                 cachemode2protval(_PAGE_CACHE_MODE_UC));
 253                 break;
 254         case _PAGE_CACHE_MODE_UC_MINUS:
 255                 prot = __pgprot(pgprot_val(prot) |
 256                                 cachemode2protval(_PAGE_CACHE_MODE_UC_MINUS));
 257                 break;
 258         case _PAGE_CACHE_MODE_WC:
 259                 prot = __pgprot(pgprot_val(prot) |
 260                                 cachemode2protval(_PAGE_CACHE_MODE_WC));
 261                 break;
 262         case _PAGE_CACHE_MODE_WT:
 263                 prot = __pgprot(pgprot_val(prot) |
 264                                 cachemode2protval(_PAGE_CACHE_MODE_WT));
 265                 break;
 266         case _PAGE_CACHE_MODE_WB:
 267                 break;
 268         }
 269 
 270         /*
 271          * Ok, go for it..
 272          */
 273         area = get_vm_area_caller(size, VM_IOREMAP, caller);
 274         if (!area)
 275                 goto err_free_memtype;
 276         area->phys_addr = phys_addr;
 277         vaddr = (unsigned long) area->addr;
 278 
 279         if (kernel_map_sync_memtype(phys_addr, size, pcm))
 280                 goto err_free_area;
 281 
 282         if (ioremap_page_range(vaddr, vaddr + size, phys_addr, prot))
 283                 goto err_free_area;
 284 
 285         ret_addr = (void __iomem *) (vaddr + offset);
 286         mmiotrace_ioremap(unaligned_phys_addr, unaligned_size, ret_addr);
 287 
 288         /*
 289          * Check if the request spans more than any BAR in the iomem resource
 290          * tree.
 291          */
 292         if (iomem_map_sanity_check(unaligned_phys_addr, unaligned_size))
 293                 pr_warn("caller %pS mapping multiple BARs\n", caller);
 294 
 295         return ret_addr;
 296 err_free_area:
 297         free_vm_area(area);
 298 err_free_memtype:
 299         free_memtype(phys_addr, phys_addr + size);
 300         return NULL;
 301 }
 302 
 303 /**
 304  * ioremap_nocache     -   map bus memory into CPU space
 305  * @phys_addr:    bus address of the memory
 306  * @size:      size of the resource to map
 307  *
 308  * ioremap_nocache performs a platform specific sequence of operations to
 309  * make bus memory CPU accessible via the readb/readw/readl/writeb/
 310  * writew/writel functions and the other mmio helpers. The returned
 311  * address is not guaranteed to be usable directly as a virtual
 312  * address.
 313  *
 314  * This version of ioremap ensures that the memory is marked uncachable
 315  * on the CPU as well as honouring existing caching rules from things like
 316  * the PCI bus. Note that there are other caches and buffers on many
 317  * busses. In particular driver authors should read up on PCI writes
 318  *
 319  * It's useful if some control registers are in such an area and
 320  * write combining or read caching is not desirable:
 321  *
 322  * Must be freed with iounmap.
 323  */
 324 void __iomem *ioremap_nocache(resource_size_t phys_addr, unsigned long size)
 325 {
 326         /*
 327          * Ideally, this should be:
 328          *      pat_enabled() ? _PAGE_CACHE_MODE_UC : _PAGE_CACHE_MODE_UC_MINUS;
 329          *
 330          * Till we fix all X drivers to use ioremap_wc(), we will use
 331          * UC MINUS. Drivers that are certain they need or can already
 332          * be converted over to strong UC can use ioremap_uc().
 333          */
 334         enum page_cache_mode pcm = _PAGE_CACHE_MODE_UC_MINUS;
 335 
 336         return __ioremap_caller(phys_addr, size, pcm,
 337                                 __builtin_return_address(0), false);
 338 }
 339 EXPORT_SYMBOL(ioremap_nocache);
 340 
 341 /**
 342  * ioremap_uc     -   map bus memory into CPU space as strongly uncachable
 343  * @phys_addr:    bus address of the memory
 344  * @size:      size of the resource to map
 345  *
 346  * ioremap_uc performs a platform specific sequence of operations to
 347  * make bus memory CPU accessible via the readb/readw/readl/writeb/
 348  * writew/writel functions and the other mmio helpers. The returned
 349  * address is not guaranteed to be usable directly as a virtual
 350  * address.
 351  *
 352  * This version of ioremap ensures that the memory is marked with a strong
 353  * preference as completely uncachable on the CPU when possible. For non-PAT
 354  * systems this ends up setting page-attribute flags PCD=1, PWT=1. For PAT
 355  * systems this will set the PAT entry for the pages as strong UC.  This call
 356  * will honor existing caching rules from things like the PCI bus. Note that
 357  * there are other caches and buffers on many busses. In particular driver
 358  * authors should read up on PCI writes.
 359  *
 360  * It's useful if some control registers are in such an area and
 361  * write combining or read caching is not desirable:
 362  *
 363  * Must be freed with iounmap.
 364  */
 365 void __iomem *ioremap_uc(resource_size_t phys_addr, unsigned long size)
 366 {
 367         enum page_cache_mode pcm = _PAGE_CACHE_MODE_UC;
 368 
 369         return __ioremap_caller(phys_addr, size, pcm,
 370                                 __builtin_return_address(0), false);
 371 }
 372 EXPORT_SYMBOL_GPL(ioremap_uc);
 373 
 374 /**
 375  * ioremap_wc   -       map memory into CPU space write combined
 376  * @phys_addr:  bus address of the memory
 377  * @size:       size of the resource to map
 378  *
 379  * This version of ioremap ensures that the memory is marked write combining.
 380  * Write combining allows faster writes to some hardware devices.
 381  *
 382  * Must be freed with iounmap.
 383  */
 384 void __iomem *ioremap_wc(resource_size_t phys_addr, unsigned long size)
 385 {
 386         return __ioremap_caller(phys_addr, size, _PAGE_CACHE_MODE_WC,
 387                                         __builtin_return_address(0), false);
 388 }
 389 EXPORT_SYMBOL(ioremap_wc);
 390 
 391 /**
 392  * ioremap_wt   -       map memory into CPU space write through
 393  * @phys_addr:  bus address of the memory
 394  * @size:       size of the resource to map
 395  *
 396  * This version of ioremap ensures that the memory is marked write through.
 397  * Write through stores data into memory while keeping the cache up-to-date.
 398  *
 399  * Must be freed with iounmap.
 400  */
 401 void __iomem *ioremap_wt(resource_size_t phys_addr, unsigned long size)
 402 {
 403         return __ioremap_caller(phys_addr, size, _PAGE_CACHE_MODE_WT,
 404                                         __builtin_return_address(0), false);
 405 }
 406 EXPORT_SYMBOL(ioremap_wt);
 407 
 408 void __iomem *ioremap_encrypted(resource_size_t phys_addr, unsigned long size)
 409 {
 410         return __ioremap_caller(phys_addr, size, _PAGE_CACHE_MODE_WB,
 411                                 __builtin_return_address(0), true);
 412 }
 413 EXPORT_SYMBOL(ioremap_encrypted);
 414 
 415 void __iomem *ioremap_cache(resource_size_t phys_addr, unsigned long size)
 416 {
 417         return __ioremap_caller(phys_addr, size, _PAGE_CACHE_MODE_WB,
 418                                 __builtin_return_address(0), false);
 419 }
 420 EXPORT_SYMBOL(ioremap_cache);
 421 
 422 void __iomem *ioremap_prot(resource_size_t phys_addr, unsigned long size,
 423                                 unsigned long prot_val)
 424 {
 425         return __ioremap_caller(phys_addr, size,
 426                                 pgprot2cachemode(__pgprot(prot_val)),
 427                                 __builtin_return_address(0), false);
 428 }
 429 EXPORT_SYMBOL(ioremap_prot);
 430 
 431 /**
 432  * iounmap - Free a IO remapping
 433  * @addr: virtual address from ioremap_*
 434  *
 435  * Caller must ensure there is only one unmapping for the same pointer.
 436  */
 437 void iounmap(volatile void __iomem *addr)
 438 {
 439         struct vm_struct *p, *o;
 440 
 441         if ((void __force *)addr <= high_memory)
 442                 return;
 443 
 444         /*
 445          * The PCI/ISA range special-casing was removed from __ioremap()
 446          * so this check, in theory, can be removed. However, there are
 447          * cases where iounmap() is called for addresses not obtained via
 448          * ioremap() (vga16fb for example). Add a warning so that these
 449          * cases can be caught and fixed.
 450          */
 451         if ((void __force *)addr >= phys_to_virt(ISA_START_ADDRESS) &&
 452             (void __force *)addr < phys_to_virt(ISA_END_ADDRESS)) {
 453                 WARN(1, "iounmap() called for ISA range not obtained using ioremap()\n");
 454                 return;
 455         }
 456 
 457         mmiotrace_iounmap(addr);
 458 
 459         addr = (volatile void __iomem *)
 460                 (PAGE_MASK & (unsigned long __force)addr);
 461 
 462         /* Use the vm area unlocked, assuming the caller
 463            ensures there isn't another iounmap for the same address
 464            in parallel. Reuse of the virtual address is prevented by
 465            leaving it in the global lists until we're done with it.
 466            cpa takes care of the direct mappings. */
 467         p = find_vm_area((void __force *)addr);
 468 
 469         if (!p) {
 470                 printk(KERN_ERR "iounmap: bad address %p\n", addr);
 471                 dump_stack();
 472                 return;
 473         }
 474 
 475         free_memtype(p->phys_addr, p->phys_addr + get_vm_area_size(p));
 476 
 477         /* Finally remove it */
 478         o = remove_vm_area((void __force *)addr);
 479         BUG_ON(p != o || o == NULL);
 480         kfree(p);
 481 }
 482 EXPORT_SYMBOL(iounmap);
 483 
 484 int __init arch_ioremap_p4d_supported(void)
 485 {
 486         return 0;
 487 }
 488 
 489 int __init arch_ioremap_pud_supported(void)
 490 {
 491 #ifdef CONFIG_X86_64
 492         return boot_cpu_has(X86_FEATURE_GBPAGES);
 493 #else
 494         return 0;
 495 #endif
 496 }
 497 
 498 int __init arch_ioremap_pmd_supported(void)
 499 {
 500         return boot_cpu_has(X86_FEATURE_PSE);
 501 }
 502 
 503 /*
 504  * Convert a physical pointer to a virtual kernel pointer for /dev/mem
 505  * access
 506  */
 507 void *xlate_dev_mem_ptr(phys_addr_t phys)
 508 {
 509         unsigned long start  = phys &  PAGE_MASK;
 510         unsigned long offset = phys & ~PAGE_MASK;
 511         void *vaddr;
 512 
 513         /* memremap() maps if RAM, otherwise falls back to ioremap() */
 514         vaddr = memremap(start, PAGE_SIZE, MEMREMAP_WB);
 515 
 516         /* Only add the offset on success and return NULL if memremap() failed */
 517         if (vaddr)
 518                 vaddr += offset;
 519 
 520         return vaddr;
 521 }
 522 
 523 void unxlate_dev_mem_ptr(phys_addr_t phys, void *addr)
 524 {
 525         memunmap((void *)((unsigned long)addr & PAGE_MASK));
 526 }
 527 
 528 /*
 529  * Examine the physical address to determine if it is an area of memory
 530  * that should be mapped decrypted.  If the memory is not part of the
 531  * kernel usable area it was accessed and created decrypted, so these
 532  * areas should be mapped decrypted. And since the encryption key can
 533  * change across reboots, persistent memory should also be mapped
 534  * decrypted.
 535  *
 536  * If SEV is active, that implies that BIOS/UEFI also ran encrypted so
 537  * only persistent memory should be mapped decrypted.
 538  */
 539 static bool memremap_should_map_decrypted(resource_size_t phys_addr,
 540                                           unsigned long size)
 541 {
 542         int is_pmem;
 543 
 544         /*
 545          * Check if the address is part of a persistent memory region.
 546          * This check covers areas added by E820, EFI and ACPI.
 547          */
 548         is_pmem = region_intersects(phys_addr, size, IORESOURCE_MEM,
 549                                     IORES_DESC_PERSISTENT_MEMORY);
 550         if (is_pmem != REGION_DISJOINT)
 551                 return true;
 552 
 553         /*
 554          * Check if the non-volatile attribute is set for an EFI
 555          * reserved area.
 556          */
 557         if (efi_enabled(EFI_BOOT)) {
 558                 switch (efi_mem_type(phys_addr)) {
 559                 case EFI_RESERVED_TYPE:
 560                         if (efi_mem_attributes(phys_addr) & EFI_MEMORY_NV)
 561                                 return true;
 562                         break;
 563                 default:
 564                         break;
 565                 }
 566         }
 567 
 568         /* Check if the address is outside kernel usable area */
 569         switch (e820__get_entry_type(phys_addr, phys_addr + size - 1)) {
 570         case E820_TYPE_RESERVED:
 571         case E820_TYPE_ACPI:
 572         case E820_TYPE_NVS:
 573         case E820_TYPE_UNUSABLE:
 574                 /* For SEV, these areas are encrypted */
 575                 if (sev_active())
 576                         break;
 577                 /* Fallthrough */
 578 
 579         case E820_TYPE_PRAM:
 580                 return true;
 581         default:
 582                 break;
 583         }
 584 
 585         return false;
 586 }
 587 
 588 /*
 589  * Examine the physical address to determine if it is EFI data. Check
 590  * it against the boot params structure and EFI tables and memory types.
 591  */
 592 static bool memremap_is_efi_data(resource_size_t phys_addr,
 593                                  unsigned long size)
 594 {
 595         u64 paddr;
 596 
 597         /* Check if the address is part of EFI boot/runtime data */
 598         if (!efi_enabled(EFI_BOOT))
 599                 return false;
 600 
 601         paddr = boot_params.efi_info.efi_memmap_hi;
 602         paddr <<= 32;
 603         paddr |= boot_params.efi_info.efi_memmap;
 604         if (phys_addr == paddr)
 605                 return true;
 606 
 607         paddr = boot_params.efi_info.efi_systab_hi;
 608         paddr <<= 32;
 609         paddr |= boot_params.efi_info.efi_systab;
 610         if (phys_addr == paddr)
 611                 return true;
 612 
 613         if (efi_is_table_address(phys_addr))
 614                 return true;
 615 
 616         switch (efi_mem_type(phys_addr)) {
 617         case EFI_BOOT_SERVICES_DATA:
 618         case EFI_RUNTIME_SERVICES_DATA:
 619                 return true;
 620         default:
 621                 break;
 622         }
 623 
 624         return false;
 625 }
 626 
 627 /*
 628  * Examine the physical address to determine if it is boot data by checking
 629  * it against the boot params setup_data chain.
 630  */
 631 static bool memremap_is_setup_data(resource_size_t phys_addr,
 632                                    unsigned long size)
 633 {
 634         struct setup_data *data;
 635         u64 paddr, paddr_next;
 636 
 637         paddr = boot_params.hdr.setup_data;
 638         while (paddr) {
 639                 unsigned int len;
 640 
 641                 if (phys_addr == paddr)
 642                         return true;
 643 
 644                 data = memremap(paddr, sizeof(*data),
 645                                 MEMREMAP_WB | MEMREMAP_DEC);
 646 
 647                 paddr_next = data->next;
 648                 len = data->len;
 649 
 650                 memunmap(data);
 651 
 652                 if ((phys_addr > paddr) && (phys_addr < (paddr + len)))
 653                         return true;
 654 
 655                 paddr = paddr_next;
 656         }
 657 
 658         return false;
 659 }
 660 
 661 /*
 662  * Examine the physical address to determine if it is boot data by checking
 663  * it against the boot params setup_data chain (early boot version).
 664  */
 665 static bool __init early_memremap_is_setup_data(resource_size_t phys_addr,
 666                                                 unsigned long size)
 667 {
 668         struct setup_data *data;
 669         u64 paddr, paddr_next;
 670 
 671         paddr = boot_params.hdr.setup_data;
 672         while (paddr) {
 673                 unsigned int len;
 674 
 675                 if (phys_addr == paddr)
 676                         return true;
 677 
 678                 data = early_memremap_decrypted(paddr, sizeof(*data));
 679 
 680                 paddr_next = data->next;
 681                 len = data->len;
 682 
 683                 early_memunmap(data, sizeof(*data));
 684 
 685                 if ((phys_addr > paddr) && (phys_addr < (paddr + len)))
 686                         return true;
 687 
 688                 paddr = paddr_next;
 689         }
 690 
 691         return false;
 692 }
 693 
 694 /*
 695  * Architecture function to determine if RAM remap is allowed. By default, a
 696  * RAM remap will map the data as encrypted. Determine if a RAM remap should
 697  * not be done so that the data will be mapped decrypted.
 698  */
 699 bool arch_memremap_can_ram_remap(resource_size_t phys_addr, unsigned long size,
 700                                  unsigned long flags)
 701 {
 702         if (!mem_encrypt_active())
 703                 return true;
 704 
 705         if (flags & MEMREMAP_ENC)
 706                 return true;
 707 
 708         if (flags & MEMREMAP_DEC)
 709                 return false;
 710 
 711         if (sme_active()) {
 712                 if (memremap_is_setup_data(phys_addr, size) ||
 713                     memremap_is_efi_data(phys_addr, size))
 714                         return false;
 715         }
 716 
 717         return !memremap_should_map_decrypted(phys_addr, size);
 718 }
 719 
 720 /*
 721  * Architecture override of __weak function to adjust the protection attributes
 722  * used when remapping memory. By default, early_memremap() will map the data
 723  * as encrypted. Determine if an encrypted mapping should not be done and set
 724  * the appropriate protection attributes.
 725  */
 726 pgprot_t __init early_memremap_pgprot_adjust(resource_size_t phys_addr,
 727                                              unsigned long size,
 728                                              pgprot_t prot)
 729 {
 730         bool encrypted_prot;
 731 
 732         if (!mem_encrypt_active())
 733                 return prot;
 734 
 735         encrypted_prot = true;
 736 
 737         if (sme_active()) {
 738                 if (early_memremap_is_setup_data(phys_addr, size) ||
 739                     memremap_is_efi_data(phys_addr, size))
 740                         encrypted_prot = false;
 741         }
 742 
 743         if (encrypted_prot && memremap_should_map_decrypted(phys_addr, size))
 744                 encrypted_prot = false;
 745 
 746         return encrypted_prot ? pgprot_encrypted(prot)
 747                               : pgprot_decrypted(prot);
 748 }
 749 
 750 bool phys_mem_access_encrypted(unsigned long phys_addr, unsigned long size)
 751 {
 752         return arch_memremap_can_ram_remap(phys_addr, size, 0);
 753 }
 754 
 755 #ifdef CONFIG_AMD_MEM_ENCRYPT
 756 /* Remap memory with encryption */
 757 void __init *early_memremap_encrypted(resource_size_t phys_addr,
 758                                       unsigned long size)
 759 {
 760         return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_ENC);
 761 }
 762 
 763 /*
 764  * Remap memory with encryption and write-protected - cannot be called
 765  * before pat_init() is called
 766  */
 767 void __init *early_memremap_encrypted_wp(resource_size_t phys_addr,
 768                                          unsigned long size)
 769 {
 770         /* Be sure the write-protect PAT entry is set for write-protect */
 771         if (__pte2cachemode_tbl[_PAGE_CACHE_MODE_WP] != _PAGE_CACHE_MODE_WP)
 772                 return NULL;
 773 
 774         return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_ENC_WP);
 775 }
 776 
 777 /* Remap memory without encryption */
 778 void __init *early_memremap_decrypted(resource_size_t phys_addr,
 779                                       unsigned long size)
 780 {
 781         return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_NOENC);
 782 }
 783 
 784 /*
 785  * Remap memory without encryption and write-protected - cannot be called
 786  * before pat_init() is called
 787  */
 788 void __init *early_memremap_decrypted_wp(resource_size_t phys_addr,
 789                                          unsigned long size)
 790 {
 791         /* Be sure the write-protect PAT entry is set for write-protect */
 792         if (__pte2cachemode_tbl[_PAGE_CACHE_MODE_WP] != _PAGE_CACHE_MODE_WP)
 793                 return NULL;
 794 
 795         return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_NOENC_WP);
 796 }
 797 #endif  /* CONFIG_AMD_MEM_ENCRYPT */
 798 
 799 static pte_t bm_pte[PAGE_SIZE/sizeof(pte_t)] __page_aligned_bss;
 800 
 801 static inline pmd_t * __init early_ioremap_pmd(unsigned long addr)
 802 {
 803         /* Don't assume we're using swapper_pg_dir at this point */
 804         pgd_t *base = __va(read_cr3_pa());
 805         pgd_t *pgd = &base[pgd_index(addr)];
 806         p4d_t *p4d = p4d_offset(pgd, addr);
 807         pud_t *pud = pud_offset(p4d, addr);
 808         pmd_t *pmd = pmd_offset(pud, addr);
 809 
 810         return pmd;
 811 }
 812 
 813 static inline pte_t * __init early_ioremap_pte(unsigned long addr)
 814 {
 815         return &bm_pte[pte_index(addr)];
 816 }
 817 
 818 bool __init is_early_ioremap_ptep(pte_t *ptep)
 819 {
 820         return ptep >= &bm_pte[0] && ptep < &bm_pte[PAGE_SIZE/sizeof(pte_t)];
 821 }
 822 
 823 void __init early_ioremap_init(void)
 824 {
 825         pmd_t *pmd;
 826 
 827 #ifdef CONFIG_X86_64
 828         BUILD_BUG_ON((fix_to_virt(0) + PAGE_SIZE) & ((1 << PMD_SHIFT) - 1));
 829 #else
 830         WARN_ON((fix_to_virt(0) + PAGE_SIZE) & ((1 << PMD_SHIFT) - 1));
 831 #endif
 832 
 833         early_ioremap_setup();
 834 
 835         pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
 836         memset(bm_pte, 0, sizeof(bm_pte));
 837         pmd_populate_kernel(&init_mm, pmd, bm_pte);
 838 
 839         /*
 840          * The boot-ioremap range spans multiple pmds, for which
 841          * we are not prepared:
 842          */
 843 #define __FIXADDR_TOP (-PAGE_SIZE)
 844         BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT)
 845                      != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT));
 846 #undef __FIXADDR_TOP
 847         if (pmd != early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))) {
 848                 WARN_ON(1);
 849                 printk(KERN_WARNING "pmd %p != %p\n",
 850                        pmd, early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END)));
 851                 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
 852                         fix_to_virt(FIX_BTMAP_BEGIN));
 853                 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END):   %08lx\n",
 854                         fix_to_virt(FIX_BTMAP_END));
 855 
 856                 printk(KERN_WARNING "FIX_BTMAP_END:       %d\n", FIX_BTMAP_END);
 857                 printk(KERN_WARNING "FIX_BTMAP_BEGIN:     %d\n",
 858                        FIX_BTMAP_BEGIN);
 859         }
 860 }
 861 
 862 void __init __early_set_fixmap(enum fixed_addresses idx,
 863                                phys_addr_t phys, pgprot_t flags)
 864 {
 865         unsigned long addr = __fix_to_virt(idx);
 866         pte_t *pte;
 867 
 868         if (idx >= __end_of_fixed_addresses) {
 869                 BUG();
 870                 return;
 871         }
 872         pte = early_ioremap_pte(addr);
 873 
 874         /* Sanitize 'prot' against any unsupported bits: */
 875         pgprot_val(flags) &= __supported_pte_mask;
 876 
 877         if (pgprot_val(flags))
 878                 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
 879         else
 880                 pte_clear(&init_mm, addr, pte);
 881         __flush_tlb_one_kernel(addr);
 882 }