88 } intel_private;  variable
90 #define INTEL_GTT_GEN	intel_private.driver->gen
91 #define IS_G33		intel_private.driver->is_g33
92 #define IS_PINEVIEW	intel_private.driver->is_pineview
93 #define IS_IRONLAKE	intel_private.driver->is_ironlake
94 #define HAS_PGTBL_EN	intel_private.driver->has_pgtbl_enable
112 	if (!pci_map_sg(intel_private.pcidev,  in intel_gtt_map_memory()
128 	pci_unmap_sg(intel_private.pcidev, sg_list,  in intel_gtt_unmap_memory()
181 	intel_private.i81x_gtt_table = gtt_table;  in i810_setup()
183 	reg_addr = pci_resource_start(intel_private.pcidev, I810_MMADR_BAR);  in i810_setup()
185 	intel_private.registers = ioremap(reg_addr, KB(64));  in i810_setup()
186 	if (!intel_private.registers)  in i810_setup()
190 	       intel_private.registers+I810_PGETBL_CTL);  in i810_setup()
192 	intel_private.gtt_phys_addr = reg_addr + I810_PTE_BASE;  in i810_setup()
194 	if ((readl(intel_private.registers+I810_DRAM_CTL)  in i810_setup()
196 		dev_info(&intel_private.pcidev->dev,  in i810_setup()
198 		intel_private.num_dcache_entries = 1024;  in i810_setup()
206 	writel(0, intel_private.registers+I810_PGETBL_CTL);  in i810_cleanup()
207 	free_gatt_pages(intel_private.i81x_gtt_table, I810_GTT_ORDER);  in i810_cleanup()
217 			> intel_private.num_dcache_entries)  in i810_insert_dcache_entries()
225 		intel_private.driver->write_entry(addr,  in i810_insert_dcache_entries()
303 	if (intel_private.needs_dmar) {  in intel_gtt_setup_scratch_page()
304 		dma_addr = pci_map_page(intel_private.pcidev, page, 0,  in intel_gtt_setup_scratch_page()
306 		if (pci_dma_mapping_error(intel_private.pcidev, dma_addr))  in intel_gtt_setup_scratch_page()
309 		intel_private.scratch_page_dma = dma_addr;  in intel_gtt_setup_scratch_page()
311 		intel_private.scratch_page_dma = page_to_phys(page);  in intel_gtt_setup_scratch_page()
313 	intel_private.scratch_page = page;  in intel_gtt_setup_scratch_page()
332 	writel_relaxed(addr | pte_flags, intel_private.gtt + entry);  in i810_write_entry()
354 	pci_read_config_word(intel_private.bridge_dev,  in intel_gtt_stolen_size()
357 	if (intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82830_HB ||  in intel_gtt_stolen_size()
358 	    intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82845G_HB) {  in intel_gtt_stolen_size()
370 			rdct = readb(intel_private.registers+I830_RDRAM_CHANNEL_TYPE);  in intel_gtt_stolen_size()
427 		dev_info(&intel_private.bridge_dev->dev, "detected %dK %s memory\n",  in intel_gtt_stolen_size()
430 		dev_info(&intel_private.bridge_dev->dev,  in intel_gtt_stolen_size()
443 	pgetbl_ctl2 = readl(intel_private.registers+I965_PGETBL_CTL2);  in i965_adjust_pgetbl_size()
445 	writel(pgetbl_ctl2, intel_private.registers+I965_PGETBL_CTL2);  in i965_adjust_pgetbl_size()
448 	pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL);  in i965_adjust_pgetbl_size()
451 	writel(pgetbl_ctl, intel_private.registers+I810_PGETBL_CTL);  in i965_adjust_pgetbl_size()
460 	pci_read_config_word(intel_private.bridge_dev,  in i965_gtt_total_entries()
479 	pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL);  in i965_gtt_total_entries()
502 		dev_info(&intel_private.pcidev->dev,  in i965_gtt_total_entries()
518 		return intel_private.gtt_mappable_entries;  in intel_gtt_total_entries()
529 		pci_read_config_dword(intel_private.bridge_dev,  in intel_gtt_mappable_entries()
540 		pci_read_config_word(intel_private.bridge_dev,  in intel_gtt_mappable_entries()
549 		aperture_size = pci_resource_len(intel_private.pcidev, 2);  in intel_gtt_mappable_entries()
557 	set_pages_wb(intel_private.scratch_page, 1);  in intel_gtt_teardown_scratch_page()
558 	pci_unmap_page(intel_private.pcidev, intel_private.scratch_page_dma,  in intel_gtt_teardown_scratch_page()
560 	__free_page(intel_private.scratch_page);  in intel_gtt_teardown_scratch_page()
565 	intel_private.driver->cleanup();  in intel_gtt_cleanup()
567 	iounmap(intel_private.gtt);  in intel_gtt_cleanup()
568 	iounmap(intel_private.registers);  in intel_gtt_cleanup()
579 	const unsigned short gpu_devid = intel_private.pcidev->device;  in needs_ilk_vtd_wa()
612 	ret = intel_private.driver->setup();  in intel_gtt_init()
616 	intel_private.gtt_mappable_entries = intel_gtt_mappable_entries();  in intel_gtt_init()
617 	intel_private.gtt_total_entries = intel_gtt_total_entries();  in intel_gtt_init()
620 	intel_private.PGETBL_save =  in intel_gtt_init()
621 		readl(intel_private.registers+I810_PGETBL_CTL)  in intel_gtt_init()
625 		intel_private.PGETBL_save |= I810_PGETBL_ENABLED;  in intel_gtt_init()
627 	dev_info(&intel_private.bridge_dev->dev,  in intel_gtt_init()
629 			intel_private.gtt_total_entries * 4,  in intel_gtt_init()
630 			intel_private.gtt_mappable_entries * 4);  in intel_gtt_init()
632 	gtt_map_size = intel_private.gtt_total_entries * 4;  in intel_gtt_init()
634 	intel_private.gtt = NULL;  in intel_gtt_init()
636 		intel_private.gtt = ioremap_wc(intel_private.gtt_phys_addr,  in intel_gtt_init()
638 	if (intel_private.gtt == NULL)  in intel_gtt_init()
639 		intel_private.gtt = ioremap(intel_private.gtt_phys_addr,  in intel_gtt_init()
641 	if (intel_private.gtt == NULL) {  in intel_gtt_init()
642 		intel_private.driver->cleanup();  in intel_gtt_init()
643 		iounmap(intel_private.registers);  in intel_gtt_init()
651 	intel_private.stolen_size = intel_gtt_stolen_size();  in intel_gtt_init()
653 	intel_private.needs_dmar = USE_PCI_DMA_API && INTEL_GTT_GEN > 2;  in intel_gtt_init()
666 	intel_private.gma_bus_addr = pci_bus_address(intel_private.pcidev, bar);  in intel_gtt_init()
677 	aper_size = (intel_private.gtt_mappable_entries << PAGE_SHIFT) / MB(1);  in intel_fake_agp_fetch_size()
719 	writel(readl(intel_private.registers+I830_HIC) | (1<<31),  in i830_chipset_flush()
720 	       intel_private.registers+I830_HIC);  in i830_chipset_flush()
722 	while (readl(intel_private.registers+I830_HIC) & (1<<31)) {  in i830_chipset_flush()
738 	writel_relaxed(addr | pte_flags, intel_private.gtt + entry);  in i830_write_entry()
748 		pci_read_config_word(intel_private.bridge_dev,  in intel_enable_gtt()
751 		pci_write_config_word(intel_private.bridge_dev,  in intel_enable_gtt()
754 		pci_read_config_word(intel_private.bridge_dev,  in intel_enable_gtt()
757 			dev_err(&intel_private.pcidev->dev,  in intel_enable_gtt()
768 		writel(0, intel_private.registers+GFX_FLSH_CNTL);  in intel_enable_gtt()
770 	reg = intel_private.registers+I810_PGETBL_CTL;  in intel_enable_gtt()
771 	writel(intel_private.PGETBL_save, reg);  in intel_enable_gtt()
773 		dev_err(&intel_private.pcidev->dev,  in intel_enable_gtt()
775 			readl(reg), intel_private.PGETBL_save);  in intel_enable_gtt()
780 		writel(0, intel_private.registers+GFX_FLSH_CNTL);  in intel_enable_gtt()
790 	reg_addr = pci_resource_start(intel_private.pcidev, I810_MMADR_BAR);  in i830_setup()
792 	intel_private.registers = ioremap(reg_addr, KB(64));  in i830_setup()
793 	if (!intel_private.registers)  in i830_setup()
796 	intel_private.gtt_phys_addr = reg_addr + I810_PTE_BASE;  in i830_setup()
821 	intel_private.clear_fake_agp = true;  in intel_fake_agp_configure()
822 	agp_bridge->gart_bus_addr = intel_private.gma_bus_addr;  in intel_fake_agp_configure()
857 			intel_private.driver->write_entry(addr, j, flags);  in intel_gtt_insert_sg_entries()
875 		intel_private.driver->write_entry(addr,  in intel_gtt_insert_pages()
886 	if (intel_private.clear_fake_agp) {  in intel_fake_agp_insert_entries()
887 		int start = intel_private.stolen_size / PAGE_SIZE;  in intel_fake_agp_insert_entries()
888 		int end = intel_private.gtt_mappable_entries;  in intel_fake_agp_insert_entries()
890 		intel_private.clear_fake_agp = false;  in intel_fake_agp_insert_entries()
899 	if (pg_start + mem->page_count > intel_private.gtt_total_entries)  in intel_fake_agp_insert_entries()
905 	if (!intel_private.driver->check_flags(type))  in intel_fake_agp_insert_entries()
911 	if (intel_private.needs_dmar) {  in intel_fake_agp_insert_entries()
938 		intel_private.driver->write_entry(intel_private.scratch_page_dma,  in intel_gtt_clear_range()
954 	if (intel_private.needs_dmar) {  in intel_fake_agp_remove_entries()
969 		if (pg_count != intel_private.num_dcache_entries)  in intel_fake_agp_alloc_by_type()
992 	ret = pci_bus_alloc_resource(intel_private.bridge_dev->bus, &intel_private.ifp_resource, PAGE_SIZE,  in intel_alloc_chipset_flush_resource()
994 				     pcibios_align_resource, intel_private.bridge_dev);  in intel_alloc_chipset_flush_resource()
1004 	pci_read_config_dword(intel_private.bridge_dev, I915_IFPADDR, &temp);  in intel_i915_setup_chipset_flush()
1007 		intel_private.resource_valid = 1;  in intel_i915_setup_chipset_flush()
1008 …pci_write_config_dword(intel_private.bridge_dev, I915_IFPADDR, (intel_private.ifp_resource.start &…  in intel_i915_setup_chipset_flush()
1012 		intel_private.resource_valid = 1;  in intel_i915_setup_chipset_flush()
1013 		intel_private.ifp_resource.start = temp;  in intel_i915_setup_chipset_flush()
1014 		intel_private.ifp_resource.end = temp + PAGE_SIZE;  in intel_i915_setup_chipset_flush()
1015 		ret = request_resource(&iomem_resource, &intel_private.ifp_resource);  in intel_i915_setup_chipset_flush()
1018 			intel_private.resource_valid = 0;  in intel_i915_setup_chipset_flush()
1027 	pci_read_config_dword(intel_private.bridge_dev, I965_IFPADDR + 4, &temp_hi);  in intel_i965_g33_setup_chipset_flush()
1028 	pci_read_config_dword(intel_private.bridge_dev, I965_IFPADDR, &temp_lo);  in intel_i965_g33_setup_chipset_flush()
1034 		intel_private.resource_valid = 1;  in intel_i965_g33_setup_chipset_flush()
1035 		pci_write_config_dword(intel_private.bridge_dev, I965_IFPADDR + 4,  in intel_i965_g33_setup_chipset_flush()
1036 			upper_32_bits(intel_private.ifp_resource.start));  in intel_i965_g33_setup_chipset_flush()
1037 …pci_write_config_dword(intel_private.bridge_dev, I965_IFPADDR, (intel_private.ifp_resource.start &…  in intel_i965_g33_setup_chipset_flush()
1044 		intel_private.resource_valid = 1;  in intel_i965_g33_setup_chipset_flush()
1045 		intel_private.ifp_resource.start = l64;  in intel_i965_g33_setup_chipset_flush()
1046 		intel_private.ifp_resource.end = l64 + PAGE_SIZE;  in intel_i965_g33_setup_chipset_flush()
1047 		ret = request_resource(&iomem_resource, &intel_private.ifp_resource);  in intel_i965_g33_setup_chipset_flush()
1050 			intel_private.resource_valid = 0;  in intel_i965_g33_setup_chipset_flush()
1057 	if (intel_private.ifp_resource.start)  in intel_i9xx_setup_flush()
1064 	intel_private.ifp_resource.name = "Intel Flush Page";  in intel_i9xx_setup_flush()
1065 	intel_private.ifp_resource.flags = IORESOURCE_MEM;  in intel_i9xx_setup_flush()
1074 	if (intel_private.ifp_resource.start)  in intel_i9xx_setup_flush()
1075 		intel_private.i9xx_flush_page = ioremap_nocache(intel_private.ifp_resource.start, PAGE_SIZE);  in intel_i9xx_setup_flush()
1076 	if (!intel_private.i9xx_flush_page)  in intel_i9xx_setup_flush()
1077 		dev_err(&intel_private.pcidev->dev,  in intel_i9xx_setup_flush()
1083 	if (intel_private.i9xx_flush_page)  in i9xx_cleanup()
1084 		iounmap(intel_private.i9xx_flush_page);  in i9xx_cleanup()
1085 	if (intel_private.resource_valid)  in i9xx_cleanup()
1086 		release_resource(&intel_private.ifp_resource);  in i9xx_cleanup()
1087 	intel_private.ifp_resource.start = 0;  in i9xx_cleanup()
1088 	intel_private.resource_valid = 0;  in i9xx_cleanup()
1093 	if (intel_private.i9xx_flush_page)  in i9xx_chipset_flush()
1094 		writel(1, intel_private.i9xx_flush_page);  in i9xx_chipset_flush()
1109 	writel_relaxed(addr | pte_flags, intel_private.gtt + entry);  in i965_write_entry()
1117 	reg_addr = pci_resource_start(intel_private.pcidev, I915_MMADR_BAR);  in i9xx_setup()
1119 	intel_private.registers = ioremap(reg_addr, size);  in i9xx_setup()
1120 	if (!intel_private.registers)  in i9xx_setup()
1125 		intel_private.gtt_phys_addr =  in i9xx_setup()
1126 			pci_resource_start(intel_private.pcidev, I915_PTE_BAR);  in i9xx_setup()
1129 		intel_private.gtt_phys_addr = reg_addr + MB(2);  in i9xx_setup()
1132 		intel_private.gtt_phys_addr = reg_addr + KB(512);  in i9xx_setup()
1340 	intel_private.pcidev = gmch_device;  in find_gmch()
1354 	if (intel_private.driver) {  in intel_gmch_probe()
1355 		intel_private.refcount++;  in intel_gmch_probe()
1363 				intel_private.pcidev = pci_dev_get(gpu_pdev);  in intel_gmch_probe()
1364 				intel_private.driver =  in intel_gmch_probe()
1370 			intel_private.driver =  in intel_gmch_probe()
1376 	if (!intel_private.driver)  in intel_gmch_probe()
1379 	intel_private.refcount++;  in intel_gmch_probe()
1384 		bridge->dev_private_data = &intel_private;  in intel_gmch_probe()
1389 	intel_private.bridge_dev = pci_dev_get(bridge_pdev);  in intel_gmch_probe()
1393 	mask = intel_private.driver->dma_mask_size;  in intel_gmch_probe()
1394 	if (pci_set_dma_mask(intel_private.pcidev, DMA_BIT_MASK(mask)))  in intel_gmch_probe()
1395 		dev_err(&intel_private.pcidev->dev,  in intel_gmch_probe()
1398 		pci_set_consistent_dma_mask(intel_private.pcidev,  in intel_gmch_probe()
1414 	*gtt_total = intel_private.gtt_total_entries << PAGE_SHIFT;  in intel_gtt_get()
1415 	*stolen_size = intel_private.stolen_size;  in intel_gtt_get()
1416 	*mappable_base = intel_private.gma_bus_addr;  in intel_gtt_get()
1417 	*mappable_end = intel_private.gtt_mappable_entries << PAGE_SHIFT;  in intel_gtt_get()
1423 	if (intel_private.driver->chipset_flush)  in intel_gtt_chipset_flush()
1424 		intel_private.driver->chipset_flush();  in intel_gtt_chipset_flush()
1430 	if (--intel_private.refcount)  in intel_gmch_remove()
1433 	if (intel_private.pcidev)  in intel_gmch_remove()
1434 		pci_dev_put(intel_private.pcidev);  in intel_gmch_remove()
1435 	if (intel_private.bridge_dev)  in intel_gmch_remove()
1436 		pci_dev_put(intel_private.bridge_dev);  in intel_gmch_remove()
1437 	intel_private.driver = NULL;  in intel_gmch_remove()