60 	dev_priv->dma_low += 8;					\
68 	dev_priv->dma_low += 8;			\
71 static void via_cmdbuf_start(drm_via_private_t *dev_priv);
72 static void via_cmdbuf_pause(drm_via_private_t *dev_priv);
73 static void via_cmdbuf_reset(drm_via_private_t *dev_priv);
74 static void via_cmdbuf_rewind(drm_via_private_t *dev_priv);
75 static int via_wait_idle(drm_via_private_t *dev_priv);
76 static void via_pad_cache(drm_via_private_t *dev_priv, int qwords);
82 static uint32_t via_cmdbuf_space(drm_via_private_t *dev_priv)  in via_cmdbuf_space()  argument
84 	uint32_t agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;  in via_cmdbuf_space()
85 	uint32_t hw_addr = *(dev_priv->hw_addr_ptr) - agp_base;  in via_cmdbuf_space()
87 	return ((hw_addr <= dev_priv->dma_low) ?  in via_cmdbuf_space()
88 		(dev_priv->dma_high + hw_addr - dev_priv->dma_low) :  in via_cmdbuf_space()
89 		(hw_addr - dev_priv->dma_low));  in via_cmdbuf_space()
96 static uint32_t via_cmdbuf_lag(drm_via_private_t *dev_priv)  in via_cmdbuf_lag()  argument
98 	uint32_t agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;  in via_cmdbuf_lag()
99 	uint32_t hw_addr = *(dev_priv->hw_addr_ptr) - agp_base;  in via_cmdbuf_lag()
101 	return ((hw_addr <= dev_priv->dma_low) ?  in via_cmdbuf_lag()
102 		(dev_priv->dma_low - hw_addr) :  in via_cmdbuf_lag()
103 		(dev_priv->dma_wrap + dev_priv->dma_low - hw_addr));  in via_cmdbuf_lag()
111 via_cmdbuf_wait(drm_via_private_t *dev_priv, unsigned int size)  in via_cmdbuf_wait()  argument
113 	uint32_t agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;  in via_cmdbuf_wait()
117 	hw_addr_ptr = dev_priv->hw_addr_ptr;  in via_cmdbuf_wait()
118 	cur_addr = dev_priv->dma_low;  in via_cmdbuf_wait()
142 static inline uint32_t *via_check_dma(drm_via_private_t * dev_priv,  in via_check_dma()  argument
145 	if ((dev_priv->dma_low + size + 4 * CMDBUF_ALIGNMENT_SIZE) >  in via_check_dma()
146 	    dev_priv->dma_high) {  in via_check_dma()
147 		via_cmdbuf_rewind(dev_priv);  in via_check_dma()
149 	if (via_cmdbuf_wait(dev_priv, size) != 0)  in via_check_dma()
152 	return (uint32_t *) (dev_priv->dma_ptr + dev_priv->dma_low);  in via_check_dma()
158 		drm_via_private_t *dev_priv =  in via_dma_cleanup()  local
161 		if (dev_priv->ring.virtual_start) {  in via_dma_cleanup()
162 			via_cmdbuf_reset(dev_priv);  in via_dma_cleanup()
164 			drm_legacy_ioremapfree(&dev_priv->ring.map, dev);  in via_dma_cleanup()
165 			dev_priv->ring.virtual_start = NULL;  in via_dma_cleanup()
174 			  drm_via_private_t *dev_priv,  in via_initialize()  argument
177 	if (!dev_priv || !dev_priv->mmio) {  in via_initialize()
182 	if (dev_priv->ring.virtual_start != NULL) {  in via_initialize()
192 	if (dev_priv->chipset == VIA_DX9_0) {  in via_initialize()
197 	dev_priv->ring.map.offset = dev->agp->base + init->offset;  in via_initialize()
198 	dev_priv->ring.map.size = init->size;  in via_initialize()
199 	dev_priv->ring.map.type = 0;  in via_initialize()
200 	dev_priv->ring.map.flags = 0;  in via_initialize()
201 	dev_priv->ring.map.mtrr = 0;  in via_initialize()
203 	drm_legacy_ioremap(&dev_priv->ring.map, dev);  in via_initialize()
205 	if (dev_priv->ring.map.handle == NULL) {  in via_initialize()
212 	dev_priv->ring.virtual_start = dev_priv->ring.map.handle;  in via_initialize()
214 	dev_priv->dma_ptr = dev_priv->ring.virtual_start;  in via_initialize()
215 	dev_priv->dma_low = 0;  in via_initialize()
216 	dev_priv->dma_high = init->size;  in via_initialize()
217 	dev_priv->dma_wrap = init->size;  in via_initialize()
218 	dev_priv->dma_offset = init->offset;  in via_initialize()
219 	dev_priv->last_pause_ptr = NULL;  in via_initialize()
220 	dev_priv->hw_addr_ptr =  in via_initialize()
221 		(volatile uint32_t *)((char *)dev_priv->mmio->handle +  in via_initialize()
224 	via_cmdbuf_start(dev_priv);  in via_initialize()
231 	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;  in via_dma_init()  local
240 			retcode = via_initialize(dev, dev_priv, init);  in via_dma_init()
249 		retcode = (dev_priv->ring.virtual_start != NULL) ?  in via_dma_init()
262 	drm_via_private_t *dev_priv;  in via_dispatch_cmdbuffer()  local
266 	dev_priv = (drm_via_private_t *) dev->dev_private;  in via_dispatch_cmdbuffer()
268 	if (dev_priv->ring.virtual_start == NULL) {  in via_dispatch_cmdbuffer()
276 	if (copy_from_user(dev_priv->pci_buf, cmd->buf, cmd->size))  in via_dispatch_cmdbuffer()
286 	     via_verify_command_stream((uint32_t *) dev_priv->pci_buf,  in via_dispatch_cmdbuffer()
291 	vb = via_check_dma(dev_priv, (cmd->size < 0x100) ? 0x102 : cmd->size);  in via_dispatch_cmdbuffer()
295 	memcpy(vb, dev_priv->pci_buf, cmd->size);  in via_dispatch_cmdbuffer()
297 	dev_priv->dma_low += cmd->size;  in via_dispatch_cmdbuffer()
305 		via_pad_cache(dev_priv, (0x100 - cmd->size) >> 3);  in via_dispatch_cmdbuffer()
306 	via_cmdbuf_pause(dev_priv);  in via_dispatch_cmdbuffer()
313 	drm_via_private_t *dev_priv = dev->dev_private;  in via_driver_dma_quiescent()  local
315 	if (!via_wait_idle(dev_priv))  in via_driver_dma_quiescent()
344 	drm_via_private_t *dev_priv = dev->dev_private;  in via_dispatch_pci_cmdbuffer()  local
349 	if (copy_from_user(dev_priv->pci_buf, cmd->buf, cmd->size))  in via_dispatch_pci_cmdbuffer()
353 	     via_verify_command_stream((uint32_t *) dev_priv->pci_buf,  in via_dispatch_pci_cmdbuffer()
359 	    via_parse_command_stream(dev, (const uint32_t *)dev_priv->pci_buf,  in via_dispatch_pci_cmdbuffer()
377 static inline uint32_t *via_align_buffer(drm_via_private_t *dev_priv,  in via_align_buffer()  argument
390 static inline uint32_t *via_get_dma(drm_via_private_t *dev_priv)  in via_get_dma()  argument
392 	return (uint32_t *) (dev_priv->dma_ptr + dev_priv->dma_low);  in via_get_dma()
400 static int via_hook_segment(drm_via_private_t *dev_priv,  in via_hook_segment()  argument
405 	volatile uint32_t *paused_at = dev_priv->last_pause_ptr;  in via_hook_segment()
411 	(void) *(volatile uint32_t *)(via_get_dma(dev_priv) - 1);  in via_hook_segment()
417 	reader = *(dev_priv->hw_addr_ptr);  in via_hook_segment()
418 	ptr = ((volatile char *)paused_at - dev_priv->dma_ptr) +  in via_hook_segment()
419 		dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr + 4;  in via_hook_segment()
421 	dev_priv->last_pause_ptr = via_get_dma(dev_priv) - 1;  in via_hook_segment()
430 	diff = (uint32_t) (ptr - reader) - dev_priv->dma_diff;  in via_hook_segment()
436 		reader = *(dev_priv->hw_addr_ptr);  in via_hook_segment()
437 		diff = (uint32_t) (ptr - reader) - dev_priv->dma_diff;  in via_hook_segment()
443 		reader = *(dev_priv->hw_addr_ptr);  in via_hook_segment()
444 		diff = (uint32_t) (ptr - reader) - dev_priv->dma_diff;  in via_hook_segment()
445 		diff &= (dev_priv->dma_high - 1);  in via_hook_segment()
446 		if (diff != 0 && diff < (dev_priv->dma_high >> 1)) {  in via_hook_segment()
449 				  ptr, reader, dev_priv->dma_diff);  in via_hook_segment()
466 static int via_wait_idle(drm_via_private_t *dev_priv)  in via_wait_idle()  argument
480 static uint32_t *via_align_cmd(drm_via_private_t *dev_priv, uint32_t cmd_type,  in via_align_cmd()  argument
490 		via_cmdbuf_wait(dev_priv, 2 * CMDBUF_ALIGNMENT_SIZE);  in via_align_cmd()
492 	vb = via_get_dma(dev_priv);  in via_align_cmd()
495 	agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;  in via_align_cmd()
497 	    ((dev_priv->dma_low & CMDBUF_ALIGNMENT_MASK) >> 3);  in via_align_cmd()
500 	    agp_base + dev_priv->dma_low - 8 + (qw_pad_count << 3);  in via_align_cmd()
505 	vb = via_align_buffer(dev_priv, vb, qw_pad_count - 1);  in via_align_cmd()
510 static void via_cmdbuf_start(drm_via_private_t *dev_priv)  in via_cmdbuf_start()  argument
521 	dev_priv->dma_low = 0;  in via_cmdbuf_start()
523 	agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;  in via_cmdbuf_start()
525 	end_addr = agp_base + dev_priv->dma_high;  in via_cmdbuf_start()
532 	dev_priv->last_pause_ptr =  in via_cmdbuf_start()
533 	    via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0,  in via_cmdbuf_start()
537 	(void) *(volatile uint32_t *)dev_priv->last_pause_ptr;  in via_cmdbuf_start()
550 	dev_priv->dma_diff = 0;  in via_cmdbuf_start()
555 	reader = *(dev_priv->hw_addr_ptr);  in via_cmdbuf_start()
556 	ptr = ((volatile char *)dev_priv->last_pause_ptr - dev_priv->dma_ptr) +  in via_cmdbuf_start()
557 	    dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr + 4;  in via_cmdbuf_start()
566 	dev_priv->dma_diff = ptr - reader;  in via_cmdbuf_start()
569 static void via_pad_cache(drm_via_private_t *dev_priv, int qwords)  in via_pad_cache()  argument
573 	via_cmdbuf_wait(dev_priv, qwords + 2);  in via_pad_cache()
574 	vb = via_get_dma(dev_priv);  in via_pad_cache()
576 	via_align_buffer(dev_priv, vb, qwords);  in via_pad_cache()
579 static inline void via_dummy_bitblt(drm_via_private_t *dev_priv)  in via_dummy_bitblt()  argument
581 	uint32_t *vb = via_get_dma(dev_priv);  in via_dummy_bitblt()
587 static void via_cmdbuf_jump(drm_via_private_t *dev_priv)  in via_cmdbuf_jump()  argument
595 	agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;  in via_cmdbuf_jump()
596 	via_align_cmd(dev_priv, HC_HAGPBpID_JUMP, 0, &jump_addr_hi,  in via_cmdbuf_jump()
599 	dev_priv->dma_wrap = dev_priv->dma_low;  in via_cmdbuf_jump()
605 	dev_priv->dma_low = 0;  in via_cmdbuf_jump()
606 	if (via_cmdbuf_wait(dev_priv, CMDBUF_ALIGNMENT_SIZE) != 0)  in via_cmdbuf_jump()
609 	via_dummy_bitblt(dev_priv);  in via_cmdbuf_jump()
610 	via_dummy_bitblt(dev_priv);  in via_cmdbuf_jump()
613 	    via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0, &pause_addr_hi,  in via_cmdbuf_jump()
615 	via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0, &pause_addr_hi,  in via_cmdbuf_jump()
619 	dma_low_save1 = dev_priv->dma_low;  in via_cmdbuf_jump()
631 		via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0, &pause_addr_hi,  in via_cmdbuf_jump()
633 	via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0, &pause_addr_hi,  in via_cmdbuf_jump()
637 	dma_low_save2 = dev_priv->dma_low;  in via_cmdbuf_jump()
638 	dev_priv->dma_low = dma_low_save1;  in via_cmdbuf_jump()
639 	via_hook_segment(dev_priv, jump_addr_hi, jump_addr_lo, 0);  in via_cmdbuf_jump()
640 	dev_priv->dma_low = dma_low_save2;  in via_cmdbuf_jump()
641 	via_hook_segment(dev_priv, pause_addr_hi, pause_addr_lo, 0);  in via_cmdbuf_jump()
645 static void via_cmdbuf_rewind(drm_via_private_t *dev_priv)  in via_cmdbuf_rewind()  argument
647 	via_cmdbuf_jump(dev_priv);  in via_cmdbuf_rewind()
650 static void via_cmdbuf_flush(drm_via_private_t *dev_priv, uint32_t cmd_type)  in via_cmdbuf_flush()  argument
654 	via_align_cmd(dev_priv, cmd_type, 0, &pause_addr_hi, &pause_addr_lo, 0);  in via_cmdbuf_flush()
655 	via_hook_segment(dev_priv, pause_addr_hi, pause_addr_lo, 0);  in via_cmdbuf_flush()
658 static void via_cmdbuf_pause(drm_via_private_t *dev_priv)  in via_cmdbuf_pause()  argument
660 	via_cmdbuf_flush(dev_priv, HC_HAGPBpID_PAUSE);  in via_cmdbuf_pause()
663 static void via_cmdbuf_reset(drm_via_private_t *dev_priv)  in via_cmdbuf_reset()  argument
665 	via_cmdbuf_flush(dev_priv, HC_HAGPBpID_STOP);  in via_cmdbuf_reset()
666 	via_wait_idle(dev_priv);  in via_cmdbuf_reset()
678 	drm_via_private_t *dev_priv;  in via_cmdbuf_size()  local
683 	dev_priv = (drm_via_private_t *) dev->dev_private;  in via_cmdbuf_size()
685 	if (dev_priv->ring.virtual_start == NULL) {  in via_cmdbuf_size()
694 		while (((tmp_size = via_cmdbuf_space(dev_priv)) < d_siz->size)  in via_cmdbuf_size()
705 		while (((tmp_size = via_cmdbuf_lag(dev_priv)) > d_siz->size)  in via_cmdbuf_size()