drm/radeon/radeon_ring.c

37  * Most engines on the GPU are fed via ring buffers.  Ring
43  * pointers are equal, the ring is idle.  When the host
44  * writes commands to the ring buffer, it increments the
48 static void radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *ring);
51  * radeon_ring_supports_scratch_reg - check if the ring supports
55  * @ring: radeon_ring structure holding ring information
57  * Check if a specific ring supports writing to scratch registers (all asics).
58  * Returns true if the ring supports writing to scratch regs, false if not.
61 				      struct radeon_ring *ring)
63 	switch (ring->idx) {
77  * @ring: radeon_ring structure holding ring information
79  * Update the free dw slots in the ring buffer (all asics).
81 void radeon_ring_free_size(struct radeon_device *rdev, struct radeon_ring *ring)
83 	uint32_t rptr = radeon_ring_get_rptr(rdev, ring);
86 	ring->ring_free_dw = rptr + (ring->ring_size / 4);
87 	ring->ring_free_dw -= ring->wptr;
88 	ring->ring_free_dw &= ring->ptr_mask;
89 	if (!ring->ring_free_dw) {
90 		/* this is an empty ring */
91 		ring->ring_free_dw = ring->ring_size / 4;
93 		radeon_ring_lockup_update(rdev, ring);
98  * radeon_ring_alloc - allocate space on the ring buffer
101  * @ring: radeon_ring structure holding ring information
102  * @ndw: number of dwords to allocate in the ring buffer
104  * Allocate @ndw dwords in the ring buffer (all asics).
107 int radeon_ring_alloc(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ndw)
111 	/* make sure we aren't trying to allocate more space than there is on the ring */
112 	if (ndw > (ring->ring_size / 4))
116 	radeon_ring_free_size(rdev, ring);
117 	ndw = (ndw + ring->align_mask) & ~ring->align_mask;
118 	while (ndw > (ring->ring_free_dw - 1)) {
119 		radeon_ring_free_size(rdev, ring);
120 		if (ndw < ring->ring_free_dw) {
123 		r = radeon_fence_wait_next(rdev, ring->idx);
127 	ring->count_dw = ndw;
128 	ring->wptr_old = ring->wptr;
133  * radeon_ring_lock - lock the ring and allocate space on it
136  * @ring: radeon_ring structure holding ring information
137  * @ndw: number of dwords to allocate in the ring buffer
139  * Lock the ring and allocate @ndw dwords in the ring buffer
143 int radeon_ring_lock(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ndw)
148 	r = radeon_ring_alloc(rdev, ring, ndw);
158  * commands on the ring buffer
161  * @ring: radeon_ring structure holding ring information
165  * execute new commands on the ring buffer (all asics).
167 void radeon_ring_commit(struct radeon_device *rdev, struct radeon_ring *ring,
170 	/* If we are emitting the HDP flush via the ring buffer, we need to
173 	if (hdp_flush && rdev->asic->ring[ring->idx]->hdp_flush)
174 		rdev->asic->ring[ring->idx]->hdp_flush(rdev, ring);
176 	while (ring->wptr & ring->align_mask) {
177 		radeon_ring_write(ring, ring->nop);
185 	radeon_ring_set_wptr(rdev, ring);
190  * commands on the ring buffer and unlock it
193  * @ring: radeon_ring structure holding ring information
196  * Call radeon_ring_commit() then unlock the ring (all asics).
198 void radeon_ring_unlock_commit(struct radeon_device *rdev, struct radeon_ring *ring,
201 	radeon_ring_commit(rdev, ring, hdp_flush);
208  * @ring: radeon_ring structure holding ring information
212 void radeon_ring_undo(struct radeon_ring *ring)
214 	ring->wptr = ring->wptr_old;
218  * radeon_ring_unlock_undo - reset the wptr and unlock the ring
221  * @ring: radeon_ring structure holding ring information
223  * Call radeon_ring_undo() then unlock the ring (all asics).
225 void radeon_ring_unlock_undo(struct radeon_device *rdev, struct radeon_ring *ring)
227 	radeon_ring_undo(ring);
235  * @ring: radeon_ring structure holding ring information
240 			       struct radeon_ring *ring)
242 	atomic_set(&ring->last_rptr, radeon_ring_get_rptr(rdev, ring));
243 	atomic64_set(&ring->last_activity, jiffies_64);
247  * radeon_ring_test_lockup() - check if ring is lockedup by recording information
249  * @ring:       radeon_ring structure holding ring information
252 bool radeon_ring_test_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
254 	uint32_t rptr = radeon_ring_get_rptr(rdev, ring);
255 	uint64_t last = atomic64_read(&ring->last_activity);
258 	if (rptr != atomic_read(&ring->last_rptr)) {
259 		/* ring is still working, no lockup */
260 		radeon_ring_lockup_update(rdev, ring);
266 		dev_err(rdev->dev, "ring %d stalled for more than %llumsec\n",
267 			ring->idx, elapsed);
275  * radeon_ring_backup - Back up the content of a ring
278  * @ring: the ring we want to back up
281  * Saves all unprocessed commits from a ring, returns the number of dwords saved.
283 unsigned radeon_ring_backup(struct radeon_device *rdev, struct radeon_ring *ring,
288 	/* just in case lock the ring */
292 	if (ring->ring_obj == NULL) {
298 	if (!radeon_fence_count_emitted(rdev, ring->idx)) {
303 	/* calculate the number of dw on the ring */
304 	if (ring->rptr_save_reg)
305 		ptr = RREG32(ring->rptr_save_reg);
307 		ptr = le32_to_cpu(*ring->next_rptr_cpu_addr);
314 	size = ring->wptr + (ring->ring_size / 4);
316 	size &= ring->ptr_mask;
322 	/* and then save the content of the ring */
329 		(*data)[i] = ring->ring[ptr++];
330 		ptr &= ring->ptr_mask;
338  * radeon_ring_restore - append saved commands to the ring again
341  * @ring: ring to append commands to
345  * Allocates space on the ring and restore the previously saved commands.
347 int radeon_ring_restore(struct radeon_device *rdev, struct radeon_ring *ring,
355 	/* restore the saved ring content */
356 	r = radeon_ring_lock(rdev, ring, size);
361 		radeon_ring_write(ring, data[i]);
364 	radeon_ring_unlock_commit(rdev, ring, false);
370  * radeon_ring_init - init driver ring struct.
373  * @ring: radeon_ring structure holding ring information
374  * @ring_size: size of the ring
376  * @nop: nop packet for this ring
378  * Initialize the driver information for the selected ring (all asics).
381 int radeon_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ring_size,
386 	ring->ring_size = ring_size;
387 	ring->rptr_offs = rptr_offs;
388 	ring->nop = nop;
389 	ring->rdev = rdev;
390 	/* Allocate ring buffer */
391 	if (ring->ring_obj == NULL) {
392 		r = radeon_bo_create(rdev, ring->ring_size, PAGE_SIZE, true,
394 				     NULL, &ring->ring_obj);
396 			dev_err(rdev->dev, "(%d) ring create failed\n", r);
399 		r = radeon_bo_reserve(ring->ring_obj, false);
402 		r = radeon_bo_pin(ring->ring_obj, RADEON_GEM_DOMAIN_GTT,
403 					&ring->gpu_addr);
405 			radeon_bo_unreserve(ring->ring_obj);
406 			dev_err(rdev->dev, "(%d) ring pin failed\n", r);
409 		r = radeon_bo_kmap(ring->ring_obj,
410 				       (void **)&ring->ring);
411 		radeon_bo_unreserve(ring->ring_obj);
413 			dev_err(rdev->dev, "(%d) ring map failed\n", r);
416 		radeon_debugfs_ring_init(rdev, ring);
418 	ring->ptr_mask = (ring->ring_size / 4) - 1;
419 	ring->ring_free_dw = ring->ring_size / 4;
421 		u32 index = RADEON_WB_RING0_NEXT_RPTR + (ring->idx * 4);
422 		ring->next_rptr_gpu_addr = rdev->wb.gpu_addr + index;
423 		ring->next_rptr_cpu_addr = &rdev->wb.wb[index/4];
425 	radeon_ring_lockup_update(rdev, ring);
430  * radeon_ring_fini - tear down the driver ring struct.
433  * @ring: radeon_ring structure holding ring information
435  * Tear down the driver information for the selected ring (all asics).
437 void radeon_ring_fini(struct radeon_device *rdev, struct radeon_ring *ring)
443 	ring_obj = ring->ring_obj;
444 	ring->ready = false;
445 	ring->ring = NULL;
446 	ring->ring_obj = NULL;
467 	struct radeon_ring *ring = m->private;
468 	struct radeon_device *rdev = ring->rdev;
473 	radeon_ring_free_size(rdev, ring);
474 	count = (ring->ring_size / 4) - ring->ring_free_dw;
476 	wptr = radeon_ring_get_wptr(rdev, ring);
480 	rptr = radeon_ring_get_rptr(rdev, ring);
484 	if (ring->rptr_save_reg) {
485 		rptr_next = RREG32(ring->rptr_save_reg);
487 			   ring->rptr_save_reg, rptr_next, rptr_next);
492 		   ring->wptr, ring->wptr);
494 		   ring->last_semaphore_signal_addr);
496 		   ring->last_semaphore_wait_addr);
497 	seq_printf(m, "%u free dwords in ring\n", ring->ring_free_dw);
498 	seq_printf(m, "%u dwords in ring\n", count);
500 	if (!ring->ring)
506 	i = (rptr + ring->ptr_mask + 1 - 32) & ring->ptr_mask;
508 		seq_printf(m, "r[%5d]=0x%08x", i, ring->ring[i]);
514 		i = (i + 1) & ring->ptr_mask;
547 static void radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *ring)
550 	const char *ring_name = radeon_debugfs_ring_idx_to_name(ring->idx);
554 		debugfs_create_file(ring_name, 0444, root, ring,