1// SPDX-License-Identifier: MIT
2/*
3 * Copyright 2022 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 *
23 */
24
25#include "amdgpu.h"
26
27/**
28 * amdgpu_mm_rdoorbell - read a doorbell dword
29 *
30 * @adev: amdgpu_device pointer
31 * @index: doorbell index
32 *
33 * Returns the value in the doorbell aperture at the
34 * requested doorbell index (CIK).
35 */
36u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
37{
38	if (amdgpu_device_skip_hw_access(adev))
39		return 0;
40
41	if (index < adev->doorbell.num_kernel_doorbells)
42		return readl(adev->doorbell.cpu_addr + index);
43
44	DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
45	return 0;
46}
47
48/**
49 * amdgpu_mm_wdoorbell - write a doorbell dword
50 *
51 * @adev: amdgpu_device pointer
52 * @index: doorbell index
53 * @v: value to write
54 *
55 * Writes @v to the doorbell aperture at the
56 * requested doorbell index (CIK).
57 */
58void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
59{
60	if (amdgpu_device_skip_hw_access(adev))
61		return;
62
63	if (index < adev->doorbell.num_kernel_doorbells)
64		writel(v, adev->doorbell.cpu_addr + index);
65	else
66		DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
67}
68
69/**
70 * amdgpu_mm_rdoorbell64 - read a doorbell Qword
71 *
72 * @adev: amdgpu_device pointer
73 * @index: doorbell index
74 *
75 * Returns the value in the doorbell aperture at the
76 * requested doorbell index (VEGA10+).
77 */
78u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
79{
80	if (amdgpu_device_skip_hw_access(adev))
81		return 0;
82
83	if (index < adev->doorbell.num_kernel_doorbells)
84		return atomic64_read((atomic64_t *)(adev->doorbell.cpu_addr + index));
85
86	DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
87	return 0;
88}
89
90/**
91 * amdgpu_mm_wdoorbell64 - write a doorbell Qword
92 *
93 * @adev: amdgpu_device pointer
94 * @index: doorbell index
95 * @v: value to write
96 *
97 * Writes @v to the doorbell aperture at the
98 * requested doorbell index (VEGA10+).
99 */
100void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
101{
102	if (amdgpu_device_skip_hw_access(adev))
103		return;
104
105	if (index < adev->doorbell.num_kernel_doorbells)
106		atomic64_set((atomic64_t *)(adev->doorbell.cpu_addr + index), v);
107	else
108		DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
109}
110
111/**
112 * amdgpu_doorbell_index_on_bar - Find doorbell's absolute offset in BAR
113 *
114 * @adev: amdgpu_device pointer
115 * @db_bo: doorbell object's bo
116 * @doorbell_index: doorbell relative index in this doorbell object
117 * @db_size: doorbell size is in byte
118 *
119 * returns doorbell's absolute index in BAR
120 */
121uint32_t amdgpu_doorbell_index_on_bar(struct amdgpu_device *adev,
122				      struct amdgpu_bo *db_bo,
123				      uint32_t doorbell_index,
124				      uint32_t db_size)
125{
126	int db_bo_offset;
127
128	db_bo_offset = amdgpu_bo_gpu_offset_no_check(db_bo);
129
130	/* doorbell index is 32 bit but doorbell's size can be 32 bit
131	 * or 64 bit, so *db_size(in byte)/4 for alignment.
132	 */
133	return db_bo_offset / sizeof(u32) + doorbell_index *
134	       DIV_ROUND_UP(db_size, 4);
135}
136
137/**
138 * amdgpu_doorbell_create_kernel_doorbells - Create kernel doorbells for graphics
139 *
140 * @adev: amdgpu_device pointer
141 *
142 * Creates doorbells for graphics driver usages.
143 * returns 0 on success, error otherwise.
144 */
145int amdgpu_doorbell_create_kernel_doorbells(struct amdgpu_device *adev)
146{
147	int r;
148	int size;
149
150	/* SI HW does not have doorbells, skip allocation */
151	if (adev->doorbell.num_kernel_doorbells == 0)
152		return 0;
153
154	/* Reserve first num_kernel_doorbells (page-aligned) for kernel ops */
155	size = ALIGN(adev->doorbell.num_kernel_doorbells * sizeof(u32), PAGE_SIZE);
156
157	/* Allocate an extra page for MES kernel usages (ring test) */
158	adev->mes.db_start_dw_offset = size / sizeof(u32);
159	size += PAGE_SIZE;
160
161	r = amdgpu_bo_create_kernel(adev,
162				    size,
163				    PAGE_SIZE,
164				    AMDGPU_GEM_DOMAIN_DOORBELL,
165				    &adev->doorbell.kernel_doorbells,
166				    NULL,
167				    (void **)&adev->doorbell.cpu_addr);
168	if (r) {
169		DRM_ERROR("Failed to allocate kernel doorbells, err=%d\n", r);
170		return r;
171	}
172
173	adev->doorbell.num_kernel_doorbells = size / sizeof(u32);
174	return 0;
175}
176
177/*
178 * GPU doorbell aperture helpers function.
179 */
180/**
181 * amdgpu_doorbell_init - Init doorbell driver information.
182 *
183 * @adev: amdgpu_device pointer
184 *
185 * Init doorbell driver information (CIK)
186 * Returns 0 on success, error on failure.
187 */
188int amdgpu_doorbell_init(struct amdgpu_device *adev)
189{
190
191	/* No doorbell on SI hardware generation */
192	if (adev->asic_type < CHIP_BONAIRE) {
193		adev->doorbell.base = 0;
194		adev->doorbell.size = 0;
195		adev->doorbell.num_kernel_doorbells = 0;
196		return 0;
197	}
198
199	if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
200		return -EINVAL;
201
202	amdgpu_asic_init_doorbell_index(adev);
203
204	/* doorbell bar mapping */
205	adev->doorbell.base = pci_resource_start(adev->pdev, 2);
206	adev->doorbell.size = pci_resource_len(adev->pdev, 2);
207
208	adev->doorbell.num_kernel_doorbells =
209		min_t(u32, adev->doorbell.size / sizeof(u32),
210		      adev->doorbell_index.max_assignment + 1);
211	if (adev->doorbell.num_kernel_doorbells == 0)
212		return -EINVAL;
213
214	/*
215	 * For Vega, reserve and map two pages on doorbell BAR since SDMA
216	 * paging queue doorbell use the second page. The
217	 * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
218	 * doorbells are in the first page. So with paging queue enabled,
219	 * the max num_kernel_doorbells should + 1 page (0x400 in dword)
220	 */
221	if (adev->asic_type >= CHIP_VEGA10)
222		adev->doorbell.num_kernel_doorbells += 0x400;
223
224	return 0;
225}
226
227/**
228 * amdgpu_doorbell_fini - Tear down doorbell driver information.
229 *
230 * @adev: amdgpu_device pointer
231 *
232 * Tear down doorbell driver information (CIK)
233 */
234void amdgpu_doorbell_fini(struct amdgpu_device *adev)
235{
236	amdgpu_bo_free_kernel(&adev->doorbell.kernel_doorbells,
237			      NULL,
238			      (void **)&adev->doorbell.cpu_addr);
239}
240