1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (C) 2023, Intel Corporation. */
3
4#include "ice.h"
5#include "ice_lib.h"
6#include "ice_irq.h"
7
8/**
9 * ice_init_irq_tracker - initialize interrupt tracker
10 * @pf: board private structure
11 * @max_vectors: maximum number of vectors that tracker can hold
12 * @num_static: number of preallocated interrupts
13 */
14static void
15ice_init_irq_tracker(struct ice_pf *pf, unsigned int max_vectors,
16		     unsigned int num_static)
17{
18	pf->irq_tracker.num_entries = max_vectors;
19	pf->irq_tracker.num_static = num_static;
20	xa_init_flags(&pf->irq_tracker.entries, XA_FLAGS_ALLOC);
21}
22
23/**
24 * ice_deinit_irq_tracker - free xarray tracker
25 * @pf: board private structure
26 */
27static void ice_deinit_irq_tracker(struct ice_pf *pf)
28{
29	xa_destroy(&pf->irq_tracker.entries);
30}
31
32/**
33 * ice_free_irq_res - free a block of resources
34 * @pf: board private structure
35 * @index: starting index previously returned by ice_get_res
36 */
37static void ice_free_irq_res(struct ice_pf *pf, u16 index)
38{
39	struct ice_irq_entry *entry;
40
41	entry = xa_erase(&pf->irq_tracker.entries, index);
42	kfree(entry);
43}
44
45/**
46 * ice_get_irq_res - get an interrupt resource
47 * @pf: board private structure
48 * @dyn_only: force entry to be dynamically allocated
49 *
50 * Allocate new irq entry in the free slot of the tracker. Since xarray
51 * is used, always allocate new entry at the lowest possible index. Set
52 * proper allocation limit for maximum tracker entries.
53 *
54 * Returns allocated irq entry or NULL on failure.
55 */
56static struct ice_irq_entry *ice_get_irq_res(struct ice_pf *pf, bool dyn_only)
57{
58	struct xa_limit limit = { .max = pf->irq_tracker.num_entries,
59				  .min = 0 };
60	unsigned int num_static = pf->irq_tracker.num_static;
61	struct ice_irq_entry *entry;
62	unsigned int index;
63	int ret;
64
65	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
66	if (!entry)
67		return NULL;
68
69	/* skip preallocated entries if the caller says so */
70	if (dyn_only)
71		limit.min = num_static;
72
73	ret = xa_alloc(&pf->irq_tracker.entries, &index, entry, limit,
74		       GFP_KERNEL);
75
76	if (ret) {
77		kfree(entry);
78		entry = NULL;
79	} else {
80		entry->index = index;
81		entry->dynamic = index >= num_static;
82	}
83
84	return entry;
85}
86
87/**
88 * ice_reduce_msix_usage - Reduce usage of MSI-X vectors
89 * @pf: board private structure
90 * @v_remain: number of remaining MSI-X vectors to be distributed
91 *
92 * Reduce the usage of MSI-X vectors when entire request cannot be fulfilled.
93 * pf->num_lan_msix and pf->num_rdma_msix values are set based on number of
94 * remaining vectors.
95 */
96static void ice_reduce_msix_usage(struct ice_pf *pf, int v_remain)
97{
98	int v_rdma;
99
100	if (!ice_is_rdma_ena(pf)) {
101		pf->num_lan_msix = v_remain;
102		return;
103	}
104
105	/* RDMA needs at least 1 interrupt in addition to AEQ MSIX */
106	v_rdma = ICE_RDMA_NUM_AEQ_MSIX + 1;
107
108	if (v_remain < ICE_MIN_LAN_TXRX_MSIX + ICE_MIN_RDMA_MSIX) {
109		dev_warn(ice_pf_to_dev(pf), "Not enough MSI-X vectors to support RDMA.\n");
110		clear_bit(ICE_FLAG_RDMA_ENA, pf->flags);
111
112		pf->num_rdma_msix = 0;
113		pf->num_lan_msix = ICE_MIN_LAN_TXRX_MSIX;
114	} else if ((v_remain < ICE_MIN_LAN_TXRX_MSIX + v_rdma) ||
115		   (v_remain - v_rdma < v_rdma)) {
116		/* Support minimum RDMA and give remaining vectors to LAN MSIX
117		 */
118		pf->num_rdma_msix = ICE_MIN_RDMA_MSIX;
119		pf->num_lan_msix = v_remain - ICE_MIN_RDMA_MSIX;
120	} else {
121		/* Split remaining MSIX with RDMA after accounting for AEQ MSIX
122		 */
123		pf->num_rdma_msix = (v_remain - ICE_RDMA_NUM_AEQ_MSIX) / 2 +
124				    ICE_RDMA_NUM_AEQ_MSIX;
125		pf->num_lan_msix = v_remain - pf->num_rdma_msix;
126	}
127}
128
129/**
130 * ice_ena_msix_range - Request a range of MSIX vectors from the OS
131 * @pf: board private structure
132 *
133 * Compute the number of MSIX vectors wanted and request from the OS. Adjust
134 * device usage if there are not enough vectors. Return the number of vectors
135 * reserved or negative on failure.
136 */
137static int ice_ena_msix_range(struct ice_pf *pf)
138{
139	int num_cpus, hw_num_msix, v_other, v_wanted, v_actual;
140	struct device *dev = ice_pf_to_dev(pf);
141	int err;
142
143	hw_num_msix = pf->hw.func_caps.common_cap.num_msix_vectors;
144	num_cpus = num_online_cpus();
145
146	/* LAN miscellaneous handler */
147	v_other = ICE_MIN_LAN_OICR_MSIX;
148
149	/* Flow Director */
150	if (test_bit(ICE_FLAG_FD_ENA, pf->flags))
151		v_other += ICE_FDIR_MSIX;
152
153	/* switchdev */
154	v_other += ICE_ESWITCH_MSIX;
155
156	v_wanted = v_other;
157
158	/* LAN traffic */
159	pf->num_lan_msix = num_cpus;
160	v_wanted += pf->num_lan_msix;
161
162	/* RDMA auxiliary driver */
163	if (ice_is_rdma_ena(pf)) {
164		pf->num_rdma_msix = num_cpus + ICE_RDMA_NUM_AEQ_MSIX;
165		v_wanted += pf->num_rdma_msix;
166	}
167
168	if (v_wanted > hw_num_msix) {
169		int v_remain;
170
171		dev_warn(dev, "not enough device MSI-X vectors. wanted = %d, available = %d\n",
172			 v_wanted, hw_num_msix);
173
174		if (hw_num_msix < ICE_MIN_MSIX) {
175			err = -ERANGE;
176			goto exit_err;
177		}
178
179		v_remain = hw_num_msix - v_other;
180		if (v_remain < ICE_MIN_LAN_TXRX_MSIX) {
181			v_other = ICE_MIN_MSIX - ICE_MIN_LAN_TXRX_MSIX;
182			v_remain = ICE_MIN_LAN_TXRX_MSIX;
183		}
184
185		ice_reduce_msix_usage(pf, v_remain);
186		v_wanted = pf->num_lan_msix + pf->num_rdma_msix + v_other;
187
188		dev_notice(dev, "Reducing request to %d MSI-X vectors for LAN traffic.\n",
189			   pf->num_lan_msix);
190		if (ice_is_rdma_ena(pf))
191			dev_notice(dev, "Reducing request to %d MSI-X vectors for RDMA.\n",
192				   pf->num_rdma_msix);
193	}
194
195	/* actually reserve the vectors */
196	v_actual = pci_alloc_irq_vectors(pf->pdev, ICE_MIN_MSIX, v_wanted,
197					 PCI_IRQ_MSIX);
198	if (v_actual < 0) {
199		dev_err(dev, "unable to reserve MSI-X vectors\n");
200		err = v_actual;
201		goto exit_err;
202	}
203
204	if (v_actual < v_wanted) {
205		dev_warn(dev, "not enough OS MSI-X vectors. requested = %d, obtained = %d\n",
206			 v_wanted, v_actual);
207
208		if (v_actual < ICE_MIN_MSIX) {
209			/* error if we can't get minimum vectors */
210			pci_free_irq_vectors(pf->pdev);
211			err = -ERANGE;
212			goto exit_err;
213		} else {
214			int v_remain = v_actual - v_other;
215
216			if (v_remain < ICE_MIN_LAN_TXRX_MSIX)
217				v_remain = ICE_MIN_LAN_TXRX_MSIX;
218
219			ice_reduce_msix_usage(pf, v_remain);
220
221			dev_notice(dev, "Enabled %d MSI-X vectors for LAN traffic.\n",
222				   pf->num_lan_msix);
223
224			if (ice_is_rdma_ena(pf))
225				dev_notice(dev, "Enabled %d MSI-X vectors for RDMA.\n",
226					   pf->num_rdma_msix);
227		}
228	}
229
230	return v_actual;
231
232exit_err:
233	pf->num_rdma_msix = 0;
234	pf->num_lan_msix = 0;
235	return err;
236}
237
238/**
239 * ice_clear_interrupt_scheme - Undo things done by ice_init_interrupt_scheme
240 * @pf: board private structure
241 */
242void ice_clear_interrupt_scheme(struct ice_pf *pf)
243{
244	pci_free_irq_vectors(pf->pdev);
245	ice_deinit_irq_tracker(pf);
246}
247
248/**
249 * ice_init_interrupt_scheme - Determine proper interrupt scheme
250 * @pf: board private structure to initialize
251 */
252int ice_init_interrupt_scheme(struct ice_pf *pf)
253{
254	int total_vectors = pf->hw.func_caps.common_cap.num_msix_vectors;
255	int vectors, max_vectors;
256
257	vectors = ice_ena_msix_range(pf);
258
259	if (vectors < 0)
260		return -ENOMEM;
261
262	if (pci_msix_can_alloc_dyn(pf->pdev))
263		max_vectors = total_vectors;
264	else
265		max_vectors = vectors;
266
267	ice_init_irq_tracker(pf, max_vectors, vectors);
268
269	return 0;
270}
271
272/**
273 * ice_alloc_irq - Allocate new interrupt vector
274 * @pf: board private structure
275 * @dyn_only: force dynamic allocation of the interrupt
276 *
277 * Allocate new interrupt vector for a given owner id.
278 * return struct msi_map with interrupt details and track
279 * allocated interrupt appropriately.
280 *
281 * This function reserves new irq entry from the irq_tracker.
282 * if according to the tracker information all interrupts that
283 * were allocated with ice_pci_alloc_irq_vectors are already used
284 * and dynamically allocated interrupts are supported then new
285 * interrupt will be allocated with pci_msix_alloc_irq_at.
286 *
287 * Some callers may only support dynamically allocated interrupts.
288 * This is indicated with dyn_only flag.
289 *
290 * On failure, return map with negative .index. The caller
291 * is expected to check returned map index.
292 *
293 */
294struct msi_map ice_alloc_irq(struct ice_pf *pf, bool dyn_only)
295{
296	int sriov_base_vector = pf->sriov_base_vector;
297	struct msi_map map = { .index = -ENOENT };
298	struct device *dev = ice_pf_to_dev(pf);
299	struct ice_irq_entry *entry;
300
301	entry = ice_get_irq_res(pf, dyn_only);
302	if (!entry)
303		return map;
304
305	/* fail if we're about to violate SRIOV vectors space */
306	if (sriov_base_vector && entry->index >= sriov_base_vector)
307		goto exit_free_res;
308
309	if (pci_msix_can_alloc_dyn(pf->pdev) && entry->dynamic) {
310		map = pci_msix_alloc_irq_at(pf->pdev, entry->index, NULL);
311		if (map.index < 0)
312			goto exit_free_res;
313		dev_dbg(dev, "allocated new irq at index %d\n", map.index);
314	} else {
315		map.index = entry->index;
316		map.virq = pci_irq_vector(pf->pdev, map.index);
317	}
318
319	return map;
320
321exit_free_res:
322	dev_err(dev, "Could not allocate irq at idx %d\n", entry->index);
323	ice_free_irq_res(pf, entry->index);
324	return map;
325}
326
327/**
328 * ice_free_irq - Free interrupt vector
329 * @pf: board private structure
330 * @map: map with interrupt details
331 *
332 * Remove allocated interrupt from the interrupt tracker. If interrupt was
333 * allocated dynamically, free respective interrupt vector.
334 */
335void ice_free_irq(struct ice_pf *pf, struct msi_map map)
336{
337	struct ice_irq_entry *entry;
338
339	entry = xa_load(&pf->irq_tracker.entries, map.index);
340
341	if (!entry) {
342		dev_err(ice_pf_to_dev(pf), "Failed to get MSIX interrupt entry at index %d",
343			map.index);
344		return;
345	}
346
347	dev_dbg(ice_pf_to_dev(pf), "Free irq at index %d\n", map.index);
348
349	if (entry->dynamic)
350		pci_msix_free_irq(pf->pdev, map);
351
352	ice_free_irq_res(pf, map.index);
353}
354
355/**
356 * ice_get_max_used_msix_vector - Get the max used interrupt vector
357 * @pf: board private structure
358 *
359 * Return index of maximum used interrupt vectors with respect to the
360 * beginning of the MSIX table. Take into account that some interrupts
361 * may have been dynamically allocated after MSIX was initially enabled.
362 */
363int ice_get_max_used_msix_vector(struct ice_pf *pf)
364{
365	unsigned long start, index, max_idx;
366	void *entry;
367
368	/* Treat all preallocated interrupts as used */
369	start = pf->irq_tracker.num_static;
370	max_idx = start - 1;
371
372	xa_for_each_start(&pf->irq_tracker.entries, index, entry, start) {
373		if (index > max_idx)
374			max_idx = index;
375	}
376
377	return max_idx;
378}
379