Lines Matching refs:wq
18 static void idxd_wq_disable_cleanup(struct idxd_wq *wq);
41 static void free_hw_descs(struct idxd_wq *wq)
45 for (i = 0; i < wq->num_descs; i++)
46 kfree(wq->hw_descs[i]);
48 kfree(wq->hw_descs);
51 static int alloc_hw_descs(struct idxd_wq *wq, int num)
53 struct device *dev = &wq->idxd->pdev->dev;
57 wq->hw_descs = kcalloc_node(num, sizeof(struct dsa_hw_desc *),
59 if (!wq->hw_descs)
63 wq->hw_descs[i] = kzalloc_node(sizeof(*wq->hw_descs[i]),
65 if (!wq->hw_descs[i]) {
66 free_hw_descs(wq);
74 static void free_descs(struct idxd_wq *wq)
78 for (i = 0; i < wq->num_descs; i++)
79 kfree(wq->descs[i]);
81 kfree(wq->descs);
84 static int alloc_descs(struct idxd_wq *wq, int num)
86 struct device *dev = &wq->idxd->pdev->dev;
90 wq->descs = kcalloc_node(num, sizeof(struct idxd_desc *),
92 if (!wq->descs)
96 wq->descs[i] = kzalloc_node(sizeof(*wq->descs[i]),
98 if (!wq->descs[i]) {
99 free_descs(wq);
108 int idxd_wq_alloc_resources(struct idxd_wq *wq)
110 struct idxd_device *idxd = wq->idxd;
114 if (wq->type != IDXD_WQT_KERNEL)
117 num_descs = wq_dedicated(wq) ? wq->size : wq->threshold;
118 wq->num_descs = num_descs;
120 rc = alloc_hw_descs(wq, num_descs);
124 wq->compls_size = num_descs * idxd->data->compl_size;
125 wq->compls = dma_alloc_coherent(dev, wq->compls_size, &wq->compls_addr, GFP_KERNEL);
126 if (!wq->compls) {
131 rc = alloc_descs(wq, num_descs);
135 rc = sbitmap_queue_init_node(&wq->sbq, num_descs, -1, false, GFP_KERNEL,
141 struct idxd_desc *desc = wq->descs[i];
143 desc->hw = wq->hw_descs[i];
145 desc->completion = &wq->compls[i];
147 desc->iax_completion = &wq->iax_compls[i];
148 desc->compl_dma = wq->compls_addr + idxd->data->compl_size * i;
150 desc->wq = wq;
157 free_descs(wq);
159 dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);
161 free_hw_descs(wq);
166 void idxd_wq_free_resources(struct idxd_wq *wq)
168 struct device *dev = &wq->idxd->pdev->dev;
170 if (wq->type != IDXD_WQT_KERNEL)
173 free_hw_descs(wq);
174 free_descs(wq);
175 dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);
176 sbitmap_queue_free(&wq->sbq);
180 int idxd_wq_enable(struct idxd_wq *wq)
182 struct idxd_device *idxd = wq->idxd;
186 if (wq->state == IDXD_WQ_ENABLED) {
187 dev_dbg(dev, "WQ %d already enabled\n", wq->id);
191 idxd_cmd_exec(idxd, IDXD_CMD_ENABLE_WQ, wq->id, &status);
199 wq->state = IDXD_WQ_ENABLED;
200 set_bit(wq->id, idxd->wq_enable_map);
201 dev_dbg(dev, "WQ %d enabled\n", wq->id);
205 int idxd_wq_disable(struct idxd_wq *wq, bool reset_config)
207 struct idxd_device *idxd = wq->idxd;
211 dev_dbg(dev, "Disabling WQ %d\n", wq->id);
213 if (wq->state != IDXD_WQ_ENABLED) {
214 dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);
218 operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);
227 idxd_wq_disable_cleanup(wq);
228 clear_bit(wq->id, idxd->wq_enable_map);
229 wq->state = IDXD_WQ_DISABLED;
230 dev_dbg(dev, "WQ %d disabled\n", wq->id);
234 void idxd_wq_drain(struct idxd_wq *wq)
236 struct idxd_device *idxd = wq->idxd;
240 if (wq->state != IDXD_WQ_ENABLED) {
241 dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);
245 dev_dbg(dev, "Draining WQ %d\n", wq->id);
246 operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);
250 void idxd_wq_reset(struct idxd_wq *wq)
252 struct idxd_device *idxd = wq->idxd;
256 if (wq->state != IDXD_WQ_ENABLED) {
257 dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);
261 operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);
263 idxd_wq_disable_cleanup(wq);
266 int idxd_wq_map_portal(struct idxd_wq *wq)
268 struct idxd_device *idxd = wq->idxd;
274 start += idxd_get_wq_portal_full_offset(wq->id, IDXD_PORTAL_LIMITED);
276 wq->portal = devm_ioremap(dev, start, IDXD_PORTAL_SIZE);
277 if (!wq->portal)
283 void idxd_wq_unmap_portal(struct idxd_wq *wq)
285 struct device *dev = &wq->idxd->pdev->dev;
287 devm_iounmap(dev, wq->portal);
288 wq->portal = NULL;
289 wq->portal_offset = 0;
297 struct idxd_wq *wq = idxd->wqs[i];
299 if (wq->portal)
300 idxd_wq_unmap_portal(wq);
304 static void __idxd_wq_set_pasid_locked(struct idxd_wq *wq, int pasid)
306 struct idxd_device *idxd = wq->idxd;
310 offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX);
315 wq->wqcfg->bits[WQCFG_PASID_IDX] = wqcfg.bits[WQCFG_PASID_IDX];
320 int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid)
324 rc = idxd_wq_disable(wq, false);
328 __idxd_wq_set_pasid_locked(wq, pasid);
330 rc = idxd_wq_enable(wq);
337 int idxd_wq_disable_pasid(struct idxd_wq *wq)
339 struct idxd_device *idxd = wq->idxd;
344 rc = idxd_wq_disable(wq, false);
348 offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX);
356 rc = idxd_wq_enable(wq);
363 static void idxd_wq_disable_cleanup(struct idxd_wq *wq)
365 struct idxd_device *idxd = wq->idxd;
367 lockdep_assert_held(&wq->wq_lock);
368 wq->state = IDXD_WQ_DISABLED;
369 memset(wq->wqcfg, 0, idxd->wqcfg_size);
370 wq->type = IDXD_WQT_NONE;
371 wq->threshold = 0;
372 wq->priority = 0;
373 wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES;
374 wq->flags = 0;
375 memset(wq->name, 0, WQ_NAME_SIZE);
376 wq->max_xfer_bytes = WQ_DEFAULT_MAX_XFER;
377 idxd_wq_set_max_batch_size(idxd->data->type, wq, WQ_DEFAULT_MAX_BATCH);
378 if (wq->opcap_bmap)
379 bitmap_copy(wq->opcap_bmap, idxd->opcap_bmap, IDXD_MAX_OPCAP_BITS);
382 static void idxd_wq_device_reset_cleanup(struct idxd_wq *wq)
384 lockdep_assert_held(&wq->wq_lock);
386 wq->size = 0;
387 wq->group = NULL;
392 struct idxd_wq *wq = container_of(ref, struct idxd_wq, wq_active);
394 complete(&wq->wq_dead);
397 int idxd_wq_init_percpu_ref(struct idxd_wq *wq)
401 memset(&wq->wq_active, 0, sizeof(wq->wq_active));
402 rc = percpu_ref_init(&wq->wq_active, idxd_wq_ref_release,
406 reinit_completion(&wq->wq_dead);
407 reinit_completion(&wq->wq_resurrect);
412 void __idxd_wq_quiesce(struct idxd_wq *wq)
414 lockdep_assert_held(&wq->wq_lock);
415 reinit_completion(&wq->wq_resurrect);
416 percpu_ref_kill(&wq->wq_active);
417 complete_all(&wq->wq_resurrect);
418 wait_for_completion(&wq->wq_dead);
422 void idxd_wq_quiesce(struct idxd_wq *wq)
424 mutex_lock(&wq->wq_lock);
425 __idxd_wq_quiesce(wq);
426 mutex_unlock(&wq->wq_lock);
714 struct idxd_wq *wq = idxd->wqs[i];
716 mutex_lock(&wq->wq_lock);
717 idxd_wq_disable_cleanup(wq);
718 idxd_wq_device_reset_cleanup(wq);
719 mutex_unlock(&wq->wq_lock);
728 * Clearing wq state is protected by wq lock.
857 dev_dbg(dev, "GRPCFG wq[%d:%d: %#x]: %#llx\n",
911 static int idxd_wq_config_write(struct idxd_wq *wq)
913 struct idxd_device *idxd = wq->idxd;
918 if (!wq->group)
923 * wq reset. This will copy back the sticky values that are present on some devices.
926 wq_offset = WQCFG_OFFSET(idxd, wq->id, i);
927 wq->wqcfg->bits[i] |= ioread32(idxd->reg_base + wq_offset);
930 if (wq->size == 0 && wq->type != IDXD_WQT_NONE)
931 wq->size = WQ_DEFAULT_QUEUE_DEPTH;
934 wq->wqcfg->wq_size = wq->size;
937 wq->wqcfg->wq_thresh = wq->threshold;
940 if (wq_dedicated(wq))
941 wq->wqcfg->mode = 1;
954 if (wq_dedicated(wq) && wq->wqcfg->pasid_en &&
956 wq->type == IDXD_WQT_KERNEL) {
961 wq->wqcfg->priority = wq->priority;
964 test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags) &&
965 !test_bit(WQ_FLAG_PRS_DISABLE, &wq->flags))
966 wq->wqcfg->bof = 1;
969 wq->wqcfg->wq_ats_disable = test_bit(WQ_FLAG_ATS_DISABLE, &wq->flags);
972 wq->wqcfg->wq_prs_disable = test_bit(WQ_FLAG_PRS_DISABLE, &wq->flags);
975 wq->wqcfg->max_xfer_shift = ilog2(wq->max_xfer_bytes);
976 idxd_wqcfg_set_max_batch_shift(idxd->data->type, wq->wqcfg, ilog2(wq->max_batch_size));
979 if (idxd->hw.wq_cap.op_config && wq->opcap_bmap) {
980 memset(wq->wqcfg->op_config, 0, IDXD_MAX_OPCAP_BITS / 8);
981 for_each_set_bit(n, wq->opcap_bmap, IDXD_MAX_OPCAP_BITS) {
985 wq->wqcfg->op_config[idx] |= BIT(pos);
989 dev_dbg(dev, "WQ %d CFGs\n", wq->id);
991 wq_offset = WQCFG_OFFSET(idxd, wq->id, i);
992 iowrite32(wq->wqcfg->bits[i], idxd->reg_base + wq_offset);
994 wq->id, i, wq_offset,
1006 struct idxd_wq *wq = idxd->wqs[i];
1008 rc = idxd_wq_config_write(wq);
1070 struct idxd_wq *wq;
1082 wq = idxd->wqs[i];
1083 group = wq->group;
1085 if (!wq->group)
1088 if (wq_shared(wq) && !wq_shared_supported(wq)) {
1090 dev_warn(dev, "No shared wq support but configured.\n");
1094 group->grpcfg.wqs[wq->id / 64] |= BIT(wq->id % 64);
1132 static int idxd_wq_load_config(struct idxd_wq *wq)
1134 struct idxd_device *idxd = wq->idxd;
1139 wqcfg_offset = WQCFG_OFFSET(idxd, wq->id, 0);
1140 memcpy_fromio(wq->wqcfg, idxd->reg_base + wqcfg_offset, idxd->wqcfg_size);
1142 wq->size = wq->wqcfg->wq_size;
1143 wq->threshold = wq->wqcfg->wq_thresh;
1146 if (wq->wqcfg->mode == 0 || wq->wqcfg->pasid_en)
1149 set_bit(WQ_FLAG_DEDICATED, &wq->flags);
1151 wq->priority = wq->wqcfg->priority;
1153 wq->max_xfer_bytes = 1ULL << wq->wqcfg->max_xfer_shift;
1154 idxd_wq_set_max_batch_size(idxd->data->type, wq, 1U << wq->wqcfg->max_batch_shift);
1157 wqcfg_offset = WQCFG_OFFSET(idxd, wq->id, i);
1158 dev_dbg(dev, "WQ[%d][%d][%#x]: %#x\n", wq->id, i, wqcfg_offset, wq->wqcfg->bits[i]);
1175 struct idxd_wq *wq;
1179 dev_dbg(dev, "GRPCFG wq[%d:%d: %#x]: %#llx\n",
1185 /* Iterate through all 64 bits and check for wq set */
1193 /* Set group assignment for wq if wq bit is set */
1195 wq = idxd->wqs[id];
1196 wq->group = group;
1239 struct idxd_wq *wq = idxd->wqs[i];
1241 rc = idxd_wq_load_config(wq);
1273 * wq is being disabled. Any remaining descriptors are
1305 void idxd_wq_free_irq(struct idxd_wq *wq)
1307 struct idxd_device *idxd = wq->idxd;
1308 struct idxd_irq_entry *ie = &wq->ie;
1310 if (wq->type != IDXD_WQT_KERNEL)
1323 int idxd_wq_request_irq(struct idxd_wq *wq)
1325 struct idxd_device *idxd = wq->idxd;
1331 if (wq->type != IDXD_WQT_KERNEL)
1334 ie = &wq->ie;
1365 int idxd_drv_enable_wq(struct idxd_wq *wq)
1367 struct idxd_device *idxd = wq->idxd;
1371 lockdep_assert_held(&wq->wq_lock);
1378 if (wq->state != IDXD_WQ_DISABLED) {
1379 dev_dbg(dev, "wq %d already enabled.\n", wq->id);
1385 if (!wq->group) {
1386 dev_dbg(dev, "wq %d not attached to group.\n", wq->id);
1391 if (strlen(wq->name) == 0) {
1393 dev_dbg(dev, "wq %d name not set.\n", wq->id);
1398 if (wq_shared(wq)) {
1399 if (!wq_shared_supported(wq)) {
1401 dev_dbg(dev, "PASID not enabled and shared wq.\n");
1405 * Shared wq with the threshold set to 0 means the user
1407 * dedicated wq but did not set threshold. A value
1408 * of 0 would effectively disable the shared wq. The
1412 if (wq->threshold == 0) {
1414 dev_dbg(dev, "Shared wq and threshold 0.\n");
1424 * A dedicated wq that is not 'kernel' type will configure pasid and
1428 if (wq_pasid_enabled(wq)) {
1429 if (is_idxd_wq_kernel(wq) || wq_shared(wq)) {
1430 u32 pasid = wq_dedicated(wq) ? idxd->pasid : 0;
1432 __idxd_wq_set_pasid_locked(wq, pasid);
1443 dev_dbg(dev, "Writing wq %d config failed: %d\n", wq->id, rc);
1447 rc = idxd_wq_enable(wq);
1449 dev_dbg(dev, "wq %d enabling failed: %d\n", wq->id, rc);
1453 rc = idxd_wq_map_portal(wq);
1456 dev_dbg(dev, "wq %d portal mapping failed: %d\n", wq->id, rc);
1460 wq->client_count = 0;
1462 rc = idxd_wq_request_irq(wq);
1465 dev_dbg(dev, "WQ %d irq setup failed: %d\n", wq->id, rc);
1469 rc = idxd_wq_alloc_resources(wq);
1476 rc = idxd_wq_init_percpu_ref(wq);
1486 idxd_wq_free_resources(wq);
1488 idxd_wq_free_irq(wq);
1490 idxd_wq_unmap_portal(wq);
1492 if (idxd_wq_disable(wq, false))
1493 dev_dbg(dev, "wq %s disable failed\n", dev_name(wq_confdev(wq)));
1499 void idxd_drv_disable_wq(struct idxd_wq *wq)
1501 struct idxd_device *idxd = wq->idxd;
1504 lockdep_assert_held(&wq->wq_lock);
1506 if (idxd_wq_refcount(wq))
1507 dev_warn(dev, "Clients has claim on wq %d: %d\n",
1508 wq->id, idxd_wq_refcount(wq));
1510 idxd_wq_unmap_portal(wq);
1511 idxd_wq_drain(wq);
1512 idxd_wq_free_irq(wq);
1513 idxd_wq_reset(wq);
1514 idxd_wq_free_resources(wq);
1515 percpu_ref_exit(&wq->wq_active);
1516 wq->type = IDXD_WQT_NONE;
1517 wq->client_count = 0;
1586 struct idxd_wq *wq = idxd->wqs[i];
1587 struct device *wq_dev = wq_confdev(wq);
1589 if (wq->state == IDXD_WQ_DISABLED)
1591 dev_warn(dev, "Active wq %d on disable %s.\n", i, dev_name(wq_dev));