1// SPDX-License-Identifier: GPL-2.0-only OR MIT 2/* Copyright (c) 2023 Imagination Technologies Ltd. */ 3 4#include "pvr_ccb.h" 5#include "pvr_cccb.h" 6#include "pvr_device.h" 7#include "pvr_gem.h" 8#include "pvr_hwrt.h" 9 10#include <linux/compiler.h> 11#include <linux/delay.h> 12#include <linux/jiffies.h> 13#include <linux/mutex.h> 14#include <linux/types.h> 15 16static __always_inline u32 17get_ccb_space(u32 w_off, u32 r_off, u32 ccb_size) 18{ 19 return (((r_off) - (w_off)) + ((ccb_size) - 1)) & ((ccb_size) - 1); 20} 21 22static void 23cccb_ctrl_init(void *cpu_ptr, void *priv) 24{ 25 struct rogue_fwif_cccb_ctl *ctrl = cpu_ptr; 26 struct pvr_cccb *pvr_cccb = priv; 27 28 WRITE_ONCE(ctrl->write_offset, 0); 29 WRITE_ONCE(ctrl->read_offset, 0); 30 WRITE_ONCE(ctrl->dep_offset, 0); 31 WRITE_ONCE(ctrl->wrap_mask, pvr_cccb->wrap_mask); 32} 33 34/** 35 * pvr_cccb_init() - Initialise a Client CCB 36 * @pvr_dev: Device pointer. 37 * @pvr_cccb: Pointer to Client CCB structure to initialise. 38 * @size_log2: Log2 size of Client CCB in bytes. 39 * @name: Name of owner of Client CCB. Used for fence context. 40 * 41 * Return: 42 * * Zero on success, or 43 * * Any error code returned by pvr_fw_object_create_and_map(). 44 */ 45int 46pvr_cccb_init(struct pvr_device *pvr_dev, struct pvr_cccb *pvr_cccb, 47 u32 size_log2, const char *name) 48{ 49 size_t size = 1 << size_log2; 50 int err; 51 52 pvr_cccb->size = size; 53 pvr_cccb->write_offset = 0; 54 pvr_cccb->wrap_mask = size - 1; 55 56 /* 57 * Map CCCB and control structure as uncached, so we don't have to flush 58 * CPU cache repeatedly when polling for space. 59 */ 60 pvr_cccb->ctrl = pvr_fw_object_create_and_map(pvr_dev, sizeof(*pvr_cccb->ctrl), 61 PVR_BO_FW_FLAGS_DEVICE_UNCACHED, 62 cccb_ctrl_init, pvr_cccb, 63 &pvr_cccb->ctrl_obj); 64 if (IS_ERR(pvr_cccb->ctrl)) 65 return PTR_ERR(pvr_cccb->ctrl); 66 67 pvr_cccb->cccb = pvr_fw_object_create_and_map(pvr_dev, size, 68 PVR_BO_FW_FLAGS_DEVICE_UNCACHED, 69 NULL, NULL, &pvr_cccb->cccb_obj); 70 if (IS_ERR(pvr_cccb->cccb)) { 71 err = PTR_ERR(pvr_cccb->cccb); 72 goto err_free_ctrl; 73 } 74 75 pvr_fw_object_get_fw_addr(pvr_cccb->ctrl_obj, &pvr_cccb->ctrl_fw_addr); 76 pvr_fw_object_get_fw_addr(pvr_cccb->cccb_obj, &pvr_cccb->cccb_fw_addr); 77 78 return 0; 79 80err_free_ctrl: 81 pvr_fw_object_unmap_and_destroy(pvr_cccb->ctrl_obj); 82 83 return err; 84} 85 86/** 87 * pvr_cccb_fini() - Release Client CCB structure 88 * @pvr_cccb: Client CCB to release. 89 */ 90void 91pvr_cccb_fini(struct pvr_cccb *pvr_cccb) 92{ 93 pvr_fw_object_unmap_and_destroy(pvr_cccb->cccb_obj); 94 pvr_fw_object_unmap_and_destroy(pvr_cccb->ctrl_obj); 95} 96 97/** 98 * pvr_cccb_cmdseq_fits() - Check if a command sequence fits in the CCCB 99 * @pvr_cccb: Target Client CCB. 100 * @size: Size of the command sequence. 101 * 102 * Check if a command sequence fits in the CCCB we have at hand. 103 * 104 * Return: 105 * * true if the command sequence fits in the CCCB, or 106 * * false otherwise. 107 */ 108bool pvr_cccb_cmdseq_fits(struct pvr_cccb *pvr_cccb, size_t size) 109{ 110 struct rogue_fwif_cccb_ctl *ctrl = pvr_cccb->ctrl; 111 u32 read_offset, remaining; 112 bool fits = false; 113 114 read_offset = READ_ONCE(ctrl->read_offset); 115 remaining = pvr_cccb->size - pvr_cccb->write_offset; 116 117 /* Always ensure we have enough room for a padding command at the end of the CCCB. 118 * If our command sequence does not fit, reserve the remaining space for a padding 119 * command. 120 */ 121 if (size + PADDING_COMMAND_SIZE > remaining) 122 size += remaining; 123 124 if (get_ccb_space(pvr_cccb->write_offset, read_offset, pvr_cccb->size) >= size) 125 fits = true; 126 127 return fits; 128} 129 130/** 131 * pvr_cccb_write_command_with_header() - Write a command + command header to a 132 * Client CCB 133 * @pvr_cccb: Target Client CCB. 134 * @cmd_type: Client CCB command type. Must be one of %ROGUE_FWIF_CCB_CMD_TYPE_*. 135 * @cmd_size: Size of command in bytes. 136 * @cmd_data: Pointer to command to write. 137 * @ext_job_ref: External job reference. 138 * @int_job_ref: Internal job reference. 139 * 140 * Caller must make sure there's enough space in CCCB to queue this command. This 141 * can be done by calling pvr_cccb_cmdseq_fits(). 142 * 143 * This function is not protected by any lock. The caller must ensure there's 144 * no concurrent caller, which should be guaranteed by the drm_sched model (job 145 * submission is serialized in drm_sched_main()). 146 */ 147void 148pvr_cccb_write_command_with_header(struct pvr_cccb *pvr_cccb, u32 cmd_type, u32 cmd_size, 149 void *cmd_data, u32 ext_job_ref, u32 int_job_ref) 150{ 151 u32 sz_with_hdr = pvr_cccb_get_size_of_cmd_with_hdr(cmd_size); 152 struct rogue_fwif_ccb_cmd_header cmd_header = { 153 .cmd_type = cmd_type, 154 .cmd_size = ALIGN(cmd_size, 8), 155 .ext_job_ref = ext_job_ref, 156 .int_job_ref = int_job_ref, 157 }; 158 struct rogue_fwif_cccb_ctl *ctrl = pvr_cccb->ctrl; 159 u32 remaining = pvr_cccb->size - pvr_cccb->write_offset; 160 u32 required_size, cccb_space, read_offset; 161 162 /* 163 * Always ensure we have enough room for a padding command at the end of 164 * the CCCB. 165 */ 166 if (remaining < sz_with_hdr + PADDING_COMMAND_SIZE) { 167 /* 168 * Command would need to wrap, so we need to pad the remainder 169 * of the CCCB. 170 */ 171 required_size = sz_with_hdr + remaining; 172 } else { 173 required_size = sz_with_hdr; 174 } 175 176 read_offset = READ_ONCE(ctrl->read_offset); 177 cccb_space = get_ccb_space(pvr_cccb->write_offset, read_offset, pvr_cccb->size); 178 if (WARN_ON(cccb_space < required_size)) 179 return; 180 181 if (required_size != sz_with_hdr) { 182 /* Add padding command */ 183 struct rogue_fwif_ccb_cmd_header pad_cmd = { 184 .cmd_type = ROGUE_FWIF_CCB_CMD_TYPE_PADDING, 185 .cmd_size = remaining - sizeof(pad_cmd), 186 }; 187 188 memcpy(&pvr_cccb->cccb[pvr_cccb->write_offset], &pad_cmd, sizeof(pad_cmd)); 189 pvr_cccb->write_offset = 0; 190 } 191 192 memcpy(&pvr_cccb->cccb[pvr_cccb->write_offset], &cmd_header, sizeof(cmd_header)); 193 memcpy(&pvr_cccb->cccb[pvr_cccb->write_offset + sizeof(cmd_header)], cmd_data, cmd_size); 194 pvr_cccb->write_offset += sz_with_hdr; 195} 196 197static void fill_cmd_kick_data(struct pvr_cccb *cccb, u32 ctx_fw_addr, 198 struct pvr_hwrt_data *hwrt, 199 struct rogue_fwif_kccb_cmd_kick_data *k) 200{ 201 k->context_fw_addr = ctx_fw_addr; 202 k->client_woff_update = cccb->write_offset; 203 k->client_wrap_mask_update = cccb->wrap_mask; 204 205 if (hwrt) { 206 u32 cleanup_state_offset = offsetof(struct rogue_fwif_hwrtdata, cleanup_state); 207 208 pvr_fw_object_get_fw_addr_offset(hwrt->fw_obj, cleanup_state_offset, 209 &k->cleanup_ctl_fw_addr[k->num_cleanup_ctl++]); 210 } 211} 212 213/** 214 * pvr_cccb_send_kccb_kick: Send KCCB kick to trigger command processing 215 * @pvr_dev: Device pointer. 216 * @pvr_cccb: Pointer to CCCB to process. 217 * @cctx_fw_addr: FW virtual address for context owning this Client CCB. 218 * @hwrt: HWRT data set associated with this kick. May be %NULL. 219 * 220 * You must call pvr_kccb_reserve_slot() and wait for the returned fence to 221 * signal (if this function didn't return NULL) before calling 222 * pvr_cccb_send_kccb_kick(). 223 */ 224void 225pvr_cccb_send_kccb_kick(struct pvr_device *pvr_dev, 226 struct pvr_cccb *pvr_cccb, u32 cctx_fw_addr, 227 struct pvr_hwrt_data *hwrt) 228{ 229 struct rogue_fwif_kccb_cmd cmd_kick = { 230 .cmd_type = ROGUE_FWIF_KCCB_CMD_KICK, 231 }; 232 233 fill_cmd_kick_data(pvr_cccb, cctx_fw_addr, hwrt, &cmd_kick.cmd_data.cmd_kick_data); 234 235 /* Make sure the writes to the CCCB are flushed before sending the KICK. */ 236 wmb(); 237 238 pvr_kccb_send_cmd_reserved_powered(pvr_dev, &cmd_kick, NULL); 239} 240 241void 242pvr_cccb_send_kccb_combined_kick(struct pvr_device *pvr_dev, 243 struct pvr_cccb *geom_cccb, 244 struct pvr_cccb *frag_cccb, 245 u32 geom_ctx_fw_addr, 246 u32 frag_ctx_fw_addr, 247 struct pvr_hwrt_data *hwrt, 248 bool frag_is_pr) 249{ 250 struct rogue_fwif_kccb_cmd cmd_kick = { 251 .cmd_type = ROGUE_FWIF_KCCB_CMD_COMBINED_GEOM_FRAG_KICK, 252 }; 253 254 fill_cmd_kick_data(geom_cccb, geom_ctx_fw_addr, hwrt, 255 &cmd_kick.cmd_data.combined_geom_frag_cmd_kick_data.geom_cmd_kick_data); 256 257 /* If this is a partial-render job, we don't attach resources to cleanup-ctl array, 258 * because the resources are already retained by the geometry job. 259 */ 260 fill_cmd_kick_data(frag_cccb, frag_ctx_fw_addr, frag_is_pr ? NULL : hwrt, 261 &cmd_kick.cmd_data.combined_geom_frag_cmd_kick_data.frag_cmd_kick_data); 262 263 /* Make sure the writes to the CCCB are flushed before sending the KICK. */ 264 wmb(); 265 266 pvr_kccb_send_cmd_reserved_powered(pvr_dev, &cmd_kick, NULL); 267} 268