1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2017 Marvell
4 *
5 * Antoine Tenart <antoine.tenart@free-electrons.com>
6 */
7
8#include <linux/dma-mapping.h>
9#include <linux/spinlock.h>
10
11#include "safexcel.h"
12
13int safexcel_init_ring_descriptors(struct safexcel_crypto_priv *priv,
14				   struct safexcel_desc_ring *cdr,
15				   struct safexcel_desc_ring *rdr)
16{
17	int i;
18	struct safexcel_command_desc *cdesc;
19	dma_addr_t atok;
20
21	/* Actual command descriptor ring */
22	cdr->offset = priv->config.cd_offset;
23	cdr->base = dmam_alloc_coherent(priv->dev,
24					cdr->offset * EIP197_DEFAULT_RING_SIZE,
25					&cdr->base_dma, GFP_KERNEL);
26	if (!cdr->base)
27		return -ENOMEM;
28	cdr->write = cdr->base;
29	cdr->base_end = cdr->base + cdr->offset * (EIP197_DEFAULT_RING_SIZE - 1);
30	cdr->read = cdr->base;
31
32	/* Command descriptor shadow ring for storing additional token data */
33	cdr->shoffset = priv->config.cdsh_offset;
34	cdr->shbase = dmam_alloc_coherent(priv->dev,
35					  cdr->shoffset *
36					  EIP197_DEFAULT_RING_SIZE,
37					  &cdr->shbase_dma, GFP_KERNEL);
38	if (!cdr->shbase)
39		return -ENOMEM;
40	cdr->shwrite = cdr->shbase;
41	cdr->shbase_end = cdr->shbase + cdr->shoffset *
42					(EIP197_DEFAULT_RING_SIZE - 1);
43
44	/*
45	 * Populate command descriptors with physical pointers to shadow descs.
46	 * Note that we only need to do this once if we don't overwrite them.
47	 */
48	cdesc = cdr->base;
49	atok = cdr->shbase_dma;
50	for (i = 0; i < EIP197_DEFAULT_RING_SIZE; i++) {
51		cdesc->atok_lo = lower_32_bits(atok);
52		cdesc->atok_hi = upper_32_bits(atok);
53		cdesc = (void *)cdesc + cdr->offset;
54		atok += cdr->shoffset;
55	}
56
57	rdr->offset = priv->config.rd_offset;
58	/* Use shoffset for result token offset here */
59	rdr->shoffset = priv->config.res_offset;
60	rdr->base = dmam_alloc_coherent(priv->dev,
61					rdr->offset * EIP197_DEFAULT_RING_SIZE,
62					&rdr->base_dma, GFP_KERNEL);
63	if (!rdr->base)
64		return -ENOMEM;
65	rdr->write = rdr->base;
66	rdr->base_end = rdr->base + rdr->offset  * (EIP197_DEFAULT_RING_SIZE - 1);
67	rdr->read = rdr->base;
68
69	return 0;
70}
71
72inline int safexcel_select_ring(struct safexcel_crypto_priv *priv)
73{
74	return (atomic_inc_return(&priv->ring_used) % priv->config.rings);
75}
76
77static void *safexcel_ring_next_cwptr(struct safexcel_crypto_priv *priv,
78				     struct safexcel_desc_ring *ring,
79				     bool first,
80				     struct safexcel_token **atoken)
81{
82	void *ptr = ring->write;
83
84	if (first)
85		*atoken = ring->shwrite;
86
87	if ((ring->write == ring->read - ring->offset) ||
88	    (ring->read == ring->base && ring->write == ring->base_end))
89		return ERR_PTR(-ENOMEM);
90
91	if (ring->write == ring->base_end) {
92		ring->write = ring->base;
93		ring->shwrite = ring->shbase;
94	} else {
95		ring->write += ring->offset;
96		ring->shwrite += ring->shoffset;
97	}
98
99	return ptr;
100}
101
102static void *safexcel_ring_next_rwptr(struct safexcel_crypto_priv *priv,
103				     struct safexcel_desc_ring *ring,
104				     struct result_data_desc **rtoken)
105{
106	void *ptr = ring->write;
107
108	/* Result token at relative offset shoffset */
109	*rtoken = ring->write + ring->shoffset;
110
111	if ((ring->write == ring->read - ring->offset) ||
112	    (ring->read == ring->base && ring->write == ring->base_end))
113		return ERR_PTR(-ENOMEM);
114
115	if (ring->write == ring->base_end)
116		ring->write = ring->base;
117	else
118		ring->write += ring->offset;
119
120	return ptr;
121}
122
123void *safexcel_ring_next_rptr(struct safexcel_crypto_priv *priv,
124			      struct safexcel_desc_ring *ring)
125{
126	void *ptr = ring->read;
127
128	if (ring->write == ring->read)
129		return ERR_PTR(-ENOENT);
130
131	if (ring->read == ring->base_end)
132		ring->read = ring->base;
133	else
134		ring->read += ring->offset;
135
136	return ptr;
137}
138
139inline void *safexcel_ring_curr_rptr(struct safexcel_crypto_priv *priv,
140				     int ring)
141{
142	struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr;
143
144	return rdr->read;
145}
146
147inline int safexcel_ring_first_rdr_index(struct safexcel_crypto_priv *priv,
148					 int ring)
149{
150	struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr;
151
152	return (rdr->read - rdr->base) / rdr->offset;
153}
154
155inline int safexcel_ring_rdr_rdesc_index(struct safexcel_crypto_priv *priv,
156					 int ring,
157					 struct safexcel_result_desc *rdesc)
158{
159	struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr;
160
161	return ((void *)rdesc - rdr->base) / rdr->offset;
162}
163
164void safexcel_ring_rollback_wptr(struct safexcel_crypto_priv *priv,
165				 struct safexcel_desc_ring *ring)
166{
167	if (ring->write == ring->read)
168		return;
169
170	if (ring->write == ring->base) {
171		ring->write = ring->base_end;
172		ring->shwrite = ring->shbase_end;
173	} else {
174		ring->write -= ring->offset;
175		ring->shwrite -= ring->shoffset;
176	}
177}
178
179struct safexcel_command_desc *safexcel_add_cdesc(struct safexcel_crypto_priv *priv,
180						 int ring_id,
181						 bool first, bool last,
182						 dma_addr_t data, u32 data_len,
183						 u32 full_data_len,
184						 dma_addr_t context,
185						 struct safexcel_token **atoken)
186{
187	struct safexcel_command_desc *cdesc;
188
189	cdesc = safexcel_ring_next_cwptr(priv, &priv->ring[ring_id].cdr,
190					 first, atoken);
191	if (IS_ERR(cdesc))
192		return cdesc;
193
194	cdesc->particle_size = data_len;
195	cdesc->rsvd0 = 0;
196	cdesc->last_seg = last;
197	cdesc->first_seg = first;
198	cdesc->additional_cdata_size = 0;
199	cdesc->rsvd1 = 0;
200	cdesc->data_lo = lower_32_bits(data);
201	cdesc->data_hi = upper_32_bits(data);
202
203	if (first) {
204		/*
205		 * Note that the length here MUST be >0 or else the EIP(1)97
206		 * may hang. Newer EIP197 firmware actually incorporates this
207		 * fix already, but that doesn't help the EIP97 and we may
208		 * also be running older firmware.
209		 */
210		cdesc->control_data.packet_length = full_data_len ?: 1;
211		cdesc->control_data.options = EIP197_OPTION_MAGIC_VALUE |
212					      EIP197_OPTION_64BIT_CTX |
213					      EIP197_OPTION_CTX_CTRL_IN_CMD |
214					      EIP197_OPTION_RC_AUTO;
215		cdesc->control_data.type = EIP197_TYPE_BCLA;
216		cdesc->control_data.context_lo = lower_32_bits(context) |
217						 EIP197_CONTEXT_SMALL;
218		cdesc->control_data.context_hi = upper_32_bits(context);
219	}
220
221	return cdesc;
222}
223
224struct safexcel_result_desc *safexcel_add_rdesc(struct safexcel_crypto_priv *priv,
225						int ring_id,
226						bool first, bool last,
227						dma_addr_t data, u32 len)
228{
229	struct safexcel_result_desc *rdesc;
230	struct result_data_desc *rtoken;
231
232	rdesc = safexcel_ring_next_rwptr(priv, &priv->ring[ring_id].rdr,
233					 &rtoken);
234	if (IS_ERR(rdesc))
235		return rdesc;
236
237	rdesc->particle_size = len;
238	rdesc->rsvd0 = 0;
239	rdesc->descriptor_overflow = 1; /* assume error */
240	rdesc->buffer_overflow = 1;     /* assume error */
241	rdesc->last_seg = last;
242	rdesc->first_seg = first;
243	rdesc->result_size = EIP197_RD64_RESULT_SIZE;
244	rdesc->rsvd1 = 0;
245	rdesc->data_lo = lower_32_bits(data);
246	rdesc->data_hi = upper_32_bits(data);
247
248	/* Clear length in result token */
249	rtoken->packet_length = 0;
250	/* Assume errors - HW will clear if not the case */
251	rtoken->error_code = 0x7fff;
252
253	return rdesc;
254}
255