1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 *
5 * Connection Data Control (CDC)
6 *
7 * Copyright IBM Corp. 2016
8 *
9 * Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
10 */
11
12#ifndef SMC_CDC_H
13#define SMC_CDC_H
14
15#include <linux/kernel.h> /* max_t */
16#include <linux/atomic.h>
17#include <linux/in.h>
18#include <linux/compiler.h>
19
20#include "smc.h"
21#include "smc_core.h"
22#include "smc_wr.h"
23
24#define	SMC_CDC_MSG_TYPE		0xFE
25
26/* in network byte order */
27union smc_cdc_cursor {		/* SMC cursor */
28	struct {
29		__be16	reserved;
30		__be16	wrap;
31		__be32	count;
32	};
33#ifdef KERNEL_HAS_ATOMIC64
34	atomic64_t	acurs;		/* for atomic processing */
35#else
36	u64		acurs;		/* for atomic processing */
37#endif
38} __aligned(8);
39
40/* in network byte order */
41struct smc_cdc_msg {
42	struct smc_wr_rx_hdr		common; /* .type = 0xFE */
43	u8				len;	/* 44 */
44	__be16				seqno;
45	__be32				token;
46	union smc_cdc_cursor		prod;
47	union smc_cdc_cursor		cons;	/* piggy backed "ack" */
48	struct smc_cdc_producer_flags	prod_flags;
49	struct smc_cdc_conn_state_flags	conn_state_flags;
50	u8				reserved[18];
51};
52
53/* SMC-D cursor format */
54union smcd_cdc_cursor {
55	struct {
56		u16	wrap;
57		u32	count;
58		struct smc_cdc_producer_flags	prod_flags;
59		struct smc_cdc_conn_state_flags	conn_state_flags;
60	} __packed;
61#ifdef KERNEL_HAS_ATOMIC64
62	atomic64_t		acurs;		/* for atomic processing */
63#else
64	u64			acurs;		/* for atomic processing */
65#endif
66} __aligned(8);
67
68/* CDC message for SMC-D */
69struct smcd_cdc_msg {
70	struct smc_wr_rx_hdr common;	/* Type = 0xFE */
71	u8 res1[7];
72	union smcd_cdc_cursor	prod;
73	union smcd_cdc_cursor	cons;
74	u8 res3[8];
75} __aligned(8);
76
77static inline bool smc_cdc_rxed_any_close(struct smc_connection *conn)
78{
79	return conn->local_rx_ctrl.conn_state_flags.peer_conn_abort ||
80	       conn->local_rx_ctrl.conn_state_flags.peer_conn_closed;
81}
82
83static inline bool smc_cdc_rxed_any_close_or_senddone(
84	struct smc_connection *conn)
85{
86	return smc_cdc_rxed_any_close(conn) ||
87	       conn->local_rx_ctrl.conn_state_flags.peer_done_writing;
88}
89
90static inline void smc_curs_add(int size, union smc_host_cursor *curs,
91				int value)
92{
93	curs->count += value;
94	if (curs->count >= size) {
95		curs->wrap++;
96		curs->count -= size;
97	}
98}
99
100/* Copy cursor src into tgt */
101static inline void smc_curs_copy(union smc_host_cursor *tgt,
102				 union smc_host_cursor *src,
103				 struct smc_connection *conn)
104{
105#ifndef KERNEL_HAS_ATOMIC64
106	unsigned long flags;
107
108	spin_lock_irqsave(&conn->acurs_lock, flags);
109	tgt->acurs = src->acurs;
110	spin_unlock_irqrestore(&conn->acurs_lock, flags);
111#else
112	atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
113#endif
114}
115
116static inline void smc_curs_copy_net(union smc_cdc_cursor *tgt,
117				     union smc_cdc_cursor *src,
118				     struct smc_connection *conn)
119{
120#ifndef KERNEL_HAS_ATOMIC64
121	unsigned long flags;
122
123	spin_lock_irqsave(&conn->acurs_lock, flags);
124	tgt->acurs = src->acurs;
125	spin_unlock_irqrestore(&conn->acurs_lock, flags);
126#else
127	atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
128#endif
129}
130
131static inline void smcd_curs_copy(union smcd_cdc_cursor *tgt,
132				  union smcd_cdc_cursor *src,
133				  struct smc_connection *conn)
134{
135#ifndef KERNEL_HAS_ATOMIC64
136	unsigned long flags;
137
138	spin_lock_irqsave(&conn->acurs_lock, flags);
139	tgt->acurs = src->acurs;
140	spin_unlock_irqrestore(&conn->acurs_lock, flags);
141#else
142	atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
143#endif
144}
145
146/* calculate cursor difference between old and new, where old <= new and
147 * difference cannot exceed size
148 */
149static inline int smc_curs_diff(unsigned int size,
150				union smc_host_cursor *old,
151				union smc_host_cursor *new)
152{
153	if (old->wrap != new->wrap)
154		return max_t(int, 0,
155			     ((size - old->count) + new->count));
156
157	return max_t(int, 0, (new->count - old->count));
158}
159
160/* calculate cursor difference between old and new - returns negative
161 * value in case old > new
162 */
163static inline int smc_curs_comp(unsigned int size,
164				union smc_host_cursor *old,
165				union smc_host_cursor *new)
166{
167	if (old->wrap > new->wrap ||
168	    (old->wrap == new->wrap && old->count > new->count))
169		return -smc_curs_diff(size, new, old);
170	return smc_curs_diff(size, old, new);
171}
172
173/* calculate cursor difference between old and new, where old <= new and
174 * difference may exceed size
175 */
176static inline int smc_curs_diff_large(unsigned int size,
177				      union smc_host_cursor *old,
178				      union smc_host_cursor *new)
179{
180	if (old->wrap < new->wrap)
181		return min_t(int,
182			     (size - old->count) + new->count +
183			     (new->wrap - old->wrap - 1) * size,
184			     size);
185
186	if (old->wrap > new->wrap) /* wrap has switched from 0xffff to 0x0000 */
187		return min_t(int,
188			     (size - old->count) + new->count +
189			     (new->wrap + 0xffff - old->wrap) * size,
190			     size);
191
192	return max_t(int, 0, (new->count - old->count));
193}
194
195static inline void smc_host_cursor_to_cdc(union smc_cdc_cursor *peer,
196					  union smc_host_cursor *local,
197					  union smc_host_cursor *save,
198					  struct smc_connection *conn)
199{
200	smc_curs_copy(save, local, conn);
201	peer->count = htonl(save->count);
202	peer->wrap = htons(save->wrap);
203	/* peer->reserved = htons(0); must be ensured by caller */
204}
205
206static inline void smc_host_msg_to_cdc(struct smc_cdc_msg *peer,
207				       struct smc_connection *conn,
208				       union smc_host_cursor *save)
209{
210	struct smc_host_cdc_msg *local = &conn->local_tx_ctrl;
211
212	peer->common.type = local->common.type;
213	peer->len = local->len;
214	peer->seqno = htons(local->seqno);
215	peer->token = htonl(local->token);
216	smc_host_cursor_to_cdc(&peer->prod, &local->prod, save, conn);
217	smc_host_cursor_to_cdc(&peer->cons, &local->cons, save, conn);
218	peer->prod_flags = local->prod_flags;
219	peer->conn_state_flags = local->conn_state_flags;
220}
221
222static inline void smc_cdc_cursor_to_host(union smc_host_cursor *local,
223					  union smc_cdc_cursor *peer,
224					  struct smc_connection *conn)
225{
226	union smc_host_cursor temp, old;
227	union smc_cdc_cursor net;
228
229	smc_curs_copy(&old, local, conn);
230	smc_curs_copy_net(&net, peer, conn);
231	temp.count = ntohl(net.count);
232	temp.wrap = ntohs(net.wrap);
233	if ((old.wrap > temp.wrap) && temp.wrap)
234		return;
235	if ((old.wrap == temp.wrap) &&
236	    (old.count > temp.count))
237		return;
238	smc_curs_copy(local, &temp, conn);
239}
240
241static inline void smcr_cdc_msg_to_host(struct smc_host_cdc_msg *local,
242					struct smc_cdc_msg *peer,
243					struct smc_connection *conn)
244{
245	local->common.type = peer->common.type;
246	local->len = peer->len;
247	local->seqno = ntohs(peer->seqno);
248	local->token = ntohl(peer->token);
249	smc_cdc_cursor_to_host(&local->prod, &peer->prod, conn);
250	smc_cdc_cursor_to_host(&local->cons, &peer->cons, conn);
251	local->prod_flags = peer->prod_flags;
252	local->conn_state_flags = peer->conn_state_flags;
253}
254
255static inline void smcd_cdc_msg_to_host(struct smc_host_cdc_msg *local,
256					struct smcd_cdc_msg *peer,
257					struct smc_connection *conn)
258{
259	union smc_host_cursor temp;
260
261	temp.wrap = peer->prod.wrap;
262	temp.count = peer->prod.count;
263	smc_curs_copy(&local->prod, &temp, conn);
264
265	temp.wrap = peer->cons.wrap;
266	temp.count = peer->cons.count;
267	smc_curs_copy(&local->cons, &temp, conn);
268	local->prod_flags = peer->cons.prod_flags;
269	local->conn_state_flags = peer->cons.conn_state_flags;
270}
271
272static inline void smc_cdc_msg_to_host(struct smc_host_cdc_msg *local,
273				       struct smc_cdc_msg *peer,
274				       struct smc_connection *conn)
275{
276	if (conn->lgr->is_smcd)
277		smcd_cdc_msg_to_host(local, (struct smcd_cdc_msg *)peer, conn);
278	else
279		smcr_cdc_msg_to_host(local, peer, conn);
280}
281
282struct smc_cdc_tx_pend {
283	struct smc_connection	*conn;		/* socket connection */
284	union smc_host_cursor	cursor;		/* tx sndbuf cursor sent */
285	union smc_host_cursor	p_cursor;	/* rx RMBE cursor produced */
286	u16			ctrl_seq;	/* conn. tx sequence # */
287};
288
289int smc_cdc_get_free_slot(struct smc_connection *conn,
290			  struct smc_link *link,
291			  struct smc_wr_buf **wr_buf,
292			  struct smc_rdma_wr **wr_rdma_buf,
293			  struct smc_cdc_tx_pend **pend);
294void smc_cdc_wait_pend_tx_wr(struct smc_connection *conn);
295int smc_cdc_msg_send(struct smc_connection *conn, struct smc_wr_buf *wr_buf,
296		     struct smc_cdc_tx_pend *pend);
297int smc_cdc_get_slot_and_msg_send(struct smc_connection *conn);
298int smcd_cdc_msg_send(struct smc_connection *conn);
299int smcr_cdc_msg_send_validation(struct smc_connection *conn,
300				 struct smc_cdc_tx_pend *pend,
301				 struct smc_wr_buf *wr_buf);
302int smc_cdc_init(void) __init;
303void smcd_cdc_rx_init(struct smc_connection *conn);
304
305#endif /* SMC_CDC_H */
306