1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 *  Shared Memory Communications over RDMA (SMC-R) and RoCE
4 *
5 *  Definitions for the SMC module (socket related)
6 *
7 *  Copyright IBM Corp. 2016
8 *
9 *  Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
10 */
11#ifndef __SMC_H
12#define __SMC_H
13
14#include <linux/socket.h>
15#include <linux/types.h>
16#include <linux/compiler.h> /* __aligned */
17#include <net/genetlink.h>
18#include <net/sock.h>
19
20#include "smc_ib.h"
21
22#define SMC_V1		1		/* SMC version V1 */
23#define SMC_V2		2		/* SMC version V2 */
24
25#define SMC_RELEASE_0 0
26#define SMC_RELEASE_1 1
27#define SMC_RELEASE	SMC_RELEASE_1 /* the latest release version */
28
29#define SMCPROTO_SMC		0	/* SMC protocol, IPv4 */
30#define SMCPROTO_SMC6		1	/* SMC protocol, IPv6 */
31
32#define SMC_AUTOCORKING_DEFAULT_SIZE	0x10000	/* 64K by default */
33
34extern struct proto smc_proto;
35extern struct proto smc_proto6;
36
37#ifdef ATOMIC64_INIT
38#define KERNEL_HAS_ATOMIC64
39#endif
40
41enum smc_state {		/* possible states of an SMC socket */
42	SMC_ACTIVE	= 1,
43	SMC_INIT	= 2,
44	SMC_CLOSED	= 7,
45	SMC_LISTEN	= 10,
46	/* normal close */
47	SMC_PEERCLOSEWAIT1	= 20,
48	SMC_PEERCLOSEWAIT2	= 21,
49	SMC_APPFINCLOSEWAIT	= 24,
50	SMC_APPCLOSEWAIT1	= 22,
51	SMC_APPCLOSEWAIT2	= 23,
52	SMC_PEERFINCLOSEWAIT	= 25,
53	/* abnormal close */
54	SMC_PEERABORTWAIT	= 26,
55	SMC_PROCESSABORT	= 27,
56};
57
58enum smc_supplemental_features {
59	SMC_SPF_EMULATED_ISM_DEV	= 0,
60};
61
62#define SMC_FEATURE_MASK \
63	(BIT(SMC_SPF_EMULATED_ISM_DEV))
64
65struct smc_link_group;
66
67struct smc_wr_rx_hdr {	/* common prefix part of LLC and CDC to demultiplex */
68	union {
69		u8 type;
70#if defined(__BIG_ENDIAN_BITFIELD)
71		struct {
72			u8 llc_version:4,
73			   llc_type:4;
74		};
75#elif defined(__LITTLE_ENDIAN_BITFIELD)
76		struct {
77			u8 llc_type:4,
78			   llc_version:4;
79		};
80#endif
81	};
82} __aligned(1);
83
84struct smc_cdc_conn_state_flags {
85#if defined(__BIG_ENDIAN_BITFIELD)
86	u8	peer_done_writing : 1;	/* Sending done indicator */
87	u8	peer_conn_closed : 1;	/* Peer connection closed indicator */
88	u8	peer_conn_abort : 1;	/* Abnormal close indicator */
89	u8	reserved : 5;
90#elif defined(__LITTLE_ENDIAN_BITFIELD)
91	u8	reserved : 5;
92	u8	peer_conn_abort : 1;
93	u8	peer_conn_closed : 1;
94	u8	peer_done_writing : 1;
95#endif
96};
97
98struct smc_cdc_producer_flags {
99#if defined(__BIG_ENDIAN_BITFIELD)
100	u8	write_blocked : 1;	/* Writing Blocked, no rx buf space */
101	u8	urg_data_pending : 1;	/* Urgent Data Pending */
102	u8	urg_data_present : 1;	/* Urgent Data Present */
103	u8	cons_curs_upd_req : 1;	/* cursor update requested */
104	u8	failover_validation : 1;/* message replay due to failover */
105	u8	reserved : 3;
106#elif defined(__LITTLE_ENDIAN_BITFIELD)
107	u8	reserved : 3;
108	u8	failover_validation : 1;
109	u8	cons_curs_upd_req : 1;
110	u8	urg_data_present : 1;
111	u8	urg_data_pending : 1;
112	u8	write_blocked : 1;
113#endif
114};
115
116/* in host byte order */
117union smc_host_cursor {	/* SMC cursor - an offset in an RMBE */
118	struct {
119		u16	reserved;
120		u16	wrap;		/* window wrap sequence number */
121		u32	count;		/* cursor (= offset) part */
122	};
123#ifdef KERNEL_HAS_ATOMIC64
124	atomic64_t		acurs;	/* for atomic processing */
125#else
126	u64			acurs;	/* for atomic processing */
127#endif
128} __aligned(8);
129
130/* in host byte order, except for flag bitfields in network byte order */
131struct smc_host_cdc_msg {		/* Connection Data Control message */
132	struct smc_wr_rx_hdr		common; /* .type = 0xFE */
133	u8				len;	/* length = 44 */
134	u16				seqno;	/* connection seq # */
135	u32				token;	/* alert_token */
136	union smc_host_cursor		prod;		/* producer cursor */
137	union smc_host_cursor		cons;		/* consumer cursor,
138							 * piggy backed "ack"
139							 */
140	struct smc_cdc_producer_flags	prod_flags;	/* conn. tx/rx status */
141	struct smc_cdc_conn_state_flags	conn_state_flags; /* peer conn. status*/
142	u8				reserved[18];
143} __aligned(8);
144
145enum smc_urg_state {
146	SMC_URG_VALID	= 1,			/* data present */
147	SMC_URG_NOTYET	= 2,			/* data pending */
148	SMC_URG_READ	= 3,			/* data was already read */
149};
150
151struct smc_mark_woken {
152	bool woken;
153	void *key;
154	wait_queue_entry_t wait_entry;
155};
156
157struct smc_connection {
158	struct rb_node		alert_node;
159	struct smc_link_group	*lgr;		/* link group of connection */
160	struct smc_link		*lnk;		/* assigned SMC-R link */
161	u32			alert_token_local; /* unique conn. id */
162	u8			peer_rmbe_idx;	/* from tcp handshake */
163	int			peer_rmbe_size;	/* size of peer rx buffer */
164	atomic_t		peer_rmbe_space;/* remaining free bytes in peer
165						 * rmbe
166						 */
167	int			rtoken_idx;	/* idx to peer RMB rkey/addr */
168
169	struct smc_buf_desc	*sndbuf_desc;	/* send buffer descriptor */
170	struct smc_buf_desc	*rmb_desc;	/* RMBE descriptor */
171	int                     rmbe_size_comp; /* compressed notation */
172	int			rmbe_update_limit;
173						/* lower limit for consumer
174						 * cursor update
175						 */
176
177	struct smc_host_cdc_msg	local_tx_ctrl;	/* host byte order staging
178						 * buffer for CDC msg send
179						 * .prod cf. TCP snd_nxt
180						 * .cons cf. TCP sends ack
181						 */
182	union smc_host_cursor	local_tx_ctrl_fin;
183						/* prod crsr - confirmed by peer
184						 */
185	union smc_host_cursor	tx_curs_prep;	/* tx - prepared data
186						 * snd_max..wmem_alloc
187						 */
188	union smc_host_cursor	tx_curs_sent;	/* tx - sent data
189						 * snd_nxt ?
190						 */
191	union smc_host_cursor	tx_curs_fin;	/* tx - confirmed by peer
192						 * snd-wnd-begin ?
193						 */
194	atomic_t		sndbuf_space;	/* remaining space in sndbuf */
195	u16			tx_cdc_seq;	/* sequence # for CDC send */
196	u16			tx_cdc_seq_fin;	/* sequence # - tx completed */
197	spinlock_t		send_lock;	/* protect wr_sends */
198	atomic_t		cdc_pend_tx_wr; /* number of pending tx CDC wqe
199						 * - inc when post wqe,
200						 * - dec on polled tx cqe
201						 */
202	wait_queue_head_t	cdc_pend_tx_wq; /* wakeup on no cdc_pend_tx_wr*/
203	struct delayed_work	tx_work;	/* retry of smc_cdc_msg_send */
204	u32			tx_off;		/* base offset in peer rmb */
205
206	struct smc_host_cdc_msg	local_rx_ctrl;	/* filled during event_handl.
207						 * .prod cf. TCP rcv_nxt
208						 * .cons cf. TCP snd_una
209						 */
210	union smc_host_cursor	rx_curs_confirmed; /* confirmed to peer
211						    * source of snd_una ?
212						    */
213	union smc_host_cursor	urg_curs;	/* points at urgent byte */
214	enum smc_urg_state	urg_state;
215	bool			urg_tx_pend;	/* urgent data staged */
216	bool			urg_rx_skip_pend;
217						/* indicate urgent oob data
218						 * read, but previous regular
219						 * data still pending
220						 */
221	char			urg_rx_byte;	/* urgent byte */
222	bool			tx_in_release_sock;
223						/* flush pending tx data in
224						 * sock release_cb()
225						 */
226	atomic_t		bytes_to_rcv;	/* arrived data,
227						 * not yet received
228						 */
229	atomic_t		splice_pending;	/* number of spliced bytes
230						 * pending processing
231						 */
232#ifndef KERNEL_HAS_ATOMIC64
233	spinlock_t		acurs_lock;	/* protect cursors */
234#endif
235	struct work_struct	close_work;	/* peer sent some closing */
236	struct work_struct	abort_work;	/* abort the connection */
237	struct tasklet_struct	rx_tsklet;	/* Receiver tasklet for SMC-D */
238	u8			rx_off;		/* receive offset:
239						 * 0 for SMC-R, 32 for SMC-D
240						 */
241	u64			peer_token;	/* SMC-D token of peer */
242	u8			killed : 1;	/* abnormal termination */
243	u8			freed : 1;	/* normal termiation */
244	u8			out_of_sync : 1; /* out of sync with peer */
245};
246
247struct smc_sock {				/* smc sock container */
248	struct sock		sk;
249	struct socket		*clcsock;	/* internal tcp socket */
250	void			(*clcsk_state_change)(struct sock *sk);
251						/* original stat_change fct. */
252	void			(*clcsk_data_ready)(struct sock *sk);
253						/* original data_ready fct. */
254	void			(*clcsk_write_space)(struct sock *sk);
255						/* original write_space fct. */
256	void			(*clcsk_error_report)(struct sock *sk);
257						/* original error_report fct. */
258	struct smc_connection	conn;		/* smc connection */
259	struct smc_sock		*listen_smc;	/* listen parent */
260	struct work_struct	connect_work;	/* handle non-blocking connect*/
261	struct work_struct	tcp_listen_work;/* handle tcp socket accepts */
262	struct work_struct	smc_listen_work;/* prepare new accept socket */
263	struct list_head	accept_q;	/* sockets to be accepted */
264	spinlock_t		accept_q_lock;	/* protects accept_q */
265	bool			limit_smc_hs;	/* put constraint on handshake */
266	bool			use_fallback;	/* fallback to tcp */
267	int			fallback_rsn;	/* reason for fallback */
268	u32			peer_diagnosis; /* decline reason from peer */
269	atomic_t                queued_smc_hs;  /* queued smc handshakes */
270	struct inet_connection_sock_af_ops		af_ops;
271	const struct inet_connection_sock_af_ops	*ori_af_ops;
272						/* original af ops */
273	int			sockopt_defer_accept;
274						/* sockopt TCP_DEFER_ACCEPT
275						 * value
276						 */
277	u8			wait_close_tx_prepared : 1;
278						/* shutdown wr or close
279						 * started, waiting for unsent
280						 * data to be sent
281						 */
282	u8			connect_nonblock : 1;
283						/* non-blocking connect in
284						 * flight
285						 */
286	struct mutex            clcsock_release_lock;
287						/* protects clcsock of a listen
288						 * socket
289						 * */
290};
291
292#define smc_sk(ptr) container_of_const(ptr, struct smc_sock, sk)
293
294static inline void smc_init_saved_callbacks(struct smc_sock *smc)
295{
296	smc->clcsk_state_change	= NULL;
297	smc->clcsk_data_ready	= NULL;
298	smc->clcsk_write_space	= NULL;
299	smc->clcsk_error_report	= NULL;
300}
301
302static inline struct smc_sock *smc_clcsock_user_data(const struct sock *clcsk)
303{
304	return (struct smc_sock *)
305	       ((uintptr_t)clcsk->sk_user_data & ~SK_USER_DATA_NOCOPY);
306}
307
308/* save target_cb in saved_cb, and replace target_cb with new_cb */
309static inline void smc_clcsock_replace_cb(void (**target_cb)(struct sock *),
310					  void (*new_cb)(struct sock *),
311					  void (**saved_cb)(struct sock *))
312{
313	/* only save once */
314	if (!*saved_cb)
315		*saved_cb = *target_cb;
316	*target_cb = new_cb;
317}
318
319/* restore target_cb to saved_cb, and reset saved_cb to NULL */
320static inline void smc_clcsock_restore_cb(void (**target_cb)(struct sock *),
321					  void (**saved_cb)(struct sock *))
322{
323	if (!*saved_cb)
324		return;
325	*target_cb = *saved_cb;
326	*saved_cb = NULL;
327}
328
329extern struct workqueue_struct	*smc_hs_wq;	/* wq for handshake work */
330extern struct workqueue_struct	*smc_close_wq;	/* wq for close work */
331
332#define SMC_SYSTEMID_LEN		8
333
334extern u8	local_systemid[SMC_SYSTEMID_LEN]; /* unique system identifier */
335
336#define ntohll(x) be64_to_cpu(x)
337#define htonll(x) cpu_to_be64(x)
338
339/* convert an u32 value into network byte order, store it into a 3 byte field */
340static inline void hton24(u8 *net, u32 host)
341{
342	__be32 t;
343
344	t = cpu_to_be32(host);
345	memcpy(net, ((u8 *)&t) + 1, 3);
346}
347
348/* convert a received 3 byte field into host byte order*/
349static inline u32 ntoh24(u8 *net)
350{
351	__be32 t = 0;
352
353	memcpy(((u8 *)&t) + 1, net, 3);
354	return be32_to_cpu(t);
355}
356
357#ifdef CONFIG_XFRM
358static inline bool using_ipsec(struct smc_sock *smc)
359{
360	return (smc->clcsock->sk->sk_policy[0] ||
361		smc->clcsock->sk->sk_policy[1]) ? true : false;
362}
363#else
364static inline bool using_ipsec(struct smc_sock *smc)
365{
366	return false;
367}
368#endif
369
370struct smc_gidlist;
371
372struct sock *smc_accept_dequeue(struct sock *parent, struct socket *new_sock);
373void smc_close_non_accepted(struct sock *sk);
374void smc_fill_gid_list(struct smc_link_group *lgr,
375		       struct smc_gidlist *gidlist,
376		       struct smc_ib_device *known_dev, u8 *known_gid);
377
378/* smc handshake limitation interface for netlink  */
379int smc_nl_dump_hs_limitation(struct sk_buff *skb, struct netlink_callback *cb);
380int smc_nl_enable_hs_limitation(struct sk_buff *skb, struct genl_info *info);
381int smc_nl_disable_hs_limitation(struct sk_buff *skb, struct genl_info *info);
382
383static inline void smc_sock_set_flag(struct sock *sk, enum sock_flags flag)
384{
385	set_bit(flag, &sk->sk_flags);
386}
387
388#endif	/* __SMC_H */
389