1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * rio_cm - RapidIO Channelized Messaging Driver
4 *
5 * Copyright 2013-2016 Integrated Device Technology, Inc.
6 * Copyright (c) 2015, Prodrive Technologies
7 * Copyright (c) 2015, RapidIO Trade Association
8 */
9
10#include <linux/module.h>
11#include <linux/kernel.h>
12#include <linux/dma-mapping.h>
13#include <linux/delay.h>
14#include <linux/sched.h>
15#include <linux/rio.h>
16#include <linux/rio_drv.h>
17#include <linux/slab.h>
18#include <linux/idr.h>
19#include <linux/interrupt.h>
20#include <linux/cdev.h>
21#include <linux/fs.h>
22#include <linux/poll.h>
23#include <linux/reboot.h>
24#include <linux/bitops.h>
25#include <linux/printk.h>
26#include <linux/rio_cm_cdev.h>
27
28#define DRV_NAME        "rio_cm"
29#define DRV_VERSION     "1.0.0"
30#define DRV_AUTHOR      "Alexandre Bounine <alexandre.bounine@idt.com>"
31#define DRV_DESC        "RapidIO Channelized Messaging Driver"
32#define DEV_NAME	"rio_cm"
33
34/* Debug output filtering masks */
35enum {
36	DBG_NONE	= 0,
37	DBG_INIT	= BIT(0), /* driver init */
38	DBG_EXIT	= BIT(1), /* driver exit */
39	DBG_MPORT	= BIT(2), /* mport add/remove */
40	DBG_RDEV	= BIT(3), /* RapidIO device add/remove */
41	DBG_CHOP	= BIT(4), /* channel operations */
42	DBG_WAIT	= BIT(5), /* waiting for events */
43	DBG_TX		= BIT(6), /* message TX */
44	DBG_TX_EVENT	= BIT(7), /* message TX event */
45	DBG_RX_DATA	= BIT(8), /* inbound data messages */
46	DBG_RX_CMD	= BIT(9), /* inbound REQ/ACK/NACK messages */
47	DBG_ALL		= ~0,
48};
49
50#ifdef DEBUG
51#define riocm_debug(level, fmt, arg...) \
52	do { \
53		if (DBG_##level & dbg_level) \
54			pr_debug(DRV_NAME ": %s " fmt "\n", \
55				__func__, ##arg); \
56	} while (0)
57#else
58#define riocm_debug(level, fmt, arg...) \
59		no_printk(KERN_DEBUG pr_fmt(DRV_NAME fmt "\n"), ##arg)
60#endif
61
62#define riocm_warn(fmt, arg...) \
63	pr_warn(DRV_NAME ": %s WARNING " fmt "\n", __func__, ##arg)
64
65#define riocm_error(fmt, arg...) \
66	pr_err(DRV_NAME ": %s ERROR " fmt "\n", __func__, ##arg)
67
68
69static int cmbox = 1;
70module_param(cmbox, int, S_IRUGO);
71MODULE_PARM_DESC(cmbox, "RapidIO Mailbox number (default 1)");
72
73static int chstart = 256;
74module_param(chstart, int, S_IRUGO);
75MODULE_PARM_DESC(chstart,
76		 "Start channel number for dynamic allocation (default 256)");
77
78#ifdef DEBUG
79static u32 dbg_level = DBG_NONE;
80module_param(dbg_level, uint, S_IWUSR | S_IRUGO);
81MODULE_PARM_DESC(dbg_level, "Debugging output level (default 0 = none)");
82#endif
83
84MODULE_AUTHOR(DRV_AUTHOR);
85MODULE_DESCRIPTION(DRV_DESC);
86MODULE_LICENSE("GPL");
87MODULE_VERSION(DRV_VERSION);
88
89#define RIOCM_TX_RING_SIZE	128
90#define RIOCM_RX_RING_SIZE	128
91#define RIOCM_CONNECT_TO	3 /* connect response TO (in sec) */
92
93#define RIOCM_MAX_CHNUM		0xffff /* Use full range of u16 field */
94#define RIOCM_CHNUM_AUTO	0
95#define RIOCM_MAX_EP_COUNT	0x10000 /* Max number of endpoints */
96
97enum rio_cm_state {
98	RIO_CM_IDLE,
99	RIO_CM_CONNECT,
100	RIO_CM_CONNECTED,
101	RIO_CM_DISCONNECT,
102	RIO_CM_CHAN_BOUND,
103	RIO_CM_LISTEN,
104	RIO_CM_DESTROYING,
105};
106
107enum rio_cm_pkt_type {
108	RIO_CM_SYS	= 0xaa,
109	RIO_CM_CHAN	= 0x55,
110};
111
112enum rio_cm_chop {
113	CM_CONN_REQ,
114	CM_CONN_ACK,
115	CM_CONN_CLOSE,
116	CM_DATA_MSG,
117};
118
119struct rio_ch_base_bhdr {
120	u32 src_id;
121	u32 dst_id;
122#define RIO_HDR_LETTER_MASK 0xffff0000
123#define RIO_HDR_MBOX_MASK   0x0000ffff
124	u8  src_mbox;
125	u8  dst_mbox;
126	u8  type;
127} __attribute__((__packed__));
128
129struct rio_ch_chan_hdr {
130	struct rio_ch_base_bhdr bhdr;
131	u8 ch_op;
132	u16 dst_ch;
133	u16 src_ch;
134	u16 msg_len;
135	u16 rsrvd;
136} __attribute__((__packed__));
137
138struct tx_req {
139	struct list_head node;
140	struct rio_dev   *rdev;
141	void		 *buffer;
142	size_t		 len;
143};
144
145struct cm_dev {
146	struct list_head	list;
147	struct rio_mport	*mport;
148	void			*rx_buf[RIOCM_RX_RING_SIZE];
149	int			rx_slots;
150	struct mutex		rx_lock;
151
152	void			*tx_buf[RIOCM_TX_RING_SIZE];
153	int			tx_slot;
154	int			tx_cnt;
155	int			tx_ack_slot;
156	struct list_head	tx_reqs;
157	spinlock_t		tx_lock;
158
159	struct list_head	peers;
160	u32			npeers;
161	struct workqueue_struct *rx_wq;
162	struct work_struct	rx_work;
163};
164
165struct chan_rx_ring {
166	void	*buf[RIOCM_RX_RING_SIZE];
167	int	head;
168	int	tail;
169	int	count;
170
171	/* Tracking RX buffers reported to upper level */
172	void	*inuse[RIOCM_RX_RING_SIZE];
173	int	inuse_cnt;
174};
175
176struct rio_channel {
177	u16			id;	/* local channel ID */
178	struct kref		ref;	/* channel refcount */
179	struct file		*filp;
180	struct cm_dev		*cmdev;	/* associated CM device object */
181	struct rio_dev		*rdev;	/* remote RapidIO device */
182	enum rio_cm_state	state;
183	int			error;
184	spinlock_t		lock;
185	void			*context;
186	u32			loc_destid;	/* local destID */
187	u32			rem_destid;	/* remote destID */
188	u16			rem_channel;	/* remote channel ID */
189	struct list_head	accept_queue;
190	struct list_head	ch_node;
191	struct completion	comp;
192	struct completion	comp_close;
193	struct chan_rx_ring	rx_ring;
194};
195
196struct cm_peer {
197	struct list_head node;
198	struct rio_dev *rdev;
199};
200
201struct rio_cm_work {
202	struct work_struct work;
203	struct cm_dev *cm;
204	void *data;
205};
206
207struct conn_req {
208	struct list_head node;
209	u32 destid;	/* requester destID */
210	u16 chan;	/* requester channel ID */
211	struct cm_dev *cmdev;
212};
213
214/*
215 * A channel_dev structure represents a CM_CDEV
216 * @cdev	Character device
217 * @dev		Associated device object
218 */
219struct channel_dev {
220	struct cdev	cdev;
221	struct device	*dev;
222};
223
224static struct rio_channel *riocm_ch_alloc(u16 ch_num);
225static void riocm_ch_free(struct kref *ref);
226static int riocm_post_send(struct cm_dev *cm, struct rio_dev *rdev,
227			   void *buffer, size_t len);
228static int riocm_ch_close(struct rio_channel *ch);
229
230static DEFINE_SPINLOCK(idr_lock);
231static DEFINE_IDR(ch_idr);
232
233static LIST_HEAD(cm_dev_list);
234static DECLARE_RWSEM(rdev_sem);
235
236static struct class *dev_class;
237static unsigned int dev_major;
238static unsigned int dev_minor_base;
239static dev_t dev_number;
240static struct channel_dev riocm_cdev;
241
242#define is_msg_capable(src_ops, dst_ops)			\
243			((src_ops & RIO_SRC_OPS_DATA_MSG) &&	\
244			 (dst_ops & RIO_DST_OPS_DATA_MSG))
245#define dev_cm_capable(dev) \
246	is_msg_capable(dev->src_ops, dev->dst_ops)
247
248static int riocm_cmp(struct rio_channel *ch, enum rio_cm_state cmp)
249{
250	int ret;
251
252	spin_lock_bh(&ch->lock);
253	ret = (ch->state == cmp);
254	spin_unlock_bh(&ch->lock);
255	return ret;
256}
257
258static int riocm_cmp_exch(struct rio_channel *ch,
259			   enum rio_cm_state cmp, enum rio_cm_state exch)
260{
261	int ret;
262
263	spin_lock_bh(&ch->lock);
264	ret = (ch->state == cmp);
265	if (ret)
266		ch->state = exch;
267	spin_unlock_bh(&ch->lock);
268	return ret;
269}
270
271static enum rio_cm_state riocm_exch(struct rio_channel *ch,
272				    enum rio_cm_state exch)
273{
274	enum rio_cm_state old;
275
276	spin_lock_bh(&ch->lock);
277	old = ch->state;
278	ch->state = exch;
279	spin_unlock_bh(&ch->lock);
280	return old;
281}
282
283static struct rio_channel *riocm_get_channel(u16 nr)
284{
285	struct rio_channel *ch;
286
287	spin_lock_bh(&idr_lock);
288	ch = idr_find(&ch_idr, nr);
289	if (ch)
290		kref_get(&ch->ref);
291	spin_unlock_bh(&idr_lock);
292	return ch;
293}
294
295static void riocm_put_channel(struct rio_channel *ch)
296{
297	kref_put(&ch->ref, riocm_ch_free);
298}
299
300static void *riocm_rx_get_msg(struct cm_dev *cm)
301{
302	void *msg;
303	int i;
304
305	msg = rio_get_inb_message(cm->mport, cmbox);
306	if (msg) {
307		for (i = 0; i < RIOCM_RX_RING_SIZE; i++) {
308			if (cm->rx_buf[i] == msg) {
309				cm->rx_buf[i] = NULL;
310				cm->rx_slots++;
311				break;
312			}
313		}
314
315		if (i == RIOCM_RX_RING_SIZE)
316			riocm_warn("no record for buffer 0x%p", msg);
317	}
318
319	return msg;
320}
321
322/*
323 * riocm_rx_fill - fills a ring of receive buffers for given cm device
324 * @cm: cm_dev object
325 * @nent: max number of entries to fill
326 *
327 * Returns: none
328 */
329static void riocm_rx_fill(struct cm_dev *cm, int nent)
330{
331	int i;
332
333	if (cm->rx_slots == 0)
334		return;
335
336	for (i = 0; i < RIOCM_RX_RING_SIZE && cm->rx_slots && nent; i++) {
337		if (cm->rx_buf[i] == NULL) {
338			cm->rx_buf[i] = kmalloc(RIO_MAX_MSG_SIZE, GFP_KERNEL);
339			if (cm->rx_buf[i] == NULL)
340				break;
341			rio_add_inb_buffer(cm->mport, cmbox, cm->rx_buf[i]);
342			cm->rx_slots--;
343			nent--;
344		}
345	}
346}
347
348/*
349 * riocm_rx_free - frees all receive buffers associated with given cm device
350 * @cm: cm_dev object
351 *
352 * Returns: none
353 */
354static void riocm_rx_free(struct cm_dev *cm)
355{
356	int i;
357
358	for (i = 0; i < RIOCM_RX_RING_SIZE; i++) {
359		if (cm->rx_buf[i] != NULL) {
360			kfree(cm->rx_buf[i]);
361			cm->rx_buf[i] = NULL;
362		}
363	}
364}
365
366/*
367 * riocm_req_handler - connection request handler
368 * @cm: cm_dev object
369 * @req_data: pointer to the request packet
370 *
371 * Returns: 0 if success, or
372 *          -EINVAL if channel is not in correct state,
373 *          -ENODEV if cannot find a channel with specified ID,
374 *          -ENOMEM if unable to allocate memory to store the request
375 */
376static int riocm_req_handler(struct cm_dev *cm, void *req_data)
377{
378	struct rio_channel *ch;
379	struct conn_req *req;
380	struct rio_ch_chan_hdr *hh = req_data;
381	u16 chnum;
382
383	chnum = ntohs(hh->dst_ch);
384
385	ch = riocm_get_channel(chnum);
386
387	if (!ch)
388		return -ENODEV;
389
390	if (ch->state != RIO_CM_LISTEN) {
391		riocm_debug(RX_CMD, "channel %d is not in listen state", chnum);
392		riocm_put_channel(ch);
393		return -EINVAL;
394	}
395
396	req = kzalloc(sizeof(*req), GFP_KERNEL);
397	if (!req) {
398		riocm_put_channel(ch);
399		return -ENOMEM;
400	}
401
402	req->destid = ntohl(hh->bhdr.src_id);
403	req->chan = ntohs(hh->src_ch);
404	req->cmdev = cm;
405
406	spin_lock_bh(&ch->lock);
407	list_add_tail(&req->node, &ch->accept_queue);
408	spin_unlock_bh(&ch->lock);
409	complete(&ch->comp);
410	riocm_put_channel(ch);
411
412	return 0;
413}
414
415/*
416 * riocm_resp_handler - response to connection request handler
417 * @resp_data: pointer to the response packet
418 *
419 * Returns: 0 if success, or
420 *          -EINVAL if channel is not in correct state,
421 *          -ENODEV if cannot find a channel with specified ID,
422 */
423static int riocm_resp_handler(void *resp_data)
424{
425	struct rio_channel *ch;
426	struct rio_ch_chan_hdr *hh = resp_data;
427	u16 chnum;
428
429	chnum = ntohs(hh->dst_ch);
430	ch = riocm_get_channel(chnum);
431	if (!ch)
432		return -ENODEV;
433
434	if (ch->state != RIO_CM_CONNECT) {
435		riocm_put_channel(ch);
436		return -EINVAL;
437	}
438
439	riocm_exch(ch, RIO_CM_CONNECTED);
440	ch->rem_channel = ntohs(hh->src_ch);
441	complete(&ch->comp);
442	riocm_put_channel(ch);
443
444	return 0;
445}
446
447/*
448 * riocm_close_handler - channel close request handler
449 * @req_data: pointer to the request packet
450 *
451 * Returns: 0 if success, or
452 *          -ENODEV if cannot find a channel with specified ID,
453 *            + error codes returned by riocm_ch_close.
454 */
455static int riocm_close_handler(void *data)
456{
457	struct rio_channel *ch;
458	struct rio_ch_chan_hdr *hh = data;
459	int ret;
460
461	riocm_debug(RX_CMD, "for ch=%d", ntohs(hh->dst_ch));
462
463	spin_lock_bh(&idr_lock);
464	ch = idr_find(&ch_idr, ntohs(hh->dst_ch));
465	if (!ch) {
466		spin_unlock_bh(&idr_lock);
467		return -ENODEV;
468	}
469	idr_remove(&ch_idr, ch->id);
470	spin_unlock_bh(&idr_lock);
471
472	riocm_exch(ch, RIO_CM_DISCONNECT);
473
474	ret = riocm_ch_close(ch);
475	if (ret)
476		riocm_debug(RX_CMD, "riocm_ch_close() returned %d", ret);
477
478	return 0;
479}
480
481/*
482 * rio_cm_handler - function that services request (non-data) packets
483 * @cm: cm_dev object
484 * @data: pointer to the packet
485 */
486static void rio_cm_handler(struct cm_dev *cm, void *data)
487{
488	struct rio_ch_chan_hdr *hdr;
489
490	if (!rio_mport_is_running(cm->mport))
491		goto out;
492
493	hdr = data;
494
495	riocm_debug(RX_CMD, "OP=%x for ch=%d from %d",
496		    hdr->ch_op, ntohs(hdr->dst_ch), ntohs(hdr->src_ch));
497
498	switch (hdr->ch_op) {
499	case CM_CONN_REQ:
500		riocm_req_handler(cm, data);
501		break;
502	case CM_CONN_ACK:
503		riocm_resp_handler(data);
504		break;
505	case CM_CONN_CLOSE:
506		riocm_close_handler(data);
507		break;
508	default:
509		riocm_error("Invalid packet header");
510		break;
511	}
512out:
513	kfree(data);
514}
515
516/*
517 * rio_rx_data_handler - received data packet handler
518 * @cm: cm_dev object
519 * @buf: data packet
520 *
521 * Returns: 0 if success, or
522 *          -ENODEV if cannot find a channel with specified ID,
523 *          -EIO if channel is not in CONNECTED state,
524 *          -ENOMEM if channel RX queue is full (packet discarded)
525 */
526static int rio_rx_data_handler(struct cm_dev *cm, void *buf)
527{
528	struct rio_ch_chan_hdr *hdr;
529	struct rio_channel *ch;
530
531	hdr = buf;
532
533	riocm_debug(RX_DATA, "for ch=%d", ntohs(hdr->dst_ch));
534
535	ch = riocm_get_channel(ntohs(hdr->dst_ch));
536	if (!ch) {
537		/* Discard data message for non-existing channel */
538		kfree(buf);
539		return -ENODEV;
540	}
541
542	/* Place pointer to the buffer into channel's RX queue */
543	spin_lock(&ch->lock);
544
545	if (ch->state != RIO_CM_CONNECTED) {
546		/* Channel is not ready to receive data, discard a packet */
547		riocm_debug(RX_DATA, "ch=%d is in wrong state=%d",
548			    ch->id, ch->state);
549		spin_unlock(&ch->lock);
550		kfree(buf);
551		riocm_put_channel(ch);
552		return -EIO;
553	}
554
555	if (ch->rx_ring.count == RIOCM_RX_RING_SIZE) {
556		/* If RX ring is full, discard a packet */
557		riocm_debug(RX_DATA, "ch=%d is full", ch->id);
558		spin_unlock(&ch->lock);
559		kfree(buf);
560		riocm_put_channel(ch);
561		return -ENOMEM;
562	}
563
564	ch->rx_ring.buf[ch->rx_ring.head] = buf;
565	ch->rx_ring.head++;
566	ch->rx_ring.count++;
567	ch->rx_ring.head %= RIOCM_RX_RING_SIZE;
568
569	complete(&ch->comp);
570
571	spin_unlock(&ch->lock);
572	riocm_put_channel(ch);
573
574	return 0;
575}
576
577/*
578 * rio_ibmsg_handler - inbound message packet handler
579 */
580static void rio_ibmsg_handler(struct work_struct *work)
581{
582	struct cm_dev *cm = container_of(work, struct cm_dev, rx_work);
583	void *data;
584	struct rio_ch_chan_hdr *hdr;
585
586	if (!rio_mport_is_running(cm->mport))
587		return;
588
589	while (1) {
590		mutex_lock(&cm->rx_lock);
591		data = riocm_rx_get_msg(cm);
592		if (data)
593			riocm_rx_fill(cm, 1);
594		mutex_unlock(&cm->rx_lock);
595
596		if (data == NULL)
597			break;
598
599		hdr = data;
600
601		if (hdr->bhdr.type != RIO_CM_CHAN) {
602			/* For now simply discard packets other than channel */
603			riocm_error("Unsupported TYPE code (0x%x). Msg dropped",
604				    hdr->bhdr.type);
605			kfree(data);
606			continue;
607		}
608
609		/* Process a channel message */
610		if (hdr->ch_op == CM_DATA_MSG)
611			rio_rx_data_handler(cm, data);
612		else
613			rio_cm_handler(cm, data);
614	}
615}
616
617static void riocm_inb_msg_event(struct rio_mport *mport, void *dev_id,
618				int mbox, int slot)
619{
620	struct cm_dev *cm = dev_id;
621
622	if (rio_mport_is_running(cm->mport) && !work_pending(&cm->rx_work))
623		queue_work(cm->rx_wq, &cm->rx_work);
624}
625
626/*
627 * rio_txcq_handler - TX completion handler
628 * @cm: cm_dev object
629 * @slot: TX queue slot
630 *
631 * TX completion handler also ensures that pending request packets are placed
632 * into transmit queue as soon as a free slot becomes available. This is done
633 * to give higher priority to request packets during high intensity data flow.
634 */
635static void rio_txcq_handler(struct cm_dev *cm, int slot)
636{
637	int ack_slot;
638
639	/* ATTN: Add TX completion notification if/when direct buffer
640	 * transfer is implemented. At this moment only correct tracking
641	 * of tx_count is important.
642	 */
643	riocm_debug(TX_EVENT, "for mport_%d slot %d tx_cnt %d",
644		    cm->mport->id, slot, cm->tx_cnt);
645
646	spin_lock(&cm->tx_lock);
647	ack_slot = cm->tx_ack_slot;
648
649	if (ack_slot == slot)
650		riocm_debug(TX_EVENT, "slot == ack_slot");
651
652	while (cm->tx_cnt && ((ack_slot != slot) ||
653	       (cm->tx_cnt == RIOCM_TX_RING_SIZE))) {
654
655		cm->tx_buf[ack_slot] = NULL;
656		++ack_slot;
657		ack_slot &= (RIOCM_TX_RING_SIZE - 1);
658		cm->tx_cnt--;
659	}
660
661	if (cm->tx_cnt < 0 || cm->tx_cnt > RIOCM_TX_RING_SIZE)
662		riocm_error("tx_cnt %d out of sync", cm->tx_cnt);
663
664	WARN_ON((cm->tx_cnt < 0) || (cm->tx_cnt > RIOCM_TX_RING_SIZE));
665
666	cm->tx_ack_slot = ack_slot;
667
668	/*
669	 * If there are pending requests, insert them into transmit queue
670	 */
671	if (!list_empty(&cm->tx_reqs) && (cm->tx_cnt < RIOCM_TX_RING_SIZE)) {
672		struct tx_req *req, *_req;
673		int rc;
674
675		list_for_each_entry_safe(req, _req, &cm->tx_reqs, node) {
676			list_del(&req->node);
677			cm->tx_buf[cm->tx_slot] = req->buffer;
678			rc = rio_add_outb_message(cm->mport, req->rdev, cmbox,
679						  req->buffer, req->len);
680			kfree(req->buffer);
681			kfree(req);
682
683			++cm->tx_cnt;
684			++cm->tx_slot;
685			cm->tx_slot &= (RIOCM_TX_RING_SIZE - 1);
686			if (cm->tx_cnt == RIOCM_TX_RING_SIZE)
687				break;
688		}
689	}
690
691	spin_unlock(&cm->tx_lock);
692}
693
694static void riocm_outb_msg_event(struct rio_mport *mport, void *dev_id,
695				 int mbox, int slot)
696{
697	struct cm_dev *cm = dev_id;
698
699	if (cm && rio_mport_is_running(cm->mport))
700		rio_txcq_handler(cm, slot);
701}
702
703static int riocm_queue_req(struct cm_dev *cm, struct rio_dev *rdev,
704			   void *buffer, size_t len)
705{
706	unsigned long flags;
707	struct tx_req *treq;
708
709	treq = kzalloc(sizeof(*treq), GFP_KERNEL);
710	if (treq == NULL)
711		return -ENOMEM;
712
713	treq->rdev = rdev;
714	treq->buffer = buffer;
715	treq->len = len;
716
717	spin_lock_irqsave(&cm->tx_lock, flags);
718	list_add_tail(&treq->node, &cm->tx_reqs);
719	spin_unlock_irqrestore(&cm->tx_lock, flags);
720	return 0;
721}
722
723/*
724 * riocm_post_send - helper function that places packet into msg TX queue
725 * @cm: cm_dev object
726 * @rdev: target RapidIO device object (required by outbound msg interface)
727 * @buffer: pointer to a packet buffer to send
728 * @len: length of data to transfer
729 * @req: request priority flag
730 *
731 * Returns: 0 if success, or error code otherwise.
732 */
733static int riocm_post_send(struct cm_dev *cm, struct rio_dev *rdev,
734			   void *buffer, size_t len)
735{
736	int rc;
737	unsigned long flags;
738
739	spin_lock_irqsave(&cm->tx_lock, flags);
740
741	if (cm->mport == NULL) {
742		rc = -ENODEV;
743		goto err_out;
744	}
745
746	if (cm->tx_cnt == RIOCM_TX_RING_SIZE) {
747		riocm_debug(TX, "Tx Queue is full");
748		rc = -EBUSY;
749		goto err_out;
750	}
751
752	cm->tx_buf[cm->tx_slot] = buffer;
753	rc = rio_add_outb_message(cm->mport, rdev, cmbox, buffer, len);
754
755	riocm_debug(TX, "Add buf@%p destid=%x tx_slot=%d tx_cnt=%d",
756		 buffer, rdev->destid, cm->tx_slot, cm->tx_cnt);
757
758	++cm->tx_cnt;
759	++cm->tx_slot;
760	cm->tx_slot &= (RIOCM_TX_RING_SIZE - 1);
761
762err_out:
763	spin_unlock_irqrestore(&cm->tx_lock, flags);
764	return rc;
765}
766
767/*
768 * riocm_ch_send - sends a data packet to a remote device
769 * @ch_id: local channel ID
770 * @buf: pointer to a data buffer to send (including CM header)
771 * @len: length of data to transfer (including CM header)
772 *
773 * ATTN: ASSUMES THAT THE HEADER SPACE IS RESERVED PART OF THE DATA PACKET
774 *
775 * Returns: 0 if success, or
776 *          -EINVAL if one or more input parameters is/are not valid,
777 *          -ENODEV if cannot find a channel with specified ID,
778 *          -EAGAIN if a channel is not in CONNECTED state,
779 *	    + error codes returned by HW send routine.
780 */
781static int riocm_ch_send(u16 ch_id, void *buf, int len)
782{
783	struct rio_channel *ch;
784	struct rio_ch_chan_hdr *hdr;
785	int ret;
786
787	if (buf == NULL || ch_id == 0 || len == 0 || len > RIO_MAX_MSG_SIZE)
788		return -EINVAL;
789
790	ch = riocm_get_channel(ch_id);
791	if (!ch) {
792		riocm_error("%s(%d) ch_%d not found", current->comm,
793			    task_pid_nr(current), ch_id);
794		return -ENODEV;
795	}
796
797	if (!riocm_cmp(ch, RIO_CM_CONNECTED)) {
798		ret = -EAGAIN;
799		goto err_out;
800	}
801
802	/*
803	 * Fill buffer header section with corresponding channel data
804	 */
805	hdr = buf;
806
807	hdr->bhdr.src_id = htonl(ch->loc_destid);
808	hdr->bhdr.dst_id = htonl(ch->rem_destid);
809	hdr->bhdr.src_mbox = cmbox;
810	hdr->bhdr.dst_mbox = cmbox;
811	hdr->bhdr.type = RIO_CM_CHAN;
812	hdr->ch_op = CM_DATA_MSG;
813	hdr->dst_ch = htons(ch->rem_channel);
814	hdr->src_ch = htons(ch->id);
815	hdr->msg_len = htons((u16)len);
816
817	/* ATTN: the function call below relies on the fact that underlying
818	 * HW-specific add_outb_message() routine copies TX data into its own
819	 * internal transfer buffer (true for all RIONET compatible mport
820	 * drivers). Must be reviewed if mport driver uses the buffer directly.
821	 */
822
823	ret = riocm_post_send(ch->cmdev, ch->rdev, buf, len);
824	if (ret)
825		riocm_debug(TX, "ch %d send_err=%d", ch->id, ret);
826err_out:
827	riocm_put_channel(ch);
828	return ret;
829}
830
831static int riocm_ch_free_rxbuf(struct rio_channel *ch, void *buf)
832{
833	int i, ret = -EINVAL;
834
835	spin_lock_bh(&ch->lock);
836
837	for (i = 0; i < RIOCM_RX_RING_SIZE; i++) {
838		if (ch->rx_ring.inuse[i] == buf) {
839			ch->rx_ring.inuse[i] = NULL;
840			ch->rx_ring.inuse_cnt--;
841			ret = 0;
842			break;
843		}
844	}
845
846	spin_unlock_bh(&ch->lock);
847
848	if (!ret)
849		kfree(buf);
850
851	return ret;
852}
853
854/*
855 * riocm_ch_receive - fetch a data packet received for the specified channel
856 * @ch: local channel ID
857 * @buf: pointer to a packet buffer
858 * @timeout: timeout to wait for incoming packet (in jiffies)
859 *
860 * Returns: 0 and valid buffer pointer if success, or NULL pointer and one of:
861 *          -EAGAIN if a channel is not in CONNECTED state,
862 *          -ENOMEM if in-use tracking queue is full,
863 *          -ETIME if wait timeout expired,
864 *	    -EINTR if wait was interrupted.
865 */
866static int riocm_ch_receive(struct rio_channel *ch, void **buf, long timeout)
867{
868	void *rxmsg = NULL;
869	int i, ret = 0;
870	long wret;
871
872	if (!riocm_cmp(ch, RIO_CM_CONNECTED)) {
873		ret = -EAGAIN;
874		goto out;
875	}
876
877	if (ch->rx_ring.inuse_cnt == RIOCM_RX_RING_SIZE) {
878		/* If we do not have entries to track buffers given to upper
879		 * layer, reject request.
880		 */
881		ret = -ENOMEM;
882		goto out;
883	}
884
885	wret = wait_for_completion_interruptible_timeout(&ch->comp, timeout);
886
887	riocm_debug(WAIT, "wait on %d returned %ld", ch->id, wret);
888
889	if (!wret)
890		ret = -ETIME;
891	else if (wret == -ERESTARTSYS)
892		ret = -EINTR;
893	else
894		ret = riocm_cmp(ch, RIO_CM_CONNECTED) ? 0 : -ECONNRESET;
895
896	if (ret)
897		goto out;
898
899	spin_lock_bh(&ch->lock);
900
901	rxmsg = ch->rx_ring.buf[ch->rx_ring.tail];
902	ch->rx_ring.buf[ch->rx_ring.tail] = NULL;
903	ch->rx_ring.count--;
904	ch->rx_ring.tail++;
905	ch->rx_ring.tail %= RIOCM_RX_RING_SIZE;
906	ret = -ENOMEM;
907
908	for (i = 0; i < RIOCM_RX_RING_SIZE; i++) {
909		if (ch->rx_ring.inuse[i] == NULL) {
910			ch->rx_ring.inuse[i] = rxmsg;
911			ch->rx_ring.inuse_cnt++;
912			ret = 0;
913			break;
914		}
915	}
916
917	if (ret) {
918		/* We have no entry to store pending message: drop it */
919		kfree(rxmsg);
920		rxmsg = NULL;
921	}
922
923	spin_unlock_bh(&ch->lock);
924out:
925	*buf = rxmsg;
926	return ret;
927}
928
929/*
930 * riocm_ch_connect - sends a connect request to a remote device
931 * @loc_ch: local channel ID
932 * @cm: CM device to send connect request
933 * @peer: target RapidIO device
934 * @rem_ch: remote channel ID
935 *
936 * Returns: 0 if success, or
937 *          -EINVAL if the channel is not in IDLE state,
938 *          -EAGAIN if no connection request available immediately,
939 *          -ETIME if ACK response timeout expired,
940 *          -EINTR if wait for response was interrupted.
941 */
942static int riocm_ch_connect(u16 loc_ch, struct cm_dev *cm,
943			    struct cm_peer *peer, u16 rem_ch)
944{
945	struct rio_channel *ch = NULL;
946	struct rio_ch_chan_hdr *hdr;
947	int ret;
948	long wret;
949
950	ch = riocm_get_channel(loc_ch);
951	if (!ch)
952		return -ENODEV;
953
954	if (!riocm_cmp_exch(ch, RIO_CM_IDLE, RIO_CM_CONNECT)) {
955		ret = -EINVAL;
956		goto conn_done;
957	}
958
959	ch->cmdev = cm;
960	ch->rdev = peer->rdev;
961	ch->context = NULL;
962	ch->loc_destid = cm->mport->host_deviceid;
963	ch->rem_channel = rem_ch;
964
965	/*
966	 * Send connect request to the remote RapidIO device
967	 */
968
969	hdr = kzalloc(sizeof(*hdr), GFP_KERNEL);
970	if (hdr == NULL) {
971		ret = -ENOMEM;
972		goto conn_done;
973	}
974
975	hdr->bhdr.src_id = htonl(ch->loc_destid);
976	hdr->bhdr.dst_id = htonl(peer->rdev->destid);
977	hdr->bhdr.src_mbox = cmbox;
978	hdr->bhdr.dst_mbox = cmbox;
979	hdr->bhdr.type = RIO_CM_CHAN;
980	hdr->ch_op = CM_CONN_REQ;
981	hdr->dst_ch = htons(rem_ch);
982	hdr->src_ch = htons(loc_ch);
983
984	/* ATTN: the function call below relies on the fact that underlying
985	 * HW-specific add_outb_message() routine copies TX data into its
986	 * internal transfer buffer. Must be reviewed if mport driver uses
987	 * this buffer directly.
988	 */
989	ret = riocm_post_send(cm, peer->rdev, hdr, sizeof(*hdr));
990
991	if (ret != -EBUSY) {
992		kfree(hdr);
993	} else {
994		ret = riocm_queue_req(cm, peer->rdev, hdr, sizeof(*hdr));
995		if (ret)
996			kfree(hdr);
997	}
998
999	if (ret) {
1000		riocm_cmp_exch(ch, RIO_CM_CONNECT, RIO_CM_IDLE);
1001		goto conn_done;
1002	}
1003
1004	/* Wait for connect response from the remote device */
1005	wret = wait_for_completion_interruptible_timeout(&ch->comp,
1006							 RIOCM_CONNECT_TO * HZ);
1007	riocm_debug(WAIT, "wait on %d returns %ld", ch->id, wret);
1008
1009	if (!wret)
1010		ret = -ETIME;
1011	else if (wret == -ERESTARTSYS)
1012		ret = -EINTR;
1013	else
1014		ret = riocm_cmp(ch, RIO_CM_CONNECTED) ? 0 : -1;
1015
1016conn_done:
1017	riocm_put_channel(ch);
1018	return ret;
1019}
1020
1021static int riocm_send_ack(struct rio_channel *ch)
1022{
1023	struct rio_ch_chan_hdr *hdr;
1024	int ret;
1025
1026	hdr = kzalloc(sizeof(*hdr), GFP_KERNEL);
1027	if (hdr == NULL)
1028		return -ENOMEM;
1029
1030	hdr->bhdr.src_id = htonl(ch->loc_destid);
1031	hdr->bhdr.dst_id = htonl(ch->rem_destid);
1032	hdr->dst_ch = htons(ch->rem_channel);
1033	hdr->src_ch = htons(ch->id);
1034	hdr->bhdr.src_mbox = cmbox;
1035	hdr->bhdr.dst_mbox = cmbox;
1036	hdr->bhdr.type = RIO_CM_CHAN;
1037	hdr->ch_op = CM_CONN_ACK;
1038
1039	/* ATTN: the function call below relies on the fact that underlying
1040	 * add_outb_message() routine copies TX data into its internal transfer
1041	 * buffer. Review if switching to direct buffer version.
1042	 */
1043	ret = riocm_post_send(ch->cmdev, ch->rdev, hdr, sizeof(*hdr));
1044
1045	if (ret == -EBUSY && !riocm_queue_req(ch->cmdev,
1046					      ch->rdev, hdr, sizeof(*hdr)))
1047		return 0;
1048	kfree(hdr);
1049
1050	if (ret)
1051		riocm_error("send ACK to ch_%d on %s failed (ret=%d)",
1052			    ch->id, rio_name(ch->rdev), ret);
1053	return ret;
1054}
1055
1056/*
1057 * riocm_ch_accept - accept incoming connection request
1058 * @ch_id: channel ID
1059 * @new_ch_id: local mport device
1060 * @timeout: wait timeout (if 0 non-blocking call, do not wait if connection
1061 *           request is not available).
1062 *
1063 * Returns: pointer to new channel struct if success, or error-valued pointer:
1064 *          -ENODEV - cannot find specified channel or mport,
1065 *          -EINVAL - the channel is not in IDLE state,
1066 *          -EAGAIN - no connection request available immediately (timeout=0),
1067 *          -ENOMEM - unable to allocate new channel,
1068 *          -ETIME - wait timeout expired,
1069 *          -EINTR - wait was interrupted.
1070 */
1071static struct rio_channel *riocm_ch_accept(u16 ch_id, u16 *new_ch_id,
1072					   long timeout)
1073{
1074	struct rio_channel *ch;
1075	struct rio_channel *new_ch;
1076	struct conn_req *req;
1077	struct cm_peer *peer;
1078	int found = 0;
1079	int err = 0;
1080	long wret;
1081
1082	ch = riocm_get_channel(ch_id);
1083	if (!ch)
1084		return ERR_PTR(-EINVAL);
1085
1086	if (!riocm_cmp(ch, RIO_CM_LISTEN)) {
1087		err = -EINVAL;
1088		goto err_put;
1089	}
1090
1091	/* Don't sleep if this is a non blocking call */
1092	if (!timeout) {
1093		if (!try_wait_for_completion(&ch->comp)) {
1094			err = -EAGAIN;
1095			goto err_put;
1096		}
1097	} else {
1098		riocm_debug(WAIT, "on %d", ch->id);
1099
1100		wret = wait_for_completion_interruptible_timeout(&ch->comp,
1101								 timeout);
1102		if (!wret) {
1103			err = -ETIME;
1104			goto err_put;
1105		} else if (wret == -ERESTARTSYS) {
1106			err = -EINTR;
1107			goto err_put;
1108		}
1109	}
1110
1111	spin_lock_bh(&ch->lock);
1112
1113	if (ch->state != RIO_CM_LISTEN) {
1114		err = -ECANCELED;
1115	} else if (list_empty(&ch->accept_queue)) {
1116		riocm_debug(WAIT, "on %d accept_queue is empty on completion",
1117			    ch->id);
1118		err = -EIO;
1119	}
1120
1121	spin_unlock_bh(&ch->lock);
1122
1123	if (err) {
1124		riocm_debug(WAIT, "on %d returns %d", ch->id, err);
1125		goto err_put;
1126	}
1127
1128	/* Create new channel for this connection */
1129	new_ch = riocm_ch_alloc(RIOCM_CHNUM_AUTO);
1130
1131	if (IS_ERR(new_ch)) {
1132		riocm_error("failed to get channel for new req (%ld)",
1133			PTR_ERR(new_ch));
1134		err = -ENOMEM;
1135		goto err_put;
1136	}
1137
1138	spin_lock_bh(&ch->lock);
1139
1140	req = list_first_entry(&ch->accept_queue, struct conn_req, node);
1141	list_del(&req->node);
1142	new_ch->cmdev = ch->cmdev;
1143	new_ch->loc_destid = ch->loc_destid;
1144	new_ch->rem_destid = req->destid;
1145	new_ch->rem_channel = req->chan;
1146
1147	spin_unlock_bh(&ch->lock);
1148	riocm_put_channel(ch);
1149	ch = NULL;
1150	kfree(req);
1151
1152	down_read(&rdev_sem);
1153	/* Find requester's device object */
1154	list_for_each_entry(peer, &new_ch->cmdev->peers, node) {
1155		if (peer->rdev->destid == new_ch->rem_destid) {
1156			riocm_debug(RX_CMD, "found matching device(%s)",
1157				    rio_name(peer->rdev));
1158			found = 1;
1159			break;
1160		}
1161	}
1162	up_read(&rdev_sem);
1163
1164	if (!found) {
1165		/* If peer device object not found, simply ignore the request */
1166		err = -ENODEV;
1167		goto err_put_new_ch;
1168	}
1169
1170	new_ch->rdev = peer->rdev;
1171	new_ch->state = RIO_CM_CONNECTED;
1172	spin_lock_init(&new_ch->lock);
1173
1174	/* Acknowledge the connection request. */
1175	riocm_send_ack(new_ch);
1176
1177	*new_ch_id = new_ch->id;
1178	return new_ch;
1179
1180err_put_new_ch:
1181	spin_lock_bh(&idr_lock);
1182	idr_remove(&ch_idr, new_ch->id);
1183	spin_unlock_bh(&idr_lock);
1184	riocm_put_channel(new_ch);
1185
1186err_put:
1187	if (ch)
1188		riocm_put_channel(ch);
1189	*new_ch_id = 0;
1190	return ERR_PTR(err);
1191}
1192
1193/*
1194 * riocm_ch_listen - puts a channel into LISTEN state
1195 * @ch_id: channel ID
1196 *
1197 * Returns: 0 if success, or
1198 *          -EINVAL if the specified channel does not exists or
1199 *                  is not in CHAN_BOUND state.
1200 */
1201static int riocm_ch_listen(u16 ch_id)
1202{
1203	struct rio_channel *ch = NULL;
1204	int ret = 0;
1205
1206	riocm_debug(CHOP, "(ch_%d)", ch_id);
1207
1208	ch = riocm_get_channel(ch_id);
1209	if (!ch)
1210		return -EINVAL;
1211	if (!riocm_cmp_exch(ch, RIO_CM_CHAN_BOUND, RIO_CM_LISTEN))
1212		ret = -EINVAL;
1213	riocm_put_channel(ch);
1214	return ret;
1215}
1216
1217/*
1218 * riocm_ch_bind - associate a channel object and an mport device
1219 * @ch_id: channel ID
1220 * @mport_id: local mport device ID
1221 * @context: pointer to the additional caller's context
1222 *
1223 * Returns: 0 if success, or
1224 *          -ENODEV if cannot find specified mport,
1225 *          -EINVAL if the specified channel does not exist or
1226 *                  is not in IDLE state.
1227 */
1228static int riocm_ch_bind(u16 ch_id, u8 mport_id, void *context)
1229{
1230	struct rio_channel *ch = NULL;
1231	struct cm_dev *cm;
1232	int rc = -ENODEV;
1233
1234	riocm_debug(CHOP, "ch_%d to mport_%d", ch_id, mport_id);
1235
1236	/* Find matching cm_dev object */
1237	down_read(&rdev_sem);
1238	list_for_each_entry(cm, &cm_dev_list, list) {
1239		if ((cm->mport->id == mport_id) &&
1240		     rio_mport_is_running(cm->mport)) {
1241			rc = 0;
1242			break;
1243		}
1244	}
1245
1246	if (rc)
1247		goto exit;
1248
1249	ch = riocm_get_channel(ch_id);
1250	if (!ch) {
1251		rc = -EINVAL;
1252		goto exit;
1253	}
1254
1255	spin_lock_bh(&ch->lock);
1256	if (ch->state != RIO_CM_IDLE) {
1257		spin_unlock_bh(&ch->lock);
1258		rc = -EINVAL;
1259		goto err_put;
1260	}
1261
1262	ch->cmdev = cm;
1263	ch->loc_destid = cm->mport->host_deviceid;
1264	ch->context = context;
1265	ch->state = RIO_CM_CHAN_BOUND;
1266	spin_unlock_bh(&ch->lock);
1267err_put:
1268	riocm_put_channel(ch);
1269exit:
1270	up_read(&rdev_sem);
1271	return rc;
1272}
1273
1274/*
1275 * riocm_ch_alloc - channel object allocation helper routine
1276 * @ch_num: channel ID (1 ... RIOCM_MAX_CHNUM, 0 = automatic)
1277 *
1278 * Return value: pointer to newly created channel object,
1279 *               or error-valued pointer
1280 */
1281static struct rio_channel *riocm_ch_alloc(u16 ch_num)
1282{
1283	int id;
1284	int start, end;
1285	struct rio_channel *ch;
1286
1287	ch = kzalloc(sizeof(*ch), GFP_KERNEL);
1288	if (!ch)
1289		return ERR_PTR(-ENOMEM);
1290
1291	if (ch_num) {
1292		/* If requested, try to obtain the specified channel ID */
1293		start = ch_num;
1294		end = ch_num + 1;
1295	} else {
1296		/* Obtain channel ID from the dynamic allocation range */
1297		start = chstart;
1298		end = RIOCM_MAX_CHNUM + 1;
1299	}
1300
1301	idr_preload(GFP_KERNEL);
1302	spin_lock_bh(&idr_lock);
1303	id = idr_alloc_cyclic(&ch_idr, ch, start, end, GFP_NOWAIT);
1304	spin_unlock_bh(&idr_lock);
1305	idr_preload_end();
1306
1307	if (id < 0) {
1308		kfree(ch);
1309		return ERR_PTR(id == -ENOSPC ? -EBUSY : id);
1310	}
1311
1312	ch->id = (u16)id;
1313	ch->state = RIO_CM_IDLE;
1314	spin_lock_init(&ch->lock);
1315	INIT_LIST_HEAD(&ch->accept_queue);
1316	INIT_LIST_HEAD(&ch->ch_node);
1317	init_completion(&ch->comp);
1318	init_completion(&ch->comp_close);
1319	kref_init(&ch->ref);
1320	ch->rx_ring.head = 0;
1321	ch->rx_ring.tail = 0;
1322	ch->rx_ring.count = 0;
1323	ch->rx_ring.inuse_cnt = 0;
1324
1325	return ch;
1326}
1327
1328/*
1329 * riocm_ch_create - creates a new channel object and allocates ID for it
1330 * @ch_num: channel ID (1 ... RIOCM_MAX_CHNUM, 0 = automatic)
1331 *
1332 * Allocates and initializes a new channel object. If the parameter ch_num > 0
1333 * and is within the valid range, riocm_ch_create tries to allocate the
1334 * specified ID for the new channel. If ch_num = 0, channel ID will be assigned
1335 * automatically from the range (chstart ... RIOCM_MAX_CHNUM).
1336 * Module parameter 'chstart' defines start of an ID range available for dynamic
1337 * allocation. Range below 'chstart' is reserved for pre-defined ID numbers.
1338 * Available channel numbers are limited by 16-bit size of channel numbers used
1339 * in the packet header.
1340 *
1341 * Return value: PTR to rio_channel structure if successful (with channel number
1342 *               updated via pointer) or error-valued pointer if error.
1343 */
1344static struct rio_channel *riocm_ch_create(u16 *ch_num)
1345{
1346	struct rio_channel *ch = NULL;
1347
1348	ch = riocm_ch_alloc(*ch_num);
1349
1350	if (IS_ERR(ch))
1351		riocm_debug(CHOP, "Failed to allocate channel %d (err=%ld)",
1352			    *ch_num, PTR_ERR(ch));
1353	else
1354		*ch_num = ch->id;
1355
1356	return ch;
1357}
1358
1359/*
1360 * riocm_ch_free - channel object release routine
1361 * @ref: pointer to a channel's kref structure
1362 */
1363static void riocm_ch_free(struct kref *ref)
1364{
1365	struct rio_channel *ch = container_of(ref, struct rio_channel, ref);
1366	int i;
1367
1368	riocm_debug(CHOP, "(ch_%d)", ch->id);
1369
1370	if (ch->rx_ring.inuse_cnt) {
1371		for (i = 0;
1372		     i < RIOCM_RX_RING_SIZE && ch->rx_ring.inuse_cnt; i++) {
1373			if (ch->rx_ring.inuse[i] != NULL) {
1374				kfree(ch->rx_ring.inuse[i]);
1375				ch->rx_ring.inuse_cnt--;
1376			}
1377		}
1378	}
1379
1380	if (ch->rx_ring.count)
1381		for (i = 0; i < RIOCM_RX_RING_SIZE && ch->rx_ring.count; i++) {
1382			if (ch->rx_ring.buf[i] != NULL) {
1383				kfree(ch->rx_ring.buf[i]);
1384				ch->rx_ring.count--;
1385			}
1386		}
1387
1388	complete(&ch->comp_close);
1389}
1390
1391static int riocm_send_close(struct rio_channel *ch)
1392{
1393	struct rio_ch_chan_hdr *hdr;
1394	int ret;
1395
1396	/*
1397	 * Send CH_CLOSE notification to the remote RapidIO device
1398	 */
1399
1400	hdr = kzalloc(sizeof(*hdr), GFP_KERNEL);
1401	if (hdr == NULL)
1402		return -ENOMEM;
1403
1404	hdr->bhdr.src_id = htonl(ch->loc_destid);
1405	hdr->bhdr.dst_id = htonl(ch->rem_destid);
1406	hdr->bhdr.src_mbox = cmbox;
1407	hdr->bhdr.dst_mbox = cmbox;
1408	hdr->bhdr.type = RIO_CM_CHAN;
1409	hdr->ch_op = CM_CONN_CLOSE;
1410	hdr->dst_ch = htons(ch->rem_channel);
1411	hdr->src_ch = htons(ch->id);
1412
1413	/* ATTN: the function call below relies on the fact that underlying
1414	 * add_outb_message() routine copies TX data into its internal transfer
1415	 * buffer. Needs to be reviewed if switched to direct buffer mode.
1416	 */
1417	ret = riocm_post_send(ch->cmdev, ch->rdev, hdr, sizeof(*hdr));
1418
1419	if (ret == -EBUSY && !riocm_queue_req(ch->cmdev, ch->rdev,
1420					      hdr, sizeof(*hdr)))
1421		return 0;
1422	kfree(hdr);
1423
1424	if (ret)
1425		riocm_error("ch(%d) send CLOSE failed (ret=%d)", ch->id, ret);
1426
1427	return ret;
1428}
1429
1430/*
1431 * riocm_ch_close - closes a channel object with specified ID (by local request)
1432 * @ch: channel to be closed
1433 */
1434static int riocm_ch_close(struct rio_channel *ch)
1435{
1436	unsigned long tmo = msecs_to_jiffies(3000);
1437	enum rio_cm_state state;
1438	long wret;
1439	int ret = 0;
1440
1441	riocm_debug(CHOP, "ch_%d by %s(%d)",
1442		    ch->id, current->comm, task_pid_nr(current));
1443
1444	state = riocm_exch(ch, RIO_CM_DESTROYING);
1445	if (state == RIO_CM_CONNECTED)
1446		riocm_send_close(ch);
1447
1448	complete_all(&ch->comp);
1449
1450	riocm_put_channel(ch);
1451	wret = wait_for_completion_interruptible_timeout(&ch->comp_close, tmo);
1452
1453	riocm_debug(WAIT, "wait on %d returns %ld", ch->id, wret);
1454
1455	if (wret == 0) {
1456		/* Timeout on wait occurred */
1457		riocm_debug(CHOP, "%s(%d) timed out waiting for ch %d",
1458		       current->comm, task_pid_nr(current), ch->id);
1459		ret = -ETIMEDOUT;
1460	} else if (wret == -ERESTARTSYS) {
1461		/* Wait_for_completion was interrupted by a signal */
1462		riocm_debug(CHOP, "%s(%d) wait for ch %d was interrupted",
1463			current->comm, task_pid_nr(current), ch->id);
1464		ret = -EINTR;
1465	}
1466
1467	if (!ret) {
1468		riocm_debug(CHOP, "ch_%d resources released", ch->id);
1469		kfree(ch);
1470	} else {
1471		riocm_debug(CHOP, "failed to release ch_%d resources", ch->id);
1472	}
1473
1474	return ret;
1475}
1476
1477/*
1478 * riocm_cdev_open() - Open character device
1479 */
1480static int riocm_cdev_open(struct inode *inode, struct file *filp)
1481{
1482	riocm_debug(INIT, "by %s(%d) filp=%p ",
1483		    current->comm, task_pid_nr(current), filp);
1484
1485	if (list_empty(&cm_dev_list))
1486		return -ENODEV;
1487
1488	return 0;
1489}
1490
1491/*
1492 * riocm_cdev_release() - Release character device
1493 */
1494static int riocm_cdev_release(struct inode *inode, struct file *filp)
1495{
1496	struct rio_channel *ch, *_c;
1497	unsigned int i;
1498	LIST_HEAD(list);
1499
1500	riocm_debug(EXIT, "by %s(%d) filp=%p",
1501		    current->comm, task_pid_nr(current), filp);
1502
1503	/* Check if there are channels associated with this file descriptor */
1504	spin_lock_bh(&idr_lock);
1505	idr_for_each_entry(&ch_idr, ch, i) {
1506		if (ch && ch->filp == filp) {
1507			riocm_debug(EXIT, "ch_%d not released by %s(%d)",
1508				    ch->id, current->comm,
1509				    task_pid_nr(current));
1510			idr_remove(&ch_idr, ch->id);
1511			list_add(&ch->ch_node, &list);
1512		}
1513	}
1514	spin_unlock_bh(&idr_lock);
1515
1516	if (!list_empty(&list)) {
1517		list_for_each_entry_safe(ch, _c, &list, ch_node) {
1518			list_del(&ch->ch_node);
1519			riocm_ch_close(ch);
1520		}
1521	}
1522
1523	return 0;
1524}
1525
1526/*
1527 * cm_ep_get_list_size() - Reports number of endpoints in the network
1528 */
1529static int cm_ep_get_list_size(void __user *arg)
1530{
1531	u32 __user *p = arg;
1532	u32 mport_id;
1533	u32 count = 0;
1534	struct cm_dev *cm;
1535
1536	if (get_user(mport_id, p))
1537		return -EFAULT;
1538	if (mport_id >= RIO_MAX_MPORTS)
1539		return -EINVAL;
1540
1541	/* Find a matching cm_dev object */
1542	down_read(&rdev_sem);
1543	list_for_each_entry(cm, &cm_dev_list, list) {
1544		if (cm->mport->id == mport_id) {
1545			count = cm->npeers;
1546			up_read(&rdev_sem);
1547			if (copy_to_user(arg, &count, sizeof(u32)))
1548				return -EFAULT;
1549			return 0;
1550		}
1551	}
1552	up_read(&rdev_sem);
1553
1554	return -ENODEV;
1555}
1556
1557/*
1558 * cm_ep_get_list() - Returns list of attached endpoints
1559 */
1560static int cm_ep_get_list(void __user *arg)
1561{
1562	struct cm_dev *cm;
1563	struct cm_peer *peer;
1564	u32 info[2];
1565	void *buf;
1566	u32 nent;
1567	u32 *entry_ptr;
1568	u32 i = 0;
1569	int ret = 0;
1570
1571	if (copy_from_user(&info, arg, sizeof(info)))
1572		return -EFAULT;
1573
1574	if (info[1] >= RIO_MAX_MPORTS || info[0] > RIOCM_MAX_EP_COUNT)
1575		return -EINVAL;
1576
1577	/* Find a matching cm_dev object */
1578	down_read(&rdev_sem);
1579	list_for_each_entry(cm, &cm_dev_list, list)
1580		if (cm->mport->id == (u8)info[1])
1581			goto found;
1582
1583	up_read(&rdev_sem);
1584	return -ENODEV;
1585
1586found:
1587	nent = min(info[0], cm->npeers);
1588	buf = kcalloc(nent + 2, sizeof(u32), GFP_KERNEL);
1589	if (!buf) {
1590		up_read(&rdev_sem);
1591		return -ENOMEM;
1592	}
1593
1594	entry_ptr = (u32 *)((uintptr_t)buf + 2*sizeof(u32));
1595
1596	list_for_each_entry(peer, &cm->peers, node) {
1597		*entry_ptr = (u32)peer->rdev->destid;
1598		entry_ptr++;
1599		if (++i == nent)
1600			break;
1601	}
1602	up_read(&rdev_sem);
1603
1604	((u32 *)buf)[0] = i; /* report an updated number of entries */
1605	((u32 *)buf)[1] = info[1]; /* put back an mport ID */
1606	if (copy_to_user(arg, buf, sizeof(u32) * (info[0] + 2)))
1607		ret = -EFAULT;
1608
1609	kfree(buf);
1610	return ret;
1611}
1612
1613/*
1614 * cm_mport_get_list() - Returns list of available local mport devices
1615 */
1616static int cm_mport_get_list(void __user *arg)
1617{
1618	int ret = 0;
1619	u32 entries;
1620	void *buf;
1621	struct cm_dev *cm;
1622	u32 *entry_ptr;
1623	int count = 0;
1624
1625	if (copy_from_user(&entries, arg, sizeof(entries)))
1626		return -EFAULT;
1627	if (entries == 0 || entries > RIO_MAX_MPORTS)
1628		return -EINVAL;
1629	buf = kcalloc(entries + 1, sizeof(u32), GFP_KERNEL);
1630	if (!buf)
1631		return -ENOMEM;
1632
1633	/* Scan all registered cm_dev objects */
1634	entry_ptr = (u32 *)((uintptr_t)buf + sizeof(u32));
1635	down_read(&rdev_sem);
1636	list_for_each_entry(cm, &cm_dev_list, list) {
1637		if (count++ < entries) {
1638			*entry_ptr = (cm->mport->id << 16) |
1639				      cm->mport->host_deviceid;
1640			entry_ptr++;
1641		}
1642	}
1643	up_read(&rdev_sem);
1644
1645	*((u32 *)buf) = count; /* report a real number of entries */
1646	if (copy_to_user(arg, buf, sizeof(u32) * (count + 1)))
1647		ret = -EFAULT;
1648
1649	kfree(buf);
1650	return ret;
1651}
1652
1653/*
1654 * cm_chan_create() - Create a message exchange channel
1655 */
1656static int cm_chan_create(struct file *filp, void __user *arg)
1657{
1658	u16 __user *p = arg;
1659	u16 ch_num;
1660	struct rio_channel *ch;
1661
1662	if (get_user(ch_num, p))
1663		return -EFAULT;
1664
1665	riocm_debug(CHOP, "ch_%d requested by %s(%d)",
1666		    ch_num, current->comm, task_pid_nr(current));
1667	ch = riocm_ch_create(&ch_num);
1668	if (IS_ERR(ch))
1669		return PTR_ERR(ch);
1670
1671	ch->filp = filp;
1672	riocm_debug(CHOP, "ch_%d created by %s(%d)",
1673		    ch_num, current->comm, task_pid_nr(current));
1674	return put_user(ch_num, p);
1675}
1676
1677/*
1678 * cm_chan_close() - Close channel
1679 * @filp:	Pointer to file object
1680 * @arg:	Channel to close
1681 */
1682static int cm_chan_close(struct file *filp, void __user *arg)
1683{
1684	u16 __user *p = arg;
1685	u16 ch_num;
1686	struct rio_channel *ch;
1687
1688	if (get_user(ch_num, p))
1689		return -EFAULT;
1690
1691	riocm_debug(CHOP, "ch_%d by %s(%d)",
1692		    ch_num, current->comm, task_pid_nr(current));
1693
1694	spin_lock_bh(&idr_lock);
1695	ch = idr_find(&ch_idr, ch_num);
1696	if (!ch) {
1697		spin_unlock_bh(&idr_lock);
1698		return 0;
1699	}
1700	if (ch->filp != filp) {
1701		spin_unlock_bh(&idr_lock);
1702		return -EINVAL;
1703	}
1704	idr_remove(&ch_idr, ch->id);
1705	spin_unlock_bh(&idr_lock);
1706
1707	return riocm_ch_close(ch);
1708}
1709
1710/*
1711 * cm_chan_bind() - Bind channel
1712 * @arg:	Channel number
1713 */
1714static int cm_chan_bind(void __user *arg)
1715{
1716	struct rio_cm_channel chan;
1717
1718	if (copy_from_user(&chan, arg, sizeof(chan)))
1719		return -EFAULT;
1720	if (chan.mport_id >= RIO_MAX_MPORTS)
1721		return -EINVAL;
1722
1723	return riocm_ch_bind(chan.id, chan.mport_id, NULL);
1724}
1725
1726/*
1727 * cm_chan_listen() - Listen on channel
1728 * @arg:	Channel number
1729 */
1730static int cm_chan_listen(void __user *arg)
1731{
1732	u16 __user *p = arg;
1733	u16 ch_num;
1734
1735	if (get_user(ch_num, p))
1736		return -EFAULT;
1737
1738	return riocm_ch_listen(ch_num);
1739}
1740
1741/*
1742 * cm_chan_accept() - Accept incoming connection
1743 * @filp:	Pointer to file object
1744 * @arg:	Channel number
1745 */
1746static int cm_chan_accept(struct file *filp, void __user *arg)
1747{
1748	struct rio_cm_accept param;
1749	long accept_to;
1750	struct rio_channel *ch;
1751
1752	if (copy_from_user(&param, arg, sizeof(param)))
1753		return -EFAULT;
1754
1755	riocm_debug(CHOP, "on ch_%d by %s(%d)",
1756		    param.ch_num, current->comm, task_pid_nr(current));
1757
1758	accept_to = param.wait_to ?
1759			msecs_to_jiffies(param.wait_to) : 0;
1760
1761	ch = riocm_ch_accept(param.ch_num, &param.ch_num, accept_to);
1762	if (IS_ERR(ch))
1763		return PTR_ERR(ch);
1764	ch->filp = filp;
1765
1766	riocm_debug(CHOP, "new ch_%d for %s(%d)",
1767		    ch->id, current->comm, task_pid_nr(current));
1768
1769	if (copy_to_user(arg, &param, sizeof(param)))
1770		return -EFAULT;
1771	return 0;
1772}
1773
1774/*
1775 * cm_chan_connect() - Connect on channel
1776 * @arg:	Channel information
1777 */
1778static int cm_chan_connect(void __user *arg)
1779{
1780	struct rio_cm_channel chan;
1781	struct cm_dev *cm;
1782	struct cm_peer *peer;
1783	int ret = -ENODEV;
1784
1785	if (copy_from_user(&chan, arg, sizeof(chan)))
1786		return -EFAULT;
1787	if (chan.mport_id >= RIO_MAX_MPORTS)
1788		return -EINVAL;
1789
1790	down_read(&rdev_sem);
1791
1792	/* Find matching cm_dev object */
1793	list_for_each_entry(cm, &cm_dev_list, list) {
1794		if (cm->mport->id == chan.mport_id) {
1795			ret = 0;
1796			break;
1797		}
1798	}
1799
1800	if (ret)
1801		goto err_out;
1802
1803	if (chan.remote_destid >= RIO_ANY_DESTID(cm->mport->sys_size)) {
1804		ret = -EINVAL;
1805		goto err_out;
1806	}
1807
1808	/* Find corresponding RapidIO endpoint device object */
1809	ret = -ENODEV;
1810
1811	list_for_each_entry(peer, &cm->peers, node) {
1812		if (peer->rdev->destid == chan.remote_destid) {
1813			ret = 0;
1814			break;
1815		}
1816	}
1817
1818	if (ret)
1819		goto err_out;
1820
1821	up_read(&rdev_sem);
1822
1823	return riocm_ch_connect(chan.id, cm, peer, chan.remote_channel);
1824err_out:
1825	up_read(&rdev_sem);
1826	return ret;
1827}
1828
1829/*
1830 * cm_chan_msg_send() - Send a message through channel
1831 * @arg:	Outbound message information
1832 */
1833static int cm_chan_msg_send(void __user *arg)
1834{
1835	struct rio_cm_msg msg;
1836	void *buf;
1837	int ret;
1838
1839	if (copy_from_user(&msg, arg, sizeof(msg)))
1840		return -EFAULT;
1841	if (msg.size > RIO_MAX_MSG_SIZE)
1842		return -EINVAL;
1843
1844	buf = memdup_user((void __user *)(uintptr_t)msg.msg, msg.size);
1845	if (IS_ERR(buf))
1846		return PTR_ERR(buf);
1847
1848	ret = riocm_ch_send(msg.ch_num, buf, msg.size);
1849
1850	kfree(buf);
1851	return ret;
1852}
1853
1854/*
1855 * cm_chan_msg_rcv() - Receive a message through channel
1856 * @arg:	Inbound message information
1857 */
1858static int cm_chan_msg_rcv(void __user *arg)
1859{
1860	struct rio_cm_msg msg;
1861	struct rio_channel *ch;
1862	void *buf;
1863	long rxto;
1864	int ret = 0, msg_size;
1865
1866	if (copy_from_user(&msg, arg, sizeof(msg)))
1867		return -EFAULT;
1868
1869	if (msg.ch_num == 0 || msg.size == 0)
1870		return -EINVAL;
1871
1872	ch = riocm_get_channel(msg.ch_num);
1873	if (!ch)
1874		return -ENODEV;
1875
1876	rxto = msg.rxto ? msecs_to_jiffies(msg.rxto) : MAX_SCHEDULE_TIMEOUT;
1877
1878	ret = riocm_ch_receive(ch, &buf, rxto);
1879	if (ret)
1880		goto out;
1881
1882	msg_size = min(msg.size, (u16)(RIO_MAX_MSG_SIZE));
1883
1884	if (copy_to_user((void __user *)(uintptr_t)msg.msg, buf, msg_size))
1885		ret = -EFAULT;
1886
1887	riocm_ch_free_rxbuf(ch, buf);
1888out:
1889	riocm_put_channel(ch);
1890	return ret;
1891}
1892
1893/*
1894 * riocm_cdev_ioctl() - IOCTL requests handler
1895 */
1896static long
1897riocm_cdev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
1898{
1899	switch (cmd) {
1900	case RIO_CM_EP_GET_LIST_SIZE:
1901		return cm_ep_get_list_size((void __user *)arg);
1902	case RIO_CM_EP_GET_LIST:
1903		return cm_ep_get_list((void __user *)arg);
1904	case RIO_CM_CHAN_CREATE:
1905		return cm_chan_create(filp, (void __user *)arg);
1906	case RIO_CM_CHAN_CLOSE:
1907		return cm_chan_close(filp, (void __user *)arg);
1908	case RIO_CM_CHAN_BIND:
1909		return cm_chan_bind((void __user *)arg);
1910	case RIO_CM_CHAN_LISTEN:
1911		return cm_chan_listen((void __user *)arg);
1912	case RIO_CM_CHAN_ACCEPT:
1913		return cm_chan_accept(filp, (void __user *)arg);
1914	case RIO_CM_CHAN_CONNECT:
1915		return cm_chan_connect((void __user *)arg);
1916	case RIO_CM_CHAN_SEND:
1917		return cm_chan_msg_send((void __user *)arg);
1918	case RIO_CM_CHAN_RECEIVE:
1919		return cm_chan_msg_rcv((void __user *)arg);
1920	case RIO_CM_MPORT_GET_LIST:
1921		return cm_mport_get_list((void __user *)arg);
1922	default:
1923		break;
1924	}
1925
1926	return -EINVAL;
1927}
1928
1929static const struct file_operations riocm_cdev_fops = {
1930	.owner		= THIS_MODULE,
1931	.open		= riocm_cdev_open,
1932	.release	= riocm_cdev_release,
1933	.unlocked_ioctl = riocm_cdev_ioctl,
1934};
1935
1936/*
1937 * riocm_add_dev - add new remote RapidIO device into channel management core
1938 * @dev: device object associated with RapidIO device
1939 * @sif: subsystem interface
1940 *
1941 * Adds the specified RapidIO device (if applicable) into peers list of
1942 * the corresponding channel management device (cm_dev).
1943 */
1944static int riocm_add_dev(struct device *dev, struct subsys_interface *sif)
1945{
1946	struct cm_peer *peer;
1947	struct rio_dev *rdev = to_rio_dev(dev);
1948	struct cm_dev *cm;
1949
1950	/* Check if the remote device has capabilities required to support CM */
1951	if (!dev_cm_capable(rdev))
1952		return 0;
1953
1954	riocm_debug(RDEV, "(%s)", rio_name(rdev));
1955
1956	peer = kmalloc(sizeof(*peer), GFP_KERNEL);
1957	if (!peer)
1958		return -ENOMEM;
1959
1960	/* Find a corresponding cm_dev object */
1961	down_write(&rdev_sem);
1962	list_for_each_entry(cm, &cm_dev_list, list) {
1963		if (cm->mport == rdev->net->hport)
1964			goto found;
1965	}
1966
1967	up_write(&rdev_sem);
1968	kfree(peer);
1969	return -ENODEV;
1970
1971found:
1972	peer->rdev = rdev;
1973	list_add_tail(&peer->node, &cm->peers);
1974	cm->npeers++;
1975
1976	up_write(&rdev_sem);
1977	return 0;
1978}
1979
1980/*
1981 * riocm_remove_dev - remove remote RapidIO device from channel management core
1982 * @dev: device object associated with RapidIO device
1983 * @sif: subsystem interface
1984 *
1985 * Removes the specified RapidIO device (if applicable) from peers list of
1986 * the corresponding channel management device (cm_dev).
1987 */
1988static void riocm_remove_dev(struct device *dev, struct subsys_interface *sif)
1989{
1990	struct rio_dev *rdev = to_rio_dev(dev);
1991	struct cm_dev *cm;
1992	struct cm_peer *peer;
1993	struct rio_channel *ch, *_c;
1994	unsigned int i;
1995	bool found = false;
1996	LIST_HEAD(list);
1997
1998	/* Check if the remote device has capabilities required to support CM */
1999	if (!dev_cm_capable(rdev))
2000		return;
2001
2002	riocm_debug(RDEV, "(%s)", rio_name(rdev));
2003
2004	/* Find matching cm_dev object */
2005	down_write(&rdev_sem);
2006	list_for_each_entry(cm, &cm_dev_list, list) {
2007		if (cm->mport == rdev->net->hport) {
2008			found = true;
2009			break;
2010		}
2011	}
2012
2013	if (!found) {
2014		up_write(&rdev_sem);
2015		return;
2016	}
2017
2018	/* Remove remote device from the list of peers */
2019	found = false;
2020	list_for_each_entry(peer, &cm->peers, node) {
2021		if (peer->rdev == rdev) {
2022			riocm_debug(RDEV, "removing peer %s", rio_name(rdev));
2023			found = true;
2024			list_del(&peer->node);
2025			cm->npeers--;
2026			kfree(peer);
2027			break;
2028		}
2029	}
2030
2031	up_write(&rdev_sem);
2032
2033	if (!found)
2034		return;
2035
2036	/*
2037	 * Release channels associated with this peer
2038	 */
2039
2040	spin_lock_bh(&idr_lock);
2041	idr_for_each_entry(&ch_idr, ch, i) {
2042		if (ch && ch->rdev == rdev) {
2043			if (atomic_read(&rdev->state) != RIO_DEVICE_SHUTDOWN)
2044				riocm_exch(ch, RIO_CM_DISCONNECT);
2045			idr_remove(&ch_idr, ch->id);
2046			list_add(&ch->ch_node, &list);
2047		}
2048	}
2049	spin_unlock_bh(&idr_lock);
2050
2051	if (!list_empty(&list)) {
2052		list_for_each_entry_safe(ch, _c, &list, ch_node) {
2053			list_del(&ch->ch_node);
2054			riocm_ch_close(ch);
2055		}
2056	}
2057}
2058
2059/*
2060 * riocm_cdev_add() - Create rio_cm char device
2061 * @devno: device number assigned to device (MAJ + MIN)
2062 */
2063static int riocm_cdev_add(dev_t devno)
2064{
2065	int ret;
2066
2067	cdev_init(&riocm_cdev.cdev, &riocm_cdev_fops);
2068	riocm_cdev.cdev.owner = THIS_MODULE;
2069	ret = cdev_add(&riocm_cdev.cdev, devno, 1);
2070	if (ret < 0) {
2071		riocm_error("Cannot register a device with error %d", ret);
2072		return ret;
2073	}
2074
2075	riocm_cdev.dev = device_create(dev_class, NULL, devno, NULL, DEV_NAME);
2076	if (IS_ERR(riocm_cdev.dev)) {
2077		cdev_del(&riocm_cdev.cdev);
2078		return PTR_ERR(riocm_cdev.dev);
2079	}
2080
2081	riocm_debug(MPORT, "Added %s cdev(%d:%d)",
2082		    DEV_NAME, MAJOR(devno), MINOR(devno));
2083
2084	return 0;
2085}
2086
2087/*
2088 * riocm_add_mport - add new local mport device into channel management core
2089 * @dev: device object associated with mport
2090 * @class_intf: class interface
2091 *
2092 * When a new mport device is added, CM immediately reserves inbound and
2093 * outbound RapidIO mailboxes that will be used.
2094 */
2095static int riocm_add_mport(struct device *dev,
2096			   struct class_interface *class_intf)
2097{
2098	int rc;
2099	int i;
2100	struct cm_dev *cm;
2101	struct rio_mport *mport = to_rio_mport(dev);
2102
2103	riocm_debug(MPORT, "add mport %s", mport->name);
2104
2105	cm = kzalloc(sizeof(*cm), GFP_KERNEL);
2106	if (!cm)
2107		return -ENOMEM;
2108
2109	cm->mport = mport;
2110
2111	rc = rio_request_outb_mbox(mport, cm, cmbox,
2112				   RIOCM_TX_RING_SIZE, riocm_outb_msg_event);
2113	if (rc) {
2114		riocm_error("failed to allocate OBMBOX_%d on %s",
2115			    cmbox, mport->name);
2116		kfree(cm);
2117		return -ENODEV;
2118	}
2119
2120	rc = rio_request_inb_mbox(mport, cm, cmbox,
2121				  RIOCM_RX_RING_SIZE, riocm_inb_msg_event);
2122	if (rc) {
2123		riocm_error("failed to allocate IBMBOX_%d on %s",
2124			    cmbox, mport->name);
2125		rio_release_outb_mbox(mport, cmbox);
2126		kfree(cm);
2127		return -ENODEV;
2128	}
2129
2130	cm->rx_wq = create_workqueue(DRV_NAME "/rxq");
2131	if (!cm->rx_wq) {
2132		rio_release_inb_mbox(mport, cmbox);
2133		rio_release_outb_mbox(mport, cmbox);
2134		kfree(cm);
2135		return -ENOMEM;
2136	}
2137
2138	/*
2139	 * Allocate and register inbound messaging buffers to be ready
2140	 * to receive channel and system management requests
2141	 */
2142	for (i = 0; i < RIOCM_RX_RING_SIZE; i++)
2143		cm->rx_buf[i] = NULL;
2144
2145	cm->rx_slots = RIOCM_RX_RING_SIZE;
2146	mutex_init(&cm->rx_lock);
2147	riocm_rx_fill(cm, RIOCM_RX_RING_SIZE);
2148	INIT_WORK(&cm->rx_work, rio_ibmsg_handler);
2149
2150	cm->tx_slot = 0;
2151	cm->tx_cnt = 0;
2152	cm->tx_ack_slot = 0;
2153	spin_lock_init(&cm->tx_lock);
2154
2155	INIT_LIST_HEAD(&cm->peers);
2156	cm->npeers = 0;
2157	INIT_LIST_HEAD(&cm->tx_reqs);
2158
2159	down_write(&rdev_sem);
2160	list_add_tail(&cm->list, &cm_dev_list);
2161	up_write(&rdev_sem);
2162
2163	return 0;
2164}
2165
2166/*
2167 * riocm_remove_mport - remove local mport device from channel management core
2168 * @dev: device object associated with mport
2169 * @class_intf: class interface
2170 *
2171 * Removes a local mport device from the list of registered devices that provide
2172 * channel management services. Returns an error if the specified mport is not
2173 * registered with the CM core.
2174 */
2175static void riocm_remove_mport(struct device *dev,
2176			       struct class_interface *class_intf)
2177{
2178	struct rio_mport *mport = to_rio_mport(dev);
2179	struct cm_dev *cm;
2180	struct cm_peer *peer, *temp;
2181	struct rio_channel *ch, *_c;
2182	unsigned int i;
2183	bool found = false;
2184	LIST_HEAD(list);
2185
2186	riocm_debug(MPORT, "%s", mport->name);
2187
2188	/* Find a matching cm_dev object */
2189	down_write(&rdev_sem);
2190	list_for_each_entry(cm, &cm_dev_list, list) {
2191		if (cm->mport == mport) {
2192			list_del(&cm->list);
2193			found = true;
2194			break;
2195		}
2196	}
2197	up_write(&rdev_sem);
2198	if (!found)
2199		return;
2200
2201	flush_workqueue(cm->rx_wq);
2202	destroy_workqueue(cm->rx_wq);
2203
2204	/* Release channels bound to this mport */
2205	spin_lock_bh(&idr_lock);
2206	idr_for_each_entry(&ch_idr, ch, i) {
2207		if (ch->cmdev == cm) {
2208			riocm_debug(RDEV, "%s drop ch_%d",
2209				    mport->name, ch->id);
2210			idr_remove(&ch_idr, ch->id);
2211			list_add(&ch->ch_node, &list);
2212		}
2213	}
2214	spin_unlock_bh(&idr_lock);
2215
2216	if (!list_empty(&list)) {
2217		list_for_each_entry_safe(ch, _c, &list, ch_node) {
2218			list_del(&ch->ch_node);
2219			riocm_ch_close(ch);
2220		}
2221	}
2222
2223	rio_release_inb_mbox(mport, cmbox);
2224	rio_release_outb_mbox(mport, cmbox);
2225
2226	/* Remove and free peer entries */
2227	if (!list_empty(&cm->peers))
2228		riocm_debug(RDEV, "ATTN: peer list not empty");
2229	list_for_each_entry_safe(peer, temp, &cm->peers, node) {
2230		riocm_debug(RDEV, "removing peer %s", rio_name(peer->rdev));
2231		list_del(&peer->node);
2232		kfree(peer);
2233	}
2234
2235	riocm_rx_free(cm);
2236	kfree(cm);
2237	riocm_debug(MPORT, "%s done", mport->name);
2238}
2239
2240static int rio_cm_shutdown(struct notifier_block *nb, unsigned long code,
2241	void *unused)
2242{
2243	struct rio_channel *ch;
2244	unsigned int i;
2245	LIST_HEAD(list);
2246
2247	riocm_debug(EXIT, ".");
2248
2249	/*
2250	 * If there are any channels left in connected state send
2251	 * close notification to the connection partner.
2252	 * First build a list of channels that require a closing
2253	 * notification because function riocm_send_close() should
2254	 * be called outside of spinlock protected code.
2255	 */
2256	spin_lock_bh(&idr_lock);
2257	idr_for_each_entry(&ch_idr, ch, i) {
2258		if (ch->state == RIO_CM_CONNECTED) {
2259			riocm_debug(EXIT, "close ch %d", ch->id);
2260			idr_remove(&ch_idr, ch->id);
2261			list_add(&ch->ch_node, &list);
2262		}
2263	}
2264	spin_unlock_bh(&idr_lock);
2265
2266	list_for_each_entry(ch, &list, ch_node)
2267		riocm_send_close(ch);
2268
2269	return NOTIFY_DONE;
2270}
2271
2272/*
2273 * riocm_interface handles addition/removal of remote RapidIO devices
2274 */
2275static struct subsys_interface riocm_interface = {
2276	.name		= "rio_cm",
2277	.subsys		= &rio_bus_type,
2278	.add_dev	= riocm_add_dev,
2279	.remove_dev	= riocm_remove_dev,
2280};
2281
2282/*
2283 * rio_mport_interface handles addition/removal local mport devices
2284 */
2285static struct class_interface rio_mport_interface __refdata = {
2286	.class = &rio_mport_class,
2287	.add_dev = riocm_add_mport,
2288	.remove_dev = riocm_remove_mport,
2289};
2290
2291static struct notifier_block rio_cm_notifier = {
2292	.notifier_call = rio_cm_shutdown,
2293};
2294
2295static int __init riocm_init(void)
2296{
2297	int ret;
2298
2299	/* Create device class needed by udev */
2300	dev_class = class_create(THIS_MODULE, DRV_NAME);
2301	if (IS_ERR(dev_class)) {
2302		riocm_error("Cannot create " DRV_NAME " class");
2303		return PTR_ERR(dev_class);
2304	}
2305
2306	ret = alloc_chrdev_region(&dev_number, 0, 1, DRV_NAME);
2307	if (ret) {
2308		class_destroy(dev_class);
2309		return ret;
2310	}
2311
2312	dev_major = MAJOR(dev_number);
2313	dev_minor_base = MINOR(dev_number);
2314	riocm_debug(INIT, "Registered class with %d major", dev_major);
2315
2316	/*
2317	 * Register as rapidio_port class interface to get notifications about
2318	 * mport additions and removals.
2319	 */
2320	ret = class_interface_register(&rio_mport_interface);
2321	if (ret) {
2322		riocm_error("class_interface_register error: %d", ret);
2323		goto err_reg;
2324	}
2325
2326	/*
2327	 * Register as RapidIO bus interface to get notifications about
2328	 * addition/removal of remote RapidIO devices.
2329	 */
2330	ret = subsys_interface_register(&riocm_interface);
2331	if (ret) {
2332		riocm_error("subsys_interface_register error: %d", ret);
2333		goto err_cl;
2334	}
2335
2336	ret = register_reboot_notifier(&rio_cm_notifier);
2337	if (ret) {
2338		riocm_error("failed to register reboot notifier (err=%d)", ret);
2339		goto err_sif;
2340	}
2341
2342	ret = riocm_cdev_add(dev_number);
2343	if (ret) {
2344		unregister_reboot_notifier(&rio_cm_notifier);
2345		ret = -ENODEV;
2346		goto err_sif;
2347	}
2348
2349	return 0;
2350err_sif:
2351	subsys_interface_unregister(&riocm_interface);
2352err_cl:
2353	class_interface_unregister(&rio_mport_interface);
2354err_reg:
2355	unregister_chrdev_region(dev_number, 1);
2356	class_destroy(dev_class);
2357	return ret;
2358}
2359
2360static void __exit riocm_exit(void)
2361{
2362	riocm_debug(EXIT, "enter");
2363	unregister_reboot_notifier(&rio_cm_notifier);
2364	subsys_interface_unregister(&riocm_interface);
2365	class_interface_unregister(&rio_mport_interface);
2366	idr_destroy(&ch_idr);
2367
2368	device_unregister(riocm_cdev.dev);
2369	cdev_del(&(riocm_cdev.cdev));
2370
2371	class_destroy(dev_class);
2372	unregister_chrdev_region(dev_number, 1);
2373}
2374
2375late_initcall(riocm_init);
2376module_exit(riocm_exit);
2377