1/*-
2 * Copyright (c) 2005-2011 Daniel Braniss <danny@cs.huji.ac.il>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * $FreeBSD$
27 */
28
29/*
30 | $Id: iscsivar.h 743 2009-08-08 10:54:53Z danny $
31 */
32#define ISCSI_MAX_LUNS		128	// don't touch this
33#if ISCSI_MAX_LUNS > 8
34/*
35 | for this to work
36 | sysctl kern.cam.cam_srch_hi=1
37 */
38#endif
39
40#ifndef ISCSI_INITIATOR_DEBUG
41#define ISCSI_INITIATOR_DEBUG 1
42#endif
43
44#ifdef ISCSI_INITIATOR_DEBUG
45extern int iscsi_debug;
46#define debug(level, fmt, args...)	do {if(level <= iscsi_debug)\
47	printf("%s: " fmt "\n", __func__ , ##args);} while(0)
48#define sdebug(level, fmt, args...)	do {if(level <= iscsi_debug)\
49     	printf("%d] %s: " fmt "\n", sp->sid, __func__ , ##args);} while(0)
50#define debug_called(level)		do {if(level <= iscsi_debug)\
51	printf("%s: called\n",  __func__);} while(0)
52#else
53#define debug(level, fmt, args...)
54#define debug_called(level)
55#define sdebug(level, fmt, args...)
56#endif /* ISCSI_INITIATOR_DEBUG */
57
58#define xdebug(fmt, args...)	printf(">>> %s: " fmt "\n", __func__ , ##args)
59
60#define MAX_SESSIONS	ISCSI_MAX_TARGETS
61#define MAX_PDUS	(MAX_SESSIONS*256) // XXX: at the moment this is arbitrary
62
63typedef uint32_t digest_t(const void *, int len, uint32_t ocrc);
64
65MALLOC_DECLARE(M_ISCSI);
66MALLOC_DECLARE(M_ISCSIBUF);
67
68#define ISOK2DIG(dig, pp)	((dig != NULL) && ((pp->ipdu.bhs.opcode & 0x1f) != ISCSI_LOGIN_CMD))
69
70#ifndef BIT
71#define BIT(n)	(1 <<(n))
72#endif
73
74#define ISC_SM_RUN	BIT(0)
75#define ISC_SM_RUNNING	BIT(1)
76
77#define ISC_LINK_UP	BIT(2)
78#define ISC_CON_RUN	BIT(3)
79#define ISC_CON_RUNNING	BIT(4)
80#define ISC_KILL	BIT(5)
81#define ISC_OQNOTEMPTY	BIT(6)
82#define ISC_OWAITING	BIT(7)
83#define ISC_FFPHASE	BIT(8)
84
85#define ISC_CAMDEVS	BIT(9)
86#define ISC_SCANWAIT	BIT(10)
87
88#define ISC_MEMWAIT	BIT(11)
89#define ISC_SIGNALED	BIT(12)
90
91#define ISC_HOLD	BIT(15)
92#define ISC_HOLDED	BIT(16)
93
94#define ISC_SHUTDOWN	BIT(31)
95
96/*
97 | some stats
98 */
99struct i_stats {
100     int	npdu;	// number of pdus malloc'ed.
101     int	nrecv;	// unprocessed received pdus
102     int	nsent;	// sent pdus
103
104     int	nrsp, max_rsp;
105     int	nrsv, max_rsv;
106     int	ncsnd, max_csnd;
107     int	nisnd, max_isnd;
108     int	nwsnd, max_wsnd;
109     int	nhld, max_hld;
110
111     struct bintime t_sent;
112     struct bintime t_recv;
113};
114
115/*
116 | one per 'session'
117 */
118
119typedef TAILQ_HEAD(, pduq) queue_t;
120
121typedef struct isc_session {
122     TAILQ_ENTRY(isc_session)	sp_link;
123     int		flags;
124     struct cdev	*dev;
125     struct socket	*soc;
126     struct file	*fp;
127     struct thread	*td;
128
129     struct proc 	*proc; // the userland process
130     int		signal;
131     struct proc 	*soc_proc;
132     struct proc	*stp;	// the sm thread
133
134     struct isc_softc	*isc;
135
136     digest_t   	*hdrDigest;     // the digest alg. if any
137     digest_t   	*dataDigest;    // the digest alg. if any
138
139     int		sid;		// Session ID
140     sn_t       	sn;             // sequence number stuff;
141     int		cws;		// current window size
142
143     int		target_nluns; // this and target_lun are
144				      // hopefully temporal till I
145				      // figure out a better way.
146     int		target_lun[ISCSI_MAX_LUNS/(sizeof(int)*8) + 1];
147
148     struct mtx		rsp_mtx;
149     struct mtx		rsv_mtx;
150     struct mtx		snd_mtx;
151     struct mtx		hld_mtx;
152     struct mtx		io_mtx;
153     queue_t		rsp;
154     queue_t		rsv;
155     queue_t		csnd;
156     queue_t		isnd;
157     queue_t		wsnd;
158     queue_t		hld;
159
160     isc_opt_t		opt;	// negotiable values
161
162     struct i_stats	stats;
163     bhs_t		bhs;
164     struct uio		uio;
165     struct iovec	iov;
166     /*
167      | cam stuff
168      */
169     struct cam_sim	*cam_sim;
170     struct cam_path	*cam_path;
171     struct mtx		cam_mtx;
172     /*
173      | sysctl stuff
174      */
175     struct sysctl_ctx_list	clist;
176     struct sysctl_oid	*oid;
177     int	douio;	//XXX: turn on/off uio on read
178} isc_session_t;
179
180typedef struct pduq {
181     TAILQ_ENTRY(pduq)	pq_link;
182
183     caddr_t		buf;
184     u_int		len;	// the total length of the pdu
185     pdu_t		pdu;
186     union ccb		*ccb;
187
188     struct uio		uio;
189     struct iovec	iov[5];	// XXX: careful ...
190     struct mbuf	*mp;
191     struct bintime	ts;
192     queue_t		*pduq;
193} pduq_t;
194/*
195 */
196struct isc_softc {
197     struct mtx		isc_mtx;
198     TAILQ_HEAD(,isc_session)	isc_sess;
199     int		nsess;
200     struct cdev	*dev;
201     char		isid[6];	// Initiator Session ID (48 bits)
202     struct unrhdr	*unit;
203     struct sx 		unit_sx;
204
205     uma_zone_t		pdu_zone;	// pool of free pdu's
206     TAILQ_HEAD(,pduq)	freepdu;
207
208#ifdef  ISCSI_INITIATOR_DEBUG
209     int		 npdu_alloc, npdu_max; // for instrumentation
210#endif
211#ifdef DO_EVENTHANDLER
212     eventhandler_tag	eh;
213#endif
214     /*
215      | sysctl stuff
216      */
217     struct sysctl_ctx_list	clist;
218     struct sysctl_oid		*oid;
219};
220
221#ifdef  ISCSI_INITIATOR_DEBUG
222extern struct mtx iscsi_dbg_mtx;
223#endif
224
225void	isc_start_receiver(isc_session_t *sp);
226void	isc_stop_receiver(isc_session_t *sp);
227
228int	isc_sendPDU(isc_session_t *sp, pduq_t *pq);
229int	isc_qout(isc_session_t *sp, pduq_t *pq);
230int	i_prepPDU(isc_session_t *sp, pduq_t *pq);
231
232int	ism_fullfeature(struct cdev *dev, int flag);
233
234int	i_pdu_flush(isc_session_t *sc);
235int	i_setopt(isc_session_t *sp, isc_opt_t *opt);
236void	i_freeopt(isc_opt_t *opt);
237
238int	ic_init(isc_session_t *sp);
239void	ic_destroy(isc_session_t *sp);
240void	ic_lost_target(isc_session_t *sp, int target);
241int	ic_getCamVals(isc_session_t *sp, iscsi_cam_t *cp);
242
243void	ism_recv(isc_session_t *sp, pduq_t *pq);
244int	ism_start(isc_session_t *sp);
245void	ism_restart(isc_session_t *sp);
246void	ism_stop(isc_session_t *sp);
247
248int	scsi_encap(struct cam_sim *sim, union ccb *ccb);
249int	scsi_decap(isc_session_t *sp, pduq_t *opq, pduq_t *pq);
250void	iscsi_r2t(isc_session_t *sp, pduq_t *opq, pduq_t *pq);
251void	iscsi_done(isc_session_t *sp, pduq_t *opq, pduq_t *pq);
252void	iscsi_reject(isc_session_t *sp, pduq_t *opq, pduq_t *pq);
253void	iscsi_async(isc_session_t *sp,  pduq_t *pq);
254void	iscsi_cleanup(isc_session_t *sp);
255int	iscsi_requeue(isc_session_t *sp);
256
257// Serial Number Arithmetic
258#define _MAXINCR	0x7FFFFFFF	// 2 ^ 31 - 1
259#define SNA_GT(i1, i2)	((i1 != i2) && (\
260	(i1 < i2 && i2 - i1 > _MAXINCR) ||\
261	(i1 > i2 && i1 - i2 < _MAXINCR))?1: 0)
262
263/*
264 | inlines
265 */
266#ifdef _CAM_CAM_XPT_SIM_H
267
268#if __FreeBSD_version <  600000
269#define CAM_LOCK(arg)
270#define CAM_ULOCK(arg)
271
272static __inline void
273XPT_DONE(isc_session_t *sp, union ccb *ccb)
274{
275     mtx_lock(&Giant);
276     xpt_done(ccb);
277     mtx_unlock(&Giant);
278}
279#elif __FreeBSD_version >= 700000
280#define CAM_LOCK(arg)	mtx_lock(&arg->cam_mtx)
281#define CAM_UNLOCK(arg)	mtx_unlock(&arg->cam_mtx)
282
283static __inline void
284XPT_DONE(isc_session_t *sp, union ccb *ccb)
285{
286     CAM_LOCK(sp);
287     xpt_done(ccb);
288     CAM_UNLOCK(sp);
289}
290#else
291//__FreeBSD_version >= 600000
292#define CAM_LOCK(arg)
293#define CAM_UNLOCK(arg)
294#define XPT_DONE(ignore, arg)	xpt_done(arg)
295#endif
296
297#endif /* _CAM_CAM_XPT_SIM_H */
298
299static __inline pduq_t *
300pdu_alloc(struct isc_softc *isc, int wait)
301{
302     pduq_t	*pq;
303
304     pq = (pduq_t *)uma_zalloc(isc->pdu_zone, wait /* M_WAITOK or M_NOWAIT*/);
305     if(pq == NULL) {
306	  debug(7, "out of mem");
307	  return NULL;
308     }
309#ifdef ISCSI_INITIATOR_DEBUG
310     mtx_lock(&iscsi_dbg_mtx);
311     isc->npdu_alloc++;
312     if(isc->npdu_alloc > isc->npdu_max)
313	  isc->npdu_max = isc->npdu_alloc;
314     mtx_unlock(&iscsi_dbg_mtx);
315#endif
316     memset(pq, 0, sizeof(pduq_t));
317
318     return pq;
319}
320
321static __inline void
322pdu_free(struct isc_softc *isc, pduq_t *pq)
323{
324     if(pq->mp)
325	  m_freem(pq->mp);
326#ifdef NO_USE_MBUF
327     if(pq->buf != NULL)
328	  free(pq->buf, M_ISCSIBUF);
329#endif
330     uma_zfree(isc->pdu_zone, pq);
331#ifdef ISCSI_INITIATOR_DEBUG
332     mtx_lock(&iscsi_dbg_mtx);
333     isc->npdu_alloc--;
334     mtx_unlock(&iscsi_dbg_mtx);
335#endif
336}
337
338static __inline void
339i_nqueue_rsp(isc_session_t *sp, pduq_t *pq)
340{
341     mtx_lock(&sp->rsp_mtx);
342     if(++sp->stats.nrsp > sp->stats.max_rsp)
343	  sp->stats.max_rsp = sp->stats.nrsp;
344     TAILQ_INSERT_TAIL(&sp->rsp, pq, pq_link);
345     mtx_unlock(&sp->rsp_mtx);
346}
347
348static __inline pduq_t *
349i_dqueue_rsp(isc_session_t *sp)
350{
351     pduq_t *pq;
352
353     mtx_lock(&sp->rsp_mtx);
354     if((pq = TAILQ_FIRST(&sp->rsp)) != NULL) {
355	  sp->stats.nrsp--;
356	  TAILQ_REMOVE(&sp->rsp, pq, pq_link);
357     }
358     mtx_unlock(&sp->rsp_mtx);
359
360     return pq;
361}
362
363static __inline void
364i_nqueue_rsv(isc_session_t *sp, pduq_t *pq)
365{
366     mtx_lock(&sp->rsv_mtx);
367     if(++sp->stats.nrsv > sp->stats.max_rsv)
368	  sp->stats.max_rsv = sp->stats.nrsv;
369     TAILQ_INSERT_TAIL(&sp->rsv, pq, pq_link);
370     mtx_unlock(&sp->rsv_mtx);
371}
372
373static __inline pduq_t *
374i_dqueue_rsv(isc_session_t *sp)
375{
376     pduq_t *pq;
377
378     mtx_lock(&sp->rsv_mtx);
379     if((pq = TAILQ_FIRST(&sp->rsv)) != NULL) {
380	  sp->stats.nrsv--;
381	  TAILQ_REMOVE(&sp->rsv, pq, pq_link);
382     }
383     mtx_unlock(&sp->rsv_mtx);
384
385     return pq;
386}
387
388static __inline void
389i_nqueue_csnd(isc_session_t *sp, pduq_t *pq)
390{
391     mtx_lock(&sp->snd_mtx);
392     if(++sp->stats.ncsnd > sp->stats.max_csnd)
393	  sp->stats.max_csnd = sp->stats.ncsnd;
394     TAILQ_INSERT_TAIL(&sp->csnd, pq, pq_link);
395     mtx_unlock(&sp->snd_mtx);
396}
397
398static __inline pduq_t *
399i_dqueue_csnd(isc_session_t *sp)
400{
401     pduq_t *pq;
402
403     mtx_lock(&sp->snd_mtx);
404     if((pq = TAILQ_FIRST(&sp->csnd)) != NULL) {
405	  sp->stats.ncsnd--;
406	  TAILQ_REMOVE(&sp->csnd, pq, pq_link);
407     }
408     mtx_unlock(&sp->snd_mtx);
409
410     return pq;
411}
412
413static __inline void
414i_nqueue_isnd(isc_session_t *sp, pduq_t *pq)
415{
416     mtx_lock(&sp->snd_mtx);
417     if(++sp->stats.nisnd > sp->stats.max_isnd)
418	  sp->stats.max_isnd = sp->stats.nisnd;
419     TAILQ_INSERT_TAIL(&sp->isnd, pq, pq_link);
420     mtx_unlock(&sp->snd_mtx);
421}
422
423static __inline pduq_t *
424i_dqueue_isnd(isc_session_t *sp)
425{
426     pduq_t *pq;
427
428     mtx_lock(&sp->snd_mtx);
429     if((pq = TAILQ_FIRST(&sp->isnd)) != NULL) {
430	  sp->stats.nisnd--;
431	  TAILQ_REMOVE(&sp->isnd, pq, pq_link);
432     }
433     mtx_unlock(&sp->snd_mtx);
434
435     return pq;
436}
437
438static __inline void
439i_nqueue_wsnd(isc_session_t *sp, pduq_t *pq)
440{
441     mtx_lock(&sp->snd_mtx);
442     if(++sp->stats.nwsnd > sp->stats.max_wsnd)
443	  sp->stats.max_wsnd = sp->stats.nwsnd;
444     TAILQ_INSERT_TAIL(&sp->wsnd, pq, pq_link);
445     mtx_unlock(&sp->snd_mtx);
446}
447
448static __inline pduq_t *
449i_dqueue_wsnd(isc_session_t *sp)
450{
451     pduq_t *pq;
452
453     mtx_lock(&sp->snd_mtx);
454     if((pq = TAILQ_FIRST(&sp->wsnd)) != NULL) {
455	  sp->stats.nwsnd--;
456	  TAILQ_REMOVE(&sp->wsnd, pq, pq_link);
457     }
458     mtx_unlock(&sp->snd_mtx);
459
460     return pq;
461}
462
463static __inline pduq_t *
464i_dqueue_snd(isc_session_t *sp, int which)
465{
466     pduq_t *pq;
467
468     pq = NULL;
469     mtx_lock(&sp->snd_mtx);
470     if((which & BIT(0)) && (pq = TAILQ_FIRST(&sp->isnd)) != NULL) {
471	  sp->stats.nisnd--;
472	  TAILQ_REMOVE(&sp->isnd, pq, pq_link);
473	  pq->pduq = &sp->isnd;	// remember where you came from
474     } else
475     if((which & BIT(1)) && (pq = TAILQ_FIRST(&sp->wsnd)) != NULL) {
476	  sp->stats.nwsnd--;
477	  TAILQ_REMOVE(&sp->wsnd, pq, pq_link);
478	  pq->pduq = &sp->wsnd;	// remember where you came from
479     } else
480     if((which & BIT(2)) && (pq = TAILQ_FIRST(&sp->csnd)) != NULL) {
481	  sp->stats.ncsnd--;
482	  TAILQ_REMOVE(&sp->csnd, pq, pq_link);
483	  pq->pduq = &sp->csnd;	// remember where you came from
484     }
485     mtx_unlock(&sp->snd_mtx);
486
487     return pq;
488}
489
490static __inline void
491i_rqueue_pdu(isc_session_t *sp, pduq_t *pq)
492{
493     mtx_lock(&sp->snd_mtx);
494     KASSERT(pq->pduq != NULL, ("pq->pduq is NULL"));
495     TAILQ_INSERT_TAIL(pq->pduq, pq, pq_link);
496     mtx_unlock(&sp->snd_mtx);
497}
498
499/*
500 | Waiting for ACK (or something :-)
501 */
502static __inline void
503i_nqueue_hld(isc_session_t *sp, pduq_t *pq)
504{
505     getbintime(&pq->ts);
506     mtx_lock(&sp->hld_mtx);
507     if(++sp->stats.nhld > sp->stats.max_hld)
508	  sp->stats.max_hld = sp->stats.nhld;
509     TAILQ_INSERT_TAIL(&sp->hld, pq, pq_link);
510     mtx_unlock(&sp->hld_mtx);
511     return;
512}
513
514static __inline void
515i_remove_hld(isc_session_t *sp, pduq_t *pq)
516{
517     mtx_lock(&sp->hld_mtx);
518     sp->stats.nhld--;
519     TAILQ_REMOVE(&sp->hld, pq, pq_link);
520     mtx_unlock(&sp->hld_mtx);
521}
522
523static __inline pduq_t *
524i_dqueue_hld(isc_session_t *sp)
525{
526     pduq_t *pq;
527
528     mtx_lock(&sp->hld_mtx);
529     if((pq = TAILQ_FIRST(&sp->hld)) != NULL) {
530	  sp->stats.nhld--;
531	  TAILQ_REMOVE(&sp->hld, pq, pq_link);
532     }
533     mtx_unlock(&sp->hld_mtx);
534
535     return pq;
536}
537
538static __inline pduq_t *
539i_search_hld(isc_session_t *sp, int itt, int keep)
540{
541     pduq_t	*pq, *tmp;
542
543     pq = NULL;
544
545     mtx_lock(&sp->hld_mtx);
546     TAILQ_FOREACH_SAFE(pq, &sp->hld, pq_link, tmp) {
547	  if(pq->pdu.ipdu.bhs.itt == itt) {
548	       if(!keep) {
549		    sp->stats.nhld--;
550		    TAILQ_REMOVE(&sp->hld, pq, pq_link);
551	       }
552	       break;
553	  }
554     }
555     mtx_unlock(&sp->hld_mtx);
556
557     return pq;
558}
559
560static __inline void
561i_acked_hld(isc_session_t *sp, pdu_t *op)
562{
563     pduq_t	*pq, *tmp;
564     u_int exp = sp->sn.expCmd;
565
566     pq = NULL;
567     mtx_lock(&sp->hld_mtx);
568     TAILQ_FOREACH_SAFE(pq, &sp->hld, pq_link, tmp) {
569	  if((op && op->ipdu.bhs.itt == pq->pdu.ipdu.bhs.itt)
570	     || (pq->ccb == NULL
571		 && (pq->pdu.ipdu.bhs.opcode != ISCSI_WRITE_DATA)
572		 && SNA_GT(exp, ntohl(pq->pdu.ipdu.bhs.ExpStSN)))) {
573	       sp->stats.nhld--;
574	       TAILQ_REMOVE(&sp->hld, pq, pq_link);
575	       pdu_free(sp->isc, pq);
576	  }
577     }
578     mtx_unlock(&sp->hld_mtx);
579}
580
581static __inline void
582i_mbufcopy(struct mbuf *mp, caddr_t dp, int len)
583{
584     struct mbuf *m;
585     caddr_t bp;
586
587     for(m = mp; m != NULL; m = m->m_next) {
588	  bp = mtod(m, caddr_t);
589	  /*
590	   | the pdu is word (4 octed) aligned
591	   | so len <= packet
592	   */
593	  memcpy(dp, bp, MIN(len, m->m_len));
594	  dp += m->m_len;
595	  len -= m->m_len;
596	  if(len <= 0)
597	       break;
598     }
599}
600