1/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
2/*
3 * Copyright(c) 2015 - 2017 Intel Corporation.
4 */
5#if !defined(__HFI1_TRACE_TX_H) || defined(TRACE_HEADER_MULTI_READ)
6#define __HFI1_TRACE_TX_H
7
8#include <linux/tracepoint.h>
9#include <linux/trace_seq.h>
10
11#include "hfi.h"
12#include "mad.h"
13#include "sdma.h"
14#include "ipoib.h"
15#include "user_sdma.h"
16
17const char *parse_sdma_flags(struct trace_seq *p, u64 desc0, u64 desc1);
18
19#define __parse_sdma_flags(desc0, desc1) parse_sdma_flags(p, desc0, desc1)
20
21#undef TRACE_SYSTEM
22#define TRACE_SYSTEM hfi1_tx
23
24TRACE_EVENT(hfi1_piofree,
25	    TP_PROTO(struct send_context *sc, int extra),
26	    TP_ARGS(sc, extra),
27	    TP_STRUCT__entry(DD_DEV_ENTRY(sc->dd)
28	    __field(u32, sw_index)
29	    __field(u32, hw_context)
30	    __field(int, extra)
31	    ),
32	    TP_fast_assign(DD_DEV_ASSIGN(sc->dd);
33	    __entry->sw_index = sc->sw_index;
34	    __entry->hw_context = sc->hw_context;
35	    __entry->extra = extra;
36	    ),
37	    TP_printk("[%s] ctxt %u(%u) extra %d",
38		      __get_str(dev),
39		      __entry->sw_index,
40		      __entry->hw_context,
41		      __entry->extra
42	    )
43);
44
45TRACE_EVENT(hfi1_wantpiointr,
46	    TP_PROTO(struct send_context *sc, u32 needint, u64 credit_ctrl),
47	    TP_ARGS(sc, needint, credit_ctrl),
48	    TP_STRUCT__entry(DD_DEV_ENTRY(sc->dd)
49			__field(u32, sw_index)
50			__field(u32, hw_context)
51			__field(u32, needint)
52			__field(u64, credit_ctrl)
53			),
54	    TP_fast_assign(DD_DEV_ASSIGN(sc->dd);
55			__entry->sw_index = sc->sw_index;
56			__entry->hw_context = sc->hw_context;
57			__entry->needint = needint;
58			__entry->credit_ctrl = credit_ctrl;
59			),
60	    TP_printk("[%s] ctxt %u(%u) on %d credit_ctrl 0x%llx",
61		      __get_str(dev),
62		      __entry->sw_index,
63		      __entry->hw_context,
64		      __entry->needint,
65		      (unsigned long long)__entry->credit_ctrl
66		      )
67);
68
69DECLARE_EVENT_CLASS(hfi1_qpsleepwakeup_template,
70		    TP_PROTO(struct rvt_qp *qp, u32 flags),
71		    TP_ARGS(qp, flags),
72		    TP_STRUCT__entry(
73		    DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device))
74		    __field(u32, qpn)
75		    __field(u32, flags)
76		    __field(u32, s_flags)
77		    __field(u32, ps_flags)
78		    __field(unsigned long, iow_flags)
79		    ),
80		    TP_fast_assign(
81		    DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device));
82		    __entry->flags = flags;
83		    __entry->qpn = qp->ibqp.qp_num;
84		    __entry->s_flags = qp->s_flags;
85		    __entry->ps_flags =
86			((struct hfi1_qp_priv *)qp->priv)->s_flags;
87		    __entry->iow_flags =
88			((struct hfi1_qp_priv *)qp->priv)->s_iowait.flags;
89		    ),
90		    TP_printk(
91		    "[%s] qpn 0x%x flags 0x%x s_flags 0x%x ps_flags 0x%x iow_flags 0x%lx",
92		    __get_str(dev),
93		    __entry->qpn,
94		    __entry->flags,
95		    __entry->s_flags,
96		    __entry->ps_flags,
97		    __entry->iow_flags
98		    )
99);
100
101DEFINE_EVENT(hfi1_qpsleepwakeup_template, hfi1_qpwakeup,
102	     TP_PROTO(struct rvt_qp *qp, u32 flags),
103	     TP_ARGS(qp, flags));
104
105DEFINE_EVENT(hfi1_qpsleepwakeup_template, hfi1_qpsleep,
106	     TP_PROTO(struct rvt_qp *qp, u32 flags),
107	     TP_ARGS(qp, flags));
108
109TRACE_EVENT(hfi1_sdma_descriptor,
110	    TP_PROTO(struct sdma_engine *sde,
111		     u64 desc0,
112		     u64 desc1,
113		     u16 e,
114		     void *descp),
115		     TP_ARGS(sde, desc0, desc1, e, descp),
116		     TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
117		     __field(void *, descp)
118		     __field(u64, desc0)
119		     __field(u64, desc1)
120		     __field(u16, e)
121		     __field(u8, idx)
122		     ),
123		     TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
124		     __entry->desc0 = desc0;
125		     __entry->desc1 = desc1;
126		     __entry->idx = sde->this_idx;
127		     __entry->descp = descp;
128		     __entry->e = e;
129		     ),
130	    TP_printk(
131	    "[%s] SDE(%u) flags:%s addr:0x%016llx gen:%u len:%u d0:%016llx d1:%016llx to %p,%u",
132	    __get_str(dev),
133	    __entry->idx,
134	    __parse_sdma_flags(__entry->desc0, __entry->desc1),
135	    (__entry->desc0 >> SDMA_DESC0_PHY_ADDR_SHIFT) &
136	    SDMA_DESC0_PHY_ADDR_MASK,
137	    (u8)((__entry->desc1 >> SDMA_DESC1_GENERATION_SHIFT) &
138	    SDMA_DESC1_GENERATION_MASK),
139	    (u16)((__entry->desc0 >> SDMA_DESC0_BYTE_COUNT_SHIFT) &
140	    SDMA_DESC0_BYTE_COUNT_MASK),
141	    __entry->desc0,
142	    __entry->desc1,
143	    __entry->descp,
144	    __entry->e
145	    )
146);
147
148TRACE_EVENT(hfi1_sdma_engine_select,
149	    TP_PROTO(struct hfi1_devdata *dd, u32 sel, u8 vl, u8 idx),
150	    TP_ARGS(dd, sel, vl, idx),
151	    TP_STRUCT__entry(DD_DEV_ENTRY(dd)
152	    __field(u32, sel)
153	    __field(u8, vl)
154	    __field(u8, idx)
155	    ),
156	    TP_fast_assign(DD_DEV_ASSIGN(dd);
157	    __entry->sel = sel;
158	    __entry->vl = vl;
159	    __entry->idx = idx;
160	    ),
161	    TP_printk("[%s] selecting SDE %u sel 0x%x vl %u",
162		      __get_str(dev),
163		      __entry->idx,
164		      __entry->sel,
165		      __entry->vl
166		      )
167);
168
169TRACE_EVENT(hfi1_sdma_user_free_queues,
170	    TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u16 subctxt),
171	    TP_ARGS(dd, ctxt, subctxt),
172	    TP_STRUCT__entry(DD_DEV_ENTRY(dd)
173			     __field(u16, ctxt)
174			     __field(u16, subctxt)
175			     ),
176	    TP_fast_assign(DD_DEV_ASSIGN(dd);
177			   __entry->ctxt = ctxt;
178			   __entry->subctxt = subctxt;
179			   ),
180	    TP_printk("[%s] SDMA [%u:%u] Freeing user SDMA queues",
181		      __get_str(dev),
182		      __entry->ctxt,
183		      __entry->subctxt
184		      )
185);
186
187TRACE_EVENT(hfi1_sdma_user_process_request,
188	    TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u16 subctxt,
189		     u16 comp_idx),
190	    TP_ARGS(dd, ctxt, subctxt, comp_idx),
191	    TP_STRUCT__entry(DD_DEV_ENTRY(dd)
192			     __field(u16, ctxt)
193			     __field(u16, subctxt)
194			     __field(u16, comp_idx)
195			     ),
196	    TP_fast_assign(DD_DEV_ASSIGN(dd);
197			   __entry->ctxt = ctxt;
198			   __entry->subctxt = subctxt;
199			   __entry->comp_idx = comp_idx;
200			   ),
201	    TP_printk("[%s] SDMA [%u:%u] Using req/comp entry: %u",
202		      __get_str(dev),
203		      __entry->ctxt,
204		      __entry->subctxt,
205		      __entry->comp_idx
206		      )
207);
208
209DECLARE_EVENT_CLASS(
210	hfi1_sdma_value_template,
211	TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u16 subctxt, u16 comp_idx,
212		 u32 value),
213	TP_ARGS(dd, ctxt, subctxt, comp_idx, value),
214	TP_STRUCT__entry(DD_DEV_ENTRY(dd)
215			 __field(u16, ctxt)
216			 __field(u16, subctxt)
217			 __field(u16, comp_idx)
218			 __field(u32, value)
219		),
220	TP_fast_assign(DD_DEV_ASSIGN(dd);
221		       __entry->ctxt = ctxt;
222		       __entry->subctxt = subctxt;
223		       __entry->comp_idx = comp_idx;
224		       __entry->value = value;
225		),
226	TP_printk("[%s] SDMA [%u:%u:%u] value: %u",
227		  __get_str(dev),
228		  __entry->ctxt,
229		  __entry->subctxt,
230		  __entry->comp_idx,
231		  __entry->value
232		)
233);
234
235DEFINE_EVENT(hfi1_sdma_value_template, hfi1_sdma_user_initial_tidoffset,
236	     TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u16 subctxt,
237		      u16 comp_idx, u32 tidoffset),
238	     TP_ARGS(dd, ctxt, subctxt, comp_idx, tidoffset));
239
240DEFINE_EVENT(hfi1_sdma_value_template, hfi1_sdma_user_data_length,
241	     TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u16 subctxt,
242		      u16 comp_idx, u32 data_len),
243	     TP_ARGS(dd, ctxt, subctxt, comp_idx, data_len));
244
245DEFINE_EVENT(hfi1_sdma_value_template, hfi1_sdma_user_compute_length,
246	     TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u16 subctxt,
247		      u16 comp_idx, u32 data_len),
248	     TP_ARGS(dd, ctxt, subctxt, comp_idx, data_len));
249
250TRACE_EVENT(hfi1_sdma_user_tid_info,
251	    TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u16 subctxt,
252		     u16 comp_idx, u32 tidoffset, u32 units, u8 shift),
253	    TP_ARGS(dd, ctxt, subctxt, comp_idx, tidoffset, units, shift),
254	    TP_STRUCT__entry(DD_DEV_ENTRY(dd)
255			     __field(u16, ctxt)
256			     __field(u16, subctxt)
257			     __field(u16, comp_idx)
258			     __field(u32, tidoffset)
259			     __field(u32, units)
260			     __field(u8, shift)
261			     ),
262	    TP_fast_assign(DD_DEV_ASSIGN(dd);
263			   __entry->ctxt = ctxt;
264			   __entry->subctxt = subctxt;
265			   __entry->comp_idx = comp_idx;
266			   __entry->tidoffset = tidoffset;
267			   __entry->units = units;
268			   __entry->shift = shift;
269			   ),
270	    TP_printk("[%s] SDMA [%u:%u:%u] TID offset %ubytes %uunits om %u",
271		      __get_str(dev),
272		      __entry->ctxt,
273		      __entry->subctxt,
274		      __entry->comp_idx,
275		      __entry->tidoffset,
276		      __entry->units,
277		      __entry->shift
278		      )
279);
280
281TRACE_EVENT(hfi1_sdma_request,
282	    TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u16 subctxt,
283		     unsigned long dim),
284	    TP_ARGS(dd, ctxt, subctxt, dim),
285	    TP_STRUCT__entry(DD_DEV_ENTRY(dd)
286			     __field(u16, ctxt)
287			     __field(u16, subctxt)
288			     __field(unsigned long, dim)
289			     ),
290	    TP_fast_assign(DD_DEV_ASSIGN(dd);
291			   __entry->ctxt = ctxt;
292			   __entry->subctxt = subctxt;
293			   __entry->dim = dim;
294			   ),
295	    TP_printk("[%s] SDMA from %u:%u (%lu)",
296		      __get_str(dev),
297		      __entry->ctxt,
298		      __entry->subctxt,
299		      __entry->dim
300		      )
301);
302
303DECLARE_EVENT_CLASS(hfi1_sdma_engine_class,
304		    TP_PROTO(struct sdma_engine *sde, u64 status),
305		    TP_ARGS(sde, status),
306		    TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
307		    __field(u64, status)
308		    __field(u8, idx)
309		    ),
310		    TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
311		    __entry->status = status;
312		    __entry->idx = sde->this_idx;
313		    ),
314		    TP_printk("[%s] SDE(%u) status %llx",
315			      __get_str(dev),
316			      __entry->idx,
317			      (unsigned long long)__entry->status
318			      )
319);
320
321DEFINE_EVENT(hfi1_sdma_engine_class, hfi1_sdma_engine_interrupt,
322	     TP_PROTO(struct sdma_engine *sde, u64 status),
323	     TP_ARGS(sde, status)
324);
325
326DEFINE_EVENT(hfi1_sdma_engine_class, hfi1_sdma_engine_progress,
327	     TP_PROTO(struct sdma_engine *sde, u64 status),
328	     TP_ARGS(sde, status)
329);
330
331DECLARE_EVENT_CLASS(hfi1_sdma_ahg_ad,
332		    TP_PROTO(struct sdma_engine *sde, int aidx),
333		    TP_ARGS(sde, aidx),
334		    TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
335		    __field(int, aidx)
336		    __field(u8, idx)
337		    ),
338		    TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
339		    __entry->idx = sde->this_idx;
340		    __entry->aidx = aidx;
341		    ),
342		    TP_printk("[%s] SDE(%u) aidx %d",
343			      __get_str(dev),
344			      __entry->idx,
345			      __entry->aidx
346			      )
347);
348
349DEFINE_EVENT(hfi1_sdma_ahg_ad, hfi1_ahg_allocate,
350	     TP_PROTO(struct sdma_engine *sde, int aidx),
351	     TP_ARGS(sde, aidx));
352
353DEFINE_EVENT(hfi1_sdma_ahg_ad, hfi1_ahg_deallocate,
354	     TP_PROTO(struct sdma_engine *sde, int aidx),
355	     TP_ARGS(sde, aidx));
356
357#ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
358TRACE_EVENT(hfi1_sdma_progress,
359	    TP_PROTO(struct sdma_engine *sde,
360		     u16 hwhead,
361		     u16 swhead,
362		     struct sdma_txreq *txp
363		     ),
364	    TP_ARGS(sde, hwhead, swhead, txp),
365	    TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
366	    __field(u64, sn)
367	    __field(u16, hwhead)
368	    __field(u16, swhead)
369	    __field(u16, txnext)
370	    __field(u16, tx_tail)
371	    __field(u16, tx_head)
372	    __field(u8, idx)
373	    ),
374	    TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
375	    __entry->hwhead = hwhead;
376	    __entry->swhead = swhead;
377	    __entry->tx_tail = sde->tx_tail;
378	    __entry->tx_head = sde->tx_head;
379	    __entry->txnext = txp ? txp->next_descq_idx : ~0;
380	    __entry->idx = sde->this_idx;
381	    __entry->sn = txp ? txp->sn : ~0;
382	    ),
383	    TP_printk(
384	    "[%s] SDE(%u) sn %llu hwhead %u swhead %u next_descq_idx %u tx_head %u tx_tail %u",
385	    __get_str(dev),
386	    __entry->idx,
387	    __entry->sn,
388	    __entry->hwhead,
389	    __entry->swhead,
390	    __entry->txnext,
391	    __entry->tx_head,
392	    __entry->tx_tail
393	    )
394);
395#else
396TRACE_EVENT(hfi1_sdma_progress,
397	    TP_PROTO(struct sdma_engine *sde,
398		     u16 hwhead, u16 swhead,
399		     struct sdma_txreq *txp
400		     ),
401	    TP_ARGS(sde, hwhead, swhead, txp),
402	    TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
403		    __field(u16, hwhead)
404		    __field(u16, swhead)
405		    __field(u16, txnext)
406		    __field(u16, tx_tail)
407		    __field(u16, tx_head)
408		    __field(u8, idx)
409		    ),
410	    TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
411		    __entry->hwhead = hwhead;
412		    __entry->swhead = swhead;
413		    __entry->tx_tail = sde->tx_tail;
414		    __entry->tx_head = sde->tx_head;
415		    __entry->txnext = txp ? txp->next_descq_idx : ~0;
416		    __entry->idx = sde->this_idx;
417		    ),
418	    TP_printk(
419		    "[%s] SDE(%u) hwhead %u swhead %u next_descq_idx %u tx_head %u tx_tail %u",
420		    __get_str(dev),
421		    __entry->idx,
422		    __entry->hwhead,
423		    __entry->swhead,
424		    __entry->txnext,
425		    __entry->tx_head,
426		    __entry->tx_tail
427	    )
428);
429#endif
430
431DECLARE_EVENT_CLASS(hfi1_sdma_sn,
432		    TP_PROTO(struct sdma_engine *sde, u64 sn),
433		    TP_ARGS(sde, sn),
434		    TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
435		    __field(u64, sn)
436		    __field(u8, idx)
437		    ),
438		    TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
439		    __entry->sn = sn;
440		    __entry->idx = sde->this_idx;
441		    ),
442		    TP_printk("[%s] SDE(%u) sn %llu",
443			      __get_str(dev),
444			      __entry->idx,
445			      __entry->sn
446			      )
447);
448
449DEFINE_EVENT(hfi1_sdma_sn, hfi1_sdma_out_sn,
450	     TP_PROTO(
451	     struct sdma_engine *sde,
452	     u64 sn
453	     ),
454	     TP_ARGS(sde, sn)
455);
456
457DEFINE_EVENT(hfi1_sdma_sn, hfi1_sdma_in_sn,
458	     TP_PROTO(struct sdma_engine *sde, u64 sn),
459	     TP_ARGS(sde, sn)
460);
461
462#define USDMA_HDR_FORMAT \
463	"[%s:%u:%u:%u] PBC=(0x%x 0x%x) LRH=(0x%x 0x%x) BTH=(0x%x 0x%x 0x%x) KDETH=(0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x) TIDVal=0x%x"
464
465TRACE_EVENT(hfi1_sdma_user_header,
466	    TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u8 subctxt, u16 req,
467		     struct hfi1_pkt_header *hdr, u32 tidval),
468	    TP_ARGS(dd, ctxt, subctxt, req, hdr, tidval),
469	    TP_STRUCT__entry(
470		    DD_DEV_ENTRY(dd)
471		    __field(u16, ctxt)
472		    __field(u8, subctxt)
473		    __field(u16, req)
474		    __field(u32, pbc0)
475		    __field(u32, pbc1)
476		    __field(u32, lrh0)
477		    __field(u32, lrh1)
478		    __field(u32, bth0)
479		    __field(u32, bth1)
480		    __field(u32, bth2)
481		    __field(u32, kdeth0)
482		    __field(u32, kdeth1)
483		    __field(u32, kdeth2)
484		    __field(u32, kdeth3)
485		    __field(u32, kdeth4)
486		    __field(u32, kdeth5)
487		    __field(u32, kdeth6)
488		    __field(u32, kdeth7)
489		    __field(u32, kdeth8)
490		    __field(u32, tidval)
491		    ),
492		    TP_fast_assign(
493		    __le32 *pbc = (__le32 *)hdr->pbc;
494		    __be32 *lrh = (__be32 *)hdr->lrh;
495		    __be32 *bth = (__be32 *)hdr->bth;
496		    __le32 *kdeth = (__le32 *)&hdr->kdeth;
497
498		    DD_DEV_ASSIGN(dd);
499		    __entry->ctxt = ctxt;
500		    __entry->subctxt = subctxt;
501		    __entry->req = req;
502		    __entry->pbc0 = le32_to_cpu(pbc[0]);
503		    __entry->pbc1 = le32_to_cpu(pbc[1]);
504		    __entry->lrh0 = be32_to_cpu(lrh[0]);
505		    __entry->lrh1 = be32_to_cpu(lrh[1]);
506		    __entry->bth0 = be32_to_cpu(bth[0]);
507		    __entry->bth1 = be32_to_cpu(bth[1]);
508		    __entry->bth2 = be32_to_cpu(bth[2]);
509		    __entry->kdeth0 = le32_to_cpu(kdeth[0]);
510		    __entry->kdeth1 = le32_to_cpu(kdeth[1]);
511		    __entry->kdeth2 = le32_to_cpu(kdeth[2]);
512		    __entry->kdeth3 = le32_to_cpu(kdeth[3]);
513		    __entry->kdeth4 = le32_to_cpu(kdeth[4]);
514		    __entry->kdeth5 = le32_to_cpu(kdeth[5]);
515		    __entry->kdeth6 = le32_to_cpu(kdeth[6]);
516		    __entry->kdeth7 = le32_to_cpu(kdeth[7]);
517		    __entry->kdeth8 = le32_to_cpu(kdeth[8]);
518		    __entry->tidval = tidval;
519	    ),
520	    TP_printk(USDMA_HDR_FORMAT,
521		      __get_str(dev),
522		      __entry->ctxt,
523		      __entry->subctxt,
524		      __entry->req,
525		      __entry->pbc1,
526		      __entry->pbc0,
527		      __entry->lrh0,
528		      __entry->lrh1,
529		      __entry->bth0,
530		      __entry->bth1,
531		      __entry->bth2,
532		      __entry->kdeth0,
533		      __entry->kdeth1,
534		      __entry->kdeth2,
535		      __entry->kdeth3,
536		      __entry->kdeth4,
537		      __entry->kdeth5,
538		      __entry->kdeth6,
539		      __entry->kdeth7,
540		      __entry->kdeth8,
541		      __entry->tidval
542	    )
543);
544
545#define SDMA_UREQ_FMT \
546	"[%s:%u:%u] ver/op=0x%x, iovcnt=%u, npkts=%u, frag=%u, idx=%u"
547TRACE_EVENT(hfi1_sdma_user_reqinfo,
548	    TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u8 subctxt, u16 *i),
549	    TP_ARGS(dd, ctxt, subctxt, i),
550	    TP_STRUCT__entry(
551		    DD_DEV_ENTRY(dd)
552		    __field(u16, ctxt)
553		    __field(u8, subctxt)
554		    __field(u8, ver_opcode)
555		    __field(u8, iovcnt)
556		    __field(u16, npkts)
557		    __field(u16, fragsize)
558		    __field(u16, comp_idx)
559	    ),
560	    TP_fast_assign(
561		    DD_DEV_ASSIGN(dd);
562		    __entry->ctxt = ctxt;
563		    __entry->subctxt = subctxt;
564		    __entry->ver_opcode = i[0] & 0xff;
565		    __entry->iovcnt = (i[0] >> 8) & 0xff;
566		    __entry->npkts = i[1];
567		    __entry->fragsize = i[2];
568		    __entry->comp_idx = i[3];
569	    ),
570	    TP_printk(SDMA_UREQ_FMT,
571		      __get_str(dev),
572		      __entry->ctxt,
573		      __entry->subctxt,
574		      __entry->ver_opcode,
575		      __entry->iovcnt,
576		      __entry->npkts,
577		      __entry->fragsize,
578		      __entry->comp_idx
579		      )
580);
581
582#define usdma_complete_name(st) { st, #st }
583#define show_usdma_complete_state(st)			\
584	__print_symbolic(st,				\
585			usdma_complete_name(FREE),	\
586			usdma_complete_name(QUEUED),	\
587			usdma_complete_name(COMPLETE), \
588			usdma_complete_name(ERROR))
589
590TRACE_EVENT(hfi1_sdma_user_completion,
591	    TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u8 subctxt, u16 idx,
592		     u8 state, int code),
593	    TP_ARGS(dd, ctxt, subctxt, idx, state, code),
594	    TP_STRUCT__entry(
595	    DD_DEV_ENTRY(dd)
596	    __field(u16, ctxt)
597	    __field(u8, subctxt)
598	    __field(u16, idx)
599	    __field(u8, state)
600	    __field(int, code)
601	    ),
602	    TP_fast_assign(
603	    DD_DEV_ASSIGN(dd);
604	    __entry->ctxt = ctxt;
605	    __entry->subctxt = subctxt;
606	    __entry->idx = idx;
607	    __entry->state = state;
608	    __entry->code = code;
609	    ),
610	    TP_printk("[%s:%u:%u:%u] SDMA completion state %s (%d)",
611		      __get_str(dev), __entry->ctxt, __entry->subctxt,
612		      __entry->idx, show_usdma_complete_state(__entry->state),
613		      __entry->code)
614);
615
616TRACE_EVENT(hfi1_usdma_defer,
617	    TP_PROTO(struct hfi1_user_sdma_pkt_q *pq,
618		     struct sdma_engine *sde,
619		     struct iowait *wait),
620	    TP_ARGS(pq, sde, wait),
621	    TP_STRUCT__entry(DD_DEV_ENTRY(pq->dd)
622			     __field(struct hfi1_user_sdma_pkt_q *, pq)
623			     __field(struct sdma_engine *, sde)
624			     __field(struct iowait *, wait)
625			     __field(int, engine)
626			     __field(int, empty)
627			     ),
628	     TP_fast_assign(DD_DEV_ASSIGN(pq->dd);
629			    __entry->pq = pq;
630			    __entry->sde = sde;
631			    __entry->wait = wait;
632			    __entry->engine = sde->this_idx;
633			    __entry->empty = list_empty(&__entry->wait->list);
634			    ),
635	     TP_printk("[%s] pq %llx sde %llx wait %llx engine %d empty %d",
636		       __get_str(dev),
637		       (unsigned long long)__entry->pq,
638		       (unsigned long long)__entry->sde,
639		       (unsigned long long)__entry->wait,
640		       __entry->engine,
641		       __entry->empty
642		)
643);
644
645TRACE_EVENT(hfi1_usdma_activate,
646	    TP_PROTO(struct hfi1_user_sdma_pkt_q *pq,
647		     struct iowait *wait,
648		     int reason),
649	    TP_ARGS(pq, wait, reason),
650	    TP_STRUCT__entry(DD_DEV_ENTRY(pq->dd)
651			     __field(struct hfi1_user_sdma_pkt_q *, pq)
652			     __field(struct iowait *, wait)
653			     __field(int, reason)
654			     ),
655	     TP_fast_assign(DD_DEV_ASSIGN(pq->dd);
656			    __entry->pq = pq;
657			    __entry->wait = wait;
658			    __entry->reason = reason;
659			    ),
660	     TP_printk("[%s] pq %llx wait %llx reason %d",
661		       __get_str(dev),
662		       (unsigned long long)__entry->pq,
663		       (unsigned long long)__entry->wait,
664		       __entry->reason
665		)
666);
667
668TRACE_EVENT(hfi1_usdma_we,
669	    TP_PROTO(struct hfi1_user_sdma_pkt_q *pq,
670		     int we_ret),
671	    TP_ARGS(pq, we_ret),
672	    TP_STRUCT__entry(DD_DEV_ENTRY(pq->dd)
673			     __field(struct hfi1_user_sdma_pkt_q *, pq)
674			     __field(int, state)
675			     __field(int, we_ret)
676			     ),
677	     TP_fast_assign(DD_DEV_ASSIGN(pq->dd);
678			    __entry->pq = pq;
679			    __entry->state = pq->state;
680			    __entry->we_ret = we_ret;
681			    ),
682	     TP_printk("[%s] pq %llx state %d we_ret %d",
683		       __get_str(dev),
684		       (unsigned long long)__entry->pq,
685		       __entry->state,
686		       __entry->we_ret
687		)
688);
689
690const char *print_u32_array(struct trace_seq *, u32 *, int);
691#define __print_u32_hex(arr, len) print_u32_array(p, arr, len)
692
693TRACE_EVENT(hfi1_sdma_user_header_ahg,
694	    TP_PROTO(struct hfi1_devdata *dd, u16 ctxt, u8 subctxt, u16 req,
695		     u8 sde, u8 ahgidx, u32 *ahg, int len, u32 tidval),
696	    TP_ARGS(dd, ctxt, subctxt, req, sde, ahgidx, ahg, len, tidval),
697	    TP_STRUCT__entry(
698	    DD_DEV_ENTRY(dd)
699	    __field(u16, ctxt)
700	    __field(u8, subctxt)
701	    __field(u16, req)
702	    __field(u8, sde)
703	    __field(u8, idx)
704	    __field(int, len)
705	    __field(u32, tidval)
706	    __array(u32, ahg, 10)
707	    ),
708	    TP_fast_assign(
709	    DD_DEV_ASSIGN(dd);
710	    __entry->ctxt = ctxt;
711	    __entry->subctxt = subctxt;
712	    __entry->req = req;
713	    __entry->sde = sde;
714	    __entry->idx = ahgidx;
715	    __entry->len = len;
716	    __entry->tidval = tidval;
717	    memcpy(__entry->ahg, ahg, len * sizeof(u32));
718	    ),
719	    TP_printk("[%s:%u:%u:%u] (SDE%u/AHG%u) ahg[0-%d]=(%s) TIDVal=0x%x",
720		      __get_str(dev),
721		      __entry->ctxt,
722		      __entry->subctxt,
723		      __entry->req,
724		      __entry->sde,
725		      __entry->idx,
726		      __entry->len - 1,
727		      __print_u32_hex(__entry->ahg, __entry->len),
728		      __entry->tidval
729		      )
730);
731
732TRACE_EVENT(hfi1_sdma_state,
733	    TP_PROTO(struct sdma_engine *sde,
734		     const char *cstate,
735		     const char *nstate
736		     ),
737	    TP_ARGS(sde, cstate, nstate),
738	    TP_STRUCT__entry(DD_DEV_ENTRY(sde->dd)
739		__string(curstate, cstate)
740		__string(newstate, nstate)
741	    ),
742	    TP_fast_assign(DD_DEV_ASSIGN(sde->dd);
743		__assign_str(curstate, cstate);
744		__assign_str(newstate, nstate);
745	    ),
746	    TP_printk("[%s] current state %s new state %s",
747		      __get_str(dev),
748		      __get_str(curstate),
749		      __get_str(newstate)
750	    )
751);
752
753#define BCT_FORMAT \
754	"shared_limit %x vls 0-7 [%x,%x][%x,%x][%x,%x][%x,%x][%x,%x][%x,%x][%x,%x][%x,%x] 15 [%x,%x]"
755
756#define BCT(field) \
757	be16_to_cpu( \
758	((struct buffer_control *)__get_dynamic_array(bct))->field \
759	)
760
761DECLARE_EVENT_CLASS(hfi1_bct_template,
762		    TP_PROTO(struct hfi1_devdata *dd,
763			     struct buffer_control *bc),
764		    TP_ARGS(dd, bc),
765		    TP_STRUCT__entry(DD_DEV_ENTRY(dd)
766		    __dynamic_array(u8, bct, sizeof(*bc))
767		    ),
768		    TP_fast_assign(DD_DEV_ASSIGN(dd);
769				   memcpy(__get_dynamic_array(bct), bc,
770					  sizeof(*bc));
771		    ),
772		    TP_printk(BCT_FORMAT,
773			      BCT(overall_shared_limit),
774
775			      BCT(vl[0].dedicated),
776			      BCT(vl[0].shared),
777
778			      BCT(vl[1].dedicated),
779			      BCT(vl[1].shared),
780
781			      BCT(vl[2].dedicated),
782			      BCT(vl[2].shared),
783
784			      BCT(vl[3].dedicated),
785			      BCT(vl[3].shared),
786
787			      BCT(vl[4].dedicated),
788			      BCT(vl[4].shared),
789
790			      BCT(vl[5].dedicated),
791			      BCT(vl[5].shared),
792
793			      BCT(vl[6].dedicated),
794			      BCT(vl[6].shared),
795
796			      BCT(vl[7].dedicated),
797			      BCT(vl[7].shared),
798
799			      BCT(vl[15].dedicated),
800			      BCT(vl[15].shared)
801		    )
802);
803
804DEFINE_EVENT(hfi1_bct_template, bct_set,
805	     TP_PROTO(struct hfi1_devdata *dd, struct buffer_control *bc),
806	     TP_ARGS(dd, bc));
807
808DEFINE_EVENT(hfi1_bct_template, bct_get,
809	     TP_PROTO(struct hfi1_devdata *dd, struct buffer_control *bc),
810	     TP_ARGS(dd, bc));
811
812TRACE_EVENT(
813	hfi1_qp_send_completion,
814	TP_PROTO(struct rvt_qp *qp, struct rvt_swqe *wqe, u32 idx),
815	TP_ARGS(qp, wqe, idx),
816	TP_STRUCT__entry(
817		DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device))
818		__field(struct rvt_swqe *, wqe)
819		__field(u64, wr_id)
820		__field(u32, qpn)
821		__field(u32, qpt)
822		__field(u32, length)
823		__field(u32, idx)
824		__field(u32, ssn)
825		__field(enum ib_wr_opcode, opcode)
826		__field(int, send_flags)
827	),
828	TP_fast_assign(
829		DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device));
830		__entry->wqe = wqe;
831		__entry->wr_id = wqe->wr.wr_id;
832		__entry->qpn = qp->ibqp.qp_num;
833		__entry->qpt = qp->ibqp.qp_type;
834		__entry->length = wqe->length;
835		__entry->idx = idx;
836		__entry->ssn = wqe->ssn;
837		__entry->opcode = wqe->wr.opcode;
838		__entry->send_flags = wqe->wr.send_flags;
839	),
840	TP_printk(
841		"[%s] qpn 0x%x qpt %u wqe %p idx %u wr_id %llx length %u ssn %u opcode %x send_flags %x",
842		__get_str(dev),
843		__entry->qpn,
844		__entry->qpt,
845		__entry->wqe,
846		__entry->idx,
847		__entry->wr_id,
848		__entry->length,
849		__entry->ssn,
850		__entry->opcode,
851		__entry->send_flags
852	)
853);
854
855DECLARE_EVENT_CLASS(
856	hfi1_do_send_template,
857	TP_PROTO(struct rvt_qp *qp, bool flag),
858	TP_ARGS(qp, flag),
859	TP_STRUCT__entry(
860		DD_DEV_ENTRY(dd_from_ibdev(qp->ibqp.device))
861		__field(u32, qpn)
862		__field(bool, flag)
863	),
864	TP_fast_assign(
865		DD_DEV_ASSIGN(dd_from_ibdev(qp->ibqp.device));
866		__entry->qpn = qp->ibqp.qp_num;
867		__entry->flag = flag;
868	),
869	TP_printk(
870		"[%s] qpn %x flag %d",
871		__get_str(dev),
872		__entry->qpn,
873		__entry->flag
874	)
875);
876
877DEFINE_EVENT(
878	hfi1_do_send_template, hfi1_rc_do_send,
879	TP_PROTO(struct rvt_qp *qp, bool flag),
880	TP_ARGS(qp, flag)
881);
882
883DEFINE_EVENT(/* event */
884	hfi1_do_send_template, hfi1_rc_do_tid_send,
885	TP_PROTO(struct rvt_qp *qp, bool flag),
886	TP_ARGS(qp, flag)
887);
888
889DEFINE_EVENT(
890	hfi1_do_send_template, hfi1_rc_expired_time_slice,
891	TP_PROTO(struct rvt_qp *qp, bool flag),
892	TP_ARGS(qp, flag)
893);
894
895DECLARE_EVENT_CLASS(/* AIP  */
896	hfi1_ipoib_txq_template,
897	TP_PROTO(struct hfi1_ipoib_txq *txq),
898	TP_ARGS(txq),
899	TP_STRUCT__entry(/* entry */
900		DD_DEV_ENTRY(txq->priv->dd)
901		__field(struct hfi1_ipoib_txq *, txq)
902		__field(struct sdma_engine *, sde)
903		__field(ulong, head)
904		__field(ulong, tail)
905		__field(uint, used)
906		__field(uint, flow)
907		__field(int, stops)
908		__field(int, no_desc)
909		__field(u8, idx)
910		__field(u8, stopped)
911	),
912	TP_fast_assign(/* assign */
913		DD_DEV_ASSIGN(txq->priv->dd);
914		__entry->txq = txq;
915		__entry->sde = txq->sde;
916		__entry->head = txq->tx_ring.head;
917		__entry->tail = txq->tx_ring.tail;
918		__entry->idx = txq->q_idx;
919		__entry->used =
920			txq->tx_ring.sent_txreqs -
921			txq->tx_ring.complete_txreqs;
922		__entry->flow = txq->flow.as_int;
923		__entry->stops = atomic_read(&txq->tx_ring.stops);
924		__entry->no_desc = atomic_read(&txq->tx_ring.no_desc);
925		__entry->stopped =
926		 __netif_subqueue_stopped(txq->priv->netdev, txq->q_idx);
927	),
928	TP_printk(/* print  */
929		"[%s] txq %llx idx %u sde %llx:%u cpu %d head %lx tail %lx flow %x used %u stops %d no_desc %d stopped %u",
930		__get_str(dev),
931		(unsigned long long)__entry->txq,
932		__entry->idx,
933		(unsigned long long)__entry->sde,
934		__entry->sde ? __entry->sde->this_idx : 0,
935		__entry->sde ? __entry->sde->cpu : 0,
936		__entry->head,
937		__entry->tail,
938		__entry->flow,
939		__entry->used,
940		__entry->stops,
941		__entry->no_desc,
942		__entry->stopped
943	)
944);
945
946DEFINE_EVENT(/* queue stop */
947	hfi1_ipoib_txq_template, hfi1_txq_stop,
948	TP_PROTO(struct hfi1_ipoib_txq *txq),
949	TP_ARGS(txq)
950);
951
952DEFINE_EVENT(/* queue wake */
953	hfi1_ipoib_txq_template, hfi1_txq_wake,
954	TP_PROTO(struct hfi1_ipoib_txq *txq),
955	TP_ARGS(txq)
956);
957
958DEFINE_EVENT(/* flow flush */
959	hfi1_ipoib_txq_template, hfi1_flow_flush,
960	TP_PROTO(struct hfi1_ipoib_txq *txq),
961	TP_ARGS(txq)
962);
963
964DEFINE_EVENT(/* flow switch */
965	hfi1_ipoib_txq_template, hfi1_flow_switch,
966	TP_PROTO(struct hfi1_ipoib_txq *txq),
967	TP_ARGS(txq)
968);
969
970DEFINE_EVENT(/* wakeup */
971	hfi1_ipoib_txq_template, hfi1_txq_wakeup,
972	TP_PROTO(struct hfi1_ipoib_txq *txq),
973	TP_ARGS(txq)
974);
975
976DEFINE_EVENT(/* full */
977	hfi1_ipoib_txq_template, hfi1_txq_full,
978	TP_PROTO(struct hfi1_ipoib_txq *txq),
979	TP_ARGS(txq)
980);
981
982DEFINE_EVENT(/* queued */
983	hfi1_ipoib_txq_template, hfi1_txq_queued,
984	TP_PROTO(struct hfi1_ipoib_txq *txq),
985	TP_ARGS(txq)
986);
987
988DEFINE_EVENT(/* xmit_stopped */
989	hfi1_ipoib_txq_template, hfi1_txq_xmit_stopped,
990	TP_PROTO(struct hfi1_ipoib_txq *txq),
991	TP_ARGS(txq)
992);
993
994DEFINE_EVENT(/* xmit_unstopped */
995	hfi1_ipoib_txq_template, hfi1_txq_xmit_unstopped,
996	TP_PROTO(struct hfi1_ipoib_txq *txq),
997	TP_ARGS(txq)
998);
999
1000DECLARE_EVENT_CLASS(/* AIP  */
1001	hfi1_ipoib_tx_template,
1002	TP_PROTO(struct ipoib_txreq *tx, u32 idx),
1003	TP_ARGS(tx, idx),
1004	TP_STRUCT__entry(/* entry */
1005		DD_DEV_ENTRY(tx->txq->priv->dd)
1006		__field(struct ipoib_txreq *, tx)
1007		__field(struct hfi1_ipoib_txq *, txq)
1008		__field(struct sk_buff *, skb)
1009		__field(ulong, idx)
1010	),
1011	TP_fast_assign(/* assign */
1012		DD_DEV_ASSIGN(tx->txq->priv->dd);
1013		__entry->tx = tx;
1014		__entry->skb = tx->skb;
1015		__entry->txq = tx->txq;
1016		__entry->idx = idx;
1017	),
1018	TP_printk(/* print  */
1019		"[%s] tx %llx txq %llx,%u skb %llx idx %lu",
1020		__get_str(dev),
1021		(unsigned long long)__entry->tx,
1022		(unsigned long long)__entry->txq,
1023		__entry->txq ? __entry->txq->q_idx : 0,
1024		(unsigned long long)__entry->skb,
1025		__entry->idx
1026	)
1027);
1028
1029DEFINE_EVENT(/* produce */
1030	hfi1_ipoib_tx_template, hfi1_tx_produce,
1031	TP_PROTO(struct ipoib_txreq *tx, u32 idx),
1032	TP_ARGS(tx, idx)
1033);
1034
1035DEFINE_EVENT(/* consume */
1036	hfi1_ipoib_tx_template, hfi1_tx_consume,
1037	TP_PROTO(struct ipoib_txreq *tx, u32 idx),
1038	TP_ARGS(tx, idx)
1039);
1040
1041DEFINE_EVENT(/* alloc_tx */
1042	hfi1_ipoib_txq_template, hfi1_txq_alloc_tx,
1043	TP_PROTO(struct hfi1_ipoib_txq *txq),
1044	TP_ARGS(txq)
1045);
1046
1047DEFINE_EVENT(/* poll */
1048	hfi1_ipoib_txq_template, hfi1_txq_poll,
1049	TP_PROTO(struct hfi1_ipoib_txq *txq),
1050	TP_ARGS(txq)
1051);
1052
1053DEFINE_EVENT(/* complete */
1054	hfi1_ipoib_txq_template, hfi1_txq_complete,
1055	TP_PROTO(struct hfi1_ipoib_txq *txq),
1056	TP_ARGS(txq)
1057);
1058
1059#endif /* __HFI1_TRACE_TX_H */
1060
1061#undef TRACE_INCLUDE_PATH
1062#undef TRACE_INCLUDE_FILE
1063#define TRACE_INCLUDE_PATH .
1064#define TRACE_INCLUDE_FILE trace_tx
1065#include <trace/define_trace.h>
1066