Lines Matching defs:ptl
335 ptl_info(packet->ptl, "packet error injection: dropping ACK packet %p\n",
347 ptl_info(packet->ptl, "packet error injection: dropping NAK packet %p\n",
359 ptl_info(packet->ptl,
387 static int ssh_ptl_write_buf(struct ssh_ptl *ptl, struct ssh_packet *packet,
395 ptl_info(packet->ptl,
402 return serdev_device_write_buf(ptl->serdev, buf, count);
419 ptl_info(packet->ptl,
431 static void ssh_ptl_rx_inject_invalid_syn(struct ssh_ptl *ptl,
448 static void ssh_ptl_rx_inject_invalid_data(struct ssh_ptl *ptl,
486 static inline int ssh_ptl_write_buf(struct ssh_ptl *ptl,
491 return serdev_device_write_buf(ptl->serdev, buf, count);
498 static inline void ssh_ptl_rx_inject_invalid_syn(struct ssh_ptl *ptl,
503 static inline void ssh_ptl_rx_inject_invalid_data(struct ssh_ptl *ptl,
516 ptl_dbg_cond(p->ptl, "ptl: releasing packet %p\n", p);
577 packet->ptl = NULL;
663 static void ssh_ptl_timeout_reaper_mod(struct ssh_ptl *ptl, ktime_t now,
669 spin_lock(&ptl->rtx_timeout.lock);
672 if (ktime_before(aexp, ptl->rtx_timeout.expires)) {
673 ptl->rtx_timeout.expires = expires;
674 mod_delayed_work(system_wq, &ptl->rtx_timeout.reaper, delta);
677 spin_unlock(&ptl->rtx_timeout.lock);
686 lockdep_assert_held(&p->ptl->queue.lock);
702 lockdep_assert_held(&p->ptl->queue.lock);
718 list_for_each(head, &p->ptl->queue.head) {
725 list_for_each_prev(head, &p->ptl->queue.head) {
741 struct ssh_ptl *ptl = packet->ptl;
744 lockdep_assert_held(&ptl->queue.lock);
746 if (test_bit(SSH_PTL_SF_SHUTDOWN_BIT, &ptl->state))
767 spin_lock(&packet->ptl->queue.lock);
769 spin_unlock(&packet->ptl->queue.lock);
776 struct ssh_ptl *ptl = packet->ptl;
778 spin_lock(&ptl->queue.lock);
781 spin_unlock(&ptl->queue.lock);
787 spin_unlock(&ptl->queue.lock);
793 struct ssh_ptl *ptl = p->ptl;
795 const ktime_t timeout = ptl->rtx_timeout.timeout;
804 spin_lock(&ptl->pending.lock);
808 spin_unlock(&ptl->pending.lock);
821 atomic_inc(&ptl->pending.count);
822 list_add_tail(&ssh_packet_get(p)->pending_node, &ptl->pending.head);
825 spin_unlock(&ptl->pending.lock);
828 ssh_ptl_timeout_reaper_mod(ptl, timestamp, timestamp + timeout);
833 struct ssh_ptl *ptl = packet->ptl;
835 spin_lock(&ptl->pending.lock);
838 spin_unlock(&ptl->pending.lock);
843 atomic_dec(&ptl->pending.count);
845 spin_unlock(&ptl->pending.lock);
853 struct ssh_ptl *ptl = READ_ONCE(p->ptl);
856 ptl_dbg_cond(ptl, "ptl: completing packet %p (status: %d)\n", p, status);
885 struct ssh_ptl *ptl = packet->ptl;
888 return !atomic_read(&ptl->pending.count);
899 return atomic_read(&ptl->pending.count) < SSH_PTL_MAX_PENDING;
902 static struct ssh_packet *ssh_ptl_tx_pop(struct ssh_ptl *ptl)
907 spin_lock(&ptl->queue.lock);
908 list_for_each_entry_safe(p, n, &ptl->queue.head, queue_node) {
950 spin_unlock(&ptl->queue.lock);
955 static struct ssh_packet *ssh_ptl_tx_next(struct ssh_ptl *ptl)
959 p = ssh_ptl_tx_pop(ptl);
964 ptl_dbg(ptl, "ptl: transmitting sequenced packet %p\n", p);
967 ptl_dbg(ptl, "ptl: transmitting non-sequenced packet %p\n", p);
975 struct ssh_ptl *ptl = packet->ptl;
977 ptl_dbg(ptl, "ptl: successfully transmitted packet %p\n", packet);
995 wake_up_all(&ptl->tx.packet_wq);
1006 ptl_err(packet->ptl, "ptl: transmission error: %d\n", status);
1007 ptl_dbg(packet->ptl, "ptl: failed to transmit packet: %p\n", packet);
1015 wake_up_all(&packet->ptl->tx.packet_wq);
1018 static long ssh_ptl_tx_wait_packet(struct ssh_ptl *ptl)
1022 status = wait_for_completion_interruptible(&ptl->tx.thread_cplt_pkt);
1023 reinit_completion(&ptl->tx.thread_cplt_pkt);
1034 static long ssh_ptl_tx_wait_transfer(struct ssh_ptl *ptl, long timeout)
1038 status = wait_for_completion_interruptible_timeout(&ptl->tx.thread_cplt_tx,
1040 reinit_completion(&ptl->tx.thread_cplt_tx);
1051 static int ssh_ptl_tx_packet(struct ssh_ptl *ptl, struct ssh_packet *packet)
1067 ptl_dbg(ptl, "tx: sending data (length: %zu)\n", packet->data.len);
1078 status = ssh_ptl_write_buf(ptl, packet, buf, len);
1087 timeout = ssh_ptl_tx_wait_transfer(ptl, timeout);
1088 if (kthread_should_stop() || !atomic_read(&ptl->tx.running))
1101 struct ssh_ptl *ptl = data;
1103 while (!kthread_should_stop() && atomic_read(&ptl->tx.running)) {
1108 packet = ssh_ptl_tx_next(ptl);
1112 ssh_ptl_tx_wait_packet(ptl);
1117 status = ssh_ptl_tx_packet(ptl, packet);
1132 * @ptl: The packet transport layer.
1138 static void ssh_ptl_tx_wakeup_packet(struct ssh_ptl *ptl)
1140 if (test_bit(SSH_PTL_SF_SHUTDOWN_BIT, &ptl->state))
1143 complete(&ptl->tx.thread_cplt_pkt);
1148 * @ptl: The packet transport layer.
1152 int ssh_ptl_tx_start(struct ssh_ptl *ptl)
1154 atomic_set_release(&ptl->tx.running, 1);
1156 ptl->tx.thread = kthread_run(ssh_ptl_tx_threadfn, ptl, "ssam_serial_hub-tx");
1157 if (IS_ERR(ptl->tx.thread))
1158 return PTR_ERR(ptl->tx.thread);
1165 * @ptl: The packet transport layer.
1169 int ssh_ptl_tx_stop(struct ssh_ptl *ptl)
1173 if (!IS_ERR_OR_NULL(ptl->tx.thread)) {
1175 atomic_set_release(&ptl->tx.running, 0);
1182 complete(&ptl->tx.thread_cplt_pkt);
1183 complete(&ptl->tx.thread_cplt_tx);
1186 status = kthread_stop(ptl->tx.thread);
1187 ptl->tx.thread = NULL;
1193 static struct ssh_packet *ssh_ptl_ack_pop(struct ssh_ptl *ptl, u8 seq_id)
1198 spin_lock(&ptl->pending.lock);
1199 list_for_each_entry_safe(p, n, &ptl->pending.head, pending_node) {
1226 atomic_dec(&ptl->pending.count);
1232 spin_unlock(&ptl->pending.lock);
1239 wait_event(packet->ptl->tx.packet_wq,
1244 static void ssh_ptl_acknowledge(struct ssh_ptl *ptl, u8 seq)
1248 p = ssh_ptl_ack_pop(ptl, seq);
1255 ptl_warn(ptl, "ptl: received ACK for non-pending packet\n");
1266 ptl_dbg(ptl, "ptl: received ACK for packet %p\n", p);
1286 ptl_err(ptl, "ptl: received ACK before packet had been fully transmitted\n");
1295 if (atomic_read(&ptl->pending.count) < SSH_PTL_MAX_PENDING)
1296 ssh_ptl_tx_wakeup_packet(ptl);
1301 * @ptl: The packet transport layer to submit the packet to.
1312 int ssh_ptl_submit(struct ssh_ptl *ptl, struct ssh_packet *p)
1328 * The ptl reference only gets set on or before the first submission.
1331 * Note that ptl may already be set from upper-layer request
1334 ptl_old = READ_ONCE(p->ptl);
1336 WRITE_ONCE(p->ptl, ptl);
1337 else if (WARN_ON(ptl_old != ptl))
1345 (atomic_read(&ptl->pending.count) < SSH_PTL_MAX_PENDING))
1346 ssh_ptl_tx_wakeup_packet(ptl);
1368 lockdep_assert_held(&packet->ptl->pending.lock);
1372 spin_lock(&packet->ptl->queue.lock);
1377 spin_unlock(&packet->ptl->queue.lock);
1388 spin_unlock(&packet->ptl->queue.lock);
1394 spin_unlock(&packet->ptl->queue.lock);
1398 static void ssh_ptl_resubmit_pending(struct ssh_ptl *ptl)
1414 spin_lock(&ptl->pending.lock);
1417 list_for_each_entry(p, &ptl->pending.head, pending_node) {
1427 spin_unlock(&ptl->pending.lock);
1430 ssh_ptl_tx_wakeup_packet(ptl);
1470 * In case the packet has never been submitted, packet->ptl is NULL. If
1471 * the packet is currently being submitted, packet->ptl may be NULL or
1473 * the memory barrier, we have guaranteed that, if packet->ptl is NULL,
1474 * the packet will never be added to the queue. If packet->ptl is
1478 if (READ_ONCE(p->ptl)) {
1481 if (atomic_read(&p->ptl->pending.count) < SSH_PTL_MAX_PENDING)
1482 ssh_ptl_tx_wakeup_packet(p->ptl);
1492 lockdep_assert_held(&p->ptl->pending.lock);
1502 struct ssh_ptl *ptl = to_ssh_ptl(work, rtx_timeout.reaper.work);
1506 ktime_t timeout = ptl->rtx_timeout.timeout;
1511 trace_ssam_ptl_timeout_reap(atomic_read(&ptl->pending.count));
1517 spin_lock(&ptl->rtx_timeout.lock);
1518 ptl->rtx_timeout.expires = KTIME_MAX;
1519 spin_unlock(&ptl->rtx_timeout.lock);
1521 spin_lock(&ptl->pending.lock);
1523 list_for_each_entry_safe(p, n, &ptl->pending.head, pending_node) {
1569 atomic_dec(&ptl->pending.count);
1573 spin_unlock(&ptl->pending.lock);
1593 ssh_ptl_timeout_reaper_mod(ptl, now, next);
1596 ssh_ptl_tx_wakeup_packet(ptl);
1599 static bool ssh_ptl_rx_retransmit_check(struct ssh_ptl *ptl, const struct ssh_frame *frame)
1623 for (i = 0; i < ARRAY_SIZE(ptl->rx.blocked.seqs); i++) {
1624 if (likely(ptl->rx.blocked.seqs[i] != frame->seq))
1627 ptl_dbg(ptl, "ptl: ignoring repeated data packet\n");
1632 ptl->rx.blocked.seqs[ptl->rx.blocked.offset] = frame->seq;
1633 ptl->rx.blocked.offset = (ptl->rx.blocked.offset + 1)
1634 % ARRAY_SIZE(ptl->rx.blocked.seqs);
1639 static void ssh_ptl_rx_dataframe(struct ssh_ptl *ptl,
1643 if (ssh_ptl_rx_retransmit_check(ptl, frame))
1646 ptl->ops.data_received(ptl, payload);
1649 static void ssh_ptl_send_ack(struct ssh_ptl *ptl, u8 seq)
1658 ptl_err(ptl, "ptl: failed to allocate ACK packet\n");
1669 ssh_ptl_submit(ptl, packet);
1673 static void ssh_ptl_send_nak(struct ssh_ptl *ptl)
1682 ptl_err(ptl, "ptl: failed to allocate NAK packet\n");
1693 ssh_ptl_submit(ptl, packet);
1697 static size_t ssh_ptl_rx_eval(struct ssh_ptl *ptl, struct ssam_span *source)
1706 ssh_ptl_rx_inject_invalid_syn(ptl, source);
1729 ptl_warn(ptl, "rx: parser: invalid start of frame, skipping\n");
1749 ssh_ptl_send_nak(ptl);
1756 ssh_ptl_rx_inject_invalid_data(ptl, &aligned);
1759 status = sshp_parse_frame(&ptl->serdev->dev, &aligned, &frame, &payload,
1770 ssh_ptl_acknowledge(ptl, frame->seq);
1774 ssh_ptl_resubmit_pending(ptl);
1778 ssh_ptl_send_ack(ptl, frame->seq);
1782 ssh_ptl_rx_dataframe(ptl, frame, &payload);
1786 ptl_warn(ptl, "ptl: received frame with unknown type %#04x\n",
1796 struct ssh_ptl *ptl = data;
1803 wait_event_interruptible(ptl->rx.wq,
1804 !kfifo_is_empty(&ptl->rx.fifo) ||
1810 n = sshp_buf_read_from_fifo(&ptl->rx.buf, &ptl->rx.fifo);
1812 ptl_dbg(ptl, "rx: received data (size: %zu)\n", n);
1814 ptl->rx.buf.ptr + ptl->rx.buf.len - n,
1818 while (offs < ptl->rx.buf.len) {
1819 sshp_buf_span_from(&ptl->rx.buf, offs, &span);
1820 n = ssh_ptl_rx_eval(ptl, &span);
1828 sshp_buf_drop(&ptl->rx.buf, offs);
1834 static void ssh_ptl_rx_wakeup(struct ssh_ptl *ptl)
1836 wake_up(&ptl->rx.wq);
1841 * @ptl: The packet transport layer.
1845 int ssh_ptl_rx_start(struct ssh_ptl *ptl)
1847 if (ptl->rx.thread)
1850 ptl->rx.thread = kthread_run(ssh_ptl_rx_threadfn, ptl,
1852 if (IS_ERR(ptl->rx.thread))
1853 return PTR_ERR(ptl->rx.thread);
1860 * @ptl: The packet transport layer.
1864 int ssh_ptl_rx_stop(struct ssh_ptl *ptl)
1868 if (ptl->rx.thread) {
1869 status = kthread_stop(ptl->rx.thread);
1870 ptl->rx.thread = NULL;
1879 * @ptl: The packet transport layer.
1890 ssize_t ssh_ptl_rx_rcvbuf(struct ssh_ptl *ptl, const u8 *buf, size_t n)
1894 if (test_bit(SSH_PTL_SF_SHUTDOWN_BIT, &ptl->state))
1897 used = kfifo_in(&ptl->rx.fifo, buf, n);
1899 ssh_ptl_rx_wakeup(ptl);
1906 * @ptl: The packet transport layer.
1917 void ssh_ptl_shutdown(struct ssh_ptl *ptl)
1925 set_bit(SSH_PTL_SF_SHUTDOWN_BIT, &ptl->state);
1936 status = ssh_ptl_rx_stop(ptl);
1938 ptl_err(ptl, "ptl: failed to stop receiver thread\n");
1940 status = ssh_ptl_tx_stop(ptl);
1942 ptl_err(ptl, "ptl: failed to stop transmitter thread\n");
1944 cancel_delayed_work_sync(&ptl->rtx_timeout.reaper);
1967 spin_lock(&ptl->queue.lock);
1968 list_for_each_entry_safe(p, n, &ptl->queue.head, queue_node) {
1976 spin_unlock(&ptl->queue.lock);
1979 spin_lock(&ptl->pending.lock);
1980 list_for_each_entry_safe(p, n, &ptl->pending.head, pending_node) {
1988 atomic_set(&ptl->pending.count, 0);
1989 spin_unlock(&ptl->pending.lock);
2015 * @ptl: The packet transport layer to initialize.
2026 int ssh_ptl_init(struct ssh_ptl *ptl, struct serdev_device *serdev,
2031 ptl->serdev = serdev;
2032 ptl->state = 0;
2034 spin_lock_init(&ptl->queue.lock);
2035 INIT_LIST_HEAD(&ptl->queue.head);
2037 spin_lock_init(&ptl->pending.lock);
2038 INIT_LIST_HEAD(&ptl->pending.head);
2039 atomic_set_release(&ptl->pending.count, 0);
2041 ptl->tx.thread = NULL;
2042 atomic_set(&ptl->tx.running, 0);
2043 init_completion(&ptl->tx.thread_cplt_pkt);
2044 init_completion(&ptl->tx.thread_cplt_tx);
2045 init_waitqueue_head(&ptl->tx.packet_wq);
2047 ptl->rx.thread = NULL;
2048 init_waitqueue_head(&ptl->rx.wq);
2050 spin_lock_init(&ptl->rtx_timeout.lock);
2051 ptl->rtx_timeout.timeout = SSH_PTL_PACKET_TIMEOUT;
2052 ptl->rtx_timeout.expires = KTIME_MAX;
2053 INIT_DELAYED_WORK(&ptl->rtx_timeout.reaper, ssh_ptl_timeout_reap);
2055 ptl->ops = *ops;
2058 for (i = 0; i < ARRAY_SIZE(ptl->rx.blocked.seqs); i++)
2059 ptl->rx.blocked.seqs[i] = U16_MAX;
2060 ptl->rx.blocked.offset = 0;
2062 status = kfifo_alloc(&ptl->rx.fifo, SSH_PTL_RX_FIFO_LEN, GFP_KERNEL);
2066 status = sshp_buf_alloc(&ptl->rx.buf, SSH_PTL_RX_BUF_LEN, GFP_KERNEL);
2068 kfifo_free(&ptl->rx.fifo);
2075 * @ptl: The packet transport layer to deinitialize.
2082 void ssh_ptl_destroy(struct ssh_ptl *ptl)
2084 kfifo_free(&ptl->rx.fifo);
2085 sshp_buf_free(&ptl->rx.buf);