154359Sroberto// SPDX-License-Identifier: GPL-2.0-only 254359Sroberto/* 354359Sroberto * Copyright (c) 2021, MediaTek Inc. 454359Sroberto * Copyright (c) 2021-2022, Intel Corporation. 554359Sroberto * 6280849Scy * Authors: 7280849Scy * Amir Hanania <amir.hanania@intel.com> 854359Sroberto * Haijun Liu <haijun.liu@mediatek.com> 954359Sroberto * Eliot Lee <eliot.lee@intel.com> 1054359Sroberto * Moises Veleta <moises.veleta@intel.com> 1154359Sroberto * Ricardo Martinez <ricardo.martinez@linux.intel.com> 1254359Sroberto * 13280849Scy * Contributors: 14280849Scy * Chiranjeevi Rapolu <chiranjeevi.rapolu@intel.com> 15132451Sroberto * Sreehari Kancharla <sreehari.kancharla@intel.com> 1654359Sroberto */ 1754359Sroberto 1854359Sroberto#include <linux/atomic.h> 1954359Sroberto#include <linux/bitfield.h> 2054359Sroberto#include <linux/delay.h> 2154359Sroberto#include <linux/device.h> 2254359Sroberto#include <linux/dma-direction.h> 2354359Sroberto#include <linux/dma-mapping.h> 2454359Sroberto#include <linux/err.h> 2554359Sroberto#include <linux/gfp.h> 2654359Sroberto#include <linux/kernel.h> 2754359Sroberto#include <linux/kthread.h> 2854359Sroberto#include <linux/list.h> 2954359Sroberto#include <linux/minmax.h> 3054359Sroberto#include <linux/netdevice.h> 3154359Sroberto#include <linux/pm_runtime.h> 32280849Scy#include <linux/sched.h> 3354359Sroberto#include <linux/spinlock.h> 3454359Sroberto#include <linux/skbuff.h> 3554359Sroberto#include <linux/types.h> 3654359Sroberto#include <linux/wait.h> 37280849Scy#include <linux/workqueue.h> 3854359Sroberto 3954359Sroberto#include "t7xx_dpmaif.h" 4054359Sroberto#include "t7xx_hif_dpmaif.h" 4154359Sroberto#include "t7xx_hif_dpmaif_tx.h" 4254359Sroberto#include "t7xx_pci.h" 43280849Scy 4454359Sroberto#define DPMAIF_SKB_TX_BURST_CNT 5 4554359Sroberto#define DPMAIF_DRB_LIST_LEN 6144 4654359Sroberto 4754359Sroberto/* DRB dtype */ 4854359Sroberto#define DES_DTYP_PD 0 49280849Scy#define DES_DTYP_MSG 1 50280849Scy 5154359Srobertostatic unsigned int t7xx_dpmaif_update_drb_rd_idx(struct dpmaif_ctrl *dpmaif_ctrl, 5254359Sroberto unsigned int q_num) 53280849Scy{ 5454359Sroberto struct dpmaif_tx_queue *txq = &dpmaif_ctrl->txq[q_num]; 5582498Sroberto unsigned int old_sw_rd_idx, new_hw_rd_idx, drb_cnt; 56280849Scy unsigned long flags; 5782498Sroberto 5882498Sroberto if (!txq->que_started) 5954359Sroberto return 0; 6054359Sroberto 61280849Scy old_sw_rd_idx = txq->drb_rd_idx; 6254359Sroberto new_hw_rd_idx = t7xx_dpmaif_ul_get_rd_idx(&dpmaif_ctrl->hw_info, q_num); 6354359Sroberto if (new_hw_rd_idx >= DPMAIF_DRB_LIST_LEN) { 6454359Sroberto dev_err(dpmaif_ctrl->dev, "Out of range read index: %u\n", new_hw_rd_idx); 65280849Scy return 0; 6654359Sroberto } 6754359Sroberto 6854359Sroberto if (old_sw_rd_idx <= new_hw_rd_idx) 6954359Sroberto drb_cnt = new_hw_rd_idx - old_sw_rd_idx; 70280849Scy else 71280849Scy drb_cnt = txq->drb_size_cnt - old_sw_rd_idx + new_hw_rd_idx; 72280849Scy 73280849Scy spin_lock_irqsave(&txq->tx_lock, flags); 74280849Scy txq->drb_rd_idx = new_hw_rd_idx; 75280849Scy spin_unlock_irqrestore(&txq->tx_lock, flags); 76280849Scy 77280849Scy return drb_cnt; 7854359Sroberto} 7954359Sroberto 8054359Srobertostatic unsigned int t7xx_dpmaif_release_tx_buffer(struct dpmaif_ctrl *dpmaif_ctrl, 81280849Scy unsigned int q_num, unsigned int release_cnt) 8254359Sroberto{ 8354359Sroberto struct dpmaif_tx_queue *txq = &dpmaif_ctrl->txq[q_num]; 8454359Sroberto struct dpmaif_callbacks *cb = dpmaif_ctrl->callbacks; 85280849Scy struct dpmaif_drb_skb *cur_drb_skb, *drb_skb_base; 8654359Sroberto struct dpmaif_drb *cur_drb, *drb_base; 8754359Sroberto unsigned int drb_cnt, i, cur_idx; 8854359Sroberto unsigned long flags; 89280849Scy 9054359Sroberto drb_skb_base = txq->drb_skb_base; 9154359Sroberto drb_base = txq->drb_base; 9254359Sroberto 93280849Scy spin_lock_irqsave(&txq->tx_lock, flags); 9454359Sroberto drb_cnt = txq->drb_size_cnt; 9554359Sroberto cur_idx = txq->drb_release_rd_idx; 9654359Sroberto spin_unlock_irqrestore(&txq->tx_lock, flags); 9754359Sroberto 98280849Scy for (i = 0; i < release_cnt; i++) { 9954359Sroberto cur_drb = drb_base + cur_idx; 10054359Sroberto if (FIELD_GET(DRB_HDR_DTYP, le32_to_cpu(cur_drb->header)) == DES_DTYP_PD) { 10154359Sroberto cur_drb_skb = drb_skb_base + cur_idx; 102280849Scy if (!cur_drb_skb->is_msg) 10354359Sroberto dma_unmap_single(dpmaif_ctrl->dev, cur_drb_skb->bus_addr, 10454359Sroberto cur_drb_skb->data_len, DMA_TO_DEVICE); 10554359Sroberto 106280849Scy if (!FIELD_GET(DRB_HDR_CONT, le32_to_cpu(cur_drb->header))) { 10754359Sroberto if (!cur_drb_skb->skb) { 10854359Sroberto dev_err(dpmaif_ctrl->dev, 10954359Sroberto "txq%u: DRB check fail, invalid skb\n", q_num); 110280849Scy continue; 111280849Scy } 11254359Sroberto 11354359Sroberto dev_kfree_skb_any(cur_drb_skb->skb); 11454359Sroberto } 11554359Sroberto 11654359Sroberto cur_drb_skb->skb = NULL; 117280849Scy } 11854359Sroberto 11954359Sroberto spin_lock_irqsave(&txq->tx_lock, flags); 12054359Sroberto cur_idx = t7xx_ring_buf_get_next_wr_idx(drb_cnt, cur_idx); 12154359Sroberto txq->drb_release_rd_idx = cur_idx; 122280849Scy spin_unlock_irqrestore(&txq->tx_lock, flags); 12354359Sroberto 12454359Sroberto if (atomic_inc_return(&txq->tx_budget) > txq->drb_size_cnt / 8) 125280849Scy cb->state_notify(dpmaif_ctrl->t7xx_dev, DMPAIF_TXQ_STATE_IRQ, txq->index); 12654359Sroberto } 12754359Sroberto 12854359Sroberto if (FIELD_GET(DRB_HDR_CONT, le32_to_cpu(cur_drb->header))) 12954359Sroberto dev_err(dpmaif_ctrl->dev, "txq%u: DRB not marked as the last one\n", q_num); 13054359Sroberto 131280849Scy return i; 13254359Sroberto} 13354359Sroberto 13454359Srobertostatic int t7xx_dpmaif_tx_release(struct dpmaif_ctrl *dpmaif_ctrl, 13554359Sroberto unsigned int q_num, unsigned int budget) 136280849Scy{ 137280849Scy struct dpmaif_tx_queue *txq = &dpmaif_ctrl->txq[q_num]; 138280849Scy unsigned int rel_cnt, real_rel_cnt; 139280849Scy 140280849Scy /* Update read index from HW */ 141280849Scy t7xx_dpmaif_update_drb_rd_idx(dpmaif_ctrl, q_num); 142280849Scy 143280849Scy rel_cnt = t7xx_ring_buf_rd_wr_count(txq->drb_size_cnt, txq->drb_release_rd_idx, 144280849Scy txq->drb_rd_idx, DPMAIF_READ); 145280849Scy 146280849Scy real_rel_cnt = min_not_zero(budget, rel_cnt); 147280849Scy if (real_rel_cnt) 14854359Sroberto real_rel_cnt = t7xx_dpmaif_release_tx_buffer(dpmaif_ctrl, q_num, real_rel_cnt); 149280849Scy 15054359Sroberto return real_rel_cnt < rel_cnt ? -EAGAIN : 0; 151280849Scy} 152280849Scy 153280849Scystatic bool t7xx_dpmaif_drb_ring_not_empty(struct dpmaif_tx_queue *txq) 154280849Scy{ 155280849Scy return !!t7xx_dpmaif_update_drb_rd_idx(txq->dpmaif_ctrl, txq->index); 15654359Sroberto} 15754359Sroberto 15854359Srobertostatic void t7xx_dpmaif_tx_done(struct work_struct *work) 15954359Sroberto{ 160280849Scy struct dpmaif_tx_queue *txq = container_of(work, struct dpmaif_tx_queue, dpmaif_tx_work); 16154359Sroberto struct dpmaif_ctrl *dpmaif_ctrl = txq->dpmaif_ctrl; 16254359Sroberto struct dpmaif_hw_info *hw_info; 16356746Sroberto int ret; 164280849Scy 16556746Sroberto ret = pm_runtime_resume_and_get(dpmaif_ctrl->dev); 16656746Sroberto if (ret < 0 && ret != -EACCES) 16754359Sroberto return; 168280849Scy 16954359Sroberto /* The device may be in low power state. Disable sleep if needed */ 17054359Sroberto t7xx_pci_disable_sleep(dpmaif_ctrl->t7xx_dev); 17154359Sroberto if (t7xx_pci_sleep_disable_complete(dpmaif_ctrl->t7xx_dev)) { 172280849Scy hw_info = &dpmaif_ctrl->hw_info; 17354359Sroberto ret = t7xx_dpmaif_tx_release(dpmaif_ctrl, txq->index, txq->drb_size_cnt); 17454359Sroberto if (ret == -EAGAIN || 17554359Sroberto (t7xx_dpmaif_ul_clr_done(hw_info, txq->index) && 176280849Scy t7xx_dpmaif_drb_ring_not_empty(txq))) { 17754359Sroberto queue_work(dpmaif_ctrl->txq[txq->index].worker, 17854359Sroberto &dpmaif_ctrl->txq[txq->index].dpmaif_tx_work); 17954359Sroberto /* Give the device time to enter the low power state */ 180280849Scy t7xx_dpmaif_clr_ip_busy_sts(hw_info); 18154359Sroberto } else { 182280849Scy t7xx_dpmaif_clr_ip_busy_sts(hw_info); 18354359Sroberto t7xx_dpmaif_unmask_ulq_intr(hw_info, txq->index); 184280849Scy } 18554359Sroberto } 186280849Scy 187280849Scy t7xx_pci_enable_sleep(dpmaif_ctrl->t7xx_dev); 18854359Sroberto pm_runtime_mark_last_busy(dpmaif_ctrl->dev); 18954359Sroberto pm_runtime_put_autosuspend(dpmaif_ctrl->dev); 19054359Sroberto} 191280849Scy 19254359Srobertostatic void t7xx_setup_msg_drb(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int q_num, 19354359Sroberto unsigned int cur_idx, unsigned int pkt_len, unsigned int count_l, 19454359Sroberto unsigned int channel_id) 19554359Sroberto{ 196280849Scy struct dpmaif_drb *drb_base = dpmaif_ctrl->txq[q_num].drb_base; 197280849Scy struct dpmaif_drb *drb = drb_base + cur_idx; 19854359Sroberto 19954359Sroberto drb->header = cpu_to_le32(FIELD_PREP(DRB_HDR_DTYP, DES_DTYP_MSG) | 200280849Scy FIELD_PREP(DRB_HDR_CONT, 1) | 201280849Scy FIELD_PREP(DRB_HDR_DATA_LEN, pkt_len)); 20254359Sroberto 20354359Sroberto drb->msg.msg_hdr = cpu_to_le32(FIELD_PREP(DRB_MSG_COUNT_L, count_l) | 20454359Sroberto FIELD_PREP(DRB_MSG_CHANNEL_ID, channel_id) | 20554359Sroberto FIELD_PREP(DRB_MSG_L4_CHK, 1)); 206280849Scy} 20754359Sroberto 20854359Srobertostatic void t7xx_setup_payload_drb(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int q_num, 20954359Sroberto unsigned int cur_idx, dma_addr_t data_addr, 210280849Scy unsigned int pkt_size, bool last_one) 21154359Sroberto{ 21254359Sroberto struct dpmaif_drb *drb_base = dpmaif_ctrl->txq[q_num].drb_base; 21354359Sroberto struct dpmaif_drb *drb = drb_base + cur_idx; 21454359Sroberto u32 header; 21554359Sroberto 21654359Sroberto header = FIELD_PREP(DRB_HDR_DTYP, DES_DTYP_PD) | FIELD_PREP(DRB_HDR_DATA_LEN, pkt_size); 21754359Sroberto if (!last_one) 21854359Sroberto header |= FIELD_PREP(DRB_HDR_CONT, 1); 21954359Sroberto 220106424Sroberto drb->header = cpu_to_le32(header); 22154359Sroberto drb->pd.data_addr_l = cpu_to_le32(lower_32_bits(data_addr)); 22254359Sroberto drb->pd.data_addr_h = cpu_to_le32(upper_32_bits(data_addr)); 22354359Sroberto} 224362716Scy 225362716Scystatic void t7xx_record_drb_skb(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int q_num, 226362716Scy unsigned int cur_idx, struct sk_buff *skb, bool is_msg, 227362716Scy bool is_frag, bool is_last_one, dma_addr_t bus_addr, 228362716Scy unsigned int data_len) 229362716Scy{ 230362716Scy struct dpmaif_drb_skb *drb_skb_base = dpmaif_ctrl->txq[q_num].drb_skb_base; 231362716Scy struct dpmaif_drb_skb *drb_skb = drb_skb_base + cur_idx; 232362716Scy 233280849Scy drb_skb->skb = skb; 234 drb_skb->bus_addr = bus_addr; 235 drb_skb->data_len = data_len; 236 drb_skb->index = cur_idx; 237 drb_skb->is_msg = is_msg; 238 drb_skb->is_frag = is_frag; 239 drb_skb->is_last = is_last_one; 240} 241 242static int t7xx_dpmaif_add_skb_to_ring(struct dpmaif_ctrl *dpmaif_ctrl, struct sk_buff *skb) 243{ 244 struct dpmaif_callbacks *cb = dpmaif_ctrl->callbacks; 245 unsigned int wr_cnt, send_cnt, payload_cnt; 246 unsigned int cur_idx, drb_wr_idx_backup; 247 struct skb_shared_info *shinfo; 248 struct dpmaif_tx_queue *txq; 249 struct t7xx_skb_cb *skb_cb; 250 unsigned long flags; 251 252 skb_cb = T7XX_SKB_CB(skb); 253 txq = &dpmaif_ctrl->txq[skb_cb->txq_number]; 254 if (!txq->que_started || dpmaif_ctrl->state != DPMAIF_STATE_PWRON) 255 return -ENODEV; 256 257 atomic_set(&txq->tx_processing, 1); 258 /* Ensure tx_processing is changed to 1 before actually begin TX flow */ 259 smp_mb(); 260 261 shinfo = skb_shinfo(skb); 262 if (shinfo->frag_list) 263 dev_warn_ratelimited(dpmaif_ctrl->dev, "frag_list not supported\n"); 264 265 payload_cnt = shinfo->nr_frags + 1; 266 /* nr_frags: frag cnt, 1: skb->data, 1: msg DRB */ 267 send_cnt = payload_cnt + 1; 268 269 spin_lock_irqsave(&txq->tx_lock, flags); 270 cur_idx = txq->drb_wr_idx; 271 drb_wr_idx_backup = cur_idx; 272 txq->drb_wr_idx += send_cnt; 273 if (txq->drb_wr_idx >= txq->drb_size_cnt) 274 txq->drb_wr_idx -= txq->drb_size_cnt; 275 t7xx_setup_msg_drb(dpmaif_ctrl, txq->index, cur_idx, skb->len, 0, skb_cb->netif_idx); 276 t7xx_record_drb_skb(dpmaif_ctrl, txq->index, cur_idx, skb, true, 0, 0, 0, 0); 277 spin_unlock_irqrestore(&txq->tx_lock, flags); 278 279 for (wr_cnt = 0; wr_cnt < payload_cnt; wr_cnt++) { 280 bool is_frag, is_last_one = wr_cnt == payload_cnt - 1; 281 unsigned int data_len; 282 dma_addr_t bus_addr; 283 void *data_addr; 284 285 if (!wr_cnt) { 286 data_len = skb_headlen(skb); 287 data_addr = skb->data; 288 is_frag = false; 289 } else { 290 skb_frag_t *frag = shinfo->frags + wr_cnt - 1; 291 292 data_len = skb_frag_size(frag); 293 data_addr = skb_frag_address(frag); 294 is_frag = true; 295 } 296 297 bus_addr = dma_map_single(dpmaif_ctrl->dev, data_addr, data_len, DMA_TO_DEVICE); 298 if (dma_mapping_error(dpmaif_ctrl->dev, bus_addr)) 299 goto unmap_buffers; 300 301 cur_idx = t7xx_ring_buf_get_next_wr_idx(txq->drb_size_cnt, cur_idx); 302 303 spin_lock_irqsave(&txq->tx_lock, flags); 304 t7xx_setup_payload_drb(dpmaif_ctrl, txq->index, cur_idx, bus_addr, data_len, 305 is_last_one); 306 t7xx_record_drb_skb(dpmaif_ctrl, txq->index, cur_idx, skb, false, is_frag, 307 is_last_one, bus_addr, data_len); 308 spin_unlock_irqrestore(&txq->tx_lock, flags); 309 } 310 311 if (atomic_sub_return(send_cnt, &txq->tx_budget) <= (MAX_SKB_FRAGS + 2)) 312 cb->state_notify(dpmaif_ctrl->t7xx_dev, DMPAIF_TXQ_STATE_FULL, txq->index); 313 314 atomic_set(&txq->tx_processing, 0); 315 316 return 0; 317 318unmap_buffers: 319 while (wr_cnt--) { 320 struct dpmaif_drb_skb *drb_skb = txq->drb_skb_base; 321 322 cur_idx = cur_idx ? cur_idx - 1 : txq->drb_size_cnt - 1; 323 drb_skb += cur_idx; 324 dma_unmap_single(dpmaif_ctrl->dev, drb_skb->bus_addr, 325 drb_skb->data_len, DMA_TO_DEVICE); 326 } 327 328 txq->drb_wr_idx = drb_wr_idx_backup; 329 atomic_set(&txq->tx_processing, 0); 330 331 return -ENOMEM; 332} 333 334static bool t7xx_tx_lists_are_all_empty(const struct dpmaif_ctrl *dpmaif_ctrl) 335{ 336 int i; 337 338 for (i = 0; i < DPMAIF_TXQ_NUM; i++) { 339 if (!skb_queue_empty(&dpmaif_ctrl->txq[i].tx_skb_head)) 340 return false; 341 } 342 343 return true; 344} 345 346/* Currently, only the default TX queue is used */ 347static struct dpmaif_tx_queue *t7xx_select_tx_queue(struct dpmaif_ctrl *dpmaif_ctrl) 348{ 349 struct dpmaif_tx_queue *txq; 350 351 txq = &dpmaif_ctrl->txq[DPMAIF_TX_DEFAULT_QUEUE]; 352 if (!txq->que_started) 353 return NULL; 354 355 return txq; 356} 357 358static unsigned int t7xx_txq_drb_wr_available(struct dpmaif_tx_queue *txq) 359{ 360 return t7xx_ring_buf_rd_wr_count(txq->drb_size_cnt, txq->drb_release_rd_idx, 361 txq->drb_wr_idx, DPMAIF_WRITE); 362} 363 364static unsigned int t7xx_skb_drb_cnt(struct sk_buff *skb) 365{ 366 /* Normal DRB (frags data + skb linear data) + msg DRB */ 367 return skb_shinfo(skb)->nr_frags + 2; 368} 369 370static int t7xx_txq_burst_send_skb(struct dpmaif_tx_queue *txq) 371{ 372 unsigned int drb_remain_cnt, i; 373 unsigned int send_drb_cnt; 374 int drb_cnt = 0; 375 int ret = 0; 376 377 drb_remain_cnt = t7xx_txq_drb_wr_available(txq); 378 379 for (i = 0; i < DPMAIF_SKB_TX_BURST_CNT; i++) { 380 struct sk_buff *skb; 381 382 skb = skb_peek(&txq->tx_skb_head); 383 if (!skb) 384 break; 385 386 send_drb_cnt = t7xx_skb_drb_cnt(skb); 387 if (drb_remain_cnt < send_drb_cnt) { 388 drb_remain_cnt = t7xx_txq_drb_wr_available(txq); 389 continue; 390 } 391 392 drb_remain_cnt -= send_drb_cnt; 393 394 ret = t7xx_dpmaif_add_skb_to_ring(txq->dpmaif_ctrl, skb); 395 if (ret < 0) { 396 dev_err(txq->dpmaif_ctrl->dev, 397 "Failed to add skb to device's ring: %d\n", ret); 398 break; 399 } 400 401 drb_cnt += send_drb_cnt; 402 skb_unlink(skb, &txq->tx_skb_head); 403 } 404 405 if (drb_cnt > 0) 406 return drb_cnt; 407 408 return ret; 409} 410 411static void t7xx_do_tx_hw_push(struct dpmaif_ctrl *dpmaif_ctrl) 412{ 413 bool wait_disable_sleep = true; 414 415 do { 416 struct dpmaif_tx_queue *txq; 417 int drb_send_cnt; 418 419 txq = t7xx_select_tx_queue(dpmaif_ctrl); 420 if (!txq) 421 return; 422 423 drb_send_cnt = t7xx_txq_burst_send_skb(txq); 424 if (drb_send_cnt <= 0) { 425 usleep_range(10, 20); 426 cond_resched(); 427 continue; 428 } 429 430 /* Wait for the PCIe resource to unlock */ 431 if (wait_disable_sleep) { 432 if (!t7xx_pci_sleep_disable_complete(dpmaif_ctrl->t7xx_dev)) 433 return; 434 435 wait_disable_sleep = false; 436 } 437 438 t7xx_dpmaif_ul_update_hw_drb_cnt(&dpmaif_ctrl->hw_info, txq->index, 439 drb_send_cnt * DPMAIF_UL_DRB_SIZE_WORD); 440 441 cond_resched(); 442 } while (!t7xx_tx_lists_are_all_empty(dpmaif_ctrl) && !kthread_should_stop() && 443 (dpmaif_ctrl->state == DPMAIF_STATE_PWRON)); 444} 445 446static int t7xx_dpmaif_tx_hw_push_thread(void *arg) 447{ 448 struct dpmaif_ctrl *dpmaif_ctrl = arg; 449 int ret; 450 451 while (!kthread_should_stop()) { 452 if (t7xx_tx_lists_are_all_empty(dpmaif_ctrl) || 453 dpmaif_ctrl->state != DPMAIF_STATE_PWRON) { 454 if (wait_event_interruptible(dpmaif_ctrl->tx_wq, 455 (!t7xx_tx_lists_are_all_empty(dpmaif_ctrl) && 456 dpmaif_ctrl->state == DPMAIF_STATE_PWRON) || 457 kthread_should_stop())) 458 continue; 459 460 if (kthread_should_stop()) 461 break; 462 } 463 464 ret = pm_runtime_resume_and_get(dpmaif_ctrl->dev); 465 if (ret < 0 && ret != -EACCES) 466 return ret; 467 468 t7xx_pci_disable_sleep(dpmaif_ctrl->t7xx_dev); 469 t7xx_do_tx_hw_push(dpmaif_ctrl); 470 t7xx_pci_enable_sleep(dpmaif_ctrl->t7xx_dev); 471 pm_runtime_mark_last_busy(dpmaif_ctrl->dev); 472 pm_runtime_put_autosuspend(dpmaif_ctrl->dev); 473 } 474 475 return 0; 476} 477 478int t7xx_dpmaif_tx_thread_init(struct dpmaif_ctrl *dpmaif_ctrl) 479{ 480 init_waitqueue_head(&dpmaif_ctrl->tx_wq); 481 dpmaif_ctrl->tx_thread = kthread_run(t7xx_dpmaif_tx_hw_push_thread, 482 dpmaif_ctrl, "dpmaif_tx_hw_push"); 483 return PTR_ERR_OR_ZERO(dpmaif_ctrl->tx_thread); 484} 485 486void t7xx_dpmaif_tx_thread_rel(struct dpmaif_ctrl *dpmaif_ctrl) 487{ 488 if (dpmaif_ctrl->tx_thread) 489 kthread_stop(dpmaif_ctrl->tx_thread); 490} 491 492/** 493 * t7xx_dpmaif_tx_send_skb() - Add skb to the transmit queue. 494 * @dpmaif_ctrl: Pointer to struct dpmaif_ctrl. 495 * @txq_number: Queue number to xmit on. 496 * @skb: Pointer to the skb to transmit. 497 * 498 * Add the skb to the queue of the skbs to be transmit. 499 * Wake up the thread that push the skbs from the queue to the HW. 500 * 501 * Return: 502 * * 0 - Success. 503 * * -EBUSY - Tx budget exhausted. 504 * In normal circumstances t7xx_dpmaif_add_skb_to_ring() must report the txq full 505 * state to prevent this error condition. 506 */ 507int t7xx_dpmaif_tx_send_skb(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int txq_number, 508 struct sk_buff *skb) 509{ 510 struct dpmaif_tx_queue *txq = &dpmaif_ctrl->txq[txq_number]; 511 struct dpmaif_callbacks *cb = dpmaif_ctrl->callbacks; 512 struct t7xx_skb_cb *skb_cb; 513 514 if (atomic_read(&txq->tx_budget) <= t7xx_skb_drb_cnt(skb)) { 515 cb->state_notify(dpmaif_ctrl->t7xx_dev, DMPAIF_TXQ_STATE_FULL, txq_number); 516 return -EBUSY; 517 } 518 519 skb_cb = T7XX_SKB_CB(skb); 520 skb_cb->txq_number = txq_number; 521 skb_queue_tail(&txq->tx_skb_head, skb); 522 wake_up(&dpmaif_ctrl->tx_wq); 523 524 return 0; 525} 526 527void t7xx_dpmaif_irq_tx_done(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int que_mask) 528{ 529 int i; 530 531 for (i = 0; i < DPMAIF_TXQ_NUM; i++) { 532 if (que_mask & BIT(i)) 533 queue_work(dpmaif_ctrl->txq[i].worker, &dpmaif_ctrl->txq[i].dpmaif_tx_work); 534 } 535} 536 537static int t7xx_dpmaif_tx_drb_buf_init(struct dpmaif_tx_queue *txq) 538{ 539 size_t brb_skb_size, brb_pd_size; 540 541 brb_pd_size = DPMAIF_DRB_LIST_LEN * sizeof(struct dpmaif_drb); 542 brb_skb_size = DPMAIF_DRB_LIST_LEN * sizeof(struct dpmaif_drb_skb); 543 544 txq->drb_size_cnt = DPMAIF_DRB_LIST_LEN; 545 546 /* For HW && AP SW */ 547 txq->drb_base = dma_alloc_coherent(txq->dpmaif_ctrl->dev, brb_pd_size, 548 &txq->drb_bus_addr, GFP_KERNEL | __GFP_ZERO); 549 if (!txq->drb_base) 550 return -ENOMEM; 551 552 /* For AP SW to record the skb information */ 553 txq->drb_skb_base = devm_kzalloc(txq->dpmaif_ctrl->dev, brb_skb_size, GFP_KERNEL); 554 if (!txq->drb_skb_base) { 555 dma_free_coherent(txq->dpmaif_ctrl->dev, brb_pd_size, 556 txq->drb_base, txq->drb_bus_addr); 557 return -ENOMEM; 558 } 559 560 return 0; 561} 562 563static void t7xx_dpmaif_tx_free_drb_skb(struct dpmaif_tx_queue *txq) 564{ 565 struct dpmaif_drb_skb *drb_skb, *drb_skb_base = txq->drb_skb_base; 566 unsigned int i; 567 568 if (!drb_skb_base) 569 return; 570 571 for (i = 0; i < txq->drb_size_cnt; i++) { 572 drb_skb = drb_skb_base + i; 573 if (!drb_skb->skb) 574 continue; 575 576 if (!drb_skb->is_msg) 577 dma_unmap_single(txq->dpmaif_ctrl->dev, drb_skb->bus_addr, 578 drb_skb->data_len, DMA_TO_DEVICE); 579 580 if (drb_skb->is_last) { 581 dev_kfree_skb(drb_skb->skb); 582 drb_skb->skb = NULL; 583 } 584 } 585} 586 587static void t7xx_dpmaif_tx_drb_buf_rel(struct dpmaif_tx_queue *txq) 588{ 589 if (txq->drb_base) 590 dma_free_coherent(txq->dpmaif_ctrl->dev, 591 txq->drb_size_cnt * sizeof(struct dpmaif_drb), 592 txq->drb_base, txq->drb_bus_addr); 593 594 t7xx_dpmaif_tx_free_drb_skb(txq); 595} 596 597/** 598 * t7xx_dpmaif_txq_init() - Initialize TX queue. 599 * @txq: Pointer to struct dpmaif_tx_queue. 600 * 601 * Initialize the TX queue data structure and allocate memory for it to use. 602 * 603 * Return: 604 * * 0 - Success. 605 * * -ERROR - Error code from failure sub-initializations. 606 */ 607int t7xx_dpmaif_txq_init(struct dpmaif_tx_queue *txq) 608{ 609 int ret; 610 611 skb_queue_head_init(&txq->tx_skb_head); 612 init_waitqueue_head(&txq->req_wq); 613 atomic_set(&txq->tx_budget, DPMAIF_DRB_LIST_LEN); 614 615 ret = t7xx_dpmaif_tx_drb_buf_init(txq); 616 if (ret) { 617 dev_err(txq->dpmaif_ctrl->dev, "Failed to initialize DRB buffers: %d\n", ret); 618 return ret; 619 } 620 621 txq->worker = alloc_ordered_workqueue("md_dpmaif_tx%d_worker", 622 WQ_MEM_RECLAIM | (txq->index ? 0 : WQ_HIGHPRI), 623 txq->index); 624 if (!txq->worker) 625 return -ENOMEM; 626 627 INIT_WORK(&txq->dpmaif_tx_work, t7xx_dpmaif_tx_done); 628 spin_lock_init(&txq->tx_lock); 629 630 return 0; 631} 632 633void t7xx_dpmaif_txq_free(struct dpmaif_tx_queue *txq) 634{ 635 if (txq->worker) 636 destroy_workqueue(txq->worker); 637 638 skb_queue_purge(&txq->tx_skb_head); 639 t7xx_dpmaif_tx_drb_buf_rel(txq); 640} 641 642void t7xx_dpmaif_tx_stop(struct dpmaif_ctrl *dpmaif_ctrl) 643{ 644 int i; 645 646 for (i = 0; i < DPMAIF_TXQ_NUM; i++) { 647 struct dpmaif_tx_queue *txq; 648 int count = 0; 649 650 txq = &dpmaif_ctrl->txq[i]; 651 txq->que_started = false; 652 /* Make sure TXQ is disabled */ 653 smp_mb(); 654 655 /* Wait for active Tx to be done */ 656 while (atomic_read(&txq->tx_processing)) { 657 if (++count >= DPMAIF_MAX_CHECK_COUNT) { 658 dev_err(dpmaif_ctrl->dev, "TX queue stop failed\n"); 659 break; 660 } 661 } 662 } 663} 664 665static void t7xx_dpmaif_txq_flush_rel(struct dpmaif_tx_queue *txq) 666{ 667 txq->que_started = false; 668 669 cancel_work_sync(&txq->dpmaif_tx_work); 670 flush_work(&txq->dpmaif_tx_work); 671 t7xx_dpmaif_tx_free_drb_skb(txq); 672 673 txq->drb_rd_idx = 0; 674 txq->drb_wr_idx = 0; 675 txq->drb_release_rd_idx = 0; 676} 677 678void t7xx_dpmaif_tx_clear(struct dpmaif_ctrl *dpmaif_ctrl) 679{ 680 int i; 681 682 for (i = 0; i < DPMAIF_TXQ_NUM; i++) 683 t7xx_dpmaif_txq_flush_rel(&dpmaif_ctrl->txq[i]); 684} 685