@@ -473,11 +473,6 @@ static inline bool stmmac_tc_active(struct stmmac_priv *priv)
netif_device_present(priv->dev) && !priv->hw_unavailable;
}
-void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue);
-void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue);
-void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue);
-void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue);
-int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags);
struct timespec64 stmmac_calc_tas_basetime(ktime_t old_base_time,
ktime_t current_time,
u64 cycle_time);
@@ -2318,6 +2318,7 @@ static void __free_dma_tx_desc_resources(struct stmmac_priv *priv,
kfree(tx_q->tx_skbuff);
tx_q->tx_skbuff = NULL;
+ tx_q->xsk_pool = NULL;
}
static void free_dma_tx_desc_resources(struct stmmac_priv *priv,
@@ -2830,6 +2831,12 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget)
bool work_done = true;
u32 tx_set_ic_bit = 0;
+ /* Nothing can be submitted while the link is down. Let NAPI complete;
+ * userspace can retry ndo_xsk_wakeup() once carrier has returned.
+ */
+ if (!netif_carrier_ok(priv->dev))
+ return true;
+
/* Avoids TX time-out as we are sharing with slow path */
txq_trans_cond_update(nq);
@@ -2844,8 +2851,7 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget)
/* We are sharing with slow path and stop XSK TX desc submission when
* available TX ring is less than threshold.
*/
- if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL) ||
- !netif_carrier_ok(priv->dev)) {
+ if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL)) {
work_done = false;
break;
}
@@ -7450,107 +7456,6 @@ static int stmmac_xdp_xmit(struct net_device *dev, int num_frames,
return nxmit;
}
-void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue)
-{
- struct stmmac_channel *ch = &priv->channel[queue];
- unsigned long flags;
-
- spin_lock_irqsave(&ch->lock, flags);
- stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 1, 0);
- spin_unlock_irqrestore(&ch->lock, flags);
-
- stmmac_stop_rx_dma(priv, queue);
- __free_dma_rx_desc_resources(priv, priv->dma_conf, queue);
-}
-
-void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue)
-{
- struct stmmac_rx_queue *rx_q = &priv->dma_conf->rx_queue[queue];
- struct stmmac_channel *ch = &priv->channel[queue];
- unsigned long flags;
- int ret;
-
- ret = __alloc_dma_rx_desc_resources(priv, priv->dma_conf, queue);
- if (ret) {
- netdev_err(priv->dev, "Failed to alloc RX desc.\n");
- return;
- }
-
- ret = __init_dma_rx_desc_rings(priv, priv->dma_conf, queue, GFP_KERNEL);
- if (ret) {
- __free_dma_rx_desc_resources(priv, priv->dma_conf, queue);
- netdev_err(priv->dev, "Failed to init RX desc.\n");
- return;
- }
-
- stmmac_reset_rx_queue(priv, queue);
- stmmac_clear_rx_descriptors(priv, priv->dma_conf, queue);
-
- stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
- rx_q->dma_rx_phy, queue);
-
- stmmac_set_queue_rx_tail_ptr(priv, rx_q, queue, rx_q->buf_alloc_num);
-
- stmmac_set_queue_rx_buf_size(priv, rx_q, queue);
-
- stmmac_start_rx_dma(priv, queue);
-
- spin_lock_irqsave(&ch->lock, flags);
- stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 1, 0);
- spin_unlock_irqrestore(&ch->lock, flags);
-}
-
-void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue)
-{
- struct stmmac_channel *ch = &priv->channel[queue];
- unsigned long flags;
-
- spin_lock_irqsave(&ch->lock, flags);
- stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 0, 1);
- spin_unlock_irqrestore(&ch->lock, flags);
-
- stmmac_stop_tx_dma(priv, queue);
- __free_dma_tx_desc_resources(priv, priv->dma_conf, queue);
-}
-
-void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue)
-{
- struct stmmac_tx_queue *tx_q = &priv->dma_conf->tx_queue[queue];
- struct stmmac_channel *ch = &priv->channel[queue];
- unsigned long flags;
- int ret;
-
- ret = __alloc_dma_tx_desc_resources(priv, priv->dma_conf, queue);
- if (ret) {
- netdev_err(priv->dev, "Failed to alloc TX desc.\n");
- return;
- }
-
- ret = __init_dma_tx_desc_rings(priv, priv->dma_conf, queue);
- if (ret) {
- __free_dma_tx_desc_resources(priv, priv->dma_conf, queue);
- netdev_err(priv->dev, "Failed to init TX desc.\n");
- return;
- }
-
- stmmac_reset_tx_queue(priv, queue);
- stmmac_clear_tx_descriptors(priv, priv->dma_conf, queue);
-
- stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
- tx_q->dma_tx_phy, queue);
-
- if (tx_q->tbs & STMMAC_TBS_AVAIL)
- stmmac_enable_tbs(priv, priv->ioaddr, 1, queue);
-
- stmmac_set_queue_tx_tail_ptr(priv, tx_q, queue, 0);
-
- stmmac_start_tx_dma(priv, queue);
-
- spin_lock_irqsave(&ch->lock, flags);
- stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 0, 1);
- spin_unlock_irqrestore(&ch->lock, flags);
-}
-
void stmmac_xdp_release(struct net_device *dev)
{
struct stmmac_priv *priv = netdev_priv(dev);
@@ -7650,6 +7555,9 @@ int stmmac_xdp_open(struct net_device *dev)
stmmac_set_queue_tx_tail_ptr(priv, tx_q, chan, 0);
+ if (tx_q->tbs & STMMAC_TBS_AVAIL)
+ stmmac_enable_tbs(priv, priv->ioaddr, 1, chan);
+
hrtimer_setup(&tx_q->txtimer, stmmac_tx_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
}
@@ -7687,7 +7595,7 @@ int stmmac_xdp_open(struct net_device *dev)
return ret;
}
-int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags)
+static int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags)
{
struct stmmac_priv *priv = netdev_priv(dev);
struct stmmac_channel *ch;
@@ -9,7 +9,6 @@
static int stmmac_xdp_enable_pool(struct stmmac_priv *priv,
struct xsk_buff_pool *pool, u16 queue)
{
- struct stmmac_channel *ch = &priv->channel[queue];
bool need_update;
u32 frame_size;
int err;
@@ -34,23 +33,23 @@ static int stmmac_xdp_enable_pool(struct stmmac_priv *priv,
need_update = priv->datapath == STMMAC_DATAPATH_RUNNING &&
stmmac_xdp_is_enabled(priv);
- if (need_update) {
- napi_disable(&ch->rx_napi);
- napi_disable(&ch->tx_napi);
- stmmac_disable_rx_queue(priv, queue);
- stmmac_disable_tx_queue(priv, queue);
- }
+ if (need_update)
+ stmmac_xdp_release(priv->dev);
set_bit(queue, priv->af_xdp_zc_qps);
if (need_update) {
- stmmac_enable_rx_queue(priv, queue);
- stmmac_enable_tx_queue(priv, queue);
- napi_enable(&ch->rxtx_napi);
-
- err = stmmac_xsk_wakeup(priv->dev, queue, XDP_WAKEUP_RX);
- if (err)
+ err = stmmac_xdp_open(priv->dev);
+ if (err) {
+ clear_bit(queue, priv->af_xdp_zc_qps);
+ xsk_pool_dma_unmap(pool, STMMAC_RX_DMA_ATTR);
+ netdev_err(priv->dev, "failed reopening after XSK pool attach: %pe; interface remains detached\n",
+ ERR_PTR(err));
return err;
+ }
+
+ /* The pool is installed even if link resolution is still pending. */
+ napi_schedule(&priv->channel[queue].rxtx_napi);
}
return 0;
@@ -58,9 +57,9 @@ static int stmmac_xdp_enable_pool(struct stmmac_priv *priv,
static int stmmac_xdp_disable_pool(struct stmmac_priv *priv, u16 queue)
{
- struct stmmac_channel *ch = &priv->channel[queue];
struct xsk_buff_pool *pool;
bool need_update;
+ int err;
if (queue >= priv->plat->rx_queues_to_use ||
queue >= priv->plat->tx_queues_to_use)
@@ -73,24 +72,21 @@ static int stmmac_xdp_disable_pool(struct stmmac_priv *priv, u16 queue)
need_update = priv->datapath == STMMAC_DATAPATH_RUNNING &&
stmmac_xdp_is_enabled(priv);
- if (need_update) {
- napi_disable(&ch->rxtx_napi);
- stmmac_disable_rx_queue(priv, queue);
- stmmac_disable_tx_queue(priv, queue);
- synchronize_rcu();
- }
+ if (need_update)
+ stmmac_xdp_release(priv->dev);
xsk_pool_dma_unmap(pool, STMMAC_RX_DMA_ATTR);
clear_bit(queue, priv->af_xdp_zc_qps);
if (need_update) {
- stmmac_enable_rx_queue(priv, queue);
- stmmac_enable_tx_queue(priv, queue);
- napi_enable(&ch->rx_napi);
- napi_enable(&ch->tx_napi);
+ err = stmmac_xdp_open(priv->dev);
+ if (err)
+ netdev_err(priv->dev, "failed reopening after XSK pool removal: %pe; interface remains detached\n",
+ ERR_PTR(err));
}
+ /* Socket teardown must be able to unmap and free the removed pool. */
return 0;
}