659 lines
21 KiB
Diff
659 lines
21 KiB
Diff
From: Arthur Kiyanovski <akiyano@amazon.com>
|
|
Date: Thu, 11 Oct 2018 11:26:20 +0300
|
|
Subject: [PATCH 06/19] net: ena: add functions for handling Low Latency Queues
|
|
in ena_netdev
|
|
Origin: https://git.kernel.org/linus/38005ca816a7ef5516dc8e59ae95716739aa75b0
|
|
|
|
This patch includes all code changes necessary in ena_netdev to enable
|
|
packet sending via the LLQ placemnt mode.
|
|
|
|
Signed-off-by: Arthur Kiyanovski <akiyano@amazon.com>
|
|
Signed-off-by: David S. Miller <davem@davemloft.net>
|
|
---
|
|
drivers/net/ethernet/amazon/ena/ena_ethtool.c | 1 +
|
|
drivers/net/ethernet/amazon/ena/ena_netdev.c | 387 +++++++++++-------
|
|
drivers/net/ethernet/amazon/ena/ena_netdev.h | 6 +
|
|
3 files changed, 251 insertions(+), 143 deletions(-)
|
|
|
|
diff --git a/drivers/net/ethernet/amazon/ena/ena_ethtool.c b/drivers/net/ethernet/amazon/ena/ena_ethtool.c
|
|
index 521607bc4393..fd28bd0d1c1e 100644
|
|
--- a/drivers/net/ethernet/amazon/ena/ena_ethtool.c
|
|
+++ b/drivers/net/ethernet/amazon/ena/ena_ethtool.c
|
|
@@ -81,6 +81,7 @@ static const struct ena_stats ena_stats_tx_strings[] = {
|
|
ENA_STAT_TX_ENTRY(doorbells),
|
|
ENA_STAT_TX_ENTRY(prepare_ctx_err),
|
|
ENA_STAT_TX_ENTRY(bad_req_id),
|
|
+ ENA_STAT_TX_ENTRY(llq_buffer_copy),
|
|
ENA_STAT_TX_ENTRY(missed_tx),
|
|
};
|
|
|
|
diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c
|
|
index e732bd2ddd32..fcdfaf0ab8a7 100644
|
|
--- a/drivers/net/ethernet/amazon/ena/ena_netdev.c
|
|
+++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c
|
|
@@ -237,6 +237,17 @@ static int ena_setup_tx_resources(struct ena_adapter *adapter, int qid)
|
|
}
|
|
}
|
|
|
|
+ size = tx_ring->tx_max_header_size;
|
|
+ tx_ring->push_buf_intermediate_buf = vzalloc_node(size, node);
|
|
+ if (!tx_ring->push_buf_intermediate_buf) {
|
|
+ tx_ring->push_buf_intermediate_buf = vzalloc(size);
|
|
+ if (!tx_ring->push_buf_intermediate_buf) {
|
|
+ vfree(tx_ring->tx_buffer_info);
|
|
+ vfree(tx_ring->free_tx_ids);
|
|
+ return -ENOMEM;
|
|
+ }
|
|
+ }
|
|
+
|
|
/* Req id ring for TX out of order completions */
|
|
for (i = 0; i < tx_ring->ring_size; i++)
|
|
tx_ring->free_tx_ids[i] = i;
|
|
@@ -265,6 +276,9 @@ static void ena_free_tx_resources(struct ena_adapter *adapter, int qid)
|
|
|
|
vfree(tx_ring->free_tx_ids);
|
|
tx_ring->free_tx_ids = NULL;
|
|
+
|
|
+ vfree(tx_ring->push_buf_intermediate_buf);
|
|
+ tx_ring->push_buf_intermediate_buf = NULL;
|
|
}
|
|
|
|
/* ena_setup_all_tx_resources - allocate I/O Tx queues resources for All queues
|
|
@@ -602,6 +616,36 @@ static void ena_free_all_rx_bufs(struct ena_adapter *adapter)
|
|
ena_free_rx_bufs(adapter, i);
|
|
}
|
|
|
|
+static inline void ena_unmap_tx_skb(struct ena_ring *tx_ring,
|
|
+ struct ena_tx_buffer *tx_info)
|
|
+{
|
|
+ struct ena_com_buf *ena_buf;
|
|
+ u32 cnt;
|
|
+ int i;
|
|
+
|
|
+ ena_buf = tx_info->bufs;
|
|
+ cnt = tx_info->num_of_bufs;
|
|
+
|
|
+ if (unlikely(!cnt))
|
|
+ return;
|
|
+
|
|
+ if (tx_info->map_linear_data) {
|
|
+ dma_unmap_single(tx_ring->dev,
|
|
+ dma_unmap_addr(ena_buf, paddr),
|
|
+ dma_unmap_len(ena_buf, len),
|
|
+ DMA_TO_DEVICE);
|
|
+ ena_buf++;
|
|
+ cnt--;
|
|
+ }
|
|
+
|
|
+ /* unmap remaining mapped pages */
|
|
+ for (i = 0; i < cnt; i++) {
|
|
+ dma_unmap_page(tx_ring->dev, dma_unmap_addr(ena_buf, paddr),
|
|
+ dma_unmap_len(ena_buf, len), DMA_TO_DEVICE);
|
|
+ ena_buf++;
|
|
+ }
|
|
+}
|
|
+
|
|
/* ena_free_tx_bufs - Free Tx Buffers per Queue
|
|
* @tx_ring: TX ring for which buffers be freed
|
|
*/
|
|
@@ -612,9 +656,6 @@ static void ena_free_tx_bufs(struct ena_ring *tx_ring)
|
|
|
|
for (i = 0; i < tx_ring->ring_size; i++) {
|
|
struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i];
|
|
- struct ena_com_buf *ena_buf;
|
|
- int nr_frags;
|
|
- int j;
|
|
|
|
if (!tx_info->skb)
|
|
continue;
|
|
@@ -630,21 +671,7 @@ static void ena_free_tx_bufs(struct ena_ring *tx_ring)
|
|
tx_ring->qid, i);
|
|
}
|
|
|
|
- ena_buf = tx_info->bufs;
|
|
- dma_unmap_single(tx_ring->dev,
|
|
- ena_buf->paddr,
|
|
- ena_buf->len,
|
|
- DMA_TO_DEVICE);
|
|
-
|
|
- /* unmap remaining mapped pages */
|
|
- nr_frags = tx_info->num_of_bufs - 1;
|
|
- for (j = 0; j < nr_frags; j++) {
|
|
- ena_buf++;
|
|
- dma_unmap_page(tx_ring->dev,
|
|
- ena_buf->paddr,
|
|
- ena_buf->len,
|
|
- DMA_TO_DEVICE);
|
|
- }
|
|
+ ena_unmap_tx_skb(tx_ring, tx_info);
|
|
|
|
dev_kfree_skb_any(tx_info->skb);
|
|
}
|
|
@@ -735,8 +762,6 @@ static int ena_clean_tx_irq(struct ena_ring *tx_ring, u32 budget)
|
|
while (tx_pkts < budget) {
|
|
struct ena_tx_buffer *tx_info;
|
|
struct sk_buff *skb;
|
|
- struct ena_com_buf *ena_buf;
|
|
- int i, nr_frags;
|
|
|
|
rc = ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq,
|
|
&req_id);
|
|
@@ -756,24 +781,7 @@ static int ena_clean_tx_irq(struct ena_ring *tx_ring, u32 budget)
|
|
tx_info->skb = NULL;
|
|
tx_info->last_jiffies = 0;
|
|
|
|
- if (likely(tx_info->num_of_bufs != 0)) {
|
|
- ena_buf = tx_info->bufs;
|
|
-
|
|
- dma_unmap_single(tx_ring->dev,
|
|
- dma_unmap_addr(ena_buf, paddr),
|
|
- dma_unmap_len(ena_buf, len),
|
|
- DMA_TO_DEVICE);
|
|
-
|
|
- /* unmap remaining mapped pages */
|
|
- nr_frags = tx_info->num_of_bufs - 1;
|
|
- for (i = 0; i < nr_frags; i++) {
|
|
- ena_buf++;
|
|
- dma_unmap_page(tx_ring->dev,
|
|
- dma_unmap_addr(ena_buf, paddr),
|
|
- dma_unmap_len(ena_buf, len),
|
|
- DMA_TO_DEVICE);
|
|
- }
|
|
- }
|
|
+ ena_unmap_tx_skb(tx_ring, tx_info);
|
|
|
|
netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,
|
|
"tx_poll: q %d skb %p completed\n", tx_ring->qid,
|
|
@@ -1300,7 +1308,6 @@ static int ena_enable_msix(struct ena_adapter *adapter, int num_queues)
|
|
|
|
/* Reserved the max msix vectors we might need */
|
|
msix_vecs = ENA_MAX_MSIX_VEC(num_queues);
|
|
-
|
|
netif_dbg(adapter, probe, adapter->netdev,
|
|
"trying to enable MSI-X, vectors %d\n", msix_vecs);
|
|
|
|
@@ -1591,7 +1598,7 @@ static int ena_up_complete(struct ena_adapter *adapter)
|
|
|
|
static int ena_create_io_tx_queue(struct ena_adapter *adapter, int qid)
|
|
{
|
|
- struct ena_com_create_io_ctx ctx = { 0 };
|
|
+ struct ena_com_create_io_ctx ctx;
|
|
struct ena_com_dev *ena_dev;
|
|
struct ena_ring *tx_ring;
|
|
u32 msix_vector;
|
|
@@ -1604,6 +1611,8 @@ static int ena_create_io_tx_queue(struct ena_adapter *adapter, int qid)
|
|
msix_vector = ENA_IO_IRQ_IDX(qid);
|
|
ena_qid = ENA_IO_TXQ_IDX(qid);
|
|
|
|
+ memset(&ctx, 0x0, sizeof(ctx));
|
|
+
|
|
ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX;
|
|
ctx.qid = ena_qid;
|
|
ctx.mem_queue_type = ena_dev->tx_mem_queue_type;
|
|
@@ -1657,7 +1666,7 @@ static int ena_create_all_io_tx_queues(struct ena_adapter *adapter)
|
|
static int ena_create_io_rx_queue(struct ena_adapter *adapter, int qid)
|
|
{
|
|
struct ena_com_dev *ena_dev;
|
|
- struct ena_com_create_io_ctx ctx = { 0 };
|
|
+ struct ena_com_create_io_ctx ctx;
|
|
struct ena_ring *rx_ring;
|
|
u32 msix_vector;
|
|
u16 ena_qid;
|
|
@@ -1669,6 +1678,8 @@ static int ena_create_io_rx_queue(struct ena_adapter *adapter, int qid)
|
|
msix_vector = ENA_IO_IRQ_IDX(qid);
|
|
ena_qid = ENA_IO_RXQ_IDX(qid);
|
|
|
|
+ memset(&ctx, 0x0, sizeof(ctx));
|
|
+
|
|
ctx.qid = ena_qid;
|
|
ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX;
|
|
ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
|
|
@@ -1986,73 +1997,70 @@ static int ena_check_and_linearize_skb(struct ena_ring *tx_ring,
|
|
return rc;
|
|
}
|
|
|
|
-/* Called with netif_tx_lock. */
|
|
-static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev)
|
|
+static int ena_tx_map_skb(struct ena_ring *tx_ring,
|
|
+ struct ena_tx_buffer *tx_info,
|
|
+ struct sk_buff *skb,
|
|
+ void **push_hdr,
|
|
+ u16 *header_len)
|
|
{
|
|
- struct ena_adapter *adapter = netdev_priv(dev);
|
|
- struct ena_tx_buffer *tx_info;
|
|
- struct ena_com_tx_ctx ena_tx_ctx;
|
|
- struct ena_ring *tx_ring;
|
|
- struct netdev_queue *txq;
|
|
+ struct ena_adapter *adapter = tx_ring->adapter;
|
|
struct ena_com_buf *ena_buf;
|
|
- void *push_hdr;
|
|
- u32 len, last_frag;
|
|
- u16 next_to_use;
|
|
- u16 req_id;
|
|
- u16 push_len;
|
|
- u16 header_len;
|
|
dma_addr_t dma;
|
|
- int qid, rc, nb_hw_desc;
|
|
- int i = -1;
|
|
-
|
|
- netif_dbg(adapter, tx_queued, dev, "%s skb %p\n", __func__, skb);
|
|
- /* Determine which tx ring we will be placed on */
|
|
- qid = skb_get_queue_mapping(skb);
|
|
- tx_ring = &adapter->tx_ring[qid];
|
|
- txq = netdev_get_tx_queue(dev, qid);
|
|
-
|
|
- rc = ena_check_and_linearize_skb(tx_ring, skb);
|
|
- if (unlikely(rc))
|
|
- goto error_drop_packet;
|
|
-
|
|
- skb_tx_timestamp(skb);
|
|
- len = skb_headlen(skb);
|
|
+ u32 skb_head_len, frag_len, last_frag;
|
|
+ u16 push_len = 0;
|
|
+ u16 delta = 0;
|
|
+ int i = 0;
|
|
|
|
- next_to_use = tx_ring->next_to_use;
|
|
- req_id = tx_ring->free_tx_ids[next_to_use];
|
|
- tx_info = &tx_ring->tx_buffer_info[req_id];
|
|
- tx_info->num_of_bufs = 0;
|
|
-
|
|
- WARN(tx_info->skb, "SKB isn't NULL req_id %d\n", req_id);
|
|
- ena_buf = tx_info->bufs;
|
|
+ skb_head_len = skb_headlen(skb);
|
|
tx_info->skb = skb;
|
|
+ ena_buf = tx_info->bufs;
|
|
|
|
if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
|
|
- /* prepared the push buffer */
|
|
- push_len = min_t(u32, len, tx_ring->tx_max_header_size);
|
|
- header_len = push_len;
|
|
- push_hdr = skb->data;
|
|
+ /* When the device is LLQ mode, the driver will copy
|
|
+ * the header into the device memory space.
|
|
+ * the ena_com layer assume the header is in a linear
|
|
+ * memory space.
|
|
+ * This assumption might be wrong since part of the header
|
|
+ * can be in the fragmented buffers.
|
|
+ * Use skb_header_pointer to make sure the header is in a
|
|
+ * linear memory space.
|
|
+ */
|
|
+
|
|
+ push_len = min_t(u32, skb->len, tx_ring->tx_max_header_size);
|
|
+ *push_hdr = skb_header_pointer(skb, 0, push_len,
|
|
+ tx_ring->push_buf_intermediate_buf);
|
|
+ *header_len = push_len;
|
|
+ if (unlikely(skb->data != *push_hdr)) {
|
|
+ u64_stats_update_begin(&tx_ring->syncp);
|
|
+ tx_ring->tx_stats.llq_buffer_copy++;
|
|
+ u64_stats_update_end(&tx_ring->syncp);
|
|
+
|
|
+ delta = push_len - skb_head_len;
|
|
+ }
|
|
} else {
|
|
- push_len = 0;
|
|
- header_len = min_t(u32, len, tx_ring->tx_max_header_size);
|
|
- push_hdr = NULL;
|
|
+ *push_hdr = NULL;
|
|
+ *header_len = min_t(u32, skb_head_len,
|
|
+ tx_ring->tx_max_header_size);
|
|
}
|
|
|
|
- netif_dbg(adapter, tx_queued, dev,
|
|
+ netif_dbg(adapter, tx_queued, adapter->netdev,
|
|
"skb: %p header_buf->vaddr: %p push_len: %d\n", skb,
|
|
- push_hdr, push_len);
|
|
+ *push_hdr, push_len);
|
|
|
|
- if (len > push_len) {
|
|
+ if (skb_head_len > push_len) {
|
|
dma = dma_map_single(tx_ring->dev, skb->data + push_len,
|
|
- len - push_len, DMA_TO_DEVICE);
|
|
- if (dma_mapping_error(tx_ring->dev, dma))
|
|
+ skb_head_len - push_len, DMA_TO_DEVICE);
|
|
+ if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
|
|
goto error_report_dma_error;
|
|
|
|
ena_buf->paddr = dma;
|
|
- ena_buf->len = len - push_len;
|
|
+ ena_buf->len = skb_head_len - push_len;
|
|
|
|
ena_buf++;
|
|
tx_info->num_of_bufs++;
|
|
+ tx_info->map_linear_data = 1;
|
|
+ } else {
|
|
+ tx_info->map_linear_data = 0;
|
|
}
|
|
|
|
last_frag = skb_shinfo(skb)->nr_frags;
|
|
@@ -2060,18 +2068,75 @@ static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev)
|
|
for (i = 0; i < last_frag; i++) {
|
|
const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
|
|
|
|
- len = skb_frag_size(frag);
|
|
- dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
|
|
- DMA_TO_DEVICE);
|
|
- if (dma_mapping_error(tx_ring->dev, dma))
|
|
+ frag_len = skb_frag_size(frag);
|
|
+
|
|
+ if (unlikely(delta >= frag_len)) {
|
|
+ delta -= frag_len;
|
|
+ continue;
|
|
+ }
|
|
+
|
|
+ dma = skb_frag_dma_map(tx_ring->dev, frag, delta,
|
|
+ frag_len - delta, DMA_TO_DEVICE);
|
|
+ if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
|
|
goto error_report_dma_error;
|
|
|
|
ena_buf->paddr = dma;
|
|
- ena_buf->len = len;
|
|
+ ena_buf->len = frag_len - delta;
|
|
ena_buf++;
|
|
+ tx_info->num_of_bufs++;
|
|
+ delta = 0;
|
|
}
|
|
|
|
- tx_info->num_of_bufs += last_frag;
|
|
+ return 0;
|
|
+
|
|
+error_report_dma_error:
|
|
+ u64_stats_update_begin(&tx_ring->syncp);
|
|
+ tx_ring->tx_stats.dma_mapping_err++;
|
|
+ u64_stats_update_end(&tx_ring->syncp);
|
|
+ netdev_warn(adapter->netdev, "failed to map skb\n");
|
|
+
|
|
+ tx_info->skb = NULL;
|
|
+
|
|
+ tx_info->num_of_bufs += i;
|
|
+ ena_unmap_tx_skb(tx_ring, tx_info);
|
|
+
|
|
+ return -EINVAL;
|
|
+}
|
|
+
|
|
+/* Called with netif_tx_lock. */
|
|
+static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev)
|
|
+{
|
|
+ struct ena_adapter *adapter = netdev_priv(dev);
|
|
+ struct ena_tx_buffer *tx_info;
|
|
+ struct ena_com_tx_ctx ena_tx_ctx;
|
|
+ struct ena_ring *tx_ring;
|
|
+ struct netdev_queue *txq;
|
|
+ void *push_hdr;
|
|
+ u16 next_to_use, req_id, header_len;
|
|
+ int qid, rc, nb_hw_desc;
|
|
+
|
|
+ netif_dbg(adapter, tx_queued, dev, "%s skb %p\n", __func__, skb);
|
|
+ /* Determine which tx ring we will be placed on */
|
|
+ qid = skb_get_queue_mapping(skb);
|
|
+ tx_ring = &adapter->tx_ring[qid];
|
|
+ txq = netdev_get_tx_queue(dev, qid);
|
|
+
|
|
+ rc = ena_check_and_linearize_skb(tx_ring, skb);
|
|
+ if (unlikely(rc))
|
|
+ goto error_drop_packet;
|
|
+
|
|
+ skb_tx_timestamp(skb);
|
|
+
|
|
+ next_to_use = tx_ring->next_to_use;
|
|
+ req_id = tx_ring->free_tx_ids[next_to_use];
|
|
+ tx_info = &tx_ring->tx_buffer_info[req_id];
|
|
+ tx_info->num_of_bufs = 0;
|
|
+
|
|
+ WARN(tx_info->skb, "SKB isn't NULL req_id %d\n", req_id);
|
|
+
|
|
+ rc = ena_tx_map_skb(tx_ring, tx_info, skb, &push_hdr, &header_len);
|
|
+ if (unlikely(rc))
|
|
+ goto error_drop_packet;
|
|
|
|
memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx));
|
|
ena_tx_ctx.ena_bufs = tx_info->bufs;
|
|
@@ -2087,14 +2152,22 @@ static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev)
|
|
rc = ena_com_prepare_tx(tx_ring->ena_com_io_sq, &ena_tx_ctx,
|
|
&nb_hw_desc);
|
|
|
|
+ /* ena_com_prepare_tx() can't fail due to overflow of tx queue,
|
|
+ * since the number of free descriptors in the queue is checked
|
|
+ * after sending the previous packet. In case there isn't enough
|
|
+ * space in the queue for the next packet, it is stopped
|
|
+ * until there is again enough available space in the queue.
|
|
+ * All other failure reasons of ena_com_prepare_tx() are fatal
|
|
+ * and therefore require a device reset.
|
|
+ */
|
|
if (unlikely(rc)) {
|
|
netif_err(adapter, tx_queued, dev,
|
|
"failed to prepare tx bufs\n");
|
|
u64_stats_update_begin(&tx_ring->syncp);
|
|
- tx_ring->tx_stats.queue_stop++;
|
|
tx_ring->tx_stats.prepare_ctx_err++;
|
|
u64_stats_update_end(&tx_ring->syncp);
|
|
- netif_tx_stop_queue(txq);
|
|
+ adapter->reset_reason = ENA_REGS_RESET_DRIVER_INVALID_STATE;
|
|
+ set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
|
|
goto error_unmap_dma;
|
|
}
|
|
|
|
@@ -2157,35 +2230,11 @@ static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev)
|
|
|
|
return NETDEV_TX_OK;
|
|
|
|
-error_report_dma_error:
|
|
- u64_stats_update_begin(&tx_ring->syncp);
|
|
- tx_ring->tx_stats.dma_mapping_err++;
|
|
- u64_stats_update_end(&tx_ring->syncp);
|
|
- netdev_warn(adapter->netdev, "failed to map skb\n");
|
|
-
|
|
- tx_info->skb = NULL;
|
|
-
|
|
error_unmap_dma:
|
|
- if (i >= 0) {
|
|
- /* save value of frag that failed */
|
|
- last_frag = i;
|
|
-
|
|
- /* start back at beginning and unmap skb */
|
|
- tx_info->skb = NULL;
|
|
- ena_buf = tx_info->bufs;
|
|
- dma_unmap_single(tx_ring->dev, dma_unmap_addr(ena_buf, paddr),
|
|
- dma_unmap_len(ena_buf, len), DMA_TO_DEVICE);
|
|
-
|
|
- /* unmap remaining mapped pages */
|
|
- for (i = 0; i < last_frag; i++) {
|
|
- ena_buf++;
|
|
- dma_unmap_page(tx_ring->dev, dma_unmap_addr(ena_buf, paddr),
|
|
- dma_unmap_len(ena_buf, len), DMA_TO_DEVICE);
|
|
- }
|
|
- }
|
|
+ ena_unmap_tx_skb(tx_ring, tx_info);
|
|
+ tx_info->skb = NULL;
|
|
|
|
error_drop_packet:
|
|
-
|
|
dev_kfree_skb(skb);
|
|
return NETDEV_TX_OK;
|
|
}
|
|
@@ -2621,7 +2670,9 @@ static int ena_restore_device(struct ena_adapter *adapter)
|
|
netif_carrier_on(adapter->netdev);
|
|
|
|
mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));
|
|
- dev_err(&pdev->dev, "Device reset completed successfully\n");
|
|
+ dev_err(&pdev->dev,
|
|
+ "Device reset completed successfully, Driver info: %s\n",
|
|
+ version);
|
|
|
|
return rc;
|
|
err_disable_msix:
|
|
@@ -2988,18 +3039,52 @@ static int ena_calc_io_queue_num(struct pci_dev *pdev,
|
|
return io_queue_num;
|
|
}
|
|
|
|
-static void ena_set_push_mode(struct pci_dev *pdev, struct ena_com_dev *ena_dev,
|
|
- struct ena_com_dev_get_features_ctx *get_feat_ctx)
|
|
+static int ena_set_queues_placement_policy(struct pci_dev *pdev,
|
|
+ struct ena_com_dev *ena_dev,
|
|
+ struct ena_admin_feature_llq_desc *llq,
|
|
+ struct ena_llq_configurations *llq_default_configurations)
|
|
{
|
|
bool has_mem_bar;
|
|
+ int rc;
|
|
+ u32 llq_feature_mask;
|
|
+
|
|
+ llq_feature_mask = 1 << ENA_ADMIN_LLQ;
|
|
+ if (!(ena_dev->supported_features & llq_feature_mask)) {
|
|
+ dev_err(&pdev->dev,
|
|
+ "LLQ is not supported Fallback to host mode policy.\n");
|
|
+ ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
|
|
+ return 0;
|
|
+ }
|
|
|
|
has_mem_bar = pci_select_bars(pdev, IORESOURCE_MEM) & BIT(ENA_MEM_BAR);
|
|
|
|
- /* Enable push mode if device supports LLQ */
|
|
- if (has_mem_bar && get_feat_ctx->max_queues.max_legacy_llq_num > 0)
|
|
- ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_DEV;
|
|
- else
|
|
+ rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations);
|
|
+ if (unlikely(rc)) {
|
|
+ dev_err(&pdev->dev,
|
|
+ "Failed to configure the device mode. Fallback to host mode policy.\n");
|
|
+ ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
|
|
+ return 0;
|
|
+ }
|
|
+
|
|
+ /* Nothing to config, exit */
|
|
+ if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
|
|
+ return 0;
|
|
+
|
|
+ if (!has_mem_bar) {
|
|
+ dev_err(&pdev->dev,
|
|
+ "ENA device does not expose LLQ bar. Fallback to host mode policy.\n");
|
|
ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
|
|
+ return 0;
|
|
+ }
|
|
+
|
|
+ ena_dev->mem_bar = devm_ioremap_wc(&pdev->dev,
|
|
+ pci_resource_start(pdev, ENA_MEM_BAR),
|
|
+ pci_resource_len(pdev, ENA_MEM_BAR));
|
|
+
|
|
+ if (!ena_dev->mem_bar)
|
|
+ return -EFAULT;
|
|
+
|
|
+ return 0;
|
|
}
|
|
|
|
static void ena_set_dev_offloads(struct ena_com_dev_get_features_ctx *feat,
|
|
@@ -3117,6 +3202,15 @@ static void ena_release_bars(struct ena_com_dev *ena_dev, struct pci_dev *pdev)
|
|
pci_release_selected_regions(pdev, release_bars);
|
|
}
|
|
|
|
+static inline void set_default_llq_configurations(struct ena_llq_configurations *llq_config)
|
|
+{
|
|
+ llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER;
|
|
+ llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_128B;
|
|
+ llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY;
|
|
+ llq_config->llq_num_decs_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2;
|
|
+ llq_config->llq_ring_entry_size_value = 128;
|
|
+}
|
|
+
|
|
static int ena_calc_queue_size(struct pci_dev *pdev,
|
|
struct ena_com_dev *ena_dev,
|
|
u16 *max_tx_sgl_size,
|
|
@@ -3165,7 +3259,9 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
|
|
static int version_printed;
|
|
struct net_device *netdev;
|
|
struct ena_adapter *adapter;
|
|
+ struct ena_llq_configurations llq_config;
|
|
struct ena_com_dev *ena_dev = NULL;
|
|
+ char *queue_type_str;
|
|
static int adapters_found;
|
|
int io_queue_num, bars, rc;
|
|
int queue_size;
|
|
@@ -3219,16 +3315,13 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
|
|
goto err_free_region;
|
|
}
|
|
|
|
- ena_set_push_mode(pdev, ena_dev, &get_feat_ctx);
|
|
+ set_default_llq_configurations(&llq_config);
|
|
|
|
- if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
|
|
- ena_dev->mem_bar = devm_ioremap_wc(&pdev->dev,
|
|
- pci_resource_start(pdev, ENA_MEM_BAR),
|
|
- pci_resource_len(pdev, ENA_MEM_BAR));
|
|
- if (!ena_dev->mem_bar) {
|
|
- rc = -EFAULT;
|
|
- goto err_device_destroy;
|
|
- }
|
|
+ rc = ena_set_queues_placement_policy(pdev, ena_dev, &get_feat_ctx.llq,
|
|
+ &llq_config);
|
|
+ if (rc) {
|
|
+ dev_err(&pdev->dev, "ena device init failed\n");
|
|
+ goto err_device_destroy;
|
|
}
|
|
|
|
/* initial Tx interrupt delay, Assumes 1 usec granularity.
|
|
@@ -3243,8 +3336,10 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
|
|
goto err_device_destroy;
|
|
}
|
|
|
|
- dev_info(&pdev->dev, "creating %d io queues. queue size: %d\n",
|
|
- io_queue_num, queue_size);
|
|
+ dev_info(&pdev->dev, "creating %d io queues. queue size: %d. LLQ is %s\n",
|
|
+ io_queue_num, queue_size,
|
|
+ (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) ?
|
|
+ "ENABLED" : "DISABLED");
|
|
|
|
/* dev zeroed in init_etherdev */
|
|
netdev = alloc_etherdev_mq(sizeof(struct ena_adapter), io_queue_num);
|
|
@@ -3334,9 +3429,15 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
|
|
timer_setup(&adapter->timer_service, ena_timer_service, 0);
|
|
mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));
|
|
|
|
- dev_info(&pdev->dev, "%s found at mem %lx, mac addr %pM Queues %d\n",
|
|
+ if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
|
|
+ queue_type_str = "Regular";
|
|
+ else
|
|
+ queue_type_str = "Low Latency";
|
|
+
|
|
+ dev_info(&pdev->dev,
|
|
+ "%s found at mem %lx, mac addr %pM Queues %d, Placement policy: %s\n",
|
|
DEVICE_NAME, (long)pci_resource_start(pdev, 0),
|
|
- netdev->dev_addr, io_queue_num);
|
|
+ netdev->dev_addr, io_queue_num, queue_type_str);
|
|
|
|
set_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
|
|
|
|
diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.h b/drivers/net/ethernet/amazon/ena/ena_netdev.h
|
|
index 7c7ae56c52cf..4fa7d2fda475 100644
|
|
--- a/drivers/net/ethernet/amazon/ena/ena_netdev.h
|
|
+++ b/drivers/net/ethernet/amazon/ena/ena_netdev.h
|
|
@@ -151,6 +151,9 @@ struct ena_tx_buffer {
|
|
/* num of buffers used by this skb */
|
|
u32 num_of_bufs;
|
|
|
|
+ /* Indicate if bufs[0] map the linear data of the skb. */
|
|
+ u8 map_linear_data;
|
|
+
|
|
/* Used for detect missing tx packets to limit the number of prints */
|
|
u32 print_once;
|
|
/* Save the last jiffies to detect missing tx packets
|
|
@@ -186,6 +189,7 @@ struct ena_stats_tx {
|
|
u64 tx_poll;
|
|
u64 doorbells;
|
|
u64 bad_req_id;
|
|
+ u64 llq_buffer_copy;
|
|
u64 missed_tx;
|
|
};
|
|
|
|
@@ -257,6 +261,8 @@ struct ena_ring {
|
|
struct ena_stats_tx tx_stats;
|
|
struct ena_stats_rx rx_stats;
|
|
};
|
|
+
|
|
+ u8 *push_buf_intermediate_buf;
|
|
int empty_rx_queue;
|
|
} ____cacheline_aligned;
|
|
|
|
--
|
|
2.19.2
|
|
|