152 lines
4.5 KiB
Diff
152 lines
4.5 KiB
Diff
From e55bc3b0e8dd9e6a902ab14916f8e4f142ef1c49 Mon Sep 17 00:00:00 2001
|
|
Message-Id: <e55bc3b0e8dd9e6a902ab14916f8e4f142ef1c49.1601675152.git.zanussi@kernel.org>
|
|
In-Reply-To: <5b5a156f9808b1acf1205606e03da117214549ea.1601675151.git.zanussi@kernel.org>
|
|
References: <5b5a156f9808b1acf1205606e03da117214549ea.1601675151.git.zanussi@kernel.org>
|
|
From: Thomas Gleixner <tglx@linutronix.de>
|
|
Date: Tue, 12 Jul 2011 15:38:34 +0200
|
|
Subject: [PATCH 197/333] net: Use skbufhead with raw lock
|
|
Origin: https://www.kernel.org/pub/linux/kernel/projects/rt/4.19/older/patches-4.19.148-rt64.tar.xz
|
|
|
|
Use the rps lock as rawlock so we can keep irq-off regions. It looks low
|
|
latency. However we can't kfree() from this context therefore we defer this
|
|
to the softirq and use the tofree_queue list for it (similar to process_queue).
|
|
|
|
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
|
|
---
|
|
include/linux/netdevice.h | 1 +
|
|
include/linux/skbuff.h | 7 +++++++
|
|
net/core/dev.c | 26 ++++++++++++++++++++------
|
|
3 files changed, 28 insertions(+), 6 deletions(-)
|
|
|
|
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
|
|
index 4e14926433ed..8fff010fff6e 100644
|
|
--- a/include/linux/netdevice.h
|
|
+++ b/include/linux/netdevice.h
|
|
@@ -2979,6 +2979,7 @@ struct softnet_data {
|
|
unsigned int dropped;
|
|
struct sk_buff_head input_pkt_queue;
|
|
struct napi_struct backlog;
|
|
+ struct sk_buff_head tofree_queue;
|
|
|
|
};
|
|
|
|
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
|
|
index cbc0294f3989..1054741c9233 100644
|
|
--- a/include/linux/skbuff.h
|
|
+++ b/include/linux/skbuff.h
|
|
@@ -287,6 +287,7 @@ struct sk_buff_head {
|
|
|
|
__u32 qlen;
|
|
spinlock_t lock;
|
|
+ raw_spinlock_t raw_lock;
|
|
};
|
|
|
|
struct sk_buff;
|
|
@@ -1718,6 +1719,12 @@ static inline void skb_queue_head_init(struct sk_buff_head *list)
|
|
__skb_queue_head_init(list);
|
|
}
|
|
|
|
+static inline void skb_queue_head_init_raw(struct sk_buff_head *list)
|
|
+{
|
|
+ raw_spin_lock_init(&list->raw_lock);
|
|
+ __skb_queue_head_init(list);
|
|
+}
|
|
+
|
|
static inline void skb_queue_head_init_class(struct sk_buff_head *list,
|
|
struct lock_class_key *class)
|
|
{
|
|
diff --git a/net/core/dev.c b/net/core/dev.c
|
|
index 414e4d7e2082..fc1dfd57d4c0 100644
|
|
--- a/net/core/dev.c
|
|
+++ b/net/core/dev.c
|
|
@@ -219,14 +219,14 @@ static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
|
|
static inline void rps_lock(struct softnet_data *sd)
|
|
{
|
|
#ifdef CONFIG_RPS
|
|
- spin_lock(&sd->input_pkt_queue.lock);
|
|
+ raw_spin_lock(&sd->input_pkt_queue.raw_lock);
|
|
#endif
|
|
}
|
|
|
|
static inline void rps_unlock(struct softnet_data *sd)
|
|
{
|
|
#ifdef CONFIG_RPS
|
|
- spin_unlock(&sd->input_pkt_queue.lock);
|
|
+ raw_spin_unlock(&sd->input_pkt_queue.raw_lock);
|
|
#endif
|
|
}
|
|
|
|
@@ -5846,7 +5846,9 @@ static int process_backlog(struct napi_struct *napi, int quota)
|
|
while (again) {
|
|
struct sk_buff *skb;
|
|
|
|
+ local_irq_disable();
|
|
while ((skb = __skb_dequeue(&sd->process_queue))) {
|
|
+ local_irq_enable();
|
|
rcu_read_lock();
|
|
__netif_receive_skb(skb);
|
|
rcu_read_unlock();
|
|
@@ -5854,9 +5856,9 @@ static int process_backlog(struct napi_struct *napi, int quota)
|
|
if (++work >= quota)
|
|
return work;
|
|
|
|
+ local_irq_disable();
|
|
}
|
|
|
|
- local_irq_disable();
|
|
rps_lock(sd);
|
|
if (skb_queue_empty(&sd->input_pkt_queue)) {
|
|
/*
|
|
@@ -6322,13 +6324,21 @@ static __latent_entropy void net_rx_action(struct softirq_action *h)
|
|
unsigned long time_limit = jiffies +
|
|
usecs_to_jiffies(netdev_budget_usecs);
|
|
int budget = netdev_budget;
|
|
+ struct sk_buff_head tofree_q;
|
|
+ struct sk_buff *skb;
|
|
LIST_HEAD(list);
|
|
LIST_HEAD(repoll);
|
|
|
|
+ __skb_queue_head_init(&tofree_q);
|
|
+
|
|
local_irq_disable();
|
|
+ skb_queue_splice_init(&sd->tofree_queue, &tofree_q);
|
|
list_splice_init(&sd->poll_list, &list);
|
|
local_irq_enable();
|
|
|
|
+ while ((skb = __skb_dequeue(&tofree_q)))
|
|
+ kfree_skb(skb);
|
|
+
|
|
for (;;) {
|
|
struct napi_struct *n;
|
|
|
|
@@ -9521,10 +9531,13 @@ static int dev_cpu_dead(unsigned int oldcpu)
|
|
netif_rx_ni(skb);
|
|
input_queue_head_incr(oldsd);
|
|
}
|
|
- while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
|
|
+ while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
|
|
netif_rx_ni(skb);
|
|
input_queue_head_incr(oldsd);
|
|
}
|
|
+ while ((skb = __skb_dequeue(&oldsd->tofree_queue))) {
|
|
+ kfree_skb(skb);
|
|
+ }
|
|
|
|
return 0;
|
|
}
|
|
@@ -9835,8 +9848,9 @@ static int __init net_dev_init(void)
|
|
|
|
INIT_WORK(flush, flush_backlog);
|
|
|
|
- skb_queue_head_init(&sd->input_pkt_queue);
|
|
- skb_queue_head_init(&sd->process_queue);
|
|
+ skb_queue_head_init_raw(&sd->input_pkt_queue);
|
|
+ skb_queue_head_init_raw(&sd->process_queue);
|
|
+ skb_queue_head_init_raw(&sd->tofree_queue);
|
|
#ifdef CONFIG_XFRM_OFFLOAD
|
|
skb_queue_head_init(&sd->xfrm_backlog);
|
|
#endif
|
|
--
|
|
2.17.1
|
|
|