summaryrefslogtreecommitdiff
path: root/drivers
diff options
context:
space:
mode:
authorDavid S. Miller <davem@davemloft.net>2021-02-16 13:14:06 -0800
committerDavid S. Miller <davem@davemloft.net>2021-02-16 13:14:06 -0800
commitb8af417e4d93caeefb89bbfbd56ec95dedd8dab5 (patch)
tree1c8d22e1aec330238830a43cc8aee0cf768ae1c7 /drivers
parent9ec5eea5b6acfae7279203097eeec5d02d01d9b7 (diff)
parent45159b27637b0fef6d5ddb86fc7c46b13c77960f (diff)
Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next
Daniel Borkmann says: ==================== pull-request: bpf-next 2021-02-16 The following pull-request contains BPF updates for your *net-next* tree. There's a small merge conflict between 7eeba1706eba ("tcp: Add receive timestamp support for receive zerocopy.") from net-next tree and 9cacf81f8161 ("bpf: Remove extra lock_sock for TCP_ZEROCOPY_RECEIVE") from bpf-next tree. Resolve as follows: [...] lock_sock(sk); err = tcp_zerocopy_receive(sk, &zc, &tss); err = BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sk, level, optname, &zc, &len, err); release_sock(sk); [...] We've added 116 non-merge commits during the last 27 day(s) which contain a total of 156 files changed, 5662 insertions(+), 1489 deletions(-). The main changes are: 1) Adds support of pointers to types with known size among global function args to overcome the limit on max # of allowed args, from Dmitrii Banshchikov. 2) Add bpf_iter for task_vma which can be used to generate information similar to /proc/pid/maps, from Song Liu. 3) Enable bpf_{g,s}etsockopt() from all sock_addr related program hooks. Allow rewriting bind user ports from BPF side below the ip_unprivileged_port_start range, both from Stanislav Fomichev. 4) Prevent recursion on fentry/fexit & sleepable programs and allow map-in-map as well as per-cpu maps for the latter, from Alexei Starovoitov. 5) Add selftest script to run BPF CI locally. Also enable BPF ringbuffer for sleepable programs, both from KP Singh. 6) Extend verifier to enable variable offset read/write access to the BPF program stack, from Andrei Matei. 7) Improve tc & XDP MTU handling and add a new bpf_check_mtu() helper to query device MTU from programs, from Jesper Dangaard Brouer. 8) Allow bpf_get_socket_cookie() helper also be called from [sleepable] BPF tracing programs, from Florent Revest. 9) Extend x86 JIT to pad JMPs with NOPs for helping image to converge when otherwise too many passes are required, from Gary Lin. 10) Verifier fixes on atomics with BPF_FETCH as well as function-by-function verification both related to zero-extension handling, from Ilya Leoshkevich. 11) Better kernel build integration of resolve_btfids tool, from Jiri Olsa. 12) Batch of AF_XDP selftest cleanups and small performance improvement for libbpf's xsk map redirect for newer kernels, from Björn Töpel. 13) Follow-up BPF doc and verifier improvements around atomics with BPF_FETCH, from Brendan Jackman. 14) Permit zero-sized data sections e.g. if ELF .rodata section contains read-only data from local variables, from Yonghong Song. 15) veth driver skb bulk-allocation for ndo_xdp_xmit, from Lorenzo Bianconi. ==================== Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'drivers')
-rw-r--r--drivers/net/veth.c94
1 files changed, 60 insertions, 34 deletions
diff --git a/drivers/net/veth.c b/drivers/net/veth.c
index 99caae7d1641..aa1a66ad2ce5 100644
--- a/drivers/net/veth.c
+++ b/drivers/net/veth.c
@@ -35,6 +35,7 @@
#define VETH_XDP_HEADROOM (XDP_PACKET_HEADROOM + NET_IP_ALIGN)
#define VETH_XDP_TX_BULK_SIZE 16
+#define VETH_XDP_BATCH 16
struct veth_stats {
u64 rx_drops;
@@ -562,20 +563,13 @@ static int veth_xdp_tx(struct veth_rq *rq, struct xdp_buff *xdp,
return 0;
}
-static struct sk_buff *veth_xdp_rcv_one(struct veth_rq *rq,
- struct xdp_frame *frame,
- struct veth_xdp_tx_bq *bq,
- struct veth_stats *stats)
+static struct xdp_frame *veth_xdp_rcv_one(struct veth_rq *rq,
+ struct xdp_frame *frame,
+ struct veth_xdp_tx_bq *bq,
+ struct veth_stats *stats)
{
- void *hard_start = frame->data - frame->headroom;
- int len = frame->len, delta = 0;
struct xdp_frame orig_frame;
struct bpf_prog *xdp_prog;
- unsigned int headroom;
- struct sk_buff *skb;
-
- /* bpf_xdp_adjust_head() assures BPF cannot access xdp_frame area */
- hard_start -= sizeof(struct xdp_frame);
rcu_read_lock();
xdp_prog = rcu_dereference(rq->xdp_prog);
@@ -590,8 +584,8 @@ static struct sk_buff *veth_xdp_rcv_one(struct veth_rq *rq,
switch (act) {
case XDP_PASS:
- delta = frame->data - xdp.data;
- len = xdp.data_end - xdp.data;
+ if (xdp_update_frame_from_buff(&xdp, frame))
+ goto err_xdp;
break;
case XDP_TX:
orig_frame = *frame;
@@ -629,19 +623,7 @@ static struct sk_buff *veth_xdp_rcv_one(struct veth_rq *rq,
}
rcu_read_unlock();
- headroom = sizeof(struct xdp_frame) + frame->headroom - delta;
- skb = veth_build_skb(hard_start, headroom, len, frame->frame_sz);
- if (!skb) {
- xdp_return_frame(frame);
- stats->rx_drops++;
- goto err;
- }
-
- xdp_release_frame(frame);
- xdp_scrub_frame(frame);
- skb->protocol = eth_type_trans(skb, rq->dev);
-err:
- return skb;
+ return frame;
err_xdp:
rcu_read_unlock();
xdp_return_frame(frame);
@@ -649,6 +631,37 @@ xdp_xmit:
return NULL;
}
+/* frames array contains VETH_XDP_BATCH at most */
+static void veth_xdp_rcv_bulk_skb(struct veth_rq *rq, void **frames,
+ int n_xdpf, struct veth_xdp_tx_bq *bq,
+ struct veth_stats *stats)
+{
+ void *skbs[VETH_XDP_BATCH];
+ int i;
+
+ if (xdp_alloc_skb_bulk(skbs, n_xdpf,
+ GFP_ATOMIC | __GFP_ZERO) < 0) {
+ for (i = 0; i < n_xdpf; i++)
+ xdp_return_frame(frames[i]);
+ stats->rx_drops += n_xdpf;
+
+ return;
+ }
+
+ for (i = 0; i < n_xdpf; i++) {
+ struct sk_buff *skb = skbs[i];
+
+ skb = __xdp_build_skb_from_frame(frames[i], skb,
+ rq->dev);
+ if (!skb) {
+ xdp_return_frame(frames[i]);
+ stats->rx_drops++;
+ continue;
+ }
+ napi_gro_receive(&rq->xdp_napi, skb);
+ }
+}
+
static struct sk_buff *veth_xdp_rcv_skb(struct veth_rq *rq,
struct sk_buff *skb,
struct veth_xdp_tx_bq *bq,
@@ -796,32 +809,45 @@ static int veth_xdp_rcv(struct veth_rq *rq, int budget,
struct veth_xdp_tx_bq *bq,
struct veth_stats *stats)
{
- int i, done = 0;
+ int i, done = 0, n_xdpf = 0;
+ void *xdpf[VETH_XDP_BATCH];
for (i = 0; i < budget; i++) {
void *ptr = __ptr_ring_consume(&rq->xdp_ring);
- struct sk_buff *skb;
if (!ptr)
break;
if (veth_is_xdp_frame(ptr)) {
+ /* ndo_xdp_xmit */
struct xdp_frame *frame = veth_ptr_to_xdp(ptr);
stats->xdp_bytes += frame->len;
- skb = veth_xdp_rcv_one(rq, frame, bq, stats);
+ frame = veth_xdp_rcv_one(rq, frame, bq, stats);
+ if (frame) {
+ /* XDP_PASS */
+ xdpf[n_xdpf++] = frame;
+ if (n_xdpf == VETH_XDP_BATCH) {
+ veth_xdp_rcv_bulk_skb(rq, xdpf, n_xdpf,
+ bq, stats);
+ n_xdpf = 0;
+ }
+ }
} else {
- skb = ptr;
+ /* ndo_start_xmit */
+ struct sk_buff *skb = ptr;
+
stats->xdp_bytes += skb->len;
skb = veth_xdp_rcv_skb(rq, skb, bq, stats);
+ if (skb)
+ napi_gro_receive(&rq->xdp_napi, skb);
}
-
- if (skb)
- napi_gro_receive(&rq->xdp_napi, skb);
-
done++;
}
+ if (n_xdpf)
+ veth_xdp_rcv_bulk_skb(rq, xdpf, n_xdpf, bq, stats);
+
u64_stats_update_begin(&rq->stats.syncp);
rq->stats.vs.xdp_redirect += stats->xdp_redirect;
rq->stats.vs.xdp_bytes += stats->xdp_bytes;