--- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -825,6 +825,9 @@ __u8 decrypted:1; #endif __u8 gro_skip:1; +#ifdef CONFIG_SHORTCUT_FE + __u8 fast_forwarded:1; +#endif #ifdef CONFIG_NET_SCHED __u16 tc_index; /* traffic control index */ #if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_BRIDGE_IGMP_SNOOPING) int br_multicast_list_adjacent(struct net_device *dev, --- a/net/Kconfig +++ b/net/Kconfig @@ -473,3 +473,6 @@ config HAVE_CBPF_JIT # Extended BPF JIT (eBPF) config HAVE_EBPF_JIT bool + +config SHORTCUT_FE + bool "Enables kernel network stack path for Shortcut Forwarding Engine --- a/net/core/dev.c +++ b/net/core/dev.c @@ -3192,8 +3192,17 @@ static int xmit_one(struct sk_buff *skb, unsigned int len; int rc; +#ifdef CONFIG_SHORTCUT_FE + /* If this skb has been fast forwarded then we don't want it to + * go to any taps (by definition we're trying to bypass them). + */ + if (!skb->fast_forwarded) { +#endif if (dev_nit_active(dev)) dev_queue_xmit_nit(skb, dev); +#ifdef CONFIG_SHORTCUT_FE + } +#endif #ifdef CONFIG_ETHERNET_PACKET_MANGLE if (!dev->eth_mangle_tx || @@ -4684,6 +4693,11 @@ void netdev_rx_handler_unregister(struct } EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister); +#ifdef CONFIG_SHORTCUT_FE +int (*athrs_fast_nat_recv)(struct sk_buff *skb) __rcu __read_mostly; +EXPORT_SYMBOL_GPL(athrs_fast_nat_recv); +#endif + /* * Limit the use of PFMEMALLOC reserves to those protocols that implement * the special handling of PFMEMALLOC skbs. @@ -4733,6 +4747,9 @@ static int __netif_receive_skb_core(stru bool deliver_exact = false; int ret = NET_RX_DROP; __be16 type; +#ifdef CONFIG_SHORTCUT_FE + int (*fast_recv)(struct sk_buff *skb); +#endif net_timestamp_check(!netdev_tstamp_prequeue, skb); @@ -4773,6 +4790,16 @@ another_round: goto out; } +#ifdef CONFIG_SHORTCUT_FE + fast_recv = rcu_dereference(athrs_fast_nat_recv); + if (fast_recv) { + if (fast_recv(skb)) { + ret = NET_RX_SUCCESS; + goto out; + } + } +#endif + if (skb_skip_tc_classify(skb)) goto skip_classify;