Compiler Explorer

Source code

// SPDX-License-Identifier: GPL-2.0

/* Code from Ethernet driver igc, modified for godbolt.org usage
 *
 * ASM reading hint:
 *  func args for x86_64 are passed as registers in following order
 *
 *   %rdi - 1st argument
 *   %rsi - 2nd argument
 *   %rdx  -3rd
 *   %rcx, %r8, %r9
 */

/* Intel ASM code generated by godbolt:

igc_rx_hash:
        mov     rax, rdi
        mov     rdi, rdx
        mov     rax, QWORD PTR [rax]
        test    BYTE PTR [rax+20], -128
        jne     .L4
        ret
.L4:
        mov     eax, DWORD PTR [rsi]
        mov     esi, DWORD PTR [rsi+4]
        and     eax, 15
        mov     edx, DWORD PTR igc_rss_type_table[0+rax*4]
        jmp     skb_set_hash
igc_rss_type_table:
        .long   1
        .long   3
        .long   2
        .long   3
        .long   2
        .long   2
        .long   3
        .long   3
        .long   3
        .long   3
        .long   1
        .long   1
        .long   1
        .long   1
        .long   1
        .long   1

#include <asm/types.h>

#if __BYTE_ORDER == __LITTLE_ENDIAN
typedef __u64 netdev_features_t;
typedef __u64 __le64;
typedef __u32 __le32;
typedef __u16 __le16;
typedef __u8 u8;
typedef __u32 u32;
#define le32_to_cpu	/* Assume Intel Little-Endian*/
#endif

enum {
	NETIF_F_SG_BIT,			/* Scatter/gather IO. */
	NETIF_F_IP_CSUM_BIT,		/* Can checksum TCP/UDP over IPv4. */
	__UNUSED_NETIF_F_1,
	NETIF_F_HW_CSUM_BIT,		/* Can checksum all the packets. */
	NETIF_F_IPV6_CSUM_BIT,		/* Can checksum TCP/UDP over IPV6 */
	NETIF_F_HIGHDMA_BIT,		/* Can DMA to high memory. */
	NETIF_F_FRAGLIST_BIT,		/* Scatter/gather IO. */
	NETIF_F_HW_VLAN_CTAG_TX_BIT,	/* Transmit VLAN CTAG HW acceleration */
	NETIF_F_HW_VLAN_CTAG_RX_BIT,	/* Receive VLAN CTAG HW acceleration */
	NETIF_F_HW_VLAN_CTAG_FILTER_BIT,/* Receive filtering on VLAN CTAGs */
	NETIF_F_VLAN_CHALLENGED_BIT,	/* Device cannot handle VLAN packets */
	NETIF_F_GSO_BIT,		/* Enable software GSO. */
	NETIF_F_LLTX_BIT,		/* LockLess TX - deprecated. Please */
					/* do not use LLTX in new drivers */
	NETIF_F_NETNS_LOCAL_BIT,	/* Does not change network namespaces */
	NETIF_F_GRO_BIT,		/* Generic receive offload */
	NETIF_F_LRO_BIT,		/* large receive offload */

/**/NETIF_F_GSO_SHIFT,		/* keep the order of SKB_GSO_* bits */
	NETIF_F_TSO_BIT			/* ... TCPv4 segmentation */
		= NETIF_F_GSO_SHIFT,
	NETIF_F_GSO_ROBUST_BIT,		/* ... ->SKB_GSO_DODGY */
	NETIF_F_TSO_ECN_BIT,		/* ... TCP ECN support */
	NETIF_F_TSO_MANGLEID_BIT,	/* ... IPV4 ID mangling allowed */
	NETIF_F_TSO6_BIT,		/* ... TCPv6 segmentation */
	NETIF_F_FSO_BIT,		/* ... FCoE segmentation */
	NETIF_F_GSO_GRE_BIT,		/* ... GRE with TSO */
	NETIF_F_GSO_GRE_CSUM_BIT,	/* ... GRE with csum with TSO */
	NETIF_F_GSO_IPXIP4_BIT,		/* ... IP4 or IP6 over IP4 with TSO */
	NETIF_F_GSO_IPXIP6_BIT,		/* ... IP4 or IP6 over IP6 with TSO */
	NETIF_F_GSO_UDP_TUNNEL_BIT,	/* ... UDP TUNNEL with TSO */
	NETIF_F_GSO_UDP_TUNNEL_CSUM_BIT,/* ... UDP TUNNEL with TSO & CSUM */
	NETIF_F_GSO_PARTIAL_BIT,	/* ... Only segment inner-most L4
					 *     in hardware and all other
					 *     headers in software.
					 */
	NETIF_F_GSO_TUNNEL_REMCSUM_BIT, /* ... TUNNEL with TSO & REMCSUM */
	NETIF_F_GSO_SCTP_BIT,		/* ... SCTP fragmentation */
	NETIF_F_GSO_ESP_BIT,		/* ... ESP with TSO */
	NETIF_F_GSO_UDP_BIT,		/* ... UFO, deprecated except tuntap */
	NETIF_F_GSO_UDP_L4_BIT,		/* ... UDP payload GSO (not UFO) */
	NETIF_F_GSO_FRAGLIST_BIT,		/* ... Fraglist GSO */
	/**/NETIF_F_GSO_LAST =		/* last bit, see GSO_MASK */
		NETIF_F_GSO_FRAGLIST_BIT,

NETIF_F_FCOE_CRC_BIT,		/* FCoE CRC32 */
	NETIF_F_SCTP_CRC_BIT,		/* SCTP checksum offload */
	NETIF_F_FCOE_MTU_BIT,		/* Supports max FCoE MTU, 2158 bytes*/
	NETIF_F_NTUPLE_BIT,		/* N-tuple filters supported */
	NETIF_F_RXHASH_BIT,		/* Receive hashing offload */
	NETIF_F_RXCSUM_BIT,		/* Receive checksumming offload */
	NETIF_F_NOCACHE_COPY_BIT,	/* Use no-cache copyfromuser */
	NETIF_F_LOOPBACK_BIT,		/* Enable loopback */
	NETIF_F_RXFCS_BIT,		/* Append FCS to skb pkt data */
	NETIF_F_RXALL_BIT,		/* Receive errored frames too */
	NETIF_F_HW_VLAN_STAG_TX_BIT,	/* Transmit VLAN STAG HW acceleration */
	NETIF_F_HW_VLAN_STAG_RX_BIT,	/* Receive VLAN STAG HW acceleration */
	NETIF_F_HW_VLAN_STAG_FILTER_BIT,/* Receive filtering on VLAN STAGs */
	NETIF_F_HW_L2FW_DOFFLOAD_BIT,	/* Allow L2 Forwarding in Hardware */

NETIF_F_HW_TC_BIT,		/* Offload TC infrastructure */
	NETIF_F_HW_ESP_BIT,		/* Hardware ESP transformation offload */
	NETIF_F_HW_ESP_TX_CSUM_BIT,	/* ESP with TX checksum offload */
	NETIF_F_RX_UDP_TUNNEL_PORT_BIT, /* Offload of RX port for UDP tunnels */
	NETIF_F_HW_TLS_TX_BIT,		/* Hardware TLS TX offload */
	NETIF_F_HW_TLS_RX_BIT,		/* Hardware TLS RX offload */

NETIF_F_GRO_HW_BIT,		/* Hardware Generic receive offload */
	NETIF_F_HW_TLS_RECORD_BIT,	/* Offload TLS record */
	NETIF_F_GRO_FRAGLIST_BIT,	/* Fraglist GRO */

NETIF_F_HW_MACSEC_BIT,		/* Offload MACsec operations */
	NETIF_F_GRO_UDP_FWD_BIT,	/* Allow UDP GRO for forwarding */

NETIF_F_HW_HSR_TAG_INS_BIT,	/* Offload HSR tag insertion */
	NETIF_F_HW_HSR_TAG_RM_BIT,	/* Offload HSR tag removal */
	NETIF_F_HW_HSR_FWD_BIT,		/* Offload HSR forwarding */
	NETIF_F_HW_HSR_DUP_BIT,		/* Offload HSR duplication */

/*
	 * Add your fresh new feature above and remember to update
	 * netdev_features_strings[] in net/ethtool/common.c and maybe
	 * some feature mask #defines below. Please also describe it
	 * in Documentation/networking/netdev-features.rst.
	 */

/**/NETDEV_FEATURE_COUNT
};

/* copy'n'paste compression ;) */
#define __NETIF_F_BIT(bit)	((netdev_features_t)1 << (bit))
#define __NETIF_F(name)		__NETIF_F_BIT(NETIF_F_##name##_BIT)

#define NETIF_F_RXHASH		__NETIF_F(RXHASH)

enum pkt_hash_types {
	PKT_HASH_TYPE_NONE,	/* Undefined type */
	PKT_HASH_TYPE_L2,	/* Input: src_MAC, dest_MAC */
	PKT_HASH_TYPE_L3,	/* Input: src_IP, dst_IP */
	PKT_HASH_TYPE_L4,	/* Input: src_IP, dst_IP, src_port, dst_port */
};

struct sk_buff;
void skb_set_hash(struct sk_buff *skb, __u32 hash, enum pkt_hash_types type);

/* Receive Descriptor - Advanced */
union igc_adv_rx_desc {
	struct {
		__le64 pkt_addr; /* Packet buffer address */
		__le64 hdr_addr; /* Header buffer address */
	} read;
	struct {
		struct {
			union {
				__le32 data;
				struct {
					__le16 pkt_info; /*RSS type, Pkt type*/
					/* Split Header, header buffer len */
					__le16 hdr_info;
				} hs_rss;
			} lo_dword;
			union {
				__le32 rss; /* RSS Hash */
				struct {
					__le16 ip_id; /* IP id */
					__le16 csum; /* Packet Checksum */
				} csum_ip;
			} hi_dword;
		} lower;
		struct {
			__le32 status_error; /* ext status/error */
			__le16 length; /* Packet length */
			__le16 vlan; /* VLAN tag */
		} upper;
	} wb;  /* writeback */
};

struct net_device {
	char			name[16];
	netdev_features_t	features;
};

struct igc_ring {
	struct net_device *netdev;      /* back pointer to net_device */
};

/* RX-desc Write-Back format RSS Type's */
enum igc_rss_type_num {
	IGC_RSS_TYPE_NO_HASH		= 0,
	IGC_RSS_TYPE_HASH_TCP_IPV4	= 1,
	IGC_RSS_TYPE_HASH_IPV4		= 2,
	IGC_RSS_TYPE_HASH_TCP_IPV6	= 3,
	IGC_RSS_TYPE_HASH_IPV6_EX	= 4,
	IGC_RSS_TYPE_HASH_IPV6		= 5,
	IGC_RSS_TYPE_HASH_TCP_IPV6_EX	= 6,
	IGC_RSS_TYPE_HASH_UDP_IPV4	= 7,
	IGC_RSS_TYPE_HASH_UDP_IPV6	= 8,
	IGC_RSS_TYPE_HASH_UDP_IPV6_EX	= 9,
	IGC_RSS_TYPE_MAX		= 10,
};
#define IGC_RSS_TYPE_MAX_TABLE		16
#define IGC_RSS_TYPE_MASK		0xF

/* igc_rss_type - Rx descriptor RSS type field */
static inline u32 igc_rss_type(union igc_adv_rx_desc *rx_desc)
{
	/* RSS Type 4-bit number: 0-9 (above 9 is reserved)
	 * Accessing the same bits via u16 (wb.lower.lo_dword.hs_rss.pkt_info)
	 * is slightly slower than via u32 (wb.lower.lo_dword.data)
	 */
	return le32_to_cpu(rx_desc->wb.lower.lo_dword.data) & IGC_RSS_TYPE_MASK;
}

/* Mapping HW RSS Type to enum pkt_hash_types */
struct igc_rss_type {
	u32 hash_type; /* can contain enum pkt_hash_types */
} igc_rss_type_table[IGC_RSS_TYPE_MAX_TABLE] = {
	[IGC_RSS_TYPE_NO_HASH].hash_type	  = PKT_HASH_TYPE_L2,
	[IGC_RSS_TYPE_HASH_TCP_IPV4].hash_type	  = PKT_HASH_TYPE_L4,
	[IGC_RSS_TYPE_HASH_IPV4].hash_type	  = PKT_HASH_TYPE_L3,
	[IGC_RSS_TYPE_HASH_TCP_IPV6].hash_type	  = PKT_HASH_TYPE_L4,
	[IGC_RSS_TYPE_HASH_IPV6_EX].hash_type	  = PKT_HASH_TYPE_L3,
	[IGC_RSS_TYPE_HASH_IPV6].hash_type	  = PKT_HASH_TYPE_L3,
	[IGC_RSS_TYPE_HASH_TCP_IPV6_EX].hash_type = PKT_HASH_TYPE_L4,
	[IGC_RSS_TYPE_HASH_UDP_IPV4].hash_type	  = PKT_HASH_TYPE_L4,
	[IGC_RSS_TYPE_HASH_UDP_IPV6].hash_type	  = PKT_HASH_TYPE_L4,
	[IGC_RSS_TYPE_HASH_UDP_IPV6_EX].hash_type = PKT_HASH_TYPE_L4,
	[10].hash_type = PKT_HASH_TYPE_L2, /* RSS Type above 9 "Reserved" by HW */
	[11].hash_type = PKT_HASH_TYPE_L2,
	[12].hash_type = PKT_HASH_TYPE_L2,
	[13].hash_type = PKT_HASH_TYPE_L2,
	[14].hash_type = PKT_HASH_TYPE_L2,
	[15].hash_type = PKT_HASH_TYPE_L2,
};

void igc_rx_hash(struct igc_ring *ring,
			       union igc_adv_rx_desc *rx_desc,
			       struct sk_buff *skb)
{
	if (ring->netdev->features & NETIF_F_RXHASH) {
		u32 rss_hash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
		u32 rss_type = igc_rss_type(rx_desc);
		enum pkt_hash_types hash_type;

hash_type = igc_rss_type_table[rss_type].hash_type;
		skb_set_hash(skb, rss_hash, hash_type);
	}
}