[Intel-wired-lan] [PATCH v3 06/11] igc: Add transmit and receive fastpath and interrupt handlers

Shannon Nelson shannon.nelson at oracle.com
Thu Jun 28 21:19:46 UTC 2018


On 6/24/2018 1:45 AM, Sasha Neftin wrote:
> This patch adds support for allocating, configuring, and freeing Tx/Rx ring
> resources.  With these changes in place the descriptor queues are in a
> state where they are ready to transmit or receive if provided buffers.
> 
> This also adds the transmit and receive fastpath and interrupt handlers.
> With this code in place the network device is now able to send and receive
> frames over the network interface using a single queue.
> 
> Sasha Neftin (v2):
> removed obsolete code
> 
> Sasha Neftin (v3):
> removed unused mac type
> code optimization, remome switch statement where it is not necessary
> resolve conflicts
> 
> Signed-off-by: Sasha Neftin <sasha.neftin at intel.com>
> ---
>   drivers/net/ethernet/intel/igc/e1000_base.h    |   15 +
>   drivers/net/ethernet/intel/igc/e1000_defines.h |   52 ++
>   drivers/net/ethernet/intel/igc/igc.h           |   68 +-
>   drivers/net/ethernet/intel/igc/igc_main.c      | 1172 ++++++++++++++++++++++--
>   4 files changed, 1252 insertions(+), 55 deletions(-)
> 
> diff --git a/drivers/net/ethernet/intel/igc/e1000_base.h b/drivers/net/ethernet/intel/igc/e1000_base.h
> index 9acc41d73f4f..5c766fb7514b 100644
> --- a/drivers/net/ethernet/intel/igc/e1000_base.h
> +++ b/drivers/net/ethernet/intel/igc/e1000_base.h
> @@ -21,6 +21,18 @@ union e1000_adv_tx_desc {
>   	} wb;
>   };
>   
> +/* Adv Transmit Descriptor Config Masks */
> +#define E1000_ADVTXD_MAC_TSTAMP	0x00080000 /* IEEE1588 Timestamp packet */
> +#define E1000_ADVTXD_DTYP_CTXT	0x00200000 /* Advanced Context Descriptor */
> +#define E1000_ADVTXD_DTYP_DATA	0x00300000 /* Advanced Data Descriptor */
> +#define E1000_ADVTXD_DCMD_EOP	0x01000000 /* End of Packet */
> +#define E1000_ADVTXD_DCMD_IFCS	0x02000000 /* Insert FCS (Ethernet CRC) */
> +#define E1000_ADVTXD_DCMD_RS	0x08000000 /* Report Status */
> +#define E1000_ADVTXD_DCMD_DEXT	0x20000000 /* Descriptor extension (1=Adv) */
> +#define E1000_ADVTXD_DCMD_VLE	0x40000000 /* VLAN pkt enable */
> +#define E1000_ADVTXD_DCMD_TSE	0x80000000 /* TCP Seg enable */
> +#define E1000_ADVTXD_PAYLEN_SHIFT	14 /* Adv desc PAYLEN shift */
> +
>   struct e1000_adv_data_desc {
>   	__le64 buffer_addr;    /* Address of the descriptor's data buffer */
>   	union {
> @@ -75,6 +87,9 @@ union e1000_adv_rx_desc {
>   	} wb;  /* writeback */
>   };
>   
> +/* Adv Transmit Descriptor Config Masks */
> +#define E1000_ADVTXD_PAYLEN_SHIFT	14 /* Adv desc PAYLEN shift */
> +
>   /* Additional Transmit Descriptor Control definitions */
>   #define E1000_TXDCTL_QUEUE_ENABLE	0x02000000 /* Ena specific Tx Queue */
>   
> diff --git a/drivers/net/ethernet/intel/igc/e1000_defines.h b/drivers/net/ethernet/intel/igc/e1000_defines.h
> index 66f8fc96dfb8..f39d93d17ba6 100644
> --- a/drivers/net/ethernet/intel/igc/e1000_defines.h
> +++ b/drivers/net/ethernet/intel/igc/e1000_defines.h
> @@ -101,6 +101,29 @@
>   #define E1000_GPIE_EIAME	0x40000000
>   #define E1000_GPIE_PBA		0x80000000
>   
> +/* Transmit Descriptor bit definitions */
> +#define E1000_TXD_DTYP_D	0x00100000 /* Data Descriptor */
> +#define E1000_TXD_DTYP_C	0x00000000 /* Context Descriptor */
> +#define E1000_TXD_POPTS_IXSM	0x01       /* Insert IP checksum */
> +#define E1000_TXD_POPTS_TXSM	0x02       /* Insert TCP/UDP checksum */
> +#define E1000_TXD_CMD_EOP	0x01000000 /* End of Packet */
> +#define E1000_TXD_CMD_IFCS	0x02000000 /* Insert FCS (Ethernet CRC) */
> +#define E1000_TXD_CMD_IC	0x04000000 /* Insert Checksum */
> +#define E1000_TXD_CMD_RS	0x08000000 /* Report Status */
> +#define E1000_TXD_CMD_RPS	0x10000000 /* Report Packet Sent */
> +#define E1000_TXD_CMD_DEXT	0x20000000 /* Desc extension (0 = legacy) */
> +#define E1000_TXD_CMD_VLE	0x40000000 /* Add VLAN tag */
> +#define E1000_TXD_CMD_IDE	0x80000000 /* Enable Tidv register */
> +#define E1000_TXD_STAT_DD	0x00000001 /* Descriptor Done */
> +#define E1000_TXD_STAT_EC	0x00000002 /* Excess Collisions */
> +#define E1000_TXD_STAT_LC	0x00000004 /* Late Collisions */
> +#define E1000_TXD_STAT_TU	0x00000008 /* Transmit underrun */
> +#define E1000_TXD_CMD_TCP	0x01000000 /* TCP packet */
> +#define E1000_TXD_CMD_IP	0x02000000 /* IP packet */
> +#define E1000_TXD_CMD_TSE	0x04000000 /* TCP Seg enable */
> +#define E1000_TXD_STAT_TC	0x00000004 /* Tx Underrun */
> +#define E1000_TXD_EXTCMD_TSTAMP	0x00000010 /* IEEE1588 Timestamp packet */
> +
>   /* Transmit Control */
>   #define E1000_TCTL_EN		0x00000002 /* enable Tx */
>   #define E1000_TCTL_PSP		0x00000008 /* pad short packets */
> @@ -130,10 +153,39 @@
>   #define E1000_RCTL_RDMTS_HALF	0x00000000 /* Rx desc min thresh size */
>   #define E1000_RCTL_BAM		0x00008000 /* broadcast enable */
>   
> +/* Receive Descriptor bit definitions */
> +#define E1000_RXD_STAT_DD	0x01    /* Descriptor Done */
> +#define E1000_RXD_STAT_EOP	0x02    /* End of Packet */
> +#define E1000_RXD_STAT_IXSM	0x04    /* Ignore checksum */
> +#define E1000_RXD_STAT_VP	0x08    /* IEEE VLAN Packet */
> +#define E1000_RXD_STAT_UDPCS	0x10    /* UDP xsum calculated */
> +#define E1000_RXD_STAT_TCPCS	0x20    /* TCP xsum calculated */
> +#define E1000_RXD_STAT_TS	0x10000 /* Pkt was time stamped */
> +
> +#define E1000_RXDEXT_STATERR_LB		0x00040000
> +#define E1000_RXDEXT_STATERR_CE		0x01000000
> +#define E1000_RXDEXT_STATERR_SE		0x02000000
> +#define E1000_RXDEXT_STATERR_SEQ	0x04000000
> +#define E1000_RXDEXT_STATERR_CXE	0x10000000
> +#define E1000_RXDEXT_STATERR_TCPE	0x20000000
> +#define E1000_RXDEXT_STATERR_IPE	0x40000000
> +#define E1000_RXDEXT_STATERR_RXE	0x80000000
> +
> +/* Same mask, but for extended and packet split descriptors */
> +#define E1000_RXDEXT_ERR_FRAME_ERR_MASK ( \
> +	E1000_RXDEXT_STATERR_CE  |            \
> +	E1000_RXDEXT_STATERR_SE  |            \
> +	E1000_RXDEXT_STATERR_SEQ |            \
> +	E1000_RXDEXT_STATERR_CXE |            \
> +	E1000_RXDEXT_STATERR_RXE)
> +
>   /* Header split receive */
>   #define E1000_RFCTL_IPV6_EX_DIS	0x00010000
>   #define E1000_RFCTL_LEF		0x00040000
>   
> +#define I225_RXPBSIZE_DEFAULT	0x000000A2 /* RXPBSIZE default */
> +#define I225_TXPBSIZE_DEFAULT	0x04000014 /* TXPBSIZE default */
> +
>   /* these buffer sizes are valid if E1000_RCTL_BSEX is 0 */
>   #define E1000_RCTL_SZ_2048	0x00000000 /* Rx buffer size 2048 */
>   #define E1000_RCTL_SZ_1024	0x00010000 /* Rx buffer size 1024 */
> diff --git a/drivers/net/ethernet/intel/igc/igc.h b/drivers/net/ethernet/intel/igc/igc.h
> index abf2e302c417..c61212ccb60e 100644
> --- a/drivers/net/ethernet/intel/igc/igc.h
> +++ b/drivers/net/ethernet/intel/igc/igc.h
> @@ -32,18 +32,36 @@ extern char igc_driver_version[];
>   #define IGC_START_ITR			648 /* ~6000 ints/sec */
>   #define IGC_FLAG_HAS_MSI		BIT(0)
>   #define IGC_FLAG_QUEUE_PAIRS		BIT(4)
> +#define IGC_FLAG_NEED_LINK_UPDATE	BIT(9)
>   #define IGC_FLAG_HAS_MSIX		BIT(13)
> +#define IGC_FLAG_VLAN_PROMISC		BIT(15)
>   
>   #define IGC_START_ITR			648 /* ~6000 ints/sec */
>   #define IGC_4K_ITR			980
>   #define IGC_20K_ITR			196
>   #define IGC_70K_ITR			56
>   
> +#define IGC_DEFAULT_ITR		3 /* dynamic */
> +#define IGC_MAX_ITR_USECS	10000
> +#define IGC_MIN_ITR_USECS	10
> +#define NON_Q_VECTORS		1
> +#define MAX_Q_VECTORS		8
> +#define MAX_MSIX_ENTRIES	10
> +
> +/* TX/RX descriptor defines */
> +#define IGC_DEFAULT_TXD		256
> +#define IGC_DEFAULT_TX_WORK	128
> +#define IGC_MIN_TXD		80
> +#define IGC_MAX_TXD		4096
> +
> +#define IGC_DEFAULT_RXD		256
> +#define IGC_MIN_RXD		80
> +#define IGC_MAX_RXD		4096
> +
>   /* Transmit and receive queues */
>   #define IGC_MAX_RX_QUEUES                 4
>   #define IGC_MAX_TX_QUEUES                 4
>   
> -#define MAX_Q_VECTORS                     10
>   #define MAX_STD_JUMBO_FRAME_SIZE        9216
>   
>   #define IGC_TX_PTHRESH			8
> @@ -87,6 +105,16 @@ extern char igc_driver_version[];
>   #define IGC_MAX_FRAME_BUILD_SKB (IGC_RXBUFFER_2048 - IGC_TS_HDR_LEN)
>   #endif
>   
> +/* How many Rx Buffers do we bundle into one write to the hardware ? */
> +#define IGC_RX_BUFFER_WRITE	16 /* Must be power of 2 */
> +
> +/* igc_test_staterr - tests bits within Rx descriptor status and error fields */
> +static inline __le32 igc_test_staterr(union e1000_adv_rx_desc *rx_desc,
> +				      const u32 stat_err_bits)
> +{
> +	return rx_desc->wb.upper.status_error & cpu_to_le32(stat_err_bits);
> +}
> +
>   enum e1000_state_t {
>   	 __IGC_TESTING,
>   	__IGC_RESETTING,
> @@ -94,6 +122,27 @@ enum e1000_state_t {
>   	 __IGC_PTP_TX_IN_PROGRESS,
>   };
>   
> +enum igc_tx_flags {
> +	/* cmd_type flags */
> +	IGC_TX_FLAGS_VLAN       = 0x01,
> +	IGC_TX_FLAGS_TSO        = 0x02,
> +	IGC_TX_FLAGS_TSTAMP     = 0x04,
> +
> +	/* olinfo flags */
> +	IGC_TX_FLAGS_IPV4       = 0x10,
> +	IGC_TX_FLAGS_CSUM       = 0x20,
> +};
> +
> +/** The largest size we can write to the descriptor is 65535.  In order to
> + * maintain a power of two alignment we have to limit ourselves to 32K.
> + **/

Don't ** these comments

> +#define IGC_MAX_TXD_PWR		15
> +#define IGC_MAX_DATA_PER_TXD	BIT(IGC_MAX_TXD_PWR)
> +
> +/* Tx Descriptors needed, worst case */
> +#define TXD_USE_COUNT(S)	DIV_ROUND_UP((S), IGC_MAX_DATA_PER_TXD)
> +#define DESC_NEEDED	(MAX_SKB_FRAGS + 4)
> +
>   /** wrapper around a pointer to a socket buffer,
>    *  so a DMA handle can be stored along with the buffer
>    **/
> @@ -125,6 +174,7 @@ struct igc_tx_queue_stats {
>   	u64 packets;
>   	u64 bytes;
>   	u64 restart_queue;
> +	u64 restart_queue2;
>   };
>   
>   struct igc_rx_queue_stats {
> @@ -183,11 +233,14 @@ struct igc_ring {
>   		/* TX */
>   		struct {
>   			struct igc_tx_queue_stats tx_stats;
> +			struct u64_stats_sync tx_syncp;
> +			struct u64_stats_sync tx_syncp2;
>   		};
>   		/* RX */
>   		struct {
>   			struct igc_rx_queue_stats rx_stats;
>   			struct igc_rx_packet_stats pkt_stats;
> +			struct u64_stats_sync rx_syncp;
>   #ifdef CONFIG_IGC_DISABLE_PACKET_SPLIT
>   			u16 rx_buffer_len;
>   #else
> @@ -264,11 +317,17 @@ struct igc_adapter {
>   	struct work_struct watchdog_task;
>   	struct work_struct dma_err_task;
>   
> +	u8  tx_timeout_factor;
> +
>   	int msg_enable;
>   	u32 max_frame_size;
> +	u32 min_frame_size;
>   
>   	/* OS defined structs */
>   	struct pci_dev *pdev;
> +	/* lock for statistics */
> +	spinlock_t stats64_lock;
> +	struct rtnl_link_stats64 stats64;
>   
>   	/* structs defined in e1000_hw.h */
>   	struct e1000_hw hw;
> @@ -281,8 +340,13 @@ struct igc_adapter {
>   	u16 tx_ring_count;
>   	u16 rx_ring_count;
>   
> +	u32 *shadow_vfta;
> +
>   	u32 rss_queues;
>   
> +	/* lock for RX network flow classification filter */
> +	spinlock_t nfc_lock;
> +
>   	struct igc_mac_addr *mac_table;
>   };
>   
> @@ -338,6 +402,8 @@ static inline unsigned int igc_rx_pg_order(struct igc_ring *ring)
>   
>   #define igc_rx_pg_size(_ring) (PAGE_SIZE << igc_rx_pg_order(_ring))
>   
> +#define IGC_TXD_DCMD	(E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_RS)
> +
>   #define IGC_RX_DESC(R, i)       \
>   	(&(((union e1000_adv_rx_desc *)((R)->desc))[i]))
>   #define IGC_TX_DESC(R, i)       \
> diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c
> index a147a1b7585e..67826041eb3c 100644
> --- a/drivers/net/ethernet/intel/igc/igc_main.c
> +++ b/drivers/net/ethernet/intel/igc/igc_main.c
> @@ -37,9 +37,13 @@ static int igc_sw_init(struct igc_adapter *);
>   static void igc_configure(struct igc_adapter *adapter);
>   static void igc_configure_tx(struct igc_adapter *);
>   static void igc_configure_rx(struct igc_adapter *adapter);
> +static void igc_clean_all_tx_rings(struct igc_adapter *);
> +static void igc_clean_all_rx_rings(struct igc_adapter *);
>   static void igc_power_down_link(struct igc_adapter *adapter);
>   static void igc_set_default_mac_filter(struct igc_adapter *adapter);
>   static void igc_set_rx_mode(struct net_device *netdev);
> +static netdev_tx_t igc_xmit_frame(struct sk_buff *skb,
> +				  struct net_device *netdev);
>   static void igc_setup_mrqc(struct igc_adapter *adapter);
>   static irqreturn_t igc_msix_ring(int irq, void *data);
>   static irqreturn_t igc_intr_msi(int irq, void *data);
> @@ -51,8 +55,11 @@ static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx);
>   static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix);
>   static int igc_alloc_q_vectors(struct igc_adapter *adapter);
>   static int igc_poll(struct napi_struct *napi, int budget);
> +static bool igc_clean_tx_irq(struct igc_q_vector *, int);
> +static int igc_clean_rx_irq(struct igc_q_vector *, int);
>   static void igc_set_interrupt_capability(struct igc_adapter *adapter,
>   					 bool msix);
> +static void igc_reset_task(struct work_struct *);
>   static void igc_reset_interrupt_capability(struct igc_adapter *adapter);
>   static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx);
>   static void igc_clear_interrupt_scheme(struct igc_adapter *adapter);
> @@ -66,6 +73,9 @@ static void igc_set_itr(struct igc_q_vector *q_vector);
>   static void igc_update_ring_itr(struct igc_q_vector *q_vector);
>   static void igc_update_itr(struct igc_q_vector *q_vector,
>   			   struct igc_ring_container *ring_container);
> +static void igc_nfc_filter_exit(struct igc_adapter *adapter);
> +static bool igc_alloc_mapped_page(struct igc_ring *rx_ring,
> +				  struct igc_rx_buffer *bi);
>   
>   enum latency_range {
>   	lowest_latency = 0,
> @@ -225,6 +235,19 @@ static void igc_clean_tx_ring(struct igc_ring *tx_ring)
>   }
>   
>   /**
> + *  igc_clean_all_tx_rings - Free Tx Buffers for all queues
> + *  @adapter: board private structure
> + **/
> +static void igc_clean_all_tx_rings(struct igc_adapter *adapter)
> +{
> +	int i;
> +
> +	for (i = 0; i < adapter->num_tx_queues; i++)
> +		if (adapter->tx_ring[i])
> +			igc_clean_tx_ring(adapter->tx_ring[i]);
> +}
> +
> +/**
>    *  igc_setup_tx_resources - allocate Tx resources (Descriptors)
>    *  @tx_ring: tx descriptor ring (for a specific queue) to setup
>    *
> @@ -333,6 +356,19 @@ void igc_clean_rx_ring(struct igc_ring *rx_ring)
>   }
>   
>   /**
> + *  igc_clean_all_rx_rings - Free Rx Buffers for all queues
> + *  @adapter: board private structure
> + **/
> +static void igc_clean_all_rx_rings(struct igc_adapter *adapter)
> +{
> +	int i;
> +
> +	for (i = 0; i < adapter->num_rx_queues; i++)
> +		if (adapter->rx_ring[i])
> +			igc_clean_rx_ring(adapter->rx_ring[i]);
> +}
> +
> +/**
>    *  igc_free_rx_resources - Free Rx Resources
>    *  @rx_ring: ring to clean the resources from
>    *
> @@ -679,60 +715,633 @@ static int igc_set_mac(struct net_device *netdev, void *p)
>   	return 0;
>   }
>   
> +static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first)
> +{
> +}
> +
> +static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size)
> +{
> +	struct net_device *netdev = tx_ring->netdev;
> +
> +	netif_stop_subqueue(netdev, tx_ring->queue_index);
> +
> +	/* Herbert's original patch had:
> +	 *  smp_mb__after_netif_stop_queue();
> +	 * but since that doesn't exist yet, just open code it.
> +	 */
> +	smp_mb();

"Herbert's original patch" more cut-n-paste stuff that should go away?

> +
> +	/* We need to check again in a case another CPU has just
> +	 * made room available.
> +	 */
> +	if (igc_desc_unused(tx_ring) < size)
> +		return -EBUSY;
> +
> +	/* A reprieve! */
> +	netif_wake_subqueue(netdev, tx_ring->queue_index);
> +
> +	u64_stats_update_begin(&tx_ring->tx_syncp2);
> +	tx_ring->tx_stats.restart_queue2++;
> +	u64_stats_update_end(&tx_ring->tx_syncp2);
> +
> +	return 0;
> +}
> +
> +static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size)
> +{
> +	if (igc_desc_unused(tx_ring) >= size)
> +		return 0;
> +	return __igc_maybe_stop_tx(tx_ring, size);
> +}
> +
> +/**#define IGC_SET_FLAG(_input, _flag, _result) \
> + *	((_flag <= _result) ? \
> + *	((u32)(_input & _flag) * (_result / _flag)) : \
> + *	((u32)(_input & _flag) / (_flag / _result)))
> + **/

Remove commented out code

> +
> +static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)
> +{
> +	/* set type for advanced descriptor with frame checksum insertion */
> +	u32 cmd_type = E1000_ADVTXD_DTYP_DATA |
> +		       E1000_ADVTXD_DCMD_DEXT |
> +		       E1000_ADVTXD_DCMD_IFCS;
> +
> +	return cmd_type;
> +}
> +
> +static void igc_tx_olinfo_status(struct igc_ring *tx_ring,
> +				 union e1000_adv_tx_desc *tx_desc,
> +				 u32 tx_flags, unsigned int paylen)
> +{
> +	u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
> +
> +	/* insert L4 checksum */
> +	if (IGC_TX_FLAGS_CSUM <= (E1000_TXD_POPTS_TXSM << 8))

Since both IGC_TX_FLAGS_CSUM and E1000_TXD_POPTS_TXSM are #defined 
constants, how is this supposed to ever be a useful if/else expression?

> +		olinfo_status |= (tx_flags & IGC_TX_FLAGS_CSUM) *
> +				  ((E1000_TXD_POPTS_TXSM << 8) /
> +				  IGC_TX_FLAGS_CSUM);
> +	else
> +		olinfo_status |= (tx_flags & IGC_TX_FLAGS_CSUM) *
> +				  (IGC_TX_FLAGS_CSUM /
> +				  (E1000_TXD_POPTS_TXSM << 8));
> +
> +	/* insert IPv4 checksum */
> +	if (IGC_TX_FLAGS_IPV4 <= (E1000_TXD_POPTS_IXSM << 8))

Same question... why bother with if/else with constant expression?

> +		olinfo_status |= (tx_flags & IGC_TX_FLAGS_IPV4) *
> +				  (((E1000_TXD_POPTS_IXSM << 8)) /
> +				  IGC_TX_FLAGS_IPV4);
> +	else
> +		olinfo_status |= (tx_flags & IGC_TX_FLAGS_IPV4) *
> +				  (IGC_TX_FLAGS_IPV4 /
> +				  (E1000_TXD_POPTS_IXSM << 8));
> +
> +	tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
> +}
> +
> +static int igc_tx_map(struct igc_ring *tx_ring,
> +		      struct igc_tx_buffer *first,
> +		      const u8 hdr_len)
> +{
> +	struct sk_buff *skb = first->skb;
> +	struct igc_tx_buffer *tx_buffer;
> +	union e1000_adv_tx_desc *tx_desc;
> +	struct skb_frag_struct *frag;
> +	dma_addr_t dma;
> +	unsigned int data_len, size;
> +	u32 tx_flags = first->tx_flags;
> +	u32 cmd_type = igc_tx_cmd_type(skb, tx_flags);
> +	u16 i = tx_ring->next_to_use;

reverse xmas tree

> +
> +	tx_desc = IGC_TX_DESC(tx_ring, i);
> +
> +	igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len);
> +
> +	size = skb_headlen(skb);
> +	data_len = skb->data_len;
> +
> +	dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
> +
> +	tx_buffer = first;
> +
> +	for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
> +		if (dma_mapping_error(tx_ring->dev, dma))
> +			goto dma_error;
> +
> +		/* record length, and DMA address */
> +		dma_unmap_len_set(tx_buffer, len, size);
> +		dma_unmap_addr_set(tx_buffer, dma, dma);
> +
> +		tx_desc->read.buffer_addr = cpu_to_le64(dma);
> +
> +		while (unlikely(size > IGC_MAX_DATA_PER_TXD)) {
> +			tx_desc->read.cmd_type_len =
> +				cpu_to_le32(cmd_type ^ IGC_MAX_DATA_PER_TXD);
> +
> +			i++;
> +			tx_desc++;
> +			if (i == tx_ring->count) {
> +				tx_desc = IGC_TX_DESC(tx_ring, 0);
> +				i = 0;
> +			}
> +			tx_desc->read.olinfo_status = 0;
> +
> +			dma += IGC_MAX_DATA_PER_TXD;
> +			size -= IGC_MAX_DATA_PER_TXD;
> +
> +			tx_desc->read.buffer_addr = cpu_to_le64(dma);
> +		}
> +
> +		if (likely(!data_len))
> +			break;
> +
> +		tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
> +
> +		i++;
> +		tx_desc++;
> +		if (i == tx_ring->count) {
> +			tx_desc = IGC_TX_DESC(tx_ring, 0);
> +			i = 0;
> +		}
> +		tx_desc->read.olinfo_status = 0;
> +
> +		size = skb_frag_size(frag);
> +		data_len -= size;
> +
> +		dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
> +				       size, DMA_TO_DEVICE);
> +
> +		tx_buffer = &tx_ring->tx_buffer_info[i];
> +	}
> +
> +	/* write last descriptor with RS and EOP bits */
> +	cmd_type |= size | IGC_TXD_DCMD;
> +	tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
> +
> +	netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
> +
> +	/* set the timestamp */
> +	first->time_stamp = jiffies;
> +
> +	/* Force memory writes to complete before letting h/w know there
> +	 * are new descriptors to fetch.  (Only applicable for weak-ordered
> +	 * memory model archs, such as IA-64).
> +	 *
> +	 * We also need this memory barrier to make certain all of the
> +	 * status bits have been updated before next_to_watch is written.
> +	 */
> +	/* comment */

comment?

> +	wmb();
> +
> +	/* set next_to_watch value indicating a packet is present */
> +	first->next_to_watch = tx_desc;
> +
> +	i++;
> +	if (i == tx_ring->count)
> +		i = 0;
> +
> +	tx_ring->next_to_use = i;
> +
> +	/* Make sure there is space in the ring for the next send. */
> +	igc_maybe_stop_tx(tx_ring, DESC_NEEDED);
> +
> +	if (netif_xmit_stopped(txring_txq(tx_ring)) || !skb->xmit_more) {
> +		writel(i, tx_ring->tail);
> +
> +		/* we need this if more than one processor can write to our tail
> +		 * at a time, it synchronizes IO on IA64/Altix systems
> +		 */
> +		mmiowb();
> +	}
> +
> +	return 0;
> +dma_error:
> +	dev_err(tx_ring->dev, "TX DMA map failed\n");
> +	tx_buffer = &tx_ring->tx_buffer_info[i];
> +
> +	/* clear dma mappings for failed tx_buffer_info map */
> +	while (tx_buffer != first) {
> +		if (dma_unmap_len(tx_buffer, len))
> +			dma_unmap_page(tx_ring->dev,
> +				       dma_unmap_addr(tx_buffer, dma),
> +				       dma_unmap_len(tx_buffer, len),
> +				       DMA_TO_DEVICE);
> +		dma_unmap_len_set(tx_buffer, len, 0);
> +
> +		if (i-- == 0)
> +			i += tx_ring->count;
> +		tx_buffer = &tx_ring->tx_buffer_info[i];
> +	}
> +
> +	if (dma_unmap_len(tx_buffer, len))
> +		dma_unmap_single(tx_ring->dev,
> +				 dma_unmap_addr(tx_buffer, dma),
> +				 dma_unmap_len(tx_buffer, len),
> +				 DMA_TO_DEVICE);
> +	dma_unmap_len_set(tx_buffer, len, 0);
> +
> +	dev_kfree_skb_any(tx_buffer->skb);
> +	tx_buffer->skb = NULL;
> +
> +	tx_ring->next_to_use = i;
> +
> +	return -1;
> +}
> +
> +netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb,
> +				struct igc_ring *tx_ring)
> +{
> +	struct igc_tx_buffer *first;
> +	u32 tx_flags = 0;
> +	unsigned short f;
> +	u16 count = TXD_USE_COUNT(skb_headlen(skb));
> +	__be16 protocol = vlan_get_protocol(skb);
> +	u8 hdr_len = 0;
> +
> +	/* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD,
> +	 *       + 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD,
> +	 *      + 2 desc gap to keep tail from touching head,
> +	 *       + 1 desc for context descriptor,
> +	 * otherwise try next time
> +	 */
> +	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
> +		count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
> +
> +	if (igc_maybe_stop_tx(tx_ring, count + 3)) {
> +		/* this is a hard error */
> +		return NETDEV_TX_BUSY;
> +	}
> +
> +	/* record the location of the first descriptor for this packet */
> +	first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
> +	first->skb = skb;
> +	first->bytecount = skb->len;
> +	first->gso_segs = 1;
> +
> +	skb_tx_timestamp(skb);
> +
> +	/* record initial flags and protocol */
> +	first->tx_flags = tx_flags;
> +	first->protocol = protocol;
> +
> +	igc_tx_csum(tx_ring, first);
> +
> +	igc_tx_map(tx_ring, first, hdr_len);
> +
> +	return NETDEV_TX_OK;
> +}
> +
> +static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter,
> +						    struct sk_buff *skb)
> +{
> +	unsigned int r_idx = skb->queue_mapping;
> +
> +	if (r_idx >= adapter->num_tx_queues)
> +		r_idx = r_idx % adapter->num_tx_queues;
> +
> +	return adapter->tx_ring[r_idx];
> +}
> +
>   static netdev_tx_t igc_xmit_frame(struct sk_buff *skb,
>   				  struct net_device *netdev)
>   {
> -	dev_kfree_skb_any(skb);
> -	return NETDEV_TX_OK;
> +	struct igc_adapter *adapter = netdev_priv(netdev);
> +
> +	/* The minimum packet size with TCTL.PSP set is 17 so pad the skb
> +	 * in order to meet this minimum size requirement.
> +	 */
> +	if (skb->len < 17) {
> +		if (skb_padto(skb, 17))
> +			return NETDEV_TX_OK;
> +		skb->len = 17;
> +	}
> +
> +	return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb));
>   }
>   
> -static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring)
> +static inline void igc_rx_hash(struct igc_ring *ring,
> +			       union e1000_adv_rx_desc *rx_desc,
> +			       struct sk_buff *skb)
>   {
> -	return ring_uses_build_skb(rx_ring) ? IGC_SKB_PAD : 0;
> +	if (ring->netdev->features & NETIF_F_RXHASH)
> +		skb_set_hash(skb,
> +			     le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
> +			     PKT_HASH_TYPE_L3);
>   }
>   
> -static bool igc_alloc_mapped_page(struct igc_ring *rx_ring,
> -				  struct igc_rx_buffer *bi)
> +/**
> + *  igc_process_skb_fields - Populate skb header fields from Rx descriptor
> + *  @rx_ring: rx descriptor ring packet is being transacted on
> + *  @rx_desc: pointer to the EOP Rx descriptor
> + *  @skb: pointer to current skb being populated
> + *
> + *  This function checks the ring, descriptor, and packet information in
> + *  order to populate the hash, checksum, VLAN, timestamp, protocol, and
> + *  other fields within the skb.
> + **/
> +static void igc_process_skb_fields(struct igc_ring *rx_ring,
> +				   union e1000_adv_rx_desc *rx_desc,
> +				   struct sk_buff *skb)
>   {
> -	struct page *page = bi->page;
> -	dma_addr_t dma;
> +	igc_rx_hash(rx_ring, rx_desc, skb);
>   
> -	/* since we are recycling buffers we should seldom need to alloc */
> -	if (likely(page))
> -		return true;
> +	skb_record_rx_queue(skb, rx_ring->queue_index);
>   
> -	/* alloc new page for storage */
> -	page = dev_alloc_pages(igc_rx_pg_order(rx_ring));
> -	if (unlikely(!page)) {
> -		rx_ring->rx_stats.alloc_failed++;
> -		return false;
> +	skb->protocol = eth_type_trans(skb, rx_ring->netdev);
> +}
> +
> +static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring,
> +					       const unsigned int size)
> +{
> +	struct igc_rx_buffer *rx_buffer;
> +
> +	rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
> +	prefetchw(rx_buffer->page);
> +
> +	/* we are reusing so sync this buffer for CPU use */
> +	dma_sync_single_range_for_cpu(rx_ring->dev,
> +				      rx_buffer->dma,
> +				      rx_buffer->page_offset,
> +				      size,
> +				      DMA_FROM_DEVICE);
> +
> +	rx_buffer->pagecnt_bias--;
> +
> +	return rx_buffer;
> +}
> +
> +/**
> + *  igc_add_rx_frag - Add contents of Rx buffer to sk_buff
> + *  @rx_ring: rx descriptor ring to transact packets on
> + *  @rx_buffer: buffer containing page to add
> + *  @skb: sk_buff to place the data into
> + *  @size: size of buffer to be added
> + *
> + *  This function will add the data contained in rx_buffer->page to the skb.
> + **/
> +static void igc_add_rx_frag(struct igc_ring *rx_ring,
> +			    struct igc_rx_buffer *rx_buffer,
> +			    struct sk_buff *skb,
> +			    unsigned int size)
> +{
> +#if (PAGE_SIZE < 8192)
> +	unsigned int truesize = igc_rx_pg_size(rx_ring) / 2;
> +#else
> +	unsigned int truesize = ring_uses_build_skb(rx_ring) ?
> +				SKB_DATA_ALIGN(IGC_SKB_PAD + size) :
> +				SKB_DATA_ALIGN(size);
> +#endif
> +	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
> +			rx_buffer->page_offset, size, truesize);
> +#if (PAGE_SIZE < 8192)
> +	rx_buffer->page_offset ^= truesize;
> +#else
> +	rx_buffer->page_offset += truesize;
> +#endif
> +}

This might look cleaner as
#if (PAGE_SIZE < 8192)
	unsigned int truesize = igc_rx_pg_size(rx_ring) / 2;
	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
			rx_buffer->page_offset, size, truesize);
	rx_buffer->page_offset ^= truesize;
#else
	unsigned int truesize = ring_uses_build_skb(rx_ring) ?
				SKB_DATA_ALIGN(IGC_SKB_PAD + size) :
				SKB_DATA_ALIGN(size);
	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
			rx_buffer->page_offset, size, truesize);
	rx_buffer->page_offset += truesize;
#endif


> +
> +static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring,
> +				     struct igc_rx_buffer *rx_buffer,
> +				     union e1000_adv_rx_desc *rx_desc,
> +				     unsigned int size)
> +{
> +	void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
> +#if (PAGE_SIZE < 8192)
> +	unsigned int truesize = igc_rx_pg_size(rx_ring) / 2;
> +#else
> +	unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
> +				SKB_DATA_ALIGN(IGC_SKB_PAD + size);
> +#endif
> +	struct sk_buff *skb;
> +
> +	/* prefetch first cache line of first page */
> +	prefetch(va);
> +#if L1_CACHE_BYTES < 128
> +	prefetch(va + L1_CACHE_BYTES);
> +#endif
> +
> +	/* build an skb around the page buffer */
> +	skb = build_skb(va - IGC_SKB_PAD, truesize);
> +	if (unlikely(!skb))
> +		return NULL;
> +
> +	/* update pointers within the skb to store the data */
> +	skb_reserve(skb, IGC_SKB_PAD);
> +	 __skb_put(skb, size);
> +
> +	/* update buffer offset */
> +#if (PAGE_SIZE < 8192)
> +	rx_buffer->page_offset ^= truesize;
> +#else
> +	rx_buffer->page_offset += truesize;
> +#endif
> +
> +	return skb;
> +}
> +
> +static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring,
> +					 struct igc_rx_buffer *rx_buffer,
> +					 union e1000_adv_rx_desc *rx_desc,
> +					 unsigned int size)
> +{
> +	void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
> +#if (PAGE_SIZE < 8192)
> +	unsigned int truesize = igc_rx_pg_size(rx_ring) / 2;
> +#else
> +	unsigned int truesize = SKB_DATA_ALIGN(size);
> +#endif
> +	unsigned int headlen;
> +	struct sk_buff *skb;
> +
> +	/* prefetch first cache line of first page */
> +	prefetch(va);
> +#if L1_CACHE_BYTES < 128
> +	prefetch(va + L1_CACHE_BYTES);
> +#endif
> +
> +	/* allocate a skb to store the frags */
> +	skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGC_RX_HDR_LEN);
> +	if (unlikely(!skb))
> +		return NULL;
> +
> +	/* Determine available headroom for copy */
> +	headlen = size;
> +	if (headlen > IGC_RX_HDR_LEN)
> +		headlen = eth_get_headlen(va, IGC_RX_HDR_LEN);
> +
> +	/* align pull length to size of long to optimize memcpy performance */
> +	memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long)));
> +
> +	/* update all of the pointers */
> +	size -= headlen;
> +	if (size) {
> +		skb_add_rx_frag(skb, 0, rx_buffer->page,
> +				(va + headlen) - page_address(rx_buffer->page),
> +				size, truesize);
> +#if (PAGE_SIZE < 8192)
> +	rx_buffer->page_offset ^= truesize;
> +#else
> +	rx_buffer->page_offset += truesize;
> +#endif
> +	} else {
> +		rx_buffer->pagecnt_bias++;
>   	}
>   
> -	/* map page for use */
> -	dma = dma_map_page_attrs(rx_ring->dev, page, 0,
> -				 igc_rx_pg_size(rx_ring),
> -				 DMA_FROM_DEVICE,
> -				 IGC_RX_DMA_ATTR);
> +	return skb;
> +}
>   
> -	/* if mapping failed free memory back to system since
> -	 * there isn't much point in holding memory we can't use
> +/**
> + *  igc_reuse_rx_page - page flip buffer and store it back on the ring
> + *  @rx_ring: rx descriptor ring to store buffers on
> + *  @old_buff: donor buffer to have page reused
> + *
> + *  Synchronizes page for reuse by the adapter
> + **/
> +static void igc_reuse_rx_page(struct igc_ring *rx_ring,
> +			      struct igc_rx_buffer *old_buff)
> +{
> +	struct igc_rx_buffer *new_buff;
> +	u16 nta = rx_ring->next_to_alloc;
> +
> +	new_buff = &rx_ring->rx_buffer_info[nta];
> +
> +	/* update, and store next to alloc */
> +	nta++;
> +	rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
> +
> +	/* Transfer page from old buffer to new buffer.
> +	 * Move each member individually to avoid possible store
> +	 * forwarding stalls.
>   	 */
> -	if (dma_mapping_error(rx_ring->dev, dma)) {
> -		__free_page(page);
> +	new_buff->dma           = old_buff->dma;
> +	new_buff->page          = old_buff->page;
> +	new_buff->page_offset   = old_buff->page_offset;
> +	new_buff->pagecnt_bias  = old_buff->pagecnt_bias;
> +}
>   
> -		rx_ring->rx_stats.alloc_failed++;
> +static inline bool igc_page_is_reserved(struct page *page)
> +{
> +	return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page);
> +}
> +
> +static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer)
> +{
> +	unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
> +	struct page *page = rx_buffer->page;
> +
> +	/* avoid re-using remote pages */
> +	if (unlikely(igc_page_is_reserved(page)))
> +		return false;
> +
> +#if (PAGE_SIZE < 8192)
> +	/* if we are only owner of page we can reuse it */
> +	if (unlikely((page_ref_count(page) - pagecnt_bias) > 1))
> +		return false;
> +#else
> +#define IGC_LAST_OFFSET \
> +	(SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048)
> +
> +	if (rx_buffer->page_offset > IGC_LAST_OFFSET)
>   		return false;
> +#endif
> +
> +	/* If we have drained the page fragment pool we need to update
> +	 * the pagecnt_bias and page count so that we fully restock the
> +	 * number of references the driver holds.
> +	 */
> +	if (unlikely(!pagecnt_bias)) {
> +		page_ref_add(page, USHRT_MAX);
> +		rx_buffer->pagecnt_bias = USHRT_MAX;
>   	}
>   
> -	bi->dma = dma;
> -	bi->page = page;
> -	bi->page_offset = igc_rx_offset(rx_ring);
> -	bi->pagecnt_bias = 1;
> +	return true;
> +}
> +
> +/**
> + *  igc_is_non_eop - process handling of non-EOP buffers
> + *  @rx_ring: Rx ring being processed
> + *  @rx_desc: Rx descriptor for current buffer
> + *  @skb: current socket buffer containing buffer in progress
> + *
> + *  This function updates next to clean.  If the buffer is an EOP buffer
> + *  this function exits returning false, otherwise it will place the
> + *  sk_buff in the next buffer to be chained and return true indicating
> + *  that this is in fact a non-EOP buffer.
> + **/
> +static bool igc_is_non_eop(struct igc_ring *rx_ring,
> +			   union e1000_adv_rx_desc *rx_desc)
> +{
> +	u32 ntc = rx_ring->next_to_clean + 1;
> +
> +	/* fetch, update, and store next to clean */
> +	ntc = (ntc < rx_ring->count) ? ntc : 0;
> +	rx_ring->next_to_clean = ntc;
> +
> +	prefetch(IGC_RX_DESC(rx_ring, ntc));
> +
> +	if (likely(igc_test_staterr(rx_desc, E1000_RXD_STAT_EOP)))
> +		return false;
>   
>   	return true;
>   }
>   
>   /**
> + *  igc_cleanup_headers - Correct corrupted or empty headers
> + *  @rx_ring: rx descriptor ring packet is being transacted on
> + *  @rx_desc: pointer to the EOP Rx descriptor
> + *  @skb: pointer to current skb being fixed
> + *
> + *  Address the case where we are pulling data in on pages only
> + *  and as such no data is present in the skb header.
> + *
> + *  In addition if skb is not at least 60 bytes we need to pad it so that
> + *  it is large enough to qualify as a valid Ethernet frame.
> + *
> + *  Returns true if an error was encountered and skb was freed.
> + **/
> +static bool igc_cleanup_headers(struct igc_ring *rx_ring,
> +				union e1000_adv_rx_desc *rx_desc,
> +				struct sk_buff *skb)
> +{
> +	if (unlikely((igc_test_staterr(rx_desc,
> +				       E1000_RXDEXT_ERR_FRAME_ERR_MASK)))) {
> +		struct net_device *netdev = rx_ring->netdev;
> +
> +		if (!(netdev->features & NETIF_F_RXALL)) {
> +			dev_kfree_skb_any(skb);
> +			return true;
> +		}
> +	}
> +
> +	/* if eth_skb_pad returns an error the skb was freed */
> +	if (eth_skb_pad(skb))
> +		return true;
> +
> +	return false;
> +}
> +
> +static void igc_put_rx_buffer(struct igc_ring *rx_ring,
> +			      struct igc_rx_buffer *rx_buffer)
> +{
> +	if (igc_can_reuse_rx_page(rx_buffer)) {
> +		/* hand second half of page back to the ring */
> +		igc_reuse_rx_page(rx_ring, rx_buffer);
> +	} else {
> +		/* We are not reusing the buffer so unmap it and free
> +		 * any references we are holding to it
> +		 */
> +		dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
> +				     igc_rx_pg_size(rx_ring), DMA_FROM_DEVICE,
> +				     IGC_RX_DMA_ATTR);
> +		__page_frag_cache_drain(rx_buffer->page,
> +					rx_buffer->pagecnt_bias);
> +	}
> +
> +	/* clear contents of rx_buffer */
> +	rx_buffer->page = NULL;
> +}
> +
> +/**
>    *  igc_alloc_rx_buffers - Replace used receive buffers; packet split
>    *  @adapter: address of board private structure
>    **/
> @@ -801,6 +1410,314 @@ void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count)
>   	}
>   }
>   
> +static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget)
> +{
> +	struct igc_ring *rx_ring = q_vector->rx.ring;
> +	struct sk_buff *skb = rx_ring->skb;
> +	unsigned int total_bytes = 0, total_packets = 0;
> +	u16 cleaned_count = igc_desc_unused(rx_ring);
> +
> +	while (likely(total_packets < budget)) {
> +		union e1000_adv_rx_desc *rx_desc;
> +		struct igc_rx_buffer *rx_buffer;
> +		unsigned int size;
> +
> +		/* return some buffers to hardware, one at a time is too slow */
> +		if (cleaned_count >= IGC_RX_BUFFER_WRITE) {
> +			igc_alloc_rx_buffers(rx_ring, cleaned_count);
> +			cleaned_count = 0;
> +		}
> +
> +		rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean);
> +		size = le16_to_cpu(rx_desc->wb.upper.length);
> +		if (!size)
> +			break;
> +
> +		/* This memory barrier is needed to keep us from reading
> +		 * any other fields out of the rx_desc until we know the
> +		 * descriptor has been written back
> +		 */
> +		dma_rmb();
> +
> +		rx_buffer = igc_get_rx_buffer(rx_ring, size);
> +
> +		/* retrieve a buffer from the ring */
> +		if (skb)
> +			igc_add_rx_frag(rx_ring, rx_buffer, skb, size);
> +		else if (ring_uses_build_skb(rx_ring))
> +			skb = igc_build_skb(rx_ring, rx_buffer, rx_desc, size);
> +		else
> +			skb = igc_construct_skb(rx_ring, rx_buffer,
> +						rx_desc, size);
> +
> +		/* exit if we failed to retrieve a buffer */
> +		if (!skb) {
> +			rx_ring->rx_stats.alloc_failed++;
> +			rx_buffer->pagecnt_bias++;
> +			break;
> +		}
> +
> +		igc_put_rx_buffer(rx_ring, rx_buffer);
> +		cleaned_count++;
> +
> +		/* fetch next buffer in frame if non-eop */
> +		if (igc_is_non_eop(rx_ring, rx_desc))
> +			continue;
> +
> +		/* verify the packet layout is correct */
> +		if (igc_cleanup_headers(rx_ring, rx_desc, skb)) {
> +			skb = NULL;
> +			continue;
> +		}
> +
> +		/* probably a little skewed due to removing CRC */
> +		total_bytes += skb->len;
> +
> +		/* populate checksum, timestamp, VLAN, and protocol */
> +		igc_process_skb_fields(rx_ring, rx_desc, skb);
> +
> +		napi_gro_receive(&q_vector->napi, skb);
> +
> +		/* reset skb pointer */
> +		skb = NULL;
> +
> +		/* update budget accounting */
> +		total_packets++;
> +	}
> +
> +	/* place incomplete frames back on ring for completion */
> +	rx_ring->skb = skb;
> +
> +	u64_stats_update_begin(&rx_ring->rx_syncp);
> +	rx_ring->rx_stats.packets += total_packets;
> +	rx_ring->rx_stats.bytes += total_bytes;
> +	u64_stats_update_end(&rx_ring->rx_syncp);
> +	q_vector->rx.total_packets += total_packets;
> +	q_vector->rx.total_bytes += total_bytes;
> +
> +	if (cleaned_count)
> +		igc_alloc_rx_buffers(rx_ring, cleaned_count);
> +
> +	return total_packets;
> +}
> +
> +static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring)
> +{
> +	return ring_uses_build_skb(rx_ring) ? IGC_SKB_PAD : 0;
> +}
> +
> +static bool igc_alloc_mapped_page(struct igc_ring *rx_ring,
> +				  struct igc_rx_buffer *bi)
> +{
> +	struct page *page = bi->page;
> +	dma_addr_t dma;
> +
> +	/* since we are recycling buffers we should seldom need to alloc */
> +	if (likely(page))
> +		return true;
> +
> +	/* alloc new page for storage */
> +	page = dev_alloc_pages(igc_rx_pg_order(rx_ring));
> +	if (unlikely(!page)) {
> +		rx_ring->rx_stats.alloc_failed++;
> +		return false;
> +	}
> +
> +	/* map page for use */
> +	dma = dma_map_page_attrs(rx_ring->dev, page, 0,
> +				 igc_rx_pg_size(rx_ring),
> +				 DMA_FROM_DEVICE,
> +				 IGC_RX_DMA_ATTR);
> +
> +	/* if mapping failed free memory back to system since
> +	 * there isn't much point in holding memory we can't use
> +	 */
> +	if (dma_mapping_error(rx_ring->dev, dma)) {
> +		__free_page(page);
> +
> +		rx_ring->rx_stats.alloc_failed++;
> +		return false;
> +	}
> +
> +	bi->dma = dma;
> +	bi->page = page;
> +	bi->page_offset = igc_rx_offset(rx_ring);
> +	bi->pagecnt_bias = 1;
> +
> +	return true;
> +}
> +
> +/**
> + *  igc_clean_tx_irq - Reclaim resources after transmit completes
> + *  @q_vector: pointer to q_vector containing needed info
> + *  @napi_budget: Used to determine if we are in netpoll
> + *
> + *  returns true if ring is completely cleaned
> + **/
> +static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget)
> +{
> +	struct igc_adapter *adapter = q_vector->adapter;
> +	struct igc_ring *tx_ring = q_vector->tx.ring;
> +	struct igc_tx_buffer *tx_buffer;
> +	union e1000_adv_tx_desc *tx_desc;
> +	unsigned int total_bytes = 0, total_packets = 0;
> +	unsigned int budget = q_vector->tx.work_limit;
> +	unsigned int i = tx_ring->next_to_clean;

reverse xmas tree

> +
> +	if (test_bit(__IGC_DOWN, &adapter->state))
> +		return true;
> +
> +	tx_buffer = &tx_ring->tx_buffer_info[i];
> +	tx_desc = IGC_TX_DESC(tx_ring, i);
> +	i -= tx_ring->count;
> +
> +	do {
> +		union e1000_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
> +
> +		/* if next_to_watch is not set then there is no work pending */
> +		if (!eop_desc)
> +			break;
> +
> +		/* prevent any other reads prior to eop_desc */
> +		smp_rmb();
> +
> +		/* if DD is not set pending work has not been completed */
> +		if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
> +			break;
> +
> +		/* clear next_to_watch to prevent false hangs */
> +		tx_buffer->next_to_watch = NULL;
> +
> +		/* update the statistics for this packet */
> +		total_bytes += tx_buffer->bytecount;
> +		total_packets += tx_buffer->gso_segs;
> +
> +		/* free the skb */
> +		napi_consume_skb(tx_buffer->skb, napi_budget);
> +
> +		/* unmap skb header data */
> +		dma_unmap_single(tx_ring->dev,
> +				 dma_unmap_addr(tx_buffer, dma),
> +				 dma_unmap_len(tx_buffer, len),
> +				 DMA_TO_DEVICE);
> +
> +		/* clear tx_buffer data */
> +		dma_unmap_len_set(tx_buffer, len, 0);
> +
> +		/* clear last DMA location and unmap remaining buffers */
> +		while (tx_desc != eop_desc) {
> +			tx_buffer++;
> +			tx_desc++;
> +			i++;
> +			if (unlikely(!i)) {
> +				i -= tx_ring->count;
> +				tx_buffer = tx_ring->tx_buffer_info;
> +				tx_desc = IGC_TX_DESC(tx_ring, 0);
> +			}
> +
> +		/* unmap any remaining paged data */

Indent the comment

> +			if (dma_unmap_len(tx_buffer, len)) {
> +				dma_unmap_page(tx_ring->dev,
> +					       dma_unmap_addr(tx_buffer, dma),
> +					       dma_unmap_len(tx_buffer, len),
> +					       DMA_TO_DEVICE);
> +				dma_unmap_len_set(tx_buffer, len, 0);
> +			}
> +		}
> +
> +		/* move us one more past the eop_desc for start of next pkt */
> +		tx_buffer++;
> +		tx_desc++;
> +		i++;
> +		if (unlikely(!i)) {
> +			i -= tx_ring->count;
> +			tx_buffer = tx_ring->tx_buffer_info;
> +			tx_desc = IGC_TX_DESC(tx_ring, 0);
> +		}
> +
> +		/* issue prefetch for next Tx descriptor */
> +		prefetch(tx_desc);
> +
> +		/* update budget accounting */
> +		budget--;
> +	} while (likely(budget));
> +
> +	netdev_tx_completed_queue(txring_txq(tx_ring),
> +				  total_packets, total_bytes);
> +
> +	i += tx_ring->count;
> +	tx_ring->next_to_clean = i;
> +	u64_stats_update_begin(&tx_ring->tx_syncp);
> +	tx_ring->tx_stats.bytes += total_bytes;
> +	tx_ring->tx_stats.packets += total_packets;
> +	u64_stats_update_end(&tx_ring->tx_syncp);
> +	q_vector->tx.total_bytes += total_bytes;
> +	q_vector->tx.total_packets += total_packets;
> +
> +	if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
> +		struct e1000_hw *hw = &adapter->hw;
> +
> +		/* Detect a transmit hang in hardware, this serializes the
> +		 * check with the clearing of time_stamp and movement of i
> +		 */
> +		clear_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
> +		if (tx_buffer->next_to_watch &&
> +		    time_after(jiffies, tx_buffer->time_stamp +
> +		    (adapter->tx_timeout_factor * HZ)) &&
> +		    !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
> +			/* detected Tx unit hang */
> +			dev_err(tx_ring->dev,
> +				"Detected Tx Unit Hang\n"
> +				"  Tx Queue             <%d>\n"
> +				"  TDH                  <%x>\n"
> +				"  TDT                  <%x>\n"
> +				"  next_to_use          <%x>\n"
> +				"  next_to_clean        <%x>\n"
> +				"buffer_info[next_to_clean]\n"
> +				"  time_stamp           <%lx>\n"
> +				"  next_to_watch        <%p>\n"
> +				"  jiffies              <%lx>\n"
> +				"  desc.status          <%x>\n",
> +				tx_ring->queue_index,
> +				rd32(E1000_TDH(tx_ring->reg_idx)),
> +				readl(tx_ring->tail),
> +				tx_ring->next_to_use,
> +				tx_ring->next_to_clean,
> +				tx_buffer->time_stamp,
> +				tx_buffer->next_to_watch,
> +				jiffies,
> +				tx_buffer->next_to_watch->wb.status);
> +				netif_stop_subqueue(tx_ring->netdev,
> +						    tx_ring->queue_index);
> +
> +			/* we are about to reset, no point in enabling stuff */
> +			return true;
> +		}
> +	}
> +
> +#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
> +	if (unlikely(total_packets &&
> +		     netif_carrier_ok(tx_ring->netdev) &&
> +		     igc_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) {
> +		/* Make sure that anybody stopping the queue after this
> +		 * sees the new next_to_clean.
> +		 */
> +		smp_mb();
> +		if (__netif_subqueue_stopped(tx_ring->netdev,
> +					     tx_ring->queue_index) &&
> +		    !(test_bit(__IGC_DOWN, &adapter->state))) {
> +			netif_wake_subqueue(tx_ring->netdev,
> +					    tx_ring->queue_index);
> +
> +			u64_stats_update_begin(&tx_ring->tx_syncp);
> +			tx_ring->tx_stats.restart_queue++;
> +			u64_stats_update_end(&tx_ring->tx_syncp);
> +		}
> +	}
> +
> +	return !!budget;
> +}
> +
>   /**
>    *  igc_ioctl - I/O control method
>    *  @netdev: network interface device structure
> @@ -851,27 +1768,97 @@ int igc_up(struct igc_adapter *adapter)
>   }
>   
>   /**
> + *  igc_update_stats - Update the board statistics counters
> + *  @adapter: board private structure
> + **/
> +void igc_update_stats(struct igc_adapter *adapter)
> +{
> +}
> +
> +/**
>    *  igc_down - Close the interface
>    *  @adapter: board private structure
>    **/
>   void igc_down(struct igc_adapter *adapter)
>   {
>   	struct net_device *netdev = adapter->netdev;
> +	struct e1000_hw *hw = &adapter->hw;
>   	int i = 0;
> +	u32 tctl, rctl;

reverse xmas tree

>   
>   	set_bit(__IGC_DOWN, &adapter->state);
>   
> +	/* disable receives in the hardware */
> +	rctl = rd32(E1000_RCTL);
> +	wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
> +	/* flush and sleep below */
> +
> +	igc_nfc_filter_exit(adapter);
> +
>   	/* set trans_start so we don't get spurious watchdogs during reset */
>   	netif_trans_update(netdev);
>   
>   	netif_carrier_off(netdev);
>   	netif_tx_stop_all_queues(netdev);
>   
> -	for (i = 0; i < adapter->num_q_vectors; i++)
> -		napi_disable(&adapter->q_vector[i]->napi);
> +	/* disable transmits in the hardware */
> +	tctl = rd32(E1000_TCTL);
> +	tctl &= ~E1000_TCTL_EN;
> +	wr32(E1000_TCTL, tctl);
> +	/* flush both disables and wait for them to finish */
> +	wrfl();
> +	usleep_range(10000, 20000);
> +
> +	igc_irq_disable(adapter);
> +
> +	adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
> +
> +	for (i = 0; i < adapter->num_q_vectors; i++) {
> +		if (adapter->q_vector[i]) {
> +			napi_synchronize(&adapter->q_vector[i]->napi);
> +			napi_disable(&adapter->q_vector[i]->napi);
> +		}
> +	}
> +
> +	del_timer_sync(&adapter->watchdog_timer);
> +	del_timer_sync(&adapter->phy_info_timer);
> +
> +	/* record the stats before reset*/
> +	spin_lock(&adapter->stats64_lock);
> +	igc_update_stats(adapter);
> +	spin_unlock(&adapter->stats64_lock);
>   
>   	adapter->link_speed = 0;
>   	adapter->link_duplex = 0;
> +
> +	if (!pci_channel_offline(adapter->pdev))
> +		igc_reset(adapter);
> +
> +	/* clear VLAN promisc flag so VFTA will be updated if necessary */
> +	adapter->flags &= ~IGC_FLAG_VLAN_PROMISC;
> +
> +	igc_clean_all_tx_rings(adapter);
> +	igc_clean_all_rx_rings(adapter);
> +}
> +
> +void igc_reinit_locked(struct igc_adapter *adapter)
> +{
> +	WARN_ON(in_interrupt());
> +	while (test_and_set_bit(__IGC_RESETTING, &adapter->state))
> +		usleep_range(1000, 2000);
> +	igc_down(adapter);
> +	igc_up(adapter);
> +	clear_bit(__IGC_RESETTING, &adapter->state);
> +}
> +
> +static void igc_reset_task(struct work_struct *work)
> +{
> +	struct igc_adapter *adapter;
> +
> +	adapter = container_of(work, struct igc_adapter, reset_task);
> +
> +	netdev_err(adapter->netdev, "Reset adapter\n");
> +	igc_reinit_locked(adapter);
>   }
>   
>   /**
> @@ -915,14 +1902,6 @@ static int igc_change_mtu(struct net_device *netdev, int new_mtu)
>   }
>   
>   /**
> - *  igc_update_stats - Update the board statistics counters
> - *  @adapter: board private structure
> - **/
> -void igc_update_stats(struct igc_adapter *adapter)
> -{
> -}
> -
> -/**
>    *  igc_get_stats - Get System Network Statistics
>    *  @netdev: network interface device structure
>    *
> @@ -1322,6 +2301,17 @@ static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx)
>   }
>   
>   /**
> + *  igc_watchdog - Timer Call-back
> + *  @data: pointer to adapter cast into an unsigned long
> + **/
> +static void igc_watchdog(struct timer_list *t)
> +{
> +	struct igc_adapter *adapter = from_timer(adapter, t, watchdog_timer);
> +	/* Do the rest outside of interrupt context */
> +	schedule_work(&adapter->watchdog_task);
> +}
> +
> +/**
>    *  igc_update_ring_itr - update the dynamic ITR value based on packet size
>    *  @q_vector: pointer to q_vector
>    *
> @@ -1642,12 +2632,12 @@ static int igc_poll(struct napi_struct *napi, int budget)
>   						     napi);
>   	bool clean_complete = true;
>   	int work_done = 0;
> -	int cleaned = 0;
>   
> -	/* TODO q->vector->tx_ring: igc_clean_tx_irq */
> +	if (q_vector->tx.ring)
> +		clean_complete = igc_clean_tx_irq(q_vector, budget);
>   
>   	if (q_vector->rx.ring) {
> -		/* TODO igc_clean_rx_irq */
> +		int cleaned = igc_clean_rx_irq(q_vector, budget);
>   
>   		work_done += cleaned;
>   		if (cleaned >= budget)
> @@ -2430,6 +3420,14 @@ static int igc_probe(struct pci_dev *pdev,
>   	netdev->min_mtu = ETH_MIN_MTU;
>   	netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;
>   
> +	/* configure RXPBSIZE and TXPBSIZE */
> +	wr32(E1000_RXPBS, I225_RXPBSIZE_DEFAULT);
> +	wr32(E1000_TXPBS, I225_TXPBSIZE_DEFAULT);
> +
> +	timer_setup(&adapter->watchdog_timer, igc_watchdog, 0);
> +
> +	INIT_WORK(&adapter->reset_task, igc_reset_task);
> +
>   	/* reset the hardware with the new settings */
>   	igc_reset(adapter);
>   
> @@ -2490,9 +3488,14 @@ static void igc_remove(struct pci_dev *pdev)
>   {
>   	struct net_device *netdev = pci_get_drvdata(pdev);
>   	struct igc_adapter *adapter = netdev_priv(netdev);
> +	struct e1000_hw *hw = &adapter->hw;
>   
>   	set_bit(__IGC_DOWN, &adapter->state);
> -	flush_scheduled_work();
> +
> +	del_timer_sync(&adapter->watchdog_timer);
> +
> +	cancel_work_sync(&adapter->reset_task);
> +	cancel_work_sync(&adapter->watchdog_task);
>   
>   	/* Release control of h/w to f/w.  If f/w is AMT enabled, this
>   	 * would have already happened in close and is redundant.
> @@ -2500,10 +3503,18 @@ static void igc_remove(struct pci_dev *pdev)
>   	igc_release_hw_control(adapter);
>   	unregister_netdev(netdev);
>   
> -	pci_release_selected_regions(pdev,
> -				     pci_select_bars(pdev, IORESOURCE_MEM));
> +	igc_clear_interrupt_scheme(adapter);
> +	pci_iounmap(pdev, adapter->io_addr);
> +	if (hw->flash_address)
> +		iounmap(hw->flash_address);
> +	pci_release_mem_regions(pdev);
>   
> +	kfree(adapter->mac_table);
> +	kfree(adapter->shadow_vfta);
>   	free_netdev(netdev);
> +
> +	pci_disable_pcie_error_reporting(pdev);
> +
>   	pci_disable_device(pdev);
>   }
>   
> @@ -2514,6 +3525,39 @@ static struct pci_driver igc_driver = {
>   	.remove   = igc_remove,
>   };
>   
> +void igc_set_flag_queue_pairs(struct igc_adapter *adapter,
> +			      const u32 max_rss_queues)
> +{
> +	/* Determine if we need to pair queues. */
> +	/* If rss_queues > half of max_rss_queues, pair the queues in
> +	 * order to conserve interrupts due to limited supply.
> +	 */
> +	if (adapter->rss_queues > (max_rss_queues / 2))
> +		adapter->flags |= IGC_FLAG_QUEUE_PAIRS;
> +	else
> +		adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS;
> +}
> +
> +unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter)
> +{
> +	unsigned int max_rss_queues;
> +
> +	/* Determine the maximum number of RSS queues supported. */
> +	max_rss_queues = IGC_MAX_RX_QUEUES;
> +
> +	return max_rss_queues;
> +}
> +
> +static void igc_init_queue_configuration(struct igc_adapter *adapter)
> +{
> +	u32 max_rss_queues;
> +
> +	max_rss_queues = igc_get_max_rss_queues(adapter);
> +	adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
> +
> +	igc_set_flag_queue_pairs(adapter, max_rss_queues);
> +}
> +
>   /**
>    *  igc_sw_init - Initialize general software structures (struct igc_adapter)
>    *  @adapter: board private structure to initialize
> @@ -2528,21 +3572,37 @@ static int igc_sw_init(struct igc_adapter *adapter)
>   	struct net_device *netdev = adapter->netdev;
>   	struct pci_dev *pdev = adapter->pdev;
>   
> -	/* PCI config space info */
> +	int size = sizeof(struct igc_mac_addr) * hw->mac.rar_entry_count;
>   
> -	hw->vendor_id = pdev->vendor;
> -	hw->device_id = pdev->device;
> -	hw->subsystem_vendor_id = pdev->subsystem_vendor;
> -	hw->subsystem_device_id = pdev->subsystem_device;
> +	pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
>   
> -	pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
> +	/* set default ring sizes */
> +	adapter->tx_ring_count = IGC_DEFAULT_TXD;
> +	adapter->rx_ring_count = IGC_DEFAULT_RXD;
>   
> -	pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
> +	/* set default ITR values */
> +	adapter->rx_itr_setting = IGC_DEFAULT_ITR;
> +	adapter->tx_itr_setting = IGC_DEFAULT_ITR;
>   
>   	/* set default work limits */
> +	adapter->tx_work_limit = IGC_DEFAULT_TX_WORK;
> +
>   	adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
> -					VLAN_HLEN;
> +				VLAN_HLEN;
> +	adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
> +
> +	spin_lock_init(&adapter->nfc_lock);
> +	spin_lock_init(&adapter->stats64_lock);
> +	/* Assume MSI-X interrupts, will be checked during IRQ allocation */
> +	adapter->flags |= IGC_FLAG_HAS_MSIX;
>   
> +	adapter->mac_table = kzalloc(size, GFP_ATOMIC);
> +	if (!adapter->mac_table)
> +		return -ENOMEM;
> +
> +	igc_init_queue_configuration(adapter);
> +
> +	/* This call may decrease the number of queues */
>   	if (igc_init_interrupt_scheme(adapter, true)) {
>   		dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
>   		return -ENOMEM;
> @@ -2556,6 +3616,10 @@ static int igc_sw_init(struct igc_adapter *adapter)
>   	return 0;
>   }
>   
> +static void igc_nfc_filter_exit(struct igc_adapter *adapter)
> +{
> +}
> +
>   /**
>    *  igc_init_module - Driver Registration Routine
>    *
> 


More information about the Intel-wired-lan mailing list