Index: /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/Makefile =================================================================== --- /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/Makefile (revision 38965) +++ /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/Makefile (working copy) @@ -1,35 +1,6 @@ -# -# BSD LICENSE -# -# Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2015-2019 Amazon.com, Inc. or its affiliates. # All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of copyright holder nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# include $(RTE_SDK)/mk/rte.vars.mk @@ -48,6 +19,7 @@ # all source are stored in SRCS-y # SRCS-$(CONFIG_RTE_LIBRTE_ENA_PMD) += ena_ethdev.c +SRCS-$(CONFIG_RTE_LIBRTE_ENA_PMD) += ena_rss.c SRCS-$(CONFIG_RTE_LIBRTE_ENA_PMD) += ena_com.c SRCS-$(CONFIG_RTE_LIBRTE_ENA_PMD) += ena_eth_com.c Index: /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/README =================================================================== --- /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/README (nonexistent) +++ /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/README (working copy) @@ -0,0 +1,18 @@ +2025/2/27 ENA dirver upgrade note: +there is official patch provied by aws to upgrade ena driver from specific DPDK version +reference link: https://github.com/amzn/amzn-drivers/tree/master/userspace/dpdk/backports/ENAv2.4.0/v16.11.11 + +here we choose to backport patches to DPDK v16.11.11 instead of v17.11.10 is because atcp is using DPDK v17.02 +and the code base is much similar with v16.11.11. + +the whole processes to upgrade ena driver are: +1. upgrade ena driver to pure DPDK v16.11.11. + please refer link for more detail operations: https://github.com/amzn/amzn-drivers/tree/master/userspace/dpdk#3-ena-pmd-backports + +2. copy whole YOUR_DPDK16.11.11_ROOT_FOLDER/drivers/net/ena folder to replace vAPV rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena folder + +3. need to fix some compiling error by update code in below files: + a. usr/click/lib/libintel_dpdk/drivers/net/ena/ena_ethdev.c + b. usr/click/lib/libintel_dpdk/lib/librte_eal/common/include/rte_common.h + c. usr/click/lib/libintel_dpdk/lib/librte_eal/common/include/rte_pci.h + Index: /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_com.h =================================================================== --- /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_com.h (revision 38965) +++ /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_com.h (working copy) @@ -1,53 +1,16 @@ -/*- -* BSD LICENSE -* -* Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. -* All rights reserved. -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following conditions -* are met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above copyright -* notice, this list of conditions and the following disclaimer in -* the documentation and/or other materials provided with the -* distribution. -* * Neither the name of copyright holder nor the names of its -* contributors may be used to endorse or promote products derived -* from this software without specific prior written permission. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. + * All rights reserved. + */ #ifndef ENA_COM #define ENA_COM #include "ena_plat.h" -#include "ena_common_defs.h" -#include "ena_admin_defs.h" -#include "ena_eth_io_defs.h" -#include "ena_regs_defs.h" -#if defined(__linux__) && !defined(__KERNEL__) -#include -#include -#define __iomem -#endif -#define ENA_MAX_NUM_IO_QUEUES 128U +#define ENA_MAX_NUM_IO_QUEUES 128U /* We need to queues for each IO (on for Tx and one for Rx) */ -#define ENA_TOTAL_NUM_QUEUES (2 * (ENA_MAX_NUM_IO_QUEUES)) +#define ENA_TOTAL_NUM_QUEUES (2 * (ENA_MAX_NUM_IO_QUEUES)) #define ENA_MAX_HANDLERS 256 @@ -60,48 +23,28 @@ #define ADMIN_CQ_SIZE(depth) ((depth) * sizeof(struct ena_admin_acq_entry)) #define ADMIN_AENQ_SIZE(depth) ((depth) * sizeof(struct ena_admin_aenq_entry)) +#define ENA_CDESC_RING_SIZE_ALIGNMENT (1 << 12) /* 4K */ + /*****************************************************************************/ /*****************************************************************************/ /* ENA adaptive interrupt moderation settings */ -#define ENA_INTR_LOWEST_USECS (0) -#define ENA_INTR_LOWEST_PKTS (3) -#define ENA_INTR_LOWEST_BYTES (2 * 1524) - -#define ENA_INTR_LOW_USECS (32) -#define ENA_INTR_LOW_PKTS (12) -#define ENA_INTR_LOW_BYTES (16 * 1024) - -#define ENA_INTR_MID_USECS (80) -#define ENA_INTR_MID_PKTS (48) -#define ENA_INTR_MID_BYTES (64 * 1024) - -#define ENA_INTR_HIGH_USECS (128) -#define ENA_INTR_HIGH_PKTS (96) -#define ENA_INTR_HIGH_BYTES (128 * 1024) - -#define ENA_INTR_HIGHEST_USECS (192) -#define ENA_INTR_HIGHEST_PKTS (128) -#define ENA_INTR_HIGHEST_BYTES (192 * 1024) - -#define ENA_INTR_INITIAL_TX_INTERVAL_USECS 196 -#define ENA_INTR_INITIAL_RX_INTERVAL_USECS 4 -#define ENA_INTR_DELAY_OLD_VALUE_WEIGHT 6 -#define ENA_INTR_DELAY_NEW_VALUE_WEIGHT 4 - -enum ena_intr_moder_level { - ENA_INTR_MODER_LOWEST = 0, - ENA_INTR_MODER_LOW, - ENA_INTR_MODER_MID, - ENA_INTR_MODER_HIGH, - ENA_INTR_MODER_HIGHEST, - ENA_INTR_MAX_NUM_OF_LEVELS, -}; - -struct ena_intr_moder_entry { - unsigned int intr_moder_interval; - unsigned int pkts_per_interval; - unsigned int bytes_per_interval; +#define ENA_INTR_INITIAL_TX_INTERVAL_USECS ENA_INTR_INITIAL_TX_INTERVAL_USECS_PLAT +#define ENA_INTR_INITIAL_RX_INTERVAL_USECS 0 +#define ENA_DEFAULT_INTR_DELAY_RESOLUTION 1 + +#define ENA_HASH_KEY_SIZE 40 + +#define ENA_HW_HINTS_NO_TIMEOUT 0xFFFF + +#define ENA_FEATURE_MAX_QUEUE_EXT_VER 1 + +struct ena_llq_configurations { + enum ena_admin_llq_header_location llq_header_location; + enum ena_admin_llq_ring_entry_size llq_ring_entry_size; + enum ena_admin_llq_stride_ctrl llq_stride_ctrl; + enum ena_admin_llq_num_descs_before_header llq_num_decs_before_header; + u16 llq_ring_entry_size_value; }; enum queue_direction { @@ -120,8 +63,8 @@ }; struct ena_com_io_desc_addr { - u8 __iomem *pbuf_dev_addr; /* LLQ address */ - u8 *virt_addr; + u8 __iomem *pbuf_dev_addr; /* LLQ address */ + u8 *virt_addr; dma_addr_t phys_addr; ena_mem_handle_t mem_handle; }; @@ -130,13 +73,23 @@ u16 mss; u16 l3_hdr_len; u16 l3_hdr_offset; - u16 l3_outer_hdr_len; /* In words */ - u16 l3_outer_hdr_offset; u16 l4_hdr_len; /* In words */ }; +struct ena_com_llq_info { + u16 header_location_ctrl; + u16 desc_stride_ctrl; + u16 desc_list_entry_size_ctrl; + u16 desc_list_entry_size; + u16 descs_num_before_header; + u16 descs_per_entry; + u16 max_entries_in_tx_burst; + bool disable_meta_caching; +}; + struct ena_com_io_cq { struct ena_com_io_desc_addr cdesc_addr; + void *bus; /* Interrupt unmask register */ u32 __iomem *unmask_reg; @@ -172,8 +125,23 @@ } ____cacheline_aligned; +struct ena_com_io_bounce_buffer_control { + u8 *base_buffer; + u16 next_to_use; + u16 buffer_size; + u16 buffers_num; /* Must be a power of 2 */ +}; + +/* This struct is to keep tracking the current location of the next llq entry */ +struct ena_com_llq_pkt_ctrl { + u8 *curr_bounce_buf; + u16 idx; + u16 descs_left_in_line; +}; + struct ena_com_io_sq { struct ena_com_io_desc_addr desc_addr; + void *bus; u32 __iomem *db_addr; u8 __iomem *header_addr; @@ -181,8 +149,13 @@ enum queue_direction direction; enum ena_admin_placement_policy_type mem_queue_type; + bool disable_meta_caching; + u32 msix_vector; struct ena_com_tx_meta cached_tx_meta; + struct ena_com_llq_info llq_info; + struct ena_com_llq_pkt_ctrl llq_buf_ctrl; + struct ena_com_io_bounce_buffer_control bounce_buf_ctrl; u16 q_depth; u16 qid; @@ -190,10 +163,12 @@ u16 idx; u16 tail; u16 next_to_comp; + u16 llq_last_copy_tail; u32 tx_max_header_size; u8 phase; u8 desc_entry_size; u8 dma_addr_bits; + u16 entries_in_tx_burst_left; } ____cacheline_aligned; struct ena_com_admin_cq { @@ -219,17 +194,21 @@ }; struct ena_com_stats_admin { - u32 aborted_cmd; - u32 submitted_cmd; - u32 completed_cmd; - u32 out_of_space; - u32 no_completion; + u64 aborted_cmd; + u64 submitted_cmd; + u64 completed_cmd; + u64 out_of_space; + u64 no_completion; }; struct ena_com_admin_queue { void *q_dmadev; + void *bus; + struct ena_com_dev *ena_dev; ena_spinlock_t q_lock; /* spinlock for the admin queue */ + struct ena_comp_ctx *comp_ctx; + u32 completion_timeout; u16 q_depth; struct ena_com_admin_cq cq; struct ena_com_admin_sq sq; @@ -237,6 +216,9 @@ /* Indicate if the admin queue should poll for completion */ bool polling; + /* Define if fallback to polling mode should occur */ + bool auto_polling; + u16 curr_cmd_id; /* Indicate that the ena was initialized and can @@ -266,6 +248,7 @@ struct ena_admin_ena_mmio_req_read_less_resp *read_resp; dma_addr_t read_resp_dma_addr; ena_mem_handle_t read_resp_mem_handle; + u32 reg_read_to; /* in us */ u16 seq_num; bool readless_supported; /* spin lock to ensure a single outstanding read */ @@ -316,6 +299,8 @@ u8 __iomem *reg_bar; void __iomem *mem_bar; void *dmadev; + void *bus; + ena_netdev *net_device; enum ena_admin_placement_policy_type tx_mem_queue_type; u32 tx_max_header_size; @@ -331,15 +316,28 @@ struct ena_host_attribute host_attr; bool adaptive_coalescing; u16 intr_delay_resolution; + + /* interrupt moderation intervals are in usec divided by + * intr_delay_resolution, which is supplied by the device. + */ u32 intr_moder_tx_interval; + u32 intr_moder_rx_interval; + struct ena_intr_moder_entry *intr_moder_tbl; + + struct ena_com_llq_info llq_info; + + u32 ena_min_poll_delay_us; }; struct ena_com_dev_get_features_ctx { struct ena_admin_queue_feature_desc max_queues; + struct ena_admin_queue_ext_feature_desc max_queue_ext; struct ena_admin_device_attr_feature_desc dev_attr; struct ena_admin_feature_aenq_desc aenq; struct ena_admin_feature_offload_desc offload; + struct ena_admin_ena_hw_hints hw_hints; + struct ena_admin_feature_llq_desc llq; }; struct ena_com_create_io_ctx { @@ -377,9 +375,9 @@ */ int ena_com_mmio_reg_read_request_init(struct ena_com_dev *ena_dev); -/* ena_com_set_mmio_read_mode - Enable/disable the mmio reg read mechanism +/* ena_com_set_mmio_read_mode - Enable/disable the indirect mmio reg read mechanism * @ena_dev: ENA communication layer struct - * @realess_supported: readless mode (enable/disable) + * @readless_supported: readless mode (enable/disable) */ void ena_com_set_mmio_read_mode(struct ena_com_dev *ena_dev, bool readless_supported); @@ -398,8 +396,6 @@ /* ena_com_admin_init - Init the admin and the async queues * @ena_dev: ENA communication layer struct * @aenq_handlers: Those handlers to be called upon event. - * @init_spinlock: Indicate if this method should init the admin spinlock or - * the spinlock was init before (for example, in a case of FLR). * * Initialize the admin submission and completion queues. * Initialize the asynchronous events notification queues. @@ -407,8 +403,7 @@ * @return - 0 on success, negative value on failure. */ int ena_com_admin_init(struct ena_com_dev *ena_dev, - struct ena_aenq_handlers *aenq_handlers, - bool init_spinlock); + struct ena_aenq_handlers *aenq_handlers); /* ena_com_admin_destroy - Destroy the admin and the async events queues. * @ena_dev: ENA communication layer struct @@ -421,14 +416,16 @@ /* ena_com_dev_reset - Perform device FLR to the device. * @ena_dev: ENA communication layer struct + * @reset_reason: Specify what is the trigger for the reset in case of an error. * * @return - 0 on success, negative value on failure. */ -int ena_com_dev_reset(struct ena_com_dev *ena_dev); +int ena_com_dev_reset(struct ena_com_dev *ena_dev, + enum ena_regs_reset_reason_types reset_reason); /* ena_com_create_io_queue - Create io queue. * @ena_dev: ENA communication layer struct - * ena_com_create_io_ctx - create context structure + * @ctx - create context structure * * Create the submission and the completion queues. * @@ -437,8 +434,9 @@ int ena_com_create_io_queue(struct ena_com_dev *ena_dev, struct ena_com_create_io_ctx *ctx); -/* ena_com_admin_destroy - Destroy IO queue with the queue id - qid. +/* ena_com_destroy_io_queue - Destroy IO queue with the queue id - qid. * @ena_dev: ENA communication layer struct + * @qid - the caller virtual queue id. */ void ena_com_destroy_io_queue(struct ena_com_dev *ena_dev, u16 qid); @@ -485,7 +483,7 @@ */ void ena_com_set_admin_polling_mode(struct ena_com_dev *ena_dev, bool polling); -/* ena_com_set_admin_polling_mode - Get the admin completion queue polling mode +/* ena_com_get_admin_polling_mode - Get the admin completion queue polling mode * @ena_dev: ENA communication layer struct * * Get the admin completion mode. @@ -495,12 +493,23 @@ * * @return state */ -bool ena_com_get_ena_admin_polling_mode(struct ena_com_dev *ena_dev); +bool ena_com_get_admin_polling_mode(struct ena_com_dev *ena_dev); + +/* ena_com_set_admin_auto_polling_mode - Enable autoswitch to polling mode + * @ena_dev: ENA communication layer struct + * @polling: Enable/Disable polling mode + * + * Set the autopolling mode. + * If autopolling is on: + * In case of missing interrupt when data is available switch to polling. + */ +void ena_com_set_admin_auto_polling_mode(struct ena_com_dev *ena_dev, + bool polling); /* ena_com_admin_q_comp_intr_handler - admin queue interrupt handler * @ena_dev: ENA communication layer struct * - * This method go over the admin completion queue and wake up all the pending + * This method goes over the admin completion queue and wakes up all the pending * threads that wait on the commands wait event. * * @note: Should be called after MSI-X interrupt. @@ -510,10 +519,10 @@ /* ena_com_aenq_intr_handler - AENQ interrupt handler * @ena_dev: ENA communication layer struct * - * This method go over the async event notification queue and call the proper + * This method goes over the async event notification queue and calls the proper * aenq handler. */ -void ena_com_aenq_intr_handler(struct ena_com_dev *dev, void *data); +void ena_com_aenq_intr_handler(struct ena_com_dev *ena_dev, void *data); /* ena_com_abort_admin_commands - Abort all the outstanding admin commands. * @ena_dev: ENA communication layer struct @@ -527,14 +536,14 @@ /* ena_com_wait_for_abort_completion - Wait for admin commands abort. * @ena_dev: ENA communication layer struct * - * This method wait until all the outstanding admin commands will be completed. + * This method waits until all the outstanding admin commands are completed. */ void ena_com_wait_for_abort_completion(struct ena_com_dev *ena_dev); /* ena_com_validate_version - Validate the device parameters * @ena_dev: ENA communication layer struct * - * This method validate the device parameters are the same as the saved + * This method verifies the device parameters are the same as the saved * parameters in ena_dev. * This method is useful after device reset, to validate the device mac address * and the device offloads are the same as before the reset. @@ -581,9 +590,8 @@ * * @return: 0 on Success and negative value otherwise. */ -int -ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev, - struct ena_com_dev_get_features_ctx *get_feat_ctx); +int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev, + struct ena_com_dev_get_features_ctx *get_feat_ctx); /* ena_com_get_dev_basic_stats - Get device basic statistics * @ena_dev: ENA communication layer struct @@ -594,6 +602,15 @@ int ena_com_get_dev_basic_stats(struct ena_com_dev *ena_dev, struct ena_admin_basic_stats *stats); +/* ena_com_get_eni_stats - Get extended network interface statistics + * @ena_dev: ENA communication layer struct + * @stats: stats return value + * + * @return: 0 on Success and negative value otherwise. + */ +int ena_com_get_eni_stats(struct ena_com_dev *ena_dev, + struct ena_admin_eni_stats *stats); + /* ena_com_set_dev_mtu - Configure the device mtu. * @ena_dev: ENA communication layer struct * @mtu: mtu value @@ -608,9 +625,8 @@ * * @return: 0 on Success and negative value otherwise. */ -int -ena_com_get_offload_settings(struct ena_com_dev *ena_dev, - struct ena_admin_feature_offload_desc *offload); +int ena_com_get_offload_settings(struct ena_com_dev *ena_dev, + struct ena_admin_feature_offload_desc *offload); /* ena_com_rss_init - Init RSS * @ena_dev: ENA communication layer struct @@ -631,6 +647,14 @@ */ void ena_com_rss_destroy(struct ena_com_dev *ena_dev); +/* ena_com_get_current_hash_function - Get RSS hash function + * @ena_dev: ENA communication layer struct + * + * Return the current hash function. + * @return: 0 or one of the ena_admin_hash_functions values. + */ +int ena_com_get_current_hash_function(struct ena_com_dev *ena_dev); + /* ena_com_fill_hash_function - Fill RSS hash function * @ena_dev: ENA communication layer struct * @func: The hash function (Toeplitz or crc) @@ -662,23 +686,32 @@ */ int ena_com_set_hash_function(struct ena_com_dev *ena_dev); -/* ena_com_get_hash_function - Retrieve the hash function and the hash key - * from the device. +/* ena_com_get_hash_function - Retrieve the hash function from the device. * @ena_dev: ENA communication layer struct * @func: hash function - * @key: hash key * - * Retrieve the hash function and the hash key from the device. + * Retrieve the hash function from the device. * - * @note: If the caller called ena_com_fill_hash_function but didn't flash + * @note: If the caller called ena_com_fill_hash_function but didn't flush * it to the device, the new configuration will be lost. * * @return: 0 on Success and negative value otherwise. */ int ena_com_get_hash_function(struct ena_com_dev *ena_dev, - enum ena_admin_hash_functions *func, - u8 *key); + enum ena_admin_hash_functions *func); +/* ena_com_get_hash_key - Retrieve the hash key + * @ena_dev: ENA communication layer struct + * @key: hash key + * + * Retrieve the hash key. + * + * @note: If the caller called ena_com_fill_hash_key but didn't flush + * it to the device, the new configuration will be lost. + * + * @return: 0 on Success and negative value otherwise. + */ +int ena_com_get_hash_key(struct ena_com_dev *ena_dev, u8 *key); /* ena_com_fill_hash_ctrl - Fill RSS hash control * @ena_dev: ENA communication layer struct. * @proto: The protocol to configure. @@ -713,7 +746,7 @@ * * Retrieve the hash control from the device. * - * @note, If the caller called ena_com_fill_hash_ctrl but didn't flash + * @note: If the caller called ena_com_fill_hash_ctrl but didn't flush * it to the device, the new configuration will be lost. * * @return: 0 on Success and negative value otherwise. @@ -765,8 +798,8 @@ * * Retrieve the RSS indirection table from the device. * - * @note: If the caller called ena_com_indirect_table_fill_entry but didn't - * flash it to the device, the new configuration will be lost. + * @note: If the caller called ena_com_indirect_table_fill_entry but didn't flush + * it to the device, the new configuration will be lost. * * @return: 0 on Success and negative value otherwise. */ @@ -791,14 +824,14 @@ /* ena_com_delete_debug_area - Free the debug area resources. * @ena_dev: ENA communication layer struct * - * Free the allocate debug area. + * Free the allocated debug area. */ void ena_com_delete_debug_area(struct ena_com_dev *ena_dev); /* ena_com_delete_host_info - Free the host info resources. * @ena_dev: ENA communication layer struct * - * Free the allocate host info. + * Free the allocated host info. */ void ena_com_delete_host_info(struct ena_com_dev *ena_dev); @@ -839,9 +872,9 @@ * @cmd_completion: command completion return value. * @cmd_comp_size: command completion size. - * Submit an admin command and then wait until the device will return a + * Submit an admin command and then wait until the device returns a * completion. - * The completion will be copyed into cmd_comp. + * The completion will be copied into cmd_comp. * * @return - 0 on success, negative value on failure. */ @@ -858,11 +891,6 @@ */ int ena_com_init_interrupt_moderation(struct ena_com_dev *ena_dev); -/* ena_com_destroy_interrupt_moderation - Destroy interrupt moderation resources - * @ena_dev: ENA communication layer struct - */ -void ena_com_destroy_interrupt_moderation(struct ena_com_dev *ena_dev); - /* ena_com_interrupt_moderation_supported - Return if interrupt moderation * capability is supported by the device. * @@ -870,13 +898,6 @@ */ bool ena_com_interrupt_moderation_supported(struct ena_com_dev *ena_dev); -/* ena_com_config_default_interrupt_moderation_table - Restore the interrupt - * moderation table back to the default parameters. - * @ena_dev: ENA communication layer struct - */ -void -ena_com_config_default_interrupt_moderation_table(struct ena_com_dev *ena_dev); - /* ena_com_update_nonadaptive_moderation_interval_tx - Update the * non-adaptive interval in Tx direction. * @ena_dev: ENA communication layer struct @@ -884,9 +905,8 @@ * * @return - 0 on success, negative value on failure. */ -int -ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev, - u32 tx_coalesce_usecs); +int ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev, + u32 tx_coalesce_usecs); /* ena_com_update_nonadaptive_moderation_interval_rx - Update the * non-adaptive interval in Rx direction. @@ -895,9 +915,8 @@ * * @return - 0 on success, negative value on failure. */ -int -ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev, - u32 rx_coalesce_usecs); +int ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev, + u32 rx_coalesce_usecs); /* ena_com_get_nonadaptive_moderation_interval_tx - Retrieve the * non-adaptive interval in Tx direction. @@ -905,8 +924,7 @@ * * @return - interval in usec */ -unsigned int -ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev); +unsigned int ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev); /* ena_com_get_nonadaptive_moderation_interval_rx - Retrieve the * non-adaptive interval in Rx direction. @@ -914,122 +932,58 @@ * * @return - interval in usec */ -unsigned int -ena_com_get_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev); +unsigned int ena_com_get_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev); -/* ena_com_init_intr_moderation_entry - Update a single entry in the interrupt - * moderation table. +/* ena_com_config_dev_mode - Configure the placement policy of the device. * @ena_dev: ENA communication layer struct - * @level: Interrupt moderation table level - * @entry: Entry value + * @llq_features: LLQ feature descriptor, retrieve via + * ena_com_get_dev_attr_feat. + * @ena_llq_config: The default driver LLQ parameters configurations + */ +int ena_com_config_dev_mode(struct ena_com_dev *ena_dev, + struct ena_admin_feature_llq_desc *llq_features, + struct ena_llq_configurations *llq_default_config); + +/* ena_com_io_sq_to_ena_dev - Extract ena_com_dev using contained field io_sq. + * @io_sq: IO submit queue struct * - * Update a single entry in the interrupt moderation table. + * @return - ena_com_dev struct extracted from io_sq */ -void ena_com_init_intr_moderation_entry(struct ena_com_dev *ena_dev, - enum ena_intr_moder_level level, - struct ena_intr_moder_entry *entry); +static inline struct ena_com_dev *ena_com_io_sq_to_ena_dev(struct ena_com_io_sq *io_sq) +{ + return container_of(io_sq, struct ena_com_dev, io_sq_queues[io_sq->qid]); +} -/* ena_com_get_intr_moderation_entry - Init ena_intr_moder_entry. - * @ena_dev: ENA communication layer struct - * @level: Interrupt moderation table level - * @entry: Entry to fill. +/* ena_com_io_cq_to_ena_dev - Extract ena_com_dev using contained field io_cq. + * @io_sq: IO submit queue struct * - * Initialize the entry according to the adaptive interrupt moderation table. + * @return - ena_com_dev struct extracted from io_sq */ -void ena_com_get_intr_moderation_entry(struct ena_com_dev *ena_dev, - enum ena_intr_moder_level level, - struct ena_intr_moder_entry *entry); +static inline struct ena_com_dev *ena_com_io_cq_to_ena_dev(struct ena_com_io_cq *io_cq) +{ + return container_of(io_cq, struct ena_com_dev, io_cq_queues[io_cq->qid]); +} -static inline bool -ena_com_get_adaptive_moderation_enabled(struct ena_com_dev *ena_dev) +static inline bool ena_com_get_adaptive_moderation_enabled(struct ena_com_dev *ena_dev) { return ena_dev->adaptive_coalescing; } -static inline void -ena_com_enable_adaptive_moderation(struct ena_com_dev *ena_dev) +static inline void ena_com_enable_adaptive_moderation(struct ena_com_dev *ena_dev) { ena_dev->adaptive_coalescing = true; } -static inline void -ena_com_disable_adaptive_moderation(struct ena_com_dev *ena_dev) +static inline void ena_com_disable_adaptive_moderation(struct ena_com_dev *ena_dev) { ena_dev->adaptive_coalescing = false; } -/* ena_com_calculate_interrupt_delay - Calculate new interrupt delay - * @ena_dev: ENA communication layer struct - * @pkts: Number of packets since the last update - * @bytes: Number of bytes received since the last update. - * @smoothed_interval: Returned interval - * @moder_tbl_idx: Current table level as input update new level as return - * value. - */ -static inline void -ena_com_calculate_interrupt_delay(struct ena_com_dev *ena_dev, - unsigned int pkts, - unsigned int bytes, - unsigned int *smoothed_interval, - unsigned int *moder_tbl_idx) -{ - enum ena_intr_moder_level curr_moder_idx, new_moder_idx; - struct ena_intr_moder_entry *curr_moder_entry; - struct ena_intr_moder_entry *pred_moder_entry; - struct ena_intr_moder_entry *new_moder_entry; - struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl; - unsigned int interval; - - /* We apply adaptive moderation on Rx path only. - * Tx uses static interrupt moderation. - */ - if (!pkts || !bytes) - /* Tx interrupt, or spurious interrupt, - * in both cases we just use same delay values - */ - return; - - curr_moder_idx = (enum ena_intr_moder_level)(*moder_tbl_idx); - if (unlikely(curr_moder_idx >= ENA_INTR_MAX_NUM_OF_LEVELS)) { - ena_trc_err("Wrong moderation index %u\n", curr_moder_idx); - return; - } - - curr_moder_entry = &intr_moder_tbl[curr_moder_idx]; - new_moder_idx = curr_moder_idx; - - if (curr_moder_idx == ENA_INTR_MODER_LOWEST) { - if ((pkts > curr_moder_entry->pkts_per_interval) || - (bytes > curr_moder_entry->bytes_per_interval)) - new_moder_idx = (enum ena_intr_moder_level)(curr_moder_idx + 1); - } else { - pred_moder_entry = &intr_moder_tbl[curr_moder_idx - 1]; - - if ((pkts <= pred_moder_entry->pkts_per_interval) || - (bytes <= pred_moder_entry->bytes_per_interval)) - new_moder_idx = (enum ena_intr_moder_level)(curr_moder_idx - 1); - else if ((pkts > curr_moder_entry->pkts_per_interval) || - (bytes > curr_moder_entry->bytes_per_interval)) { - if (curr_moder_idx != ENA_INTR_MODER_HIGHEST) - new_moder_idx = (enum ena_intr_moder_level)(curr_moder_idx + 1); - } - } - new_moder_entry = &intr_moder_tbl[new_moder_idx]; - - interval = new_moder_entry->intr_moder_interval; - *smoothed_interval = ( - (interval * ENA_INTR_DELAY_NEW_VALUE_WEIGHT + - ENA_INTR_DELAY_OLD_VALUE_WEIGHT * (*smoothed_interval)) + 5) / - 10; - - *moder_tbl_idx = new_moder_idx; -} - /* ena_com_update_intr_reg - Prepare interrupt register * @intr_reg: interrupt register to update. * @rx_delay_interval: Rx interval in usecs * @tx_delay_interval: Tx interval in usecs - * @unmask: unask enable/disable + * @unmask: unmask enable/disable * * Prepare interrupt update register with the supplied parameters. */ @@ -1044,17 +998,28 @@ intr_reg->intr_control |= (tx_delay_interval << ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_SHIFT) - & ENA_ETH_IO_INTR_REG_RX_INTR_DELAY_MASK; + & ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_MASK; if (unmask) intr_reg->intr_control |= ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK; } -int ena_com_get_dev_extended_stats(struct ena_com_dev *ena_dev, char *buff, - u32 len); +static inline u8 *ena_com_get_next_bounce_buffer(struct ena_com_io_bounce_buffer_control *bounce_buf_ctrl) +{ + u16 size, buffers_num; + u8 *buf; + + size = bounce_buf_ctrl->buffer_size; + buffers_num = bounce_buf_ctrl->buffers_num; + + buf = bounce_buf_ctrl->base_buffer + + (bounce_buf_ctrl->next_to_use++ & (buffers_num - 1)) * size; -int ena_com_extended_stats_set_func_queue(struct ena_com_dev *ena_dev, - u32 funct_queue); + prefetchw(bounce_buf_ctrl->base_buffer + + (bounce_buf_ctrl->next_to_use & (buffers_num - 1)) * size); + + return buf; +} #if defined(__cplusplus) } Index: /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_com.c =================================================================== --- /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_com.c (revision 38965) +++ /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_com.c (working copy) @@ -1,35 +1,7 @@ -/*- -* BSD LICENSE -* -* Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. -* All rights reserved. -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following conditions -* are met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above copyright -* notice, this list of conditions and the following disclaimer in -* the documentation and/or other materials provided with the -* distribution. -* * Neither the name of copyright holder nor the names of its -* contributors may be used to endorse or promote products derived -* from this software without specific prior written permission. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. + * All rights reserved. + */ #include "ena_com.h" @@ -37,15 +9,11 @@ /*****************************************************************************/ /* Timeout in micro-sec */ -#define ADMIN_CMD_TIMEOUT_US (1000000) +#define ADMIN_CMD_TIMEOUT_US (3000000) -#define ENA_ASYNC_QUEUE_DEPTH 4 +#define ENA_ASYNC_QUEUE_DEPTH 16 #define ENA_ADMIN_QUEUE_DEPTH 32 -#define MIN_ENA_VER (((ENA_COMMON_SPEC_VERSION_MAJOR) << \ - ENA_REGS_VERSION_MAJOR_VERSION_SHIFT) \ - | (ENA_COMMON_SPEC_VERSION_MINOR)) - #define ENA_CTRL_MAJOR 0 #define ENA_CTRL_MINOR 0 #define ENA_CTRL_SUB_MINOR 1 @@ -62,7 +30,13 @@ #define ENA_MMIO_READ_TIMEOUT 0xFFFFFFFF -static int ena_alloc_cnt; +#define ENA_COM_BOUNCE_BUFFER_CNTRL_CNT 4 + +#define ENA_REGS_ADMIN_INTR_MASK 1 + +#define ENA_MIN_ADMIN_POLL_US 100 + +#define ENA_MAX_ADMIN_POLL_US 5000 /*****************************************************************************/ /*****************************************************************************/ @@ -86,143 +60,156 @@ bool occupied; }; -static inline int ena_com_mem_addr_set(struct ena_com_dev *ena_dev, +struct ena_com_stats_ctx { + struct ena_admin_aq_get_stats_cmd get_cmd; + struct ena_admin_acq_get_stats_resp get_resp; +}; + +static int ena_com_mem_addr_set(struct ena_com_dev *ena_dev, struct ena_common_mem_addr *ena_addr, dma_addr_t addr) { if ((addr & GENMASK_ULL(ena_dev->dma_addr_bits - 1, 0)) != addr) { - ena_trc_err("dma address has more bits that the device supports\n"); + ena_trc_err(ena_dev, "DMA address has more bits than the device supports\n"); return ENA_COM_INVAL; } - ena_addr->mem_addr_low = (u32)addr; - ena_addr->mem_addr_high = - ((addr & GENMASK_ULL(ena_dev->dma_addr_bits - 1, 32)) >> 32); + ena_addr->mem_addr_low = lower_32_bits(addr); + ena_addr->mem_addr_high = (u16)upper_32_bits(addr); return 0; } -static int ena_com_admin_init_sq(struct ena_com_admin_queue *queue) +static int ena_com_admin_init_sq(struct ena_com_admin_queue *admin_queue) { - ENA_MEM_ALLOC_COHERENT(queue->q_dmadev, - ADMIN_SQ_SIZE(queue->q_depth), - queue->sq.entries, - queue->sq.dma_addr, - queue->sq.mem_handle); + struct ena_com_dev *ena_dev = admin_queue->ena_dev; + struct ena_com_admin_sq *sq = &admin_queue->sq; + u16 size = ADMIN_SQ_SIZE(admin_queue->q_depth); - if (!queue->sq.entries) { - ena_trc_err("memory allocation failed"); + ENA_MEM_ALLOC_COHERENT(admin_queue->q_dmadev, size, sq->entries, sq->dma_addr, + sq->mem_handle); + + if (!sq->entries) { + ena_trc_err(ena_dev, "Memory allocation failed\n"); return ENA_COM_NO_MEM; } - queue->sq.head = 0; - queue->sq.tail = 0; - queue->sq.phase = 1; + sq->head = 0; + sq->tail = 0; + sq->phase = 1; - queue->sq.db_addr = NULL; + sq->db_addr = NULL; return 0; } -static int ena_com_admin_init_cq(struct ena_com_admin_queue *queue) +static int ena_com_admin_init_cq(struct ena_com_admin_queue *admin_queue) { - ENA_MEM_ALLOC_COHERENT(queue->q_dmadev, - ADMIN_CQ_SIZE(queue->q_depth), - queue->cq.entries, - queue->cq.dma_addr, - queue->cq.mem_handle); + struct ena_com_dev *ena_dev = admin_queue->ena_dev; + struct ena_com_admin_cq *cq = &admin_queue->cq; + u16 size = ADMIN_CQ_SIZE(admin_queue->q_depth); + + ENA_MEM_ALLOC_COHERENT(admin_queue->q_dmadev, size, cq->entries, cq->dma_addr, + cq->mem_handle); - if (!queue->cq.entries) { - ena_trc_err("memory allocation failed"); + if (!cq->entries) { + ena_trc_err(ena_dev, "Memory allocation failed\n"); return ENA_COM_NO_MEM; } - queue->cq.head = 0; - queue->cq.phase = 1; + cq->head = 0; + cq->phase = 1; return 0; } -static int ena_com_admin_init_aenq(struct ena_com_dev *dev, +static int ena_com_admin_init_aenq(struct ena_com_dev *ena_dev, struct ena_aenq_handlers *aenq_handlers) { + struct ena_com_aenq *aenq = &ena_dev->aenq; u32 addr_low, addr_high, aenq_caps; + u16 size; - dev->aenq.q_depth = ENA_ASYNC_QUEUE_DEPTH; - ENA_MEM_ALLOC_COHERENT(dev->dmadev, - ADMIN_AENQ_SIZE(dev->aenq.q_depth), - dev->aenq.entries, - dev->aenq.dma_addr, - dev->aenq.mem_handle); + ena_dev->aenq.q_depth = ENA_ASYNC_QUEUE_DEPTH; + size = ADMIN_AENQ_SIZE(ENA_ASYNC_QUEUE_DEPTH); + ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, size, + aenq->entries, + aenq->dma_addr, + aenq->mem_handle); - if (!dev->aenq.entries) { - ena_trc_err("memory allocation failed"); + if (!aenq->entries) { + ena_trc_err(ena_dev, "Memory allocation failed\n"); return ENA_COM_NO_MEM; } - dev->aenq.head = dev->aenq.q_depth; - dev->aenq.phase = 1; + aenq->head = aenq->q_depth; + aenq->phase = 1; - addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(dev->aenq.dma_addr); - addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(dev->aenq.dma_addr); + addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(aenq->dma_addr); + addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(aenq->dma_addr); - ENA_REG_WRITE32(addr_low, (unsigned char *)dev->reg_bar - + ENA_REGS_AENQ_BASE_LO_OFF); - ENA_REG_WRITE32(addr_high, (unsigned char *)dev->reg_bar - + ENA_REGS_AENQ_BASE_HI_OFF); + ENA_REG_WRITE32(ena_dev->bus, addr_low, ena_dev->reg_bar + ENA_REGS_AENQ_BASE_LO_OFF); + ENA_REG_WRITE32(ena_dev->bus, addr_high, ena_dev->reg_bar + ENA_REGS_AENQ_BASE_HI_OFF); aenq_caps = 0; - aenq_caps |= dev->aenq.q_depth & ENA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK; + aenq_caps |= ena_dev->aenq.q_depth & ENA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK; aenq_caps |= (sizeof(struct ena_admin_aenq_entry) << ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_SHIFT) & ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_MASK; + ENA_REG_WRITE32(ena_dev->bus, aenq_caps, ena_dev->reg_bar + ENA_REGS_AENQ_CAPS_OFF); - ENA_REG_WRITE32(aenq_caps, (unsigned char *)dev->reg_bar - + ENA_REGS_AENQ_CAPS_OFF); - - if (unlikely(!aenq_handlers)) - ena_trc_err("aenq handlers pointer is NULL\n"); + if (unlikely(!aenq_handlers)) { + ena_trc_err(ena_dev, "AENQ handlers pointer is NULL\n"); + return ENA_COM_INVAL; + } - dev->aenq.aenq_handlers = aenq_handlers; + aenq->aenq_handlers = aenq_handlers; return 0; } -static inline void comp_ctxt_release(struct ena_com_admin_queue *queue, +static void comp_ctxt_release(struct ena_com_admin_queue *queue, struct ena_comp_ctx *comp_ctx) { comp_ctx->occupied = false; ATOMIC32_DEC(&queue->outstanding_cmds); } -static struct ena_comp_ctx *get_comp_ctxt(struct ena_com_admin_queue *queue, +static struct ena_comp_ctx *get_comp_ctxt(struct ena_com_admin_queue *admin_queue, u16 command_id, bool capture) { - if (unlikely(command_id >= queue->q_depth)) { - ena_trc_err("command id is larger than the queue size. cmd_id: %u queue size %d\n", - command_id, queue->q_depth); + if (unlikely(command_id >= admin_queue->q_depth)) { + ena_trc_err(admin_queue->ena_dev, + "Command id is larger than the queue size. cmd_id: %u queue size %d\n", + command_id, admin_queue->q_depth); + return NULL; + } + + if (unlikely(!admin_queue->comp_ctx)) { + ena_trc_err(admin_queue->ena_dev, + "Completion context is NULL\n"); return NULL; } - if (unlikely(queue->comp_ctx[command_id].occupied && capture)) { - ena_trc_err("Completion context is occupied\n"); + if (unlikely(admin_queue->comp_ctx[command_id].occupied && capture)) { + ena_trc_err(admin_queue->ena_dev, + "Completion context is occupied\n"); return NULL; } if (capture) { - ATOMIC32_INC(&queue->outstanding_cmds); - queue->comp_ctx[command_id].occupied = true; + ATOMIC32_INC(&admin_queue->outstanding_cmds); + admin_queue->comp_ctx[command_id].occupied = true; } - return &queue->comp_ctx[command_id]; + return &admin_queue->comp_ctx[command_id]; } -static struct ena_comp_ctx * -__ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue, - struct ena_admin_aq_entry *cmd, - size_t cmd_size_in_bytes, - struct ena_admin_acq_entry *comp, - size_t comp_size_in_bytes) +static struct ena_comp_ctx *__ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue, + struct ena_admin_aq_entry *cmd, + size_t cmd_size_in_bytes, + struct ena_admin_acq_entry *comp, + size_t comp_size_in_bytes) { struct ena_comp_ctx *comp_ctx; u16 tail_masked, cmd_id; @@ -234,12 +221,9 @@ tail_masked = admin_queue->sq.tail & queue_size_mask; /* In case of queue FULL */ - cnt = admin_queue->sq.tail - admin_queue->sq.head; + cnt = (u16)ATOMIC32_READ(&admin_queue->outstanding_cmds); if (cnt >= admin_queue->q_depth) { - ena_trc_dbg("admin queue is FULL (tail %d head %d depth: %d)\n", - admin_queue->sq.tail, - admin_queue->sq.head, - admin_queue->q_depth); + ena_trc_dbg(admin_queue->ena_dev, "Admin queue is full.\n"); admin_queue->stats.out_of_space++; return ERR_PTR(ENA_COM_NO_SPACE); } @@ -253,6 +237,8 @@ ENA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK; comp_ctx = get_comp_ctxt(admin_queue, cmd_id, true); + if (unlikely(!comp_ctx)) + return ERR_PTR(ENA_COM_INVAL); comp_ctx->status = ENA_CMD_SUBMITTED; comp_ctx->comp_size = (u32)comp_size_in_bytes; @@ -272,25 +258,28 @@ if (unlikely((admin_queue->sq.tail & queue_size_mask) == 0)) admin_queue->sq.phase = !admin_queue->sq.phase; - ENA_REG_WRITE32(admin_queue->sq.tail, admin_queue->sq.db_addr); + ENA_DB_SYNC(&admin_queue->sq.mem_handle); + ENA_REG_WRITE32(admin_queue->bus, admin_queue->sq.tail, + admin_queue->sq.db_addr); return comp_ctx; } -static inline int ena_com_init_comp_ctxt(struct ena_com_admin_queue *queue) +static int ena_com_init_comp_ctxt(struct ena_com_admin_queue *admin_queue) { - size_t size = queue->q_depth * sizeof(struct ena_comp_ctx); + struct ena_com_dev *ena_dev = admin_queue->ena_dev; + size_t size = admin_queue->q_depth * sizeof(struct ena_comp_ctx); struct ena_comp_ctx *comp_ctx; u16 i; - queue->comp_ctx = ENA_MEM_ALLOC(queue->q_dmadev, size); - if (unlikely(!queue->comp_ctx)) { - ena_trc_err("memory allocation failed"); + admin_queue->comp_ctx = ENA_MEM_ALLOC(admin_queue->q_dmadev, size); + if (unlikely(!admin_queue->comp_ctx)) { + ena_trc_err(ena_dev, "Memory allocation failed\n"); return ENA_COM_NO_MEM; } - for (i = 0; i < queue->q_depth; i++) { - comp_ctx = get_comp_ctxt(queue, i, false); + for (i = 0; i < admin_queue->q_depth; i++) { + comp_ctx = get_comp_ctxt(admin_queue, i, false); if (comp_ctx) ENA_WAIT_EVENT_INIT(comp_ctx->wait_event); } @@ -298,12 +287,11 @@ return 0; } -static struct ena_comp_ctx * -ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue, - struct ena_admin_aq_entry *cmd, - size_t cmd_size_in_bytes, - struct ena_admin_acq_entry *comp, - size_t comp_size_in_bytes) +static struct ena_comp_ctx *ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue, + struct ena_admin_aq_entry *cmd, + size_t cmd_size_in_bytes, + struct ena_admin_acq_entry *comp, + size_t comp_size_in_bytes) { unsigned long flags = 0; struct ena_comp_ctx *comp_ctx; @@ -317,7 +305,7 @@ cmd_size_in_bytes, comp, comp_size_in_bytes); - if (unlikely(IS_ERR(comp_ctx))) + if (IS_ERR(comp_ctx)) admin_queue->running_state = false; ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); @@ -331,44 +319,79 @@ size_t size; int dev_node = 0; - ENA_TOUCH(ctx); - - memset(&io_sq->desc_addr, 0x0, sizeof(struct ena_com_io_desc_addr)); + memset(&io_sq->desc_addr, 0x0, sizeof(io_sq->desc_addr)); + io_sq->dma_addr_bits = (u8)ena_dev->dma_addr_bits; io_sq->desc_entry_size = (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ? sizeof(struct ena_eth_io_tx_desc) : sizeof(struct ena_eth_io_rx_desc); size = io_sq->desc_entry_size * io_sq->q_depth; + io_sq->bus = ena_dev->bus; if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) { ENA_MEM_ALLOC_COHERENT_NODE(ena_dev->dmadev, size, io_sq->desc_addr.virt_addr, io_sq->desc_addr.phys_addr, + io_sq->desc_addr.mem_handle, ctx->numa_node, dev_node); - if (!io_sq->desc_addr.virt_addr) + if (!io_sq->desc_addr.virt_addr) { ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, size, io_sq->desc_addr.virt_addr, io_sq->desc_addr.phys_addr, io_sq->desc_addr.mem_handle); - } else { + } + + if (!io_sq->desc_addr.virt_addr) { + ena_trc_err(ena_dev, "Memory allocation failed\n"); + return ENA_COM_NO_MEM; + } + } + + if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { + /* Allocate bounce buffers */ + io_sq->bounce_buf_ctrl.buffer_size = + ena_dev->llq_info.desc_list_entry_size; + io_sq->bounce_buf_ctrl.buffers_num = + ENA_COM_BOUNCE_BUFFER_CNTRL_CNT; + io_sq->bounce_buf_ctrl.next_to_use = 0; + + size = io_sq->bounce_buf_ctrl.buffer_size * + io_sq->bounce_buf_ctrl.buffers_num; + ENA_MEM_ALLOC_NODE(ena_dev->dmadev, size, - io_sq->desc_addr.virt_addr, + io_sq->bounce_buf_ctrl.base_buffer, ctx->numa_node, dev_node); - if (!io_sq->desc_addr.virt_addr) - io_sq->desc_addr.virt_addr = - ENA_MEM_ALLOC(ena_dev->dmadev, size); - } + if (!io_sq->bounce_buf_ctrl.base_buffer) + io_sq->bounce_buf_ctrl.base_buffer = ENA_MEM_ALLOC(ena_dev->dmadev, size); - if (!io_sq->desc_addr.virt_addr) { - ena_trc_err("memory allocation failed"); - return ENA_COM_NO_MEM; + if (!io_sq->bounce_buf_ctrl.base_buffer) { + ena_trc_err(ena_dev, "Bounce buffer memory allocation failed\n"); + return ENA_COM_NO_MEM; + } + + memcpy(&io_sq->llq_info, &ena_dev->llq_info, + sizeof(io_sq->llq_info)); + + /* Initiate the first bounce buffer */ + io_sq->llq_buf_ctrl.curr_bounce_buf = + ena_com_get_next_bounce_buffer(&io_sq->bounce_buf_ctrl); + memset(io_sq->llq_buf_ctrl.curr_bounce_buf, + 0x0, io_sq->llq_info.desc_list_entry_size); + io_sq->llq_buf_ctrl.descs_left_in_line = + io_sq->llq_info.descs_num_before_header; + io_sq->disable_meta_caching = + io_sq->llq_info.disable_meta_caching; + + if (io_sq->llq_info.max_entries_in_tx_burst > 0) + io_sq->entries_in_tx_burst_left = + io_sq->llq_info.max_entries_in_tx_burst; } io_sq->tail = 0; @@ -385,8 +408,7 @@ size_t size; int prev_node = 0; - ENA_TOUCH(ctx); - memset(&io_cq->cdesc_addr, 0x0, sizeof(struct ena_com_io_desc_addr)); + memset(&io_cq->cdesc_addr, 0x0, sizeof(io_cq->cdesc_addr)); /* Use the basic completion descriptor for Rx */ io_cq->cdesc_entry_size_in_bytes = @@ -395,22 +417,27 @@ sizeof(struct ena_eth_io_rx_cdesc_base); size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth; + io_cq->bus = ena_dev->bus; - ENA_MEM_ALLOC_COHERENT_NODE(ena_dev->dmadev, - size, - io_cq->cdesc_addr.virt_addr, - io_cq->cdesc_addr.phys_addr, - ctx->numa_node, - prev_node); - if (!io_cq->cdesc_addr.virt_addr) - ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, - size, - io_cq->cdesc_addr.virt_addr, - io_cq->cdesc_addr.phys_addr, - io_cq->cdesc_addr.mem_handle); + ENA_MEM_ALLOC_COHERENT_NODE_ALIGNED(ena_dev->dmadev, + size, + io_cq->cdesc_addr.virt_addr, + io_cq->cdesc_addr.phys_addr, + io_cq->cdesc_addr.mem_handle, + ctx->numa_node, + prev_node, + ENA_CDESC_RING_SIZE_ALIGNMENT); + if (!io_cq->cdesc_addr.virt_addr) { + ENA_MEM_ALLOC_COHERENT_ALIGNED(ena_dev->dmadev, + size, + io_cq->cdesc_addr.virt_addr, + io_cq->cdesc_addr.phys_addr, + io_cq->cdesc_addr.mem_handle, + ENA_CDESC_RING_SIZE_ALIGNMENT); + } if (!io_cq->cdesc_addr.virt_addr) { - ena_trc_err("memory allocation failed"); + ena_trc_err(ena_dev, "Memory allocation failed\n"); return ENA_COM_NO_MEM; } @@ -420,9 +447,8 @@ return 0; } -static void -ena_com_handle_single_admin_completion(struct ena_com_admin_queue *admin_queue, - struct ena_admin_acq_entry *cqe) +static void ena_com_handle_single_admin_completion(struct ena_com_admin_queue *admin_queue, + struct ena_admin_acq_entry *cqe) { struct ena_comp_ctx *comp_ctx; u16 cmd_id; @@ -432,7 +458,8 @@ comp_ctx = get_comp_ctxt(admin_queue, cmd_id, false); if (unlikely(!comp_ctx)) { - ena_trc_err("comp_ctx is NULL. Changing the admin queue running state\n"); + ena_trc_err(admin_queue->ena_dev, + "comp_ctx is NULL. Changing the admin queue running state\n"); admin_queue->running_state = false; return; } @@ -447,8 +474,7 @@ ENA_WAIT_EVENT_SIGNAL(comp_ctx->wait_event); } -static void -ena_com_handle_admin_completion(struct ena_com_admin_queue *admin_queue) +static void ena_com_handle_admin_completion(struct ena_com_admin_queue *admin_queue) { struct ena_admin_acq_entry *cqe = NULL; u16 comp_num = 0; @@ -461,12 +487,12 @@ cqe = &admin_queue->cq.entries[head_masked]; /* Go over all the completions */ - while ((cqe->acq_common_descriptor.flags & + while ((READ_ONCE8(cqe->acq_common_descriptor.flags) & ENA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK) == phase) { /* Do not read the rest of the completion entry before the * phase bit was validated */ - rmb(); + dma_rmb(); ena_com_handle_single_admin_completion(admin_queue, cqe); head_masked++; @@ -485,46 +511,60 @@ admin_queue->stats.completed_cmd += comp_num; } -static int ena_com_comp_status_to_errno(u8 comp_status) +static int ena_com_comp_status_to_errno(struct ena_com_admin_queue *admin_queue, + u8 comp_status) { if (unlikely(comp_status != 0)) - ena_trc_err("admin command failed[%u]\n", comp_status); - - if (unlikely(comp_status > ENA_ADMIN_UNKNOWN_ERROR)) - return ENA_COM_INVAL; + ena_trc_err(admin_queue->ena_dev, + "Admin command failed[%u]\n", comp_status); switch (comp_status) { case ENA_ADMIN_SUCCESS: - return 0; + return ENA_COM_OK; case ENA_ADMIN_RESOURCE_ALLOCATION_FAILURE: return ENA_COM_NO_MEM; case ENA_ADMIN_UNSUPPORTED_OPCODE: - return ENA_COM_PERMISSION; + return ENA_COM_UNSUPPORTED; case ENA_ADMIN_BAD_OPCODE: case ENA_ADMIN_MALFORMED_REQUEST: case ENA_ADMIN_ILLEGAL_PARAMETER: case ENA_ADMIN_UNKNOWN_ERROR: return ENA_COM_INVAL; + case ENA_ADMIN_RESOURCE_BUSY: + return ENA_COM_TRY_AGAIN; } - return 0; + return ENA_COM_INVAL; +} + +static void ena_delay_exponential_backoff_us(u32 exp, u32 delay_us) +{ + delay_us = ENA_MAX32(ENA_MIN_ADMIN_POLL_US, delay_us); + delay_us = ENA_MIN32(delay_us * (1U << exp), ENA_MAX_ADMIN_POLL_US); + ENA_USLEEP(delay_us); } -static int -ena_com_wait_and_process_admin_cq_polling( - struct ena_comp_ctx *comp_ctx, - struct ena_com_admin_queue *admin_queue) +static int ena_com_wait_and_process_admin_cq_polling(struct ena_comp_ctx *comp_ctx, + struct ena_com_admin_queue *admin_queue) { unsigned long flags = 0; - u64 start_time; + ena_time_t timeout; int ret; + u32 exp = 0; + + timeout = ENA_GET_SYSTEM_TIMEOUT(admin_queue->completion_timeout); - start_time = ENA_GET_SYSTEM_USECS(); + while (1) { + ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); + ena_com_handle_admin_completion(admin_queue); + ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); - while (comp_ctx->status == ENA_CMD_SUBMITTED) { - if ((ENA_GET_SYSTEM_USECS() - start_time) > - ADMIN_CMD_TIMEOUT_US) { - ena_trc_err("Wait for completion (polling) timeout\n"); + if (comp_ctx->status != ENA_CMD_SUBMITTED) + break; + + if (ENA_TIME_EXPIRE(timeout)) { + ena_trc_err(admin_queue->ena_dev, + "Wait for completion (polling) timeout\n"); /* ENA didn't have any completion */ ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); admin_queue->stats.no_completion++; @@ -535,13 +575,12 @@ goto err; } - ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); - ena_com_handle_admin_completion(admin_queue); - ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); + ena_delay_exponential_backoff_us(exp++, + admin_queue->ena_dev->ena_min_poll_delay_us); } if (unlikely(comp_ctx->status == ENA_CMD_ABORTED)) { - ena_trc_err("Command was aborted\n"); + ena_trc_err(admin_queue->ena_dev, "Command was aborted\n"); ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); admin_queue->stats.aborted_cmd++; ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); @@ -549,25 +588,193 @@ goto err; } - ENA_ASSERT(comp_ctx->status == ENA_CMD_COMPLETED, - "Invalid comp status %d\n", comp_ctx->status); + ENA_WARN(comp_ctx->status != ENA_CMD_COMPLETED, + admin_queue->ena_dev, "Invalid comp status %d\n", + comp_ctx->status); - ret = ena_com_comp_status_to_errno(comp_ctx->comp_status); + ret = ena_com_comp_status_to_errno(admin_queue, comp_ctx->comp_status); err: comp_ctxt_release(admin_queue, comp_ctx); return ret; } -static int -ena_com_wait_and_process_admin_cq_interrupts( - struct ena_comp_ctx *comp_ctx, - struct ena_com_admin_queue *admin_queue) +/* + * Set the LLQ configurations of the firmware + * + * The driver provides only the enabled feature values to the device, + * which in turn, checks if they are supported. + */ +static int ena_com_set_llq(struct ena_com_dev *ena_dev) +{ + struct ena_com_admin_queue *admin_queue; + struct ena_admin_set_feat_cmd cmd; + struct ena_admin_set_feat_resp resp; + struct ena_com_llq_info *llq_info = &ena_dev->llq_info; + int ret; + + memset(&cmd, 0x0, sizeof(cmd)); + admin_queue = &ena_dev->admin_queue; + + cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; + cmd.feat_common.feature_id = ENA_ADMIN_LLQ; + + cmd.u.llq.header_location_ctrl_enabled = llq_info->header_location_ctrl; + cmd.u.llq.entry_size_ctrl_enabled = llq_info->desc_list_entry_size_ctrl; + cmd.u.llq.desc_num_before_header_enabled = llq_info->descs_num_before_header; + cmd.u.llq.descriptors_stride_ctrl_enabled = llq_info->desc_stride_ctrl; + + cmd.u.llq.accel_mode.u.set.enabled_flags = + BIT(ENA_ADMIN_DISABLE_META_CACHING) | + BIT(ENA_ADMIN_LIMIT_TX_BURST); + + ret = ena_com_execute_admin_command(admin_queue, + (struct ena_admin_aq_entry *)&cmd, + sizeof(cmd), + (struct ena_admin_acq_entry *)&resp, + sizeof(resp)); + + if (unlikely(ret)) + ena_trc_err(ena_dev, "Failed to set LLQ configurations: %d\n", ret); + + return ret; +} + +static int ena_com_config_llq_info(struct ena_com_dev *ena_dev, + struct ena_admin_feature_llq_desc *llq_features, + struct ena_llq_configurations *llq_default_cfg) +{ + struct ena_com_llq_info *llq_info = &ena_dev->llq_info; + struct ena_admin_accel_mode_get llq_accel_mode_get; + u16 supported_feat; + int rc; + + memset(llq_info, 0, sizeof(*llq_info)); + + supported_feat = llq_features->header_location_ctrl_supported; + + if (likely(supported_feat & llq_default_cfg->llq_header_location)) { + llq_info->header_location_ctrl = + llq_default_cfg->llq_header_location; + } else { + ena_trc_err(ena_dev, "Invalid header location control, supported: 0x%x\n", + supported_feat); + return -EINVAL; + } + + if (likely(llq_info->header_location_ctrl == ENA_ADMIN_INLINE_HEADER)) { + supported_feat = llq_features->descriptors_stride_ctrl_supported; + if (likely(supported_feat & llq_default_cfg->llq_stride_ctrl)) { + llq_info->desc_stride_ctrl = llq_default_cfg->llq_stride_ctrl; + } else { + if (supported_feat & ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY) { + llq_info->desc_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY; + } else if (supported_feat & ENA_ADMIN_SINGLE_DESC_PER_ENTRY) { + llq_info->desc_stride_ctrl = ENA_ADMIN_SINGLE_DESC_PER_ENTRY; + } else { + ena_trc_err(ena_dev, "Invalid desc_stride_ctrl, supported: 0x%x\n", + supported_feat); + return -EINVAL; + } + + ena_trc_err(ena_dev, "Default llq stride ctrl is not supported, performing fallback, default: 0x%x, supported: 0x%x, used: 0x%x\n", + llq_default_cfg->llq_stride_ctrl, + supported_feat, + llq_info->desc_stride_ctrl); + } + } else { + llq_info->desc_stride_ctrl = 0; + } + + supported_feat = llq_features->entry_size_ctrl_supported; + if (likely(supported_feat & llq_default_cfg->llq_ring_entry_size)) { + llq_info->desc_list_entry_size_ctrl = llq_default_cfg->llq_ring_entry_size; + llq_info->desc_list_entry_size = llq_default_cfg->llq_ring_entry_size_value; + } else { + if (supported_feat & ENA_ADMIN_LIST_ENTRY_SIZE_128B) { + llq_info->desc_list_entry_size_ctrl = ENA_ADMIN_LIST_ENTRY_SIZE_128B; + llq_info->desc_list_entry_size = 128; + } else if (supported_feat & ENA_ADMIN_LIST_ENTRY_SIZE_192B) { + llq_info->desc_list_entry_size_ctrl = ENA_ADMIN_LIST_ENTRY_SIZE_192B; + llq_info->desc_list_entry_size = 192; + } else if (supported_feat & ENA_ADMIN_LIST_ENTRY_SIZE_256B) { + llq_info->desc_list_entry_size_ctrl = ENA_ADMIN_LIST_ENTRY_SIZE_256B; + llq_info->desc_list_entry_size = 256; + } else { + ena_trc_err(ena_dev, "Invalid entry_size_ctrl, supported: 0x%x\n", + supported_feat); + return -EINVAL; + } + + ena_trc_err(ena_dev, "Default llq ring entry size is not supported, performing fallback, default: 0x%x, supported: 0x%x, used: 0x%x\n", + llq_default_cfg->llq_ring_entry_size, + supported_feat, + llq_info->desc_list_entry_size); + } + if (unlikely(llq_info->desc_list_entry_size & 0x7)) { + /* The desc list entry size should be whole multiply of 8 + * This requirement comes from __iowrite64_copy() + */ + ena_trc_err(ena_dev, "Illegal entry size %d\n", + llq_info->desc_list_entry_size); + return -EINVAL; + } + + if (llq_info->desc_stride_ctrl == ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY) + llq_info->descs_per_entry = llq_info->desc_list_entry_size / + sizeof(struct ena_eth_io_tx_desc); + else + llq_info->descs_per_entry = 1; + + supported_feat = llq_features->desc_num_before_header_supported; + if (likely(supported_feat & llq_default_cfg->llq_num_decs_before_header)) { + llq_info->descs_num_before_header = llq_default_cfg->llq_num_decs_before_header; + } else { + if (supported_feat & ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2) { + llq_info->descs_num_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2; + } else if (supported_feat & ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_1) { + llq_info->descs_num_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_1; + } else if (supported_feat & ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_4) { + llq_info->descs_num_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_4; + } else if (supported_feat & ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_8) { + llq_info->descs_num_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_8; + } else { + ena_trc_err(ena_dev, "Invalid descs_num_before_header, supported: 0x%x\n", + supported_feat); + return -EINVAL; + } + + ena_trc_err(ena_dev, "Default llq num descs before header is not supported, performing fallback, default: 0x%x, supported: 0x%x, used: 0x%x\n", + llq_default_cfg->llq_num_decs_before_header, + supported_feat, + llq_info->descs_num_before_header); + } + /* Check for accelerated queue supported */ + llq_accel_mode_get = llq_features->accel_mode.u.get; + + llq_info->disable_meta_caching = + !!(llq_accel_mode_get.supported_flags & + BIT(ENA_ADMIN_DISABLE_META_CACHING)); + + if (llq_accel_mode_get.supported_flags & BIT(ENA_ADMIN_LIMIT_TX_BURST)) + llq_info->max_entries_in_tx_burst = + llq_accel_mode_get.max_tx_burst_size / + llq_default_cfg->llq_ring_entry_size_value; + + rc = ena_com_set_llq(ena_dev); + if (rc) + ena_trc_err(ena_dev, "Cannot set LLQ configuration: %d\n", rc); + + return rc; +} + +static int ena_com_wait_and_process_admin_cq_interrupts(struct ena_comp_ctx *comp_ctx, + struct ena_com_admin_queue *admin_queue) { unsigned long flags = 0; - int ret = 0; + int ret; ENA_WAIT_EVENT_WAIT(comp_ctx->wait_event, - ADMIN_CMD_TIMEOUT_US); + admin_queue->completion_timeout); /* In case the command wasn't completed find out the root cause. * There might be 2 kinds of errors @@ -580,19 +787,30 @@ admin_queue->stats.no_completion++; ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); - if (comp_ctx->status == ENA_CMD_COMPLETED) - ena_trc_err("The ena device have completion but the driver didn't receive any MSI-X interrupt (cmd %d)\n", - comp_ctx->cmd_opcode); - else - ena_trc_err("The ena device doesn't send any completion for the admin cmd %d status %d\n", + if (comp_ctx->status == ENA_CMD_COMPLETED) { + ena_trc_err(admin_queue->ena_dev, + "The ena device sent a completion but the driver didn't receive a MSI-X interrupt (cmd %d), autopolling mode is %s\n", + comp_ctx->cmd_opcode, admin_queue->auto_polling ? "ON" : "OFF"); + /* Check if fallback to polling is enabled */ + if (admin_queue->auto_polling) + admin_queue->polling = true; + } else { + ena_trc_err(admin_queue->ena_dev, + "The ena device didn't send a completion for the admin cmd %d status %d\n", comp_ctx->cmd_opcode, comp_ctx->status); - - admin_queue->running_state = false; - ret = ENA_COM_TIMER_EXPIRED; - goto err; + } + /* Check if shifted to polling mode. + * This will happen if there is a completion without an interrupt + * and autopolling mode is enabled. Continuing normal execution in such case + */ + if (!admin_queue->polling) { + admin_queue->running_state = false; + ret = ENA_COM_TIMER_EXPIRED; + goto err; + } } - ret = ena_com_comp_status_to_errno(comp_ctx->comp_status); + ret = ena_com_comp_status_to_errno(admin_queue, comp_ctx->comp_status); err: comp_ctxt_release(admin_queue, comp_ctx); return ret; @@ -607,16 +825,18 @@ struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read; volatile struct ena_admin_ena_mmio_req_read_less_resp *read_resp = mmio_read->read_resp; - u32 mmio_read_reg, ret; + u32 mmio_read_reg, ret, i; unsigned long flags = 0; - int i; + u32 timeout = mmio_read->reg_read_to; ENA_MIGHT_SLEEP(); + if (timeout == 0) + timeout = ENA_REG_READ_TIMEOUT; + /* If readless is disabled, perform regular read */ if (!mmio_read->readless_supported) - return ENA_REG_READ32((unsigned char *)ena_dev->reg_bar + - offset); + return ENA_REG_READ32(ena_dev->bus, ena_dev->reg_bar + offset); ENA_SPINLOCK_LOCK(mmio_read->lock, flags); mmio_read->seq_num++; @@ -627,23 +847,18 @@ mmio_read_reg |= mmio_read->seq_num & ENA_REGS_MMIO_REG_READ_REQ_ID_MASK; - /* make sure read_resp->req_id get updated before the hw can write - * there - */ - wmb(); - - ENA_REG_WRITE32(mmio_read_reg, (unsigned char *)ena_dev->reg_bar - + ENA_REGS_MMIO_REG_READ_OFF); + ENA_REG_WRITE32(ena_dev->bus, mmio_read_reg, + ena_dev->reg_bar + ENA_REGS_MMIO_REG_READ_OFF); - for (i = 0; i < ENA_REG_READ_TIMEOUT; i++) { - if (read_resp->req_id == mmio_read->seq_num) + for (i = 0; i < timeout; i++) { + if (READ_ONCE16(read_resp->req_id) == mmio_read->seq_num) break; ENA_UDELAY(1); } - if (unlikely(i == ENA_REG_READ_TIMEOUT)) { - ena_trc_err("reading reg failed for timeout. expected: req id[%hu] offset[%hu] actual: req id[%hu] offset[%hu]\n", + if (unlikely(i == timeout)) { + ena_trc_err(ena_dev, "Reading reg failed for timeout. expected: req id[%hu] offset[%hu] actual: req id[%hu] offset[%hu]\n", mmio_read->seq_num, offset, read_resp->req_id, @@ -653,7 +868,7 @@ } if (read_resp->reg_off != offset) { - ena_trc_err("reading failed for wrong offset value"); + ena_trc_err(ena_dev, "Read failure: wrong offset provided\n"); ret = ENA_MMIO_READ_TIMEOUT; } else { ret = read_resp->reg_val; @@ -671,9 +886,8 @@ * It is expected that the IRQ called ena_com_handle_admin_completion * to mark the completions. */ -static int -ena_com_wait_and_process_admin_cq(struct ena_comp_ctx *comp_ctx, - struct ena_com_admin_queue *admin_queue) +static int ena_com_wait_and_process_admin_cq(struct ena_comp_ctx *comp_ctx, + struct ena_com_admin_queue *admin_queue) { if (admin_queue->polling) return ena_com_wait_and_process_admin_cq_polling(comp_ctx, @@ -692,7 +906,7 @@ u8 direction; int ret; - memset(&destroy_cmd, 0x0, sizeof(struct ena_admin_aq_destroy_sq_cmd)); + memset(&destroy_cmd, 0x0, sizeof(destroy_cmd)); if (io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) direction = ENA_ADMIN_SQ_DIRECTION_TX; @@ -706,15 +920,14 @@ destroy_cmd.sq.sq_idx = io_sq->idx; destroy_cmd.aq_common_descriptor.opcode = ENA_ADMIN_DESTROY_SQ; - ret = ena_com_execute_admin_command( - admin_queue, - (struct ena_admin_aq_entry *)&destroy_cmd, - sizeof(destroy_cmd), - (struct ena_admin_acq_entry *)&destroy_resp, - sizeof(destroy_resp)); + ret = ena_com_execute_admin_command(admin_queue, + (struct ena_admin_aq_entry *)&destroy_cmd, + sizeof(destroy_cmd), + (struct ena_admin_acq_entry *)&destroy_resp, + sizeof(destroy_resp)); if (unlikely(ret && (ret != ENA_COM_NO_DEVICE))) - ena_trc_err("failed to destroy io sq error: %d\n", ret); + ena_trc_err(ena_dev, "Failed to destroy io sq error: %d\n", ret); return ret; } @@ -740,30 +953,37 @@ if (io_sq->desc_addr.virt_addr) { size = io_sq->desc_entry_size * io_sq->q_depth; - if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) - ENA_MEM_FREE_COHERENT(ena_dev->dmadev, - size, - io_sq->desc_addr.virt_addr, - io_sq->desc_addr.phys_addr, - io_sq->desc_addr.mem_handle); - else - ENA_MEM_FREE(ena_dev->dmadev, - io_sq->desc_addr.virt_addr); + ENA_MEM_FREE_COHERENT(ena_dev->dmadev, + size, + io_sq->desc_addr.virt_addr, + io_sq->desc_addr.phys_addr, + io_sq->desc_addr.mem_handle); io_sq->desc_addr.virt_addr = NULL; } + + if (io_sq->bounce_buf_ctrl.base_buffer) { + ENA_MEM_FREE(ena_dev->dmadev, + io_sq->bounce_buf_ctrl.base_buffer, + (io_sq->llq_info.desc_list_entry_size * ENA_COM_BOUNCE_BUFFER_CNTRL_CNT)); + io_sq->bounce_buf_ctrl.base_buffer = NULL; + } } -static int wait_for_reset_state(struct ena_com_dev *ena_dev, - u32 timeout, u16 exp_state) +static int wait_for_reset_state(struct ena_com_dev *ena_dev, u32 timeout, + u16 exp_state) { - u32 val, i; + u32 val, exp = 0; + ena_time_t timeout_stamp; - for (i = 0; i < timeout; i++) { + /* Convert timeout from resolution of 100ms to us resolution. */ + timeout_stamp = ENA_GET_SYSTEM_TIMEOUT(100 * 1000 * timeout); + + while (1) { val = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF); if (unlikely(val == ENA_MMIO_READ_TIMEOUT)) { - ena_trc_err("Reg read timeout occurred\n"); + ena_trc_err(ena_dev, "Reg read timeout occurred\n"); return ENA_COM_TIMER_EXPIRED; } @@ -771,16 +991,15 @@ exp_state) return 0; - /* The resolution of the timeout is 100ms */ - ENA_MSLEEP(100); - } + if (ENA_TIME_EXPIRE(timeout_stamp)) + return ENA_COM_TIMER_EXPIRED; - return ENA_COM_TIMER_EXPIRED; + ena_delay_exponential_backoff_us(exp++, ena_dev->ena_min_poll_delay_us); + } } -static bool -ena_com_check_supported_feature_id(struct ena_com_dev *ena_dev, - enum ena_admin_aq_feature_id feature_id) +static bool ena_com_check_supported_feature_id(struct ena_com_dev *ena_dev, + enum ena_admin_aq_feature_id feature_id) { u32 feature_mask = 1 << feature_id; @@ -796,20 +1015,16 @@ struct ena_admin_get_feat_resp *get_resp, enum ena_admin_aq_feature_id feature_id, dma_addr_t control_buf_dma_addr, - u32 control_buff_size) + u32 control_buff_size, + u8 feature_ver) { struct ena_com_admin_queue *admin_queue; struct ena_admin_get_feat_cmd get_cmd; int ret; - if (!ena_dev) { - ena_trc_err("%s : ena_dev is NULL\n", __func__); - return ENA_COM_NO_DEVICE; - } - if (!ena_com_check_supported_feature_id(ena_dev, feature_id)) { - ena_trc_info("Feature %d isn't supported\n", feature_id); - return ENA_COM_PERMISSION; + ena_trc_dbg(ena_dev, "Feature %d isn't supported\n", feature_id); + return ENA_COM_UNSUPPORTED; } memset(&get_cmd, 0x0, sizeof(get_cmd)); @@ -827,12 +1042,12 @@ &get_cmd.control_buffer.address, control_buf_dma_addr); if (unlikely(ret)) { - ena_trc_err("memory address set failed\n"); + ena_trc_err(ena_dev, "Memory address set failed\n"); return ret; } get_cmd.control_buffer.length = control_buff_size; - + get_cmd.feat_common.feature_version = feature_ver; get_cmd.feat_common.feature_id = feature_id; ret = ena_com_execute_admin_command(admin_queue, @@ -844,7 +1059,7 @@ sizeof(*get_resp)); if (unlikely(ret)) - ena_trc_err("Failed to submit get_feature command %d error: %d\n", + ena_trc_err(ena_dev, "Failed to submit get_feature command %d error: %d\n", feature_id, ret); return ret; @@ -852,19 +1067,41 @@ static int ena_com_get_feature(struct ena_com_dev *ena_dev, struct ena_admin_get_feat_resp *get_resp, - enum ena_admin_aq_feature_id feature_id) + enum ena_admin_aq_feature_id feature_id, + u8 feature_ver) { return ena_com_get_feature_ex(ena_dev, get_resp, feature_id, 0, - 0); + 0, + feature_ver); +} + +int ena_com_get_current_hash_function(struct ena_com_dev *ena_dev) +{ + return ena_dev->rss.hash_func; +} + +static void ena_com_hash_key_fill_default_key(struct ena_com_dev *ena_dev) +{ + struct ena_admin_feature_rss_flow_hash_control *hash_key = + (ena_dev->rss).hash_key; + + ENA_RSS_FILL_KEY(&hash_key->key, sizeof(hash_key->key)); + /* The key buffer is stored in the device in an array of + * uint32 elements. + */ + hash_key->key_parts = ENA_ADMIN_RSS_KEY_PARTS; } static int ena_com_hash_key_allocate(struct ena_com_dev *ena_dev) { struct ena_rss *rss = &ena_dev->rss; + if (!ena_com_check_supported_feature_id(ena_dev, ENA_ADMIN_RSS_HASH_FUNCTION)) + return ENA_COM_UNSUPPORTED; + ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, sizeof(*rss->hash_key), rss->hash_key, @@ -928,13 +1165,13 @@ int ret; ret = ena_com_get_feature(ena_dev, &get_resp, - ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG); + ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG, 0); if (unlikely(ret)) return ret; if ((get_resp.u.ind_table.min_size > log_size) || (get_resp.u.ind_table.max_size < log_size)) { - ena_trc_err("indirect table size doesn't fit. requested size: %d while min is:%d and max %d\n", + ena_trc_err(ena_dev, "Indirect table size doesn't fit. requested size: %d while min is:%d and max %d\n", 1 << log_size, 1 << get_resp.u.ind_table.min_size, 1 << get_resp.u.ind_table.max_size); @@ -945,10 +1182,10 @@ sizeof(struct ena_admin_rss_ind_table_entry); ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, - tbl_size, - rss->rss_ind_tbl, - rss->rss_ind_tbl_dma_addr, - rss->rss_ind_tbl_mem_handle); + tbl_size, + rss->rss_ind_tbl, + rss->rss_ind_tbl_dma_addr, + rss->rss_ind_tbl_mem_handle); if (unlikely(!rss->rss_ind_tbl)) goto mem_err1; @@ -992,7 +1229,9 @@ rss->rss_ind_tbl = NULL; if (rss->host_rss_ind_tbl) - ENA_MEM_FREE(ena_dev->dmadev, rss->host_rss_ind_tbl); + ENA_MEM_FREE(ena_dev->dmadev, + rss->host_rss_ind_tbl, + ((1ULL << rss->tbl_log_size) * sizeof(u16))); rss->host_rss_ind_tbl = NULL; } @@ -1005,7 +1244,7 @@ u8 direction; int ret; - memset(&create_cmd, 0x0, sizeof(struct ena_admin_aq_create_sq_cmd)); + memset(&create_cmd, 0x0, sizeof(create_cmd)); create_cmd.aq_common_descriptor.opcode = ENA_ADMIN_CREATE_SQ; @@ -1036,19 +1275,18 @@ &create_cmd.sq_ba, io_sq->desc_addr.phys_addr); if (unlikely(ret)) { - ena_trc_err("memory address set failed\n"); + ena_trc_err(ena_dev, "Memory address set failed\n"); return ret; } } - ret = ena_com_execute_admin_command( - admin_queue, - (struct ena_admin_aq_entry *)&create_cmd, - sizeof(create_cmd), - (struct ena_admin_acq_entry *)&cmd_completion, - sizeof(cmd_completion)); + ret = ena_com_execute_admin_command(admin_queue, + (struct ena_admin_aq_entry *)&create_cmd, + sizeof(create_cmd), + (struct ena_admin_acq_entry *)&cmd_completion, + sizeof(cmd_completion)); if (unlikely(ret)) { - ena_trc_err("Failed to create IO SQ. error: %d\n", ret); + ena_trc_err(ena_dev, "Failed to create IO SQ. error: %d\n", ret); return ret; } @@ -1066,7 +1304,7 @@ cmd_completion.llq_descriptors_offset); } - ena_trc_dbg("created sq[%u], depth[%u]\n", io_sq->idx, io_sq->q_depth); + ena_trc_dbg(ena_dev, "Created sq[%u], depth[%u]\n", io_sq->idx, io_sq->q_depth); return ret; } @@ -1094,64 +1332,29 @@ return 0; } -static int ena_com_ind_tbl_convert_from_device(struct ena_com_dev *ena_dev) +static void ena_com_update_intr_delay_resolution(struct ena_com_dev *ena_dev, + u16 intr_delay_resolution) { - u16 dev_idx_to_host_tbl[ENA_TOTAL_NUM_QUEUES] = { (u16)-1 }; - struct ena_rss *rss = &ena_dev->rss; - u8 idx; - u16 i; + u16 prev_intr_delay_resolution = ena_dev->intr_delay_resolution; - for (i = 0; i < ENA_TOTAL_NUM_QUEUES; i++) - dev_idx_to_host_tbl[ena_dev->io_sq_queues[i].idx] = i; - - for (i = 0; i < 1 << rss->tbl_log_size; i++) { - if (rss->rss_ind_tbl[i].cq_idx > ENA_TOTAL_NUM_QUEUES) - return ENA_COM_INVAL; - idx = (u8)rss->rss_ind_tbl[i].cq_idx; - - if (dev_idx_to_host_tbl[idx] > ENA_TOTAL_NUM_QUEUES) - return ENA_COM_INVAL; - - rss->host_rss_ind_tbl[i] = dev_idx_to_host_tbl[idx]; + if (unlikely(!intr_delay_resolution)) { + ena_trc_err(ena_dev, "Illegal intr_delay_resolution provided. Going to use default 1 usec resolution\n"); + intr_delay_resolution = ENA_DEFAULT_INTR_DELAY_RESOLUTION; } - return 0; -} - -static int ena_com_init_interrupt_moderation_table(struct ena_com_dev *ena_dev) -{ - size_t size; - - size = sizeof(struct ena_intr_moder_entry) * ENA_INTR_MAX_NUM_OF_LEVELS; - - ena_dev->intr_moder_tbl = ENA_MEM_ALLOC(ena_dev->dmadev, size); - if (!ena_dev->intr_moder_tbl) - return ENA_COM_NO_MEM; - - ena_com_config_default_interrupt_moderation_table(ena_dev); - - return 0; -} - -static void -ena_com_update_intr_delay_resolution(struct ena_com_dev *ena_dev, - u16 intr_delay_resolution) -{ - struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl; - unsigned int i; - - if (!intr_delay_resolution) { - ena_trc_err("Illegal intr_delay_resolution provided. Going to use default 1 usec resolution\n"); - intr_delay_resolution = 1; - } - ena_dev->intr_delay_resolution = intr_delay_resolution; - /* update Rx */ - for (i = 0; i < ENA_INTR_MAX_NUM_OF_LEVELS; i++) - intr_moder_tbl[i].intr_moder_interval /= intr_delay_resolution; + ena_dev->intr_moder_rx_interval = + ena_dev->intr_moder_rx_interval * + prev_intr_delay_resolution / + intr_delay_resolution; /* update Tx */ - ena_dev->intr_moder_tx_interval /= intr_delay_resolution; + ena_dev->intr_moder_tx_interval = + ena_dev->intr_moder_tx_interval * + prev_intr_delay_resolution / + intr_delay_resolution; + + ena_dev->intr_delay_resolution = intr_delay_resolution; } /*****************************************************************************/ @@ -1165,24 +1368,31 @@ size_t comp_size) { struct ena_comp_ctx *comp_ctx; - int ret = 0; + int ret; comp_ctx = ena_com_submit_admin_cmd(admin_queue, cmd, cmd_size, comp, comp_size); - if (unlikely(IS_ERR(comp_ctx))) { - ena_trc_err("Failed to submit command [%ld]\n", - PTR_ERR(comp_ctx)); - return PTR_ERR(comp_ctx); + if (IS_ERR(comp_ctx)) { + if (comp_ctx == ERR_PTR(ENA_COM_NO_DEVICE)) + ena_trc_dbg(admin_queue->ena_dev, + "Failed to submit command [%ld]\n", + PTR_ERR(comp_ctx)); + else + ena_trc_err(admin_queue->ena_dev, + "Failed to submit command [%ld]\n", + PTR_ERR(comp_ctx)); + + return (int)PTR_ERR(comp_ctx); } ret = ena_com_wait_and_process_admin_cq(comp_ctx, admin_queue); if (unlikely(ret)) { if (admin_queue->running_state) - ena_trc_err("Failed to process command. ret = %d\n", - ret); + ena_trc_err(admin_queue->ena_dev, + "Failed to process command. ret = %d\n", ret); else - ena_trc_dbg("Failed to process command. ret = %d\n", - ret); + ena_trc_dbg(admin_queue->ena_dev, + "Failed to process command. ret = %d\n", ret); } return ret; } @@ -1195,7 +1405,7 @@ struct ena_admin_acq_create_cq_resp_desc cmd_completion; int ret; - memset(&create_cmd, 0x0, sizeof(struct ena_admin_aq_create_cq_cmd)); + memset(&create_cmd, 0x0, sizeof(create_cmd)); create_cmd.aq_common_descriptor.opcode = ENA_ADMIN_CREATE_CQ; @@ -1211,18 +1421,17 @@ &create_cmd.cq_ba, io_cq->cdesc_addr.phys_addr); if (unlikely(ret)) { - ena_trc_err("memory address set failed\n"); + ena_trc_err(ena_dev, "Memory address set failed\n"); return ret; } - ret = ena_com_execute_admin_command( - admin_queue, - (struct ena_admin_aq_entry *)&create_cmd, - sizeof(create_cmd), - (struct ena_admin_acq_entry *)&cmd_completion, - sizeof(cmd_completion)); + ret = ena_com_execute_admin_command(admin_queue, + (struct ena_admin_aq_entry *)&create_cmd, + sizeof(create_cmd), + (struct ena_admin_acq_entry *)&cmd_completion, + sizeof(cmd_completion)); if (unlikely(ret)) { - ena_trc_err("Failed to create IO CQ. error: %d\n", ret); + ena_trc_err(ena_dev, "Failed to create IO CQ. error: %d\n", ret); return ret; } @@ -1241,7 +1450,7 @@ (u32 __iomem *)((uintptr_t)ena_dev->reg_bar + cmd_completion.numa_node_register_offset); - ena_trc_dbg("created cq[%u], depth[%u]\n", io_cq->idx, io_cq->q_depth); + ena_trc_dbg(ena_dev, "Created cq[%u], depth[%u]\n", io_cq->idx, io_cq->q_depth); return ret; } @@ -1251,7 +1460,7 @@ struct ena_com_io_cq **io_cq) { if (qid >= ENA_TOTAL_NUM_QUEUES) { - ena_trc_err("Invalid queue number %d but the max is %d\n", + ena_trc_err(ena_dev, "Invalid queue number %d but the max is %d\n", qid, ENA_TOTAL_NUM_QUEUES); return ENA_COM_INVAL; } @@ -1286,11 +1495,12 @@ { struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; unsigned long flags = 0; + u32 exp = 0; ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); while (ATOMIC32_READ(&admin_queue->outstanding_cmds) != 0) { ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); - ENA_MSLEEP(20); + ena_delay_exponential_backoff_us(exp++, ena_dev->ena_min_poll_delay_us); ENA_SPINLOCK_LOCK(admin_queue->q_lock, flags); } ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags); @@ -1304,20 +1514,19 @@ struct ena_admin_acq_destroy_cq_resp_desc destroy_resp; int ret; - memset(&destroy_cmd, 0x0, sizeof(struct ena_admin_aq_destroy_sq_cmd)); + memset(&destroy_cmd, 0x0, sizeof(destroy_cmd)); destroy_cmd.cq_idx = io_cq->idx; destroy_cmd.aq_common_descriptor.opcode = ENA_ADMIN_DESTROY_CQ; - ret = ena_com_execute_admin_command( - admin_queue, - (struct ena_admin_aq_entry *)&destroy_cmd, - sizeof(destroy_cmd), - (struct ena_admin_acq_entry *)&destroy_resp, - sizeof(destroy_resp)); + ret = ena_com_execute_admin_command(admin_queue, + (struct ena_admin_aq_entry *)&destroy_cmd, + sizeof(destroy_cmd), + (struct ena_admin_acq_entry *)&destroy_resp, + sizeof(destroy_resp)); if (unlikely(ret && (ret != ENA_COM_NO_DEVICE))) - ena_trc_err("Failed to destroy IO CQ. error: %d\n", ret); + ena_trc_err(ena_dev, "Failed to destroy IO CQ. error: %d\n", ret); return ret; } @@ -1341,13 +1550,12 @@ { u16 depth = ena_dev->aenq.q_depth; - ENA_ASSERT(ena_dev->aenq.head == depth, "Invalid AENQ state\n"); + ENA_WARN(ena_dev->aenq.head != depth, ena_dev, "Invalid AENQ state\n"); /* Init head_db to mark that all entries in the queue * are initially available */ - ENA_REG_WRITE32(depth, (unsigned char *)ena_dev->reg_bar - + ENA_REGS_AENQ_HEAD_DB_OFF); + ENA_REG_WRITE32(ena_dev->bus, depth, ena_dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF); } int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag) @@ -1356,24 +1564,19 @@ struct ena_admin_set_feat_cmd cmd; struct ena_admin_set_feat_resp resp; struct ena_admin_get_feat_resp get_resp; - int ret = 0; - - if (unlikely(!ena_dev)) { - ena_trc_err("%s : ena_dev is NULL\n", __func__); - return ENA_COM_NO_DEVICE; - } + int ret; - ret = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_AENQ_CONFIG); + ret = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_AENQ_CONFIG, 0); if (ret) { - ena_trc_info("Can't get aenq configuration\n"); + ena_trc_info(ena_dev, "Can't get aenq configuration\n"); return ret; } if ((get_resp.u.aenq.supported_groups & groups_flag) != groups_flag) { - ena_trc_warn("Trying to set unsupported aenq events. supported flag: %x asked flag: %x\n", + ena_trc_warn(ena_dev, "Trying to set unsupported aenq events. supported flag: 0x%x asked flag: 0x%x\n", get_resp.u.aenq.supported_groups, groups_flag); - return ENA_COM_PERMISSION; + return ENA_COM_UNSUPPORTED; } memset(&cmd, 0x0, sizeof(cmd)); @@ -1391,7 +1594,7 @@ sizeof(resp)); if (unlikely(ret)) - ena_trc_err("Failed to config AENQ ret: %d\n", ret); + ena_trc_err(ena_dev, "Failed to config AENQ ret: %d\n", ret); return ret; } @@ -1399,20 +1602,20 @@ int ena_com_get_dma_width(struct ena_com_dev *ena_dev) { u32 caps = ena_com_reg_bar_read32(ena_dev, ENA_REGS_CAPS_OFF); - int width; + u32 width; if (unlikely(caps == ENA_MMIO_READ_TIMEOUT)) { - ena_trc_err("Reg read timeout occurred\n"); + ena_trc_err(ena_dev, "Reg read timeout occurred\n"); return ENA_COM_TIMER_EXPIRED; } width = (caps & ENA_REGS_CAPS_DMA_ADDR_WIDTH_MASK) >> ENA_REGS_CAPS_DMA_ADDR_WIDTH_SHIFT; - ena_trc_dbg("ENA dma width: %d\n", width); + ena_trc_dbg(ena_dev, "ENA dma width: %d\n", width); if ((width < 32) || width > ENA_MAX_PHYS_ADDR_SIZE_BITS) { - ena_trc_err("DMA width illegal value: %d\n", width); + ena_trc_err(ena_dev, "DMA width illegal value: %d\n", width); return ENA_COM_INVAL; } @@ -1436,21 +1639,16 @@ if (unlikely((ver == ENA_MMIO_READ_TIMEOUT) || (ctrl_ver == ENA_MMIO_READ_TIMEOUT))) { - ena_trc_err("Reg read timeout occurred\n"); + ena_trc_err(ena_dev, "Reg read timeout occurred\n"); return ENA_COM_TIMER_EXPIRED; } - ena_trc_info("ena device version: %d.%d\n", + ena_trc_info(ena_dev, "ENA device version: %d.%d\n", (ver & ENA_REGS_VERSION_MAJOR_VERSION_MASK) >> ENA_REGS_VERSION_MAJOR_VERSION_SHIFT, ver & ENA_REGS_VERSION_MINOR_VERSION_MASK); - if (ver < MIN_ENA_VER) { - ena_trc_err("ENA version is lower than the minimal version the driver supports\n"); - return -1; - } - - ena_trc_info("ena controller version: %d.%d.%d implementation version %d\n", + ena_trc_info(ena_dev, "ENA controller version: %d.%d.%d implementation version %d\n", (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) >> ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT, (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK) @@ -1466,54 +1664,82 @@ /* Validate the ctrl version without the implementation ID */ if (ctrl_ver_masked < MIN_ENA_CTRL_VER) { - ena_trc_err("ENA ctrl version is lower than the minimal ctrl version the driver supports\n"); + ena_trc_err(ena_dev, "ENA ctrl version is lower than the minimal ctrl version the driver supports\n"); return -1; } return 0; } -void ena_com_admin_destroy(struct ena_com_dev *ena_dev) -{ - struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; +static void +ena_com_free_ena_admin_queue_comp_ctx(struct ena_com_dev *ena_dev, + struct ena_com_admin_queue *admin_queue) - if (!admin_queue) +{ + if (!admin_queue->comp_ctx) return; - if (admin_queue->comp_ctx) - ENA_MEM_FREE(ena_dev->dmadev, admin_queue->comp_ctx); - admin_queue->comp_ctx = NULL; + ENA_WAIT_EVENTS_DESTROY(admin_queue); + ENA_MEM_FREE(ena_dev->dmadev, + admin_queue->comp_ctx, + (admin_queue->q_depth * sizeof(struct ena_comp_ctx))); - if (admin_queue->sq.entries) - ENA_MEM_FREE_COHERENT(ena_dev->dmadev, - ADMIN_SQ_SIZE(admin_queue->q_depth), - admin_queue->sq.entries, - admin_queue->sq.dma_addr, - admin_queue->sq.mem_handle); - admin_queue->sq.entries = NULL; + admin_queue->comp_ctx = NULL; +} - if (admin_queue->cq.entries) - ENA_MEM_FREE_COHERENT(ena_dev->dmadev, - ADMIN_CQ_SIZE(admin_queue->q_depth), - admin_queue->cq.entries, - admin_queue->cq.dma_addr, - admin_queue->cq.mem_handle); - admin_queue->cq.entries = NULL; +void ena_com_admin_destroy(struct ena_com_dev *ena_dev) +{ + struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; + struct ena_com_admin_cq *cq = &admin_queue->cq; + struct ena_com_admin_sq *sq = &admin_queue->sq; + struct ena_com_aenq *aenq = &ena_dev->aenq; + u16 size; + + ena_com_free_ena_admin_queue_comp_ctx(ena_dev, admin_queue); + + size = ADMIN_SQ_SIZE(admin_queue->q_depth); + if (sq->entries) + ENA_MEM_FREE_COHERENT(ena_dev->dmadev, size, sq->entries, + sq->dma_addr, sq->mem_handle); + sq->entries = NULL; + + size = ADMIN_CQ_SIZE(admin_queue->q_depth); + if (cq->entries) + ENA_MEM_FREE_COHERENT(ena_dev->dmadev, size, cq->entries, + cq->dma_addr, cq->mem_handle); + cq->entries = NULL; + size = ADMIN_AENQ_SIZE(aenq->q_depth); if (ena_dev->aenq.entries) - ENA_MEM_FREE_COHERENT(ena_dev->dmadev, - ADMIN_AENQ_SIZE(ena_dev->aenq.q_depth), - ena_dev->aenq.entries, - ena_dev->aenq.dma_addr, - ena_dev->aenq.mem_handle); - ena_dev->aenq.entries = NULL; + ENA_MEM_FREE_COHERENT(ena_dev->dmadev, size, aenq->entries, + aenq->dma_addr, aenq->mem_handle); + aenq->entries = NULL; + ENA_SPINLOCK_DESTROY(admin_queue->q_lock); } void ena_com_set_admin_polling_mode(struct ena_com_dev *ena_dev, bool polling) { + u32 mask_value = 0; + + if (polling) + mask_value = ENA_REGS_ADMIN_INTR_MASK; + + ENA_REG_WRITE32(ena_dev->bus, mask_value, + ena_dev->reg_bar + ENA_REGS_INTR_MASK_OFF); ena_dev->admin_queue.polling = polling; } +bool ena_com_get_admin_polling_mode(struct ena_com_dev *ena_dev) +{ + return ena_dev->admin_queue.polling; +} + +void ena_com_set_admin_auto_polling_mode(struct ena_com_dev *ena_dev, + bool polling) +{ + ena_dev->admin_queue.auto_polling = polling; +} + int ena_com_mmio_reg_read_request_init(struct ena_com_dev *ena_dev) { struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read; @@ -1525,7 +1751,7 @@ mmio_read->read_resp_dma_addr, mmio_read->read_resp_mem_handle); if (unlikely(!mmio_read->read_resp)) - return ENA_COM_NO_MEM; + goto err; ena_com_mmio_reg_read_request_write_dev_addr(ena_dev); @@ -1534,10 +1760,13 @@ mmio_read->readless_supported = true; return 0; + +err: + ENA_SPINLOCK_DESTROY(mmio_read->lock); + return ENA_COM_NO_MEM; } -void -ena_com_set_mmio_read_mode(struct ena_com_dev *ena_dev, bool readless_supported) +void ena_com_set_mmio_read_mode(struct ena_com_dev *ena_dev, bool readless_supported) { struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read; @@ -1548,10 +1777,8 @@ { struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read; - ENA_REG_WRITE32(0x0, (unsigned char *)ena_dev->reg_bar - + ENA_REGS_MMIO_RESP_LO_OFF); - ENA_REG_WRITE32(0x0, (unsigned char *)ena_dev->reg_bar - + ENA_REGS_MMIO_RESP_HI_OFF); + ENA_REG_WRITE32(ena_dev->bus, 0x0, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_LO_OFF); + ENA_REG_WRITE32(ena_dev->bus, 0x0, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_HI_OFF); ENA_MEM_FREE_COHERENT(ena_dev->dmadev, sizeof(*mmio_read->read_resp), @@ -1560,6 +1787,7 @@ mmio_read->read_resp_mem_handle); mmio_read->read_resp = NULL; + ENA_SPINLOCK_DESTROY(mmio_read->lock); } void ena_com_mmio_reg_read_request_write_dev_addr(struct ena_com_dev *ena_dev) @@ -1570,15 +1798,12 @@ addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(mmio_read->read_resp_dma_addr); addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(mmio_read->read_resp_dma_addr); - ENA_REG_WRITE32(addr_low, (unsigned char *)ena_dev->reg_bar - + ENA_REGS_MMIO_RESP_LO_OFF); - ENA_REG_WRITE32(addr_high, (unsigned char *)ena_dev->reg_bar - + ENA_REGS_MMIO_RESP_HI_OFF); + ENA_REG_WRITE32(ena_dev->bus, addr_low, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_LO_OFF); + ENA_REG_WRITE32(ena_dev->bus, addr_high, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_HI_OFF); } int ena_com_admin_init(struct ena_com_dev *ena_dev, - struct ena_aenq_handlers *aenq_handlers, - bool init_spinlock) + struct ena_aenq_handlers *aenq_handlers) { struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; u32 aq_caps, acq_caps, dev_sts, addr_low, addr_high; @@ -1587,25 +1812,25 @@ dev_sts = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF); if (unlikely(dev_sts == ENA_MMIO_READ_TIMEOUT)) { - ena_trc_err("Reg read timeout occurred\n"); + ena_trc_err(ena_dev, "Reg read timeout occurred\n"); return ENA_COM_TIMER_EXPIRED; } if (!(dev_sts & ENA_REGS_DEV_STS_READY_MASK)) { - ena_trc_err("Device isn't ready, abort com init\n"); + ena_trc_err(ena_dev, "Device isn't ready, abort com init\n"); return ENA_COM_NO_DEVICE; } admin_queue->q_depth = ENA_ADMIN_QUEUE_DEPTH; + admin_queue->bus = ena_dev->bus; admin_queue->q_dmadev = ena_dev->dmadev; admin_queue->polling = false; admin_queue->curr_cmd_id = 0; ATOMIC32_SET(&admin_queue->outstanding_cmds, 0); - if (init_spinlock) - ENA_SPINLOCK_INIT(admin_queue->q_lock); + ENA_SPINLOCK_INIT(admin_queue->q_lock); ret = ena_com_init_comp_ctxt(admin_queue); if (ret) @@ -1619,24 +1844,20 @@ if (ret) goto error; - admin_queue->sq.db_addr = (u32 __iomem *) - ((unsigned char *)ena_dev->reg_bar + ENA_REGS_AQ_DB_OFF); + admin_queue->sq.db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar + + ENA_REGS_AQ_DB_OFF); addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->sq.dma_addr); addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->sq.dma_addr); - ENA_REG_WRITE32(addr_low, (unsigned char *)ena_dev->reg_bar - + ENA_REGS_AQ_BASE_LO_OFF); - ENA_REG_WRITE32(addr_high, (unsigned char *)ena_dev->reg_bar - + ENA_REGS_AQ_BASE_HI_OFF); + ENA_REG_WRITE32(ena_dev->bus, addr_low, ena_dev->reg_bar + ENA_REGS_AQ_BASE_LO_OFF); + ENA_REG_WRITE32(ena_dev->bus, addr_high, ena_dev->reg_bar + ENA_REGS_AQ_BASE_HI_OFF); addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->cq.dma_addr); addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->cq.dma_addr); - ENA_REG_WRITE32(addr_low, (unsigned char *)ena_dev->reg_bar - + ENA_REGS_ACQ_BASE_LO_OFF); - ENA_REG_WRITE32(addr_high, (unsigned char *)ena_dev->reg_bar - + ENA_REGS_ACQ_BASE_HI_OFF); + ENA_REG_WRITE32(ena_dev->bus, addr_low, ena_dev->reg_bar + ENA_REGS_ACQ_BASE_LO_OFF); + ENA_REG_WRITE32(ena_dev->bus, addr_high, ena_dev->reg_bar + ENA_REGS_ACQ_BASE_HI_OFF); aq_caps = 0; aq_caps |= admin_queue->q_depth & ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK; @@ -1650,14 +1871,13 @@ ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_SHIFT) & ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_MASK; - ENA_REG_WRITE32(aq_caps, (unsigned char *)ena_dev->reg_bar - + ENA_REGS_AQ_CAPS_OFF); - ENA_REG_WRITE32(acq_caps, (unsigned char *)ena_dev->reg_bar - + ENA_REGS_ACQ_CAPS_OFF); + ENA_REG_WRITE32(ena_dev->bus, aq_caps, ena_dev->reg_bar + ENA_REGS_AQ_CAPS_OFF); + ENA_REG_WRITE32(ena_dev->bus, acq_caps, ena_dev->reg_bar + ENA_REGS_ACQ_CAPS_OFF); ret = ena_com_admin_init_aenq(ena_dev, aenq_handlers); if (ret) goto error; + admin_queue->ena_dev = ena_dev; admin_queue->running_state = true; return 0; @@ -1672,10 +1892,10 @@ { struct ena_com_io_sq *io_sq; struct ena_com_io_cq *io_cq; - int ret = 0; + int ret; if (ctx->qid >= ENA_TOTAL_NUM_QUEUES) { - ena_trc_err("Qid (%d) is bigger than max num of queues (%d)\n", + ena_trc_err(ena_dev, "Qid (%d) is bigger than max num of queues (%d)\n", ctx->qid, ENA_TOTAL_NUM_QUEUES); return ENA_COM_INVAL; } @@ -1683,8 +1903,8 @@ io_sq = &ena_dev->io_sq_queues[ctx->qid]; io_cq = &ena_dev->io_cq_queues[ctx->qid]; - memset(io_sq, 0x0, sizeof(struct ena_com_io_sq)); - memset(io_cq, 0x0, sizeof(struct ena_com_io_cq)); + memset(io_sq, 0x0, sizeof(*io_sq)); + memset(io_cq, 0x0, sizeof(*io_cq)); /* Init CQ */ io_cq->q_depth = ctx->queue_size; @@ -1734,7 +1954,7 @@ struct ena_com_io_cq *io_cq; if (qid >= ENA_TOTAL_NUM_QUEUES) { - ena_trc_err("Qid (%d) is bigger than max num of queues (%d)\n", + ena_trc_err(ena_dev, "Qid (%d) is bigger than max num of queues (%d)\n", qid, ENA_TOTAL_NUM_QUEUES); return; } @@ -1751,7 +1971,7 @@ int ena_com_get_link_params(struct ena_com_dev *ena_dev, struct ena_admin_get_feat_resp *resp) { - return ena_com_get_feature(ena_dev, resp, ENA_ADMIN_LINK_CONFIG); + return ena_com_get_feature(ena_dev, resp, ENA_ADMIN_LINK_CONFIG, 0); } int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev, @@ -1761,25 +1981,43 @@ int rc; rc = ena_com_get_feature(ena_dev, &get_resp, - ENA_ADMIN_DEVICE_ATTRIBUTES); + ENA_ADMIN_DEVICE_ATTRIBUTES, 0); if (rc) return rc; memcpy(&get_feat_ctx->dev_attr, &get_resp.u.dev_attr, sizeof(get_resp.u.dev_attr)); + ena_dev->supported_features = get_resp.u.dev_attr.supported_features; - rc = ena_com_get_feature(ena_dev, &get_resp, - ENA_ADMIN_MAX_QUEUES_NUM); - if (rc) - return rc; + if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) { + rc = ena_com_get_feature(ena_dev, &get_resp, + ENA_ADMIN_MAX_QUEUES_EXT, + ENA_FEATURE_MAX_QUEUE_EXT_VER); + if (rc) + return rc; + + if (get_resp.u.max_queue_ext.version != ENA_FEATURE_MAX_QUEUE_EXT_VER) + return -EINVAL; + + memcpy(&get_feat_ctx->max_queue_ext, &get_resp.u.max_queue_ext, + sizeof(get_resp.u.max_queue_ext)); + ena_dev->tx_max_header_size = + get_resp.u.max_queue_ext.max_queue_ext.max_tx_header_size; + } else { + rc = ena_com_get_feature(ena_dev, &get_resp, + ENA_ADMIN_MAX_QUEUES_NUM, 0); + memcpy(&get_feat_ctx->max_queues, &get_resp.u.max_queue, + sizeof(get_resp.u.max_queue)); + ena_dev->tx_max_header_size = + get_resp.u.max_queue.max_header_size; - memcpy(&get_feat_ctx->max_queues, &get_resp.u.max_queue, - sizeof(get_resp.u.max_queue)); - ena_dev->tx_max_header_size = get_resp.u.max_queue.max_header_size; + if (rc) + return rc; + } rc = ena_com_get_feature(ena_dev, &get_resp, - ENA_ADMIN_AENQ_CONFIG); + ENA_ADMIN_AENQ_CONFIG, 0); if (rc) return rc; @@ -1787,13 +2025,35 @@ sizeof(get_resp.u.aenq)); rc = ena_com_get_feature(ena_dev, &get_resp, - ENA_ADMIN_STATELESS_OFFLOAD_CONFIG); + ENA_ADMIN_STATELESS_OFFLOAD_CONFIG, 0); if (rc) return rc; memcpy(&get_feat_ctx->offload, &get_resp.u.offload, sizeof(get_resp.u.offload)); + /* Driver hints isn't mandatory admin command. So in case the + * command isn't supported set driver hints to 0 + */ + rc = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_HW_HINTS, 0); + + if (!rc) + memcpy(&get_feat_ctx->hw_hints, &get_resp.u.hw_hints, + sizeof(get_resp.u.hw_hints)); + else if (rc == ENA_COM_UNSUPPORTED) + memset(&get_feat_ctx->hw_hints, 0x0, sizeof(get_feat_ctx->hw_hints)); + else + return rc; + + rc = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_LLQ, 0); + if (!rc) + memcpy(&get_feat_ctx->llq, &get_resp.u.llq, + sizeof(get_resp.u.llq)); + else if (rc == ENA_COM_UNSUPPORTED) + memset(&get_feat_ctx->llq, 0x0, sizeof(get_feat_ctx->llq)); + else + return rc; + return 0; } @@ -1805,10 +2065,10 @@ /* ena_handle_specific_aenq_event: * return the handler that is relevant to the specific event group */ -static ena_aenq_handler ena_com_get_specific_aenq_cb(struct ena_com_dev *dev, +static ena_aenq_handler ena_com_get_specific_aenq_cb(struct ena_com_dev *ena_dev, u16 group) { - struct ena_aenq_handlers *aenq_handlers = dev->aenq.aenq_handlers; + struct ena_aenq_handlers *aenq_handlers = ena_dev->aenq.aenq_handlers; if ((group < ENA_MAX_HANDLERS) && aenq_handlers->handlers[group]) return aenq_handlers->handlers[group]; @@ -1820,11 +2080,12 @@ * handles the aenq incoming events. * pop events from the queue and apply the specific handler */ -void ena_com_aenq_intr_handler(struct ena_com_dev *dev, void *data) +void ena_com_aenq_intr_handler(struct ena_com_dev *ena_dev, void *data) { struct ena_admin_aenq_entry *aenq_e; struct ena_admin_aenq_common_desc *aenq_common; - struct ena_com_aenq *aenq = &dev->aenq; + struct ena_com_aenq *aenq = &ena_dev->aenq; + u64 timestamp; ena_aenq_handler handler_cb; u16 masked_head, processed = 0; u8 phase; @@ -1835,16 +2096,23 @@ aenq_common = &aenq_e->aenq_common_desc; /* Go over all the events */ - while ((aenq_common->flags & ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK) == - phase) { - ena_trc_dbg("AENQ! Group[%x] Syndrom[%x] timestamp: [%llus]\n", + while ((READ_ONCE8(aenq_common->flags) & + ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK) == phase) { + /* Make sure the phase bit (ownership) is as expected before + * reading the rest of the descriptor. + */ + dma_rmb(); + + timestamp = (u64)aenq_common->timestamp_low | + ((u64)aenq_common->timestamp_high << 32); + ENA_TOUCH(timestamp); /* In case debug is disabled */ + ena_trc_dbg(ena_dev, "AENQ! Group[%x] Syndrome[%x] timestamp: [%" ENA_PRIu64 "s]\n", aenq_common->group, - aenq_common->syndrom, - (unsigned long long)aenq_common->timestamp_low + - ((u64)aenq_common->timestamp_high << 32)); + aenq_common->syndrome, + timestamp); /* Handle specific event*/ - handler_cb = ena_com_get_specific_aenq_cb(dev, + handler_cb = ena_com_get_specific_aenq_cb(ena_dev, aenq_common->group); handler_cb(data, aenq_e); /* call the actual event handler*/ @@ -1869,11 +2137,13 @@ /* write the aenq doorbell after all AENQ descriptors were read */ mb(); - ENA_REG_WRITE32((u32)aenq->head, (unsigned char *)dev->reg_bar - + ENA_REGS_AENQ_HEAD_DB_OFF); + ENA_REG_WRITE32_RELAXED(ena_dev->bus, (u32)aenq->head, + ena_dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF); + mmiowb(); } -int ena_com_dev_reset(struct ena_com_dev *ena_dev) +int ena_com_dev_reset(struct ena_com_dev *ena_dev, + enum ena_regs_reset_reason_types reset_reason) { u32 stat, timeout, cap, reset_val; int rc; @@ -1883,26 +2153,27 @@ if (unlikely((stat == ENA_MMIO_READ_TIMEOUT) || (cap == ENA_MMIO_READ_TIMEOUT))) { - ena_trc_err("Reg read32 timeout occurred\n"); + ena_trc_err(ena_dev, "Reg read32 timeout occurred\n"); return ENA_COM_TIMER_EXPIRED; } if ((stat & ENA_REGS_DEV_STS_READY_MASK) == 0) { - ena_trc_err("Device isn't ready, can't reset device\n"); + ena_trc_err(ena_dev, "Device isn't ready, can't reset device\n"); return ENA_COM_INVAL; } timeout = (cap & ENA_REGS_CAPS_RESET_TIMEOUT_MASK) >> ENA_REGS_CAPS_RESET_TIMEOUT_SHIFT; if (timeout == 0) { - ena_trc_err("Invalid timeout value\n"); + ena_trc_err(ena_dev, "Invalid timeout value\n"); return ENA_COM_INVAL; } /* start reset */ reset_val = ENA_REGS_DEV_CTL_DEV_RESET_MASK; - ENA_REG_WRITE32(reset_val, (unsigned char *)ena_dev->reg_bar - + ENA_REGS_DEV_CTL_OFF); + reset_val |= (reset_reason << ENA_REGS_DEV_CTL_RESET_REASON_SHIFT) & + ENA_REGS_DEV_CTL_RESET_REASON_MASK; + ENA_REG_WRITE32(ena_dev->bus, reset_val, ena_dev->reg_bar + ENA_REGS_DEV_CTL_OFF); /* Write again the MMIO read request address */ ena_com_mmio_reg_read_request_write_dev_addr(ena_dev); @@ -1910,34 +2181,37 @@ rc = wait_for_reset_state(ena_dev, timeout, ENA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK); if (rc != 0) { - ena_trc_err("Reset indication didn't turn on\n"); + ena_trc_err(ena_dev, "Reset indication didn't turn on\n"); return rc; } /* reset done */ - ENA_REG_WRITE32(0, (unsigned char *)ena_dev->reg_bar - + ENA_REGS_DEV_CTL_OFF); + ENA_REG_WRITE32(ena_dev->bus, 0, ena_dev->reg_bar + ENA_REGS_DEV_CTL_OFF); rc = wait_for_reset_state(ena_dev, timeout, 0); if (rc != 0) { - ena_trc_err("Reset indication didn't turn off\n"); + ena_trc_err(ena_dev, "Reset indication didn't turn off\n"); return rc; } + timeout = (cap & ENA_REGS_CAPS_ADMIN_CMD_TO_MASK) >> + ENA_REGS_CAPS_ADMIN_CMD_TO_SHIFT; + if (timeout) + /* the resolution of timeout reg is 100ms */ + ena_dev->admin_queue.completion_timeout = timeout * 100000; + else + ena_dev->admin_queue.completion_timeout = ADMIN_CMD_TIMEOUT_US; + return 0; } static int ena_get_dev_stats(struct ena_com_dev *ena_dev, - struct ena_admin_aq_get_stats_cmd *get_cmd, - struct ena_admin_acq_get_stats_resp *get_resp, + struct ena_com_stats_ctx *ctx, enum ena_admin_get_stats_type type) { + struct ena_admin_aq_get_stats_cmd *get_cmd = &ctx->get_cmd; + struct ena_admin_acq_get_stats_resp *get_resp = &ctx->get_resp; struct ena_com_admin_queue *admin_queue; - int ret = 0; - - if (!ena_dev) { - ena_trc_err("%s : ena_dev is NULL\n", __func__); - return ENA_COM_NO_DEVICE; - } + int ret; admin_queue = &ena_dev->admin_queue; @@ -1945,76 +2219,45 @@ get_cmd->aq_common_descriptor.flags = 0; get_cmd->type = type; - ret = ena_com_execute_admin_command( - admin_queue, - (struct ena_admin_aq_entry *)get_cmd, - sizeof(*get_cmd), - (struct ena_admin_acq_entry *)get_resp, - sizeof(*get_resp)); + ret = ena_com_execute_admin_command(admin_queue, + (struct ena_admin_aq_entry *)get_cmd, + sizeof(*get_cmd), + (struct ena_admin_acq_entry *)get_resp, + sizeof(*get_resp)); if (unlikely(ret)) - ena_trc_err("Failed to get stats. error: %d\n", ret); + ena_trc_err(ena_dev, "Failed to get stats. error: %d\n", ret); return ret; } -int ena_com_get_dev_basic_stats(struct ena_com_dev *ena_dev, - struct ena_admin_basic_stats *stats) +int ena_com_get_eni_stats(struct ena_com_dev *ena_dev, + struct ena_admin_eni_stats *stats) { - int ret = 0; - struct ena_admin_aq_get_stats_cmd get_cmd; - struct ena_admin_acq_get_stats_resp get_resp; + struct ena_com_stats_ctx ctx; + int ret; - memset(&get_cmd, 0x0, sizeof(get_cmd)); - ret = ena_get_dev_stats(ena_dev, &get_cmd, &get_resp, - ENA_ADMIN_GET_STATS_TYPE_BASIC); + memset(&ctx, 0x0, sizeof(ctx)); + ret = ena_get_dev_stats(ena_dev, &ctx, ENA_ADMIN_GET_STATS_TYPE_ENI); if (likely(ret == 0)) - memcpy(stats, &get_resp.basic_stats, - sizeof(get_resp.basic_stats)); + memcpy(stats, &ctx.get_resp.u.eni_stats, + sizeof(ctx.get_resp.u.eni_stats)); return ret; } -int ena_com_get_dev_extended_stats(struct ena_com_dev *ena_dev, char *buff, - u32 len) +int ena_com_get_dev_basic_stats(struct ena_com_dev *ena_dev, + struct ena_admin_basic_stats *stats) { - int ret = 0; - struct ena_admin_aq_get_stats_cmd get_cmd; - struct ena_admin_acq_get_stats_resp get_resp; - ena_mem_handle_t mem_handle = 0; - void *virt_addr; - dma_addr_t phys_addr; - - ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, len, - virt_addr, phys_addr, mem_handle); - if (!virt_addr) { - ret = ENA_COM_NO_MEM; - goto done; - } - memset(&get_cmd, 0x0, sizeof(get_cmd)); - ret = ena_com_mem_addr_set(ena_dev, - &get_cmd.u.control_buffer.address, - phys_addr); - if (unlikely(ret)) { - ena_trc_err("memory address set failed\n"); - return ret; - } - get_cmd.u.control_buffer.length = len; + struct ena_com_stats_ctx ctx; + int ret; - get_cmd.device_id = ena_dev->stats_func; - get_cmd.queue_idx = ena_dev->stats_queue; + memset(&ctx, 0x0, sizeof(ctx)); + ret = ena_get_dev_stats(ena_dev, &ctx, ENA_ADMIN_GET_STATS_TYPE_BASIC); + if (likely(ret == 0)) + memcpy(stats, &ctx.get_resp.u.basic_stats, + sizeof(ctx.get_resp.u.basic_stats)); - ret = ena_get_dev_stats(ena_dev, &get_cmd, &get_resp, - ENA_ADMIN_GET_STATS_TYPE_EXTENDED); - if (ret < 0) - goto free_ext_stats_mem; - - ret = snprintf(buff, len, "%s", (char *)virt_addr); - -free_ext_stats_mem: - ENA_MEM_FREE_COHERENT(ena_dev->dmadev, len, virt_addr, phys_addr, - mem_handle); -done: return ret; } @@ -2023,16 +2266,11 @@ struct ena_com_admin_queue *admin_queue; struct ena_admin_set_feat_cmd cmd; struct ena_admin_set_feat_resp resp; - int ret = 0; - - if (unlikely(!ena_dev)) { - ena_trc_err("%s : ena_dev is NULL\n", __func__); - return ENA_COM_NO_DEVICE; - } + int ret; if (!ena_com_check_supported_feature_id(ena_dev, ENA_ADMIN_MTU)) { - ena_trc_info("Feature %d isn't supported\n", ENA_ADMIN_MTU); - return ENA_COM_PERMISSION; + ena_trc_dbg(ena_dev, "Feature %d isn't supported\n", ENA_ADMIN_MTU); + return ENA_COM_UNSUPPORTED; } memset(&cmd, 0x0, sizeof(cmd)); @@ -2041,7 +2279,7 @@ cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; cmd.aq_common_descriptor.flags = 0; cmd.feat_common.feature_id = ENA_ADMIN_MTU; - cmd.u.mtu.mtu = mtu; + cmd.u.mtu.mtu = (u32)mtu; ret = ena_com_execute_admin_command(admin_queue, (struct ena_admin_aq_entry *)&cmd, @@ -2049,11 +2287,10 @@ (struct ena_admin_acq_entry *)&resp, sizeof(resp)); - if (unlikely(ret)) { - ena_trc_err("Failed to set mtu %d. error: %d\n", mtu, ret); - return ENA_COM_INVAL; - } - return 0; + if (unlikely(ret)) + ena_trc_err(ena_dev, "Failed to set mtu %d. error: %d\n", mtu, ret); + + return ret; } int ena_com_get_offload_settings(struct ena_com_dev *ena_dev, @@ -2063,10 +2300,10 @@ struct ena_admin_get_feat_resp resp; ret = ena_com_get_feature(ena_dev, &resp, - ENA_ADMIN_STATELESS_OFFLOAD_CONFIG); + ENA_ADMIN_STATELESS_OFFLOAD_CONFIG, 0); if (unlikely(ret)) { - ena_trc_err("Failed to get offload capabilities %d\n", ret); - return ENA_COM_INVAL; + ena_trc_err(ena_dev, "Failed to get offload capabilities %d\n", ret); + return ret; } memcpy(offload, &resp.u.offload, sizeof(resp.u.offload)); @@ -2085,21 +2322,21 @@ if (!ena_com_check_supported_feature_id(ena_dev, ENA_ADMIN_RSS_HASH_FUNCTION)) { - ena_trc_info("Feature %d isn't supported\n", - ENA_ADMIN_RSS_HASH_FUNCTION); - return ENA_COM_PERMISSION; + ena_trc_dbg(ena_dev, "Feature %d isn't supported\n", + ENA_ADMIN_RSS_HASH_FUNCTION); + return ENA_COM_UNSUPPORTED; } /* Validate hash function is supported */ ret = ena_com_get_feature(ena_dev, &get_resp, - ENA_ADMIN_RSS_HASH_FUNCTION); + ENA_ADMIN_RSS_HASH_FUNCTION, 0); if (unlikely(ret)) return ret; - if (get_resp.u.flow_hash_func.supported_func & (1 << rss->hash_func)) { - ena_trc_err("Func hash %d isn't supported by device, abort\n", + if (!(get_resp.u.flow_hash_func.supported_func & BIT(rss->hash_func))) { + ena_trc_err(ena_dev, "Func hash %d isn't supported by device, abort\n", rss->hash_func); - return ENA_COM_PERMISSION; + return ENA_COM_UNSUPPORTED; } memset(&cmd, 0x0, sizeof(cmd)); @@ -2115,7 +2352,7 @@ &cmd.control_buffer.address, rss->hash_key_dma_addr); if (unlikely(ret)) { - ena_trc_err("memory address set failed\n"); + ena_trc_err(ena_dev, "Memory address set failed\n"); return ret; } @@ -2127,7 +2364,7 @@ (struct ena_admin_acq_entry *)&resp, sizeof(resp)); if (unlikely(ret)) { - ena_trc_err("Failed to set hash function %d. error: %d\n", + ena_trc_err(ena_dev, "Failed to set hash function %d. error: %d\n", rss->hash_func, ret); return ENA_COM_INVAL; } @@ -2139,12 +2376,14 @@ enum ena_admin_hash_functions func, const u8 *key, u16 key_len, u32 init_val) { - struct ena_rss *rss = &ena_dev->rss; + struct ena_admin_feature_rss_flow_hash_control *hash_key; struct ena_admin_get_feat_resp get_resp; - struct ena_admin_feature_rss_flow_hash_control *hash_key = - rss->hash_key; + enum ena_admin_hash_functions old_func; + struct ena_rss *rss = &ena_dev->rss; int rc; + hash_key = rss->hash_key; + /* Make sure size is a mult of DWs */ if (unlikely(key_len & 0x3)) return ENA_COM_INVAL; @@ -2152,67 +2391,82 @@ rc = ena_com_get_feature_ex(ena_dev, &get_resp, ENA_ADMIN_RSS_HASH_FUNCTION, rss->hash_key_dma_addr, - sizeof(*rss->hash_key)); + sizeof(*rss->hash_key), 0); if (unlikely(rc)) return rc; - if (!((1 << func) & get_resp.u.flow_hash_func.supported_func)) { - ena_trc_err("Flow hash function %d isn't supported\n", func); - return ENA_COM_PERMISSION; + if (!(BIT(func) & get_resp.u.flow_hash_func.supported_func)) { + ena_trc_err(ena_dev, "Flow hash function %d isn't supported\n", func); + return ENA_COM_UNSUPPORTED; } switch (func) { case ENA_ADMIN_TOEPLITZ: - if (key_len > sizeof(hash_key->key)) { - ena_trc_err("key len (%hu) is bigger than the max supported (%zu)\n", - key_len, sizeof(hash_key->key)); - return ENA_COM_INVAL; + if (key) { + if (key_len != sizeof(hash_key->key)) { + ena_trc_err(ena_dev, "key len (%hu) doesn't equal the supported size (%zu)\n", + key_len, sizeof(hash_key->key)); + return ENA_COM_INVAL; + } + memcpy(hash_key->key, key, key_len); + rss->hash_init_val = init_val; + hash_key->key_parts = key_len / sizeof(hash_key->key[0]); } - - memcpy(hash_key->key, key, key_len); - rss->hash_init_val = init_val; - hash_key->keys_num = key_len >> 2; break; case ENA_ADMIN_CRC32: rss->hash_init_val = init_val; break; default: - ena_trc_err("Invalid hash function (%d)\n", func); + ena_trc_err(ena_dev, "Invalid hash function (%d)\n", func); return ENA_COM_INVAL; } + old_func = rss->hash_func; + rss->hash_func = func; rc = ena_com_set_hash_function(ena_dev); /* Restore the old function */ if (unlikely(rc)) - ena_com_get_hash_function(ena_dev, NULL, NULL); + rss->hash_func = old_func; return rc; } int ena_com_get_hash_function(struct ena_com_dev *ena_dev, - enum ena_admin_hash_functions *func, - u8 *key) + enum ena_admin_hash_functions *func) { struct ena_rss *rss = &ena_dev->rss; struct ena_admin_get_feat_resp get_resp; - struct ena_admin_feature_rss_flow_hash_control *hash_key = - rss->hash_key; int rc; + if (unlikely(!func)) + return ENA_COM_INVAL; + rc = ena_com_get_feature_ex(ena_dev, &get_resp, ENA_ADMIN_RSS_HASH_FUNCTION, rss->hash_key_dma_addr, - sizeof(*rss->hash_key)); + sizeof(*rss->hash_key), 0); if (unlikely(rc)) return rc; - rss->hash_func = (enum ena_admin_hash_functions)get_resp.u.flow_hash_func.selected_func; - if (func) - *func = rss->hash_func; + /* ENA_FFS() returns 1 in case the lsb is set */ + rss->hash_func = ENA_FFS(get_resp.u.flow_hash_func.selected_func); + if (rss->hash_func) + rss->hash_func--; + + *func = rss->hash_func; + + return 0; +} + +int ena_com_get_hash_key(struct ena_com_dev *ena_dev, u8 *key) +{ + struct ena_admin_feature_rss_flow_hash_control *hash_key = + ena_dev->rss.hash_key; if (key) - memcpy(key, hash_key->key, (size_t)(hash_key->keys_num) << 2); + memcpy(key, hash_key->key, + (size_t)(hash_key->key_parts) * sizeof(hash_key->key[0])); return 0; } @@ -2228,7 +2482,7 @@ rc = ena_com_get_feature_ex(ena_dev, &get_resp, ENA_ADMIN_RSS_HASH_INPUT, rss->hash_ctrl_dma_addr, - sizeof(*rss->hash_ctrl)); + sizeof(*rss->hash_ctrl), 0); if (unlikely(rc)) return rc; @@ -2249,11 +2503,13 @@ if (!ena_com_check_supported_feature_id(ena_dev, ENA_ADMIN_RSS_HASH_INPUT)) { - ena_trc_info("Feature %d isn't supported\n", - ENA_ADMIN_RSS_HASH_INPUT); - return ENA_COM_PERMISSION; + ena_trc_dbg(ena_dev, "Feature %d isn't supported\n", + ENA_ADMIN_RSS_HASH_INPUT); + return ENA_COM_UNSUPPORTED; } + memset(&cmd, 0x0, sizeof(cmd)); + cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; cmd.aq_common_descriptor.flags = ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK; @@ -2266,7 +2522,7 @@ &cmd.control_buffer.address, rss->hash_ctrl_dma_addr); if (unlikely(ret)) { - ena_trc_err("memory address set failed\n"); + ena_trc_err(ena_dev, "Memory address set failed\n"); return ret; } cmd.control_buffer.length = sizeof(*hash_ctrl); @@ -2276,12 +2532,10 @@ sizeof(cmd), (struct ena_admin_acq_entry *)&resp, sizeof(resp)); - if (unlikely(ret)) { - ena_trc_err("Failed to set hash input. error: %d\n", ret); - ret = ENA_COM_INVAL; - } + if (unlikely(ret)) + ena_trc_err(ena_dev, "Failed to set hash input. error: %d\n", ret); - return 0; + return ret; } int ena_com_set_default_hash_ctrl(struct ena_com_dev *ena_dev) @@ -2293,7 +2547,7 @@ int rc, i; /* Get the supported hash input */ - rc = ena_com_get_hash_ctrl(ena_dev, (enum ena_admin_flow_hash_proto)0, NULL); + rc = ena_com_get_hash_ctrl(ena_dev, 0, NULL); if (unlikely(rc)) return rc; @@ -2322,17 +2576,17 @@ hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4_FRAG].fields = ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA; - hash_ctrl->selected_fields[ENA_ADMIN_RSS_IP4_FRAG].fields = + hash_ctrl->selected_fields[ENA_ADMIN_RSS_NOT_IP].fields = ENA_ADMIN_RSS_L2_DA | ENA_ADMIN_RSS_L2_SA; for (i = 0; i < ENA_ADMIN_RSS_PROTO_NUM; i++) { available_fields = hash_ctrl->selected_fields[i].fields & hash_ctrl->supported_fields[i].fields; if (available_fields != hash_ctrl->selected_fields[i].fields) { - ena_trc_err("hash control doesn't support all the desire configuration. proto %x supported %x selected %x\n", + ena_trc_err(ena_dev, "Hash control doesn't support all the desire configuration. proto %x supported %x selected %x\n", i, hash_ctrl->supported_fields[i].fields, hash_ctrl->selected_fields[i].fields); - return ENA_COM_PERMISSION; + return ENA_COM_UNSUPPORTED; } } @@ -2340,7 +2594,7 @@ /* In case of failure, restore the old hash ctrl */ if (unlikely(rc)) - ena_com_get_hash_ctrl(ena_dev, (enum ena_admin_flow_hash_proto)0, NULL); + ena_com_get_hash_ctrl(ena_dev, 0, NULL); return rc; } @@ -2355,7 +2609,7 @@ int rc; if (proto >= ENA_ADMIN_RSS_PROTO_NUM) { - ena_trc_err("Invalid proto num (%u)\n", proto); + ena_trc_err(ena_dev, "Invalid proto num (%u)\n", proto); return ENA_COM_INVAL; } @@ -2367,7 +2621,7 @@ /* Make sure all the fields are supported */ supported_fields = hash_ctrl->supported_fields[proto].fields; if ((hash_fields & supported_fields) != hash_fields) { - ena_trc_err("proto %d doesn't support the required fields %x. supports only: %x\n", + ena_trc_err(ena_dev, "Proto %d doesn't support the required fields %x. supports only: %x\n", proto, hash_fields, supported_fields); } @@ -2377,7 +2631,7 @@ /* In case of failure, restore the old hash ctrl */ if (unlikely(rc)) - ena_com_get_hash_ctrl(ena_dev, (enum ena_admin_flow_hash_proto)0, NULL); + ena_com_get_hash_ctrl(ena_dev, 0, NULL); return 0; } @@ -2404,19 +2658,18 @@ struct ena_rss *rss = &ena_dev->rss; struct ena_admin_set_feat_cmd cmd; struct ena_admin_set_feat_resp resp; - int ret = 0; + int ret; - if (!ena_com_check_supported_feature_id( - ena_dev, - ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG)) { - ena_trc_info("Feature %d isn't supported\n", - ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG); - return ENA_COM_PERMISSION; + if (!ena_com_check_supported_feature_id(ena_dev, + ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG)) { + ena_trc_dbg(ena_dev, "Feature %d isn't supported\n", + ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG); + return ENA_COM_UNSUPPORTED; } ret = ena_com_ind_tbl_convert_to_device(ena_dev); if (ret) { - ena_trc_err("Failed to convert host indirection table to device table\n"); + ena_trc_err(ena_dev, "Failed to convert host indirection table to device table\n"); return ret; } @@ -2425,7 +2678,7 @@ cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; cmd.aq_common_descriptor.flags = ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK; - cmd.feat_common.feature_id = ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG; + cmd.feat_common.feature_id = ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG; cmd.u.ind_table.size = rss->tbl_log_size; cmd.u.ind_table.inline_index = 0xFFFFFFFF; @@ -2433,11 +2686,11 @@ &cmd.control_buffer.address, rss->rss_ind_tbl_dma_addr); if (unlikely(ret)) { - ena_trc_err("memory address set failed\n"); + ena_trc_err(ena_dev, "Memory address set failed\n"); return ret; } - cmd.control_buffer.length = (1ULL << rss->tbl_log_size) * + cmd.control_buffer.length = (u32)(1ULL << rss->tbl_log_size) * sizeof(struct ena_admin_rss_ind_table_entry); ret = ena_com_execute_admin_command(admin_queue, @@ -2446,12 +2699,10 @@ (struct ena_admin_acq_entry *)&resp, sizeof(resp)); - if (unlikely(ret)) { - ena_trc_err("Failed to set indirect table. error: %d\n", ret); - return ENA_COM_INVAL; - } + if (unlikely(ret)) + ena_trc_err(ena_dev, "Failed to set indirect table. error: %d\n", ret); - return 0; + return ret; } int ena_com_indirect_table_get(struct ena_com_dev *ena_dev, u32 *ind_tbl) @@ -2461,23 +2712,19 @@ u32 tbl_size; int i, rc; - tbl_size = (1ULL << rss->tbl_log_size) * + tbl_size = (u32)(1ULL << rss->tbl_log_size) * sizeof(struct ena_admin_rss_ind_table_entry); rc = ena_com_get_feature_ex(ena_dev, &get_resp, - ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG, + ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG, rss->rss_ind_tbl_dma_addr, - tbl_size); + tbl_size, 0); if (unlikely(rc)) return rc; if (!ind_tbl) return 0; - rc = ena_com_ind_tbl_convert_from_device(ena_dev); - if (unlikely(rc)) - return rc; - for (i = 0; i < (1 << rss->tbl_log_size); i++) ind_tbl[i] = rss->host_rss_ind_tbl[i]; @@ -2494,8 +2741,14 @@ if (unlikely(rc)) goto err_indr_tbl; + /* The following function might return unsupported in case the + * device doesn't support setting the key / hash function. We can safely + * ignore this error and have indirection table support only. + */ rc = ena_com_hash_key_allocate(ena_dev); - if (unlikely(rc)) + if (likely(!rc)) + ena_com_hash_key_fill_default_key(ena_dev); + else if (rc != ENA_COM_UNSUPPORTED) goto err_hash_key; rc = ena_com_hash_ctrl_init(ena_dev); @@ -2534,21 +2787,26 @@ if (unlikely(!host_attr->host_info)) return ENA_COM_NO_MEM; + host_attr->host_info->ena_spec_version = ((ENA_COMMON_SPEC_VERSION_MAJOR << + ENA_REGS_VERSION_MAJOR_VERSION_SHIFT) | + (ENA_COMMON_SPEC_VERSION_MINOR)); + return 0; } int ena_com_allocate_debug_area(struct ena_com_dev *ena_dev, - u32 debug_area_size) { + u32 debug_area_size) +{ struct ena_host_attribute *host_attr = &ena_dev->host_attr; - ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, - debug_area_size, - host_attr->debug_area_virt_addr, - host_attr->debug_area_dma_addr, - host_attr->debug_area_dma_handle); - if (unlikely(!host_attr->debug_area_virt_addr)) { - host_attr->debug_area_size = 0; - return ENA_COM_NO_MEM; + ENA_MEM_ALLOC_COHERENT(ena_dev->dmadev, + debug_area_size, + host_attr->debug_area_virt_addr, + host_attr->debug_area_dma_addr, + host_attr->debug_area_dma_handle); + if (unlikely(!host_attr->debug_area_virt_addr)) { + host_attr->debug_area_size = 0; + return ENA_COM_NO_MEM; } host_attr->debug_area_size = debug_area_size; @@ -2590,6 +2848,7 @@ struct ena_com_admin_queue *admin_queue; struct ena_admin_set_feat_cmd cmd; struct ena_admin_set_feat_resp resp; + int ret; /* Host attribute config is called before ena_com_get_dev_attr_feat @@ -2606,7 +2865,7 @@ &cmd.u.host_attr.debug_ba, host_attr->debug_area_dma_addr); if (unlikely(ret)) { - ena_trc_err("memory address set failed\n"); + ena_trc_err(ena_dev, "Memory address set failed\n"); return ret; } @@ -2614,7 +2873,7 @@ &cmd.u.host_attr.os_info_ba, host_attr->host_info_dma_addr); if (unlikely(ret)) { - ena_trc_err("memory address set failed\n"); + ena_trc_err(ena_dev, "Memory address set failed\n"); return ret; } @@ -2627,7 +2886,7 @@ sizeof(resp)); if (unlikely(ret)) - ena_trc_err("Failed to set host attributes: %d\n", ret); + ena_trc_err(ena_dev, "Failed to set host attributes: %d\n", ret); return ret; } @@ -2635,49 +2894,41 @@ /* Interrupt moderation */ bool ena_com_interrupt_moderation_supported(struct ena_com_dev *ena_dev) { - return ena_com_check_supported_feature_id( - ena_dev, - ENA_ADMIN_INTERRUPT_MODERATION); + return ena_com_check_supported_feature_id(ena_dev, + ENA_ADMIN_INTERRUPT_MODERATION); } -int -ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev, - u32 tx_coalesce_usecs) +static int ena_com_update_nonadaptive_moderation_interval(struct ena_com_dev *ena_dev, + u32 coalesce_usecs, + u32 intr_delay_resolution, + u32 *intr_moder_interval) { - if (!ena_dev->intr_delay_resolution) { - ena_trc_err("Illegal interrupt delay granularity value\n"); + if (!intr_delay_resolution) { + ena_trc_err(ena_dev, "Illegal interrupt delay granularity value\n"); return ENA_COM_FAULT; } - ena_dev->intr_moder_tx_interval = tx_coalesce_usecs / - ena_dev->intr_delay_resolution; + *intr_moder_interval = coalesce_usecs / intr_delay_resolution; return 0; } -int -ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev, - u32 rx_coalesce_usecs) +int ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev, + u32 tx_coalesce_usecs) { - if (!ena_dev->intr_delay_resolution) { - ena_trc_err("Illegal interrupt delay granularity value\n"); - return ENA_COM_FAULT; - } - - /* We use LOWEST entry of moderation table for storing - * nonadaptive interrupt coalescing values - */ - ena_dev->intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval = - rx_coalesce_usecs / ena_dev->intr_delay_resolution; - - return 0; + return ena_com_update_nonadaptive_moderation_interval(ena_dev, + tx_coalesce_usecs, + ena_dev->intr_delay_resolution, + &ena_dev->intr_moder_tx_interval); } -void ena_com_destroy_interrupt_moderation(struct ena_com_dev *ena_dev) +int ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev, + u32 rx_coalesce_usecs) { - if (ena_dev->intr_moder_tbl) - ENA_MEM_FREE(ena_dev->dmadev, ena_dev->intr_moder_tbl); - ena_dev->intr_moder_tbl = NULL; + return ena_com_update_nonadaptive_moderation_interval(ena_dev, + rx_coalesce_usecs, + ena_dev->intr_delay_resolution, + &ena_dev->intr_moder_rx_interval); } int ena_com_init_interrupt_moderation(struct ena_com_dev *ena_dev) @@ -2687,16 +2938,16 @@ int rc; rc = ena_com_get_feature(ena_dev, &get_resp, - ENA_ADMIN_INTERRUPT_MODERATION); + ENA_ADMIN_INTERRUPT_MODERATION, 0); if (rc) { - if (rc == ENA_COM_PERMISSION) { - ena_trc_info("Feature %d isn't supported\n", - ENA_ADMIN_INTERRUPT_MODERATION); + if (rc == ENA_COM_UNSUPPORTED) { + ena_trc_dbg(ena_dev, "Feature %d isn't supported\n", + ENA_ADMIN_INTERRUPT_MODERATION); rc = 0; } else { - ena_trc_err("Failed to get interrupt moderation admin cmd. rc: %d\n", - rc); + ena_trc_err(ena_dev, + "Failed to get interrupt moderation admin cmd. rc: %d\n", rc); } /* no moderation supported, disable adaptive support */ @@ -2704,112 +2955,51 @@ return rc; } - rc = ena_com_init_interrupt_moderation_table(ena_dev); - if (rc) - goto err; - /* if moderation is supported by device we set adaptive moderation */ delay_resolution = get_resp.u.intr_moderation.intr_delay_resolution; ena_com_update_intr_delay_resolution(ena_dev, delay_resolution); - ena_com_enable_adaptive_moderation(ena_dev); - - return 0; -err: - ena_com_destroy_interrupt_moderation(ena_dev); - return rc; -} -void -ena_com_config_default_interrupt_moderation_table(struct ena_com_dev *ena_dev) -{ - struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl; - - if (!intr_moder_tbl) - return; + /* Disable adaptive moderation by default - can be enabled later */ + ena_com_disable_adaptive_moderation(ena_dev); - intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval = - ENA_INTR_LOWEST_USECS; - intr_moder_tbl[ENA_INTR_MODER_LOWEST].pkts_per_interval = - ENA_INTR_LOWEST_PKTS; - intr_moder_tbl[ENA_INTR_MODER_LOWEST].bytes_per_interval = - ENA_INTR_LOWEST_BYTES; - - intr_moder_tbl[ENA_INTR_MODER_LOW].intr_moder_interval = - ENA_INTR_LOW_USECS; - intr_moder_tbl[ENA_INTR_MODER_LOW].pkts_per_interval = - ENA_INTR_LOW_PKTS; - intr_moder_tbl[ENA_INTR_MODER_LOW].bytes_per_interval = - ENA_INTR_LOW_BYTES; - - intr_moder_tbl[ENA_INTR_MODER_MID].intr_moder_interval = - ENA_INTR_MID_USECS; - intr_moder_tbl[ENA_INTR_MODER_MID].pkts_per_interval = - ENA_INTR_MID_PKTS; - intr_moder_tbl[ENA_INTR_MODER_MID].bytes_per_interval = - ENA_INTR_MID_BYTES; - - intr_moder_tbl[ENA_INTR_MODER_HIGH].intr_moder_interval = - ENA_INTR_HIGH_USECS; - intr_moder_tbl[ENA_INTR_MODER_HIGH].pkts_per_interval = - ENA_INTR_HIGH_PKTS; - intr_moder_tbl[ENA_INTR_MODER_HIGH].bytes_per_interval = - ENA_INTR_HIGH_BYTES; - - intr_moder_tbl[ENA_INTR_MODER_HIGHEST].intr_moder_interval = - ENA_INTR_HIGHEST_USECS; - intr_moder_tbl[ENA_INTR_MODER_HIGHEST].pkts_per_interval = - ENA_INTR_HIGHEST_PKTS; - intr_moder_tbl[ENA_INTR_MODER_HIGHEST].bytes_per_interval = - ENA_INTR_HIGHEST_BYTES; + return 0; } -unsigned int -ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev) +unsigned int ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev) { return ena_dev->intr_moder_tx_interval; } -unsigned int -ena_com_get_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev) +unsigned int ena_com_get_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev) { - struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl; - - if (intr_moder_tbl) - return intr_moder_tbl[ENA_INTR_MODER_LOWEST].intr_moder_interval; - - return 0; + return ena_dev->intr_moder_rx_interval; } -void ena_com_init_intr_moderation_entry(struct ena_com_dev *ena_dev, - enum ena_intr_moder_level level, - struct ena_intr_moder_entry *entry) +int ena_com_config_dev_mode(struct ena_com_dev *ena_dev, + struct ena_admin_feature_llq_desc *llq_features, + struct ena_llq_configurations *llq_default_cfg) { - struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl; + struct ena_com_llq_info *llq_info = &ena_dev->llq_info; + int rc; - if (level >= ENA_INTR_MAX_NUM_OF_LEVELS) - return; + if (!llq_features->max_llq_num) { + ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; + return 0; + } - intr_moder_tbl[level].intr_moder_interval = entry->intr_moder_interval; - if (ena_dev->intr_delay_resolution) - intr_moder_tbl[level].intr_moder_interval /= - ena_dev->intr_delay_resolution; - intr_moder_tbl[level].pkts_per_interval = entry->pkts_per_interval; - intr_moder_tbl[level].bytes_per_interval = entry->bytes_per_interval; -} + rc = ena_com_config_llq_info(ena_dev, llq_features, llq_default_cfg); + if (rc) + return rc; -void ena_com_get_intr_moderation_entry(struct ena_com_dev *ena_dev, - enum ena_intr_moder_level level, - struct ena_intr_moder_entry *entry) -{ - struct ena_intr_moder_entry *intr_moder_tbl = ena_dev->intr_moder_tbl; + ena_dev->tx_max_header_size = llq_info->desc_list_entry_size - + (llq_info->descs_num_before_header * sizeof(struct ena_eth_io_tx_desc)); - if (level >= ENA_INTR_MAX_NUM_OF_LEVELS) - return; + if (unlikely(ena_dev->tx_max_header_size == 0)) { + ena_trc_err(ena_dev, "The size of the LLQ entry is smaller than needed\n"); + return -EINVAL; + } - entry->intr_moder_interval = intr_moder_tbl[level].intr_moder_interval; - if (ena_dev->intr_delay_resolution) - entry->intr_moder_interval *= ena_dev->intr_delay_resolution; - entry->pkts_per_interval = - intr_moder_tbl[level].pkts_per_interval; - entry->bytes_per_interval = intr_moder_tbl[level].bytes_per_interval; + ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_DEV; + + return 0; } Index: /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_defs/ena_admin_defs.h =================================================================== --- /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_defs/ena_admin_defs.h (revision 38965) +++ /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_defs/ena_admin_defs.h (working copy) @@ -1,207 +1,115 @@ -/*- -* BSD LICENSE -* -* Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. -* All rights reserved. -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following conditions -* are met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above copyright -* notice, this list of conditions and the following disclaimer in -* the documentation and/or other materials provided with the -* distribution. -* * Neither the name of copyright holder nor the names of its -* contributors may be used to endorse or promote products derived -* from this software without specific prior written permission. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. + * All rights reserved. + */ #ifndef _ENA_ADMIN_H_ #define _ENA_ADMIN_H_ -/* admin commands opcodes */ -enum ena_admin_aq_opcode { - /* create submission queue */ - ENA_ADMIN_CREATE_SQ = 1, - - /* destroy submission queue */ - ENA_ADMIN_DESTROY_SQ = 2, - - /* create completion queue */ - ENA_ADMIN_CREATE_CQ = 3, +#define ENA_ADMIN_RSS_KEY_PARTS 10 - /* destroy completion queue */ - ENA_ADMIN_DESTROY_CQ = 4, - - /* get capabilities of particular feature */ - ENA_ADMIN_GET_FEATURE = 8, - - /* get capabilities of particular feature */ - ENA_ADMIN_SET_FEATURE = 9, - - /* get statistics */ - ENA_ADMIN_GET_STATS = 11, +enum ena_admin_aq_opcode { + ENA_ADMIN_CREATE_SQ = 1, + ENA_ADMIN_DESTROY_SQ = 2, + ENA_ADMIN_CREATE_CQ = 3, + ENA_ADMIN_DESTROY_CQ = 4, + ENA_ADMIN_GET_FEATURE = 8, + ENA_ADMIN_SET_FEATURE = 9, + ENA_ADMIN_GET_STATS = 11, }; -/* admin command completion status codes */ enum ena_admin_aq_completion_status { - /* Request completed successfully */ - ENA_ADMIN_SUCCESS = 0, - - /* no resources to satisfy request */ - ENA_ADMIN_RESOURCE_ALLOCATION_FAILURE = 1, - - /* Bad opcode in request descriptor */ - ENA_ADMIN_BAD_OPCODE = 2, - - /* Unsupported opcode in request descriptor */ - ENA_ADMIN_UNSUPPORTED_OPCODE = 3, - - /* Wrong request format */ - ENA_ADMIN_MALFORMED_REQUEST = 4, - - /* One of parameters is not valid. Provided in ACQ entry - * extended_status - */ - ENA_ADMIN_ILLEGAL_PARAMETER = 5, - - /* unexpected error */ - ENA_ADMIN_UNKNOWN_ERROR = 6, + ENA_ADMIN_SUCCESS = 0, + ENA_ADMIN_RESOURCE_ALLOCATION_FAILURE = 1, + ENA_ADMIN_BAD_OPCODE = 2, + ENA_ADMIN_UNSUPPORTED_OPCODE = 3, + ENA_ADMIN_MALFORMED_REQUEST = 4, + /* Additional status is provided in ACQ entry extended_status */ + ENA_ADMIN_ILLEGAL_PARAMETER = 5, + ENA_ADMIN_UNKNOWN_ERROR = 6, + ENA_ADMIN_RESOURCE_BUSY = 7, }; -/* get/set feature subcommands opcodes */ +/* subcommands for the set/get feature admin commands */ enum ena_admin_aq_feature_id { - /* list of all supported attributes/capabilities in the ENA */ - ENA_ADMIN_DEVICE_ATTRIBUTES = 1, - - /* max number of supported queues per for every queues type */ - ENA_ADMIN_MAX_QUEUES_NUM = 2, - - /* Receive Side Scaling (RSS) function */ - ENA_ADMIN_RSS_HASH_FUNCTION = 10, - - /* stateless TCP/UDP/IP offload capabilities. */ - ENA_ADMIN_STATELESS_OFFLOAD_CONFIG = 11, - - /* Multiple tuples flow table configuration */ - ENA_ADMIN_RSS_REDIRECTION_TABLE_CONFIG = 12, - - /* max MTU, current MTU */ - ENA_ADMIN_MTU = 14, - - /* Receive Side Scaling (RSS) hash input */ - ENA_ADMIN_RSS_HASH_INPUT = 18, - - /* interrupt moderation parameters */ - ENA_ADMIN_INTERRUPT_MODERATION = 20, - - /* AENQ configuration */ - ENA_ADMIN_AENQ_CONFIG = 26, - - /* Link configuration */ - ENA_ADMIN_LINK_CONFIG = 27, - - /* Host attributes configuration */ - ENA_ADMIN_HOST_ATTR_CONFIG = 28, - - /* Number of valid opcodes */ - ENA_ADMIN_FEATURES_OPCODE_NUM = 32, + ENA_ADMIN_DEVICE_ATTRIBUTES = 1, + ENA_ADMIN_MAX_QUEUES_NUM = 2, + ENA_ADMIN_HW_HINTS = 3, + ENA_ADMIN_LLQ = 4, + ENA_ADMIN_EXTRA_PROPERTIES_STRINGS = 5, + ENA_ADMIN_EXTRA_PROPERTIES_FLAGS = 6, + ENA_ADMIN_MAX_QUEUES_EXT = 7, + ENA_ADMIN_RSS_HASH_FUNCTION = 10, + ENA_ADMIN_STATELESS_OFFLOAD_CONFIG = 11, + ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG = 12, + ENA_ADMIN_MTU = 14, + ENA_ADMIN_RSS_HASH_INPUT = 18, + ENA_ADMIN_INTERRUPT_MODERATION = 20, + ENA_ADMIN_AENQ_CONFIG = 26, + ENA_ADMIN_LINK_CONFIG = 27, + ENA_ADMIN_HOST_ATTR_CONFIG = 28, + ENA_ADMIN_FEATURES_OPCODE_NUM = 32, }; -/* descriptors and headers placement */ enum ena_admin_placement_policy_type { - /* descriptors and headers are in OS memory */ - ENA_ADMIN_PLACEMENT_POLICY_HOST = 1, - - /* descriptors and headers in device memory (a.k.a Low Latency + /* descriptors and headers are in host memory */ + ENA_ADMIN_PLACEMENT_POLICY_HOST = 1, + /* descriptors and headers are in device memory (a.k.a Low Latency * Queue) */ - ENA_ADMIN_PLACEMENT_POLICY_DEV = 3, + ENA_ADMIN_PLACEMENT_POLICY_DEV = 3, }; -/* link speeds */ enum ena_admin_link_types { - ENA_ADMIN_LINK_SPEED_1G = 0x1, - - ENA_ADMIN_LINK_SPEED_2_HALF_G = 0x2, - - ENA_ADMIN_LINK_SPEED_5G = 0x4, - - ENA_ADMIN_LINK_SPEED_10G = 0x8, - - ENA_ADMIN_LINK_SPEED_25G = 0x10, - - ENA_ADMIN_LINK_SPEED_40G = 0x20, - - ENA_ADMIN_LINK_SPEED_50G = 0x40, - - ENA_ADMIN_LINK_SPEED_100G = 0x80, - - ENA_ADMIN_LINK_SPEED_200G = 0x100, - - ENA_ADMIN_LINK_SPEED_400G = 0x200, + ENA_ADMIN_LINK_SPEED_1G = 0x1, + ENA_ADMIN_LINK_SPEED_2_HALF_G = 0x2, + ENA_ADMIN_LINK_SPEED_5G = 0x4, + ENA_ADMIN_LINK_SPEED_10G = 0x8, + ENA_ADMIN_LINK_SPEED_25G = 0x10, + ENA_ADMIN_LINK_SPEED_40G = 0x20, + ENA_ADMIN_LINK_SPEED_50G = 0x40, + ENA_ADMIN_LINK_SPEED_100G = 0x80, + ENA_ADMIN_LINK_SPEED_200G = 0x100, + ENA_ADMIN_LINK_SPEED_400G = 0x200, }; -/* completion queue update policy */ enum ena_admin_completion_policy_type { - /* cqe for each sq descriptor */ - ENA_ADMIN_COMPLETION_POLICY_DESC = 0, - - /* cqe upon request in sq descriptor */ - ENA_ADMIN_COMPLETION_POLICY_DESC_ON_DEMAND = 1, - + /* completion queue entry for each sq descriptor */ + ENA_ADMIN_COMPLETION_POLICY_DESC = 0, + /* completion queue entry upon request in sq descriptor */ + ENA_ADMIN_COMPLETION_POLICY_DESC_ON_DEMAND = 1, /* current queue head pointer is updated in OS memory upon sq * descriptor request */ - ENA_ADMIN_COMPLETION_POLICY_HEAD_ON_DEMAND = 2, - + ENA_ADMIN_COMPLETION_POLICY_HEAD_ON_DEMAND = 2, /* current queue head pointer is updated in OS memory for each sq * descriptor */ - ENA_ADMIN_COMPLETION_POLICY_HEAD = 3, + ENA_ADMIN_COMPLETION_POLICY_HEAD = 3, }; -/* type of get statistics command */ +/* basic stats return ena_admin_basic_stats while extanded stats return a + * buffer (string format) with additional statistics per queue and per + * device id + */ enum ena_admin_get_stats_type { - /* Basic statistics */ - ENA_ADMIN_GET_STATS_TYPE_BASIC = 0, - - /* Extended statistics */ - ENA_ADMIN_GET_STATS_TYPE_EXTENDED = 1, + ENA_ADMIN_GET_STATS_TYPE_BASIC = 0, + ENA_ADMIN_GET_STATS_TYPE_EXTENDED = 1, + /* extra HW stats for specific network interface */ + ENA_ADMIN_GET_STATS_TYPE_ENI = 2, }; -/* scope of get statistics command */ enum ena_admin_get_stats_scope { - ENA_ADMIN_SPECIFIC_QUEUE = 0, - - ENA_ADMIN_ETH_TRAFFIC = 1, + ENA_ADMIN_SPECIFIC_QUEUE = 0, + ENA_ADMIN_ETH_TRAFFIC = 1, }; -/* ENA Admin Queue (AQ) common descriptor */ struct ena_admin_aq_common_desc { - /* word 0 : */ - /* command identificator to associate it with the completion - * 11:0 : command_id + /* 11:0 : command_id * 15:12 : reserved12 */ uint16_t command_id; - /* as appears in ena_aq_opcode */ + /* as appears in ena_admin_aq_opcode */ uint8_t opcode; /* 0 : phase @@ -214,24 +122,17 @@ uint8_t flags; }; -/* used in ena_aq_entry. Can point directly to control data, or to a page - * list chunk. Used also at the end of indirect mode page list chunks, for - * chaining. +/* used in ena_admin_aq_entry. Can point directly to control data, or to a + * page list chunk. Used also at the end of indirect mode page list chunks, + * for chaining. */ struct ena_admin_ctrl_buff_info { - /* word 0 : indicates length of the buffer pointed by - * control_buffer_address. - */ uint32_t length; - /* words 1:2 : points to control buffer (direct or indirect) */ struct ena_common_mem_addr address; }; -/* submission queue full identification */ struct ena_admin_sq { - /* word 0 : */ - /* queue id */ uint16_t sq_idx; /* 4:0 : reserved @@ -242,36 +143,25 @@ uint8_t reserved1; }; -/* AQ entry format */ struct ena_admin_aq_entry { - /* words 0 : */ struct ena_admin_aq_common_desc aq_common_descriptor; - /* words 1:3 : */ union { - /* command specific inline data */ uint32_t inline_data_w1[3]; - /* words 1:3 : points to control buffer (direct or - * indirect, chained if needed) - */ struct ena_admin_ctrl_buff_info control_buffer; } u; - /* command specific inline data */ uint32_t inline_data_w4[12]; }; -/* ENA Admin Completion Queue (ACQ) common descriptor */ struct ena_admin_acq_common_desc { - /* word 0 : */ /* command identifier to associate it with the aq descriptor * 11:0 : command_id * 15:12 : reserved12 */ uint16_t command; - /* status of request execution */ uint8_t status; /* 0 : phase @@ -279,33 +169,23 @@ */ uint8_t flags; - /* word 1 : */ - /* provides additional info */ uint16_t extended_status; - /* submission queue head index, serves as a hint what AQ entries can - * be revoked + /* indicates to the driver which AQ entry has been consumed by the + * device and could be reused */ uint16_t sq_head_indx; }; -/* ACQ entry format */ struct ena_admin_acq_entry { - /* words 0:1 : */ struct ena_admin_acq_common_desc acq_common_descriptor; - /* response type specific data */ uint32_t response_specific_data[14]; }; -/* ENA AQ Create Submission Queue command. Placed in control buffer pointed - * by AQ entry - */ struct ena_admin_aq_create_sq_cmd { - /* words 0 : */ struct ena_admin_aq_common_desc aq_common_descriptor; - /* word 1 : */ /* 4:0 : reserved0_w1 * 7:5 : sq_direction - 0x1 - Tx, 0x2 - Rx */ @@ -337,94 +217,69 @@ */ uint8_t sq_caps_3; - /* word 2 : */ - /* associated completion queue id. This CQ must be created prior to - * SQ creation + /* associated completion queue id. This CQ must be created prior to SQ + * creation */ uint16_t cq_idx; /* submission queue depth in entries */ uint16_t sq_depth; - /* words 3:4 : SQ physical base address in OS memory. This field - * should not be used for Low Latency queues. Has to be page - * aligned. + /* SQ physical base address in OS memory. This field should not be + * used for Low Latency queues. Has to be page aligned. */ struct ena_common_mem_addr sq_ba; - /* words 5:6 : specifies queue head writeback location in OS - * memory. Valid if completion_policy is set to - * completion_policy_head_on_demand or completion_policy_head. Has - * to be cache aligned + /* specifies queue head writeback location in OS memory. Valid if + * completion_policy is set to completion_policy_head_on_demand or + * completion_policy_head. Has to be cache aligned */ struct ena_common_mem_addr sq_head_writeback; - /* word 7 : reserved word */ uint32_t reserved0_w7; - /* word 8 : reserved word */ uint32_t reserved0_w8; }; -/* submission queue direction */ enum ena_admin_sq_direction { - ENA_ADMIN_SQ_DIRECTION_TX = 1, - - ENA_ADMIN_SQ_DIRECTION_RX = 2, + ENA_ADMIN_SQ_DIRECTION_TX = 1, + ENA_ADMIN_SQ_DIRECTION_RX = 2, }; -/* ENA Response for Create SQ Command. Appears in ACQ entry as - * response_specific_data - */ struct ena_admin_acq_create_sq_resp_desc { - /* words 0:1 : Common Admin Queue completion descriptor */ struct ena_admin_acq_common_desc acq_common_desc; - /* word 2 : */ - /* sq identifier */ uint16_t sq_idx; uint16_t reserved; - /* word 3 : queue doorbell address as an offset to PCIe MMIO REG BAR */ + /* queue doorbell address as an offset to PCIe MMIO REG BAR */ uint32_t sq_doorbell_offset; - /* word 4 : low latency queue ring base address as an offset to - * PCIe MMIO LLQ_MEM BAR + /* low latency queue ring base address as an offset to PCIe MMIO + * LLQ_MEM BAR */ uint32_t llq_descriptors_offset; - /* word 5 : low latency queue headers' memory as an offset to PCIe - * MMIO LLQ_MEM BAR + /* low latency queue headers' memory as an offset to PCIe MMIO + * LLQ_MEM BAR */ uint32_t llq_headers_offset; }; -/* ENA AQ Destroy Submission Queue command. Placed in control buffer - * pointed by AQ entry - */ struct ena_admin_aq_destroy_sq_cmd { - /* words 0 : */ struct ena_admin_aq_common_desc aq_common_descriptor; - /* words 1 : */ struct ena_admin_sq sq; }; -/* ENA Response for Destroy SQ Command. Appears in ACQ entry as - * response_specific_data - */ struct ena_admin_acq_destroy_sq_resp_desc { - /* words 0:1 : Common Admin Queue completion descriptor */ struct ena_admin_acq_common_desc acq_common_desc; }; -/* ENA AQ Create Completion Queue command */ struct ena_admin_aq_create_cq_cmd { - /* words 0 : */ struct ena_admin_aq_common_desc aq_common_descriptor; - /* word 1 : */ /* 4:0 : reserved5 * 5 : interrupt_mode_enabled - if set, cq operates * in interrupt mode, otherwise - polling @@ -441,62 +296,39 @@ /* completion queue depth in # of entries. must be power of 2 */ uint16_t cq_depth; - /* word 2 : msix vector assigned to this cq */ + /* msix vector assigned to this cq */ uint32_t msix_vector; - /* words 3:4 : cq physical base address in OS memory. CQ must be - * physically contiguous + /* cq physical base address in OS memory. CQ must be physically + * contiguous */ struct ena_common_mem_addr cq_ba; }; -/* ENA Response for Create CQ Command. Appears in ACQ entry as response - * specific data - */ struct ena_admin_acq_create_cq_resp_desc { - /* words 0:1 : Common Admin Queue completion descriptor */ struct ena_admin_acq_common_desc acq_common_desc; - /* word 2 : */ - /* cq identifier */ uint16_t cq_idx; - /* actual cq depth in # of entries */ + /* actual cq depth in number of entries */ uint16_t cq_actual_depth; - /* word 3 : cpu numa node address as an offset to PCIe MMIO REG BAR */ uint32_t numa_node_register_offset; - /* word 4 : completion head doorbell address as an offset to PCIe - * MMIO REG BAR - */ uint32_t cq_head_db_register_offset; - /* word 5 : interrupt unmask register address as an offset into - * PCIe MMIO REG BAR - */ uint32_t cq_interrupt_unmask_register_offset; }; -/* ENA AQ Destroy Completion Queue command. Placed in control buffer - * pointed by AQ entry - */ struct ena_admin_aq_destroy_cq_cmd { - /* words 0 : */ struct ena_admin_aq_common_desc aq_common_descriptor; - /* word 1 : */ - /* associated queue id. */ uint16_t cq_idx; uint16_t reserved1; }; -/* ENA Response for Destroy CQ Command. Appears in ACQ entry as - * response_specific_data - */ struct ena_admin_acq_destroy_cq_resp_desc { - /* words 0:1 : Common Admin Queue completion descriptor */ struct ena_admin_acq_common_desc acq_common_desc; }; @@ -504,21 +336,15 @@ * buffer pointed by AQ entry */ struct ena_admin_aq_get_stats_cmd { - /* words 0 : */ struct ena_admin_aq_common_desc aq_common_descriptor; - /* words 1:3 : */ union { /* command specific inline data */ uint32_t inline_data_w1[3]; - /* words 1:3 : points to control buffer (direct or - * indirect, chained if needed) - */ struct ena_admin_ctrl_buff_info control_buffer; } u; - /* word 4 : */ /* stats type as defined in enum ena_admin_get_stats_type */ uint8_t type; @@ -527,101 +353,116 @@ uint16_t reserved3; - /* word 5 : */ /* queue id. used when scope is specific_queue */ uint16_t queue_idx; /* device id, value 0xFFFF means mine. only privileged device can get - * stats of other device + * stats of other device */ uint16_t device_id; }; /* Basic Statistics Command. */ struct ena_admin_basic_stats { - /* word 0 : */ uint32_t tx_bytes_low; - /* word 1 : */ uint32_t tx_bytes_high; - /* word 2 : */ uint32_t tx_pkts_low; - /* word 3 : */ uint32_t tx_pkts_high; - /* word 4 : */ uint32_t rx_bytes_low; - /* word 5 : */ uint32_t rx_bytes_high; - /* word 6 : */ uint32_t rx_pkts_low; - /* word 7 : */ uint32_t rx_pkts_high; - /* word 8 : */ uint32_t rx_drops_low; - /* word 9 : */ uint32_t rx_drops_high; + + uint32_t tx_drops_low; + + uint32_t tx_drops_high; +}; + +/* ENI Statistics Command. */ +struct ena_admin_eni_stats { + /* The number of packets shaped due to inbound aggregate BW + * allowance being exceeded + */ + uint64_t bw_in_allowance_exceeded; + + /* The number of packets shaped due to outbound aggregate BW + * allowance being exceeded + */ + uint64_t bw_out_allowance_exceeded; + + /* The number of packets shaped due to PPS allowance being exceeded */ + uint64_t pps_allowance_exceeded; + + /* The number of packets shaped due to connection tracking + * allowance being exceeded and leading to failure in establishment + * of new connections + */ + uint64_t conntrack_allowance_exceeded; + + /* The number of packets shaped due to linklocal packet rate + * allowance being exceeded + */ + uint64_t linklocal_allowance_exceeded; }; -/* ENA Response for Get Statistics Command. Appears in ACQ entry as - * response_specific_data - */ struct ena_admin_acq_get_stats_resp { - /* words 0:1 : Common Admin Queue completion descriptor */ struct ena_admin_acq_common_desc acq_common_desc; - /* words 2:11 : */ - struct ena_admin_basic_stats basic_stats; + union { + uint64_t raw[7]; + + struct ena_admin_basic_stats basic_stats; + + struct ena_admin_eni_stats eni_stats; + } u; }; -/* ENA Get/Set Feature common descriptor. Appears as inline word in - * ena_aq_entry - */ struct ena_admin_get_set_feature_common_desc { - /* word 0 : */ /* 1:0 : select - 0x1 - current value; 0x3 - default * value * 7:3 : reserved3 */ uint8_t flags; - /* as appears in ena_feature_id */ + /* as appears in ena_admin_aq_feature_id */ uint8_t feature_id; - /* reserved16 */ - uint16_t reserved16; + /* The driver specifies the max feature version it supports and the + * device responds with the currently supported feature version. The + * field is zero based + */ + uint8_t feature_version; + + uint8_t reserved8; }; -/* ENA Device Attributes Feature descriptor. */ struct ena_admin_device_attr_feature_desc { - /* word 0 : implementation id */ uint32_t impl_id; - /* word 1 : device version */ uint32_t device_version; - /* word 2 : bit map of which bits are supported value of 1 - * indicated that this feature is supported and can perform SET/GET - * for it + /* bitmap of ena_admin_aq_feature_id, which represents supported + * subcommands for the set/get feature admin commands. */ uint32_t supported_features; - /* word 3 : */ uint32_t reserved3; - /* word 4 : Indicates how many bits are used physical address - * access. - */ + /* Indicates how many bits are used physical address access. */ uint32_t phys_addr_width; - /* word 5 : Indicates how many bits are used virtual address access. */ + /* Indicates how many bits are used virtual address access. */ uint32_t virt_addr_width; /* unicast MAC address (in Network byte order) */ @@ -629,36 +470,168 @@ uint8_t reserved7[2]; - /* word 8 : Max supported MTU value */ uint32_t max_mtu; }; -/* ENA Max Queues Feature descriptor. */ +enum ena_admin_llq_header_location { + /* header is in descriptor list */ + ENA_ADMIN_INLINE_HEADER = 1, + /* header in a separate ring, implies 16B descriptor list entry */ + ENA_ADMIN_HEADER_RING = 2, +}; + +enum ena_admin_llq_ring_entry_size { + ENA_ADMIN_LIST_ENTRY_SIZE_128B = 1, + ENA_ADMIN_LIST_ENTRY_SIZE_192B = 2, + ENA_ADMIN_LIST_ENTRY_SIZE_256B = 4, +}; + +enum ena_admin_llq_num_descs_before_header { + ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_0 = 0, + ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_1 = 1, + ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2 = 2, + ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_4 = 4, + ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_8 = 8, +}; + +/* packet descriptor list entry always starts with one or more descriptors, + * followed by a header. The rest of the descriptors are located in the + * beginning of the subsequent entry. Stride refers to how the rest of the + * descriptors are placed. This field is relevant only for inline header + * mode + */ +enum ena_admin_llq_stride_ctrl { + ENA_ADMIN_SINGLE_DESC_PER_ENTRY = 1, + ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY = 2, +}; + +enum ena_admin_accel_mode_feat { + ENA_ADMIN_DISABLE_META_CACHING = 0, + ENA_ADMIN_LIMIT_TX_BURST = 1, +}; + +struct ena_admin_accel_mode_get { + /* bit field of enum ena_admin_accel_mode_feat */ + uint16_t supported_flags; + + /* maximum burst size between two doorbells. The size is in bytes */ + uint16_t max_tx_burst_size; +}; + +struct ena_admin_accel_mode_set { + /* bit field of enum ena_admin_accel_mode_feat */ + uint16_t enabled_flags; + + uint16_t reserved; +}; + +struct ena_admin_accel_mode_req { + union { + uint32_t raw[2]; + + struct ena_admin_accel_mode_get get; + + struct ena_admin_accel_mode_set set; + } u; +}; + +struct ena_admin_feature_llq_desc { + uint32_t max_llq_num; + + uint32_t max_llq_depth; + + /* specify the header locations the device supports. bitfield of enum + * ena_admin_llq_header_location. + */ + uint16_t header_location_ctrl_supported; + + /* the header location the driver selected to use. */ + uint16_t header_location_ctrl_enabled; + + /* if inline header is specified - this is the size of descriptor list + * entry. If header in a separate ring is specified - this is the size + * of header ring entry. bitfield of enum ena_admin_llq_ring_entry_size. + * specify the entry sizes the device supports + */ + uint16_t entry_size_ctrl_supported; + + /* the entry size the driver selected to use. */ + uint16_t entry_size_ctrl_enabled; + + /* valid only if inline header is specified. First entry associated with + * the packet includes descriptors and header. Rest of the entries + * occupied by descriptors. This parameter defines the max number of + * descriptors precedding the header in the first entry. The field is + * bitfield of enum ena_admin_llq_num_descs_before_header and specify + * the values the device supports + */ + uint16_t desc_num_before_header_supported; + + /* the desire field the driver selected to use */ + uint16_t desc_num_before_header_enabled; + + /* valid only if inline was chosen. bitfield of enum + * ena_admin_llq_stride_ctrl + */ + uint16_t descriptors_stride_ctrl_supported; + + /* the stride control the driver selected to use */ + uint16_t descriptors_stride_ctrl_enabled; + + /* reserved */ + uint32_t reserved1; + + /* accelerated low latency queues requirement. driver needs to + * support those requirements in order to use accelerated llq + */ + struct ena_admin_accel_mode_req accel_mode; +}; + +struct ena_admin_queue_ext_feature_fields { + uint32_t max_tx_sq_num; + + uint32_t max_tx_cq_num; + + uint32_t max_rx_sq_num; + + uint32_t max_rx_cq_num; + + uint32_t max_tx_sq_depth; + + uint32_t max_tx_cq_depth; + + uint32_t max_rx_sq_depth; + + uint32_t max_rx_cq_depth; + + uint32_t max_tx_header_size; + + /* Maximum Descriptors number, including meta descriptor, allowed for a + * single Tx packet + */ + uint16_t max_per_packet_tx_descs; + + /* Maximum Descriptors number allowed for a single Rx packet */ + uint16_t max_per_packet_rx_descs; +}; + struct ena_admin_queue_feature_desc { - /* word 0 : Max number of submission queues (including LLQs) */ uint32_t max_sq_num; - /* word 1 : Max submission queue depth */ uint32_t max_sq_depth; - /* word 2 : Max number of completion queues */ uint32_t max_cq_num; - /* word 3 : Max completion queue depth */ uint32_t max_cq_depth; - /* word 4 : Max number of LLQ submission queues */ - uint32_t max_llq_num; + uint32_t max_legacy_llq_num; - /* word 5 : Max submission queue depth of LLQ */ - uint32_t max_llq_depth; + uint32_t max_legacy_llq_depth; - /* word 6 : Max header size */ uint32_t max_header_size; - /* word 7 : */ - /* Maximum Descriptors number, including meta descriptors, allowed - * for a single Tx packet + /* Maximum Descriptors number, including meta descriptor, allowed for a + * single Tx packet */ uint16_t max_packet_tx_descs; @@ -666,86 +639,77 @@ uint16_t max_packet_rx_descs; }; -/* ENA MTU Set Feature descriptor. */ struct ena_admin_set_feature_mtu_desc { - /* word 0 : mtu payload size (exclude L2) */ + /* exclude L2 */ uint32_t mtu; }; -/* ENA host attributes Set Feature descriptor. */ +struct ena_admin_get_extra_properties_strings_desc { + uint32_t count; +}; + +struct ena_admin_get_extra_properties_flags_desc { + uint32_t flags; +}; + struct ena_admin_set_feature_host_attr_desc { - /* words 0:1 : host OS info base address in OS memory. host info is - * 4KB of physically contiguous + /* host OS info base address in OS memory. host info is 4KB of + * physically contiguous */ struct ena_common_mem_addr os_info_ba; - /* words 2:3 : host debug area base address in OS memory. debug - * area must be physically contiguous + /* host debug area base address in OS memory. debug area must be + * physically contiguous */ struct ena_common_mem_addr debug_ba; - /* word 4 : debug area size */ + /* debug area size */ uint32_t debug_area_size; }; -/* ENA Interrupt Moderation Get Feature descriptor. */ struct ena_admin_feature_intr_moder_desc { - /* word 0 : */ /* interrupt delay granularity in usec */ uint16_t intr_delay_resolution; uint16_t reserved; }; -/* ENA Link Get Feature descriptor. */ struct ena_admin_get_feature_link_desc { - /* word 0 : Link speed in Mb */ + /* Link speed in Mb */ uint32_t speed; - /* word 1 : supported speeds (bit field of enum ena_admin_link - * types) - */ + /* bit field of enum ena_admin_link types */ uint32_t supported; - /* word 2 : */ - /* 0 : autoneg - auto negotiation + /* 0 : autoneg * 1 : duplex - Full Duplex * 31:2 : reserved2 */ uint32_t flags; }; -/* ENA AENQ Feature descriptor. */ struct ena_admin_feature_aenq_desc { - /* word 0 : bitmask for AENQ groups the device can report */ + /* bitmask for AENQ groups the device can report */ uint32_t supported_groups; - /* word 1 : bitmask for AENQ groups to report */ + /* bitmask for AENQ groups to report */ uint32_t enabled_groups; }; -/* ENA Stateless Offload Feature descriptor. */ struct ena_admin_feature_offload_desc { - /* word 0 : */ - /* Trasmit side stateless offload - * 0 : TX_L3_csum_ipv4 - IPv4 checksum - * 1 : TX_L4_ipv4_csum_part - TCP/UDP over IPv4 - * checksum, the checksum field should be initialized - * with pseudo header checksum - * 2 : TX_L4_ipv4_csum_full - TCP/UDP over IPv4 - * checksum - * 3 : TX_L4_ipv6_csum_part - TCP/UDP over IPv6 - * checksum, the checksum field should be initialized - * with pseudo header checksum - * 4 : TX_L4_ipv6_csum_full - TCP/UDP over IPv6 - * checksum - * 5 : tso_ipv4 - TCP/IPv4 Segmentation Offloading - * 6 : tso_ipv6 - TCP/IPv6 Segmentation Offloading - * 7 : tso_ecn - TCP Segmentation with ECN + /* 0 : TX_L3_csum_ipv4 + * 1 : TX_L4_ipv4_csum_part - The checksum field + * should be initialized with pseudo header checksum + * 2 : TX_L4_ipv4_csum_full + * 3 : TX_L4_ipv6_csum_part - The checksum field + * should be initialized with pseudo header checksum + * 4 : TX_L4_ipv6_csum_full + * 5 : tso_ipv4 + * 6 : tso_ipv6 + * 7 : tso_ecn */ uint32_t tx; - /* word 1 : */ /* Receive side supported stateless offload * 0 : RX_L3_csum_ipv4 - IPv4 checksum * 1 : RX_L4_ipv4_csum - TCP/UDP/IPv4 checksum @@ -754,118 +718,78 @@ */ uint32_t rx_supported; - /* word 2 : */ - /* Receive side enabled stateless offload */ uint32_t rx_enabled; }; -/* hash functions */ enum ena_admin_hash_functions { - /* Toeplitz hash */ - ENA_ADMIN_TOEPLITZ = 1, - - /* CRC32 hash */ - ENA_ADMIN_CRC32 = 2, + ENA_ADMIN_TOEPLITZ = 1, + ENA_ADMIN_CRC32 = 2, }; -/* ENA RSS flow hash control buffer structure */ struct ena_admin_feature_rss_flow_hash_control { - /* word 0 : number of valid keys */ - uint32_t keys_num; + uint32_t key_parts; - /* word 1 : */ uint32_t reserved; - /* Toeplitz keys */ - uint32_t key[10]; + uint32_t key[ENA_ADMIN_RSS_KEY_PARTS]; }; -/* ENA RSS Flow Hash Function */ struct ena_admin_feature_rss_flow_hash_function { - /* word 0 : */ - /* supported hash functions - * 7:0 : funcs - supported hash functions (bitmask - * accroding to ena_admin_hash_functions) - */ + /* 7:0 : funcs - bitmask of ena_admin_hash_functions */ uint32_t supported_func; - /* word 1 : */ - /* selected hash func - * 7:0 : selected_func - selected hash function - * (bitmask accroding to ena_admin_hash_functions) + /* 7:0 : selected_func - bitmask of + * ena_admin_hash_functions */ uint32_t selected_func; - /* word 2 : initial value */ + /* initial value */ uint32_t init_val; }; /* RSS flow hash protocols */ enum ena_admin_flow_hash_proto { - /* tcp/ipv4 */ - ENA_ADMIN_RSS_TCP4 = 0, - - /* udp/ipv4 */ - ENA_ADMIN_RSS_UDP4 = 1, - - /* tcp/ipv6 */ - ENA_ADMIN_RSS_TCP6 = 2, - - /* udp/ipv6 */ - ENA_ADMIN_RSS_UDP6 = 3, - - /* ipv4 not tcp/udp */ - ENA_ADMIN_RSS_IP4 = 4, - - /* ipv6 not tcp/udp */ - ENA_ADMIN_RSS_IP6 = 5, - - /* fragmented ipv4 */ - ENA_ADMIN_RSS_IP4_FRAG = 6, - - /* not ipv4/6 */ - ENA_ADMIN_RSS_NOT_IP = 7, - - /* max number of protocols */ - ENA_ADMIN_RSS_PROTO_NUM = 16, + ENA_ADMIN_RSS_TCP4 = 0, + ENA_ADMIN_RSS_UDP4 = 1, + ENA_ADMIN_RSS_TCP6 = 2, + ENA_ADMIN_RSS_UDP6 = 3, + ENA_ADMIN_RSS_IP4 = 4, + ENA_ADMIN_RSS_IP6 = 5, + ENA_ADMIN_RSS_IP4_FRAG = 6, + ENA_ADMIN_RSS_NOT_IP = 7, + /* TCPv6 with extension header */ + ENA_ADMIN_RSS_TCP6_EX = 8, + /* IPv6 with extension header */ + ENA_ADMIN_RSS_IP6_EX = 9, + ENA_ADMIN_RSS_PROTO_NUM = 16, }; /* RSS flow hash fields */ enum ena_admin_flow_hash_fields { /* Ethernet Dest Addr */ - ENA_ADMIN_RSS_L2_DA = 0, - + ENA_ADMIN_RSS_L2_DA = BIT(0), /* Ethernet Src Addr */ - ENA_ADMIN_RSS_L2_SA = 1, - + ENA_ADMIN_RSS_L2_SA = BIT(1), /* ipv4/6 Dest Addr */ - ENA_ADMIN_RSS_L3_DA = 2, - + ENA_ADMIN_RSS_L3_DA = BIT(2), /* ipv4/6 Src Addr */ - ENA_ADMIN_RSS_L3_SA = 5, - + ENA_ADMIN_RSS_L3_SA = BIT(3), /* tcp/udp Dest Port */ - ENA_ADMIN_RSS_L4_DP = 6, - + ENA_ADMIN_RSS_L4_DP = BIT(4), /* tcp/udp Src Port */ - ENA_ADMIN_RSS_L4_SP = 7, + ENA_ADMIN_RSS_L4_SP = BIT(5), }; -/* hash input fields for flow protocol */ struct ena_admin_proto_input { - /* word 0 : */ /* flow hash fields (bitwise according to ena_admin_flow_hash_fields) */ uint16_t fields; uint16_t reserved2; }; -/* ENA RSS hash control buffer structure */ struct ena_admin_feature_rss_hash_control { - /* supported input fields */ struct ena_admin_proto_input supported_fields[ENA_ADMIN_RSS_PROTO_NUM]; - /* selected input fields */ struct ena_admin_proto_input selected_fields[ENA_ADMIN_RSS_PROTO_NUM]; struct ena_admin_proto_input reserved2[ENA_ADMIN_RSS_PROTO_NUM]; @@ -873,11 +797,9 @@ struct ena_admin_proto_input reserved3[ENA_ADMIN_RSS_PROTO_NUM]; }; -/* ENA RSS flow hash input */ struct ena_admin_feature_rss_flow_hash_input { - /* word 0 : */ /* supported hash input sorting - * 1 : L3_sort - support swap L3 addresses if DA + * 1 : L3_sort - support swap L3 addresses if DA is * smaller than SA * 2 : L4_sort - support swap L4 ports if DP smaller * SP @@ -893,292 +815,314 @@ uint16_t enabled_input_sort; }; -/* Operating system type */ enum ena_admin_os_type { - /* Linux OS */ - ENA_ADMIN_OS_LINUX = 1, - - /* Windows OS */ - ENA_ADMIN_OS_WIN = 2, - - /* DPDK OS */ - ENA_ADMIN_OS_DPDK = 3, - - /* FreeBSD OS */ - ENA_ADMIN_OS_FREEBSD = 4, - - /* PXE OS */ - ENA_ADMIN_OS_IPXE = 5, + ENA_ADMIN_OS_LINUX = 1, + ENA_ADMIN_OS_WIN = 2, + ENA_ADMIN_OS_DPDK = 3, + ENA_ADMIN_OS_FREEBSD = 4, + ENA_ADMIN_OS_IPXE = 5, + ENA_ADMIN_OS_ESXI = 6, + ENA_ADMIN_OS_GROUPS_NUM = 6, }; -/* host info */ struct ena_admin_host_info { - /* word 0 : OS type defined in enum ena_os_type */ + /* defined in enum ena_admin_os_type */ uint32_t os_type; /* os distribution string format */ uint8_t os_dist_str[128]; - /* word 33 : OS distribution numeric format */ + /* OS distribution numeric format */ uint32_t os_dist; /* kernel version string format */ uint8_t kernel_ver_str[32]; - /* word 42 : Kernel version numeric format */ + /* Kernel version numeric format */ uint32_t kernel_ver; - /* word 43 : */ - /* driver version - * 7:0 : major - major - * 15:8 : minor - minor - * 23:16 : sub_minor - sub minor + /* 7:0 : major + * 15:8 : minor + * 23:16 : sub_minor + * 31:24 : module_type */ uint32_t driver_version; /* features bitmap */ - uint32_t supported_network_features[4]; + uint32_t supported_network_features[2]; + + /* ENA spec version of driver */ + uint16_t ena_spec_version; + + /* ENA device's Bus, Device and Function + * 2:0 : function + * 7:3 : device + * 15:8 : bus + */ + uint16_t bdf; + + /* Number of CPUs */ + uint16_t num_cpus; + + uint16_t reserved; + + /* 0 : reserved + * 1 : rx_offset + * 2 : interrupt_moderation + * 3 : rx_buf_mirroring + * 4 : rss_configurable_function_key + * 31:5 : reserved + */ + uint32_t driver_supported_features; }; -/* ENA RSS indirection table entry */ struct ena_admin_rss_ind_table_entry { - /* word 0 : */ - /* cq identifier */ uint16_t cq_idx; uint16_t reserved; }; -/* ENA RSS indirection table */ struct ena_admin_feature_rss_ind_table { - /* word 0 : */ /* min supported table size (2^min_size) */ uint16_t min_size; /* max supported table size (2^max_size) */ uint16_t max_size; - /* word 1 : */ /* table size (2^size) */ uint16_t size; - uint16_t reserved; + /* 0 : one_entry_update - The ENA device supports + * setting a single RSS table entry + */ + uint8_t flags; + + uint8_t reserved; - /* word 2 : index of the inline entry. 0xFFFFFFFF means invalid */ + /* index of the inline entry. 0xFFFFFFFF means invalid */ uint32_t inline_index; - /* words 3 : used for updating single entry, ignored when setting - * the entire table through the control buffer. + /* used for updating single entry, ignored when setting the entire + * table through the control buffer. */ struct ena_admin_rss_ind_table_entry inline_entry; }; -/* ENA Get Feature command */ +/* When hint value is 0, driver should use it's own predefined value */ +struct ena_admin_ena_hw_hints { + /* value in ms */ + uint16_t mmio_read_timeout; + + /* value in ms */ + uint16_t driver_watchdog_timeout; + + /* Per packet tx completion timeout. value in ms */ + uint16_t missing_tx_completion_timeout; + + uint16_t missed_tx_completion_count_threshold_to_reset; + + /* value in ms */ + uint16_t admin_completion_tx_timeout; + + uint16_t netdev_wd_timeout; + + uint16_t max_tx_sgl_size; + + uint16_t max_rx_sgl_size; + + uint16_t reserved[8]; +}; + struct ena_admin_get_feat_cmd { - /* words 0 : */ struct ena_admin_aq_common_desc aq_common_descriptor; - /* words 1:3 : points to control buffer (direct or indirect, - * chained if needed) - */ struct ena_admin_ctrl_buff_info control_buffer; - /* words 4 : */ struct ena_admin_get_set_feature_common_desc feat_common; - /* words 5:15 : */ + uint32_t raw[11]; +}; + +struct ena_admin_queue_ext_feature_desc { + /* version */ + uint8_t version; + + uint8_t reserved1[3]; + union { - /* raw words */ - uint32_t raw[11]; - } u; + struct ena_admin_queue_ext_feature_fields max_queue_ext; + + uint32_t raw[10]; + }; }; -/* ENA Get Feature command response */ struct ena_admin_get_feat_resp { - /* words 0:1 : */ struct ena_admin_acq_common_desc acq_common_desc; - /* words 2:15 : */ union { - /* raw words */ uint32_t raw[14]; - /* words 2:10 : Get Device Attributes */ struct ena_admin_device_attr_feature_desc dev_attr; - /* words 2:5 : Max queues num */ + struct ena_admin_feature_llq_desc llq; + struct ena_admin_queue_feature_desc max_queue; - /* words 2:3 : AENQ configuration */ + struct ena_admin_queue_ext_feature_desc max_queue_ext; + struct ena_admin_feature_aenq_desc aenq; - /* words 2:4 : Get Link configuration */ struct ena_admin_get_feature_link_desc link; - /* words 2:4 : offload configuration */ struct ena_admin_feature_offload_desc offload; - /* words 2:4 : rss flow hash function */ struct ena_admin_feature_rss_flow_hash_function flow_hash_func; - /* words 2 : rss flow hash input */ struct ena_admin_feature_rss_flow_hash_input flow_hash_input; - /* words 2:3 : rss indirection table */ struct ena_admin_feature_rss_ind_table ind_table; - /* words 2 : interrupt moderation configuration */ struct ena_admin_feature_intr_moder_desc intr_moderation; + + struct ena_admin_ena_hw_hints hw_hints; + + struct ena_admin_get_extra_properties_strings_desc extra_properties_strings; + + struct ena_admin_get_extra_properties_flags_desc extra_properties_flags; } u; }; -/* ENA Set Feature command */ struct ena_admin_set_feat_cmd { - /* words 0 : */ struct ena_admin_aq_common_desc aq_common_descriptor; - /* words 1:3 : points to control buffer (direct or indirect, - * chained if needed) - */ struct ena_admin_ctrl_buff_info control_buffer; - /* words 4 : */ struct ena_admin_get_set_feature_common_desc feat_common; - /* words 5:15 : */ union { - /* raw words */ uint32_t raw[11]; - /* words 5 : mtu size */ + /* mtu size */ struct ena_admin_set_feature_mtu_desc mtu; - /* words 5:7 : host attributes */ + /* host attributes */ struct ena_admin_set_feature_host_attr_desc host_attr; - /* words 5:6 : AENQ configuration */ + /* AENQ configuration */ struct ena_admin_feature_aenq_desc aenq; - /* words 5:7 : rss flow hash function */ + /* rss flow hash function */ struct ena_admin_feature_rss_flow_hash_function flow_hash_func; - /* words 5 : rss flow hash input */ + /* rss flow hash input */ struct ena_admin_feature_rss_flow_hash_input flow_hash_input; - /* words 5:6 : rss indirection table */ + /* rss indirection table */ struct ena_admin_feature_rss_ind_table ind_table; + + /* LLQ configuration */ + struct ena_admin_feature_llq_desc llq; } u; }; -/* ENA Set Feature command response */ struct ena_admin_set_feat_resp { - /* words 0:1 : */ struct ena_admin_acq_common_desc acq_common_desc; - /* words 2:15 : */ union { - /* raw words */ uint32_t raw[14]; } u; }; -/* ENA Asynchronous Event Notification Queue descriptor. */ struct ena_admin_aenq_common_desc { - /* word 0 : */ uint16_t group; - uint16_t syndrom; + uint16_t syndrome; - /* word 1 : */ - /* 0 : phase */ + /* 0 : phase + * 7:1 : reserved - MBZ + */ uint8_t flags; uint8_t reserved1[3]; - /* word 2 : Timestamp LSB */ uint32_t timestamp_low; - /* word 3 : Timestamp MSB */ uint32_t timestamp_high; }; /* asynchronous event notification groups */ enum ena_admin_aenq_group { - /* Link State Change */ - ENA_ADMIN_LINK_CHANGE = 0, - - ENA_ADMIN_FATAL_ERROR = 1, - - ENA_ADMIN_WARNING = 2, - - ENA_ADMIN_NOTIFICATION = 3, - - ENA_ADMIN_KEEP_ALIVE = 4, - - ENA_ADMIN_AENQ_GROUPS_NUM = 5, + ENA_ADMIN_LINK_CHANGE = 0, + ENA_ADMIN_FATAL_ERROR = 1, + ENA_ADMIN_WARNING = 2, + ENA_ADMIN_NOTIFICATION = 3, + ENA_ADMIN_KEEP_ALIVE = 4, + ENA_ADMIN_AENQ_GROUPS_NUM = 5, }; -/* syndorm of AENQ notification group */ -enum ena_admin_aenq_notification_syndrom { - ENA_ADMIN_SUSPEND = 0, - - ENA_ADMIN_RESUME = 1, +enum ena_admin_aenq_notification_syndrome { + ENA_ADMIN_SUSPEND = 0, + ENA_ADMIN_RESUME = 1, + ENA_ADMIN_UPDATE_HINTS = 2, }; -/* ENA Asynchronous Event Notification generic descriptor. */ struct ena_admin_aenq_entry { - /* words 0:3 : */ struct ena_admin_aenq_common_desc aenq_common_desc; /* command specific inline data */ uint32_t inline_data_w4[12]; }; -/* ENA Asynchronous Event Notification Queue Link Change descriptor. */ struct ena_admin_aenq_link_change_desc { - /* words 0:3 : */ struct ena_admin_aenq_common_desc aenq_common_desc; - /* word 4 : */ /* 0 : link_status */ uint32_t flags; }; -/* ENA MMIO Readless response interface */ +struct ena_admin_aenq_keep_alive_desc { + struct ena_admin_aenq_common_desc aenq_common_desc; + + uint32_t rx_drops_low; + + uint32_t rx_drops_high; + + uint32_t tx_drops_low; + + uint32_t tx_drops_high; +}; + struct ena_admin_ena_mmio_req_read_less_resp { - /* word 0 : */ - /* request id */ uint16_t req_id; - /* register offset */ uint16_t reg_off; - /* word 1 : value is valid when poll is cleared */ + /* value is valid when poll is cleared */ uint32_t reg_val; }; /* aq_common_desc */ -#define ENA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK GENMASK(11, 0) -#define ENA_ADMIN_AQ_COMMON_DESC_PHASE_MASK BIT(0) -#define ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_SHIFT 1 -#define ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_MASK BIT(1) -#define ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_SHIFT 2 -#define ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK BIT(2) +#define ENA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK GENMASK(11, 0) +#define ENA_ADMIN_AQ_COMMON_DESC_PHASE_MASK BIT(0) +#define ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_SHIFT 1 +#define ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_MASK BIT(1) +#define ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_SHIFT 2 +#define ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK BIT(2) /* sq */ -#define ENA_ADMIN_SQ_SQ_DIRECTION_SHIFT 5 -#define ENA_ADMIN_SQ_SQ_DIRECTION_MASK GENMASK(7, 5) +#define ENA_ADMIN_SQ_SQ_DIRECTION_SHIFT 5 +#define ENA_ADMIN_SQ_SQ_DIRECTION_MASK GENMASK(7, 5) /* acq_common_desc */ -#define ENA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK GENMASK(11, 0) -#define ENA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK BIT(0) +#define ENA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK GENMASK(11, 0) +#define ENA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK BIT(0) /* aq_create_sq_cmd */ -#define ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_SHIFT 5 -#define ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_MASK GENMASK(7, 5) -#define ENA_ADMIN_AQ_CREATE_SQ_CMD_PLACEMENT_POLICY_MASK GENMASK(3, 0) -#define ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_SHIFT 4 -#define ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_MASK GENMASK(6, 4) +#define ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_SHIFT 5 +#define ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_MASK GENMASK(7, 5) +#define ENA_ADMIN_AQ_CREATE_SQ_CMD_PLACEMENT_POLICY_MASK GENMASK(3, 0) +#define ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_SHIFT 4 +#define ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_MASK GENMASK(6, 4) #define ENA_ADMIN_AQ_CREATE_SQ_CMD_IS_PHYSICALLY_CONTIGUOUS_MASK BIT(0) /* aq_create_cq_cmd */ @@ -1187,12 +1131,12 @@ #define ENA_ADMIN_AQ_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK GENMASK(4, 0) /* get_set_feature_common_desc */ -#define ENA_ADMIN_GET_SET_FEATURE_COMMON_DESC_SELECT_MASK GENMASK(1, 0) +#define ENA_ADMIN_GET_SET_FEATURE_COMMON_DESC_SELECT_MASK GENMASK(1, 0) /* get_feature_link_desc */ -#define ENA_ADMIN_GET_FEATURE_LINK_DESC_AUTONEG_MASK BIT(0) -#define ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_SHIFT 1 -#define ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_MASK BIT(1) +#define ENA_ADMIN_GET_FEATURE_LINK_DESC_AUTONEG_MASK BIT(0) +#define ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_SHIFT 1 +#define ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_MASK BIT(1) /* feature_offload_desc */ #define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK BIT(0) @@ -1204,699 +1148,545 @@ #define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK BIT(3) #define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_SHIFT 4 #define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK BIT(4) -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_SHIFT 5 -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK BIT(5) -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_SHIFT 6 -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK BIT(6) -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_SHIFT 7 -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_MASK BIT(7) +#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_SHIFT 5 +#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK BIT(5) +#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_SHIFT 6 +#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK BIT(6) +#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_SHIFT 7 +#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_MASK BIT(7) #define ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK BIT(0) #define ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_SHIFT 1 #define ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK BIT(1) #define ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_SHIFT 2 #define ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK BIT(2) -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_SHIFT 3 -#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_MASK BIT(3) +#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_SHIFT 3 +#define ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_MASK BIT(3) /* feature_rss_flow_hash_function */ #define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_FUNCS_MASK GENMASK(7, 0) -#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_SELECTED_FUNC_MASK \ - GENMASK(7, 0) +#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_SELECTED_FUNC_MASK GENMASK(7, 0) /* feature_rss_flow_hash_input */ #define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_SHIFT 1 -#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_MASK BIT(1) +#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_MASK BIT(1) #define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_SHIFT 2 -#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_MASK BIT(2) +#define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_MASK BIT(2) #define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_SHIFT 1 #define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_MASK BIT(1) #define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_SHIFT 2 #define ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_MASK BIT(2) /* host_info */ -#define ENA_ADMIN_HOST_INFO_MAJOR_MASK GENMASK(7, 0) -#define ENA_ADMIN_HOST_INFO_MINOR_SHIFT 8 -#define ENA_ADMIN_HOST_INFO_MINOR_MASK GENMASK(15, 8) -#define ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT 16 -#define ENA_ADMIN_HOST_INFO_SUB_MINOR_MASK GENMASK(23, 16) +#define ENA_ADMIN_HOST_INFO_MAJOR_MASK GENMASK(7, 0) +#define ENA_ADMIN_HOST_INFO_MINOR_SHIFT 8 +#define ENA_ADMIN_HOST_INFO_MINOR_MASK GENMASK(15, 8) +#define ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT 16 +#define ENA_ADMIN_HOST_INFO_SUB_MINOR_MASK GENMASK(23, 16) +#define ENA_ADMIN_HOST_INFO_MODULE_TYPE_SHIFT 24 +#define ENA_ADMIN_HOST_INFO_MODULE_TYPE_MASK GENMASK(31, 24) +#define ENA_ADMIN_HOST_INFO_FUNCTION_MASK GENMASK(2, 0) +#define ENA_ADMIN_HOST_INFO_DEVICE_SHIFT 3 +#define ENA_ADMIN_HOST_INFO_DEVICE_MASK GENMASK(7, 3) +#define ENA_ADMIN_HOST_INFO_BUS_SHIFT 8 +#define ENA_ADMIN_HOST_INFO_BUS_MASK GENMASK(15, 8) +#define ENA_ADMIN_HOST_INFO_RX_OFFSET_SHIFT 1 +#define ENA_ADMIN_HOST_INFO_RX_OFFSET_MASK BIT(1) +#define ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_SHIFT 2 +#define ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_MASK BIT(2) +#define ENA_ADMIN_HOST_INFO_RX_BUF_MIRRORING_SHIFT 3 +#define ENA_ADMIN_HOST_INFO_RX_BUF_MIRRORING_MASK BIT(3) +#define ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_SHIFT 4 +#define ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_MASK BIT(4) + +/* feature_rss_ind_table */ +#define ENA_ADMIN_FEATURE_RSS_IND_TABLE_ONE_ENTRY_UPDATE_MASK BIT(0) /* aenq_common_desc */ -#define ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK BIT(0) +#define ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK BIT(0) /* aenq_link_change_desc */ -#define ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK BIT(0) +#define ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK BIT(0) -#if !defined(ENA_DEFS_LINUX_MAINLINE) -static inline uint16_t -get_ena_admin_aq_common_desc_command_id( - const struct ena_admin_aq_common_desc *p) +#if !defined(DEFS_LINUX_MAINLINE) +static inline uint16_t get_ena_admin_aq_common_desc_command_id(const struct ena_admin_aq_common_desc *p) { return p->command_id & ENA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK; } -static inline void -set_ena_admin_aq_common_desc_command_id(struct ena_admin_aq_common_desc *p, - uint16_t val) +static inline void set_ena_admin_aq_common_desc_command_id(struct ena_admin_aq_common_desc *p, uint16_t val) { p->command_id |= val & ENA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK; } -static inline uint8_t -get_ena_admin_aq_common_desc_phase(const struct ena_admin_aq_common_desc *p) +static inline uint8_t get_ena_admin_aq_common_desc_phase(const struct ena_admin_aq_common_desc *p) { return p->flags & ENA_ADMIN_AQ_COMMON_DESC_PHASE_MASK; } -static inline void -set_ena_admin_aq_common_desc_phase(struct ena_admin_aq_common_desc *p, - uint8_t val) +static inline void set_ena_admin_aq_common_desc_phase(struct ena_admin_aq_common_desc *p, uint8_t val) { p->flags |= val & ENA_ADMIN_AQ_COMMON_DESC_PHASE_MASK; } -static inline uint8_t -get_ena_admin_aq_common_desc_ctrl_data( - const struct ena_admin_aq_common_desc *p) +static inline uint8_t get_ena_admin_aq_common_desc_ctrl_data(const struct ena_admin_aq_common_desc *p) { - return (p->flags & ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_MASK) >> - ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_SHIFT; + return (p->flags & ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_MASK) >> ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_SHIFT; } -static inline void -set_ena_admin_aq_common_desc_ctrl_data(struct ena_admin_aq_common_desc *p, - uint8_t val) +static inline void set_ena_admin_aq_common_desc_ctrl_data(struct ena_admin_aq_common_desc *p, uint8_t val) { - p->flags |= (val << ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_SHIFT) - & ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_MASK; + p->flags |= (val << ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_SHIFT) & ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_MASK; } -static inline uint8_t -get_ena_admin_aq_common_desc_ctrl_data_indirect( - const struct ena_admin_aq_common_desc *p) +static inline uint8_t get_ena_admin_aq_common_desc_ctrl_data_indirect(const struct ena_admin_aq_common_desc *p) { - return (p->flags & ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK) - >> ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_SHIFT; + return (p->flags & ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK) >> ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_SHIFT; } -static inline void -set_ena_admin_aq_common_desc_ctrl_data_indirect( - struct ena_admin_aq_common_desc *p, - uint8_t val) +static inline void set_ena_admin_aq_common_desc_ctrl_data_indirect(struct ena_admin_aq_common_desc *p, uint8_t val) { - p->flags |= (val << ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_SHIFT) - & ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK; + p->flags |= (val << ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_SHIFT) & ENA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK; } -static inline uint8_t -get_ena_admin_sq_sq_direction(const struct ena_admin_sq *p) +static inline uint8_t get_ena_admin_sq_sq_direction(const struct ena_admin_sq *p) { - return (p->sq_identity & ENA_ADMIN_SQ_SQ_DIRECTION_MASK) - >> ENA_ADMIN_SQ_SQ_DIRECTION_SHIFT; + return (p->sq_identity & ENA_ADMIN_SQ_SQ_DIRECTION_MASK) >> ENA_ADMIN_SQ_SQ_DIRECTION_SHIFT; } -static inline void -set_ena_admin_sq_sq_direction(struct ena_admin_sq *p, uint8_t val) +static inline void set_ena_admin_sq_sq_direction(struct ena_admin_sq *p, uint8_t val) { - p->sq_identity |= (val << ENA_ADMIN_SQ_SQ_DIRECTION_SHIFT) & - ENA_ADMIN_SQ_SQ_DIRECTION_MASK; + p->sq_identity |= (val << ENA_ADMIN_SQ_SQ_DIRECTION_SHIFT) & ENA_ADMIN_SQ_SQ_DIRECTION_MASK; } -static inline uint16_t -get_ena_admin_acq_common_desc_command_id( - const struct ena_admin_acq_common_desc *p) +static inline uint16_t get_ena_admin_acq_common_desc_command_id(const struct ena_admin_acq_common_desc *p) { return p->command & ENA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK; } -static inline void -set_ena_admin_acq_common_desc_command_id(struct ena_admin_acq_common_desc *p, - uint16_t val) +static inline void set_ena_admin_acq_common_desc_command_id(struct ena_admin_acq_common_desc *p, uint16_t val) { p->command |= val & ENA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK; } -static inline uint8_t -get_ena_admin_acq_common_desc_phase(const struct ena_admin_acq_common_desc *p) +static inline uint8_t get_ena_admin_acq_common_desc_phase(const struct ena_admin_acq_common_desc *p) { return p->flags & ENA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK; } -static inline void -set_ena_admin_acq_common_desc_phase(struct ena_admin_acq_common_desc *p, - uint8_t val) +static inline void set_ena_admin_acq_common_desc_phase(struct ena_admin_acq_common_desc *p, uint8_t val) { p->flags |= val & ENA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK; } -static inline uint8_t -get_ena_admin_aq_create_sq_cmd_sq_direction( - const struct ena_admin_aq_create_sq_cmd *p) +static inline uint8_t get_ena_admin_aq_create_sq_cmd_sq_direction(const struct ena_admin_aq_create_sq_cmd *p) { - return (p->sq_identity & ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_MASK) - >> ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_SHIFT; + return (p->sq_identity & ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_MASK) >> ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_SHIFT; } -static inline void -set_ena_admin_aq_create_sq_cmd_sq_direction( - struct ena_admin_aq_create_sq_cmd *p, - uint8_t val) +static inline void set_ena_admin_aq_create_sq_cmd_sq_direction(struct ena_admin_aq_create_sq_cmd *p, uint8_t val) { - p->sq_identity |= (val << - ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_SHIFT) - & ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_MASK; + p->sq_identity |= (val << ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_SHIFT) & ENA_ADMIN_AQ_CREATE_SQ_CMD_SQ_DIRECTION_MASK; } -static inline uint8_t -get_ena_admin_aq_create_sq_cmd_placement_policy( - const struct ena_admin_aq_create_sq_cmd *p) +static inline uint8_t get_ena_admin_aq_create_sq_cmd_placement_policy(const struct ena_admin_aq_create_sq_cmd *p) { return p->sq_caps_2 & ENA_ADMIN_AQ_CREATE_SQ_CMD_PLACEMENT_POLICY_MASK; } -static inline void -set_ena_admin_aq_create_sq_cmd_placement_policy( - struct ena_admin_aq_create_sq_cmd *p, - uint8_t val) +static inline void set_ena_admin_aq_create_sq_cmd_placement_policy(struct ena_admin_aq_create_sq_cmd *p, uint8_t val) { p->sq_caps_2 |= val & ENA_ADMIN_AQ_CREATE_SQ_CMD_PLACEMENT_POLICY_MASK; } -static inline uint8_t -get_ena_admin_aq_create_sq_cmd_completion_policy( - const struct ena_admin_aq_create_sq_cmd *p) +static inline uint8_t get_ena_admin_aq_create_sq_cmd_completion_policy(const struct ena_admin_aq_create_sq_cmd *p) { - return (p->sq_caps_2 - & ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_MASK) - >> ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_SHIFT; + return (p->sq_caps_2 & ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_MASK) >> ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_SHIFT; } -static inline void -set_ena_admin_aq_create_sq_cmd_completion_policy( - struct ena_admin_aq_create_sq_cmd *p, - uint8_t val) +static inline void set_ena_admin_aq_create_sq_cmd_completion_policy(struct ena_admin_aq_create_sq_cmd *p, uint8_t val) { - p->sq_caps_2 |= - (val << ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_SHIFT) - & ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_MASK; + p->sq_caps_2 |= (val << ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_SHIFT) & ENA_ADMIN_AQ_CREATE_SQ_CMD_COMPLETION_POLICY_MASK; } -static inline uint8_t -get_ena_admin_aq_create_sq_cmd_is_physically_contiguous( - const struct ena_admin_aq_create_sq_cmd *p) +static inline uint8_t get_ena_admin_aq_create_sq_cmd_is_physically_contiguous(const struct ena_admin_aq_create_sq_cmd *p) { - return p->sq_caps_3 & - ENA_ADMIN_AQ_CREATE_SQ_CMD_IS_PHYSICALLY_CONTIGUOUS_MASK; + return p->sq_caps_3 & ENA_ADMIN_AQ_CREATE_SQ_CMD_IS_PHYSICALLY_CONTIGUOUS_MASK; } -static inline void -set_ena_admin_aq_create_sq_cmd_is_physically_contiguous( - struct ena_admin_aq_create_sq_cmd *p, - uint8_t val) +static inline void set_ena_admin_aq_create_sq_cmd_is_physically_contiguous(struct ena_admin_aq_create_sq_cmd *p, uint8_t val) { - p->sq_caps_3 |= val & - ENA_ADMIN_AQ_CREATE_SQ_CMD_IS_PHYSICALLY_CONTIGUOUS_MASK; + p->sq_caps_3 |= val & ENA_ADMIN_AQ_CREATE_SQ_CMD_IS_PHYSICALLY_CONTIGUOUS_MASK; } -static inline uint8_t -get_ena_admin_aq_create_cq_cmd_interrupt_mode_enabled( - const struct ena_admin_aq_create_cq_cmd *p) +static inline uint8_t get_ena_admin_aq_create_cq_cmd_interrupt_mode_enabled(const struct ena_admin_aq_create_cq_cmd *p) { - return (p->cq_caps_1 & - ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK) - >> ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_SHIFT; + return (p->cq_caps_1 & ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK) >> ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_SHIFT; } -static inline void -set_ena_admin_aq_create_cq_cmd_interrupt_mode_enabled( - struct ena_admin_aq_create_cq_cmd *p, - uint8_t val) +static inline void set_ena_admin_aq_create_cq_cmd_interrupt_mode_enabled(struct ena_admin_aq_create_cq_cmd *p, uint8_t val) { - p->cq_caps_1 |= - (val << ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_SHIFT) - & ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK; + p->cq_caps_1 |= (val << ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_SHIFT) & ENA_ADMIN_AQ_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK; } -static inline uint8_t -get_ena_admin_aq_create_cq_cmd_cq_entry_size_words( - const struct ena_admin_aq_create_cq_cmd *p) +static inline uint8_t get_ena_admin_aq_create_cq_cmd_cq_entry_size_words(const struct ena_admin_aq_create_cq_cmd *p) { - return p->cq_caps_2 - & ENA_ADMIN_AQ_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK; + return p->cq_caps_2 & ENA_ADMIN_AQ_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK; } -static inline void -set_ena_admin_aq_create_cq_cmd_cq_entry_size_words( - struct ena_admin_aq_create_cq_cmd *p, - uint8_t val) +static inline void set_ena_admin_aq_create_cq_cmd_cq_entry_size_words(struct ena_admin_aq_create_cq_cmd *p, uint8_t val) { - p->cq_caps_2 |= - val & ENA_ADMIN_AQ_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK; + p->cq_caps_2 |= val & ENA_ADMIN_AQ_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK; } -static inline uint8_t -get_ena_admin_get_set_feature_common_desc_select( - const struct ena_admin_get_set_feature_common_desc *p) +static inline uint8_t get_ena_admin_get_set_feature_common_desc_select(const struct ena_admin_get_set_feature_common_desc *p) { return p->flags & ENA_ADMIN_GET_SET_FEATURE_COMMON_DESC_SELECT_MASK; } -static inline void -set_ena_admin_get_set_feature_common_desc_select( - struct ena_admin_get_set_feature_common_desc *p, - uint8_t val) +static inline void set_ena_admin_get_set_feature_common_desc_select(struct ena_admin_get_set_feature_common_desc *p, uint8_t val) { p->flags |= val & ENA_ADMIN_GET_SET_FEATURE_COMMON_DESC_SELECT_MASK; } -static inline uint32_t -get_ena_admin_get_feature_link_desc_autoneg( - const struct ena_admin_get_feature_link_desc *p) +static inline uint32_t get_ena_admin_get_feature_link_desc_autoneg(const struct ena_admin_get_feature_link_desc *p) { return p->flags & ENA_ADMIN_GET_FEATURE_LINK_DESC_AUTONEG_MASK; } -static inline void -set_ena_admin_get_feature_link_desc_autoneg( - struct ena_admin_get_feature_link_desc *p, - uint32_t val) +static inline void set_ena_admin_get_feature_link_desc_autoneg(struct ena_admin_get_feature_link_desc *p, uint32_t val) { p->flags |= val & ENA_ADMIN_GET_FEATURE_LINK_DESC_AUTONEG_MASK; } -static inline uint32_t -get_ena_admin_get_feature_link_desc_duplex( - const struct ena_admin_get_feature_link_desc *p) +static inline uint32_t get_ena_admin_get_feature_link_desc_duplex(const struct ena_admin_get_feature_link_desc *p) { - return (p->flags & ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_MASK) - >> ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_SHIFT; + return (p->flags & ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_MASK) >> ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_SHIFT; } -static inline void -set_ena_admin_get_feature_link_desc_duplex( - struct ena_admin_get_feature_link_desc *p, - uint32_t val) +static inline void set_ena_admin_get_feature_link_desc_duplex(struct ena_admin_get_feature_link_desc *p, uint32_t val) { - p->flags |= (val << ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_SHIFT) - & ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_MASK; + p->flags |= (val << ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_SHIFT) & ENA_ADMIN_GET_FEATURE_LINK_DESC_DUPLEX_MASK; } -static inline uint32_t -get_ena_admin_feature_offload_desc_TX_L3_csum_ipv4( - const struct ena_admin_feature_offload_desc *p) +static inline uint32_t get_ena_admin_feature_offload_desc_TX_L3_csum_ipv4(const struct ena_admin_feature_offload_desc *p) { return p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK; } -static inline void -set_ena_admin_feature_offload_desc_TX_L3_csum_ipv4( - struct ena_admin_feature_offload_desc *p, - uint32_t val) +static inline void set_ena_admin_feature_offload_desc_TX_L3_csum_ipv4(struct ena_admin_feature_offload_desc *p, uint32_t val) { p->tx |= val & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK; } -static inline uint32_t -get_ena_admin_feature_offload_desc_TX_L4_ipv4_csum_part( - const struct ena_admin_feature_offload_desc *p) +static inline uint32_t get_ena_admin_feature_offload_desc_TX_L4_ipv4_csum_part(const struct ena_admin_feature_offload_desc *p) { - return (p->tx & - ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK) - >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_SHIFT; + return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_SHIFT; } -static inline void -set_ena_admin_feature_offload_desc_TX_L4_ipv4_csum_part( - struct ena_admin_feature_offload_desc *p, - uint32_t val) +static inline void set_ena_admin_feature_offload_desc_TX_L4_ipv4_csum_part(struct ena_admin_feature_offload_desc *p, uint32_t val) { - p->tx |= (val << - ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_SHIFT) - & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK; + p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK; } -static inline uint32_t -get_ena_admin_feature_offload_desc_TX_L4_ipv4_csum_full( - const struct ena_admin_feature_offload_desc *p) +static inline uint32_t get_ena_admin_feature_offload_desc_TX_L4_ipv4_csum_full(const struct ena_admin_feature_offload_desc *p) { - return (p->tx & - ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK) - >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_SHIFT; + return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_SHIFT; } -static inline void -set_ena_admin_feature_offload_desc_TX_L4_ipv4_csum_full( - struct ena_admin_feature_offload_desc *p, - uint32_t val) +static inline void set_ena_admin_feature_offload_desc_TX_L4_ipv4_csum_full(struct ena_admin_feature_offload_desc *p, uint32_t val) { - p->tx |= (val << - ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_SHIFT) - & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK; + p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK; } -static inline uint32_t -get_ena_admin_feature_offload_desc_TX_L4_ipv6_csum_part( - const struct ena_admin_feature_offload_desc *p) +static inline uint32_t get_ena_admin_feature_offload_desc_TX_L4_ipv6_csum_part(const struct ena_admin_feature_offload_desc *p) { - return (p->tx & - ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK) - >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_SHIFT; + return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_SHIFT; } -static inline void -set_ena_admin_feature_offload_desc_TX_L4_ipv6_csum_part( - struct ena_admin_feature_offload_desc *p, - uint32_t val) +static inline void set_ena_admin_feature_offload_desc_TX_L4_ipv6_csum_part(struct ena_admin_feature_offload_desc *p, uint32_t val) { - p->tx |= (val << - ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_SHIFT) - & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK; + p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK; } -static inline uint32_t -get_ena_admin_feature_offload_desc_TX_L4_ipv6_csum_full( - const struct ena_admin_feature_offload_desc *p) +static inline uint32_t get_ena_admin_feature_offload_desc_TX_L4_ipv6_csum_full(const struct ena_admin_feature_offload_desc *p) { - return (p->tx & - ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK) - >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_SHIFT; + return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_SHIFT; } -static inline void -set_ena_admin_feature_offload_desc_TX_L4_ipv6_csum_full( - struct ena_admin_feature_offload_desc *p, - uint32_t val) +static inline void set_ena_admin_feature_offload_desc_TX_L4_ipv6_csum_full(struct ena_admin_feature_offload_desc *p, uint32_t val) { - p->tx |= (val << - ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_SHIFT) - & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK; + p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK; } -static inline uint32_t -get_ena_admin_feature_offload_desc_tso_ipv4( - const struct ena_admin_feature_offload_desc *p) +static inline uint32_t get_ena_admin_feature_offload_desc_tso_ipv4(const struct ena_admin_feature_offload_desc *p) { - return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) - >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_SHIFT; + return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_SHIFT; } -static inline void -set_ena_admin_feature_offload_desc_tso_ipv4( - struct ena_admin_feature_offload_desc *p, - uint32_t val) +static inline void set_ena_admin_feature_offload_desc_tso_ipv4(struct ena_admin_feature_offload_desc *p, uint32_t val) { - p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_SHIFT) - & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK; + p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK; } -static inline uint32_t -get_ena_admin_feature_offload_desc_tso_ipv6( - const struct ena_admin_feature_offload_desc *p) +static inline uint32_t get_ena_admin_feature_offload_desc_tso_ipv6(const struct ena_admin_feature_offload_desc *p) { - return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK) - >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_SHIFT; + return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_SHIFT; } -static inline void -set_ena_admin_feature_offload_desc_tso_ipv6( - struct ena_admin_feature_offload_desc *p, - uint32_t val) +static inline void set_ena_admin_feature_offload_desc_tso_ipv6(struct ena_admin_feature_offload_desc *p, uint32_t val) { - p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_SHIFT) - & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK; + p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK; } -static inline uint32_t -get_ena_admin_feature_offload_desc_tso_ecn( - const struct ena_admin_feature_offload_desc *p) +static inline uint32_t get_ena_admin_feature_offload_desc_tso_ecn(const struct ena_admin_feature_offload_desc *p) { - return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_MASK) - >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_SHIFT; + return (p->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_SHIFT; } -static inline void -set_ena_admin_feature_offload_desc_tso_ecn( - struct ena_admin_feature_offload_desc *p, - uint32_t val) +static inline void set_ena_admin_feature_offload_desc_tso_ecn(struct ena_admin_feature_offload_desc *p, uint32_t val) { - p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_SHIFT) - & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_MASK; + p->tx |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_MASK; } -static inline uint32_t -get_ena_admin_feature_offload_desc_RX_L3_csum_ipv4( - const struct ena_admin_feature_offload_desc *p) +static inline uint32_t get_ena_admin_feature_offload_desc_RX_L3_csum_ipv4(const struct ena_admin_feature_offload_desc *p) { - return p->rx_supported & - ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK; + return p->rx_supported & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK; } -static inline void -set_ena_admin_feature_offload_desc_RX_L3_csum_ipv4( - struct ena_admin_feature_offload_desc *p, - uint32_t val) +static inline void set_ena_admin_feature_offload_desc_RX_L3_csum_ipv4(struct ena_admin_feature_offload_desc *p, uint32_t val) { - p->rx_supported |= - val & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK; + p->rx_supported |= val & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK; } -static inline uint32_t -get_ena_admin_feature_offload_desc_RX_L4_ipv4_csum( - const struct ena_admin_feature_offload_desc *p) +static inline uint32_t get_ena_admin_feature_offload_desc_RX_L4_ipv4_csum(const struct ena_admin_feature_offload_desc *p) { - return (p->rx_supported & - ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK) - >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_SHIFT; + return (p->rx_supported & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_SHIFT; } -static inline void -set_ena_admin_feature_offload_desc_RX_L4_ipv4_csum( - struct ena_admin_feature_offload_desc *p, - uint32_t val) +static inline void set_ena_admin_feature_offload_desc_RX_L4_ipv4_csum(struct ena_admin_feature_offload_desc *p, uint32_t val) { - p->rx_supported |= - (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_SHIFT) - & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK; + p->rx_supported |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK; } -static inline uint32_t -get_ena_admin_feature_offload_desc_RX_L4_ipv6_csum( - const struct ena_admin_feature_offload_desc *p) +static inline uint32_t get_ena_admin_feature_offload_desc_RX_L4_ipv6_csum(const struct ena_admin_feature_offload_desc *p) { - return (p->rx_supported & - ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK) - >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_SHIFT; + return (p->rx_supported & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_SHIFT; } -static inline void -set_ena_admin_feature_offload_desc_RX_L4_ipv6_csum( - struct ena_admin_feature_offload_desc *p, - uint32_t val) +static inline void set_ena_admin_feature_offload_desc_RX_L4_ipv6_csum(struct ena_admin_feature_offload_desc *p, uint32_t val) { - p->rx_supported |= - (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_SHIFT) - & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK; + p->rx_supported |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK; } -static inline uint32_t -get_ena_admin_feature_offload_desc_RX_hash( - const struct ena_admin_feature_offload_desc *p) +static inline uint32_t get_ena_admin_feature_offload_desc_RX_hash(const struct ena_admin_feature_offload_desc *p) { - return (p->rx_supported & - ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_MASK) - >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_SHIFT; + return (p->rx_supported & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_MASK) >> ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_SHIFT; } -static inline void -set_ena_admin_feature_offload_desc_RX_hash( - struct ena_admin_feature_offload_desc *p, - uint32_t val) +static inline void set_ena_admin_feature_offload_desc_RX_hash(struct ena_admin_feature_offload_desc *p, uint32_t val) { - p->rx_supported |= - (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_SHIFT) - & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_MASK; + p->rx_supported |= (val << ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_SHIFT) & ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_MASK; } -static inline uint32_t -get_ena_admin_feature_rss_flow_hash_function_funcs( - const struct ena_admin_feature_rss_flow_hash_function *p) +static inline uint32_t get_ena_admin_feature_rss_flow_hash_function_funcs(const struct ena_admin_feature_rss_flow_hash_function *p) { - return p->supported_func & - ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_FUNCS_MASK; + return p->supported_func & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_FUNCS_MASK; } -static inline void -set_ena_admin_feature_rss_flow_hash_function_funcs( - struct ena_admin_feature_rss_flow_hash_function *p, - uint32_t val) +static inline void set_ena_admin_feature_rss_flow_hash_function_funcs(struct ena_admin_feature_rss_flow_hash_function *p, uint32_t val) { - p->supported_func |= - val & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_FUNCS_MASK; + p->supported_func |= val & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_FUNCS_MASK; } -static inline uint32_t -get_ena_admin_feature_rss_flow_hash_function_selected_func( - const struct ena_admin_feature_rss_flow_hash_function *p) +static inline uint32_t get_ena_admin_feature_rss_flow_hash_function_selected_func(const struct ena_admin_feature_rss_flow_hash_function *p) { - return p->selected_func & - ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_SELECTED_FUNC_MASK; + return p->selected_func & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_SELECTED_FUNC_MASK; } -static inline void -set_ena_admin_feature_rss_flow_hash_function_selected_func( - struct ena_admin_feature_rss_flow_hash_function *p, - uint32_t val) +static inline void set_ena_admin_feature_rss_flow_hash_function_selected_func(struct ena_admin_feature_rss_flow_hash_function *p, uint32_t val) { - p->selected_func |= - val & - ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_SELECTED_FUNC_MASK; + p->selected_func |= val & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_FUNCTION_SELECTED_FUNC_MASK; } -static inline uint16_t -get_ena_admin_feature_rss_flow_hash_input_L3_sort( - const struct ena_admin_feature_rss_flow_hash_input *p) +static inline uint16_t get_ena_admin_feature_rss_flow_hash_input_L3_sort(const struct ena_admin_feature_rss_flow_hash_input *p) { - return (p->supported_input_sort & - ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_MASK) - >> ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_SHIFT; + return (p->supported_input_sort & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_MASK) >> ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_SHIFT; } -static inline void -set_ena_admin_feature_rss_flow_hash_input_L3_sort( - struct ena_admin_feature_rss_flow_hash_input *p, uint16_t val) +static inline void set_ena_admin_feature_rss_flow_hash_input_L3_sort(struct ena_admin_feature_rss_flow_hash_input *p, uint16_t val) { - p->supported_input_sort |= - (val << ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_SHIFT) - & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_MASK; + p->supported_input_sort |= (val << ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_SHIFT) & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L3_SORT_MASK; } -static inline uint16_t -get_ena_admin_feature_rss_flow_hash_input_L4_sort( - const struct ena_admin_feature_rss_flow_hash_input *p) +static inline uint16_t get_ena_admin_feature_rss_flow_hash_input_L4_sort(const struct ena_admin_feature_rss_flow_hash_input *p) { - return (p->supported_input_sort & - ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_MASK) - >> ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_SHIFT; + return (p->supported_input_sort & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_MASK) >> ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_SHIFT; } -static inline void -set_ena_admin_feature_rss_flow_hash_input_L4_sort( - struct ena_admin_feature_rss_flow_hash_input *p, - uint16_t val) +static inline void set_ena_admin_feature_rss_flow_hash_input_L4_sort(struct ena_admin_feature_rss_flow_hash_input *p, uint16_t val) { - p->supported_input_sort |= - (val << ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_SHIFT) - & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_MASK; + p->supported_input_sort |= (val << ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_SHIFT) & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_L4_SORT_MASK; } -static inline uint16_t -get_ena_admin_feature_rss_flow_hash_input_enable_L3_sort( - const struct ena_admin_feature_rss_flow_hash_input *p) +static inline uint16_t get_ena_admin_feature_rss_flow_hash_input_enable_L3_sort(const struct ena_admin_feature_rss_flow_hash_input *p) { - return (p->enabled_input_sort & - ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_MASK) - >> ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_SHIFT; + return (p->enabled_input_sort & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_MASK) >> ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_SHIFT; } -static inline void -set_ena_admin_feature_rss_flow_hash_input_enable_L3_sort( - struct ena_admin_feature_rss_flow_hash_input *p, - uint16_t val) +static inline void set_ena_admin_feature_rss_flow_hash_input_enable_L3_sort(struct ena_admin_feature_rss_flow_hash_input *p, uint16_t val) { - p->enabled_input_sort |= - (val << - ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_SHIFT) - & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_MASK; + p->enabled_input_sort |= (val << ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_SHIFT) & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L3_SORT_MASK; } -static inline uint16_t -get_ena_admin_feature_rss_flow_hash_input_enable_L4_sort( - const struct ena_admin_feature_rss_flow_hash_input *p) +static inline uint16_t get_ena_admin_feature_rss_flow_hash_input_enable_L4_sort(const struct ena_admin_feature_rss_flow_hash_input *p) { - return (p->enabled_input_sort & - ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_MASK) - >> ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_SHIFT; + return (p->enabled_input_sort & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_MASK) >> ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_SHIFT; } -static inline void -set_ena_admin_feature_rss_flow_hash_input_enable_L4_sort( - struct ena_admin_feature_rss_flow_hash_input *p, - uint16_t val) +static inline void set_ena_admin_feature_rss_flow_hash_input_enable_L4_sort(struct ena_admin_feature_rss_flow_hash_input *p, uint16_t val) { - p->enabled_input_sort |= - (val << - ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_SHIFT) - & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_MASK; + p->enabled_input_sort |= (val << ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_SHIFT) & ENA_ADMIN_FEATURE_RSS_FLOW_HASH_INPUT_ENABLE_L4_SORT_MASK; } -static inline uint32_t -get_ena_admin_host_info_major(const struct ena_admin_host_info *p) +static inline uint32_t get_ena_admin_host_info_major(const struct ena_admin_host_info *p) { return p->driver_version & ENA_ADMIN_HOST_INFO_MAJOR_MASK; } -static inline void -set_ena_admin_host_info_major(struct ena_admin_host_info *p, uint32_t val) +static inline void set_ena_admin_host_info_major(struct ena_admin_host_info *p, uint32_t val) { p->driver_version |= val & ENA_ADMIN_HOST_INFO_MAJOR_MASK; } -static inline uint32_t -get_ena_admin_host_info_minor(const struct ena_admin_host_info *p) +static inline uint32_t get_ena_admin_host_info_minor(const struct ena_admin_host_info *p) +{ + return (p->driver_version & ENA_ADMIN_HOST_INFO_MINOR_MASK) >> ENA_ADMIN_HOST_INFO_MINOR_SHIFT; +} + +static inline void set_ena_admin_host_info_minor(struct ena_admin_host_info *p, uint32_t val) +{ + p->driver_version |= (val << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) & ENA_ADMIN_HOST_INFO_MINOR_MASK; +} + +static inline uint32_t get_ena_admin_host_info_sub_minor(const struct ena_admin_host_info *p) +{ + return (p->driver_version & ENA_ADMIN_HOST_INFO_SUB_MINOR_MASK) >> ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT; +} + +static inline void set_ena_admin_host_info_sub_minor(struct ena_admin_host_info *p, uint32_t val) +{ + p->driver_version |= (val << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT) & ENA_ADMIN_HOST_INFO_SUB_MINOR_MASK; +} + +static inline uint32_t get_ena_admin_host_info_module_type(const struct ena_admin_host_info *p) +{ + return (p->driver_version & ENA_ADMIN_HOST_INFO_MODULE_TYPE_MASK) >> ENA_ADMIN_HOST_INFO_MODULE_TYPE_SHIFT; +} + +static inline void set_ena_admin_host_info_module_type(struct ena_admin_host_info *p, uint32_t val) +{ + p->driver_version |= (val << ENA_ADMIN_HOST_INFO_MODULE_TYPE_SHIFT) & ENA_ADMIN_HOST_INFO_MODULE_TYPE_MASK; +} + +static inline uint16_t get_ena_admin_host_info_function(const struct ena_admin_host_info *p) +{ + return p->bdf & ENA_ADMIN_HOST_INFO_FUNCTION_MASK; +} + +static inline void set_ena_admin_host_info_function(struct ena_admin_host_info *p, uint16_t val) +{ + p->bdf |= val & ENA_ADMIN_HOST_INFO_FUNCTION_MASK; +} + +static inline uint16_t get_ena_admin_host_info_device(const struct ena_admin_host_info *p) +{ + return (p->bdf & ENA_ADMIN_HOST_INFO_DEVICE_MASK) >> ENA_ADMIN_HOST_INFO_DEVICE_SHIFT; +} + +static inline void set_ena_admin_host_info_device(struct ena_admin_host_info *p, uint16_t val) +{ + p->bdf |= (val << ENA_ADMIN_HOST_INFO_DEVICE_SHIFT) & ENA_ADMIN_HOST_INFO_DEVICE_MASK; +} + +static inline uint16_t get_ena_admin_host_info_bus(const struct ena_admin_host_info *p) +{ + return (p->bdf & ENA_ADMIN_HOST_INFO_BUS_MASK) >> ENA_ADMIN_HOST_INFO_BUS_SHIFT; +} + +static inline void set_ena_admin_host_info_bus(struct ena_admin_host_info *p, uint16_t val) +{ + p->bdf |= (val << ENA_ADMIN_HOST_INFO_BUS_SHIFT) & ENA_ADMIN_HOST_INFO_BUS_MASK; +} + +static inline uint32_t get_ena_admin_host_info_rx_offset(const struct ena_admin_host_info *p) +{ + return (p->driver_supported_features & ENA_ADMIN_HOST_INFO_RX_OFFSET_MASK) >> ENA_ADMIN_HOST_INFO_RX_OFFSET_SHIFT; +} + +static inline void set_ena_admin_host_info_rx_offset(struct ena_admin_host_info *p, uint32_t val) +{ + p->driver_supported_features |= (val << ENA_ADMIN_HOST_INFO_RX_OFFSET_SHIFT) & ENA_ADMIN_HOST_INFO_RX_OFFSET_MASK; +} + +static inline uint32_t get_ena_admin_host_info_interrupt_moderation(const struct ena_admin_host_info *p) +{ + return (p->driver_supported_features & ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_MASK) >> ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_SHIFT; +} + +static inline void set_ena_admin_host_info_interrupt_moderation(struct ena_admin_host_info *p, uint32_t val) +{ + p->driver_supported_features |= (val << ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_SHIFT) & ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_MASK; +} + +static inline uint32_t get_ena_admin_host_info_rx_buf_mirroring(const struct ena_admin_host_info *p) +{ + return (p->driver_supported_features & ENA_ADMIN_HOST_INFO_RX_BUF_MIRRORING_MASK) >> ENA_ADMIN_HOST_INFO_RX_BUF_MIRRORING_SHIFT; +} + +static inline void set_ena_admin_host_info_rx_buf_mirroring(struct ena_admin_host_info *p, uint32_t val) +{ + p->driver_supported_features |= (val << ENA_ADMIN_HOST_INFO_RX_BUF_MIRRORING_SHIFT) & ENA_ADMIN_HOST_INFO_RX_BUF_MIRRORING_MASK; +} + +static inline uint32_t get_ena_admin_host_info_rss_configurable_function_key(const struct ena_admin_host_info *p) { - return (p->driver_version & ENA_ADMIN_HOST_INFO_MINOR_MASK) - >> ENA_ADMIN_HOST_INFO_MINOR_SHIFT; + return (p->driver_supported_features & ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_MASK) >> ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_SHIFT; } -static inline void -set_ena_admin_host_info_minor(struct ena_admin_host_info *p, uint32_t val) +static inline void set_ena_admin_host_info_rss_configurable_function_key(struct ena_admin_host_info *p, uint32_t val) { - p->driver_version |= (val << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) - & ENA_ADMIN_HOST_INFO_MINOR_MASK; + p->driver_supported_features |= (val << ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_SHIFT) & ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_MASK; } -static inline uint32_t -get_ena_admin_host_info_sub_minor(const struct ena_admin_host_info *p) +static inline uint8_t get_ena_admin_feature_rss_ind_table_one_entry_update(const struct ena_admin_feature_rss_ind_table *p) { - return (p->driver_version & ENA_ADMIN_HOST_INFO_SUB_MINOR_MASK) - >> ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT; + return p->flags & ENA_ADMIN_FEATURE_RSS_IND_TABLE_ONE_ENTRY_UPDATE_MASK; } -static inline void -set_ena_admin_host_info_sub_minor(struct ena_admin_host_info *p, uint32_t val) +static inline void set_ena_admin_feature_rss_ind_table_one_entry_update(struct ena_admin_feature_rss_ind_table *p, uint8_t val) { - p->driver_version |= (val << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT) - & ENA_ADMIN_HOST_INFO_SUB_MINOR_MASK; + p->flags |= val & ENA_ADMIN_FEATURE_RSS_IND_TABLE_ONE_ENTRY_UPDATE_MASK; } -static inline uint8_t -get_ena_admin_aenq_common_desc_phase( - const struct ena_admin_aenq_common_desc *p) +static inline uint8_t get_ena_admin_aenq_common_desc_phase(const struct ena_admin_aenq_common_desc *p) { return p->flags & ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK; } -static inline void -set_ena_admin_aenq_common_desc_phase( - struct ena_admin_aenq_common_desc *p, - uint8_t val) +static inline void set_ena_admin_aenq_common_desc_phase(struct ena_admin_aenq_common_desc *p, uint8_t val) { p->flags |= val & ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK; } -static inline uint32_t -get_ena_admin_aenq_link_change_desc_link_status( - const struct ena_admin_aenq_link_change_desc *p) +static inline uint32_t get_ena_admin_aenq_link_change_desc_link_status(const struct ena_admin_aenq_link_change_desc *p) { return p->flags & ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK; } -static inline void -set_ena_admin_aenq_link_change_desc_link_status( - struct ena_admin_aenq_link_change_desc *p, - uint32_t val) +static inline void set_ena_admin_aenq_link_change_desc_link_status(struct ena_admin_aenq_link_change_desc *p, uint32_t val) { p->flags |= val & ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK; } -#endif /* !defined(ENA_DEFS_LINUX_MAINLINE) */ -#endif /*_ENA_ADMIN_H_ */ +#endif /* !defined(DEFS_LINUX_MAINLINE) */ +#endif /* _ENA_ADMIN_H_ */ Index: /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_defs/ena_common_defs.h =================================================================== --- /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_defs/ena_common_defs.h (revision 38965) +++ /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_defs/ena_common_defs.h (working copy) @@ -1,54 +1,22 @@ -/*- -* BSD LICENSE -* -* Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. -* All rights reserved. -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following conditions -* are met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above copyright -* notice, this list of conditions and the following disclaimer in -* the documentation and/or other materials provided with the -* distribution. -* * Neither the name of copyright holder nor the names of its -* contributors may be used to endorse or promote products derived -* from this software without specific prior written permission. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. + * All rights reserved. + */ #ifndef _ENA_COMMON_H_ #define _ENA_COMMON_H_ -/* spec version */ -#define ENA_COMMON_SPEC_VERSION_MAJOR 0 /* spec version major */ -#define ENA_COMMON_SPEC_VERSION_MINOR 10 /* spec version minor */ +#define ENA_COMMON_SPEC_VERSION_MAJOR 2 +#define ENA_COMMON_SPEC_VERSION_MINOR 0 /* ENA operates with 48-bit memory addresses. ena_mem_addr_t */ struct ena_common_mem_addr { - /* word 0 : low 32 bit of the memory address */ uint32_t mem_addr_low; - /* word 1 : */ - /* high 16 bits of the memory address */ uint16_t mem_addr_high; /* MBZ */ uint16_t reserved16; }; -#endif /*_ENA_COMMON_H_ */ +#endif /* _ENA_COMMON_H_ */ Index: /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_defs/ena_eth_io_defs.h =================================================================== --- /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_defs/ena_eth_io_defs.h (revision 38965) +++ /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_defs/ena_eth_io_defs.h (working copy) @@ -1,68 +1,27 @@ -/*- -* BSD LICENSE -* -* Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. -* All rights reserved. -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following conditions -* are met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above copyright -* notice, this list of conditions and the following disclaimer in -* the documentation and/or other materials provided with the -* distribution. -* * Neither the name of copyright holder nor the names of its -* contributors may be used to endorse or promote products derived -* from this software without specific prior written permission. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. + * All rights reserved. + */ #ifndef _ENA_ETH_IO_H_ #define _ENA_ETH_IO_H_ -/* Layer 3 protocol index */ enum ena_eth_io_l3_proto_index { - ENA_ETH_IO_L3_PROTO_UNKNOWN = 0, - - ENA_ETH_IO_L3_PROTO_IPV4 = 8, - - ENA_ETH_IO_L3_PROTO_IPV6 = 11, - - ENA_ETH_IO_L3_PROTO_FCOE = 21, - - ENA_ETH_IO_L3_PROTO_ROCE = 22, + ENA_ETH_IO_L3_PROTO_UNKNOWN = 0, + ENA_ETH_IO_L3_PROTO_IPV4 = 8, + ENA_ETH_IO_L3_PROTO_IPV6 = 11, + ENA_ETH_IO_L3_PROTO_FCOE = 21, + ENA_ETH_IO_L3_PROTO_ROCE = 22, }; -/* Layer 4 protocol index */ enum ena_eth_io_l4_proto_index { - ENA_ETH_IO_L4_PROTO_UNKNOWN = 0, - - ENA_ETH_IO_L4_PROTO_TCP = 12, - - ENA_ETH_IO_L4_PROTO_UDP = 13, - - ENA_ETH_IO_L4_PROTO_ROUTEABLE_ROCE = 23, + ENA_ETH_IO_L4_PROTO_UNKNOWN = 0, + ENA_ETH_IO_L4_PROTO_TCP = 12, + ENA_ETH_IO_L4_PROTO_UDP = 13, + ENA_ETH_IO_L4_PROTO_ROUTEABLE_ROCE = 23, }; -/* ENA IO Queue Tx descriptor */ struct ena_eth_io_tx_desc { - /* word 0 : */ - /* length, request id and control flags - * 15:0 : length - Buffer length in bytes, must + /* 15:0 : length - Buffer length in bytes, must * include any packet trailers that the ENA supposed * to update like End-to-End CRC, Authentication GMAC * etc. This length must not include the @@ -85,9 +44,7 @@ */ uint32_t len_ctrl; - /* word 1 : */ - /* ethernet control - * 3:0 : l3_proto_idx - L3 protocol. This field + /* 3:0 : l3_proto_idx - L3 protocol. This field * required when l3_csum_en,l3_csum or tso_en are set. * 4 : DF - IPv4 DF, must be 0 if packet is IPv4 and * DF flags of the IPv4 header is 0. Otherwise must @@ -119,10 +76,8 @@ */ uint32_t meta_ctrl; - /* word 2 : Buffer address bits[31:0] */ uint32_t buff_addr_lo; - /* word 3 : */ /* address high and header size * 15:0 : addr_hi - Buffer Pointer[47:32] * 23:16 : reserved16_w2 @@ -141,20 +96,16 @@ uint32_t buff_addr_hi_hdr_sz; }; -/* ENA IO Queue Tx Meta descriptor */ struct ena_eth_io_tx_meta_desc { - /* word 0 : */ - /* length, request id and control flags - * 9:0 : req_id_lo - Request ID[9:0] + /* 9:0 : req_id_lo - Request ID[9:0] * 11:10 : reserved10 - MBZ * 12 : reserved12 - MBZ * 13 : reserved13 - MBZ * 14 : ext_valid - if set, offset fields in Word2 - * are valid Also MSS High in Word 0 and Outer L3 - * Offset High in WORD 0 and bits [31:24] in Word 3 - * 15 : word3_valid - If set Crypto Info[23:0] of - * Word 3 is valid - * 19:16 : mss_hi_ptp + * are valid Also MSS High in Word 0 and bits [31:24] + * in Word 3 + * 15 : reserved15 + * 19:16 : mss_hi * 20 : eth_meta_type - 0: Tx Metadata Descriptor, 1: * Extended Metadata Descriptor * 21 : meta_store - Store extended metadata in queue @@ -175,19 +126,13 @@ */ uint32_t len_ctrl; - /* word 1 : */ - /* word 1 - * 5:0 : req_id_hi + /* 5:0 : req_id_hi * 31:6 : reserved6 - MBZ */ uint32_t word1; - /* word 2 : */ - /* word 2 - * 7:0 : l3_hdr_len - the header length L3 IP header. - * 15:8 : l3_hdr_off - the offset of the first byte - * in the L3 header from the beginning of the to-be - * transmitted packet. + /* 7:0 : l3_hdr_len + * 15:8 : l3_hdr_off * 21:16 : l4_hdr_len_in_words - counts the L4 header * length in words. there is an explicit assumption * that L4 header appears right after L3 header and @@ -196,13 +141,10 @@ */ uint32_t word2; - /* word 3 : */ uint32_t reserved; }; -/* ENA IO Queue Tx completions descriptor */ struct ena_eth_io_tx_cdesc { - /* word 0 : */ /* Request ID[15:0] */ uint16_t req_id; @@ -214,24 +156,19 @@ */ uint8_t flags; - /* word 1 : */ uint16_t sub_qid; - /* indicates location of submission queue head */ uint16_t sq_head_idx; }; -/* ENA IO Queue Rx descriptor */ struct ena_eth_io_rx_desc { - /* word 0 : */ /* In bytes. 0 means 64KB */ uint16_t length; /* MBZ */ uint8_t reserved2; - /* control flags - * 0 : phase + /* 0 : phase * 1 : reserved1 - MBZ * 2 : first - Indicates first descriptor in * transaction @@ -242,32 +179,27 @@ */ uint8_t ctrl; - /* word 1 : */ uint16_t req_id; /* MBZ */ uint16_t reserved6; - /* word 2 : Buffer address bits[31:0] */ uint32_t buff_addr_lo; - /* word 3 : */ - /* Buffer Address bits[47:16] */ uint16_t buff_addr_hi; /* MBZ */ uint16_t reserved16_w3; }; -/* ENA IO Queue Rx Completion Base Descriptor (4-word format). Note: all - * ethernet parsing information are valid only when last=1 +/* 4-word format Note: all ethernet parsing information are valid only when + * last=1 */ struct ena_eth_io_rx_cdesc_base { - /* word 0 : */ - /* 4:0 : l3_proto_idx - L3 protocol index - * 6:5 : src_vlan_cnt - Source VLAN count + /* 4:0 : l3_proto_idx + * 6:5 : src_vlan_cnt * 7 : reserved7 - MBZ - * 12:8 : l4_proto_idx - L4 protocol index + * 12:8 : l4_proto_idx * 13 : l3_csum_err - when set, either the L3 * checksum error detected, or, the controller didn't * validate the checksum. This bit is valid only when @@ -276,9 +208,13 @@ * checksum error detected, or, the controller didn't * validate the checksum. This bit is valid only when * l4_proto_idx indicates TCP/UDP packet, and, - * ipv4_frag is not set + * ipv4_frag is not set. This bit is valid only when + * l4_csum_checked below is set. * 15 : ipv4_frag - Indicates IPv4 fragmented packet - * 23:16 : reserved16 + * 16 : l4_csum_checked - L4 checksum was verified + * (could be OK or error), when cleared the status of + * checksum is unknown + * 23:17 : reserved17 - MBZ * 24 : phase * 25 : l3_csum2 - second checksum engine result * 26 : first - Indicates first descriptor in @@ -292,56 +228,45 @@ */ uint32_t status; - /* word 1 : */ uint16_t length; uint16_t req_id; - /* word 2 : 32-bit hash result */ + /* 32-bit hash result */ uint32_t hash; - /* word 3 : */ - /* submission queue number */ uint16_t sub_qid; - uint16_t reserved; + uint8_t offset; + + uint8_t reserved; }; -/* ENA IO Queue Rx Completion Descriptor (8-word format) */ +/* 8-word format */ struct ena_eth_io_rx_cdesc_ext { - /* words 0:3 : Rx Completion Extended */ struct ena_eth_io_rx_cdesc_base base; - /* word 4 : Completed Buffer address bits[31:0] */ uint32_t buff_addr_lo; - /* word 5 : */ - /* the buffer address used bits[47:32] */ uint16_t buff_addr_hi; uint16_t reserved16; - /* word 6 : Reserved */ uint32_t reserved_w6; - /* word 7 : Reserved */ uint32_t reserved_w7; }; -/* ENA Interrupt Unmask Register */ struct ena_eth_io_intr_reg { - /* word 0 : */ - /* 14:0 : rx_intr_delay - rx interrupt delay value - * 29:15 : tx_intr_delay - tx interrupt delay value - * 30 : intr_unmask - if set, unmasks interrupt + /* 14:0 : rx_intr_delay + * 29:15 : tx_intr_delay + * 30 : intr_unmask * 31 : reserved */ uint32_t intr_control; }; -/* ENA NUMA Node configuration register */ struct ena_eth_io_numa_node_cfg_reg { - /* word 0 : */ /* 7:0 : numa * 30:8 : reserved * 31 : enabled @@ -350,917 +275,668 @@ }; /* tx_desc */ -#define ENA_ETH_IO_TX_DESC_LENGTH_MASK GENMASK(15, 0) -#define ENA_ETH_IO_TX_DESC_REQ_ID_HI_SHIFT 16 -#define ENA_ETH_IO_TX_DESC_REQ_ID_HI_MASK GENMASK(21, 16) -#define ENA_ETH_IO_TX_DESC_META_DESC_SHIFT 23 -#define ENA_ETH_IO_TX_DESC_META_DESC_MASK BIT(23) -#define ENA_ETH_IO_TX_DESC_PHASE_SHIFT 24 -#define ENA_ETH_IO_TX_DESC_PHASE_MASK BIT(24) -#define ENA_ETH_IO_TX_DESC_FIRST_SHIFT 26 -#define ENA_ETH_IO_TX_DESC_FIRST_MASK BIT(26) -#define ENA_ETH_IO_TX_DESC_LAST_SHIFT 27 -#define ENA_ETH_IO_TX_DESC_LAST_MASK BIT(27) -#define ENA_ETH_IO_TX_DESC_COMP_REQ_SHIFT 28 -#define ENA_ETH_IO_TX_DESC_COMP_REQ_MASK BIT(28) -#define ENA_ETH_IO_TX_DESC_L3_PROTO_IDX_MASK GENMASK(3, 0) -#define ENA_ETH_IO_TX_DESC_DF_SHIFT 4 -#define ENA_ETH_IO_TX_DESC_DF_MASK BIT(4) -#define ENA_ETH_IO_TX_DESC_TSO_EN_SHIFT 7 -#define ENA_ETH_IO_TX_DESC_TSO_EN_MASK BIT(7) -#define ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_SHIFT 8 -#define ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_MASK GENMASK(12, 8) -#define ENA_ETH_IO_TX_DESC_L3_CSUM_EN_SHIFT 13 -#define ENA_ETH_IO_TX_DESC_L3_CSUM_EN_MASK BIT(13) -#define ENA_ETH_IO_TX_DESC_L4_CSUM_EN_SHIFT 14 -#define ENA_ETH_IO_TX_DESC_L4_CSUM_EN_MASK BIT(14) -#define ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_SHIFT 15 -#define ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_MASK BIT(15) -#define ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_SHIFT 17 -#define ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_MASK BIT(17) -#define ENA_ETH_IO_TX_DESC_REQ_ID_LO_SHIFT 22 -#define ENA_ETH_IO_TX_DESC_REQ_ID_LO_MASK GENMASK(31, 22) -#define ENA_ETH_IO_TX_DESC_ADDR_HI_MASK GENMASK(15, 0) -#define ENA_ETH_IO_TX_DESC_HEADER_LENGTH_SHIFT 24 -#define ENA_ETH_IO_TX_DESC_HEADER_LENGTH_MASK GENMASK(31, 24) +#define ENA_ETH_IO_TX_DESC_LENGTH_MASK GENMASK(15, 0) +#define ENA_ETH_IO_TX_DESC_REQ_ID_HI_SHIFT 16 +#define ENA_ETH_IO_TX_DESC_REQ_ID_HI_MASK GENMASK(21, 16) +#define ENA_ETH_IO_TX_DESC_META_DESC_SHIFT 23 +#define ENA_ETH_IO_TX_DESC_META_DESC_MASK BIT(23) +#define ENA_ETH_IO_TX_DESC_PHASE_SHIFT 24 +#define ENA_ETH_IO_TX_DESC_PHASE_MASK BIT(24) +#define ENA_ETH_IO_TX_DESC_FIRST_SHIFT 26 +#define ENA_ETH_IO_TX_DESC_FIRST_MASK BIT(26) +#define ENA_ETH_IO_TX_DESC_LAST_SHIFT 27 +#define ENA_ETH_IO_TX_DESC_LAST_MASK BIT(27) +#define ENA_ETH_IO_TX_DESC_COMP_REQ_SHIFT 28 +#define ENA_ETH_IO_TX_DESC_COMP_REQ_MASK BIT(28) +#define ENA_ETH_IO_TX_DESC_L3_PROTO_IDX_MASK GENMASK(3, 0) +#define ENA_ETH_IO_TX_DESC_DF_SHIFT 4 +#define ENA_ETH_IO_TX_DESC_DF_MASK BIT(4) +#define ENA_ETH_IO_TX_DESC_TSO_EN_SHIFT 7 +#define ENA_ETH_IO_TX_DESC_TSO_EN_MASK BIT(7) +#define ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_SHIFT 8 +#define ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_MASK GENMASK(12, 8) +#define ENA_ETH_IO_TX_DESC_L3_CSUM_EN_SHIFT 13 +#define ENA_ETH_IO_TX_DESC_L3_CSUM_EN_MASK BIT(13) +#define ENA_ETH_IO_TX_DESC_L4_CSUM_EN_SHIFT 14 +#define ENA_ETH_IO_TX_DESC_L4_CSUM_EN_MASK BIT(14) +#define ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_SHIFT 15 +#define ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_MASK BIT(15) +#define ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_SHIFT 17 +#define ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_MASK BIT(17) +#define ENA_ETH_IO_TX_DESC_REQ_ID_LO_SHIFT 22 +#define ENA_ETH_IO_TX_DESC_REQ_ID_LO_MASK GENMASK(31, 22) +#define ENA_ETH_IO_TX_DESC_ADDR_HI_MASK GENMASK(15, 0) +#define ENA_ETH_IO_TX_DESC_HEADER_LENGTH_SHIFT 24 +#define ENA_ETH_IO_TX_DESC_HEADER_LENGTH_MASK GENMASK(31, 24) /* tx_meta_desc */ -#define ENA_ETH_IO_TX_META_DESC_REQ_ID_LO_MASK GENMASK(9, 0) -#define ENA_ETH_IO_TX_META_DESC_EXT_VALID_SHIFT 14 -#define ENA_ETH_IO_TX_META_DESC_EXT_VALID_MASK BIT(14) -#define ENA_ETH_IO_TX_META_DESC_WORD3_VALID_SHIFT 15 -#define ENA_ETH_IO_TX_META_DESC_WORD3_VALID_MASK BIT(15) -#define ENA_ETH_IO_TX_META_DESC_MSS_HI_PTP_SHIFT 16 -#define ENA_ETH_IO_TX_META_DESC_MSS_HI_PTP_MASK GENMASK(19, 16) -#define ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_SHIFT 20 -#define ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_MASK BIT(20) -#define ENA_ETH_IO_TX_META_DESC_META_STORE_SHIFT 21 -#define ENA_ETH_IO_TX_META_DESC_META_STORE_MASK BIT(21) -#define ENA_ETH_IO_TX_META_DESC_META_DESC_SHIFT 23 -#define ENA_ETH_IO_TX_META_DESC_META_DESC_MASK BIT(23) -#define ENA_ETH_IO_TX_META_DESC_PHASE_SHIFT 24 -#define ENA_ETH_IO_TX_META_DESC_PHASE_MASK BIT(24) -#define ENA_ETH_IO_TX_META_DESC_FIRST_SHIFT 26 -#define ENA_ETH_IO_TX_META_DESC_FIRST_MASK BIT(26) -#define ENA_ETH_IO_TX_META_DESC_LAST_SHIFT 27 -#define ENA_ETH_IO_TX_META_DESC_LAST_MASK BIT(27) -#define ENA_ETH_IO_TX_META_DESC_COMP_REQ_SHIFT 28 -#define ENA_ETH_IO_TX_META_DESC_COMP_REQ_MASK BIT(28) -#define ENA_ETH_IO_TX_META_DESC_REQ_ID_HI_MASK GENMASK(5, 0) -#define ENA_ETH_IO_TX_META_DESC_L3_HDR_LEN_MASK GENMASK(7, 0) -#define ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_SHIFT 8 -#define ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_MASK GENMASK(15, 8) -#define ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_SHIFT 16 -#define ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_MASK GENMASK(21, 16) -#define ENA_ETH_IO_TX_META_DESC_MSS_LO_SHIFT 22 -#define ENA_ETH_IO_TX_META_DESC_MSS_LO_MASK GENMASK(31, 22) +#define ENA_ETH_IO_TX_META_DESC_REQ_ID_LO_MASK GENMASK(9, 0) +#define ENA_ETH_IO_TX_META_DESC_EXT_VALID_SHIFT 14 +#define ENA_ETH_IO_TX_META_DESC_EXT_VALID_MASK BIT(14) +#define ENA_ETH_IO_TX_META_DESC_MSS_HI_SHIFT 16 +#define ENA_ETH_IO_TX_META_DESC_MSS_HI_MASK GENMASK(19, 16) +#define ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_SHIFT 20 +#define ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_MASK BIT(20) +#define ENA_ETH_IO_TX_META_DESC_META_STORE_SHIFT 21 +#define ENA_ETH_IO_TX_META_DESC_META_STORE_MASK BIT(21) +#define ENA_ETH_IO_TX_META_DESC_META_DESC_SHIFT 23 +#define ENA_ETH_IO_TX_META_DESC_META_DESC_MASK BIT(23) +#define ENA_ETH_IO_TX_META_DESC_PHASE_SHIFT 24 +#define ENA_ETH_IO_TX_META_DESC_PHASE_MASK BIT(24) +#define ENA_ETH_IO_TX_META_DESC_FIRST_SHIFT 26 +#define ENA_ETH_IO_TX_META_DESC_FIRST_MASK BIT(26) +#define ENA_ETH_IO_TX_META_DESC_LAST_SHIFT 27 +#define ENA_ETH_IO_TX_META_DESC_LAST_MASK BIT(27) +#define ENA_ETH_IO_TX_META_DESC_COMP_REQ_SHIFT 28 +#define ENA_ETH_IO_TX_META_DESC_COMP_REQ_MASK BIT(28) +#define ENA_ETH_IO_TX_META_DESC_REQ_ID_HI_MASK GENMASK(5, 0) +#define ENA_ETH_IO_TX_META_DESC_L3_HDR_LEN_MASK GENMASK(7, 0) +#define ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_SHIFT 8 +#define ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_MASK GENMASK(15, 8) +#define ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_SHIFT 16 +#define ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_MASK GENMASK(21, 16) +#define ENA_ETH_IO_TX_META_DESC_MSS_LO_SHIFT 22 +#define ENA_ETH_IO_TX_META_DESC_MSS_LO_MASK GENMASK(31, 22) /* tx_cdesc */ -#define ENA_ETH_IO_TX_CDESC_PHASE_MASK BIT(0) +#define ENA_ETH_IO_TX_CDESC_PHASE_MASK BIT(0) /* rx_desc */ -#define ENA_ETH_IO_RX_DESC_PHASE_MASK BIT(0) -#define ENA_ETH_IO_RX_DESC_FIRST_SHIFT 2 -#define ENA_ETH_IO_RX_DESC_FIRST_MASK BIT(2) -#define ENA_ETH_IO_RX_DESC_LAST_SHIFT 3 -#define ENA_ETH_IO_RX_DESC_LAST_MASK BIT(3) -#define ENA_ETH_IO_RX_DESC_COMP_REQ_SHIFT 4 -#define ENA_ETH_IO_RX_DESC_COMP_REQ_MASK BIT(4) +#define ENA_ETH_IO_RX_DESC_PHASE_MASK BIT(0) +#define ENA_ETH_IO_RX_DESC_FIRST_SHIFT 2 +#define ENA_ETH_IO_RX_DESC_FIRST_MASK BIT(2) +#define ENA_ETH_IO_RX_DESC_LAST_SHIFT 3 +#define ENA_ETH_IO_RX_DESC_LAST_MASK BIT(3) +#define ENA_ETH_IO_RX_DESC_COMP_REQ_SHIFT 4 +#define ENA_ETH_IO_RX_DESC_COMP_REQ_MASK BIT(4) /* rx_cdesc_base */ -#define ENA_ETH_IO_RX_CDESC_BASE_L3_PROTO_IDX_MASK GENMASK(4, 0) -#define ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_SHIFT 5 -#define ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_MASK GENMASK(6, 5) -#define ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_SHIFT 8 -#define ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_MASK GENMASK(12, 8) -#define ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_SHIFT 13 -#define ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_MASK BIT(13) -#define ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_SHIFT 14 -#define ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_MASK BIT(14) -#define ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_SHIFT 15 -#define ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_MASK BIT(15) -#define ENA_ETH_IO_RX_CDESC_BASE_PHASE_SHIFT 24 -#define ENA_ETH_IO_RX_CDESC_BASE_PHASE_MASK BIT(24) -#define ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_SHIFT 25 -#define ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_MASK BIT(25) -#define ENA_ETH_IO_RX_CDESC_BASE_FIRST_SHIFT 26 -#define ENA_ETH_IO_RX_CDESC_BASE_FIRST_MASK BIT(26) -#define ENA_ETH_IO_RX_CDESC_BASE_LAST_SHIFT 27 -#define ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK BIT(27) -#define ENA_ETH_IO_RX_CDESC_BASE_BUFFER_SHIFT 30 -#define ENA_ETH_IO_RX_CDESC_BASE_BUFFER_MASK BIT(30) +#define ENA_ETH_IO_RX_CDESC_BASE_L3_PROTO_IDX_MASK GENMASK(4, 0) +#define ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_SHIFT 5 +#define ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_MASK GENMASK(6, 5) +#define ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_SHIFT 8 +#define ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_MASK GENMASK(12, 8) +#define ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_SHIFT 13 +#define ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_MASK BIT(13) +#define ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_SHIFT 14 +#define ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_MASK BIT(14) +#define ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_SHIFT 15 +#define ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_MASK BIT(15) +#define ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_CHECKED_SHIFT 16 +#define ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_CHECKED_MASK BIT(16) +#define ENA_ETH_IO_RX_CDESC_BASE_PHASE_SHIFT 24 +#define ENA_ETH_IO_RX_CDESC_BASE_PHASE_MASK BIT(24) +#define ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_SHIFT 25 +#define ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_MASK BIT(25) +#define ENA_ETH_IO_RX_CDESC_BASE_FIRST_SHIFT 26 +#define ENA_ETH_IO_RX_CDESC_BASE_FIRST_MASK BIT(26) +#define ENA_ETH_IO_RX_CDESC_BASE_LAST_SHIFT 27 +#define ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK BIT(27) +#define ENA_ETH_IO_RX_CDESC_BASE_BUFFER_SHIFT 30 +#define ENA_ETH_IO_RX_CDESC_BASE_BUFFER_MASK BIT(30) /* intr_reg */ -#define ENA_ETH_IO_INTR_REG_RX_INTR_DELAY_MASK GENMASK(14, 0) -#define ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_SHIFT 15 -#define ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_MASK GENMASK(29, 15) -#define ENA_ETH_IO_INTR_REG_INTR_UNMASK_SHIFT 30 -#define ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK BIT(30) +#define ENA_ETH_IO_INTR_REG_RX_INTR_DELAY_MASK GENMASK(14, 0) +#define ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_SHIFT 15 +#define ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_MASK GENMASK(29, 15) +#define ENA_ETH_IO_INTR_REG_INTR_UNMASK_SHIFT 30 +#define ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK BIT(30) /* numa_node_cfg_reg */ -#define ENA_ETH_IO_NUMA_NODE_CFG_REG_NUMA_MASK GENMASK(7, 0) -#define ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_SHIFT 31 -#define ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_MASK BIT(31) - -#if !defined(ENA_DEFS_LINUX_MAINLINE) -static inline uint32_t get_ena_eth_io_tx_desc_length( - const struct ena_eth_io_tx_desc *p) +#define ENA_ETH_IO_NUMA_NODE_CFG_REG_NUMA_MASK GENMASK(7, 0) +#define ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_SHIFT 31 +#define ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_MASK BIT(31) + +#if !defined(DEFS_LINUX_MAINLINE) +static inline uint32_t get_ena_eth_io_tx_desc_length(const struct ena_eth_io_tx_desc *p) { return p->len_ctrl & ENA_ETH_IO_TX_DESC_LENGTH_MASK; } -static inline void set_ena_eth_io_tx_desc_length( - struct ena_eth_io_tx_desc *p, - uint32_t val) +static inline void set_ena_eth_io_tx_desc_length(struct ena_eth_io_tx_desc *p, uint32_t val) { p->len_ctrl |= val & ENA_ETH_IO_TX_DESC_LENGTH_MASK; } -static inline uint32_t get_ena_eth_io_tx_desc_req_id_hi( - const struct ena_eth_io_tx_desc *p) +static inline uint32_t get_ena_eth_io_tx_desc_req_id_hi(const struct ena_eth_io_tx_desc *p) { - return (p->len_ctrl & ENA_ETH_IO_TX_DESC_REQ_ID_HI_MASK) - >> ENA_ETH_IO_TX_DESC_REQ_ID_HI_SHIFT; + return (p->len_ctrl & ENA_ETH_IO_TX_DESC_REQ_ID_HI_MASK) >> ENA_ETH_IO_TX_DESC_REQ_ID_HI_SHIFT; } -static inline void set_ena_eth_io_tx_desc_req_id_hi( - struct ena_eth_io_tx_desc *p, - uint32_t val) +static inline void set_ena_eth_io_tx_desc_req_id_hi(struct ena_eth_io_tx_desc *p, uint32_t val) { - p->len_ctrl |= - (val << ENA_ETH_IO_TX_DESC_REQ_ID_HI_SHIFT) - & ENA_ETH_IO_TX_DESC_REQ_ID_HI_MASK; + p->len_ctrl |= (val << ENA_ETH_IO_TX_DESC_REQ_ID_HI_SHIFT) & ENA_ETH_IO_TX_DESC_REQ_ID_HI_MASK; } -static inline uint32_t get_ena_eth_io_tx_desc_meta_desc( - const struct ena_eth_io_tx_desc *p) +static inline uint32_t get_ena_eth_io_tx_desc_meta_desc(const struct ena_eth_io_tx_desc *p) { - return (p->len_ctrl & ENA_ETH_IO_TX_DESC_META_DESC_MASK) - >> ENA_ETH_IO_TX_DESC_META_DESC_SHIFT; + return (p->len_ctrl & ENA_ETH_IO_TX_DESC_META_DESC_MASK) >> ENA_ETH_IO_TX_DESC_META_DESC_SHIFT; } -static inline void set_ena_eth_io_tx_desc_meta_desc( - struct ena_eth_io_tx_desc *p, - uint32_t val) +static inline void set_ena_eth_io_tx_desc_meta_desc(struct ena_eth_io_tx_desc *p, uint32_t val) { - p->len_ctrl |= - (val << ENA_ETH_IO_TX_DESC_META_DESC_SHIFT) - & ENA_ETH_IO_TX_DESC_META_DESC_MASK; + p->len_ctrl |= (val << ENA_ETH_IO_TX_DESC_META_DESC_SHIFT) & ENA_ETH_IO_TX_DESC_META_DESC_MASK; } -static inline uint32_t get_ena_eth_io_tx_desc_phase( - const struct ena_eth_io_tx_desc *p) +static inline uint32_t get_ena_eth_io_tx_desc_phase(const struct ena_eth_io_tx_desc *p) { - return (p->len_ctrl & ENA_ETH_IO_TX_DESC_PHASE_MASK) - >> ENA_ETH_IO_TX_DESC_PHASE_SHIFT; + return (p->len_ctrl & ENA_ETH_IO_TX_DESC_PHASE_MASK) >> ENA_ETH_IO_TX_DESC_PHASE_SHIFT; } -static inline void set_ena_eth_io_tx_desc_phase( - struct ena_eth_io_tx_desc *p, - uint32_t val) +static inline void set_ena_eth_io_tx_desc_phase(struct ena_eth_io_tx_desc *p, uint32_t val) { - p->len_ctrl |= - (val << ENA_ETH_IO_TX_DESC_PHASE_SHIFT) - & ENA_ETH_IO_TX_DESC_PHASE_MASK; + p->len_ctrl |= (val << ENA_ETH_IO_TX_DESC_PHASE_SHIFT) & ENA_ETH_IO_TX_DESC_PHASE_MASK; } -static inline uint32_t get_ena_eth_io_tx_desc_first( - const struct ena_eth_io_tx_desc *p) +static inline uint32_t get_ena_eth_io_tx_desc_first(const struct ena_eth_io_tx_desc *p) { - return (p->len_ctrl & ENA_ETH_IO_TX_DESC_FIRST_MASK) - >> ENA_ETH_IO_TX_DESC_FIRST_SHIFT; + return (p->len_ctrl & ENA_ETH_IO_TX_DESC_FIRST_MASK) >> ENA_ETH_IO_TX_DESC_FIRST_SHIFT; } -static inline void set_ena_eth_io_tx_desc_first( - struct ena_eth_io_tx_desc *p, - uint32_t val) +static inline void set_ena_eth_io_tx_desc_first(struct ena_eth_io_tx_desc *p, uint32_t val) { - p->len_ctrl |= - (val << ENA_ETH_IO_TX_DESC_FIRST_SHIFT) - & ENA_ETH_IO_TX_DESC_FIRST_MASK; + p->len_ctrl |= (val << ENA_ETH_IO_TX_DESC_FIRST_SHIFT) & ENA_ETH_IO_TX_DESC_FIRST_MASK; } -static inline uint32_t get_ena_eth_io_tx_desc_last( - const struct ena_eth_io_tx_desc *p) +static inline uint32_t get_ena_eth_io_tx_desc_last(const struct ena_eth_io_tx_desc *p) { - return (p->len_ctrl & ENA_ETH_IO_TX_DESC_LAST_MASK) - >> ENA_ETH_IO_TX_DESC_LAST_SHIFT; + return (p->len_ctrl & ENA_ETH_IO_TX_DESC_LAST_MASK) >> ENA_ETH_IO_TX_DESC_LAST_SHIFT; } -static inline void set_ena_eth_io_tx_desc_last( - struct ena_eth_io_tx_desc *p, - uint32_t val) +static inline void set_ena_eth_io_tx_desc_last(struct ena_eth_io_tx_desc *p, uint32_t val) { - p->len_ctrl |= - (val << ENA_ETH_IO_TX_DESC_LAST_SHIFT) - & ENA_ETH_IO_TX_DESC_LAST_MASK; + p->len_ctrl |= (val << ENA_ETH_IO_TX_DESC_LAST_SHIFT) & ENA_ETH_IO_TX_DESC_LAST_MASK; } -static inline uint32_t get_ena_eth_io_tx_desc_comp_req( - const struct ena_eth_io_tx_desc *p) +static inline uint32_t get_ena_eth_io_tx_desc_comp_req(const struct ena_eth_io_tx_desc *p) { - return (p->len_ctrl & ENA_ETH_IO_TX_DESC_COMP_REQ_MASK) - >> ENA_ETH_IO_TX_DESC_COMP_REQ_SHIFT; + return (p->len_ctrl & ENA_ETH_IO_TX_DESC_COMP_REQ_MASK) >> ENA_ETH_IO_TX_DESC_COMP_REQ_SHIFT; } -static inline void set_ena_eth_io_tx_desc_comp_req( - struct ena_eth_io_tx_desc *p, - uint32_t val) +static inline void set_ena_eth_io_tx_desc_comp_req(struct ena_eth_io_tx_desc *p, uint32_t val) { - p->len_ctrl |= - (val << ENA_ETH_IO_TX_DESC_COMP_REQ_SHIFT) - & ENA_ETH_IO_TX_DESC_COMP_REQ_MASK; + p->len_ctrl |= (val << ENA_ETH_IO_TX_DESC_COMP_REQ_SHIFT) & ENA_ETH_IO_TX_DESC_COMP_REQ_MASK; } -static inline uint32_t get_ena_eth_io_tx_desc_l3_proto_idx( - const struct ena_eth_io_tx_desc *p) +static inline uint32_t get_ena_eth_io_tx_desc_l3_proto_idx(const struct ena_eth_io_tx_desc *p) { return p->meta_ctrl & ENA_ETH_IO_TX_DESC_L3_PROTO_IDX_MASK; } -static inline void set_ena_eth_io_tx_desc_l3_proto_idx( - struct ena_eth_io_tx_desc *p, - uint32_t val) +static inline void set_ena_eth_io_tx_desc_l3_proto_idx(struct ena_eth_io_tx_desc *p, uint32_t val) { p->meta_ctrl |= val & ENA_ETH_IO_TX_DESC_L3_PROTO_IDX_MASK; } -static inline uint32_t get_ena_eth_io_tx_desc_DF( - const struct ena_eth_io_tx_desc *p) +static inline uint32_t get_ena_eth_io_tx_desc_DF(const struct ena_eth_io_tx_desc *p) { - return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_DF_MASK) - >> ENA_ETH_IO_TX_DESC_DF_SHIFT; + return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_DF_MASK) >> ENA_ETH_IO_TX_DESC_DF_SHIFT; } -static inline void set_ena_eth_io_tx_desc_DF( - struct ena_eth_io_tx_desc *p, - uint32_t val) +static inline void set_ena_eth_io_tx_desc_DF(struct ena_eth_io_tx_desc *p, uint32_t val) { - p->meta_ctrl |= - (val << ENA_ETH_IO_TX_DESC_DF_SHIFT) - & ENA_ETH_IO_TX_DESC_DF_MASK; + p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_DF_SHIFT) & ENA_ETH_IO_TX_DESC_DF_MASK; } -static inline uint32_t get_ena_eth_io_tx_desc_tso_en( - const struct ena_eth_io_tx_desc *p) +static inline uint32_t get_ena_eth_io_tx_desc_tso_en(const struct ena_eth_io_tx_desc *p) { - return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_TSO_EN_MASK) - >> ENA_ETH_IO_TX_DESC_TSO_EN_SHIFT; + return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_TSO_EN_MASK) >> ENA_ETH_IO_TX_DESC_TSO_EN_SHIFT; } -static inline void set_ena_eth_io_tx_desc_tso_en( - struct ena_eth_io_tx_desc *p, - uint32_t val) +static inline void set_ena_eth_io_tx_desc_tso_en(struct ena_eth_io_tx_desc *p, uint32_t val) { - p->meta_ctrl |= - (val << ENA_ETH_IO_TX_DESC_TSO_EN_SHIFT) - & ENA_ETH_IO_TX_DESC_TSO_EN_MASK; + p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_TSO_EN_SHIFT) & ENA_ETH_IO_TX_DESC_TSO_EN_MASK; } -static inline uint32_t get_ena_eth_io_tx_desc_l4_proto_idx( - const struct ena_eth_io_tx_desc *p) +static inline uint32_t get_ena_eth_io_tx_desc_l4_proto_idx(const struct ena_eth_io_tx_desc *p) { - return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_MASK) - >> ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_SHIFT; + return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_MASK) >> ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_SHIFT; } -static inline void set_ena_eth_io_tx_desc_l4_proto_idx( - struct ena_eth_io_tx_desc *p, - uint32_t val) +static inline void set_ena_eth_io_tx_desc_l4_proto_idx(struct ena_eth_io_tx_desc *p, uint32_t val) { - p->meta_ctrl |= - (val << ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_SHIFT) - & ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_MASK; + p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_SHIFT) & ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_MASK; } -static inline uint32_t get_ena_eth_io_tx_desc_l3_csum_en( - const struct ena_eth_io_tx_desc *p) +static inline uint32_t get_ena_eth_io_tx_desc_l3_csum_en(const struct ena_eth_io_tx_desc *p) { - return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_L3_CSUM_EN_MASK) - >> ENA_ETH_IO_TX_DESC_L3_CSUM_EN_SHIFT; + return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_L3_CSUM_EN_MASK) >> ENA_ETH_IO_TX_DESC_L3_CSUM_EN_SHIFT; } -static inline void set_ena_eth_io_tx_desc_l3_csum_en( - struct ena_eth_io_tx_desc *p, - uint32_t val) +static inline void set_ena_eth_io_tx_desc_l3_csum_en(struct ena_eth_io_tx_desc *p, uint32_t val) { - p->meta_ctrl |= - (val << ENA_ETH_IO_TX_DESC_L3_CSUM_EN_SHIFT) - & ENA_ETH_IO_TX_DESC_L3_CSUM_EN_MASK; + p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_L3_CSUM_EN_SHIFT) & ENA_ETH_IO_TX_DESC_L3_CSUM_EN_MASK; } -static inline uint32_t get_ena_eth_io_tx_desc_l4_csum_en( - const struct ena_eth_io_tx_desc *p) +static inline uint32_t get_ena_eth_io_tx_desc_l4_csum_en(const struct ena_eth_io_tx_desc *p) { - return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_L4_CSUM_EN_MASK) - >> ENA_ETH_IO_TX_DESC_L4_CSUM_EN_SHIFT; + return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_L4_CSUM_EN_MASK) >> ENA_ETH_IO_TX_DESC_L4_CSUM_EN_SHIFT; } -static inline void set_ena_eth_io_tx_desc_l4_csum_en( - struct ena_eth_io_tx_desc *p, - uint32_t val) +static inline void set_ena_eth_io_tx_desc_l4_csum_en(struct ena_eth_io_tx_desc *p, uint32_t val) { - p->meta_ctrl |= - (val << ENA_ETH_IO_TX_DESC_L4_CSUM_EN_SHIFT) - & ENA_ETH_IO_TX_DESC_L4_CSUM_EN_MASK; + p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_L4_CSUM_EN_SHIFT) & ENA_ETH_IO_TX_DESC_L4_CSUM_EN_MASK; } -static inline uint32_t get_ena_eth_io_tx_desc_ethernet_fcs_dis( - const struct ena_eth_io_tx_desc *p) +static inline uint32_t get_ena_eth_io_tx_desc_ethernet_fcs_dis(const struct ena_eth_io_tx_desc *p) { - return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_MASK) - >> ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_SHIFT; + return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_MASK) >> ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_SHIFT; } -static inline void set_ena_eth_io_tx_desc_ethernet_fcs_dis( - struct ena_eth_io_tx_desc *p, - uint32_t val) +static inline void set_ena_eth_io_tx_desc_ethernet_fcs_dis(struct ena_eth_io_tx_desc *p, uint32_t val) { - p->meta_ctrl |= - (val << ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_SHIFT) - & ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_MASK; + p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_SHIFT) & ENA_ETH_IO_TX_DESC_ETHERNET_FCS_DIS_MASK; } -static inline uint32_t get_ena_eth_io_tx_desc_l4_csum_partial( - const struct ena_eth_io_tx_desc *p) +static inline uint32_t get_ena_eth_io_tx_desc_l4_csum_partial(const struct ena_eth_io_tx_desc *p) { - return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_MASK) - >> ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_SHIFT; + return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_MASK) >> ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_SHIFT; } -static inline void set_ena_eth_io_tx_desc_l4_csum_partial( - struct ena_eth_io_tx_desc *p, - uint32_t val) +static inline void set_ena_eth_io_tx_desc_l4_csum_partial(struct ena_eth_io_tx_desc *p, uint32_t val) { - p->meta_ctrl |= - (val << ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_SHIFT) - & ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_MASK; + p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_SHIFT) & ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_MASK; } -static inline uint32_t get_ena_eth_io_tx_desc_req_id_lo( - const struct ena_eth_io_tx_desc *p) +static inline uint32_t get_ena_eth_io_tx_desc_req_id_lo(const struct ena_eth_io_tx_desc *p) { - return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_REQ_ID_LO_MASK) - >> ENA_ETH_IO_TX_DESC_REQ_ID_LO_SHIFT; + return (p->meta_ctrl & ENA_ETH_IO_TX_DESC_REQ_ID_LO_MASK) >> ENA_ETH_IO_TX_DESC_REQ_ID_LO_SHIFT; } -static inline void set_ena_eth_io_tx_desc_req_id_lo( - struct ena_eth_io_tx_desc *p, uint32_t val) +static inline void set_ena_eth_io_tx_desc_req_id_lo(struct ena_eth_io_tx_desc *p, uint32_t val) { - p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_REQ_ID_LO_SHIFT) - & ENA_ETH_IO_TX_DESC_REQ_ID_LO_MASK; + p->meta_ctrl |= (val << ENA_ETH_IO_TX_DESC_REQ_ID_LO_SHIFT) & ENA_ETH_IO_TX_DESC_REQ_ID_LO_MASK; } -static inline uint32_t get_ena_eth_io_tx_desc_addr_hi( - const struct ena_eth_io_tx_desc *p) +static inline uint32_t get_ena_eth_io_tx_desc_addr_hi(const struct ena_eth_io_tx_desc *p) { return p->buff_addr_hi_hdr_sz & ENA_ETH_IO_TX_DESC_ADDR_HI_MASK; } -static inline void set_ena_eth_io_tx_desc_addr_hi( - struct ena_eth_io_tx_desc *p, - uint32_t val) +static inline void set_ena_eth_io_tx_desc_addr_hi(struct ena_eth_io_tx_desc *p, uint32_t val) { p->buff_addr_hi_hdr_sz |= val & ENA_ETH_IO_TX_DESC_ADDR_HI_MASK; } -static inline uint32_t get_ena_eth_io_tx_desc_header_length( - const struct ena_eth_io_tx_desc *p) +static inline uint32_t get_ena_eth_io_tx_desc_header_length(const struct ena_eth_io_tx_desc *p) { - return (p->buff_addr_hi_hdr_sz & ENA_ETH_IO_TX_DESC_HEADER_LENGTH_MASK) - >> ENA_ETH_IO_TX_DESC_HEADER_LENGTH_SHIFT; + return (p->buff_addr_hi_hdr_sz & ENA_ETH_IO_TX_DESC_HEADER_LENGTH_MASK) >> ENA_ETH_IO_TX_DESC_HEADER_LENGTH_SHIFT; } -static inline void set_ena_eth_io_tx_desc_header_length( - struct ena_eth_io_tx_desc *p, - uint32_t val) +static inline void set_ena_eth_io_tx_desc_header_length(struct ena_eth_io_tx_desc *p, uint32_t val) { - p->buff_addr_hi_hdr_sz |= - (val << ENA_ETH_IO_TX_DESC_HEADER_LENGTH_SHIFT) - & ENA_ETH_IO_TX_DESC_HEADER_LENGTH_MASK; + p->buff_addr_hi_hdr_sz |= (val << ENA_ETH_IO_TX_DESC_HEADER_LENGTH_SHIFT) & ENA_ETH_IO_TX_DESC_HEADER_LENGTH_MASK; } -static inline uint32_t get_ena_eth_io_tx_meta_desc_req_id_lo( - const struct ena_eth_io_tx_meta_desc *p) +static inline uint32_t get_ena_eth_io_tx_meta_desc_req_id_lo(const struct ena_eth_io_tx_meta_desc *p) { return p->len_ctrl & ENA_ETH_IO_TX_META_DESC_REQ_ID_LO_MASK; } -static inline void set_ena_eth_io_tx_meta_desc_req_id_lo( - struct ena_eth_io_tx_meta_desc *p, - uint32_t val) +static inline void set_ena_eth_io_tx_meta_desc_req_id_lo(struct ena_eth_io_tx_meta_desc *p, uint32_t val) { p->len_ctrl |= val & ENA_ETH_IO_TX_META_DESC_REQ_ID_LO_MASK; } -static inline uint32_t get_ena_eth_io_tx_meta_desc_ext_valid( - const struct ena_eth_io_tx_meta_desc *p) +static inline uint32_t get_ena_eth_io_tx_meta_desc_ext_valid(const struct ena_eth_io_tx_meta_desc *p) { - return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_EXT_VALID_MASK) - >> ENA_ETH_IO_TX_META_DESC_EXT_VALID_SHIFT; + return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_EXT_VALID_MASK) >> ENA_ETH_IO_TX_META_DESC_EXT_VALID_SHIFT; } -static inline void set_ena_eth_io_tx_meta_desc_ext_valid( - struct ena_eth_io_tx_meta_desc *p, uint32_t val) +static inline void set_ena_eth_io_tx_meta_desc_ext_valid(struct ena_eth_io_tx_meta_desc *p, uint32_t val) { - p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_EXT_VALID_SHIFT) - & ENA_ETH_IO_TX_META_DESC_EXT_VALID_MASK; + p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_EXT_VALID_SHIFT) & ENA_ETH_IO_TX_META_DESC_EXT_VALID_MASK; } -static inline uint32_t get_ena_eth_io_tx_meta_desc_word3_valid( - const struct ena_eth_io_tx_meta_desc *p) +static inline uint32_t get_ena_eth_io_tx_meta_desc_mss_hi(const struct ena_eth_io_tx_meta_desc *p) { - return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_WORD3_VALID_MASK) - >> ENA_ETH_IO_TX_META_DESC_WORD3_VALID_SHIFT; + return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_MSS_HI_MASK) >> ENA_ETH_IO_TX_META_DESC_MSS_HI_SHIFT; } -static inline void set_ena_eth_io_tx_meta_desc_word3_valid( - struct ena_eth_io_tx_meta_desc *p, uint32_t val) +static inline void set_ena_eth_io_tx_meta_desc_mss_hi(struct ena_eth_io_tx_meta_desc *p, uint32_t val) { - p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_WORD3_VALID_SHIFT) - & ENA_ETH_IO_TX_META_DESC_WORD3_VALID_MASK; + p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_MSS_HI_SHIFT) & ENA_ETH_IO_TX_META_DESC_MSS_HI_MASK; } -static inline uint32_t get_ena_eth_io_tx_meta_desc_mss_hi_ptp( - const struct ena_eth_io_tx_meta_desc *p) +static inline uint32_t get_ena_eth_io_tx_meta_desc_eth_meta_type(const struct ena_eth_io_tx_meta_desc *p) { - return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_MSS_HI_PTP_MASK) - >> ENA_ETH_IO_TX_META_DESC_MSS_HI_PTP_SHIFT; + return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_MASK) >> ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_SHIFT; } -static inline void set_ena_eth_io_tx_meta_desc_mss_hi_ptp( - struct ena_eth_io_tx_meta_desc *p, uint32_t val) +static inline void set_ena_eth_io_tx_meta_desc_eth_meta_type(struct ena_eth_io_tx_meta_desc *p, uint32_t val) { - p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_MSS_HI_PTP_SHIFT) - & ENA_ETH_IO_TX_META_DESC_MSS_HI_PTP_MASK; + p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_SHIFT) & ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_MASK; } -static inline uint32_t get_ena_eth_io_tx_meta_desc_eth_meta_type( - const struct ena_eth_io_tx_meta_desc *p) +static inline uint32_t get_ena_eth_io_tx_meta_desc_meta_store(const struct ena_eth_io_tx_meta_desc *p) { - return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_MASK) - >> ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_SHIFT; + return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_META_STORE_MASK) >> ENA_ETH_IO_TX_META_DESC_META_STORE_SHIFT; } -static inline void set_ena_eth_io_tx_meta_desc_eth_meta_type( - struct ena_eth_io_tx_meta_desc *p, uint32_t val) +static inline void set_ena_eth_io_tx_meta_desc_meta_store(struct ena_eth_io_tx_meta_desc *p, uint32_t val) { - p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_SHIFT) - & ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_MASK; + p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_META_STORE_SHIFT) & ENA_ETH_IO_TX_META_DESC_META_STORE_MASK; } -static inline uint32_t get_ena_eth_io_tx_meta_desc_meta_store( - const struct ena_eth_io_tx_meta_desc *p) +static inline uint32_t get_ena_eth_io_tx_meta_desc_meta_desc(const struct ena_eth_io_tx_meta_desc *p) { - return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_META_STORE_MASK) - >> ENA_ETH_IO_TX_META_DESC_META_STORE_SHIFT; + return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_META_DESC_MASK) >> ENA_ETH_IO_TX_META_DESC_META_DESC_SHIFT; } -static inline void set_ena_eth_io_tx_meta_desc_meta_store( - struct ena_eth_io_tx_meta_desc *p, uint32_t val) +static inline void set_ena_eth_io_tx_meta_desc_meta_desc(struct ena_eth_io_tx_meta_desc *p, uint32_t val) { - p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_META_STORE_SHIFT) - & ENA_ETH_IO_TX_META_DESC_META_STORE_MASK; + p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_META_DESC_SHIFT) & ENA_ETH_IO_TX_META_DESC_META_DESC_MASK; } -static inline uint32_t get_ena_eth_io_tx_meta_desc_meta_desc( - const struct ena_eth_io_tx_meta_desc *p) +static inline uint32_t get_ena_eth_io_tx_meta_desc_phase(const struct ena_eth_io_tx_meta_desc *p) { - return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_META_DESC_MASK) - >> ENA_ETH_IO_TX_META_DESC_META_DESC_SHIFT; + return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_PHASE_MASK) >> ENA_ETH_IO_TX_META_DESC_PHASE_SHIFT; } -static inline void set_ena_eth_io_tx_meta_desc_meta_desc( - struct ena_eth_io_tx_meta_desc *p, uint32_t val) +static inline void set_ena_eth_io_tx_meta_desc_phase(struct ena_eth_io_tx_meta_desc *p, uint32_t val) { - p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_META_DESC_SHIFT) - & ENA_ETH_IO_TX_META_DESC_META_DESC_MASK; + p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_PHASE_SHIFT) & ENA_ETH_IO_TX_META_DESC_PHASE_MASK; } -static inline uint32_t get_ena_eth_io_tx_meta_desc_phase( - const struct ena_eth_io_tx_meta_desc *p) +static inline uint32_t get_ena_eth_io_tx_meta_desc_first(const struct ena_eth_io_tx_meta_desc *p) { - return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_PHASE_MASK) - >> ENA_ETH_IO_TX_META_DESC_PHASE_SHIFT; + return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_FIRST_MASK) >> ENA_ETH_IO_TX_META_DESC_FIRST_SHIFT; } -static inline void set_ena_eth_io_tx_meta_desc_phase( - struct ena_eth_io_tx_meta_desc *p, uint32_t val) +static inline void set_ena_eth_io_tx_meta_desc_first(struct ena_eth_io_tx_meta_desc *p, uint32_t val) { - p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_PHASE_SHIFT) - & ENA_ETH_IO_TX_META_DESC_PHASE_MASK; + p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_FIRST_SHIFT) & ENA_ETH_IO_TX_META_DESC_FIRST_MASK; } -static inline uint32_t get_ena_eth_io_tx_meta_desc_first( - const struct ena_eth_io_tx_meta_desc *p) +static inline uint32_t get_ena_eth_io_tx_meta_desc_last(const struct ena_eth_io_tx_meta_desc *p) { - return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_FIRST_MASK) - >> ENA_ETH_IO_TX_META_DESC_FIRST_SHIFT; + return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_LAST_MASK) >> ENA_ETH_IO_TX_META_DESC_LAST_SHIFT; } -static inline void set_ena_eth_io_tx_meta_desc_first( - struct ena_eth_io_tx_meta_desc *p, uint32_t val) +static inline void set_ena_eth_io_tx_meta_desc_last(struct ena_eth_io_tx_meta_desc *p, uint32_t val) { - p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_FIRST_SHIFT) - & ENA_ETH_IO_TX_META_DESC_FIRST_MASK; + p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_LAST_SHIFT) & ENA_ETH_IO_TX_META_DESC_LAST_MASK; } -static inline uint32_t get_ena_eth_io_tx_meta_desc_last( - const struct ena_eth_io_tx_meta_desc *p) +static inline uint32_t get_ena_eth_io_tx_meta_desc_comp_req(const struct ena_eth_io_tx_meta_desc *p) { - return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_LAST_MASK) - >> ENA_ETH_IO_TX_META_DESC_LAST_SHIFT; + return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_COMP_REQ_MASK) >> ENA_ETH_IO_TX_META_DESC_COMP_REQ_SHIFT; } -static inline void set_ena_eth_io_tx_meta_desc_last( - struct ena_eth_io_tx_meta_desc *p, uint32_t val) +static inline void set_ena_eth_io_tx_meta_desc_comp_req(struct ena_eth_io_tx_meta_desc *p, uint32_t val) { - p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_LAST_SHIFT) - & ENA_ETH_IO_TX_META_DESC_LAST_MASK; + p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_COMP_REQ_SHIFT) & ENA_ETH_IO_TX_META_DESC_COMP_REQ_MASK; } -static inline uint32_t get_ena_eth_io_tx_meta_desc_comp_req( - const struct ena_eth_io_tx_meta_desc *p) -{ - return (p->len_ctrl & ENA_ETH_IO_TX_META_DESC_COMP_REQ_MASK) - >> ENA_ETH_IO_TX_META_DESC_COMP_REQ_SHIFT; -} - -static inline void set_ena_eth_io_tx_meta_desc_comp_req( - struct ena_eth_io_tx_meta_desc *p, uint32_t val) -{ - p->len_ctrl |= (val << ENA_ETH_IO_TX_META_DESC_COMP_REQ_SHIFT) - & ENA_ETH_IO_TX_META_DESC_COMP_REQ_MASK; -} - -static inline uint32_t get_ena_eth_io_tx_meta_desc_req_id_hi( - const struct ena_eth_io_tx_meta_desc *p) +static inline uint32_t get_ena_eth_io_tx_meta_desc_req_id_hi(const struct ena_eth_io_tx_meta_desc *p) { return p->word1 & ENA_ETH_IO_TX_META_DESC_REQ_ID_HI_MASK; } -static inline void set_ena_eth_io_tx_meta_desc_req_id_hi( - struct ena_eth_io_tx_meta_desc *p, - uint32_t val) +static inline void set_ena_eth_io_tx_meta_desc_req_id_hi(struct ena_eth_io_tx_meta_desc *p, uint32_t val) { p->word1 |= val & ENA_ETH_IO_TX_META_DESC_REQ_ID_HI_MASK; } -static inline uint32_t get_ena_eth_io_tx_meta_desc_l3_hdr_len( - const struct ena_eth_io_tx_meta_desc *p) +static inline uint32_t get_ena_eth_io_tx_meta_desc_l3_hdr_len(const struct ena_eth_io_tx_meta_desc *p) { return p->word2 & ENA_ETH_IO_TX_META_DESC_L3_HDR_LEN_MASK; } -static inline void set_ena_eth_io_tx_meta_desc_l3_hdr_len( - struct ena_eth_io_tx_meta_desc *p, - uint32_t val) +static inline void set_ena_eth_io_tx_meta_desc_l3_hdr_len(struct ena_eth_io_tx_meta_desc *p, uint32_t val) { p->word2 |= val & ENA_ETH_IO_TX_META_DESC_L3_HDR_LEN_MASK; } -static inline uint32_t get_ena_eth_io_tx_meta_desc_l3_hdr_off( - const struct ena_eth_io_tx_meta_desc *p) +static inline uint32_t get_ena_eth_io_tx_meta_desc_l3_hdr_off(const struct ena_eth_io_tx_meta_desc *p) { - return (p->word2 & ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_MASK) - >> ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_SHIFT; + return (p->word2 & ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_MASK) >> ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_SHIFT; } -static inline void set_ena_eth_io_tx_meta_desc_l3_hdr_off( - struct ena_eth_io_tx_meta_desc *p, - uint32_t val) +static inline void set_ena_eth_io_tx_meta_desc_l3_hdr_off(struct ena_eth_io_tx_meta_desc *p, uint32_t val) { - p->word2 |= - (val << ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_SHIFT) - & ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_MASK; + p->word2 |= (val << ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_SHIFT) & ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_MASK; } -static inline uint32_t get_ena_eth_io_tx_meta_desc_l4_hdr_len_in_words( - const struct ena_eth_io_tx_meta_desc *p) +static inline uint32_t get_ena_eth_io_tx_meta_desc_l4_hdr_len_in_words(const struct ena_eth_io_tx_meta_desc *p) { - return (p->word2 & ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_MASK) - >> ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_SHIFT; + return (p->word2 & ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_MASK) >> ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_SHIFT; } -static inline void set_ena_eth_io_tx_meta_desc_l4_hdr_len_in_words( - struct ena_eth_io_tx_meta_desc *p, - uint32_t val) +static inline void set_ena_eth_io_tx_meta_desc_l4_hdr_len_in_words(struct ena_eth_io_tx_meta_desc *p, uint32_t val) { - p->word2 |= - (val << ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_SHIFT) - & ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_MASK; + p->word2 |= (val << ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_SHIFT) & ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_MASK; } -static inline uint32_t get_ena_eth_io_tx_meta_desc_mss_lo( - const struct ena_eth_io_tx_meta_desc *p) +static inline uint32_t get_ena_eth_io_tx_meta_desc_mss_lo(const struct ena_eth_io_tx_meta_desc *p) { - return (p->word2 & ENA_ETH_IO_TX_META_DESC_MSS_LO_MASK) - >> ENA_ETH_IO_TX_META_DESC_MSS_LO_SHIFT; + return (p->word2 & ENA_ETH_IO_TX_META_DESC_MSS_LO_MASK) >> ENA_ETH_IO_TX_META_DESC_MSS_LO_SHIFT; } -static inline void set_ena_eth_io_tx_meta_desc_mss_lo( - struct ena_eth_io_tx_meta_desc *p, - uint32_t val) +static inline void set_ena_eth_io_tx_meta_desc_mss_lo(struct ena_eth_io_tx_meta_desc *p, uint32_t val) { - p->word2 |= - (val << ENA_ETH_IO_TX_META_DESC_MSS_LO_SHIFT) - & ENA_ETH_IO_TX_META_DESC_MSS_LO_MASK; + p->word2 |= (val << ENA_ETH_IO_TX_META_DESC_MSS_LO_SHIFT) & ENA_ETH_IO_TX_META_DESC_MSS_LO_MASK; } -static inline uint8_t get_ena_eth_io_tx_cdesc_phase( - const struct ena_eth_io_tx_cdesc *p) +static inline uint8_t get_ena_eth_io_tx_cdesc_phase(const struct ena_eth_io_tx_cdesc *p) { return p->flags & ENA_ETH_IO_TX_CDESC_PHASE_MASK; } -static inline void set_ena_eth_io_tx_cdesc_phase( - struct ena_eth_io_tx_cdesc *p, - uint8_t val) +static inline void set_ena_eth_io_tx_cdesc_phase(struct ena_eth_io_tx_cdesc *p, uint8_t val) { p->flags |= val & ENA_ETH_IO_TX_CDESC_PHASE_MASK; } -static inline uint8_t get_ena_eth_io_rx_desc_phase( - const struct ena_eth_io_rx_desc *p) +static inline uint8_t get_ena_eth_io_rx_desc_phase(const struct ena_eth_io_rx_desc *p) { return p->ctrl & ENA_ETH_IO_RX_DESC_PHASE_MASK; } -static inline void set_ena_eth_io_rx_desc_phase( - struct ena_eth_io_rx_desc *p, - uint8_t val) +static inline void set_ena_eth_io_rx_desc_phase(struct ena_eth_io_rx_desc *p, uint8_t val) { p->ctrl |= val & ENA_ETH_IO_RX_DESC_PHASE_MASK; } -static inline uint8_t get_ena_eth_io_rx_desc_first( - const struct ena_eth_io_rx_desc *p) +static inline uint8_t get_ena_eth_io_rx_desc_first(const struct ena_eth_io_rx_desc *p) { - return (p->ctrl & ENA_ETH_IO_RX_DESC_FIRST_MASK) - >> ENA_ETH_IO_RX_DESC_FIRST_SHIFT; + return (p->ctrl & ENA_ETH_IO_RX_DESC_FIRST_MASK) >> ENA_ETH_IO_RX_DESC_FIRST_SHIFT; } -static inline void set_ena_eth_io_rx_desc_first( - struct ena_eth_io_rx_desc *p, - uint8_t val) +static inline void set_ena_eth_io_rx_desc_first(struct ena_eth_io_rx_desc *p, uint8_t val) { - p->ctrl |= - (val << ENA_ETH_IO_RX_DESC_FIRST_SHIFT) - & ENA_ETH_IO_RX_DESC_FIRST_MASK; + p->ctrl |= (val << ENA_ETH_IO_RX_DESC_FIRST_SHIFT) & ENA_ETH_IO_RX_DESC_FIRST_MASK; } -static inline uint8_t get_ena_eth_io_rx_desc_last( - const struct ena_eth_io_rx_desc *p) +static inline uint8_t get_ena_eth_io_rx_desc_last(const struct ena_eth_io_rx_desc *p) { - return (p->ctrl & ENA_ETH_IO_RX_DESC_LAST_MASK) - >> ENA_ETH_IO_RX_DESC_LAST_SHIFT; + return (p->ctrl & ENA_ETH_IO_RX_DESC_LAST_MASK) >> ENA_ETH_IO_RX_DESC_LAST_SHIFT; } -static inline void set_ena_eth_io_rx_desc_last( - struct ena_eth_io_rx_desc *p, - uint8_t val) +static inline void set_ena_eth_io_rx_desc_last(struct ena_eth_io_rx_desc *p, uint8_t val) { - p->ctrl |= - (val << ENA_ETH_IO_RX_DESC_LAST_SHIFT) - & ENA_ETH_IO_RX_DESC_LAST_MASK; + p->ctrl |= (val << ENA_ETH_IO_RX_DESC_LAST_SHIFT) & ENA_ETH_IO_RX_DESC_LAST_MASK; } -static inline uint8_t get_ena_eth_io_rx_desc_comp_req( - const struct ena_eth_io_rx_desc *p) +static inline uint8_t get_ena_eth_io_rx_desc_comp_req(const struct ena_eth_io_rx_desc *p) { - return (p->ctrl & ENA_ETH_IO_RX_DESC_COMP_REQ_MASK) - >> ENA_ETH_IO_RX_DESC_COMP_REQ_SHIFT; + return (p->ctrl & ENA_ETH_IO_RX_DESC_COMP_REQ_MASK) >> ENA_ETH_IO_RX_DESC_COMP_REQ_SHIFT; } -static inline void set_ena_eth_io_rx_desc_comp_req( - struct ena_eth_io_rx_desc *p, - uint8_t val) +static inline void set_ena_eth_io_rx_desc_comp_req(struct ena_eth_io_rx_desc *p, uint8_t val) { - p->ctrl |= - (val << ENA_ETH_IO_RX_DESC_COMP_REQ_SHIFT) - & ENA_ETH_IO_RX_DESC_COMP_REQ_MASK; + p->ctrl |= (val << ENA_ETH_IO_RX_DESC_COMP_REQ_SHIFT) & ENA_ETH_IO_RX_DESC_COMP_REQ_MASK; } -static inline uint32_t get_ena_eth_io_rx_cdesc_base_l3_proto_idx( - const struct ena_eth_io_rx_cdesc_base *p) +static inline uint32_t get_ena_eth_io_rx_cdesc_base_l3_proto_idx(const struct ena_eth_io_rx_cdesc_base *p) { return p->status & ENA_ETH_IO_RX_CDESC_BASE_L3_PROTO_IDX_MASK; } -static inline void set_ena_eth_io_rx_cdesc_base_l3_proto_idx( - struct ena_eth_io_rx_cdesc_base *p, - uint32_t val) +static inline void set_ena_eth_io_rx_cdesc_base_l3_proto_idx(struct ena_eth_io_rx_cdesc_base *p, uint32_t val) { p->status |= val & ENA_ETH_IO_RX_CDESC_BASE_L3_PROTO_IDX_MASK; } -static inline uint32_t get_ena_eth_io_rx_cdesc_base_src_vlan_cnt( - const struct ena_eth_io_rx_cdesc_base *p) +static inline uint32_t get_ena_eth_io_rx_cdesc_base_src_vlan_cnt(const struct ena_eth_io_rx_cdesc_base *p) +{ + return (p->status & ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_SHIFT; +} + +static inline void set_ena_eth_io_rx_cdesc_base_src_vlan_cnt(struct ena_eth_io_rx_cdesc_base *p, uint32_t val) +{ + p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_MASK; +} + +static inline uint32_t get_ena_eth_io_rx_cdesc_base_l4_proto_idx(const struct ena_eth_io_rx_cdesc_base *p) { - return (p->status & ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_MASK) - >> ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_SHIFT; + return (p->status & ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_SHIFT; } -static inline void set_ena_eth_io_rx_cdesc_base_src_vlan_cnt( - struct ena_eth_io_rx_cdesc_base *p, - uint32_t val) +static inline void set_ena_eth_io_rx_cdesc_base_l4_proto_idx(struct ena_eth_io_rx_cdesc_base *p, uint32_t val) { - p->status |= - (val << ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_SHIFT) - & ENA_ETH_IO_RX_CDESC_BASE_SRC_VLAN_CNT_MASK; + p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_MASK; } -static inline uint32_t get_ena_eth_io_rx_cdesc_base_l4_proto_idx( - const struct ena_eth_io_rx_cdesc_base *p) +static inline uint32_t get_ena_eth_io_rx_cdesc_base_l3_csum_err(const struct ena_eth_io_rx_cdesc_base *p) { - return (p->status & ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_MASK) - >> ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_SHIFT; + return (p->status & ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_SHIFT; } -static inline void set_ena_eth_io_rx_cdesc_base_l4_proto_idx( - struct ena_eth_io_rx_cdesc_base *p, uint32_t val) +static inline void set_ena_eth_io_rx_cdesc_base_l3_csum_err(struct ena_eth_io_rx_cdesc_base *p, uint32_t val) { - p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_SHIFT) - & ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_MASK; + p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_MASK; } -static inline uint32_t get_ena_eth_io_rx_cdesc_base_l3_csum_err( - const struct ena_eth_io_rx_cdesc_base *p) +static inline uint32_t get_ena_eth_io_rx_cdesc_base_l4_csum_err(const struct ena_eth_io_rx_cdesc_base *p) { - return (p->status & ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_MASK) - >> ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_SHIFT; + return (p->status & ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_SHIFT; } -static inline void set_ena_eth_io_rx_cdesc_base_l3_csum_err( - struct ena_eth_io_rx_cdesc_base *p, uint32_t val) +static inline void set_ena_eth_io_rx_cdesc_base_l4_csum_err(struct ena_eth_io_rx_cdesc_base *p, uint32_t val) { - p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_SHIFT) - & ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_MASK; + p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_MASK; } -static inline uint32_t get_ena_eth_io_rx_cdesc_base_l4_csum_err( - const struct ena_eth_io_rx_cdesc_base *p) +static inline uint32_t get_ena_eth_io_rx_cdesc_base_ipv4_frag(const struct ena_eth_io_rx_cdesc_base *p) { - return (p->status & ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_MASK) - >> ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_SHIFT; + return (p->status & ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_SHIFT; } -static inline void set_ena_eth_io_rx_cdesc_base_l4_csum_err( - struct ena_eth_io_rx_cdesc_base *p, uint32_t val) +static inline void set_ena_eth_io_rx_cdesc_base_ipv4_frag(struct ena_eth_io_rx_cdesc_base *p, uint32_t val) { - p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_SHIFT) - & ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_MASK; + p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_MASK; } -static inline uint32_t get_ena_eth_io_rx_cdesc_base_ipv4_frag( - const struct ena_eth_io_rx_cdesc_base *p) +static inline uint32_t get_ena_eth_io_rx_cdesc_base_l4_csum_checked(const struct ena_eth_io_rx_cdesc_base *p) { - return (p->status & ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_MASK) - >> ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_SHIFT; + return (p->status & ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_CHECKED_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_CHECKED_SHIFT; } -static inline void set_ena_eth_io_rx_cdesc_base_ipv4_frag( - struct ena_eth_io_rx_cdesc_base *p, uint32_t val) +static inline void set_ena_eth_io_rx_cdesc_base_l4_csum_checked(struct ena_eth_io_rx_cdesc_base *p, uint32_t val) { - p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_SHIFT) - & ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_MASK; + p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_CHECKED_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_CHECKED_MASK; } -static inline uint32_t get_ena_eth_io_rx_cdesc_base_phase( - const struct ena_eth_io_rx_cdesc_base *p) +static inline uint32_t get_ena_eth_io_rx_cdesc_base_phase(const struct ena_eth_io_rx_cdesc_base *p) { - return (p->status & ENA_ETH_IO_RX_CDESC_BASE_PHASE_MASK) - >> ENA_ETH_IO_RX_CDESC_BASE_PHASE_SHIFT; + return (p->status & ENA_ETH_IO_RX_CDESC_BASE_PHASE_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_PHASE_SHIFT; } -static inline void set_ena_eth_io_rx_cdesc_base_phase( - struct ena_eth_io_rx_cdesc_base *p, uint32_t val) +static inline void set_ena_eth_io_rx_cdesc_base_phase(struct ena_eth_io_rx_cdesc_base *p, uint32_t val) { - p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_PHASE_SHIFT) - & ENA_ETH_IO_RX_CDESC_BASE_PHASE_MASK; + p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_PHASE_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_PHASE_MASK; } -static inline uint32_t get_ena_eth_io_rx_cdesc_base_l3_csum2( - const struct ena_eth_io_rx_cdesc_base *p) +static inline uint32_t get_ena_eth_io_rx_cdesc_base_l3_csum2(const struct ena_eth_io_rx_cdesc_base *p) { - return (p->status & ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_MASK) - >> ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_SHIFT; + return (p->status & ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_SHIFT; } -static inline void set_ena_eth_io_rx_cdesc_base_l3_csum2( - struct ena_eth_io_rx_cdesc_base *p, uint32_t val) +static inline void set_ena_eth_io_rx_cdesc_base_l3_csum2(struct ena_eth_io_rx_cdesc_base *p, uint32_t val) { - p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_SHIFT) - & ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_MASK; + p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM2_MASK; } -static inline uint32_t get_ena_eth_io_rx_cdesc_base_first( - const struct ena_eth_io_rx_cdesc_base *p) +static inline uint32_t get_ena_eth_io_rx_cdesc_base_first(const struct ena_eth_io_rx_cdesc_base *p) { - return (p->status & ENA_ETH_IO_RX_CDESC_BASE_FIRST_MASK) - >> ENA_ETH_IO_RX_CDESC_BASE_FIRST_SHIFT; + return (p->status & ENA_ETH_IO_RX_CDESC_BASE_FIRST_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_FIRST_SHIFT; } -static inline void set_ena_eth_io_rx_cdesc_base_first( - struct ena_eth_io_rx_cdesc_base *p, uint32_t val) +static inline void set_ena_eth_io_rx_cdesc_base_first(struct ena_eth_io_rx_cdesc_base *p, uint32_t val) { - p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_FIRST_SHIFT) - & ENA_ETH_IO_RX_CDESC_BASE_FIRST_MASK; + p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_FIRST_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_FIRST_MASK; } -static inline uint32_t get_ena_eth_io_rx_cdesc_base_last( - const struct ena_eth_io_rx_cdesc_base *p) +static inline uint32_t get_ena_eth_io_rx_cdesc_base_last(const struct ena_eth_io_rx_cdesc_base *p) { - return (p->status & ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK) - >> ENA_ETH_IO_RX_CDESC_BASE_LAST_SHIFT; + return (p->status & ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_LAST_SHIFT; } -static inline void set_ena_eth_io_rx_cdesc_base_last( - struct ena_eth_io_rx_cdesc_base *p, uint32_t val) +static inline void set_ena_eth_io_rx_cdesc_base_last(struct ena_eth_io_rx_cdesc_base *p, uint32_t val) { - p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_LAST_SHIFT) - & ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK; + p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_LAST_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK; } -static inline uint32_t get_ena_eth_io_rx_cdesc_base_buffer( - const struct ena_eth_io_rx_cdesc_base *p) +static inline uint32_t get_ena_eth_io_rx_cdesc_base_buffer(const struct ena_eth_io_rx_cdesc_base *p) { - return (p->status & ENA_ETH_IO_RX_CDESC_BASE_BUFFER_MASK) - >> ENA_ETH_IO_RX_CDESC_BASE_BUFFER_SHIFT; + return (p->status & ENA_ETH_IO_RX_CDESC_BASE_BUFFER_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_BUFFER_SHIFT; } -static inline void set_ena_eth_io_rx_cdesc_base_buffer( - struct ena_eth_io_rx_cdesc_base *p, uint32_t val) +static inline void set_ena_eth_io_rx_cdesc_base_buffer(struct ena_eth_io_rx_cdesc_base *p, uint32_t val) { - p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_BUFFER_SHIFT) - & ENA_ETH_IO_RX_CDESC_BASE_BUFFER_MASK; + p->status |= (val << ENA_ETH_IO_RX_CDESC_BASE_BUFFER_SHIFT) & ENA_ETH_IO_RX_CDESC_BASE_BUFFER_MASK; } -static inline uint32_t get_ena_eth_io_intr_reg_rx_intr_delay( - const struct ena_eth_io_intr_reg *p) +static inline uint32_t get_ena_eth_io_intr_reg_rx_intr_delay(const struct ena_eth_io_intr_reg *p) { return p->intr_control & ENA_ETH_IO_INTR_REG_RX_INTR_DELAY_MASK; } -static inline void set_ena_eth_io_intr_reg_rx_intr_delay( - struct ena_eth_io_intr_reg *p, uint32_t val) +static inline void set_ena_eth_io_intr_reg_rx_intr_delay(struct ena_eth_io_intr_reg *p, uint32_t val) { p->intr_control |= val & ENA_ETH_IO_INTR_REG_RX_INTR_DELAY_MASK; } -static inline uint32_t get_ena_eth_io_intr_reg_tx_intr_delay( - const struct ena_eth_io_intr_reg *p) +static inline uint32_t get_ena_eth_io_intr_reg_tx_intr_delay(const struct ena_eth_io_intr_reg *p) { - return (p->intr_control & ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_MASK) - >> ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_SHIFT; + return (p->intr_control & ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_MASK) >> ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_SHIFT; } -static inline void set_ena_eth_io_intr_reg_tx_intr_delay( - struct ena_eth_io_intr_reg *p, uint32_t val) +static inline void set_ena_eth_io_intr_reg_tx_intr_delay(struct ena_eth_io_intr_reg *p, uint32_t val) { - p->intr_control |= (val << ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_SHIFT) - & ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_MASK; + p->intr_control |= (val << ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_SHIFT) & ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_MASK; } -static inline uint32_t get_ena_eth_io_intr_reg_intr_unmask( - const struct ena_eth_io_intr_reg *p) +static inline uint32_t get_ena_eth_io_intr_reg_intr_unmask(const struct ena_eth_io_intr_reg *p) { - return (p->intr_control & ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK) - >> ENA_ETH_IO_INTR_REG_INTR_UNMASK_SHIFT; + return (p->intr_control & ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK) >> ENA_ETH_IO_INTR_REG_INTR_UNMASK_SHIFT; } -static inline void set_ena_eth_io_intr_reg_intr_unmask( - struct ena_eth_io_intr_reg *p, uint32_t val) +static inline void set_ena_eth_io_intr_reg_intr_unmask(struct ena_eth_io_intr_reg *p, uint32_t val) { - p->intr_control |= (val << ENA_ETH_IO_INTR_REG_INTR_UNMASK_SHIFT) - & ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK; + p->intr_control |= (val << ENA_ETH_IO_INTR_REG_INTR_UNMASK_SHIFT) & ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK; } -static inline uint32_t get_ena_eth_io_numa_node_cfg_reg_numa( - const struct ena_eth_io_numa_node_cfg_reg *p) +static inline uint32_t get_ena_eth_io_numa_node_cfg_reg_numa(const struct ena_eth_io_numa_node_cfg_reg *p) { return p->numa_cfg & ENA_ETH_IO_NUMA_NODE_CFG_REG_NUMA_MASK; } -static inline void set_ena_eth_io_numa_node_cfg_reg_numa( - struct ena_eth_io_numa_node_cfg_reg *p, uint32_t val) +static inline void set_ena_eth_io_numa_node_cfg_reg_numa(struct ena_eth_io_numa_node_cfg_reg *p, uint32_t val) { p->numa_cfg |= val & ENA_ETH_IO_NUMA_NODE_CFG_REG_NUMA_MASK; } -static inline uint32_t get_ena_eth_io_numa_node_cfg_reg_enabled( - const struct ena_eth_io_numa_node_cfg_reg *p) +static inline uint32_t get_ena_eth_io_numa_node_cfg_reg_enabled(const struct ena_eth_io_numa_node_cfg_reg *p) { - return (p->numa_cfg & ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_MASK) - >> ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_SHIFT; + return (p->numa_cfg & ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_MASK) >> ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_SHIFT; } -static inline void set_ena_eth_io_numa_node_cfg_reg_enabled( - struct ena_eth_io_numa_node_cfg_reg *p, uint32_t val) +static inline void set_ena_eth_io_numa_node_cfg_reg_enabled(struct ena_eth_io_numa_node_cfg_reg *p, uint32_t val) { - p->numa_cfg |= (val << ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_SHIFT) - & ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_MASK; + p->numa_cfg |= (val << ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_SHIFT) & ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_MASK; } -#endif /* !defined(ENA_DEFS_LINUX_MAINLINE) */ -#endif /*_ENA_ETH_IO_H_ */ +#endif /* !defined(DEFS_LINUX_MAINLINE) */ +#endif /* _ENA_ETH_IO_H_ */ Index: /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_defs/ena_gen_info.h =================================================================== --- /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_defs/ena_gen_info.h (revision 38965) +++ /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_defs/ena_gen_info.h (working copy) @@ -1,35 +1,6 @@ -/*- -* BSD LICENSE -* -* Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. -* All rights reserved. -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following conditions -* are met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above copyright -* notice, this list of conditions and the following disclaimer in -* the documentation and/or other materials provided with the -* distribution. -* * Neither the name of copyright holder nor the names of its -* contributors may be used to endorse or promote products derived -* from this software without specific prior written permission. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -#define ENA_GEN_DATE "Sun Jun 5 10:24:39 IDT 2016" -#define ENA_GEN_COMMIT "17146ed" +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. + * All rights reserved. + */ +#define ENA_GEN_DATE "Fri Sep 18 17:09:00 IDT 2020" +#define ENA_GEN_COMMIT "0f80d82" Index: /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_defs/ena_includes.h =================================================================== --- /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_defs/ena_includes.h (revision 38965) +++ /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_defs/ena_includes.h (working copy) @@ -1,39 +1,9 @@ -/*- -* BSD LICENSE -* -* Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. -* All rights reserved. -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following conditions -* are met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above copyright -* notice, this list of conditions and the following disclaimer in -* the documentation and/or other materials provided with the -* distribution. -* * Neither the name of copyright holder nor the names of its -* contributors may be used to endorse or promote products derived -* from this software without specific prior written permission. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2019 Amazon.com, Inc. or its affiliates. + * All rights reserved. + */ #include "ena_common_defs.h" #include "ena_regs_defs.h" #include "ena_admin_defs.h" #include "ena_eth_io_defs.h" -#include "ena_efa_admin_defs.h" -#include "ena_efa_io_defs.h" Index: /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_defs/ena_regs_defs.h =================================================================== --- /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_defs/ena_regs_defs.h (revision 38965) +++ /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_defs/ena_regs_defs.h (working copy) @@ -1,135 +1,131 @@ -/*- -* BSD LICENSE -* -* Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. -* All rights reserved. -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following conditions -* are met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above copyright -* notice, this list of conditions and the following disclaimer in -* the documentation and/or other materials provided with the -* distribution. -* * Neither the name of copyright holder nor the names of its -* contributors may be used to endorse or promote products derived -* from this software without specific prior written permission. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. + * All rights reserved. + */ #ifndef _ENA_REGS_H_ #define _ENA_REGS_H_ +enum ena_regs_reset_reason_types { + ENA_REGS_RESET_NORMAL = 0, + ENA_REGS_RESET_KEEP_ALIVE_TO = 1, + ENA_REGS_RESET_ADMIN_TO = 2, + ENA_REGS_RESET_MISS_TX_CMPL = 3, + ENA_REGS_RESET_INV_RX_REQ_ID = 4, + ENA_REGS_RESET_INV_TX_REQ_ID = 5, + ENA_REGS_RESET_TOO_MANY_RX_DESCS = 6, + ENA_REGS_RESET_INIT_ERR = 7, + ENA_REGS_RESET_DRIVER_INVALID_STATE = 8, + ENA_REGS_RESET_OS_TRIGGER = 9, + ENA_REGS_RESET_OS_NETDEV_WD = 10, + ENA_REGS_RESET_SHUTDOWN = 11, + ENA_REGS_RESET_USER_TRIGGER = 12, + ENA_REGS_RESET_GENERIC = 13, + ENA_REGS_RESET_MISS_INTERRUPT = 14, + ENA_REGS_RESET_LAST, +}; + /* ena_registers offsets */ -#define ENA_REGS_VERSION_OFF 0x0 -#define ENA_REGS_CONTROLLER_VERSION_OFF 0x4 -#define ENA_REGS_CAPS_OFF 0x8 -#define ENA_REGS_CAPS_EXT_OFF 0xc -#define ENA_REGS_AQ_BASE_LO_OFF 0x10 -#define ENA_REGS_AQ_BASE_HI_OFF 0x14 -#define ENA_REGS_AQ_CAPS_OFF 0x18 -#define ENA_REGS_ACQ_BASE_LO_OFF 0x20 -#define ENA_REGS_ACQ_BASE_HI_OFF 0x24 -#define ENA_REGS_ACQ_CAPS_OFF 0x28 -#define ENA_REGS_AQ_DB_OFF 0x2c -#define ENA_REGS_ACQ_TAIL_OFF 0x30 -#define ENA_REGS_AENQ_CAPS_OFF 0x34 -#define ENA_REGS_AENQ_BASE_LO_OFF 0x38 -#define ENA_REGS_AENQ_BASE_HI_OFF 0x3c -#define ENA_REGS_AENQ_HEAD_DB_OFF 0x40 -#define ENA_REGS_AENQ_TAIL_OFF 0x44 -#define ENA_REGS_INTR_MASK_OFF 0x4c -#define ENA_REGS_DEV_CTL_OFF 0x54 -#define ENA_REGS_DEV_STS_OFF 0x58 -#define ENA_REGS_MMIO_REG_READ_OFF 0x5c -#define ENA_REGS_MMIO_RESP_LO_OFF 0x60 -#define ENA_REGS_MMIO_RESP_HI_OFF 0x64 -#define ENA_REGS_RSS_IND_ENTRY_UPDATE_OFF 0x68 + +/* 0 base */ +#define ENA_REGS_VERSION_OFF 0x0 +#define ENA_REGS_CONTROLLER_VERSION_OFF 0x4 +#define ENA_REGS_CAPS_OFF 0x8 +#define ENA_REGS_CAPS_EXT_OFF 0xc +#define ENA_REGS_AQ_BASE_LO_OFF 0x10 +#define ENA_REGS_AQ_BASE_HI_OFF 0x14 +#define ENA_REGS_AQ_CAPS_OFF 0x18 +#define ENA_REGS_ACQ_BASE_LO_OFF 0x20 +#define ENA_REGS_ACQ_BASE_HI_OFF 0x24 +#define ENA_REGS_ACQ_CAPS_OFF 0x28 +#define ENA_REGS_AQ_DB_OFF 0x2c +#define ENA_REGS_ACQ_TAIL_OFF 0x30 +#define ENA_REGS_AENQ_CAPS_OFF 0x34 +#define ENA_REGS_AENQ_BASE_LO_OFF 0x38 +#define ENA_REGS_AENQ_BASE_HI_OFF 0x3c +#define ENA_REGS_AENQ_HEAD_DB_OFF 0x40 +#define ENA_REGS_AENQ_TAIL_OFF 0x44 +#define ENA_REGS_INTR_MASK_OFF 0x4c +#define ENA_REGS_DEV_CTL_OFF 0x54 +#define ENA_REGS_DEV_STS_OFF 0x58 +#define ENA_REGS_MMIO_REG_READ_OFF 0x5c +#define ENA_REGS_MMIO_RESP_LO_OFF 0x60 +#define ENA_REGS_MMIO_RESP_HI_OFF 0x64 +#define ENA_REGS_RSS_IND_ENTRY_UPDATE_OFF 0x68 /* version register */ -#define ENA_REGS_VERSION_MINOR_VERSION_MASK 0xff -#define ENA_REGS_VERSION_MAJOR_VERSION_SHIFT 8 -#define ENA_REGS_VERSION_MAJOR_VERSION_MASK 0xff00 +#define ENA_REGS_VERSION_MINOR_VERSION_MASK 0xff +#define ENA_REGS_VERSION_MAJOR_VERSION_SHIFT 8 +#define ENA_REGS_VERSION_MAJOR_VERSION_MASK 0xff00 /* controller_version register */ -#define ENA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK 0xff -#define ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT 8 -#define ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK 0xff00 -#define ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT 16 -#define ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK 0xff0000 -#define ENA_REGS_CONTROLLER_VERSION_IMPL_ID_SHIFT 24 -#define ENA_REGS_CONTROLLER_VERSION_IMPL_ID_MASK 0xff000000 +#define ENA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK 0xff +#define ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT 8 +#define ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK 0xff00 +#define ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT 16 +#define ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK 0xff0000 +#define ENA_REGS_CONTROLLER_VERSION_IMPL_ID_SHIFT 24 +#define ENA_REGS_CONTROLLER_VERSION_IMPL_ID_MASK 0xff000000 /* caps register */ -#define ENA_REGS_CAPS_CONTIGUOUS_QUEUE_REQUIRED_MASK 0x1 -#define ENA_REGS_CAPS_RESET_TIMEOUT_SHIFT 1 -#define ENA_REGS_CAPS_RESET_TIMEOUT_MASK 0x3e -#define ENA_REGS_CAPS_DMA_ADDR_WIDTH_SHIFT 8 -#define ENA_REGS_CAPS_DMA_ADDR_WIDTH_MASK 0xff00 +#define ENA_REGS_CAPS_CONTIGUOUS_QUEUE_REQUIRED_MASK 0x1 +#define ENA_REGS_CAPS_RESET_TIMEOUT_SHIFT 1 +#define ENA_REGS_CAPS_RESET_TIMEOUT_MASK 0x3e +#define ENA_REGS_CAPS_DMA_ADDR_WIDTH_SHIFT 8 +#define ENA_REGS_CAPS_DMA_ADDR_WIDTH_MASK 0xff00 +#define ENA_REGS_CAPS_ADMIN_CMD_TO_SHIFT 16 +#define ENA_REGS_CAPS_ADMIN_CMD_TO_MASK 0xf0000 /* aq_caps register */ -#define ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK 0xffff -#define ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_SHIFT 16 -#define ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_MASK 0xffff0000 +#define ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK 0xffff +#define ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_SHIFT 16 +#define ENA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_MASK 0xffff0000 /* acq_caps register */ -#define ENA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK 0xffff -#define ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_SHIFT 16 -#define ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_MASK 0xffff0000 +#define ENA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK 0xffff +#define ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_SHIFT 16 +#define ENA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_MASK 0xffff0000 /* aenq_caps register */ -#define ENA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK 0xffff -#define ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_SHIFT 16 -#define ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_MASK 0xffff0000 +#define ENA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK 0xffff +#define ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_SHIFT 16 +#define ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_MASK 0xffff0000 /* dev_ctl register */ -#define ENA_REGS_DEV_CTL_DEV_RESET_MASK 0x1 -#define ENA_REGS_DEV_CTL_AQ_RESTART_SHIFT 1 -#define ENA_REGS_DEV_CTL_AQ_RESTART_MASK 0x2 -#define ENA_REGS_DEV_CTL_QUIESCENT_SHIFT 2 -#define ENA_REGS_DEV_CTL_QUIESCENT_MASK 0x4 -#define ENA_REGS_DEV_CTL_IO_RESUME_SHIFT 3 -#define ENA_REGS_DEV_CTL_IO_RESUME_MASK 0x8 +#define ENA_REGS_DEV_CTL_DEV_RESET_MASK 0x1 +#define ENA_REGS_DEV_CTL_AQ_RESTART_SHIFT 1 +#define ENA_REGS_DEV_CTL_AQ_RESTART_MASK 0x2 +#define ENA_REGS_DEV_CTL_QUIESCENT_SHIFT 2 +#define ENA_REGS_DEV_CTL_QUIESCENT_MASK 0x4 +#define ENA_REGS_DEV_CTL_IO_RESUME_SHIFT 3 +#define ENA_REGS_DEV_CTL_IO_RESUME_MASK 0x8 +#define ENA_REGS_DEV_CTL_RESET_REASON_SHIFT 28 +#define ENA_REGS_DEV_CTL_RESET_REASON_MASK 0xf0000000 /* dev_sts register */ -#define ENA_REGS_DEV_STS_READY_MASK 0x1 -#define ENA_REGS_DEV_STS_AQ_RESTART_IN_PROGRESS_SHIFT 1 -#define ENA_REGS_DEV_STS_AQ_RESTART_IN_PROGRESS_MASK 0x2 -#define ENA_REGS_DEV_STS_AQ_RESTART_FINISHED_SHIFT 2 -#define ENA_REGS_DEV_STS_AQ_RESTART_FINISHED_MASK 0x4 -#define ENA_REGS_DEV_STS_RESET_IN_PROGRESS_SHIFT 3 -#define ENA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK 0x8 -#define ENA_REGS_DEV_STS_RESET_FINISHED_SHIFT 4 -#define ENA_REGS_DEV_STS_RESET_FINISHED_MASK 0x10 -#define ENA_REGS_DEV_STS_FATAL_ERROR_SHIFT 5 -#define ENA_REGS_DEV_STS_FATAL_ERROR_MASK 0x20 -#define ENA_REGS_DEV_STS_QUIESCENT_STATE_IN_PROGRESS_SHIFT 6 -#define ENA_REGS_DEV_STS_QUIESCENT_STATE_IN_PROGRESS_MASK 0x40 -#define ENA_REGS_DEV_STS_QUIESCENT_STATE_ACHIEVED_SHIFT 7 -#define ENA_REGS_DEV_STS_QUIESCENT_STATE_ACHIEVED_MASK 0x80 +#define ENA_REGS_DEV_STS_READY_MASK 0x1 +#define ENA_REGS_DEV_STS_AQ_RESTART_IN_PROGRESS_SHIFT 1 +#define ENA_REGS_DEV_STS_AQ_RESTART_IN_PROGRESS_MASK 0x2 +#define ENA_REGS_DEV_STS_AQ_RESTART_FINISHED_SHIFT 2 +#define ENA_REGS_DEV_STS_AQ_RESTART_FINISHED_MASK 0x4 +#define ENA_REGS_DEV_STS_RESET_IN_PROGRESS_SHIFT 3 +#define ENA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK 0x8 +#define ENA_REGS_DEV_STS_RESET_FINISHED_SHIFT 4 +#define ENA_REGS_DEV_STS_RESET_FINISHED_MASK 0x10 +#define ENA_REGS_DEV_STS_FATAL_ERROR_SHIFT 5 +#define ENA_REGS_DEV_STS_FATAL_ERROR_MASK 0x20 +#define ENA_REGS_DEV_STS_QUIESCENT_STATE_IN_PROGRESS_SHIFT 6 +#define ENA_REGS_DEV_STS_QUIESCENT_STATE_IN_PROGRESS_MASK 0x40 +#define ENA_REGS_DEV_STS_QUIESCENT_STATE_ACHIEVED_SHIFT 7 +#define ENA_REGS_DEV_STS_QUIESCENT_STATE_ACHIEVED_MASK 0x80 /* mmio_reg_read register */ -#define ENA_REGS_MMIO_REG_READ_REQ_ID_MASK 0xffff -#define ENA_REGS_MMIO_REG_READ_REG_OFF_SHIFT 16 -#define ENA_REGS_MMIO_REG_READ_REG_OFF_MASK 0xffff0000 +#define ENA_REGS_MMIO_REG_READ_REQ_ID_MASK 0xffff +#define ENA_REGS_MMIO_REG_READ_REG_OFF_SHIFT 16 +#define ENA_REGS_MMIO_REG_READ_REG_OFF_MASK 0xffff0000 /* rss_ind_entry_update register */ -#define ENA_REGS_RSS_IND_ENTRY_UPDATE_INDEX_MASK 0xffff -#define ENA_REGS_RSS_IND_ENTRY_UPDATE_CQ_IDX_SHIFT 16 -#define ENA_REGS_RSS_IND_ENTRY_UPDATE_CQ_IDX_MASK 0xffff0000 +#define ENA_REGS_RSS_IND_ENTRY_UPDATE_INDEX_MASK 0xffff +#define ENA_REGS_RSS_IND_ENTRY_UPDATE_CQ_IDX_SHIFT 16 +#define ENA_REGS_RSS_IND_ENTRY_UPDATE_CQ_IDX_MASK 0xffff0000 -#endif /*_ENA_REGS_H_ */ +#endif /* _ENA_REGS_H_ */ Index: /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_eth_com.h =================================================================== --- /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_eth_com.h (revision 38965) +++ /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_eth_com.h (working copy) @@ -1,35 +1,7 @@ -/*- -* BSD LICENSE -* -* Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. -* All rights reserved. -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following conditions -* are met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above copyright -* notice, this list of conditions and the following disclaimer in -* the documentation and/or other materials provided with the -* distribution. -* * Neither the name of copyright holder nor the names of its -* contributors may be used to endorse or promote products derived -* from this software without specific prior written permission. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. + * All rights reserved. + */ #ifndef ENA_ETH_COM_H_ #define ENA_ETH_COM_H_ @@ -71,11 +43,13 @@ enum ena_eth_io_l4_proto_index l4_proto; bool l3_csum_err; bool l4_csum_err; + u8 l4_csum_checked; /* fragmented packet */ bool frag; u32 hash; u16 descs; int max_bufs; + u8 pkt_offset; }; int ena_com_prepare_tx(struct ena_com_io_sq *io_sq, @@ -90,15 +64,15 @@ struct ena_com_buf *ena_buf, u16 req_id); -int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq, u16 *req_id); +bool ena_com_cq_empty(struct ena_com_io_cq *io_cq); static inline void ena_com_unmask_intr(struct ena_com_io_cq *io_cq, struct ena_eth_io_intr_reg *intr_reg) { - ENA_REG_WRITE32(intr_reg->intr_control, io_cq->unmask_reg); + ENA_REG_WRITE32(io_cq->bus, intr_reg->intr_control, io_cq->unmask_reg); } -static inline int ena_com_sq_empty_space(struct ena_com_io_sq *io_sq) +static inline int ena_com_free_q_entries(struct ena_com_io_sq *io_sq) { u16 tail, next_to_comp, cnt; @@ -109,16 +83,90 @@ return io_sq->q_depth - 1 - cnt; } -static inline int ena_com_write_sq_doorbell(struct ena_com_io_sq *io_sq) +/* Check if the submission queue has enough space to hold required_buffers */ +static inline bool ena_com_sq_have_enough_space(struct ena_com_io_sq *io_sq, + u16 required_buffers) { - u16 tail; + int temp; - tail = io_sq->tail; + if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) + return ena_com_free_q_entries(io_sq) >= required_buffers; + + /* This calculation doesn't need to be 100% accurate. So to reduce + * the calculation overhead just Subtract 2 lines from the free descs + * (one for the header line and one to compensate the devision + * down calculation. + */ + temp = required_buffers / io_sq->llq_info.descs_per_entry + 2; + + return ena_com_free_q_entries(io_sq) > temp; +} + +static inline bool ena_com_meta_desc_changed(struct ena_com_io_sq *io_sq, + struct ena_com_tx_ctx *ena_tx_ctx) +{ + if (!ena_tx_ctx->meta_valid) + return false; + + return !!memcmp(&io_sq->cached_tx_meta, + &ena_tx_ctx->ena_meta, + sizeof(struct ena_com_tx_meta)); +} + +static inline bool is_llq_max_tx_burst_exists(struct ena_com_io_sq *io_sq) +{ + return (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) && + io_sq->llq_info.max_entries_in_tx_burst > 0; +} + +static inline bool ena_com_is_doorbell_needed(struct ena_com_io_sq *io_sq, + struct ena_com_tx_ctx *ena_tx_ctx) +{ + struct ena_com_llq_info *llq_info; + int descs_after_first_entry; + int num_entries_needed = 1; + u16 num_descs; + + if (!is_llq_max_tx_burst_exists(io_sq)) + return false; + + llq_info = &io_sq->llq_info; + num_descs = ena_tx_ctx->num_bufs; + + if (llq_info->disable_meta_caching || + unlikely(ena_com_meta_desc_changed(io_sq, ena_tx_ctx))) + ++num_descs; + + if (num_descs > llq_info->descs_num_before_header) { + descs_after_first_entry = num_descs - llq_info->descs_num_before_header; + num_entries_needed += DIV_ROUND_UP(descs_after_first_entry, + llq_info->descs_per_entry); + } + + ena_trc_dbg(ena_com_io_sq_to_ena_dev(io_sq), + "Queue: %d num_descs: %d num_entries_needed: %d\n", + io_sq->qid, num_descs, num_entries_needed); + + return num_entries_needed > io_sq->entries_in_tx_burst_left; +} - ena_trc_dbg("write submission queue doorbell for queue: %d tail: %d\n", +static inline int ena_com_write_sq_doorbell(struct ena_com_io_sq *io_sq) +{ + u16 max_entries_in_tx_burst = io_sq->llq_info.max_entries_in_tx_burst; + u16 tail = io_sq->tail; + + ena_trc_dbg(ena_com_io_sq_to_ena_dev(io_sq), + "Write submission queue doorbell for queue: %d tail: %d\n", io_sq->qid, tail); - ENA_REG_WRITE32(tail, io_sq->db_addr); + ENA_REG_WRITE32(io_sq->bus, tail, io_sq->db_addr); + + if (is_llq_max_tx_burst_exists(io_sq)) { + ena_trc_dbg(ena_com_io_sq_to_ena_dev(io_sq), + "Reset available entries in tx burst for queue %d to %d\n", + io_sq->qid, max_entries_in_tx_burst); + io_sq->entries_in_tx_burst_left = max_entries_in_tx_burst; + } return 0; } @@ -128,15 +176,18 @@ u16 unreported_comp, head; bool need_update; - head = io_cq->head; - unreported_comp = head - io_cq->last_head_update; - need_update = unreported_comp > (io_cq->q_depth / ENA_COMP_HEAD_THRESH); - - if (io_cq->cq_head_db_reg && need_update) { - ena_trc_dbg("Write completion queue doorbell for queue %d: head: %d\n", - io_cq->qid, head); - ENA_REG_WRITE32(head, io_cq->cq_head_db_reg); - io_cq->last_head_update = head; + if (unlikely(io_cq->cq_head_db_reg)) { + head = io_cq->head; + unreported_comp = head - io_cq->last_head_update; + need_update = unreported_comp > (io_cq->q_depth / ENA_COMP_HEAD_THRESH); + + if (unlikely(need_update)) { + ena_trc_dbg(ena_com_io_cq_to_ena_dev(io_cq), + "Write completion queue doorbell for queue %d: head: %d\n", + io_cq->qid, head); + ENA_REG_WRITE32(io_cq->bus, head, io_cq->cq_head_db_reg); + io_cq->last_head_update = head; + } } return 0; @@ -153,7 +204,7 @@ numa_cfg.numa_cfg = (numa_node & ENA_ETH_IO_NUMA_NODE_CFG_REG_NUMA_MASK) | ENA_ETH_IO_NUMA_NODE_CFG_REG_ENABLED_MASK; - ENA_REG_WRITE32(numa_cfg.numa_cfg, io_cq->numa_node_cfg_reg); + ENA_REG_WRITE32(io_cq->bus, numa_cfg.numa_cfg, io_cq->numa_node_cfg_reg); } static inline void ena_com_comp_ack(struct ena_com_io_sq *io_sq, u16 elem) @@ -161,6 +212,51 @@ io_sq->next_to_comp += elem; } +static inline void ena_com_cq_inc_head(struct ena_com_io_cq *io_cq) +{ + io_cq->head++; + + /* Switch phase bit in case of wrap around */ + if (unlikely((io_cq->head & (io_cq->q_depth - 1)) == 0)) + io_cq->phase ^= 1; +} + +static inline int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq, + u16 *req_id) +{ + u8 expected_phase, cdesc_phase; + struct ena_eth_io_tx_cdesc *cdesc; + u16 masked_head; + + masked_head = io_cq->head & (io_cq->q_depth - 1); + expected_phase = io_cq->phase; + + cdesc = (struct ena_eth_io_tx_cdesc *) + ((uintptr_t)io_cq->cdesc_addr.virt_addr + + (masked_head * io_cq->cdesc_entry_size_in_bytes)); + + /* When the current completion descriptor phase isn't the same as the + * expected, it mean that the device still didn't update + * this completion. + */ + cdesc_phase = READ_ONCE16(cdesc->flags) & ENA_ETH_IO_TX_CDESC_PHASE_MASK; + if (cdesc_phase != expected_phase) + return ENA_COM_TRY_AGAIN; + + dma_rmb(); + + *req_id = READ_ONCE16(cdesc->req_id); + if (unlikely(*req_id >= io_cq->q_depth)) { + ena_trc_err(ena_com_io_cq_to_ena_dev(io_cq), + "Invalid req id %d\n", cdesc->req_id); + return ENA_COM_INVAL; + } + + ena_com_cq_inc_head(io_cq); + + return 0; +} + #if defined(__cplusplus) } #endif Index: /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_eth_com.c =================================================================== --- /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_eth_com.c (revision 38965) +++ /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_eth_com.c (working copy) @@ -1,39 +1,11 @@ -/*- -* BSD LICENSE -* -* Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. -* All rights reserved. -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following conditions -* are met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above copyright -* notice, this list of conditions and the following disclaimer in -* the documentation and/or other materials provided with the -* distribution. -* * Neither the name of copyright holder nor the names of its -* contributors may be used to endorse or promote products derived -* from this software without specific prior written permission. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. + * All rights reserved. + */ #include "ena_eth_com.h" -static inline struct ena_eth_io_rx_cdesc_base *ena_com_get_next_rx_cdesc( +static struct ena_eth_io_rx_cdesc_base *ena_com_get_next_rx_cdesc( struct ena_com_io_cq *io_cq) { struct ena_eth_io_rx_cdesc_base *cdesc; @@ -43,29 +15,24 @@ head_masked = io_cq->head & (io_cq->q_depth - 1); expected_phase = io_cq->phase; - cdesc = (struct ena_eth_io_rx_cdesc_base *) - ((unsigned char *)io_cq->cdesc_addr.virt_addr + cdesc = (struct ena_eth_io_rx_cdesc_base *)(io_cq->cdesc_addr.virt_addr + (head_masked * io_cq->cdesc_entry_size_in_bytes)); - desc_phase = (cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_PHASE_MASK) >> + desc_phase = (READ_ONCE32(cdesc->status) & ENA_ETH_IO_RX_CDESC_BASE_PHASE_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_PHASE_SHIFT; if (desc_phase != expected_phase) return NULL; - return cdesc; -} - -static inline void ena_com_cq_inc_head(struct ena_com_io_cq *io_cq) -{ - io_cq->head++; + /* Make sure we read the rest of the descriptor after the phase bit + * has been read + */ + dma_rmb(); - /* Switch phase bit in case of wrap around */ - if (unlikely((io_cq->head & (io_cq->q_depth - 1)) == 0)) - io_cq->phase ^= 1; + return cdesc; } -static inline void *get_sq_desc(struct ena_com_io_sq *io_sq) +static void *get_sq_desc_regular_queue(struct ena_com_io_sq *io_sq) { u16 tail_masked; u32 offset; @@ -74,62 +41,198 @@ offset = tail_masked * io_sq->desc_entry_size; - return (unsigned char *)io_sq->desc_addr.virt_addr + offset; + return (void *)((uintptr_t)io_sq->desc_addr.virt_addr + offset); } -static inline void ena_com_copy_curr_sq_desc_to_dev(struct ena_com_io_sq *io_sq) +static int ena_com_write_bounce_buffer_to_dev(struct ena_com_io_sq *io_sq, + u8 *bounce_buffer) { - u16 tail_masked = io_sq->tail & (io_sq->q_depth - 1); - u32 offset = tail_masked * io_sq->desc_entry_size; + struct ena_com_llq_info *llq_info = &io_sq->llq_info; - /* In case this queue isn't a LLQ */ - if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) - return; + u16 dst_tail_mask; + u32 dst_offset; - memcpy_toio((unsigned char *)io_sq->desc_addr.pbuf_dev_addr + offset, - (unsigned char *)io_sq->desc_addr.virt_addr + offset, - io_sq->desc_entry_size); -} + dst_tail_mask = io_sq->tail & (io_sq->q_depth - 1); + dst_offset = dst_tail_mask * llq_info->desc_list_entry_size; + + if (is_llq_max_tx_burst_exists(io_sq)) { + if (unlikely(!io_sq->entries_in_tx_burst_left)) { + ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq), + "Error: trying to send more packets than tx burst allows\n"); + return ENA_COM_NO_SPACE; + } + + io_sq->entries_in_tx_burst_left--; + ena_trc_dbg(ena_com_io_sq_to_ena_dev(io_sq), + "Decreasing entries_in_tx_burst_left of queue %d to %d\n", + io_sq->qid, io_sq->entries_in_tx_burst_left); + } + + /* Make sure everything was written into the bounce buffer before + * writing the bounce buffer to the device + */ + wmb(); + + /* The line is completed. Copy it to dev */ + ENA_MEMCPY_TO_DEVICE_64(io_sq->desc_addr.pbuf_dev_addr + dst_offset, + bounce_buffer, + llq_info->desc_list_entry_size); -static inline void ena_com_sq_update_tail(struct ena_com_io_sq *io_sq) -{ io_sq->tail++; /* Switch phase bit in case of wrap around */ if (unlikely((io_sq->tail & (io_sq->q_depth - 1)) == 0)) io_sq->phase ^= 1; + + return ENA_COM_OK; } -static inline int ena_com_write_header(struct ena_com_io_sq *io_sq, - u8 *head_src, u16 header_len) +static int ena_com_write_header_to_bounce(struct ena_com_io_sq *io_sq, + u8 *header_src, + u16 header_len) { - u16 tail_masked = io_sq->tail & (io_sq->q_depth - 1); - u8 __iomem *dev_head_addr = - io_sq->header_addr + (tail_masked * io_sq->tx_max_header_size); + struct ena_com_llq_pkt_ctrl *pkt_ctrl = &io_sq->llq_buf_ctrl; + struct ena_com_llq_info *llq_info = &io_sq->llq_info; + u8 *bounce_buffer = pkt_ctrl->curr_bounce_buf; + u16 header_offset; - if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) + if (unlikely(io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)) return 0; - if (unlikely(!io_sq->header_addr)) { - ena_trc_err("Push buffer header ptr is NULL\n"); - return ENA_COM_INVAL; + header_offset = + llq_info->descs_num_before_header * io_sq->desc_entry_size; + + if (unlikely((header_offset + header_len) > llq_info->desc_list_entry_size)) { + ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq), + "Trying to write header larger than llq entry can accommodate\n"); + return ENA_COM_FAULT; + } + + if (unlikely(!bounce_buffer)) { + ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq), + "Bounce buffer is NULL\n"); + return ENA_COM_FAULT; } - memcpy_toio(dev_head_addr, head_src, header_len); + memcpy(bounce_buffer + header_offset, header_src, header_len); return 0; } -static inline struct ena_eth_io_rx_cdesc_base * +static void *get_sq_desc_llq(struct ena_com_io_sq *io_sq) +{ + struct ena_com_llq_pkt_ctrl *pkt_ctrl = &io_sq->llq_buf_ctrl; + u8 *bounce_buffer; + void *sq_desc; + + bounce_buffer = pkt_ctrl->curr_bounce_buf; + + if (unlikely(!bounce_buffer)) { + ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq), + "Bounce buffer is NULL\n"); + return NULL; + } + + sq_desc = bounce_buffer + pkt_ctrl->idx * io_sq->desc_entry_size; + pkt_ctrl->idx++; + pkt_ctrl->descs_left_in_line--; + + return sq_desc; +} + +static int ena_com_close_bounce_buffer(struct ena_com_io_sq *io_sq) +{ + struct ena_com_llq_pkt_ctrl *pkt_ctrl = &io_sq->llq_buf_ctrl; + struct ena_com_llq_info *llq_info = &io_sq->llq_info; + int rc; + + if (unlikely(io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)) + return ENA_COM_OK; + + /* bounce buffer was used, so write it and get a new one */ + if (pkt_ctrl->idx) { + rc = ena_com_write_bounce_buffer_to_dev(io_sq, + pkt_ctrl->curr_bounce_buf); + if (unlikely(rc)) { + ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq), + "Failed to write bounce buffer to device\n"); + return rc; + } + + pkt_ctrl->curr_bounce_buf = + ena_com_get_next_bounce_buffer(&io_sq->bounce_buf_ctrl); + memset(io_sq->llq_buf_ctrl.curr_bounce_buf, + 0x0, llq_info->desc_list_entry_size); + } + + pkt_ctrl->idx = 0; + pkt_ctrl->descs_left_in_line = llq_info->descs_num_before_header; + return ENA_COM_OK; +} + +static void *get_sq_desc(struct ena_com_io_sq *io_sq) +{ + if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) + return get_sq_desc_llq(io_sq); + + return get_sq_desc_regular_queue(io_sq); +} + +static int ena_com_sq_update_llq_tail(struct ena_com_io_sq *io_sq) +{ + struct ena_com_llq_pkt_ctrl *pkt_ctrl = &io_sq->llq_buf_ctrl; + struct ena_com_llq_info *llq_info = &io_sq->llq_info; + int rc; + + if (!pkt_ctrl->descs_left_in_line) { + rc = ena_com_write_bounce_buffer_to_dev(io_sq, + pkt_ctrl->curr_bounce_buf); + if (unlikely(rc)) { + ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq), + "Failed to write bounce buffer to device\n"); + return rc; + } + + pkt_ctrl->curr_bounce_buf = + ena_com_get_next_bounce_buffer(&io_sq->bounce_buf_ctrl); + memset(io_sq->llq_buf_ctrl.curr_bounce_buf, + 0x0, llq_info->desc_list_entry_size); + + pkt_ctrl->idx = 0; + if (unlikely(llq_info->desc_stride_ctrl == ENA_ADMIN_SINGLE_DESC_PER_ENTRY)) + pkt_ctrl->descs_left_in_line = 1; + else + pkt_ctrl->descs_left_in_line = + llq_info->desc_list_entry_size / io_sq->desc_entry_size; + } + + return ENA_COM_OK; +} + +static int ena_com_sq_update_tail(struct ena_com_io_sq *io_sq) +{ + if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) + return ena_com_sq_update_llq_tail(io_sq); + + io_sq->tail++; + + /* Switch phase bit in case of wrap around */ + if (unlikely((io_sq->tail & (io_sq->q_depth - 1)) == 0)) + io_sq->phase ^= 1; + + return ENA_COM_OK; +} + +static struct ena_eth_io_rx_cdesc_base * ena_com_rx_cdesc_idx_to_ptr(struct ena_com_io_cq *io_cq, u16 idx) { idx &= (io_cq->q_depth - 1); return (struct ena_eth_io_rx_cdesc_base *) - ((unsigned char *)io_cq->cdesc_addr.virt_addr + - idx * io_cq->cdesc_entry_size_in_bytes); + ((uintptr_t)io_cq->cdesc_addr.virt_addr + + idx * io_cq->cdesc_entry_size_in_bytes); } -static inline int ena_com_cdesc_rx_pkt_get(struct ena_com_io_cq *io_cq, +static u16 ena_com_cdesc_rx_pkt_get(struct ena_com_io_cq *io_cq, u16 *first_cdesc_idx) { struct ena_eth_io_rx_cdesc_base *cdesc; @@ -143,7 +246,7 @@ ena_com_cq_inc_head(io_cq); count++; - last = (cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK) >> + last = (READ_ONCE32(cdesc->status) & ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_LAST_SHIFT; } while (!last); @@ -156,7 +259,8 @@ io_cq->cur_rx_pkt_cdesc_count = 0; io_cq->cur_rx_pkt_cdesc_start_idx = head_masked; - ena_trc_dbg("ena q_id: %d packets were completed. first desc idx %u descs# %d\n", + ena_trc_dbg(ena_com_io_cq_to_ena_dev(io_cq), + "ENA q_id: %d packets were completed. first desc idx %u descs# %d\n", io_cq->qid, *first_cdesc_idx, count); } else { io_cq->cur_rx_pkt_cdesc_count += count; @@ -166,31 +270,15 @@ return count; } -static inline bool ena_com_meta_desc_changed(struct ena_com_io_sq *io_sq, - struct ena_com_tx_ctx *ena_tx_ctx) -{ - int rc; - - if (ena_tx_ctx->meta_valid) { - rc = memcmp(&io_sq->cached_tx_meta, - &ena_tx_ctx->ena_meta, - sizeof(struct ena_com_tx_meta)); - - if (unlikely(rc != 0)) - return true; - } - - return false; -} - -static inline void ena_com_create_and_store_tx_meta_desc( - struct ena_com_io_sq *io_sq, - struct ena_com_tx_ctx *ena_tx_ctx) +static int ena_com_create_meta(struct ena_com_io_sq *io_sq, + struct ena_com_tx_meta *ena_meta) { struct ena_eth_io_tx_meta_desc *meta_desc = NULL; - struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta; meta_desc = get_sq_desc(io_sq); + if (unlikely(!meta_desc)) + return ENA_COM_FAULT; + memset(meta_desc, 0x0, sizeof(struct ena_eth_io_tx_meta_desc)); meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_META_DESC_MASK; @@ -198,62 +286,90 @@ meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_EXT_VALID_MASK; /* bits 0-9 of the mss */ - meta_desc->word2 |= (ena_meta->mss << + meta_desc->word2 |= ((u32)ena_meta->mss << ENA_ETH_IO_TX_META_DESC_MSS_LO_SHIFT) & ENA_ETH_IO_TX_META_DESC_MSS_LO_MASK; /* bits 10-13 of the mss */ meta_desc->len_ctrl |= ((ena_meta->mss >> 10) << - ENA_ETH_IO_TX_META_DESC_MSS_HI_PTP_SHIFT) & - ENA_ETH_IO_TX_META_DESC_MSS_HI_PTP_MASK; + ENA_ETH_IO_TX_META_DESC_MSS_HI_SHIFT) & + ENA_ETH_IO_TX_META_DESC_MSS_HI_MASK; /* Extended meta desc */ meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_MASK; - meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_META_STORE_MASK; - meta_desc->len_ctrl |= (io_sq->phase << + meta_desc->len_ctrl |= ((u32)io_sq->phase << ENA_ETH_IO_TX_META_DESC_PHASE_SHIFT) & ENA_ETH_IO_TX_META_DESC_PHASE_MASK; meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_FIRST_MASK; + meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_META_STORE_MASK; + meta_desc->word2 |= ena_meta->l3_hdr_len & ENA_ETH_IO_TX_META_DESC_L3_HDR_LEN_MASK; meta_desc->word2 |= (ena_meta->l3_hdr_offset << ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_SHIFT) & ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_MASK; - meta_desc->word2 |= (ena_meta->l4_hdr_len << + meta_desc->word2 |= ((u32)ena_meta->l4_hdr_len << ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_SHIFT) & ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_MASK; - meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_META_STORE_MASK; + return ena_com_sq_update_tail(io_sq); +} + +static int ena_com_create_and_store_tx_meta_desc(struct ena_com_io_sq *io_sq, + struct ena_com_tx_ctx *ena_tx_ctx, + bool *have_meta) +{ + struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta; + + /* When disable meta caching is set, don't bother to save the meta and + * compare it to the stored version, just create the meta + */ + if (io_sq->disable_meta_caching) { + if (unlikely(!ena_tx_ctx->meta_valid)) + return ENA_COM_INVAL; - /* Cached the meta desc */ - memcpy(&io_sq->cached_tx_meta, ena_meta, - sizeof(struct ena_com_tx_meta)); - - ena_com_copy_curr_sq_desc_to_dev(io_sq); - ena_com_sq_update_tail(io_sq); -} - -static inline void ena_com_rx_set_flags(struct ena_com_rx_ctx *ena_rx_ctx, - struct ena_eth_io_rx_cdesc_base *cdesc) -{ - ena_rx_ctx->l3_proto = (enum ena_eth_io_l3_proto_index)(cdesc->status & - ENA_ETH_IO_RX_CDESC_BASE_L3_PROTO_IDX_MASK); - ena_rx_ctx->l4_proto = (enum ena_eth_io_l4_proto_index) - ((cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_MASK) >> - ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_SHIFT); + *have_meta = true; + return ena_com_create_meta(io_sq, ena_meta); + } + + if (ena_com_meta_desc_changed(io_sq, ena_tx_ctx)) { + *have_meta = true; + /* Cache the meta desc */ + memcpy(&io_sq->cached_tx_meta, ena_meta, + sizeof(struct ena_com_tx_meta)); + return ena_com_create_meta(io_sq, ena_meta); + } + + *have_meta = false; + return ENA_COM_OK; +} + +static void ena_com_rx_set_flags(struct ena_com_io_cq *io_cq, + struct ena_com_rx_ctx *ena_rx_ctx, + struct ena_eth_io_rx_cdesc_base *cdesc) +{ + ena_rx_ctx->l3_proto = cdesc->status & + ENA_ETH_IO_RX_CDESC_BASE_L3_PROTO_IDX_MASK; + ena_rx_ctx->l4_proto = + (cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_MASK) >> + ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_SHIFT; ena_rx_ctx->l3_csum_err = - (cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_MASK) >> - ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_SHIFT; + !!((cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_MASK) >> + ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_SHIFT); ena_rx_ctx->l4_csum_err = - (cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_MASK) >> - ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_SHIFT; + !!((cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_MASK) >> + ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_SHIFT); + ena_rx_ctx->l4_csum_checked = + !!((cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_CHECKED_MASK) >> + ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_CHECKED_SHIFT); ena_rx_ctx->hash = cdesc->hash; ena_rx_ctx->frag = (cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_SHIFT; - ena_trc_dbg("ena_rx_ctx->l3_proto %d ena_rx_ctx->l4_proto %d\nena_rx_ctx->l3_csum_err %d ena_rx_ctx->l4_csum_err %d\nhash frag %d frag: %d cdesc_status: %x\n", + ena_trc_dbg(ena_com_io_cq_to_ena_dev(io_cq), + "l3_proto %d l4_proto %d l3_csum_err %d l4_csum_err %d hash %d frag %d cdesc_status %x\n", ena_rx_ctx->l3_proto, ena_rx_ctx->l4_proto, ena_rx_ctx->l3_csum_err, @@ -273,61 +389,78 @@ { struct ena_eth_io_tx_desc *desc = NULL; struct ena_com_buf *ena_bufs = ena_tx_ctx->ena_bufs; - void *push_header = ena_tx_ctx->push_header; + void *buffer_to_push = ena_tx_ctx->push_header; u16 header_len = ena_tx_ctx->header_len; u16 num_bufs = ena_tx_ctx->num_bufs; - int total_desc, i, rc; + u16 start_tail = io_sq->tail; + int i, rc; bool have_meta; u64 addr_hi; - ENA_ASSERT(io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX, - "wrong Q type"); + ENA_WARN(io_sq->direction != ENA_COM_IO_QUEUE_DIRECTION_TX, + ena_com_io_sq_to_ena_dev(io_sq), "wrong Q type"); /* num_bufs +1 for potential meta desc */ - if (ena_com_sq_empty_space(io_sq) < (num_bufs + 1)) { - ena_trc_err("Not enough space in the tx queue\n"); + if (unlikely(!ena_com_sq_have_enough_space(io_sq, num_bufs + 1))) { + ena_trc_dbg(ena_com_io_sq_to_ena_dev(io_sq), + "Not enough space in the tx queue\n"); return ENA_COM_NO_MEM; } if (unlikely(header_len > io_sq->tx_max_header_size)) { - ena_trc_err("header size is too large %d max header: %d\n", + ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq), + "Header size is too large %d max header: %d\n", header_len, io_sq->tx_max_header_size); return ENA_COM_INVAL; } - /* start with pushing the header (if needed) */ - rc = ena_com_write_header(io_sq, push_header, header_len); + if (unlikely(io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV + && !buffer_to_push)) { + ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq), + "Push header wasn't provided on LLQ mode\n"); + return ENA_COM_INVAL; + } + + rc = ena_com_write_header_to_bounce(io_sq, buffer_to_push, header_len); if (unlikely(rc)) return rc; - have_meta = ena_tx_ctx->meta_valid && ena_com_meta_desc_changed(io_sq, - ena_tx_ctx); - if (have_meta) - ena_com_create_and_store_tx_meta_desc(io_sq, ena_tx_ctx); + rc = ena_com_create_and_store_tx_meta_desc(io_sq, ena_tx_ctx, &have_meta); + if (unlikely(rc)) { + ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq), + "Failed to create and store tx meta desc\n"); + return rc; + } - /* If the caller doesn't want send packets */ + /* If the caller doesn't want to send packets */ if (unlikely(!num_bufs && !header_len)) { - *nb_hw_desc = have_meta ? 0 : 1; - return 0; + rc = ena_com_close_bounce_buffer(io_sq); + if (rc) + ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq), + "Failed to write buffers to LLQ\n"); + *nb_hw_desc = io_sq->tail - start_tail; + return rc; } desc = get_sq_desc(io_sq); + if (unlikely(!desc)) + return ENA_COM_FAULT; memset(desc, 0x0, sizeof(struct ena_eth_io_tx_desc)); /* Set first desc when we don't have meta descriptor */ if (!have_meta) desc->len_ctrl |= ENA_ETH_IO_TX_DESC_FIRST_MASK; - desc->buff_addr_hi_hdr_sz |= (header_len << + desc->buff_addr_hi_hdr_sz |= ((u32)header_len << ENA_ETH_IO_TX_DESC_HEADER_LENGTH_SHIFT) & ENA_ETH_IO_TX_DESC_HEADER_LENGTH_MASK; - desc->len_ctrl |= (io_sq->phase << ENA_ETH_IO_TX_DESC_PHASE_SHIFT) & + desc->len_ctrl |= ((u32)io_sq->phase << ENA_ETH_IO_TX_DESC_PHASE_SHIFT) & ENA_ETH_IO_TX_DESC_PHASE_MASK; desc->len_ctrl |= ENA_ETH_IO_TX_DESC_COMP_REQ_MASK; /* Bits 0-9 */ - desc->meta_ctrl |= (ena_tx_ctx->req_id << + desc->meta_ctrl |= ((u32)ena_tx_ctx->req_id << ENA_ETH_IO_TX_DESC_REQ_ID_LO_SHIFT) & ENA_ETH_IO_TX_DESC_REQ_ID_LO_MASK; @@ -363,13 +496,20 @@ for (i = 0; i < num_bufs; i++) { /* The first desc share the same desc as the header */ if (likely(i != 0)) { - ena_com_copy_curr_sq_desc_to_dev(io_sq); - ena_com_sq_update_tail(io_sq); + rc = ena_com_sq_update_tail(io_sq); + if (unlikely(rc)) { + ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq), + "Failed to update sq tail\n"); + return rc; + } desc = get_sq_desc(io_sq); + if (unlikely(!desc)) + return ENA_COM_FAULT; + memset(desc, 0x0, sizeof(struct ena_eth_io_tx_desc)); - desc->len_ctrl |= (io_sq->phase << + desc->len_ctrl |= ((u32)io_sq->phase << ENA_ETH_IO_TX_DESC_PHASE_SHIFT) & ENA_ETH_IO_TX_DESC_PHASE_MASK; } @@ -389,15 +529,20 @@ /* set the last desc indicator */ desc->len_ctrl |= ENA_ETH_IO_TX_DESC_LAST_MASK; - ena_com_copy_curr_sq_desc_to_dev(io_sq); - - ena_com_sq_update_tail(io_sq); + rc = ena_com_sq_update_tail(io_sq); + if (unlikely(rc)) { + ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq), + "Failed to update sq tail of the last descriptor\n"); + return rc; + } - total_desc = ENA_MAX16(num_bufs, 1); - total_desc += have_meta ? 1 : 0; + rc = ena_com_close_bounce_buffer(io_sq); + if (rc) + ena_trc_err(ena_com_io_sq_to_ena_dev(io_sq), + "Failed when closing bounce buffer\n"); - *nb_hw_desc = total_desc; - return 0; + *nb_hw_desc = io_sq->tail - start_tail; + return rc; } int ena_com_rx_pkt(struct ena_com_io_cq *io_cq, @@ -406,12 +551,13 @@ { struct ena_com_rx_buf_info *ena_buf = &ena_rx_ctx->ena_bufs[0]; struct ena_eth_io_rx_cdesc_base *cdesc = NULL; + u16 q_depth = io_cq->q_depth; u16 cdesc_idx = 0; u16 nb_hw_desc; - u16 i; + u16 i = 0; - ENA_ASSERT(io_cq->direction == ENA_COM_IO_QUEUE_DIRECTION_RX, - "wrong Q type"); + ENA_WARN(io_cq->direction != ENA_COM_IO_QUEUE_DIRECTION_RX, + ena_com_io_cq_to_ena_dev(io_cq), "wrong Q type"); nb_hw_desc = ena_com_cdesc_rx_pkt_get(io_cq, &cdesc_idx); if (nb_hw_desc == 0) { @@ -419,33 +565,45 @@ return 0; } - ena_trc_dbg("fetch rx packet: queue %d completed desc: %d\n", + ena_trc_dbg(ena_com_io_cq_to_ena_dev(io_cq), + "Fetch rx packet: queue %d completed desc: %d\n", io_cq->qid, nb_hw_desc); if (unlikely(nb_hw_desc > ena_rx_ctx->max_bufs)) { - ena_trc_err("Too many RX cdescs (%d) > MAX(%d)\n", + ena_trc_err(ena_com_io_cq_to_ena_dev(io_cq), + "Too many RX cdescs (%d) > MAX(%d)\n", nb_hw_desc, ena_rx_ctx->max_bufs); return ENA_COM_NO_SPACE; } - for (i = 0; i < nb_hw_desc; i++) { + cdesc = ena_com_rx_cdesc_idx_to_ptr(io_cq, cdesc_idx); + ena_rx_ctx->pkt_offset = cdesc->offset; + + do { + ena_buf[i].len = cdesc->length; + ena_buf[i].req_id = cdesc->req_id; + if (unlikely(ena_buf[i].req_id >= q_depth)) + return ENA_COM_EIO; + + if (++i >= nb_hw_desc) + break; + cdesc = ena_com_rx_cdesc_idx_to_ptr(io_cq, cdesc_idx + i); - ena_buf->len = cdesc->length; - ena_buf->req_id = cdesc->req_id; - ena_buf++; - } + } while (1); /* Update SQ head ptr */ io_sq->next_to_comp += nb_hw_desc; - ena_trc_dbg("[%s][QID#%d] Updating SQ head to: %d\n", __func__, + ena_trc_dbg(ena_com_io_cq_to_ena_dev(io_cq), + "[%s][QID#%d] Updating SQ head to: %d\n", __func__, io_sq->qid, io_sq->next_to_comp); /* Get rx flags from the last pkt */ - ena_com_rx_set_flags(ena_rx_ctx, cdesc); + ena_com_rx_set_flags(io_cq, ena_rx_ctx, cdesc); ena_rx_ctx->descs = nb_hw_desc; + return 0; } @@ -455,58 +613,45 @@ { struct ena_eth_io_rx_desc *desc; - ENA_ASSERT(io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_RX, - "wrong Q type"); + ENA_WARN(io_sq->direction != ENA_COM_IO_QUEUE_DIRECTION_RX, + ena_com_io_sq_to_ena_dev(io_sq), "wrong Q type"); - if (unlikely(ena_com_sq_empty_space(io_sq) == 0)) + if (unlikely(!ena_com_sq_have_enough_space(io_sq, 1))) return ENA_COM_NO_SPACE; desc = get_sq_desc(io_sq); + if (unlikely(!desc)) + return ENA_COM_FAULT; + memset(desc, 0x0, sizeof(struct ena_eth_io_rx_desc)); desc->length = ena_buf->len; - desc->ctrl |= ENA_ETH_IO_RX_DESC_FIRST_MASK; - desc->ctrl |= ENA_ETH_IO_RX_DESC_LAST_MASK; - desc->ctrl |= io_sq->phase & ENA_ETH_IO_RX_DESC_PHASE_MASK; - desc->ctrl |= ENA_ETH_IO_RX_DESC_COMP_REQ_MASK; + desc->ctrl = ENA_ETH_IO_RX_DESC_FIRST_MASK | + ENA_ETH_IO_RX_DESC_LAST_MASK | + ENA_ETH_IO_RX_DESC_COMP_REQ_MASK | + (io_sq->phase & ENA_ETH_IO_RX_DESC_PHASE_MASK); desc->req_id = req_id; + ena_trc_dbg(ena_com_io_sq_to_ena_dev(io_sq), + "[%s] Adding single RX desc, Queue: %u, req_id: %u\n", + __func__, io_sq->qid, req_id); + desc->buff_addr_lo = (u32)ena_buf->paddr; desc->buff_addr_hi = - ((ena_buf->paddr & - GENMASK_ULL(io_sq->dma_addr_bits - 1, 32)) >> 32); - - ena_com_sq_update_tail(io_sq); + ((ena_buf->paddr & GENMASK_ULL(io_sq->dma_addr_bits - 1, 32)) >> 32); - return 0; + return ena_com_sq_update_tail(io_sq); } -int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq, u16 *req_id) +bool ena_com_cq_empty(struct ena_com_io_cq *io_cq) { - u8 expected_phase, cdesc_phase; - struct ena_eth_io_tx_cdesc *cdesc; - u16 masked_head; - - masked_head = io_cq->head & (io_cq->q_depth - 1); - expected_phase = io_cq->phase; - - cdesc = (struct ena_eth_io_tx_cdesc *) - ((unsigned char *)io_cq->cdesc_addr.virt_addr - + (masked_head * io_cq->cdesc_entry_size_in_bytes)); - - /* When the current completion descriptor phase isn't the same as the - * expected, it mean that the device still didn't update - * this completion. - */ - cdesc_phase = cdesc->flags & ENA_ETH_IO_TX_CDESC_PHASE_MASK; - if (cdesc_phase != expected_phase) - return ENA_COM_TRY_AGAIN; - - ena_com_cq_inc_head(io_cq); - - *req_id = cdesc->req_id; + struct ena_eth_io_rx_cdesc_base *cdesc; - return 0; + cdesc = ena_com_get_next_rx_cdesc(io_cq); + if (cdesc) + return false; + else + return true; } Index: /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_plat.h =================================================================== --- /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_plat.h (revision 38965) +++ /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_plat.h (working copy) @@ -1,51 +1,27 @@ -/*- -* BSD LICENSE -* -* Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. -* All rights reserved. -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following conditions -* are met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above copyright -* notice, this list of conditions and the following disclaimer in -* the documentation and/or other materials provided with the -* distribution. -* * Neither the name of copyright holder nor the names of its -* contributors may be used to endorse or promote products derived -* from this software without specific prior written permission. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2019 Amazon.com, Inc. or its affiliates. + * All rights reserved. + */ #ifndef ENA_PLAT_H_ #define ENA_PLAT_H_ #if defined(ENA_IPXE) -#include "ena_plat_ipxe.h" +#include #elif defined(__linux__) #if defined(__KERNEL__) -#include "ena_plat_linux.h" +#include #else -#include "ena_plat_dpdk.h" +#include #endif #elif defined(__FreeBSD__) -#include "ena_plat_dpdk.h" +#if defined(_KERNEL) +#include +#else +#include +#endif #elif defined(_WIN32) -#include "ena_plat_windows.h" +#include #else #error "Invalid platform" #endif Index: /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_plat_dpdk.h =================================================================== --- /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_plat_dpdk.h (revision 38965) +++ /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/base/ena_plat_dpdk.h (working copy) @@ -1,35 +1,7 @@ -/*- -* BSD LICENSE -* -* Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. -* All rights reserved. -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following conditions -* are met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above copyright -* notice, this list of conditions and the following disclaimer in -* the documentation and/or other materials provided with the -* distribution. -* * Neither the name of copyright holder nor the names of its -* contributors may be used to endorse or promote products derived -* from this software without specific prior written permission. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. + * All rights reserved. + */ #ifndef DPDK_ENA_COM_ENA_PLAT_DPDK_H_ #define DPDK_ENA_COM_ENA_PLAT_DPDK_H_ @@ -38,30 +10,35 @@ #include #include #include +#include #include #include #include #include #include -#include #include #include #include +#include #include #include +#include typedef uint64_t u64; typedef uint32_t u32; typedef uint16_t u16; typedef uint8_t u8; +typedef struct rte_eth_dev ena_netdev; typedef uint64_t dma_addr_t; + #ifndef ETIME #define ETIME ETIMEDOUT #endif +#define ENA_PRIu64 PRIu64 #define ena_atomic32_t rte_atomic32_t #define ena_mem_handle_t const struct rte_memzone * @@ -73,160 +50,273 @@ #define ENA_COM_INVAL -EINVAL #define ENA_COM_NO_SPACE -ENOSPC #define ENA_COM_NO_DEVICE -ENODEV -#define ENA_COM_PERMISSION -EPERM #define ENA_COM_TIMER_EXPIRED -ETIME #define ENA_COM_FAULT -EFAULT #define ENA_COM_TRY_AGAIN -EAGAIN +#define ENA_COM_UNSUPPORTED -EOPNOTSUPP +#define ENA_COM_EIO -EIO #define ____cacheline_aligned __rte_cache_aligned #define ENA_ABORT() abort() -#define ENA_MSLEEP(x) rte_delay_ms(x) -#define ENA_UDELAY(x) rte_delay_us(x) +#define ENA_MSLEEP(x) rte_delay_us(x * 1000) +#define ENA_USLEEP(x) rte_delay_us(x) +#define ENA_UDELAY(x) rte_delay_us_block(x) #define ENA_TOUCH(x) ((void)(x)) -#define memcpy_toio memcpy +/* Avoid nested declaration on arm64, as it may define rte_memcpy as memcpy. */ +#if defined(RTE_ARCH_X86) +#undef memcpy +#define memcpy rte_memcpy +#endif #define wmb rte_wmb -#define rmb rte_wmb +#define rmb rte_rmb #define mb rte_mb +#define mmiowb rte_wmb #define __iomem +#ifndef READ_ONCE +#define READ_ONCE(var) (*((volatile typeof(var) *)(&(var)))) +#endif + +#define READ_ONCE8(var) READ_ONCE(var) +#define READ_ONCE16(var) READ_ONCE(var) +#define READ_ONCE32(var) READ_ONCE(var) + #define US_PER_S 1000000 -#define ENA_GET_SYSTEM_USECS() \ +#define ENA_GET_SYSTEM_USECS() \ (rte_get_timer_cycles() * US_PER_S / rte_get_timer_hz()) -#if RTE_LOG_LEVEL >= RTE_LOG_DEBUG -#define ENA_ASSERT(cond, format, arg...) \ - do { \ - if (unlikely(!(cond))) { \ - RTE_LOG(ERR, PMD, format, ##arg); \ - rte_panic("line %d\tassert \"" #cond "\"" \ - "failed\n", __LINE__); \ - } \ - } while (0) -#else -#define ENA_ASSERT(cond, format, arg...) do {} while (0) -#endif - -#define ENA_MAX32(x, y) RTE_MAX((x), (y)) -#define ENA_MAX16(x, y) RTE_MAX((x), (y)) -#define ENA_MAX8(x, y) RTE_MAX((x), (y)) -#define ENA_MIN32(x, y) RTE_MIN((x), (y)) -#define ENA_MIN16(x, y) RTE_MIN((x), (y)) -#define ENA_MIN8(x, y) RTE_MIN((x), (y)) +#define ENA_MAX_T(type, x, y) RTE_MAX((type)(x), (type)(y)) +#define ENA_MAX32(x, y) ENA_MAX_T(uint32_t, (x), (y)) +#define ENA_MAX16(x, y) ENA_MAX_T(uint16_t, (x), (y)) +#define ENA_MAX8(x, y) ENA_MAX_T(uint8_t, (x), (y)) +#define ENA_MIN_T(type, x, y) RTE_MIN((type)(x), (type)(y)) +#define ENA_MIN32(x, y) ENA_MIN_T(uint32_t, (x), (y)) +#define ENA_MIN16(x, y) ENA_MIN_T(uint16_t, (x), (y)) +#define ENA_MIN8(x, y) ENA_MIN_T(uint8_t, (x), (y)) +#define BITS_PER_LONG_LONG (__SIZEOF_LONG_LONG__ * 8) #define U64_C(x) x ## ULL #define BIT(nr) (1UL << (nr)) #define BITS_PER_LONG (__SIZEOF_LONG__ * 8) #define GENMASK(h, l) (((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h)))) -#define GENMASK_ULL(h, l) (((U64_C(1) << ((h) - (l) + 1)) - 1) << (l)) +#define GENMASK_ULL(h, l) (((~0ULL) - (1ULL << (l)) + 1) & \ + (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h)))) #ifdef RTE_LIBRTE_ENA_COM_DEBUG -#define ena_trc_dbg(format, arg...) \ - RTE_LOG(DEBUG, PMD, "[ENA_COM: %s] " format, __func__, ##arg) -#define ena_trc_info(format, arg...) \ - RTE_LOG(INFO, PMD, "[ENA_COM: %s] " format, __func__, ##arg) -#define ena_trc_warn(format, arg...) \ - RTE_LOG(ERR, PMD, "[ENA_COM: %s] " format, __func__, ##arg) -#define ena_trc_err(format, arg...) \ - RTE_LOG(ERR, PMD, "[ENA_COM: %s] " format, __func__, ##arg) +#define ena_trc_log(dev, level, fmt, arg...) \ + ( \ + ENA_TOUCH(dev), \ + RTE_LOG(level, PMD, \ + "[ENA_COM: %s]" fmt, __func__, ##arg) \ + ) + +#define ena_trc_dbg(dev, format, arg...) ena_trc_log(dev, DEBUG, format, ##arg) +#define ena_trc_info(dev, format, arg...) ena_trc_log(dev, INFO, format, ##arg) +#define ena_trc_warn(dev, format, arg...) \ + ena_trc_log(dev, WARNING, format, ##arg) +#define ena_trc_err(dev, format, arg...) ena_trc_log(dev, ERR, format, ##arg) #else -#define ena_trc_dbg(format, arg...) do { } while (0) -#define ena_trc_info(format, arg...) do { } while (0) -#define ena_trc_warn(format, arg...) do { } while (0) -#define ena_trc_err(format, arg...) do { } while (0) +#define ena_trc_dbg(dev, format, arg...) ENA_TOUCH(dev) +#define ena_trc_info(dev, format, arg...) ENA_TOUCH(dev) +#define ena_trc_warn(dev, format, arg...) ENA_TOUCH(dev) +#define ena_trc_err(dev, format, arg...) ENA_TOUCH(dev) #endif /* RTE_LIBRTE_ENA_COM_DEBUG */ +#define ENA_WARN(cond, dev, format, arg...) \ + do { \ + if (unlikely(cond)) { \ + ena_trc_err(dev, \ + "Warn failed on %s:%s:%d:" format, \ + __FILE__, __func__, __LINE__, ##arg); \ + } \ + } while (0) + /* Spinlock related methods */ #define ena_spinlock_t rte_spinlock_t -#define ENA_SPINLOCK_INIT(spinlock) rte_spinlock_init(&spinlock) -#define ENA_SPINLOCK_LOCK(spinlock, flags) \ - ({(void)flags; rte_spinlock_lock(&spinlock); }) -#define ENA_SPINLOCK_UNLOCK(spinlock, flags) \ +#define ENA_SPINLOCK_INIT(spinlock) rte_spinlock_init(&(spinlock)) +#define ENA_SPINLOCK_LOCK(spinlock, flags) \ + ({(void)flags; rte_spinlock_lock(&(spinlock)); }) +#define ENA_SPINLOCK_UNLOCK(spinlock, flags) \ ({(void)flags; rte_spinlock_unlock(&(spinlock)); }) +#define ENA_SPINLOCK_DESTROY(spinlock) ((void)(spinlock)) -#define q_waitqueue_t \ - struct { \ - pthread_cond_t cond; \ - pthread_mutex_t mutex; \ - } - -#define ena_wait_queue_t q_waitqueue_t - -#define ENA_WAIT_EVENT_INIT(waitqueue) \ - do { \ - pthread_mutex_init(&(waitqueue).mutex, NULL); \ - pthread_cond_init(&(waitqueue).cond, NULL); \ - } while (0) - -#define ENA_WAIT_EVENT_WAIT(waitevent, timeout) \ - do { \ - struct timespec wait; \ - struct timeval now; \ - unsigned long timeout_us; \ - gettimeofday(&now, NULL); \ - wait.tv_sec = now.tv_sec + timeout / 1000000UL; \ - timeout_us = timeout % 1000000UL; \ - wait.tv_nsec = (now.tv_usec + timeout_us) * 1000UL; \ - pthread_mutex_lock(&waitevent.mutex); \ - pthread_cond_timedwait(&waitevent.cond, \ - &waitevent.mutex, &wait); \ - pthread_mutex_unlock(&waitevent.mutex); \ +typedef struct { + pthread_cond_t cond; + pthread_mutex_t mutex; + uint8_t flag; +} ena_wait_event_t; + +#define ENA_WAIT_EVENT_INIT(waitevent) \ + do { \ + ena_wait_event_t *_we = &(waitevent); \ + pthread_mutex_init(&_we->mutex, NULL); \ + pthread_cond_init(&_we->cond, NULL); \ + _we->flag = 0; \ + } while (0) + +#define ENA_WAIT_EVENT_WAIT(waitevent, timeout) \ + do { \ + ena_wait_event_t *_we = &(waitevent); \ + typeof(timeout) _tmo = (timeout); \ + int ret = 0; \ + struct timespec wait; \ + struct timeval now; \ + unsigned long timeout_us; \ + gettimeofday(&now, NULL); \ + wait.tv_sec = now.tv_sec + _tmo / 1000000UL; \ + timeout_us = _tmo % 1000000UL; \ + wait.tv_nsec = (now.tv_usec + timeout_us) * 1000UL; \ + pthread_mutex_lock(&_we->mutex); \ + while (ret == 0 && !_we->flag) { \ + ret = pthread_cond_timedwait(&_we->cond, \ + &_we->mutex, &wait); \ + } \ + /* Asserts only if not working on ena_wait_event_t */ \ + if (unlikely(ret != 0 && ret != ETIMEDOUT)) \ + ena_trc_err(NULL, \ + "Invalid wait event. pthread ret: %d\n", ret); \ + else if (unlikely(ret == ETIMEDOUT)) \ + ena_trc_err(NULL, \ + "Timeout waiting for " #waitevent "\n"); \ + _we->flag = 0; \ + pthread_mutex_unlock(&_we->mutex); \ + } while (0) +#define ENA_WAIT_EVENT_SIGNAL(waitevent) \ + do { \ + ena_wait_event_t *_we = &(waitevent); \ + pthread_mutex_lock(&_we->mutex); \ + _we->flag = 1; \ + pthread_cond_signal(&_we->cond); \ + pthread_mutex_unlock(&_we->mutex); \ } while (0) -#define ENA_WAIT_EVENT_SIGNAL(waitevent) pthread_cond_signal(&waitevent.cond) /* pthread condition doesn't need to be rearmed after usage */ #define ENA_WAIT_EVENT_CLEAR(...) +#define ENA_WAIT_EVENT_DESTROY(waitevent) ((void)(waitevent)) -#define ena_wait_event_t ena_wait_queue_t #define ENA_MIGHT_SLEEP() -#define ENA_MEM_ALLOC_COHERENT(dmadev, size, virt, phys, handle) \ - do { \ - const struct rte_memzone *mz; \ - char z_name[RTE_MEMZONE_NAMESIZE]; \ - ENA_TOUCH(dmadev); ENA_TOUCH(handle); \ - snprintf(z_name, sizeof(z_name), \ - "ena_alloc_%d", ena_alloc_cnt++); \ - mz = rte_memzone_reserve(z_name, size, SOCKET_ID_ANY, 0); \ - memset(mz->addr, 0, size); \ - virt = mz->addr; \ - phys = mz->phys_addr; \ - handle = mz; \ - } while (0) -#define ENA_MEM_FREE_COHERENT(dmadev, size, virt, phys, handle) \ - ({ ENA_TOUCH(size); ENA_TOUCH(phys); \ - ENA_TOUCH(dmadev); \ - rte_memzone_free(handle); }) - -#define ENA_MEM_ALLOC_COHERENT_NODE(dmadev, size, virt, phys, node, dev_node) \ - do { \ - const struct rte_memzone *mz; \ - char z_name[RTE_MEMZONE_NAMESIZE]; \ - ENA_TOUCH(dmadev); ENA_TOUCH(dev_node); \ - snprintf(z_name, sizeof(z_name), \ - "ena_alloc_%d", ena_alloc_cnt++); \ - mz = rte_memzone_reserve(z_name, size, node, 0); \ - virt = mz->addr; \ - phys = mz->phys_addr; \ - } while (0) - -#define ENA_MEM_ALLOC_NODE(dmadev, size, virt, node, dev_node) \ - do { \ - const struct rte_memzone *mz; \ - char z_name[RTE_MEMZONE_NAMESIZE]; \ - ENA_TOUCH(dmadev); ENA_TOUCH(dev_node); \ - snprintf(z_name, sizeof(z_name), \ - "ena_alloc_%d", ena_alloc_cnt++); \ - mz = rte_memzone_reserve(z_name, size, node, 0); \ - virt = mz->addr; \ +#define ena_time_t uint64_t +#define ENA_TIME_EXPIRE(timeout) (timeout < rte_get_timer_cycles()) +#define ENA_GET_SYSTEM_TIMEOUT(timeout_us) \ + ((timeout_us) * rte_get_timer_hz() / 1000000 + rte_get_timer_cycles()) + +/* + * Each rte_memzone should have unique name. + * To satisfy it, count number of allocations and add it to name. + */ +extern rte_atomic64_t ena_alloc_cnt; + +#define ENA_MEM_ALLOC_COHERENT_ALIGNED( \ + dmadev, size, virt, phys, mem_handle, alignment) \ + do { \ + const struct rte_memzone *mz = NULL; \ + ENA_TOUCH(dmadev); \ + if ((size) > 0) { \ + char z_name[RTE_MEMZONE_NAMESIZE]; \ + snprintf(z_name, sizeof(z_name), \ + "ena_alloc_%" PRIi64 "", \ + rte_atomic64_add_return(&ena_alloc_cnt, 1)); \ + mz = rte_memzone_reserve_aligned(z_name, (size), \ + SOCKET_ID_ANY, \ + 0, \ + alignment); \ + mem_handle = mz; \ + } \ + if (mz == NULL) { \ + virt = NULL; \ + phys = 0; \ + } else { \ + memset(mz->addr, 0, (size)); \ + virt = mz->addr; \ + phys = mz->phys_addr; \ + } \ + } while (0) +#define ENA_MEM_ALLOC_COHERENT(dmadev, size, virt, phys, mem_handle) \ + ENA_MEM_ALLOC_COHERENT_ALIGNED(dmadev, size, virt, phys, \ + mem_handle, RTE_CACHE_LINE_SIZE) +#define ENA_MEM_FREE_COHERENT(dmadev, size, virt, phys, mem_handle) \ + ({ ENA_TOUCH(size); ENA_TOUCH(phys); ENA_TOUCH(dmadev); \ + rte_memzone_free(mem_handle); }) + +#define ENA_MEM_ALLOC_COHERENT_NODE_ALIGNED( \ + dmadev, size, virt, phys, mem_handle, node, dev_node, alignment) \ + do { \ + const struct rte_memzone *mz = NULL; \ + ENA_TOUCH(dmadev); ENA_TOUCH(dev_node); \ + if ((size) > 0) { \ + char z_name[RTE_MEMZONE_NAMESIZE]; \ + snprintf(z_name, sizeof(z_name), \ + "ena_alloc_%" PRIi64 "", \ + rte_atomic64_add_return(&ena_alloc_cnt, 1)); \ + mz = rte_memzone_reserve_aligned(z_name, (size), \ + node, 0, alignment); \ + mem_handle = mz; \ + } \ + if (mz == NULL) { \ + virt = NULL; \ + phys = 0; \ + } else { \ + memset(mz->addr, 0, (size)); \ + virt = mz->addr; \ + phys = mz->phys_addr; \ + } \ + } while (0) +#define ENA_MEM_ALLOC_COHERENT_NODE( \ + dmadev, size, virt, phys, mem_handle, node, dev_node) \ + ENA_MEM_ALLOC_COHERENT_NODE_ALIGNED(dmadev, size, virt, phys, \ + mem_handle, node, dev_node, RTE_CACHE_LINE_SIZE) +#define ENA_MEM_ALLOC_NODE(dmadev, size, virt, node, dev_node) \ + do { \ + ENA_TOUCH(dmadev); ENA_TOUCH(dev_node); \ + virt = rte_zmalloc_socket(NULL, size, 0, node); \ } while (0) #define ENA_MEM_ALLOC(dmadev, size) rte_zmalloc(NULL, size, 1) -#define ENA_MEM_FREE(dmadev, ptr) ({ENA_TOUCH(dmadev); rte_free(ptr); }) +#define ENA_MEM_FREE(dmadev, ptr, size) \ + ({ ENA_TOUCH(dmadev); ENA_TOUCH(size); rte_free(ptr); }) -#define ENA_REG_WRITE32(value, reg) rte_write32_relaxed((value), (reg)) -#define ENA_REG_READ32(reg) rte_read32_relaxed((reg)) +#define ENA_DB_SYNC(mem_handle) ((void)mem_handle) + +static inline void writel_relaxed(u32 value, volatile void *addr) +{ + *(volatile u32 *)addr = value; +} + +static inline void writeq_relaxed(u64 value, volatile void *addr) +{ + *(volatile u64 *)addr = value; +} + +static inline u32 readl_relaxed(const volatile void *addr) +{ + return *(const volatile u32 *)addr; +} + +static inline void writel(u32 value, volatile void *addr) +{ + rte_wmb(); + writel_relaxed(value, addr); +} + +static inline u32 readl(const volatile void *addr) +{ + u32 val; + val = readl_relaxed(addr); + rte_rmb(); + return val; +} + +#define ENA_REG_WRITE32(bus, value, reg) \ + ({ (void)(bus); writel((value), (reg)); }) +#define ENA_REG_WRITE32_RELAXED(bus, value, reg) \ + ({ (void)(bus); writel_relaxed((value), (reg)); }) +#define ENA_REG_READ32(bus, reg) \ + ({ (void)(bus); readl((reg)); }) +#define ENA_REG_REQAD32_RELAXED(bus, reg) \ + ({ (void)(bus); readl_relaxed((reg)); }) #define ATOMIC32_INC(i32_ptr) rte_atomic32_inc(i32_ptr) #define ATOMIC32_DEC(i32_ptr) rte_atomic32_dec(i32_ptr) @@ -236,10 +326,45 @@ #define msleep(x) rte_delay_us(x * 1000) #define udelay(x) rte_delay_us(x) +#define dma_rmb() rmb() + #define MAX_ERRNO 4095 #define IS_ERR(x) (((unsigned long)x) >= (unsigned long)-MAX_ERRNO) #define ERR_PTR(error) ((void *)(long)error) #define PTR_ERR(error) ((long)(void *)error) #define might_sleep() +#define prefetch(x) rte_prefetch0(x) +#define prefetchw(x) prefetch(x) + +#define lower_32_bits(x) ((uint32_t)(x)) +#define upper_32_bits(x) ((uint32_t)(((x) >> 16) >> 16)) + +#define ENA_TIME_EXPIRE(timeout) (timeout < rte_get_timer_cycles()) +#define ENA_GET_SYSTEM_TIMEOUT(timeout_us) \ + ((timeout_us) * rte_get_timer_hz() / 1000000 + rte_get_timer_cycles()) +#define ENA_WAIT_EVENTS_DESTROY(admin_queue) ((void)(admin_queue)) + +/* The size must be 8 byte align */ +#define ENA_MEMCPY_TO_DEVICE_64(dst, src, size) \ + do { \ + int count, i; \ + uint64_t *to = (uint64_t *)(dst); \ + const uint64_t *from = (const uint64_t *)(src); \ + count = (size) / 8; \ + for (i = 0; i < count; i++, from++, to++) \ + writeq_relaxed(*from, to); \ + } while(0) + +#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) + +#define ENA_FFS(x) ffs(x) + +void ena_rss_key_fill(void *key, size_t size); + +#define ENA_RSS_FILL_KEY(key, size) ena_rss_key_fill(key, size) + +#define ENA_INTR_INITIAL_TX_INTERVAL_USECS_PLAT 0 + +#include "ena_includes.h" #endif /* DPDK_ENA_COM_ENA_PLAT_DPDK_H_ */ Index: /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/ena_ethdev.h =================================================================== --- /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/ena_ethdev.h (revision 38965) +++ /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/ena_ethdev.h (working copy) @@ -1,40 +1,20 @@ -/*- -* BSD LICENSE -* -* Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. -* All rights reserved. -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following conditions -* are met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above copyright -* notice, this list of conditions and the following disclaimer in -* the documentation and/or other materials provided with the -* distribution. -* * Neither the name of copyright holder nor the names of its -* contributors may be used to endorse or promote products derived -* from this software without specific prior written permission. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. + * All rights reserved. + */ #ifndef _ENA_ETHDEV_H_ #define _ENA_ETHDEV_H_ +#include +#include +#include +#include +#include #include +#include +#include +#include #include "ena_com.h" @@ -42,13 +22,47 @@ #define ENA_MEM_BAR 2 #define ENA_MAX_NUM_QUEUES 128 -#define ENA_DEFAULT_RING_SIZE (1024) #define ENA_MIN_FRAME_LEN 64 #define ENA_NAME_MAX_LEN 20 #define ENA_PKT_MAX_BUFS 17 +#define ENA_RX_BUF_MIN_SIZE 1400 +#define ENA_DEFAULT_RING_SIZE 1024 + +#define ENA_MIN_MTU 128 #define ENA_MMIO_DISABLE_REG_READ BIT(0) +#define ENA_WD_TIMEOUT_SEC 3 +#define ENA_DEVICE_KALIVE_TIMEOUT (ENA_WD_TIMEOUT_SEC * rte_get_timer_hz()) + +/* While processing submitted and completed descriptors (rx and tx path + * respectively) in a loop it is desired to: + * - perform batch submissions while populating sumbissmion queue + * - avoid blocking transmission of other packets during cleanup phase + * Hence the utilization ratio of 1/8 of a queue size or max value if the size + * of the ring is very big - like 8k Rx rings. + */ +#define ENA_REFILL_THRESH_DIVIDER 8 +#define ENA_REFILL_THRESH_PACKET 256 + +#define ENA_IDX_NEXT_MASKED(idx, mask) (((idx) + 1) & (mask)) +#define ENA_IDX_ADD_MASKED(idx, n, mask) (((idx) + (n)) & (mask)) + +#define ENA_RX_RSS_TABLE_LOG_SIZE 7 +#define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE) + +#define ENA_HASH_KEY_SIZE 40 + +#define ENA_ALL_RSS_HF (ETH_RSS_NONFRAG_IPV4_TCP | ETH_RSS_NONFRAG_IPV4_UDP | \ + ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_NONFRAG_IPV6_UDP) + +#define ENA_IO_TXQ_IDX(q) (2 * (q)) +#define ENA_IO_RXQ_IDX(q) (2 * (q) + 1) +/* Reversed version of ENA_IO_RXQ_IDX */ +#define ENA_IO_RXQ_IDX_REV(q) (((q) - 1) / 2) + +extern struct ena_shared_data *ena_shared_data; + struct ena_adapter; enum ena_ring_type { @@ -63,19 +77,66 @@ struct ena_com_buf bufs[ENA_PKT_MAX_BUFS]; }; +/* Rx buffer holds only pointer to the mbuf - may be expanded in the future */ +struct ena_rx_buffer { + struct rte_mbuf *mbuf; + struct ena_com_buf ena_buf; +}; + +struct ena_calc_queue_size_ctx { + struct ena_com_dev_get_features_ctx *get_feat_ctx; + struct ena_com_dev *ena_dev; + u32 max_rx_queue_size; + u32 max_tx_queue_size; + u16 max_tx_sgl_size; + u16 max_rx_sgl_size; +}; + +struct ena_stats_tx { + u64 cnt; + u64 bytes; + u64 prepare_ctx_err; + u64 linearize; + u64 linearize_failed; + u64 tx_poll; + u64 doorbells; + u64 bad_req_id; + u64 available_desc; +}; + +struct ena_stats_rx { + u64 cnt; + u64 bytes; + u64 refill_partial; + u64 bad_csum; + u64 mbuf_alloc_fail; + u64 bad_desc_num; + u64 bad_req_id; +}; + struct ena_ring { u16 next_to_use; u16 next_to_clean; enum ena_ring_type type; enum ena_admin_placement_policy_type tx_mem_queue_type; - /* Holds the empty requests for TX OOO completions */ - uint16_t *empty_tx_reqs; + + /* Indicate there are Tx packets pushed to the device and wait for db */ + bool pkts_without_db; + + /* Holds the empty requests for TX/RX OOO completions */ + union { + uint16_t *empty_tx_reqs; + uint16_t *empty_rx_reqs; + }; + union { struct ena_tx_buffer *tx_buffer_info; /* contex of tx packet */ - struct rte_mbuf **rx_buffer_info; /* contex of rx packet */ + struct ena_rx_buffer *rx_buffer_info; /* contex of rx packet */ }; + struct rte_mbuf **rx_refill_buffer; unsigned int ring_size; /* number of tx/rx_buffer_info's entries */ + unsigned int size_mask; struct ena_com_io_cq *ena_com_io_cq; struct ena_com_io_sq *ena_com_io_sq; @@ -89,80 +150,110 @@ /* Max length PMD can push to device for LLQ */ uint8_t tx_max_header_size; int configured; + + uint8_t *push_buf_intermediate_buf; + struct ena_adapter *adapter; + u16 sgl_size; + + bool disable_meta_caching; + + union { + struct ena_stats_rx rx_stats; + struct ena_stats_tx tx_stats; + }; + + unsigned int numa_socket_id; } __rte_cache_aligned; enum ena_adapter_state { ENA_ADAPTER_STATE_FREE = 0, ENA_ADAPTER_STATE_INIT = 1, - ENA_ADAPTER_STATE_RUNNING = 2, + ENA_ADAPTER_STATE_RUNNING = 2, ENA_ADAPTER_STATE_STOPPED = 3, ENA_ADAPTER_STATE_CONFIG = 4, + ENA_ADAPTER_STATE_CLOSED = 5, }; struct ena_driver_stats { rte_atomic64_t ierrors; rte_atomic64_t oerrors; rte_atomic64_t rx_nombuf; + u64 rx_drops; }; struct ena_stats_dev { - u64 tx_timeout; - u64 io_suspend; - u64 io_resume; u64 wd_expired; - u64 interface_up; - u64 interface_down; - u64 admin_q_pause; + u64 dev_start; + u64 dev_stop; + /* + * Tx drops cannot be reported as the driver statistic, because DPDK + * rte_eth_stats structure isn't providing appropriate field for that. + * As a workaround it is being published as an extended statistic. + */ + u64 tx_drops; }; -struct ena_stats_tx { - u64 cnt; - u64 bytes; - u64 queue_stop; - u64 prepare_ctx_err; - u64 queue_wakeup; - u64 dma_mapping_err; - u64 linearize; - u64 linearize_failed; - u64 tx_poll; - u64 doorbells; - u64 missing_tx_comp; - u64 bad_req_id; +struct ena_stats_eni { + /* + * The number of packets shaped due to inbound aggregate BW + * allowance being exceeded + */ + uint64_t bw_in_allowance_exceeded; + /* + * The number of packets shaped due to outbound aggregate BW + * allowance being exceeded + */ + uint64_t bw_out_allowance_exceeded; + /* The number of packets shaped due to PPS allowance being exceeded */ + uint64_t pps_allowance_exceeded; + /* + * The number of packets shaped due to connection tracking + * allowance being exceeded and leading to failure in establishment + * of new connections + */ + uint64_t conntrack_allowance_exceeded; + /* + * The number of packets shaped due to linklocal packet rate + * allowance being exceeded + */ + uint64_t linklocal_allowance_exceeded; }; -struct ena_stats_rx { - u64 cnt; - u64 bytes; - u64 refil_partial; - u64 bad_csum; - u64 page_alloc_fail; - u64 skb_alloc_fail; - u64 dma_mapping_err; - u64 bad_desc_num; - u64 small_copy_len_pkt; +struct ena_offloads { + bool tso4_supported; + bool tx_csum_supported; + bool rx_csum_supported; + bool rss_hash_supported; }; /* board specific private data structure */ struct ena_adapter { /* OS defined structs */ - struct rte_pci_device *pdev; - struct rte_eth_dev_data *rte_eth_dev_data; - struct rte_eth_dev *rte_dev; + struct rte_eth_dev_data *edev_data; struct ena_com_dev ena_dev __rte_cache_aligned; /* TX */ struct ena_ring tx_ring[ENA_MAX_NUM_QUEUES] __rte_cache_aligned; - int tx_ring_size; + u32 max_tx_ring_size; + u16 max_tx_sgl_size; /* RX */ struct ena_ring rx_ring[ENA_MAX_NUM_QUEUES] __rte_cache_aligned; - int rx_ring_size; + u32 max_rx_ring_size; + u16 max_rx_sgl_size; - u16 num_queues; + u32 max_num_io_queues; u16 max_mtu; - u8 tso4_supported; + struct ena_offloads offloads; + + /* The admin queue isn't protected by the lock and is used to + * retrieve statistics from the device. As there is no guarantee that + * application won't try to get statistics from multiple threads, it is + * safer to lock the queue to avoid admin queue failure. + */ + rte_spinlock_t admin_lock; int id_number; char name[ENA_NAME_MAX_LEN]; @@ -174,6 +265,34 @@ struct ena_driver_stats *drv_stats; enum ena_adapter_state state; + bool link_status; + + enum ena_regs_reset_reason_types reset_reason; + + struct rte_timer timer_wd; + uint64_t timestamp_wd; + uint64_t keep_alive_timeout; + + struct ena_stats_dev dev_stats; + struct ena_stats_eni eni_stats; + + bool trigger_reset; + + bool wd_state; + + bool use_large_llq_hdr; }; +int ena_rss_reta_update(struct rte_eth_dev *dev, + struct rte_eth_rss_reta_entry64 *reta_conf, + uint16_t reta_size); +int ena_rss_reta_query(struct rte_eth_dev *dev, + struct rte_eth_rss_reta_entry64 *reta_conf, + uint16_t reta_size); +int ena_rss_hash_update(struct rte_eth_dev *dev, + struct rte_eth_rss_conf *rss_conf); +int ena_rss_hash_conf_get(struct rte_eth_dev *dev, + struct rte_eth_rss_conf *rss_conf); +int ena_rss_configure(struct ena_adapter *adapter); + #endif /* _ENA_ETHDEV_H_ */ Index: /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/ena_ethdev.c =================================================================== --- /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/ena_ethdev.c (revision 38965) +++ /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/ena_ethdev.c (working copy) @@ -1,45 +1,14 @@ -/*- -* BSD LICENSE -* -* Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. -* All rights reserved. -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following conditions -* are met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above copyright -* notice, this list of conditions and the following disclaimer in -* the documentation and/or other materials provided with the -* distribution. -* * Neither the name of copyright holder nor the names of its -* contributors may be used to endorse or promote products derived -* from this software without specific prior written permission. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. + * All rights reserved. + */ -#include -#include +#include #include -#include -#include #include #include #include -#include +#include #include "ena_ethdev.h" #include "ena_logs.h" @@ -52,38 +21,24 @@ #include #include -#define DRV_MODULE_VER_MAJOR 1 -#define DRV_MODULE_VER_MINOR 0 +#define DRV_MODULE_VER_MAJOR 2 +#define DRV_MODULE_VER_MINOR 4 #define DRV_MODULE_VER_SUBMINOR 0 -#define ENA_IO_TXQ_IDX(q) (2 * (q)) -#define ENA_IO_RXQ_IDX(q) (2 * (q) + 1) -/*reverse version of ENA_IO_RXQ_IDX*/ -#define ENA_IO_RXQ_IDX_REV(q) ((q - 1) / 2) - -/* While processing submitted and completed descriptors (rx and tx path - * respectively) in a loop it is desired to: - * - perform batch submissions while populating sumbissmion queue - * - avoid blocking transmission of other packets during cleanup phase - * Hence the utilization ratio of 1/8 of a queue size. - */ -#define ENA_RING_DESCS_RATIO(ring_size) (ring_size / 8) - #define __MERGE_64B_H_L(h, l) (((uint64_t)h << 32) | l) -#define TEST_BIT(val, bit_shift) (val & (1UL << bit_shift)) #define GET_L4_HDR_LEN(mbuf) \ ((rte_pktmbuf_mtod_offset(mbuf, struct tcp_hdr *, \ mbuf->l3_len + mbuf->l2_len)->data_off) >> 4) -#define ENA_RX_RSS_TABLE_LOG_SIZE 7 -#define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE) -#define ENA_HASH_KEY_SIZE 40 -#define ENA_ETH_SS_STATS 0xFF #define ETH_GSTRING_LEN 32 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) +#define ENA_MIN_RING_DESC 128 + +#define ENA_PTYPE_HAS_HASH (RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP) + enum ethtool_stringset { ETH_SS_TEST = 0, ETH_SS_STATS, @@ -94,11 +49,6 @@ int stat_offset; }; -#define ENA_STAT_ENA_COM_ENTRY(stat) { \ - .name = #stat, \ - .stat_offset = offsetof(struct ena_com_stats_admin, stat) \ -} - #define ENA_STAT_ENTRY(stat, stat_type) { \ .name = #stat, \ .stat_offset = offsetof(struct ena_stats_##stat_type, stat) \ @@ -110,86 +60,91 @@ #define ENA_STAT_TX_ENTRY(stat) \ ENA_STAT_ENTRY(stat, tx) +#define ENA_STAT_ENI_ENTRY(stat) \ + ENA_STAT_ENTRY(stat, eni) + #define ENA_STAT_GLOBAL_ENTRY(stat) \ ENA_STAT_ENTRY(stat, dev) +/* Device arguments */ +#define ENA_DEVARG_LARGE_LLQ_HDR "large_llq_hdr" + +/* + * Each rte_memzone should have unique name. + * To satisfy it, count number of allocation and add it to name. + */ +rte_atomic64_t ena_alloc_cnt; + static const struct ena_stats ena_stats_global_strings[] = { - ENA_STAT_GLOBAL_ENTRY(tx_timeout), - ENA_STAT_GLOBAL_ENTRY(io_suspend), - ENA_STAT_GLOBAL_ENTRY(io_resume), ENA_STAT_GLOBAL_ENTRY(wd_expired), - ENA_STAT_GLOBAL_ENTRY(interface_up), - ENA_STAT_GLOBAL_ENTRY(interface_down), - ENA_STAT_GLOBAL_ENTRY(admin_q_pause), + ENA_STAT_GLOBAL_ENTRY(dev_start), + ENA_STAT_GLOBAL_ENTRY(dev_stop), + ENA_STAT_GLOBAL_ENTRY(tx_drops), +}; + +static const struct ena_stats ena_stats_eni_strings[] = { + ENA_STAT_ENI_ENTRY(bw_in_allowance_exceeded), + ENA_STAT_ENI_ENTRY(bw_out_allowance_exceeded), + ENA_STAT_ENI_ENTRY(pps_allowance_exceeded), + ENA_STAT_ENI_ENTRY(conntrack_allowance_exceeded), + ENA_STAT_ENI_ENTRY(linklocal_allowance_exceeded), }; static const struct ena_stats ena_stats_tx_strings[] = { ENA_STAT_TX_ENTRY(cnt), ENA_STAT_TX_ENTRY(bytes), - ENA_STAT_TX_ENTRY(queue_stop), - ENA_STAT_TX_ENTRY(queue_wakeup), - ENA_STAT_TX_ENTRY(dma_mapping_err), + ENA_STAT_TX_ENTRY(prepare_ctx_err), ENA_STAT_TX_ENTRY(linearize), ENA_STAT_TX_ENTRY(linearize_failed), ENA_STAT_TX_ENTRY(tx_poll), ENA_STAT_TX_ENTRY(doorbells), - ENA_STAT_TX_ENTRY(prepare_ctx_err), - ENA_STAT_TX_ENTRY(missing_tx_comp), ENA_STAT_TX_ENTRY(bad_req_id), + ENA_STAT_TX_ENTRY(available_desc), }; static const struct ena_stats ena_stats_rx_strings[] = { ENA_STAT_RX_ENTRY(cnt), ENA_STAT_RX_ENTRY(bytes), - ENA_STAT_RX_ENTRY(refil_partial), + ENA_STAT_RX_ENTRY(refill_partial), ENA_STAT_RX_ENTRY(bad_csum), - ENA_STAT_RX_ENTRY(page_alloc_fail), - ENA_STAT_RX_ENTRY(skb_alloc_fail), - ENA_STAT_RX_ENTRY(dma_mapping_err), + ENA_STAT_RX_ENTRY(mbuf_alloc_fail), ENA_STAT_RX_ENTRY(bad_desc_num), - ENA_STAT_RX_ENTRY(small_copy_len_pkt), -}; - -static const struct ena_stats ena_stats_ena_com_strings[] = { - ENA_STAT_ENA_COM_ENTRY(aborted_cmd), - ENA_STAT_ENA_COM_ENTRY(submitted_cmd), - ENA_STAT_ENA_COM_ENTRY(completed_cmd), - ENA_STAT_ENA_COM_ENTRY(out_of_space), - ENA_STAT_ENA_COM_ENTRY(no_completion), + ENA_STAT_RX_ENTRY(bad_req_id), }; #define ENA_STATS_ARRAY_GLOBAL ARRAY_SIZE(ena_stats_global_strings) +#define ENA_STATS_ARRAY_ENI ARRAY_SIZE(ena_stats_eni_strings) #define ENA_STATS_ARRAY_TX ARRAY_SIZE(ena_stats_tx_strings) #define ENA_STATS_ARRAY_RX ARRAY_SIZE(ena_stats_rx_strings) -#define ENA_STATS_ARRAY_ENA_COM ARRAY_SIZE(ena_stats_ena_com_strings) /** Vendor ID used by Amazon devices */ #define PCI_VENDOR_ID_AMAZON 0x1D0F /** Amazon devices */ -#define PCI_DEVICE_ID_ENA_VF 0xEC20 -#define PCI_DEVICE_ID_ENA_LLQ_VF 0xEC21 +#define PCI_DEVICE_ID_ENA_VF 0xEC20 +#define PCI_DEVICE_ID_ENA_VF_RSERV0 0xEC21 -#define ENA_TX_OFFLOAD_MASK (\ - PKT_TX_L4_MASK | \ - PKT_TX_IP_CKSUM | \ - PKT_TX_TCP_SEG) - -#define ENA_TX_OFFLOAD_NOTSUP_MASK \ - (PKT_TX_OFFLOAD_MASK ^ ENA_TX_OFFLOAD_MASK) - -static const struct rte_pci_id pci_id_ena_map[] = { +static struct rte_pci_id pci_id_ena_map[] = { { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_VF) }, - { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_LLQ_VF) }, + { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_VF_RSERV0) }, { .device_id = 0 }, }; +static struct ena_aenq_handlers aenq_handlers; + static int ena_device_init(struct ena_com_dev *ena_dev, - struct ena_com_dev_get_features_ctx *get_feat_ctx); + struct rte_pci_device *pdev, + struct ena_com_dev_get_features_ctx *get_feat_ctx, + bool *wd_state); static int ena_dev_configure(struct rte_eth_dev *dev); +static void ena_tx_map_mbuf(struct ena_ring *tx_ring, + struct ena_tx_buffer *tx_info, + struct rte_mbuf *mbuf, + void **push_header, + uint16_t *header_len); +static int ena_xmit_mbuf(struct ena_ring *tx_ring, struct rte_mbuf *mbuf); +static void ena_tx_cleanup(struct ena_ring *tx_ring); static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); -static uint16_t eth_ena_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, - uint16_t nb_pkts); static int ena_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc, unsigned int socket_id, const struct rte_eth_txconf *tx_conf); @@ -197,12 +152,22 @@ uint16_t nb_desc, unsigned int socket_id, const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp); +static inline void ena_init_rx_mbuf(struct rte_mbuf *mbuf, uint16_t len); +static struct rte_mbuf *ena_rx_mbuf(struct ena_ring *rx_ring, + struct ena_com_rx_buf_info *ena_bufs, + uint32_t descs, + uint16_t *next_to_clean, + uint8_t offset); static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); +static int ena_add_single_rx_desc(struct ena_com_io_sq *io_sq, + struct rte_mbuf *mbuf, uint16_t id); static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count); -static void ena_init_rings(struct ena_adapter *adapter); +static void ena_init_rings(struct ena_adapter *adapter, + bool disable_meta_caching); static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); static int ena_start(struct rte_eth_dev *dev); +static void ena_stop(struct rte_eth_dev *dev); static void ena_close(struct rte_eth_dev *dev); static void ena_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); static void ena_rx_queue_release_all(struct rte_eth_dev *dev); @@ -213,19 +178,38 @@ static void ena_tx_queue_release_bufs(struct ena_ring *ring); static int ena_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete); -static int ena_queue_restart(struct ena_ring *ring); -static int ena_queue_restart_all(struct rte_eth_dev *dev, - enum ena_ring_type ring_type); +static int ena_create_io_queue(struct rte_eth_dev *dev, struct ena_ring *ring); +static void ena_queue_stop(struct ena_ring *ring); +static void ena_queue_stop_all(struct rte_eth_dev *dev, + enum ena_ring_type ring_type); +static int ena_queue_start(struct rte_eth_dev *dev, struct ena_ring *ring); +static int ena_queue_start_all(struct rte_eth_dev *dev, + enum ena_ring_type ring_type); static void ena_stats_restart(struct rte_eth_dev *dev); static void ena_infos_get(__rte_unused struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info); -static int ena_rss_reta_update(struct rte_eth_dev *dev, - struct rte_eth_rss_reta_entry64 *reta_conf, - uint16_t reta_size); -static int ena_rss_reta_query(struct rte_eth_dev *dev, - struct rte_eth_rss_reta_entry64 *reta_conf, - uint16_t reta_size); -static int ena_get_sset_count(struct rte_eth_dev *dev, int sset); +static void ena_interrupt_handler_rte(struct rte_intr_handle *intr_handle, + void *cb_arg); +static void ena_timer_wd_callback(struct rte_timer *timer, void *arg); +static void ena_destroy_device(struct rte_eth_dev *eth_dev); +static int eth_ena_dev_init(struct rte_eth_dev *eth_dev); +static int ena_xstats_get_names(struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, + unsigned int n); +static int ena_xstats_get(struct rte_eth_dev *dev, + struct rte_eth_xstat *stats, + unsigned int n); +static int ena_process_bool_devarg(const char *key, + const char *value, + void *opaque); +static int ena_parse_devargs(struct ena_adapter *adapter, + struct rte_devargs *devargs); +static int ena_copy_eni_stats(struct ena_adapter *adapter); +static int ena_setup_rx_intr(struct rte_eth_dev *dev); +static int ena_rx_queue_intr_enable(struct rte_eth_dev *dev, + uint16_t queue_id); +static int ena_rx_queue_intr_disable(struct rte_eth_dev *dev, + uint16_t queue_id); static const struct eth_dev_ops ena_dev_ops = { .dev_configure = ena_dev_configure, @@ -233,53 +217,64 @@ .rx_queue_setup = ena_rx_queue_setup, .tx_queue_setup = ena_tx_queue_setup, .dev_start = ena_start, + .dev_stop = ena_stop, .link_update = ena_link_update, .stats_get = ena_stats_get, + .xstats_get_names = ena_xstats_get_names, + .xstats_get = ena_xstats_get, .mtu_set = ena_mtu_set, .rx_queue_release = ena_rx_queue_release, .tx_queue_release = ena_tx_queue_release, .dev_close = ena_close, .reta_update = ena_rss_reta_update, .reta_query = ena_rss_reta_query, + .rx_queue_intr_enable = ena_rx_queue_intr_enable, + .rx_queue_intr_disable = ena_rx_queue_intr_disable, + .rss_hash_update = ena_rss_hash_update, + .rss_hash_conf_get = ena_rss_hash_conf_get, }; -#define NUMA_NO_NODE SOCKET_ID_ANY - -static inline int ena_cpu_to_node(int cpu) -{ - struct rte_config *config = rte_eal_get_configuration(); - - if (likely(cpu < RTE_MAX_MEMZONE)) - return config->mem_config->memzone[cpu].socket_id; - - return NUMA_NO_NODE; -} - static inline void ena_rx_mbuf_prepare(struct rte_mbuf *mbuf, struct ena_com_rx_ctx *ena_rx_ctx) { uint64_t ol_flags = 0; + uint32_t packet_type = 0; if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) - ol_flags |= PKT_TX_TCP_CKSUM; + packet_type |= RTE_PTYPE_L4_TCP; else if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP) - ol_flags |= PKT_TX_UDP_CKSUM; + packet_type |= RTE_PTYPE_L4_UDP; + + if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) { + packet_type |= RTE_PTYPE_L3_IPV4; + if (unlikely(ena_rx_ctx->l3_csum_err)) + ol_flags |= PKT_RX_IP_CKSUM_BAD; + else + ol_flags |= PKT_RX_IP_CKSUM_GOOD; + } else if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV6) { + packet_type |= RTE_PTYPE_L3_IPV6; + } - if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) - ol_flags |= PKT_TX_IPV4; - else if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV6) - ol_flags |= PKT_TX_IPV6; - - if (unlikely(ena_rx_ctx->l4_csum_err)) - ol_flags |= PKT_RX_L4_CKSUM_BAD; - if (unlikely(ena_rx_ctx->l3_csum_err)) - ol_flags |= PKT_RX_IP_CKSUM_BAD; + if (!ena_rx_ctx->l4_csum_checked || ena_rx_ctx->frag) + ol_flags |= PKT_RX_L4_CKSUM_UNKNOWN; + else + if (unlikely(ena_rx_ctx->l4_csum_err)) + ol_flags |= PKT_RX_L4_CKSUM_BAD; + else + ol_flags |= PKT_RX_L4_CKSUM_GOOD; + + if (likely((packet_type & ENA_PTYPE_HAS_HASH) && !ena_rx_ctx->frag)) { + ol_flags |= PKT_RX_RSS_HASH; + mbuf->hash.rss = ena_rx_ctx->hash; + } mbuf->ol_flags = ol_flags; + mbuf->packet_type = packet_type; } static inline void ena_tx_mbuf_prepare(struct rte_mbuf *mbuf, - struct ena_com_tx_ctx *ena_tx_ctx) + struct ena_com_tx_ctx *ena_tx_ctx, + bool disable_meta_caching) { struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta; @@ -327,16 +322,38 @@ ena_meta->mss = mbuf->tso_segsz; ena_meta->l3_hdr_len = mbuf->l3_len; ena_meta->l3_hdr_offset = mbuf->l2_len; - /* this param needed only for TSO */ - ena_meta->l3_outer_hdr_len = 0; - ena_meta->l3_outer_hdr_offset = 0; ena_tx_ctx->meta_valid = true; + } else if (disable_meta_caching) { + memset(ena_meta, 0, sizeof(*ena_meta)); + ena_tx_ctx->meta_valid = true; } else { ena_tx_ctx->meta_valid = false; } } +static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id) +{ + struct ena_tx_buffer *tx_info = NULL; + + if (likely(req_id < tx_ring->ring_size)) { + tx_info = &tx_ring->tx_buffer_info[req_id]; + if (likely(tx_info->mbuf)) + return 0; + } + + if (tx_info) + PMD_TX_LOG(ERR, "tx_info doesn't have valid mbuf\n"); + else + PMD_TX_LOG(ERR, "Invalid req_id: %hu\n", req_id); + + /* Trigger device reset */ + ++tx_ring->tx_stats.bad_req_id; + tx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID; + tx_ring->adapter->trigger_reset = true; + return -EFAULT; +} + static void ena_config_host_info(struct ena_com_dev *ena_dev) { struct ena_admin_host_info *host_info; @@ -345,7 +362,7 @@ /* Allocate only the host info */ rc = ena_com_allocate_host_info(ena_dev); if (rc) { - RTE_LOG(ERR, PMD, "Cannot allocate host info\n"); + PMD_INIT_LOG(ERR, "Cannot allocate host info\n"); return; } @@ -365,12 +382,20 @@ (DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) | (DRV_MODULE_VER_SUBMINOR << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT); + host_info->num_cpus = rte_lcore_count(); + + host_info->driver_supported_features = + ENA_ADMIN_HOST_INFO_RX_OFFSET_MASK | + ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_MASK; rc = ena_com_set_host_attributes(ena_dev); if (rc) { - RTE_LOG(ERR, PMD, "Cannot set host attributes\n"); - if (rc != -EPERM) - goto err; + if (rc == -ENA_COM_UNSUPPORTED) + PMD_INIT_LOG(WARNING, "Cannot set host attributes\n"); + else + PMD_INIT_LOG(ERR, "Cannot set host attributes\n"); + + goto err; } return; @@ -379,24 +404,12 @@ ena_com_delete_host_info(ena_dev); } -static int -ena_get_sset_count(struct rte_eth_dev *dev, int sset) +/* This function calculates the number of xstats based on the current config */ +static unsigned int ena_xstats_calc_num(struct rte_eth_dev_data *data) { - if (sset != ETH_SS_STATS) - return -EOPNOTSUPP; - - /* Workaround for clang: - * touch internal structures to prevent - * compiler error - */ - ENA_TOUCH(ena_stats_global_strings); - ENA_TOUCH(ena_stats_tx_strings); - ENA_TOUCH(ena_stats_rx_strings); - ENA_TOUCH(ena_stats_ena_com_strings); - - return dev->data->nb_tx_queues * - (ENA_STATS_ARRAY_TX + ENA_STATS_ARRAY_RX) + - ENA_STATS_ARRAY_GLOBAL + ENA_STATS_ARRAY_ENA_COM; + return ENA_STATS_ARRAY_GLOBAL + ENA_STATS_ARRAY_ENI + + (data->nb_tx_queues * ENA_STATS_ARRAY_TX) + + (data->nb_rx_queues * ENA_STATS_ARRAY_RX); } static void ena_config_debug_area(struct ena_adapter *adapter) @@ -404,26 +417,25 @@ u32 debug_area_size; int rc, ss_count; - ss_count = ena_get_sset_count(adapter->rte_dev, ETH_SS_STATS); - if (ss_count <= 0) { - RTE_LOG(ERR, PMD, "SS count is negative\n"); - return; - } + ss_count = ena_xstats_calc_num(adapter->edev_data); /* allocate 32 bytes for each string and 64bit for the value */ debug_area_size = ss_count * ETH_GSTRING_LEN + sizeof(u64) * ss_count; rc = ena_com_allocate_debug_area(&adapter->ena_dev, debug_area_size); if (rc) { - RTE_LOG(ERR, PMD, "Cannot allocate debug area\n"); + PMD_INIT_LOG(ERR, "Cannot allocate debug area\n"); return; } rc = ena_com_set_host_attributes(&adapter->ena_dev); if (rc) { - RTE_LOG(WARNING, PMD, "Cannot set host attributes\n"); - if (rc != -EPERM) - goto err; + if (rc == -ENA_COM_UNSUPPORTED) + PMD_INIT_LOG(WARNING, "Cannot set host attributes\n"); + else + PMD_INIT_LOG(ERR, "Cannot set host attributes\n"); + + goto err; } return; @@ -433,159 +445,30 @@ static void ena_close(struct rte_eth_dev *dev) { - struct ena_adapter *adapter = - (struct ena_adapter *)(dev->data->dev_private); - - adapter->state = ENA_ADAPTER_STATE_STOPPED; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; + struct ena_adapter *adapter = dev->data->dev_private; + + if (adapter->state == ENA_ADAPTER_STATE_RUNNING) + ena_stop(dev); + adapter->state = ENA_ADAPTER_STATE_CLOSED; ena_rx_queue_release_all(dev); ena_tx_queue_release_all(dev); -} - -static int ena_rss_reta_update(struct rte_eth_dev *dev, - struct rte_eth_rss_reta_entry64 *reta_conf, - uint16_t reta_size) -{ - struct ena_adapter *adapter = - (struct ena_adapter *)(dev->data->dev_private); - struct ena_com_dev *ena_dev = &adapter->ena_dev; - int ret, i; - u16 entry_value; - int conf_idx; - int idx; - - if ((reta_size == 0) || (reta_conf == NULL)) - return -EINVAL; - - if (reta_size > ENA_RX_RSS_TABLE_SIZE) { - RTE_LOG(WARNING, PMD, - "indirection table %d is bigger than supported (%d)\n", - reta_size, ENA_RX_RSS_TABLE_SIZE); - ret = -EINVAL; - goto err; - } - - for (i = 0 ; i < reta_size ; i++) { - /* each reta_conf is for 64 entries. - * to support 128 we use 2 conf of 64 - */ - conf_idx = i / RTE_RETA_GROUP_SIZE; - idx = i % RTE_RETA_GROUP_SIZE; - if (TEST_BIT(reta_conf[conf_idx].mask, idx)) { - entry_value = - ENA_IO_RXQ_IDX(reta_conf[conf_idx].reta[idx]); - ret = ena_com_indirect_table_fill_entry(ena_dev, - i, - entry_value); - if (unlikely(ret && (ret != ENA_COM_PERMISSION))) { - RTE_LOG(ERR, PMD, - "Cannot fill indirect table\n"); - ret = -ENOTSUP; - goto err; - } - } - } - - ret = ena_com_indirect_table_set(ena_dev); - if (unlikely(ret && (ret != ENA_COM_PERMISSION))) { - RTE_LOG(ERR, PMD, "Cannot flush the indirect table\n"); - ret = -ENOTSUP; - goto err; - } - - RTE_LOG(DEBUG, PMD, "%s(): RSS configured %d entries for port %d\n", - __func__, reta_size, adapter->rte_dev->data->port_id); -err: - return ret; -} - -/* Query redirection table. */ -static int ena_rss_reta_query(struct rte_eth_dev *dev, - struct rte_eth_rss_reta_entry64 *reta_conf, - uint16_t reta_size) -{ - struct ena_adapter *adapter = - (struct ena_adapter *)(dev->data->dev_private); - struct ena_com_dev *ena_dev = &adapter->ena_dev; - int ret; - int i; - u32 indirect_table[ENA_RX_RSS_TABLE_SIZE] = {0}; - int reta_conf_idx; - int reta_idx; - - if (reta_size == 0 || reta_conf == NULL || - (reta_size > RTE_RETA_GROUP_SIZE && ((reta_conf + 1) == NULL))) - return -EINVAL; - - ret = ena_com_indirect_table_get(ena_dev, indirect_table); - if (unlikely(ret && (ret != ENA_COM_PERMISSION))) { - RTE_LOG(ERR, PMD, "cannot get indirect table\n"); - ret = -ENOTSUP; - goto err; - } - - for (i = 0 ; i < reta_size ; i++) { - reta_conf_idx = i / RTE_RETA_GROUP_SIZE; - reta_idx = i % RTE_RETA_GROUP_SIZE; - if (TEST_BIT(reta_conf[reta_conf_idx].mask, reta_idx)) - reta_conf[reta_conf_idx].reta[reta_idx] = - ENA_IO_RXQ_IDX_REV(indirect_table[i]); - } -err: - return ret; -} - -static int ena_rss_init_default(struct ena_adapter *adapter) -{ - struct ena_com_dev *ena_dev = &adapter->ena_dev; - uint16_t nb_rx_queues = adapter->rte_dev->data->nb_rx_queues; - int rc, i; - u32 val; - - rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE); - if (unlikely(rc)) { - RTE_LOG(ERR, PMD, "Cannot init indirect table\n"); - goto err_rss_init; - } - for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { - val = i % nb_rx_queues; - rc = ena_com_indirect_table_fill_entry(ena_dev, i, - ENA_IO_RXQ_IDX(val)); - if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { - RTE_LOG(ERR, PMD, "Cannot fill indirect table\n"); - goto err_fill_indir; - } - } + rte_free(adapter->drv_stats); + adapter->drv_stats = NULL; - rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL, - ENA_HASH_KEY_SIZE, 0xFFFFFFFF); - if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { - RTE_LOG(INFO, PMD, "Cannot fill hash function\n"); - goto err_fill_indir; - } - - rc = ena_com_set_default_hash_ctrl(ena_dev); - if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { - RTE_LOG(INFO, PMD, "Cannot fill hash control\n"); - goto err_fill_indir; - } - - rc = ena_com_indirect_table_set(ena_dev); - if (unlikely(rc && (rc != ENA_COM_PERMISSION))) { - RTE_LOG(ERR, PMD, "Cannot flush the indirect table\n"); - goto err_fill_indir; - } - RTE_LOG(DEBUG, PMD, "RSS configured for port %d\n", - adapter->rte_dev->data->port_id); - - return 0; - -err_fill_indir: - ena_com_rss_destroy(ena_dev); -err_rss_init: - - return rc; + rte_intr_disable(intr_handle); + rte_intr_callback_unregister(intr_handle, + ena_interrupt_handler_rte, + dev); + + /* + * MAC is not allocated dynamically. Setting NULL should prevent from + * release of the resource in the rte_eth_dev_release_port(). + */ + dev->data->mac_addrs = NULL; } static void ena_rx_queue_release_all(struct rte_eth_dev *dev) @@ -611,51 +494,34 @@ static void ena_rx_queue_release(void *queue) { struct ena_ring *ring = (struct ena_ring *)queue; - struct ena_adapter *adapter = ring->adapter; - int ena_qid; - - ena_assert_msg(ring->configured, - "API violation - releasing not configured queue"); - ena_assert_msg(ring->adapter->state != ENA_ADAPTER_STATE_RUNNING, - "API violation"); - - /* Destroy HW queue */ - ena_qid = ENA_IO_RXQ_IDX(ring->id); - ena_com_destroy_io_queue(&adapter->ena_dev, ena_qid); - - /* Free all bufs */ - ena_rx_queue_release_bufs(ring); /* Free ring resources */ if (ring->rx_buffer_info) rte_free(ring->rx_buffer_info); ring->rx_buffer_info = NULL; + if (ring->rx_refill_buffer) + rte_free(ring->rx_refill_buffer); + ring->rx_refill_buffer = NULL; + + if (ring->empty_rx_reqs) + rte_free(ring->empty_rx_reqs); + ring->empty_rx_reqs = NULL; + ring->configured = 0; - RTE_LOG(NOTICE, PMD, "RX Queue %d:%d released\n", + PMD_DRV_LOG(NOTICE, "Rx queue %d:%d released\n", ring->port_id, ring->id); } static void ena_tx_queue_release(void *queue) { struct ena_ring *ring = (struct ena_ring *)queue; - struct ena_adapter *adapter = ring->adapter; - int ena_qid; - - ena_assert_msg(ring->configured, - "API violation. Releasing not configured queue"); - ena_assert_msg(ring->adapter->state != ENA_ADAPTER_STATE_RUNNING, - "API violation"); - - /* Destroy HW queue */ - ena_qid = ENA_IO_TXQ_IDX(ring->id); - ena_com_destroy_io_queue(&adapter->ena_dev, ena_qid); - - /* Free all bufs */ - ena_tx_queue_release_bufs(ring); /* Free ring resources */ + if (ring->push_buf_intermediate_buf) + rte_free(ring->push_buf_intermediate_buf); + if (ring->tx_buffer_info) rte_free(ring->tx_buffer_info); @@ -664,40 +530,38 @@ ring->empty_tx_reqs = NULL; ring->tx_buffer_info = NULL; + ring->push_buf_intermediate_buf = NULL; ring->configured = 0; - RTE_LOG(NOTICE, PMD, "TX Queue %d:%d released\n", + PMD_DRV_LOG(NOTICE, "Tx queue %d:%d released\n", ring->port_id, ring->id); } static void ena_rx_queue_release_bufs(struct ena_ring *ring) { - unsigned int ring_mask = ring->ring_size - 1; - - while (ring->next_to_clean != ring->next_to_use) { - struct rte_mbuf *m = - ring->rx_buffer_info[ring->next_to_clean & ring_mask]; - - if (m) - __rte_mbuf_raw_free(m); + unsigned int i; - ring->next_to_clean++; + for (i = 0; i < ring->ring_size; ++i) { + struct ena_rx_buffer *rx_info = &ring->rx_buffer_info[i]; + if (rx_info->mbuf) { + __rte_mbuf_raw_free(rx_info->mbuf); + rx_info->mbuf = NULL; + } } } static void ena_tx_queue_release_bufs(struct ena_ring *ring) { - unsigned int ring_mask = ring->ring_size - 1; + unsigned int i; - while (ring->next_to_clean != ring->next_to_use) { - struct ena_tx_buffer *tx_buf = - &ring->tx_buffer_info[ring->next_to_clean & ring_mask]; + for (i = 0; i < ring->ring_size; ++i) { + struct ena_tx_buffer *tx_buf = &ring->tx_buffer_info[i]; - if (tx_buf->mbuf) + if (tx_buf->mbuf) { rte_pktmbuf_free(tx_buf->mbuf); - - ring->next_to_clean++; + tx_buf->mbuf = NULL; + } } } @@ -705,59 +569,71 @@ __rte_unused int wait_to_complete) { struct rte_eth_link *link = &dev->data->dev_link; + struct ena_adapter *adapter = dev->data->dev_private; - link->link_status = 1; - link->link_speed = ETH_SPEED_NUM_10G; + link->link_status = adapter->link_status ? ETH_LINK_UP : ETH_LINK_DOWN; + link->link_speed = ETH_SPEED_NUM_NONE; link->link_duplex = ETH_LINK_FULL_DUPLEX; return 0; } -static int ena_queue_restart_all(struct rte_eth_dev *dev, - enum ena_ring_type ring_type) +static int ena_queue_start_all(struct rte_eth_dev *dev, + enum ena_ring_type ring_type) { - struct ena_adapter *adapter = - (struct ena_adapter *)(dev->data->dev_private); + struct ena_adapter *adapter = dev->data->dev_private; struct ena_ring *queues = NULL; + int nb_queues; int i = 0; int rc = 0; - queues = (ring_type == ENA_RING_TYPE_RX) ? - adapter->rx_ring : adapter->tx_ring; - - for (i = 0; i < adapter->num_queues; i++) { + if (ring_type == ENA_RING_TYPE_RX) { + queues = adapter->rx_ring; + nb_queues = dev->data->nb_rx_queues; + } else { + queues = adapter->tx_ring; + nb_queues = dev->data->nb_tx_queues; + } + for (i = 0; i < nb_queues; i++) { if (queues[i].configured) { if (ring_type == ENA_RING_TYPE_RX) { ena_assert_msg( dev->data->rx_queues[i] == &queues[i], - "Inconsistent state of rx queues\n"); + "Inconsistent state of Rx queues\n"); } else { ena_assert_msg( dev->data->tx_queues[i] == &queues[i], - "Inconsistent state of tx queues\n"); + "Inconsistent state of Tx queues\n"); } - rc = ena_queue_restart(&queues[i]); + rc = ena_queue_start(dev, &queues[i]); if (rc) { PMD_INIT_LOG(ERR, - "failed to restart queue %d type(%d)", - i, ring_type); - return -1; + "Failed to start queue[%d] of type(%d)\n", + i, ring_type); + goto err; } } } return 0; + +err: + while (i--) + if (queues[i].configured) + ena_queue_stop(&queues[i]); + + return rc; } static uint32_t ena_get_mtu_conf(struct ena_adapter *adapter) { uint32_t max_frame_len = adapter->max_mtu; - if (adapter->rte_eth_dev_data->dev_conf.rxmode.jumbo_frame == 1) + if (adapter->edev_data->dev_conf.rxmode.jumbo_frame == 1) max_frame_len = - adapter->rte_eth_dev_data->dev_conf.rxmode.max_rx_pkt_len; + adapter->edev_data->dev_conf.rxmode.max_rx_pkt_len; return max_frame_len; } @@ -766,67 +642,127 @@ { uint32_t max_frame_len = ena_get_mtu_conf(adapter); - if (max_frame_len > adapter->max_mtu) { - PMD_INIT_LOG(ERR, "Unsupported MTU of %d", max_frame_len); - return -1; + if (max_frame_len > adapter->max_mtu || max_frame_len < ENA_MIN_MTU) { + PMD_INIT_LOG(ERR, + "Unsupported MTU of %d. Max MTU: %d, min MTU: %d\n", + max_frame_len, adapter->max_mtu, ENA_MIN_MTU); + return ENA_COM_UNSUPPORTED; } return 0; } static int -ena_calc_queue_size(struct ena_com_dev *ena_dev, - struct ena_com_dev_get_features_ctx *get_feat_ctx) +ena_calc_io_queue_size(struct ena_calc_queue_size_ctx *ctx, + bool use_large_llq_hdr) { - uint32_t queue_size = ENA_DEFAULT_RING_SIZE; - - queue_size = RTE_MIN(queue_size, - get_feat_ctx->max_queues.max_cq_depth); - queue_size = RTE_MIN(queue_size, - get_feat_ctx->max_queues.max_sq_depth); + struct ena_admin_feature_llq_desc *llq = &ctx->get_feat_ctx->llq; + struct ena_com_dev *ena_dev = ctx->ena_dev; + uint32_t max_tx_queue_size; + uint32_t max_rx_queue_size; + + if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) { + struct ena_admin_queue_ext_feature_fields *max_queue_ext = + &ctx->get_feat_ctx->max_queue_ext.max_queue_ext; + max_rx_queue_size = RTE_MIN(max_queue_ext->max_rx_cq_depth, + max_queue_ext->max_rx_sq_depth); + max_tx_queue_size = max_queue_ext->max_tx_cq_depth; + + if (ena_dev->tx_mem_queue_type == + ENA_ADMIN_PLACEMENT_POLICY_DEV) { + max_tx_queue_size = RTE_MIN(max_tx_queue_size, + llq->max_llq_depth); + } else { + max_tx_queue_size = RTE_MIN(max_tx_queue_size, + max_queue_ext->max_tx_sq_depth); + } - if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) - queue_size = RTE_MIN(queue_size, - get_feat_ctx->max_queues.max_llq_depth); + ctx->max_rx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS, + max_queue_ext->max_per_packet_rx_descs); + ctx->max_tx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS, + max_queue_ext->max_per_packet_tx_descs); + } else { + struct ena_admin_queue_feature_desc *max_queues = + &ctx->get_feat_ctx->max_queues; + max_rx_queue_size = RTE_MIN(max_queues->max_cq_depth, + max_queues->max_sq_depth); + max_tx_queue_size = max_queues->max_cq_depth; + + if (ena_dev->tx_mem_queue_type == + ENA_ADMIN_PLACEMENT_POLICY_DEV) { + max_tx_queue_size = RTE_MIN(max_tx_queue_size, + llq->max_llq_depth); + } else { + max_tx_queue_size = RTE_MIN(max_tx_queue_size, + max_queues->max_sq_depth); + } - /* Round down to power of 2 */ - if (!rte_is_power_of_2(queue_size)) - queue_size = rte_align32pow2(queue_size >> 1); + ctx->max_rx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS, + max_queues->max_packet_rx_descs); + ctx->max_tx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS, + max_queues->max_packet_tx_descs); + } + + /* Round down to the nearest power of 2 */ + max_rx_queue_size = rte_align32prevpow2(max_rx_queue_size); + max_tx_queue_size = rte_align32prevpow2(max_tx_queue_size); + + if (use_large_llq_hdr) { + if ((llq->entry_size_ctrl_supported & + ENA_ADMIN_LIST_ENTRY_SIZE_256B) && + (ena_dev->tx_mem_queue_type == + ENA_ADMIN_PLACEMENT_POLICY_DEV)) { + max_tx_queue_size /= 2; + PMD_INIT_LOG(INFO, + "Forcing large headers and decreasing maximum Tx queue size to %d\n", + max_tx_queue_size); + } else { + PMD_INIT_LOG(ERR, + "Forcing large headers failed: LLQ is disabled or device does not support large headers\n"); + } + } - if (queue_size == 0) { - PMD_INIT_LOG(ERR, "Invalid queue size"); + if (unlikely(max_rx_queue_size == 0 || max_tx_queue_size == 0)) { + PMD_INIT_LOG(ERR, "Invalid queue size\n"); return -EFAULT; } - return queue_size; + ctx->max_tx_queue_size = max_tx_queue_size; + ctx->max_rx_queue_size = max_rx_queue_size; + + return 0; } static void ena_stats_restart(struct rte_eth_dev *dev) { - struct ena_adapter *adapter = - (struct ena_adapter *)(dev->data->dev_private); + struct ena_adapter *adapter = dev->data->dev_private; rte_atomic64_init(&adapter->drv_stats->ierrors); rte_atomic64_init(&adapter->drv_stats->oerrors); rte_atomic64_init(&adapter->drv_stats->rx_nombuf); + adapter->drv_stats->rx_drops = 0; } static void ena_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { struct ena_admin_basic_stats ena_stats; - struct ena_adapter *adapter = - (struct ena_adapter *)(dev->data->dev_private); + struct ena_adapter *adapter = dev->data->dev_private; struct ena_com_dev *ena_dev = &adapter->ena_dev; int rc; + int i; + int max_rings_stats; if (rte_eal_process_type() != RTE_PROC_PRIMARY) return; memset(&ena_stats, 0, sizeof(ena_stats)); + + rte_spinlock_lock(&adapter->admin_lock); rc = ena_com_get_dev_basic_stats(ena_dev, &ena_stats); + rte_spinlock_unlock(&adapter->admin_lock); if (unlikely(rc)) { - RTE_LOG(ERR, PMD, "Could not retrieve statistics from ENA"); + PMD_DRV_LOG(ERR, "Could not retrieve statistics from ENA\n"); return; } @@ -839,13 +775,32 @@ ena_stats.rx_bytes_low); stats->obytes = __MERGE_64B_H_L(ena_stats.tx_bytes_high, ena_stats.tx_bytes_low); - stats->imissed = __MERGE_64B_H_L(ena_stats.rx_drops_high, - ena_stats.rx_drops_low); /* Driver related stats */ + stats->imissed = adapter->drv_stats->rx_drops; stats->ierrors = rte_atomic64_read(&adapter->drv_stats->ierrors); stats->oerrors = rte_atomic64_read(&adapter->drv_stats->oerrors); stats->rx_nombuf = rte_atomic64_read(&adapter->drv_stats->rx_nombuf); + + max_rings_stats = RTE_MIN(dev->data->nb_rx_queues, + RTE_ETHDEV_QUEUE_STAT_CNTRS); + for (i = 0; i < max_rings_stats; ++i) { + struct ena_stats_rx *rx_stats = &adapter->rx_ring[i].rx_stats; + + stats->q_ibytes[i] = rx_stats->bytes; + stats->q_ipackets[i] = rx_stats->cnt; + stats->q_errors[i] = rx_stats->bad_desc_num + + rx_stats->bad_req_id; + } + + max_rings_stats = RTE_MIN(dev->data->nb_tx_queues, + RTE_ETHDEV_QUEUE_STAT_CNTRS); + for (i = 0; i < max_rings_stats; ++i) { + struct ena_stats_tx *tx_stats = &adapter->tx_ring[i].tx_stats; + + stats->q_obytes[i] = tx_stats->bytes; + stats->q_opackets[i] = tx_stats->cnt; + } } static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) @@ -854,88 +809,253 @@ struct ena_com_dev *ena_dev; int rc = 0; - ena_assert_msg(dev->data != NULL, "Uninitialized device"); - ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device"); - adapter = (struct ena_adapter *)(dev->data->dev_private); + ena_assert_msg(dev->data != NULL, "Uninitialized device\n"); + ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device\n"); + adapter = dev->data->dev_private; ena_dev = &adapter->ena_dev; - ena_assert_msg(ena_dev != NULL, "Uninitialized device"); + ena_assert_msg(ena_dev != NULL, "Uninitialized device\n"); - if (mtu > ena_get_mtu_conf(adapter)) { - RTE_LOG(ERR, PMD, - "Given MTU (%d) exceeds maximum MTU supported (%d)\n", - mtu, ena_get_mtu_conf(adapter)); - rc = -EINVAL; - goto err; + if (mtu > ena_get_mtu_conf(adapter) || mtu < ENA_MIN_MTU) { + PMD_DRV_LOG(ERR, + "Invalid MTU setting. New MTU: %d, max MTU: %d, min MTU: %d\n", + mtu, ena_get_mtu_conf(adapter), ENA_MIN_MTU); + return -EINVAL; } rc = ena_com_set_dev_mtu(ena_dev, mtu); if (rc) - RTE_LOG(ERR, PMD, "Could not set MTU: %d\n", mtu); + PMD_DRV_LOG(ERR, "Could not set MTU: %d\n", mtu); else - RTE_LOG(NOTICE, PMD, "Set MTU: %d\n", mtu); + PMD_DRV_LOG(NOTICE, "MTU set to: %d\n", mtu); -err: return rc; } static int ena_start(struct rte_eth_dev *dev) { - struct ena_adapter *adapter = - (struct ena_adapter *)(dev->data->dev_private); + struct ena_adapter *adapter = dev->data->dev_private; + uint64_t ticks; int rc = 0; - if (!(adapter->state == ENA_ADAPTER_STATE_CONFIG || - adapter->state == ENA_ADAPTER_STATE_STOPPED)) { - PMD_INIT_LOG(ERR, "API violation"); - return -1; + /* Cannot allocate memory in secondary process */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) { + PMD_DRV_LOG(WARNING, "dev_start not supported in secondary.\n"); + return -EPERM; } rc = ena_check_valid_conf(adapter); if (rc) return rc; - rc = ena_queue_restart_all(dev, ENA_RING_TYPE_RX); + rc = ena_setup_rx_intr(dev); if (rc) return rc; - rc = ena_queue_restart_all(dev, ENA_RING_TYPE_TX); + rc = ena_queue_start_all(dev, ENA_RING_TYPE_RX); if (rc) return rc; - if (adapter->rte_dev->data->dev_conf.rxmode.mq_mode & - ETH_MQ_RX_RSS_FLAG) { - rc = ena_rss_init_default(adapter); + rc = ena_queue_start_all(dev, ENA_RING_TYPE_TX); + if (rc) + goto err_start_tx; + + if (adapter->edev_data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) { + rc = ena_rss_configure(adapter); if (rc) - return rc; + goto err_rss_init; } ena_stats_restart(dev); + adapter->timestamp_wd = rte_get_timer_cycles(); + adapter->keep_alive_timeout = ENA_DEVICE_KALIVE_TIMEOUT; + + ticks = rte_get_timer_hz(); + rte_timer_reset(&adapter->timer_wd, ticks, PERIODICAL, rte_lcore_id(), + ena_timer_wd_callback, dev); + + ++adapter->dev_stats.dev_start; adapter->state = ENA_ADAPTER_STATE_RUNNING; return 0; + +err_rss_init: + ena_queue_stop_all(dev, ENA_RING_TYPE_TX); +err_start_tx: + ena_queue_stop_all(dev, ENA_RING_TYPE_RX); + return rc; +} + +static void ena_stop(struct rte_eth_dev *dev) +{ + struct ena_adapter *adapter = dev->data->dev_private; + struct ena_com_dev *ena_dev = &adapter->ena_dev; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; + int rc; + + /* Cannot free memory in secondary process */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) { + PMD_DRV_LOG(WARNING, "dev_stop not supported in secondary.\n"); + return; + } + + rte_timer_stop_sync(&adapter->timer_wd); + ena_queue_stop_all(dev, ENA_RING_TYPE_TX); + ena_queue_stop_all(dev, ENA_RING_TYPE_RX); + + if (adapter->trigger_reset) { + rc = ena_com_dev_reset(ena_dev, adapter->reset_reason); + if (rc) + PMD_DRV_LOG(ERR, "Device reset failed, rc: %d\n", rc); + } + + rte_intr_disable(intr_handle); + + rte_intr_efd_disable(intr_handle); + if (intr_handle->intr_vec != NULL) { + rte_free(intr_handle->intr_vec); + intr_handle->intr_vec = NULL; + } + + rte_intr_enable(intr_handle); + + ++adapter->dev_stats.dev_stop; + adapter->state = ENA_ADAPTER_STATE_STOPPED; +} + +static int ena_create_io_queue(struct rte_eth_dev *dev, struct ena_ring *ring) +{ + struct ena_adapter *adapter = ring->adapter; + struct ena_com_dev *ena_dev = &adapter->ena_dev; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; + struct ena_com_create_io_ctx ctx = + /* policy set to _HOST just to satisfy icc compiler */ + { ENA_ADMIN_PLACEMENT_POLICY_HOST, + 0, 0, 0, 0, 0 }; + uint16_t ena_qid; + unsigned int i; + int rc; + + ctx.msix_vector = -1; + if (ring->type == ENA_RING_TYPE_TX) { + ena_qid = ENA_IO_TXQ_IDX(ring->id); + ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX; + ctx.mem_queue_type = ena_dev->tx_mem_queue_type; + for (i = 0; i < ring->ring_size; i++) + ring->empty_tx_reqs[i] = i; + } else { + ena_qid = ENA_IO_RXQ_IDX(ring->id); + ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX; + if (rte_intr_dp_is_en(intr_handle)) + ctx.msix_vector = intr_handle->intr_vec[ring->id]; + for (i = 0; i < ring->ring_size; i++) + ring->empty_rx_reqs[i] = i; + } + ctx.queue_size = ring->ring_size; + ctx.qid = ena_qid; + ctx.numa_node = ring->numa_socket_id; + + rc = ena_com_create_io_queue(ena_dev, &ctx); + if (rc) { + PMD_INIT_LOG(ERR, + "Failed to create IO queue[%d] (qid:%d), rc: %d\n", + ring->id, ena_qid, rc); + return rc; + } + + rc = ena_com_get_io_handlers(ena_dev, ena_qid, + &ring->ena_com_io_sq, + &ring->ena_com_io_cq); + if (rc) { + PMD_INIT_LOG(ERR, + "Failed to get IO queue[%d] handlers, rc: %d\n", + ring->id, rc); + ena_com_destroy_io_queue(ena_dev, ena_qid); + return rc; + } + + if (ring->type == ENA_RING_TYPE_TX) + ena_com_update_numa_node(ring->ena_com_io_cq, ctx.numa_node); + + /* Start with Rx interrupts being masked. */ + if (ring->type == ENA_RING_TYPE_RX && rte_intr_dp_is_en(intr_handle)) + ena_rx_queue_intr_disable(dev, ring->id); + + return 0; +} + +static void ena_queue_stop(struct ena_ring *ring) +{ + struct ena_com_dev *ena_dev = &ring->adapter->ena_dev; + + if (ring->type == ENA_RING_TYPE_RX) { + ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(ring->id)); + ena_rx_queue_release_bufs(ring); + } else { + ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(ring->id)); + ena_tx_queue_release_bufs(ring); + } +} + +static void ena_queue_stop_all(struct rte_eth_dev *dev, + enum ena_ring_type ring_type) +{ + struct ena_adapter *adapter = dev->data->dev_private; + struct ena_ring *queues = NULL; + uint16_t nb_queues, i; + + if (ring_type == ENA_RING_TYPE_RX) { + queues = adapter->rx_ring; + nb_queues = dev->data->nb_rx_queues; + } else { + queues = adapter->tx_ring; + nb_queues = dev->data->nb_tx_queues; + } + + for (i = 0; i < nb_queues; ++i) + if (queues[i].configured) + ena_queue_stop(&queues[i]); } -static int ena_queue_restart(struct ena_ring *ring) +static int ena_queue_start(struct rte_eth_dev *dev, struct ena_ring *ring) { + struct rte_mempool_cache *cache; int rc, bufs_num; ena_assert_msg(ring->configured == 1, - "Trying to restart unconfigured queue\n"); + "Trying to start unconfigured queue\n"); + + rc = ena_create_io_queue(dev, ring); + if (rc) { + PMD_INIT_LOG(ERR, "Failed to create IO queue\n"); + return rc; + } ring->next_to_clean = 0; ring->next_to_use = 0; - if (ring->type == ENA_RING_TYPE_TX) + if (ring->type == ENA_RING_TYPE_TX) { + ring->tx_stats.available_desc = + ena_com_free_q_entries(ring->ena_com_io_sq); return 0; + } bufs_num = ring->ring_size - 1; rc = ena_populate_rx_queue(ring, bufs_num); if (rc != bufs_num) { - PMD_INIT_LOG(ERR, "Failed to populate rx ring !"); - return (-1); + ena_com_destroy_io_queue(&ring->adapter->ena_dev, + ENA_IO_RXQ_IDX(ring->id)); + PMD_INIT_LOG(ERR, "Failed to populate Rx ring\n"); + return ENA_COM_FAULT; } + /* Flush per-core RX buffers pools cache as they can be used on other + * cores as well. + */ + cache = rte_mempool_default_cache(ring->mb_pool, rte_lcore_id()); + rte_mempool_cache_flush(cache, ring->mb_pool); return 0; } @@ -943,84 +1063,51 @@ static int ena_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc, - __rte_unused unsigned int socket_id, + unsigned int socket_id, __rte_unused const struct rte_eth_txconf *tx_conf) { - struct ena_com_create_io_ctx ctx = - /* policy set to _HOST just to satisfy icc compiler */ - { ENA_ADMIN_PLACEMENT_POLICY_HOST, - ENA_COM_IO_QUEUE_DIRECTION_TX, 0, 0, 0, 0 }; struct ena_ring *txq = NULL; - struct ena_adapter *adapter = - (struct ena_adapter *)(dev->data->dev_private); + struct ena_adapter *adapter = dev->data->dev_private; unsigned int i; - int ena_qid; - int rc; - struct ena_com_dev *ena_dev = &adapter->ena_dev; txq = &adapter->tx_ring[queue_idx]; if (txq->configured) { - RTE_LOG(CRIT, PMD, - "API violation. Queue %d is already configured\n", + PMD_INIT_LOG(CRIT, + "API violation. Queue[%d] is already configured\n", queue_idx); - return -1; + return ENA_COM_FAULT; } if (!rte_is_power_of_2(nb_desc)) { - RTE_LOG(ERR, PMD, - "Unsupported size of RX queue: %d is not a power of 2.", + PMD_INIT_LOG(ERR, + "Unsupported size of Tx queue: %d is not a power of 2.\n", nb_desc); return -EINVAL; } - if (nb_desc > adapter->tx_ring_size) { - RTE_LOG(ERR, PMD, - "Unsupported size of TX queue (max size: %d)\n", - adapter->tx_ring_size); + if (nb_desc > adapter->max_tx_ring_size) { + PMD_INIT_LOG(ERR, + "Unsupported size of Tx queue (max size: %d)\n", + adapter->max_tx_ring_size); return -EINVAL; } - ena_qid = ENA_IO_TXQ_IDX(queue_idx); - - ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX; - ctx.qid = ena_qid; - ctx.msix_vector = -1; /* admin interrupts not used */ - ctx.mem_queue_type = ena_dev->tx_mem_queue_type; - ctx.queue_size = adapter->tx_ring_size; - ctx.numa_node = ena_cpu_to_node(queue_idx); - - rc = ena_com_create_io_queue(ena_dev, &ctx); - if (rc) { - RTE_LOG(ERR, PMD, - "failed to create io TX queue #%d (qid:%d) rc: %d\n", - queue_idx, ena_qid, rc); - } - txq->ena_com_io_cq = &ena_dev->io_cq_queues[ena_qid]; - txq->ena_com_io_sq = &ena_dev->io_sq_queues[ena_qid]; - - rc = ena_com_get_io_handlers(ena_dev, ena_qid, - &txq->ena_com_io_sq, - &txq->ena_com_io_cq); - if (rc) { - RTE_LOG(ERR, PMD, - "Failed to get TX queue handlers. TX queue num %d rc: %d\n", - queue_idx, rc); - ena_com_destroy_io_queue(ena_dev, ena_qid); - goto err; - } - txq->port_id = dev->data->port_id; txq->next_to_clean = 0; txq->next_to_use = 0; txq->ring_size = nb_desc; + txq->size_mask = nb_desc - 1; + txq->numa_socket_id = socket_id; + txq->pkts_without_db = false; txq->tx_buffer_info = rte_zmalloc("txq->tx_buffer_info", sizeof(struct ena_tx_buffer) * txq->ring_size, RTE_CACHE_LINE_SIZE); if (!txq->tx_buffer_info) { - RTE_LOG(ERR, PMD, "failed to alloc mem for tx buffer info\n"); + PMD_INIT_LOG(ERR, + "Failed to allocate memory for Tx buffer info\n"); return -ENOMEM; } @@ -1028,105 +1115,143 @@ sizeof(u16) * txq->ring_size, RTE_CACHE_LINE_SIZE); if (!txq->empty_tx_reqs) { - RTE_LOG(ERR, PMD, "failed to alloc mem for tx reqs\n"); + PMD_INIT_LOG(ERR, + "Failed to allocate memory for empty Tx requests\n"); rte_free(txq->tx_buffer_info); return -ENOMEM; } + + txq->push_buf_intermediate_buf = + rte_zmalloc("txq->push_buf_intermediate_buf", + txq->tx_max_header_size, + RTE_CACHE_LINE_SIZE); + if (!txq->push_buf_intermediate_buf) { + PMD_INIT_LOG(ERR, "Failed to alloc push buffer for LLQ\n"); + rte_free(txq->tx_buffer_info); + rte_free(txq->empty_tx_reqs); + return -ENOMEM; + } + for (i = 0; i < txq->ring_size; i++) txq->empty_tx_reqs[i] = i; /* Store pointer to this queue in upper layer */ txq->configured = 1; dev->data->tx_queues[queue_idx] = txq; -err: - return rc; + + return 0; } static int ena_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc, - __rte_unused unsigned int socket_id, + unsigned int socket_id, __rte_unused const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp) { - struct ena_com_create_io_ctx ctx = - /* policy set to _HOST just to satisfy icc compiler */ - { ENA_ADMIN_PLACEMENT_POLICY_HOST, - ENA_COM_IO_QUEUE_DIRECTION_RX, 0, 0, 0, 0 }; - struct ena_adapter *adapter = - (struct ena_adapter *)(dev->data->dev_private); + struct ena_adapter *adapter = dev->data->dev_private; struct ena_ring *rxq = NULL; - uint16_t ena_qid = 0; - int rc = 0; - struct ena_com_dev *ena_dev = &adapter->ena_dev; + size_t buffer_size; + int i; rxq = &adapter->rx_ring[queue_idx]; if (rxq->configured) { - RTE_LOG(CRIT, PMD, - "API violation. Queue %d is already configured\n", + PMD_INIT_LOG(CRIT, + "API violation. Queue[%d] is already configured\n", queue_idx); - return -1; + return ENA_COM_FAULT; } if (!rte_is_power_of_2(nb_desc)) { - RTE_LOG(ERR, PMD, - "Unsupported size of TX queue: %d is not a power of 2.", + PMD_INIT_LOG(ERR, + "Unsupported size of Rx queue: %d is not a power of 2.\n", nb_desc); return -EINVAL; } - if (nb_desc > adapter->rx_ring_size) { - RTE_LOG(ERR, PMD, - "Unsupported size of RX queue (max size: %d)\n", - adapter->rx_ring_size); + if (nb_desc > adapter->max_rx_ring_size) { + PMD_INIT_LOG(ERR, + "Unsupported size of Rx queue (max size: %d)\n", + adapter->max_rx_ring_size); return -EINVAL; } - ena_qid = ENA_IO_RXQ_IDX(queue_idx); - - ctx.qid = ena_qid; - ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX; - ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; - ctx.msix_vector = -1; /* admin interrupts not used */ - ctx.queue_size = adapter->rx_ring_size; - ctx.numa_node = ena_cpu_to_node(queue_idx); - - rc = ena_com_create_io_queue(ena_dev, &ctx); - if (rc) - RTE_LOG(ERR, PMD, "failed to create io RX queue #%d rc: %d\n", - queue_idx, rc); - - rxq->ena_com_io_cq = &ena_dev->io_cq_queues[ena_qid]; - rxq->ena_com_io_sq = &ena_dev->io_sq_queues[ena_qid]; - - rc = ena_com_get_io_handlers(ena_dev, ena_qid, - &rxq->ena_com_io_sq, - &rxq->ena_com_io_cq); - if (rc) { - RTE_LOG(ERR, PMD, - "Failed to get RX queue handlers. RX queue num %d rc: %d\n", - queue_idx, rc); - ena_com_destroy_io_queue(ena_dev, ena_qid); + /* ENA isn't supporting buffers smaller than 1400 bytes */ + buffer_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM; + if (buffer_size < ENA_RX_BUF_MIN_SIZE) { + PMD_INIT_LOG(ERR, + "Unsupported size of Rx buffer: %zu (min size: %d)\n", + buffer_size, ENA_RX_BUF_MIN_SIZE); + return -EINVAL; } rxq->port_id = dev->data->port_id; rxq->next_to_clean = 0; rxq->next_to_use = 0; rxq->ring_size = nb_desc; + rxq->size_mask = nb_desc - 1; + rxq->numa_socket_id = socket_id; rxq->mb_pool = mp; rxq->rx_buffer_info = rte_zmalloc("rxq->buffer_info", - sizeof(struct rte_mbuf *) * nb_desc, - RTE_CACHE_LINE_SIZE); + sizeof(struct ena_rx_buffer) * nb_desc, + RTE_CACHE_LINE_SIZE); if (!rxq->rx_buffer_info) { - RTE_LOG(ERR, PMD, "failed to alloc mem for rx buffer info\n"); + PMD_INIT_LOG(ERR, + "Failed to allocate memory for Rx buffer info\n"); + return -ENOMEM; + } + + rxq->rx_refill_buffer = rte_zmalloc("rxq->rx_refill_buffer", + sizeof(struct rte_mbuf *) * nb_desc, + RTE_CACHE_LINE_SIZE); + + if (!rxq->rx_refill_buffer) { + PMD_INIT_LOG(ERR, + "Failed to allocate memory for Rx refill buffer\n"); + rte_free(rxq->rx_buffer_info); + rxq->rx_buffer_info = NULL; + return -ENOMEM; + } + + rxq->empty_rx_reqs = rte_zmalloc("rxq->empty_rx_reqs", + sizeof(uint16_t) * nb_desc, + RTE_CACHE_LINE_SIZE); + if (!rxq->empty_rx_reqs) { + PMD_INIT_LOG(ERR, + "Failed to allocate memory for empty Rx requests\n"); + rte_free(rxq->rx_buffer_info); + rxq->rx_buffer_info = NULL; + rte_free(rxq->rx_refill_buffer); + rxq->rx_refill_buffer = NULL; return -ENOMEM; } + for (i = 0; i < nb_desc; i++) + rxq->empty_rx_reqs[i] = i; + /* Store pointer to this queue in upper layer */ rxq->configured = 1; dev->data->rx_queues[queue_idx] = rxq; + return 0; +} + +static int ena_add_single_rx_desc(struct ena_com_io_sq *io_sq, + struct rte_mbuf *mbuf, uint16_t id) +{ + struct ena_com_buf ebuf; + int rc; + + /* prepare physical address for DMA transaction */ + ebuf.paddr = mbuf->buf_physaddr + RTE_PKTMBUF_HEADROOM; + ebuf.len = mbuf->buf_len - RTE_PKTMBUF_HEADROOM; + + /* pass resource to device */ + rc = ena_com_add_single_rx_desc(io_sq, &ebuf, id); + if (unlikely(rc != 0)) + PMD_RX_LOG(WARNING, "Failed adding Rx desc\n"); + return rc; } @@ -1134,54 +1259,62 @@ { unsigned int i; int rc; - uint16_t ring_size = rxq->ring_size; - uint16_t ring_mask = ring_size - 1; uint16_t next_to_use = rxq->next_to_use; + uint16_t req_id; +#ifdef RTE_LIBRTE_ENA_DEBUG_RX uint16_t in_use; - struct rte_mbuf **mbufs = &rxq->rx_buffer_info[0]; +#endif + struct rte_mbuf **mbufs = rxq->rx_refill_buffer; if (unlikely(!count)) return 0; - in_use = rxq->next_to_use - rxq->next_to_clean; - ena_assert_msg(((in_use + count) <= ring_size), "bad ring state"); - - count = RTE_MIN(count, - (uint16_t)(ring_size - (next_to_use & ring_mask))); +#ifdef RTE_LIBRTE_ENA_DEBUG_RX + in_use = rxq->ring_size - 1 - + ena_com_free_q_entries(rxq->ena_com_io_sq); + if (unlikely((in_use + count) >= rxq->ring_size)) + PMD_RX_LOG(ERR, "Bad Rx ring state\n"); +#endif /* get resources for incoming packets */ - rc = rte_mempool_get_bulk(rxq->mb_pool, - (void **)(&mbufs[next_to_use & ring_mask]), - count); + rc = rte_mempool_get_bulk(rxq->mb_pool, (void **)mbufs, count); if (unlikely(rc < 0)) { rte_atomic64_inc(&rxq->adapter->drv_stats->rx_nombuf); - PMD_RX_LOG(DEBUG, "there are no enough free buffers"); + ++rxq->rx_stats.mbuf_alloc_fail; + PMD_RX_LOG(DEBUG, "There are not enough free buffers\n"); return 0; } for (i = 0; i < count; i++) { - uint16_t next_to_use_masked = next_to_use & ring_mask; - struct rte_mbuf *mbuf = mbufs[next_to_use_masked]; - struct ena_com_buf ebuf; - - rte_prefetch0(mbufs[((next_to_use + 4) & ring_mask)]); - /* prepare physical address for DMA transaction */ - ebuf.paddr = mbuf->buf_physaddr + RTE_PKTMBUF_HEADROOM; - ebuf.len = mbuf->buf_len - RTE_PKTMBUF_HEADROOM; - /* pass resource to device */ - rc = ena_com_add_single_rx_desc(rxq->ena_com_io_sq, - &ebuf, next_to_use_masked); - if (unlikely(rc)) { - RTE_LOG(WARNING, PMD, "failed adding rx desc\n"); + struct rte_mbuf *mbuf = mbufs[i]; + struct ena_rx_buffer *rx_info; + + if (likely((i + 4) < count)) + rte_prefetch0(mbufs[i + 4]); + + req_id = rxq->empty_rx_reqs[next_to_use]; + rx_info = &rxq->rx_buffer_info[req_id]; + + rc = ena_add_single_rx_desc(rxq->ena_com_io_sq, mbuf, req_id); + if (unlikely(rc != 0)) break; - } - next_to_use++; + + rx_info->mbuf = mbuf; + next_to_use = ENA_IDX_NEXT_MASKED(next_to_use, rxq->size_mask); + } + + if (unlikely(i < count)) { + PMD_RX_LOG(WARNING, + "Refilled Rx queue[%d] with only %d/%d buffers\n", + rxq->id, i, count); + rte_mempool_put_bulk(rxq->mb_pool, (void **)(&mbufs[i]), + count - i); + ++rxq->rx_stats.refill_partial; } /* When we submitted free recources to device... */ - if (i > 0) { - /* ...let HW know that it can fill buffers with data */ - rte_wmb(); + if (likely(i > 0)) { + /* ...let HW know that it can fill buffers with data. */ ena_com_write_sq_doorbell(rxq->ena_com_io_sq); rxq->next_to_use = next_to_use; @@ -1191,47 +1324,48 @@ } static int ena_device_init(struct ena_com_dev *ena_dev, - struct ena_com_dev_get_features_ctx *get_feat_ctx) + struct rte_pci_device *pdev, + struct ena_com_dev_get_features_ctx *get_feat_ctx, + bool *wd_state) { + uint32_t aenq_groups; int rc; bool readless_supported; /* Initialize mmio registers */ rc = ena_com_mmio_reg_read_request_init(ena_dev); if (rc) { - RTE_LOG(ERR, PMD, "failed to init mmio read less\n"); + PMD_INIT_LOG(ERR, "Failed to init MMIO read less\n"); return rc; } /* The PCIe configuration space revision id indicate if mmio reg * read is disabled. */ - readless_supported = - !(((struct rte_pci_device *)ena_dev->dmadev)->id.class_id - & ENA_MMIO_DISABLE_REG_READ); + readless_supported = !(pdev->id.class_id & ENA_MMIO_DISABLE_REG_READ); ena_com_set_mmio_read_mode(ena_dev, readless_supported); /* reset device */ - rc = ena_com_dev_reset(ena_dev); + rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL); if (rc) { - RTE_LOG(ERR, PMD, "cannot reset device\n"); + PMD_INIT_LOG(ERR, "Cannot reset device\n"); goto err_mmio_read_less; } /* check FW version */ rc = ena_com_validate_version(ena_dev); if (rc) { - RTE_LOG(ERR, PMD, "device version is too low\n"); + PMD_INIT_LOG(ERR, "Device version is too low\n"); goto err_mmio_read_less; } ena_dev->dma_addr_bits = ena_com_get_dma_width(ena_dev); /* ENA device administration layer init */ - rc = ena_com_admin_init(ena_dev, NULL, true); + rc = ena_com_admin_init(ena_dev, &aenq_handlers); if (rc) { - RTE_LOG(ERR, PMD, - "cannot initialize ena admin queue with device\n"); + PMD_INIT_LOG(ERR, + "Cannot initialize ENA admin queue\n"); goto err_mmio_read_less; } @@ -1246,11 +1380,26 @@ /* Get Device Attributes and features */ rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx); if (rc) { - RTE_LOG(ERR, PMD, - "cannot get attribute for ena device rc= %d\n", rc); + PMD_INIT_LOG(ERR, + "Cannot get attribute for ENA device, rc: %d\n", rc); + goto err_admin_init; + } + + aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) | + BIT(ENA_ADMIN_NOTIFICATION) | + BIT(ENA_ADMIN_KEEP_ALIVE) | + BIT(ENA_ADMIN_FATAL_ERROR) | + BIT(ENA_ADMIN_WARNING); + + aenq_groups &= get_feat_ctx->aenq.supported_groups; + rc = ena_com_set_aenq_config(ena_dev, aenq_groups); + if (rc) { + PMD_INIT_LOG(ERR, "Cannot configure AENQ groups, rc: %d\n", rc); goto err_admin_init; } + *wd_state = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE)); + return 0; err_admin_init: @@ -1262,123 +1411,384 @@ return rc; } +static void +ena_interrupt_handler_rte(__rte_unused struct rte_intr_handle *intr_handle, + void *cb_arg) +{ + struct rte_eth_dev *dev = cb_arg; + struct ena_adapter *adapter = dev->data->dev_private; + struct ena_com_dev *ena_dev = &adapter->ena_dev; + + ena_com_admin_q_comp_intr_handler(ena_dev); + if (likely(adapter->state != ENA_ADAPTER_STATE_CLOSED)) + ena_com_aenq_intr_handler(ena_dev, dev); +} + +static void check_for_missing_keep_alive(struct ena_adapter *adapter) +{ + if (!adapter->wd_state) + return; + + if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT) + return; + + if (unlikely((rte_get_timer_cycles() - adapter->timestamp_wd) >= + adapter->keep_alive_timeout)) { + PMD_DRV_LOG(ERR, "Keep alive timeout\n"); + adapter->reset_reason = ENA_REGS_RESET_KEEP_ALIVE_TO; + adapter->trigger_reset = true; + ++adapter->dev_stats.wd_expired; + } +} + +/* Check if admin queue is enabled */ +static void check_for_admin_com_state(struct ena_adapter *adapter) +{ + if (unlikely(!ena_com_get_admin_running_state(&adapter->ena_dev))) { + PMD_DRV_LOG(ERR, "ENA admin queue is not in running state\n"); + adapter->reset_reason = ENA_REGS_RESET_ADMIN_TO; + adapter->trigger_reset = true; + } +} + +static void ena_timer_wd_callback(__rte_unused struct rte_timer *timer, + void *arg) +{ + struct rte_eth_dev *dev = arg; + struct ena_adapter *adapter = dev->data->dev_private; + + check_for_missing_keep_alive(adapter); + check_for_admin_com_state(adapter); + + if (unlikely(adapter->trigger_reset)) { + PMD_DRV_LOG(ERR, "Trigger reset is on\n"); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, NULL); + adapter->trigger_reset = false; + } +} + +static inline void +set_default_llq_configurations(struct ena_llq_configurations *llq_config, + struct ena_admin_feature_llq_desc *llq, + bool use_large_llq_hdr) +{ + llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER; + llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY; + llq_config->llq_num_decs_before_header = + ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2; + + if (use_large_llq_hdr && + (llq->entry_size_ctrl_supported & ENA_ADMIN_LIST_ENTRY_SIZE_256B)) { + llq_config->llq_ring_entry_size = + ENA_ADMIN_LIST_ENTRY_SIZE_256B; + llq_config->llq_ring_entry_size_value = 256; + } else { + llq_config->llq_ring_entry_size = + ENA_ADMIN_LIST_ENTRY_SIZE_128B; + llq_config->llq_ring_entry_size_value = 128; + } +} + +static int +ena_set_queues_placement_policy(struct ena_adapter *adapter, + struct ena_com_dev *ena_dev, + struct ena_admin_feature_llq_desc *llq, + struct ena_llq_configurations *llq_default_configurations) +{ + int rc; + u32 llq_feature_mask; + + llq_feature_mask = 1 << ENA_ADMIN_LLQ; + if (!(ena_dev->supported_features & llq_feature_mask)) { + PMD_INIT_LOG(INFO, + "LLQ is not supported. Fallback to host mode policy.\n"); + ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; + return 0; + } + + rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations); + if (unlikely(rc)) { + PMD_INIT_LOG(WARNING, + "Failed to config dev mode. Fallback to host mode policy.\n"); + ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; + return 0; + } + + /* Nothing to config, exit */ + if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) + return 0; + + if (!adapter->dev_mem_base) { + PMD_INIT_LOG(ERR, + "Unable to access LLQ BAR resource. Fallback to host mode policy.\n"); + ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; + return 0; + } + + ena_dev->mem_bar = adapter->dev_mem_base; + + return 0; +} + +static uint32_t ena_calc_max_io_queue_num(struct ena_com_dev *ena_dev, + struct ena_com_dev_get_features_ctx *get_feat_ctx) +{ + uint32_t io_tx_sq_num, io_tx_cq_num, io_rx_num, max_num_io_queues; + + /* Regular queues capabilities */ + if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) { + struct ena_admin_queue_ext_feature_fields *max_queue_ext = + &get_feat_ctx->max_queue_ext.max_queue_ext; + io_rx_num = RTE_MIN(max_queue_ext->max_rx_sq_num, + max_queue_ext->max_rx_cq_num); + io_tx_sq_num = max_queue_ext->max_tx_sq_num; + io_tx_cq_num = max_queue_ext->max_tx_cq_num; + } else { + struct ena_admin_queue_feature_desc *max_queues = + &get_feat_ctx->max_queues; + io_tx_sq_num = max_queues->max_sq_num; + io_tx_cq_num = max_queues->max_cq_num; + io_rx_num = RTE_MIN(io_tx_sq_num, io_tx_cq_num); + } + + /* In case of LLQ use the llq number in the get feature cmd */ + if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) + io_tx_sq_num = get_feat_ctx->llq.max_llq_num; + + max_num_io_queues = RTE_MIN(ENA_MAX_NUM_IO_QUEUES, io_rx_num); + max_num_io_queues = RTE_MIN(max_num_io_queues, io_tx_sq_num); + max_num_io_queues = RTE_MIN(max_num_io_queues, io_tx_cq_num); + + if (unlikely(max_num_io_queues == 0)) { + PMD_INIT_LOG(ERR, "Number of IO queues cannot not be 0\n"); + return -EFAULT; + } + + return max_num_io_queues; +} + static int eth_ena_dev_init(struct rte_eth_dev *eth_dev) { + struct ena_calc_queue_size_ctx calc_queue_ctx = { 0 }; struct rte_pci_device *pci_dev; - struct ena_adapter *adapter = - (struct ena_adapter *)(eth_dev->data->dev_private); + struct rte_intr_handle *intr_handle; + struct ena_adapter *adapter = eth_dev->data->dev_private; struct ena_com_dev *ena_dev = &adapter->ena_dev; struct ena_com_dev_get_features_ctx get_feat_ctx; - int queue_size, rc; - + struct ena_llq_configurations llq_config; + const char *queue_type_str; + uint32_t max_num_io_queues; + int rc; static int adapters_found; - - memset(adapter, 0, sizeof(struct ena_adapter)); - ena_dev = &adapter->ena_dev; + bool disable_meta_caching; + bool wd_state = false; eth_dev->dev_ops = &ena_dev_ops; eth_dev->rx_pkt_burst = ð_ena_recv_pkts; eth_dev->tx_pkt_burst = ð_ena_xmit_pkts; - eth_dev->tx_pkt_prepare = ð_ena_prep_pkts; - adapter->rte_eth_dev_data = eth_dev->data; - adapter->rte_dev = eth_dev; if (rte_eal_process_type() != RTE_PROC_PRIMARY) return 0; - pci_dev = RTE_DEV_TO_PCI(eth_dev->device); - adapter->pdev = pci_dev; + memset(adapter, 0, sizeof(struct ena_adapter)); + ena_dev = &adapter->ena_dev; + + adapter->edev_data = eth_dev->data; + + pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); - PMD_INIT_LOG(INFO, "Initializing %x:%x:%x.%d", + PMD_INIT_LOG(INFO, "Initializing %x:%x:%x.%d\n", pci_dev->addr.domain, pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function); + intr_handle = &pci_dev->intr_handle; + adapter->regs = pci_dev->mem_resource[ENA_REGS_BAR].addr; adapter->dev_mem_base = pci_dev->mem_resource[ENA_MEM_BAR].addr; - /* Present ENA_MEM_BAR indicates available LLQ mode. - * Use corresponding policy - */ - if (adapter->dev_mem_base) - ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_DEV; - else if (adapter->regs) - ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; - else - PMD_INIT_LOG(CRIT, "Failed to access registers BAR(%d)", + if (!adapter->regs) { + PMD_INIT_LOG(CRIT, "Failed to access registers BAR(%d)\n", ENA_REGS_BAR); + return -ENXIO; + } ena_dev->reg_bar = adapter->regs; - ena_dev->dmadev = adapter->pdev; + /* This is a dummy pointer for ena_com functions. */ + ena_dev->dmadev = adapter; adapter->id_number = adapters_found; snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", adapter->id_number); + rc = ena_parse_devargs(adapter, pci_dev->device.devargs); + if (rc != 0) { + PMD_INIT_LOG(CRIT, "Failed to parse devargs\n"); + goto err; + } + /* device specific initialization routine */ - rc = ena_device_init(ena_dev, &get_feat_ctx); + rc = ena_device_init(ena_dev, pci_dev, &get_feat_ctx, &wd_state); if (rc) { - PMD_INIT_LOG(CRIT, "Failed to init ENA device"); - return -1; + PMD_INIT_LOG(CRIT, "Failed to init ENA device\n"); + goto err; } + adapter->wd_state = wd_state; - if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { - if (get_feat_ctx.max_queues.max_llq_num == 0) { - PMD_INIT_LOG(ERR, - "Trying to use LLQ but llq_num is 0.\n" - "Fall back into regular queues."); - ena_dev->tx_mem_queue_type = - ENA_ADMIN_PLACEMENT_POLICY_HOST; - adapter->num_queues = - get_feat_ctx.max_queues.max_sq_num; - } else { - adapter->num_queues = - get_feat_ctx.max_queues.max_llq_num; - } - } else { - adapter->num_queues = get_feat_ctx.max_queues.max_sq_num; + set_default_llq_configurations(&llq_config, &get_feat_ctx.llq, + adapter->use_large_llq_hdr); + rc = ena_set_queues_placement_policy(adapter, ena_dev, + &get_feat_ctx.llq, &llq_config); + if (unlikely(rc)) { + PMD_INIT_LOG(CRIT, "Failed to set placement policy\n"); + return rc; } - queue_size = ena_calc_queue_size(ena_dev, &get_feat_ctx); - if ((queue_size <= 0) || (adapter->num_queues <= 0)) - return -EFAULT; + if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) + queue_type_str = "Regular"; + else + queue_type_str = "Low latency"; + PMD_INIT_LOG(INFO, "Placement policy: %s\n", queue_type_str); + + calc_queue_ctx.ena_dev = ena_dev; + calc_queue_ctx.get_feat_ctx = &get_feat_ctx; + + max_num_io_queues = ena_calc_max_io_queue_num(ena_dev, &get_feat_ctx); + rc = ena_calc_io_queue_size(&calc_queue_ctx, + adapter->use_large_llq_hdr); + if (unlikely((rc != 0) || (max_num_io_queues == 0))) { + rc = -EFAULT; + goto err_device_destroy; + } + + adapter->max_tx_ring_size = calc_queue_ctx.max_tx_queue_size; + adapter->max_rx_ring_size = calc_queue_ctx.max_rx_queue_size; + adapter->max_tx_sgl_size = calc_queue_ctx.max_tx_sgl_size; + adapter->max_rx_sgl_size = calc_queue_ctx.max_rx_sgl_size; + adapter->max_num_io_queues = max_num_io_queues; - adapter->tx_ring_size = queue_size; - adapter->rx_ring_size = queue_size; + if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { + disable_meta_caching = + !!(get_feat_ctx.llq.accel_mode.u.get.supported_flags & + BIT(ENA_ADMIN_DISABLE_META_CACHING)); + } else { + disable_meta_caching = false; + } /* prepare ring structures */ - ena_init_rings(adapter); + ena_init_rings(adapter, disable_meta_caching); ena_config_debug_area(adapter); /* Set max MTU for this device */ adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu; - /* set device support for TSO */ - adapter->tso4_supported = get_feat_ctx.offload.tx & - ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK; + /* set device support for offloads */ + adapter->offloads.tso4_supported = (get_feat_ctx.offload.tx & + ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) != 0; + adapter->offloads.tx_csum_supported = (get_feat_ctx.offload.tx & + ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK) != 0; + adapter->offloads.rx_csum_supported = + (get_feat_ctx.offload.rx_supported & + ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK) != 0; + adapter->offloads.rss_hash_supported = + (get_feat_ctx.offload.rx_supported & + ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_MASK) != 0; /* Copy MAC address and point DPDK to it */ eth_dev->data->mac_addrs = (struct ether_addr *)adapter->mac_addr; ether_addr_copy((struct ether_addr *)get_feat_ctx.dev_attr.mac_addr, (struct ether_addr *)adapter->mac_addr); + rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE); + if (unlikely(rc != 0)) { + PMD_DRV_LOG(ERR, "Failed to initialize RSS in ENA device\n"); + goto err_delete_debug_area; + } + adapter->drv_stats = rte_zmalloc("adapter stats", sizeof(*adapter->drv_stats), RTE_CACHE_LINE_SIZE); if (!adapter->drv_stats) { - RTE_LOG(ERR, PMD, "failed to alloc mem for adapter stats\n"); - return -ENOMEM; + PMD_INIT_LOG(ERR, + "Failed to allocate memory for adapter statistics\n"); + rc = -ENOMEM; + goto err_rss_destroy; } rte_eth_copy_pci_info(eth_dev, pci_dev); + rte_spinlock_init(&adapter->admin_lock); + + rte_intr_callback_register(intr_handle, + ena_interrupt_handler_rte, + eth_dev); + rte_intr_enable(intr_handle); + ena_com_set_admin_polling_mode(ena_dev, false); + ena_com_admin_aenq_enable(ena_dev); + + if (adapters_found == 0) + rte_timer_subsystem_init(); + rte_timer_init(&adapter->timer_wd); + adapters_found++; adapter->state = ENA_ADAPTER_STATE_INIT; return 0; + +err_rss_destroy: + ena_com_rss_destroy(ena_dev); +err_delete_debug_area: + ena_com_delete_debug_area(ena_dev); + +err_device_destroy: + ena_com_delete_host_info(ena_dev); + ena_com_admin_destroy(ena_dev); + +err: + return rc; +} + +static void ena_destroy_device(struct rte_eth_dev *eth_dev) +{ + struct ena_adapter *adapter = eth_dev->data->dev_private; + struct ena_com_dev *ena_dev = &adapter->ena_dev; + + if (adapter->state == ENA_ADAPTER_STATE_FREE) + return; + + ena_com_set_admin_running_state(ena_dev, false); + + if (adapter->state != ENA_ADAPTER_STATE_CLOSED) + ena_close(eth_dev); + + ena_com_rss_destroy(ena_dev); + + ena_com_delete_debug_area(ena_dev); + ena_com_delete_host_info(ena_dev); + + ena_com_abort_admin_commands(ena_dev); + ena_com_wait_for_abort_completion(ena_dev); + ena_com_admin_destroy(ena_dev); + ena_com_mmio_reg_read_request_destroy(ena_dev); + + adapter->state = ENA_ADAPTER_STATE_FREE; +} + +static int eth_ena_dev_uninit(struct rte_eth_dev *eth_dev) +{ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + + ena_destroy_device(eth_dev); + + eth_dev->dev_ops = NULL; + eth_dev->rx_pkt_burst = NULL; + eth_dev->tx_pkt_burst = NULL; + + return 0; } static int ena_dev_configure(struct rte_eth_dev *dev) @@ -1386,34 +1796,17 @@ struct ena_adapter *adapter = (struct ena_adapter *)(dev->data->dev_private); - if (!(adapter->state == ENA_ADAPTER_STATE_INIT || - adapter->state == ENA_ADAPTER_STATE_STOPPED)) { - PMD_INIT_LOG(ERR, "Illegal adapter state: %d", - adapter->state); - return -1; - } - - switch (adapter->state) { - case ENA_ADAPTER_STATE_INIT: - case ENA_ADAPTER_STATE_STOPPED: - adapter->state = ENA_ADAPTER_STATE_CONFIG; - break; - case ENA_ADAPTER_STATE_CONFIG: - RTE_LOG(WARNING, PMD, - "Ivalid driver state while trying to configure device\n"); - break; - default: - break; - } + adapter->state = ENA_ADAPTER_STATE_CONFIG; return 0; } -static void ena_init_rings(struct ena_adapter *adapter) +static void ena_init_rings(struct ena_adapter *adapter, + bool disable_meta_caching) { - int i; + size_t i; - for (i = 0; i < adapter->num_queues; i++) { + for (i = 0; i < adapter->max_num_io_queues; i++) { struct ena_ring *ring = &adapter->tx_ring[i]; ring->configured = 0; @@ -1422,15 +1815,18 @@ ring->id = i; ring->tx_mem_queue_type = adapter->ena_dev.tx_mem_queue_type; ring->tx_max_header_size = adapter->ena_dev.tx_max_header_size; + ring->sgl_size = adapter->max_tx_sgl_size; + ring->disable_meta_caching = disable_meta_caching; } - for (i = 0; i < adapter->num_queues; i++) { + for (i = 0; i < adapter->max_num_io_queues; i++) { struct ena_ring *ring = &adapter->rx_ring[i]; ring->configured = 0; ring->type = ENA_RING_TYPE_RX; ring->adapter = adapter; ring->id = i; + ring->sgl_size = adapter->max_rx_sgl_size; } } @@ -1439,18 +1835,14 @@ { struct ena_adapter *adapter; struct ena_com_dev *ena_dev; - struct ena_com_dev_get_features_ctx feat; uint32_t rx_feat = 0, tx_feat = 0; - int rc = 0; - ena_assert_msg(dev->data != NULL, "Uninitialized device"); - ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device"); - adapter = (struct ena_adapter *)(dev->data->dev_private); + ena_assert_msg(dev->data != NULL, "Uninitialized device\n"); + ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device\n"); + adapter = dev->data->dev_private; ena_dev = &adapter->ena_dev; - ena_assert_msg(ena_dev != NULL, "Uninitialized device"); - - dev_info->pci_dev = RTE_DEV_TO_PCI(dev->device); + ena_assert_msg(ena_dev != NULL, "Uninitialized device\n"); dev_info->speed_capa = ETH_LINK_SPEED_1G | @@ -1462,26 +1854,16 @@ ETH_LINK_SPEED_50G | ETH_LINK_SPEED_100G; - /* Get supported features from HW */ - rc = ena_com_get_dev_attr_feat(ena_dev, &feat); - if (unlikely(rc)) { - RTE_LOG(ERR, PMD, - "Cannot get attribute for ena device rc= %d\n", rc); - return; - } - /* Set Tx & Rx features available for device */ - if (feat.offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) + if (adapter->offloads.tso4_supported) tx_feat |= DEV_TX_OFFLOAD_TCP_TSO; - if (feat.offload.tx & - ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK) + if (adapter->offloads.tx_csum_supported) tx_feat |= DEV_TX_OFFLOAD_IPV4_CKSUM | DEV_TX_OFFLOAD_UDP_CKSUM | DEV_TX_OFFLOAD_TCP_CKSUM; - if (feat.offload.rx_supported & - ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK) + if (adapter->offloads.rx_csum_supported) rx_feat |= DEV_RX_OFFLOAD_IPV4_CKSUM | DEV_RX_OFFLOAD_UDP_CKSUM | DEV_RX_OFFLOAD_TCP_CKSUM; @@ -1490,306 +1872,913 @@ dev_info->rx_offload_capa = rx_feat; dev_info->tx_offload_capa = tx_feat; + dev_info->flow_type_rss_offloads = ENA_ALL_RSS_HF; + dev_info->hash_key_size = ENA_HASH_KEY_SIZE; + dev_info->min_rx_bufsize = ENA_MIN_FRAME_LEN; dev_info->max_rx_pktlen = adapter->max_mtu; dev_info->max_mac_addrs = 1; - dev_info->max_rx_queues = adapter->num_queues; - dev_info->max_tx_queues = adapter->num_queues; + dev_info->max_rx_queues = adapter->max_num_io_queues; + dev_info->max_tx_queues = adapter->max_num_io_queues; dev_info->reta_size = ENA_RX_RSS_TABLE_SIZE; + + dev_info->rx_desc_lim.nb_max = adapter->max_rx_ring_size; + dev_info->rx_desc_lim.nb_min = ENA_MIN_RING_DESC; + + dev_info->tx_desc_lim.nb_max = adapter->max_tx_ring_size; + dev_info->tx_desc_lim.nb_min = ENA_MIN_RING_DESC; +} + +static inline void ena_init_rx_mbuf(struct rte_mbuf *mbuf, uint16_t len) +{ + mbuf->data_len = len; + mbuf->data_off = RTE_PKTMBUF_HEADROOM; + mbuf->refcnt = 1; + mbuf->next = NULL; +} + +static struct rte_mbuf *ena_rx_mbuf(struct ena_ring *rx_ring, + struct ena_com_rx_buf_info *ena_bufs, + uint32_t descs, + uint16_t *next_to_clean, + uint8_t offset) +{ + struct rte_mbuf *mbuf; + struct rte_mbuf *mbuf_head; + struct ena_rx_buffer *rx_info; + int rc; + uint16_t ntc, len, req_id, buf = 0; + + if (unlikely(descs == 0)) + return NULL; + + ntc = *next_to_clean; + + len = ena_bufs[buf].len; + req_id = ena_bufs[buf].req_id; + + rx_info = &rx_ring->rx_buffer_info[req_id]; + + mbuf = rx_info->mbuf; + RTE_ASSERT(mbuf != NULL); + + ena_init_rx_mbuf(mbuf, len); + + /* Fill the mbuf head with the data specific for 1st segment. */ + mbuf_head = mbuf; + mbuf_head->nb_segs = descs; + mbuf_head->port = rx_ring->port_id; + mbuf_head->pkt_len = len; + mbuf_head->data_off += offset; + + rx_info->mbuf = NULL; + rx_ring->empty_rx_reqs[ntc] = req_id; + ntc = ENA_IDX_NEXT_MASKED(ntc, rx_ring->size_mask); + + while (--descs) { + ++buf; + len = ena_bufs[buf].len; + req_id = ena_bufs[buf].req_id; + + rx_info = &rx_ring->rx_buffer_info[req_id]; + RTE_ASSERT(rx_info->mbuf != NULL); + + if (unlikely(len == 0)) { + /* + * Some devices can pass descriptor with the length 0. + * To avoid confusion, the PMD is simply putting the + * descriptor back, as it was never used. We'll avoid + * mbuf allocation that way. + */ + rc = ena_add_single_rx_desc(rx_ring->ena_com_io_sq, + rx_info->mbuf, req_id); + if (unlikely(rc != 0)) { + /* Free the mbuf in case of an error. */ + __rte_mbuf_raw_free(rx_info->mbuf); + } else { + /* + * If there was no error, just exit the loop as + * 0 length descriptor is always the last one. + */ + break; + } + } else { + /* Create an mbuf chain. */ + mbuf->next = rx_info->mbuf; + mbuf = mbuf->next; + + ena_init_rx_mbuf(mbuf, len); + mbuf_head->pkt_len += len; + } + + /* + * Mark the descriptor as depleted and perform necessary + * cleanup. + * This code will execute in two cases: + * 1. Descriptor len was greater than 0 - normal situation. + * 2. Descriptor len was 0 and we failed to add the descriptor + * to the device. In that situation, we should try to add + * the mbuf again in the populate routine and mark the + * descriptor as used up by the device. + */ + rx_info->mbuf = NULL; + rx_ring->empty_rx_reqs[ntc] = req_id; + ntc = ENA_IDX_NEXT_MASKED(ntc, rx_ring->size_mask); + } + + *next_to_clean = ntc; + + return mbuf_head; } static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { struct ena_ring *rx_ring = (struct ena_ring *)(rx_queue); - unsigned int ring_size = rx_ring->ring_size; - unsigned int ring_mask = ring_size - 1; + unsigned int free_queue_entries; + unsigned int refill_threshold; uint16_t next_to_clean = rx_ring->next_to_clean; - uint16_t desc_in_use = 0; - unsigned int recv_idx = 0; - struct rte_mbuf *mbuf = NULL; - struct rte_mbuf *mbuf_head = NULL; - struct rte_mbuf *mbuf_prev = NULL; - struct rte_mbuf **rx_buff_info = rx_ring->rx_buffer_info; - unsigned int completed; - + uint16_t descs_in_use; + struct rte_mbuf *mbuf; + uint16_t completed; struct ena_com_rx_ctx ena_rx_ctx; - int rc = 0; + int i, rc = 0; +#ifdef RTE_LIBRTE_ENA_DEBUG_TX /* Check adapter state */ if (unlikely(rx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { - RTE_LOG(ALERT, PMD, + PMD_RX_LOG(ALERT, "Trying to receive pkts while device is NOT running\n"); return 0; } +#endif - desc_in_use = rx_ring->next_to_use - next_to_clean; - if (unlikely(nb_pkts > desc_in_use)) - nb_pkts = desc_in_use; + descs_in_use = rx_ring->ring_size - + ena_com_free_q_entries(rx_ring->ena_com_io_sq) - 1; + nb_pkts = RTE_MIN(descs_in_use, nb_pkts); for (completed = 0; completed < nb_pkts; completed++) { - int segments = 0; - - ena_rx_ctx.max_bufs = rx_ring->ring_size; + ena_rx_ctx.max_bufs = rx_ring->sgl_size; ena_rx_ctx.ena_bufs = rx_ring->ena_bufs; ena_rx_ctx.descs = 0; + ena_rx_ctx.pkt_offset = 0; /* receive packet context */ rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq, rx_ring->ena_com_io_sq, &ena_rx_ctx); if (unlikely(rc)) { - RTE_LOG(ERR, PMD, "ena_com_rx_pkt error %d\n", rc); + PMD_RX_LOG(ERR, + "Failed to get the packet from the device, rc: %d\n", + rc); + if (rc == ENA_COM_NO_SPACE) { + ++rx_ring->rx_stats.bad_desc_num; + rx_ring->adapter->reset_reason = + ENA_REGS_RESET_TOO_MANY_RX_DESCS; + } else { + ++rx_ring->rx_stats.bad_req_id; + rx_ring->adapter->reset_reason = + ENA_REGS_RESET_INV_RX_REQ_ID; + } + rx_ring->adapter->trigger_reset = true; return 0; } - if (unlikely(ena_rx_ctx.descs == 0)) - break; - - while (segments < ena_rx_ctx.descs) { - mbuf = rx_buff_info[next_to_clean & ring_mask]; - mbuf->data_len = ena_rx_ctx.ena_bufs[segments].len; - mbuf->data_off = RTE_PKTMBUF_HEADROOM; - mbuf->refcnt = 1; - mbuf->next = NULL; - if (segments == 0) { - mbuf->nb_segs = ena_rx_ctx.descs; - mbuf->port = rx_ring->port_id; - mbuf->pkt_len = 0; - mbuf_head = mbuf; - } else { - /* for multi-segment pkts create mbuf chain */ - mbuf_prev->next = mbuf; + mbuf = ena_rx_mbuf(rx_ring, + ena_rx_ctx.ena_bufs, + ena_rx_ctx.descs, + &next_to_clean, + ena_rx_ctx.pkt_offset); + if (unlikely(mbuf == NULL)) { + for (i = 0; i < ena_rx_ctx.descs; ++i) { + rx_ring->empty_rx_reqs[next_to_clean] = + rx_ring->ena_bufs[i].req_id; + next_to_clean = ENA_IDX_NEXT_MASKED( + next_to_clean, rx_ring->size_mask); } - mbuf_head->pkt_len += mbuf->data_len; - - mbuf_prev = mbuf; - segments++; - next_to_clean++; + break; } /* fill mbuf attributes if any */ - ena_rx_mbuf_prepare(mbuf_head, &ena_rx_ctx); - mbuf_head->hash.rss = (uint32_t)rx_ring->id; + ena_rx_mbuf_prepare(mbuf, &ena_rx_ctx); + + if (unlikely(mbuf->ol_flags & + (PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD))) { + rte_atomic64_inc(&rx_ring->adapter->drv_stats->ierrors); + ++rx_ring->rx_stats.bad_csum; + } - /* pass to DPDK application head mbuf */ - rx_pkts[recv_idx] = mbuf_head; - recv_idx++; + rx_pkts[completed] = mbuf; + rx_ring->rx_stats.bytes += mbuf->pkt_len; } + rx_ring->rx_stats.cnt += completed; + rx_ring->next_to_clean = next_to_clean; + + free_queue_entries = ena_com_free_q_entries(rx_ring->ena_com_io_sq); + refill_threshold = + RTE_MIN(rx_ring->ring_size / ENA_REFILL_THRESH_DIVIDER, + (unsigned int)ENA_REFILL_THRESH_PACKET); + /* Burst refill to save doorbells, memory barriers, const interval */ - if (ring_size - desc_in_use > ENA_RING_DESCS_RATIO(ring_size)) - ena_populate_rx_queue(rx_ring, ring_size - desc_in_use); + if (free_queue_entries > refill_threshold) { + ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq); + ena_populate_rx_queue(rx_ring, free_queue_entries); + } - rx_ring->next_to_clean = next_to_clean; + return completed; +} + +static void ena_update_hints(struct ena_adapter *adapter, + struct ena_admin_ena_hw_hints *hints) +{ + if (hints->admin_completion_tx_timeout) + adapter->ena_dev.admin_queue.completion_timeout = + hints->admin_completion_tx_timeout * 1000; - return recv_idx; + if (hints->mmio_read_timeout) + /* convert to usec */ + adapter->ena_dev.mmio_read.reg_read_to = + hints->mmio_read_timeout * 1000; + + if (hints->driver_watchdog_timeout) { + if (hints->driver_watchdog_timeout == ENA_HW_HINTS_NO_TIMEOUT) + adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT; + else + // Convert msecs to ticks + adapter->keep_alive_timeout = + (hints->driver_watchdog_timeout * + rte_get_timer_hz()) / 1000; + } } -static uint16_t -eth_ena_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, - uint16_t nb_pkts) +static int ena_check_space_and_linearize_mbuf(struct ena_ring *tx_ring, + struct rte_mbuf *mbuf) { - int32_t ret; - uint32_t i; - struct rte_mbuf *m; - struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue); - struct ipv4_hdr *ip_hdr; - uint64_t ol_flags; - uint16_t frag_field; - - /* ENA needs partial checksum for TSO packets only, skip early */ - if (!tx_ring->adapter->tso4_supported) - return nb_pkts; - - for (i = 0; i != nb_pkts; i++) { - m = tx_pkts[i]; - ol_flags = m->ol_flags; - - if ((ol_flags & ENA_TX_OFFLOAD_NOTSUP_MASK) != 0 || - (ol_flags & PKT_TX_L4_MASK) == - PKT_TX_SCTP_CKSUM) { - rte_errno = -ENOTSUP; - return i; - } + struct ena_com_dev *ena_dev; + int num_segments, header_len, rc; -#ifdef RTE_LIBRTE_ETHDEV_DEBUG - ret = rte_validate_tx_offload(m); - if (ret != 0) { - rte_errno = ret; - return i; - } -#endif + ena_dev = &tx_ring->adapter->ena_dev; + num_segments = mbuf->nb_segs; + header_len = mbuf->data_len; + + if (likely(num_segments < tx_ring->sgl_size)) + goto checkspace; + + if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV && + (num_segments == tx_ring->sgl_size) && + (header_len < tx_ring->tx_max_header_size)) + goto checkspace; + + /* Checking for space for 2 additional metadata descriptors due to + * possible header split and metadata descriptor. Linearization will + * be needed so we reduce the segments number from num_segments to 1 + */ + if (!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq, 3)) { + PMD_TX_LOG(DEBUG, "Not enough space in the Tx queue\n"); + return ENA_COM_NO_MEM; + } + ++tx_ring->tx_stats.linearize; + rc = rte_pktmbuf_linearize(mbuf); + if (unlikely(rc)) { + PMD_TX_LOG(WARNING, "Mbuf linearize failed\n"); + rte_atomic64_inc(&tx_ring->adapter->drv_stats->ierrors); + ++tx_ring->tx_stats.linearize_failed; + return rc; + } - if (!(m->ol_flags & PKT_TX_IPV4)) - continue; + return 0; - ip_hdr = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *, - m->l2_len); - frag_field = rte_be_to_cpu_16(ip_hdr->fragment_offset); - if (frag_field & IPV4_HDR_DF_FLAG) - continue; +checkspace: + /* Checking for space for 2 additional metadata descriptors due to + * possible header split and metadata descriptor + */ + if (!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq, + num_segments + 2)) { + PMD_TX_LOG(DEBUG, "Not enough space in the Tx queue\n"); + return ENA_COM_NO_MEM; + } + + return 0; +} - /* In case we are supposed to TSO and have DF not set (DF=0) - * hardware must be provided with partial checksum, otherwise - * it will take care of necessary calculations. +static void ena_tx_map_mbuf(struct ena_ring *tx_ring, + struct ena_tx_buffer *tx_info, + struct rte_mbuf *mbuf, + void **push_header, + uint16_t *header_len) +{ + struct ena_com_buf *ena_buf; + uint16_t delta, seg_len, push_len; + + delta = 0; + seg_len = mbuf->data_len; + + tx_info->mbuf = mbuf; + ena_buf = tx_info->bufs; + + if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { + /* + * Tx header might be (and will be in most cases) smaller than + * tx_max_header_size. But it's not an issue to send more data + * to the device, than actually needed if the mbuf size is + * greater than tx_max_header_size. */ + push_len = RTE_MIN(mbuf->pkt_len, tx_ring->tx_max_header_size); + *header_len = push_len; - ret = rte_net_intel_cksum_flags_prepare(m, - ol_flags & ~PKT_TX_TCP_SEG); - if (ret != 0) { - rte_errno = ret; - return i; + if (likely(push_len <= seg_len)) { + /* If the push header is in the single segment, then + * just point it to the 1st mbuf data. + */ + *push_header = rte_pktmbuf_mtod(mbuf, uint8_t *); + } else { + /* If the push header lays in the several segments, copy + * it to the intermediate buffer. + */ + rte_pktmbuf_read(mbuf, 0, push_len, + tx_ring->push_buf_intermediate_buf); + *push_header = tx_ring->push_buf_intermediate_buf; + delta = push_len - seg_len; + } + } else { + *push_header = NULL; + *header_len = 0; + push_len = 0; + } + + /* Process first segment taking into consideration pushed header */ + if (seg_len > push_len) { + ena_buf->paddr = mbuf->buf_physaddr + + mbuf->data_off + + push_len; + ena_buf->len = seg_len - push_len; + ena_buf++; + tx_info->num_of_bufs++; + } + + while ((mbuf = mbuf->next) != NULL) { + seg_len = mbuf->data_len; + + /* Skip mbufs if whole data is pushed as a header */ + if (unlikely(delta > seg_len)) { + delta -= seg_len; + continue; } + + ena_buf->paddr = mbuf->buf_physaddr + mbuf->data_off + delta; + ena_buf->len = seg_len - delta; + ena_buf++; + tx_info->num_of_bufs++; + + delta = 0; } +} - return i; +static int ena_xmit_mbuf(struct ena_ring *tx_ring, struct rte_mbuf *mbuf) +{ + struct ena_tx_buffer *tx_info; + struct ena_com_tx_ctx ena_tx_ctx = { 0 }; + uint16_t next_to_use; + uint16_t header_len; + uint16_t req_id; + void *push_header; + int nb_hw_desc; + int rc; + + rc = ena_check_space_and_linearize_mbuf(tx_ring, mbuf); + if (unlikely(rc)) + return rc; + + next_to_use = tx_ring->next_to_use; + + req_id = tx_ring->empty_tx_reqs[next_to_use]; + tx_info = &tx_ring->tx_buffer_info[req_id]; + tx_info->num_of_bufs = 0; + + ena_tx_map_mbuf(tx_ring, tx_info, mbuf, &push_header, &header_len); + + ena_tx_ctx.ena_bufs = tx_info->bufs; + ena_tx_ctx.push_header = push_header; + ena_tx_ctx.num_bufs = tx_info->num_of_bufs; + ena_tx_ctx.req_id = req_id; + ena_tx_ctx.header_len = header_len; + + /* Set Tx offloads flags, if applicable */ + ena_tx_mbuf_prepare(mbuf, &ena_tx_ctx, tx_ring->disable_meta_caching); + + if (unlikely(ena_com_is_doorbell_needed(tx_ring->ena_com_io_sq, + &ena_tx_ctx))) { + PMD_TX_LOG(DEBUG, + "LLQ Tx max burst size of queue %d achieved, writing doorbell to send burst\n", + tx_ring->id); + ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); + tx_ring->tx_stats.doorbells++; + tx_ring->pkts_without_db = false; + } + + /* prepare the packet's descriptors to dma engine */ + rc = ena_com_prepare_tx(tx_ring->ena_com_io_sq, &ena_tx_ctx, + &nb_hw_desc); + if (unlikely(rc)) { + PMD_DRV_LOG(ERR, "Failed to prepare Tx buffers, rc: %d\n", rc); + ++tx_ring->tx_stats.prepare_ctx_err; + tx_ring->adapter->reset_reason = + ENA_REGS_RESET_DRIVER_INVALID_STATE; + tx_ring->adapter->trigger_reset = true; + return rc; + } + + tx_info->tx_descs = nb_hw_desc; + + tx_ring->tx_stats.cnt++; + tx_ring->tx_stats.bytes += mbuf->pkt_len; + + tx_ring->next_to_use = ENA_IDX_NEXT_MASKED(next_to_use, + tx_ring->size_mask); + + return 0; +} + +static void ena_tx_cleanup(struct ena_ring *tx_ring) +{ + unsigned int cleanup_budget; + unsigned int total_tx_descs = 0; + uint16_t next_to_clean = tx_ring->next_to_clean; + + cleanup_budget = RTE_MIN(tx_ring->ring_size / ENA_REFILL_THRESH_DIVIDER, + (unsigned int)ENA_REFILL_THRESH_PACKET); + + while (likely(total_tx_descs < cleanup_budget)) { + struct rte_mbuf *mbuf; + struct ena_tx_buffer *tx_info; + uint16_t req_id; + + if (ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq, &req_id) != 0) + break; + + if (unlikely(validate_tx_req_id(tx_ring, req_id) != 0)) + break; + + /* Get Tx info & store how many descs were processed */ + tx_info = &tx_ring->tx_buffer_info[req_id]; + + mbuf = tx_info->mbuf; + rte_pktmbuf_free(mbuf); + + tx_info->mbuf = NULL; + tx_ring->empty_tx_reqs[next_to_clean] = req_id; + + total_tx_descs += tx_info->tx_descs; + + /* Put back descriptor to the ring for reuse */ + next_to_clean = ENA_IDX_NEXT_MASKED(next_to_clean, + tx_ring->size_mask); + } + + if (likely(total_tx_descs > 0)) { + /* acknowledge completion of sent packets */ + tx_ring->next_to_clean = next_to_clean; + ena_com_comp_ack(tx_ring->ena_com_io_sq, total_tx_descs); + ena_com_update_dev_comp_head(tx_ring->ena_com_io_cq); + } } static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue); - uint16_t next_to_use = tx_ring->next_to_use; - uint16_t next_to_clean = tx_ring->next_to_clean; - struct rte_mbuf *mbuf; - unsigned int ring_size = tx_ring->ring_size; - unsigned int ring_mask = ring_size - 1; - struct ena_com_tx_ctx ena_tx_ctx; - struct ena_tx_buffer *tx_info; - struct ena_com_buf *ebuf; - uint16_t rc, req_id, total_tx_descs = 0; - uint16_t sent_idx = 0, empty_tx_reqs; - int nb_hw_desc; + uint16_t sent_idx = 0; +#ifdef RTE_LIBRTE_ENA_DEBUG_TX /* Check adapter state */ if (unlikely(tx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { - RTE_LOG(ALERT, PMD, + PMD_TX_LOG(ALERT, "Trying to xmit pkts while device is NOT running\n"); return 0; } - - empty_tx_reqs = ring_size - (next_to_use - next_to_clean); - if (nb_pkts > empty_tx_reqs) - nb_pkts = empty_tx_reqs; +#endif for (sent_idx = 0; sent_idx < nb_pkts; sent_idx++) { - mbuf = tx_pkts[sent_idx]; + if (ena_xmit_mbuf(tx_ring, tx_pkts[sent_idx])) + break; + tx_ring->pkts_without_db = true; + rte_prefetch0(tx_pkts[ENA_IDX_ADD_MASKED(sent_idx, 4, + tx_ring->size_mask)]); + } - req_id = tx_ring->empty_tx_reqs[next_to_use & ring_mask]; - tx_info = &tx_ring->tx_buffer_info[req_id]; - tx_info->mbuf = mbuf; - tx_info->num_of_bufs = 0; - ebuf = tx_info->bufs; - - /* Prepare TX context */ - memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx)); - memset(&ena_tx_ctx.ena_meta, 0x0, - sizeof(struct ena_com_tx_meta)); - ena_tx_ctx.ena_bufs = ebuf; - ena_tx_ctx.req_id = req_id; - if (tx_ring->tx_mem_queue_type == - ENA_ADMIN_PLACEMENT_POLICY_DEV) { - /* prepare the push buffer with - * virtual address of the data - */ - ena_tx_ctx.header_len = - RTE_MIN(mbuf->data_len, - tx_ring->tx_max_header_size); - ena_tx_ctx.push_header = - (void *)((char *)mbuf->buf_addr + - mbuf->data_off); - } /* there's no else as we take advantage of memset zeroing */ + tx_ring->tx_stats.available_desc = + ena_com_free_q_entries(tx_ring->ena_com_io_sq); + + /* If there are ready packets to be xmitted... */ + if (likely(tx_ring->pkts_without_db)) { + /* ...let HW do its best :-) */ + ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); + tx_ring->tx_stats.doorbells++; + tx_ring->pkts_without_db = false; + } - /* Set TX offloads flags, if applicable */ - ena_tx_mbuf_prepare(mbuf, &ena_tx_ctx); + ena_tx_cleanup(tx_ring); - if (unlikely(mbuf->ol_flags & - (PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD))) - rte_atomic64_inc(&tx_ring->adapter->drv_stats->ierrors); + tx_ring->tx_stats.available_desc = + ena_com_free_q_entries(tx_ring->ena_com_io_sq); + tx_ring->tx_stats.tx_poll++; - rte_prefetch0(tx_pkts[(sent_idx + 4) & ring_mask]); + return sent_idx; +} - /* Process first segment taking into - * consideration pushed header - */ - if (mbuf->data_len > ena_tx_ctx.header_len) { - ebuf->paddr = mbuf->buf_physaddr + - mbuf->data_off + - ena_tx_ctx.header_len; - ebuf->len = mbuf->data_len - ena_tx_ctx.header_len; - ebuf++; - tx_info->num_of_bufs++; +int ena_copy_eni_stats(struct ena_adapter *adapter) +{ + struct ena_admin_eni_stats admin_eni_stats; + int rc; + + rte_spinlock_lock(&adapter->admin_lock); + rc = ena_com_get_eni_stats(&adapter->ena_dev, &admin_eni_stats); + rte_spinlock_unlock(&adapter->admin_lock); + if (rc != 0) { + if (rc == ENA_COM_UNSUPPORTED) { + PMD_DRV_LOG(DEBUG, + "Retrieving ENI metrics is not supported\n"); + } else { + PMD_DRV_LOG(WARNING, + "Failed to get ENI metrics, rc: %d\n", rc); } + return rc; + } + + rte_memcpy(&adapter->eni_stats, &admin_eni_stats, + sizeof(struct ena_stats_eni)); + + return 0; +} + +/** + * DPDK callback to retrieve names of extended device statistics + * + * @param dev + * Pointer to Ethernet device structure. + * @param[out] xstats_names + * Buffer to insert names into. + * @param n + * Number of names. + * + * @return + * Number of xstats names. + */ +static int ena_xstats_get_names(struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, + unsigned int n) +{ + unsigned int xstats_count = ena_xstats_calc_num(dev->data); + unsigned int stat, i, count = 0; + + if (n < xstats_count || !xstats_names) + return xstats_count; + + for (stat = 0; stat < ENA_STATS_ARRAY_GLOBAL; stat++, count++) + strcpy(xstats_names[count].name, + ena_stats_global_strings[stat].name); + + for (stat = 0; stat < ENA_STATS_ARRAY_ENI; stat++, count++) + strcpy(xstats_names[count].name, + ena_stats_eni_strings[stat].name); + + for (stat = 0; stat < ENA_STATS_ARRAY_RX; stat++) + for (i = 0; i < dev->data->nb_rx_queues; i++, count++) + snprintf(xstats_names[count].name, + sizeof(xstats_names[count].name), + "rx_q%d_%s", i, + ena_stats_rx_strings[stat].name); + + for (stat = 0; stat < ENA_STATS_ARRAY_TX; stat++) + for (i = 0; i < dev->data->nb_tx_queues; i++, count++) + snprintf(xstats_names[count].name, + sizeof(xstats_names[count].name), + "tx_q%d_%s", i, + ena_stats_tx_strings[stat].name); + + return xstats_count; +} + +/** + * DPDK callback to get extended device statistics. + * + * @param dev + * Pointer to Ethernet device structure. + * @param[out] stats + * Stats table output buffer. + * @param n + * The size of the stats table. + * + * @return + * Number of xstats on success, negative on failure. + */ +static int ena_xstats_get(struct rte_eth_dev *dev, + struct rte_eth_xstat *xstats, + unsigned int n) +{ + struct ena_adapter *adapter = dev->data->dev_private; + unsigned int xstats_count = ena_xstats_calc_num(dev->data); + unsigned int stat, i, count = 0; + int stat_offset; + void *stats_begin; + + if (n < xstats_count) + return xstats_count; + + if (!xstats) + return 0; - while ((mbuf = mbuf->next) != NULL) { - ebuf->paddr = mbuf->buf_physaddr + mbuf->data_off; - ebuf->len = mbuf->data_len; - ebuf++; - tx_info->num_of_bufs++; + for (stat = 0; stat < ENA_STATS_ARRAY_GLOBAL; stat++, count++) { + stat_offset = ena_stats_global_strings[stat].stat_offset; + stats_begin = &adapter->dev_stats; + + xstats[count].id = count; + xstats[count].value = *((uint64_t *) + ((char *)stats_begin + stat_offset)); + } + + /* Even if the function below fails, we should copy previous (or initial + * values) to keep structure of rte_eth_xstat consistent. + */ + ena_copy_eni_stats(adapter); + for (stat = 0; stat < ENA_STATS_ARRAY_ENI; stat++, count++) { + stat_offset = ena_stats_eni_strings[stat].stat_offset; + stats_begin = &adapter->eni_stats; + + xstats[count].id = count; + xstats[count].value = *((uint64_t *) + ((char *)stats_begin + stat_offset)); + } + + for (stat = 0; stat < ENA_STATS_ARRAY_RX; stat++) { + for (i = 0; i < dev->data->nb_rx_queues; i++, count++) { + stat_offset = ena_stats_rx_strings[stat].stat_offset; + stats_begin = &adapter->rx_ring[i].rx_stats; + + xstats[count].id = count; + xstats[count].value = *((uint64_t *) + ((char *)stats_begin + stat_offset)); } + } - ena_tx_ctx.num_bufs = tx_info->num_of_bufs; + for (stat = 0; stat < ENA_STATS_ARRAY_TX; stat++) { + for (i = 0; i < dev->data->nb_tx_queues; i++, count++) { + stat_offset = ena_stats_tx_strings[stat].stat_offset; + stats_begin = &adapter->tx_ring[i].rx_stats; + + xstats[count].id = count; + xstats[count].value = *((uint64_t *) + ((char *)stats_begin + stat_offset)); + } + } - /* Write data to device */ - rc = ena_com_prepare_tx(tx_ring->ena_com_io_sq, - &ena_tx_ctx, &nb_hw_desc); - if (unlikely(rc)) - break; + return count; +} - tx_info->tx_descs = nb_hw_desc; +static int ena_process_bool_devarg(const char *key, + const char *value, + void *opaque) +{ + struct ena_adapter *adapter = opaque; + bool bool_value; - next_to_use++; + /* Parse the value. */ + if (strcmp(value, "1") == 0) { + bool_value = true; + } else if (strcmp(value, "0") == 0) { + bool_value = false; + } else { + PMD_INIT_LOG(ERR, + "Invalid value: '%s' for key '%s'. Accepted: '0' or '1'\n", + value, key); + return -EINVAL; } - /* If there are ready packets to be xmitted... */ - if (sent_idx > 0) { - /* ...let HW do its best :-) */ - rte_wmb(); - ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); + /* Now, assign it to the proper adapter field. */ + if (strcmp(key, ENA_DEVARG_LARGE_LLQ_HDR) == 0) + adapter->use_large_llq_hdr = bool_value; + + return 0; +} + +static int ena_parse_devargs(struct ena_adapter *adapter, + struct rte_devargs *devargs) +{ + static const char * allowed_args[] = { + ENA_DEVARG_LARGE_LLQ_HDR, + NULL, + }; + struct rte_kvargs *kvlist; + int rc; + + if (devargs == NULL) + return 0; - tx_ring->next_to_use = next_to_use; + kvlist = rte_kvargs_parse(devargs->args, allowed_args); + if (kvlist == NULL) { + PMD_INIT_LOG(ERR, "Invalid device arguments: %s\n", + devargs->args); + return -EINVAL; } - /* Clear complete packets */ - while (ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq, &req_id) >= 0) { - /* Get Tx info & store how many descs were processed */ - tx_info = &tx_ring->tx_buffer_info[req_id]; - total_tx_descs += tx_info->tx_descs; + rc = rte_kvargs_process(kvlist, ENA_DEVARG_LARGE_LLQ_HDR, + ena_process_bool_devarg, adapter); - /* Free whole mbuf chain */ - mbuf = tx_info->mbuf; - rte_pktmbuf_free(mbuf); + rte_kvargs_free(kvlist); - /* Put back descriptor to the ring for reuse */ - tx_ring->empty_tx_reqs[next_to_clean & ring_mask] = req_id; - next_to_clean++; + return rc; +} - /* If too many descs to clean, leave it for another run */ - if (unlikely(total_tx_descs > ENA_RING_DESCS_RATIO(ring_size))) - break; +static int ena_setup_rx_intr(struct rte_eth_dev *dev) +{ + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; + int rc; + uint16_t vectors_nb, i; + bool rx_intr_requested = dev->data->dev_conf.intr_conf.rxq; + + if (!rx_intr_requested) + return 0; + + if (!rte_intr_cap_multiple(intr_handle)) { + PMD_DRV_LOG(ERR, + "Rx interrupt requested, but it isn't supported by the PCI driver\n"); + return -ENOTSUP; } - if (total_tx_descs > 0) { - /* acknowledge completion of sent packets */ - ena_com_comp_ack(tx_ring->ena_com_io_sq, total_tx_descs); - tx_ring->next_to_clean = next_to_clean; + /* Disable interrupt mapping before the configuration starts. */ + rte_intr_disable(intr_handle); + + /* Verify if there are enough vectors available. */ + vectors_nb = dev->data->nb_rx_queues; + if (vectors_nb > RTE_MAX_RXTX_INTR_VEC_ID) { + PMD_DRV_LOG(ERR, + "Too many Rx interrupts requested, maximum number: %d\n", + RTE_MAX_RXTX_INTR_VEC_ID); + rc = -ENOTSUP; + goto enable_intr; } - return sent_idx; + intr_handle->intr_vec = rte_zmalloc("intr_vec", + dev->data->nb_rx_queues * sizeof(*intr_handle->intr_vec), 0); + if (intr_handle->intr_vec == NULL) { + PMD_DRV_LOG(ERR, + "Failed to allocate interrupt vector for %d queues\n", + dev->data->nb_rx_queues); + rc = -ENOMEM; + goto enable_intr; + } + + rc = rte_intr_efd_enable(intr_handle, vectors_nb); + if (rc != 0) + goto free_intr_vec; + + if (!rte_intr_allow_others(intr_handle)) { + PMD_DRV_LOG(ERR, + "Not enough interrupts available to use both ENA Admin and Rx interrupts\n"); + goto disable_intr_efd; + } + + for (i = 0; i < vectors_nb; ++i) + intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + i; + + rte_intr_enable(intr_handle); + return 0; + +disable_intr_efd: + rte_intr_efd_disable(intr_handle); +free_intr_vec: + rte_free(intr_handle->intr_vec); + intr_handle->intr_vec = NULL; +enable_intr: + rte_intr_enable(intr_handle); + return rc; +} + +static void ena_rx_queue_intr_set(struct rte_eth_dev *dev, + uint16_t queue_id, + bool unmask) +{ + struct ena_adapter *adapter = dev->data->dev_private; + struct ena_ring *rxq = &adapter->rx_ring[queue_id]; + struct ena_eth_io_intr_reg intr_reg; + + ena_com_update_intr_reg(&intr_reg, 0, 0, unmask); + ena_com_unmask_intr(rxq->ena_com_io_cq, &intr_reg); +} + +static int ena_rx_queue_intr_enable(struct rte_eth_dev *dev, + uint16_t queue_id) +{ + ena_rx_queue_intr_set(dev, queue_id, true); + + return 0; +} + +static int ena_rx_queue_intr_disable(struct rte_eth_dev *dev, + uint16_t queue_id) +{ + ena_rx_queue_intr_set(dev, queue_id, false); + + return 0; } static struct eth_driver rte_ena_pmd = { .pci_drv = { .id_table = pci_id_ena_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | + RTE_PCI_DRV_WC_ACTIVATE, .probe = rte_eth_dev_pci_probe, .remove = rte_eth_dev_pci_remove, }, .eth_dev_init = eth_ena_dev_init, + .eth_dev_uninit = eth_ena_dev_uninit, .dev_private_size = sizeof(struct ena_adapter), }; RTE_PMD_REGISTER_PCI(net_ena, rte_ena_pmd.pci_drv); RTE_PMD_REGISTER_PCI_TABLE(net_ena, pci_id_ena_map); -RTE_PMD_REGISTER_KMOD_DEP(net_ena, "* igb_uio | uio_pci_generic | vfio"); +RTE_PMD_REGISTER_PARAM_STRING(net_ena, ENA_DEVARG_LARGE_LLQ_HDR "=<0|1>"); + +/****************************************************************************** + ******************************** AENQ Handlers ******************************* + *****************************************************************************/ +static void ena_update_on_link_change(void *adapter_data, + struct ena_admin_aenq_entry *aenq_e) +{ + struct rte_eth_dev *eth_dev = adapter_data; + struct ena_adapter *adapter = eth_dev->data->dev_private; + struct ena_admin_aenq_link_change_desc *aenq_link_desc; + uint32_t status; + + aenq_link_desc = (struct ena_admin_aenq_link_change_desc *)aenq_e; + + status = get_ena_admin_aenq_link_change_desc_link_status(aenq_link_desc); + adapter->link_status = status; + + ena_link_update(eth_dev, 0); + _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL); +} + +static void ena_notification(void *adapter_data, + struct ena_admin_aenq_entry *aenq_e) +{ + struct rte_eth_dev *eth_dev = adapter_data; + struct ena_adapter *adapter = eth_dev->data->dev_private; + struct ena_admin_ena_hw_hints *hints; + + if (aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION) + PMD_DRV_LOG(WARNING, "Invalid AENQ group: %x. Expected: %x\n", + aenq_e->aenq_common_desc.group, + ENA_ADMIN_NOTIFICATION); + + switch (aenq_e->aenq_common_desc.syndrome) { + case ENA_ADMIN_UPDATE_HINTS: + hints = (struct ena_admin_ena_hw_hints *) + (&aenq_e->inline_data_w4); + ena_update_hints(adapter, hints); + break; + default: + PMD_DRV_LOG(ERR, "Invalid AENQ notification link state: %d\n", + aenq_e->aenq_common_desc.syndrome); + } +} + +static void ena_keep_alive(void *adapter_data, + __rte_unused struct ena_admin_aenq_entry *aenq_e) +{ + struct rte_eth_dev *eth_dev = adapter_data; + struct ena_adapter *adapter = eth_dev->data->dev_private; + struct ena_admin_aenq_keep_alive_desc *desc; + uint64_t rx_drops; + uint64_t tx_drops; + + adapter->timestamp_wd = rte_get_timer_cycles(); + + desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e; + rx_drops = ((uint64_t)desc->rx_drops_high << 32) | desc->rx_drops_low; + tx_drops = ((uint64_t)desc->tx_drops_high << 32) | desc->tx_drops_low; + + adapter->drv_stats->rx_drops = rx_drops; + adapter->dev_stats.tx_drops = tx_drops; +} + +/** + * This handler will called for unknown event group or unimplemented handlers + **/ +static void unimplemented_aenq_handler(__rte_unused void *data, + __rte_unused struct ena_admin_aenq_entry *aenq_e) +{ + PMD_DRV_LOG(ERR, + "Unknown event was received or event with unimplemented handler\n"); +} + +static struct ena_aenq_handlers aenq_handlers = { + .handlers = { + [ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change, + [ENA_ADMIN_NOTIFICATION] = ena_notification, + [ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive + }, + .unimplemented_handler = unimplemented_aenq_handler +}; Index: /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/ena_logs.h =================================================================== --- /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/ena_logs.h (revision 38965) +++ /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/ena_logs.h (working copy) @@ -1,35 +1,7 @@ -/*- -* BSD LICENSE -* -* Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. -* All rights reserved. -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following conditions -* are met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above copyright -* notice, this list of conditions and the following disclaimer in -* the documentation and/or other materials provided with the -* distribution. -* * Neither the name of copyright holder nor the names of its -* contributors may be used to endorse or promote products derived -* from this software without specific prior written permission. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2019 Amazon.com, Inc. or its affiliates. + * All rights reserved. + */ #ifndef _ENA_LOGS_H_ #define _ENA_LOGS_H_ @@ -37,32 +9,25 @@ #define RTE_LOGTYPE_ENA RTE_LOGTYPE_USER1 #define PMD_INIT_LOG(level, fmt, args...) \ - RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args) + RTE_LOG(level, PMD, "%s(): " fmt, __func__, ## args) #ifdef RTE_LIBRTE_ENA_DEBUG_RX #define PMD_RX_LOG(level, fmt, args...) \ - RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args) + RTE_LOG(level, PMD, "%s(): " fmt, __func__, ## args) #else #define PMD_RX_LOG(level, fmt, args...) do { } while (0) #endif #ifdef RTE_LIBRTE_ENA_DEBUG_TX #define PMD_TX_LOG(level, fmt, args...) \ - RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args) + RTE_LOG(level, PMD, "%s(): " fmt, __func__, ## args) #else #define PMD_TX_LOG(level, fmt, args...) do { } while (0) #endif -#ifdef RTE_LIBRTE_ENA_DEBUG_TX_FREE -#define PMD_TX_FREE_LOG(level, fmt, args...) \ - RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args) -#else -#define PMD_TX_FREE_LOG(level, fmt, args...) do { } while (0) -#endif - #ifdef RTE_LIBRTE_ENA_DEBUG_DRIVER #define PMD_DRV_LOG(level, fmt, args...) \ - RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args) + RTE_LOG(level, PMD, "%s(): " fmt, __func__, ## args) #else #define PMD_DRV_LOG(level, fmt, args...) do { } while (0) #endif Index: /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/ena_platform.h =================================================================== --- /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/ena_platform.h (revision 38965) +++ /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/ena_platform.h (working copy) @@ -1,51 +1,11 @@ -/*- -* BSD LICENSE -* -* Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates. -* All rights reserved. -* -* Redistribution and use in source and binary forms, with or without -* modification, are permitted provided that the following conditions -* are met: -* -* * Redistributions of source code must retain the above copyright -* notice, this list of conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above copyright -* notice, this list of conditions and the following disclaimer in -* the documentation and/or other materials provided with the -* distribution. -* * Neither the name of copyright holder nor the names of its -* contributors may be used to endorse or promote products derived -* from this software without specific prior written permission. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2015-2019 Amazon.com, Inc. or its affiliates. + * All rights reserved. + */ #ifndef __ENA_PLATFORM_H__ #define __ENA_PLATFORM_H__ -#define swap16_to_le(x) (x) - -#define swap32_to_le(x) (x) - -#define swap64_to_le(x) (x) - -#define swap16_from_le(x) (x) - -#define swap32_from_le(x) (x) - -#define swap64_from_le(x) (x) - #define ena_assert_msg(cond, msg) \ do { \ if (unlikely(!(cond))) { \ Index: /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/ena_rss.c =================================================================== --- /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/ena_rss.c (nonexistent) +++ /branches/rel_apv_10_7/usr/click/lib/libintel_dpdk/drivers/net/ena/ena_rss.c (working copy) @@ -0,0 +1,536 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright (c) 2020 Amazon.com, Inc. or its affiliates. + * All rights reserved. + */ + +#include "ena_ethdev.h" +#include "ena_logs.h" + +#include + +#define TEST_BIT(val, bit_shift) ((val) & (1UL << (bit_shift))) + +#define ENA_HF_RSS_ALL_L2 (ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA) +#define ENA_HF_RSS_ALL_L3 (ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA) +#define ENA_HF_RSS_ALL_L4 (ENA_ADMIN_RSS_L4_SP | ENA_ADMIN_RSS_L4_DP) +#define ENA_HF_RSS_ALL_L3_L4 (ENA_HF_RSS_ALL_L3 | ENA_HF_RSS_ALL_L4) +#define ENA_HF_RSS_ALL_L2_L3_L4 (ENA_HF_RSS_ALL_L2 | ENA_HF_RSS_ALL_L3_L4) + +enum ena_rss_hash_fields { + ENA_HF_RSS_TCP4 = ENA_HF_RSS_ALL_L3_L4, + ENA_HF_RSS_UDP4 = ENA_HF_RSS_ALL_L3_L4, + ENA_HF_RSS_TCP6 = ENA_HF_RSS_ALL_L3_L4, + ENA_HF_RSS_UDP6 = ENA_HF_RSS_ALL_L3_L4, + ENA_HF_RSS_IP4 = ENA_HF_RSS_ALL_L3, + ENA_HF_RSS_IP6 = ENA_HF_RSS_ALL_L3, + ENA_HF_RSS_IP4_FRAG = ENA_HF_RSS_ALL_L3, + ENA_HF_RSS_NOT_IP = ENA_HF_RSS_ALL_L2, + ENA_HF_RSS_TCP6_EX = ENA_HF_RSS_ALL_L3_L4, + ENA_HF_RSS_IP6_EX = ENA_HF_RSS_ALL_L3, +}; + +static int ena_fill_indirect_table_default(struct ena_com_dev *ena_dev, + size_t tbl_size, + size_t queue_num); +static uint64_t ena_admin_hf_to_eth_hf(enum ena_admin_flow_hash_proto proto, + uint16_t field); +static uint16_t ena_eth_hf_to_admin_hf(enum ena_admin_flow_hash_proto proto); +static int ena_set_hash_fields(struct ena_com_dev *ena_dev, uint64_t rss_hf); +static int ena_rss_hash_set(struct ena_com_dev *ena_dev, + struct rte_eth_rss_conf *rss_conf, + bool default_allowed); +static void ena_reorder_rss_hash_key(uint8_t *reordered_key, + uint8_t *key, + size_t key_size); +static int ena_get_rss_hash_key(struct ena_com_dev *ena_dev, uint8_t *rss_key); + +void ena_rss_key_fill(void *key, size_t size) +{ + static bool key_generated; + static uint8_t default_key[ENA_HASH_KEY_SIZE]; + size_t i; + + RTE_ASSERT(size <= ENA_HASH_KEY_SIZE); + + if (!key_generated) { + for (i = 0; i < ENA_HASH_KEY_SIZE; ++i) + default_key[i] = rte_rand() & 0xff; + key_generated = true; + } + + rte_memcpy(key, default_key, size); +} + +int ena_rss_reta_update(struct rte_eth_dev *dev, + struct rte_eth_rss_reta_entry64 *reta_conf, + uint16_t reta_size) +{ + struct ena_adapter *adapter = dev->data->dev_private; + struct ena_com_dev *ena_dev = &adapter->ena_dev; + int rc, i; + u16 entry_value; + int conf_idx; + int idx; + + if (reta_size == 0 || reta_conf == NULL) + return -EINVAL; + + if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) { + PMD_DRV_LOG(ERR, + "RSS was not configured for the PMD\n"); + return -ENOTSUP; + } + + if (reta_size > ENA_RX_RSS_TABLE_SIZE) { + PMD_DRV_LOG(WARNING, + "Requested indirection table size (%d) is bigger than supported: %d\n", + reta_size, ENA_RX_RSS_TABLE_SIZE); + return -EINVAL; + } + + for (i = 0 ; i < reta_size ; i++) { + /* Each reta_conf is for 64 entries. + * To support 128 we use 2 conf of 64. + */ + conf_idx = i / RTE_RETA_GROUP_SIZE; + idx = i % RTE_RETA_GROUP_SIZE; + if (TEST_BIT(reta_conf[conf_idx].mask, idx)) { + entry_value = + ENA_IO_RXQ_IDX(reta_conf[conf_idx].reta[idx]); + + rc = ena_com_indirect_table_fill_entry(ena_dev, i, + entry_value); + if (unlikely(rc != 0)) { + PMD_DRV_LOG(ERR, + "Cannot fill indirection table\n"); + return rc; + } + } + } + + rte_spinlock_lock(&adapter->admin_lock); + rc = ena_com_indirect_table_set(ena_dev); + rte_spinlock_unlock(&adapter->admin_lock); + if (unlikely(rc != 0)) { + PMD_DRV_LOG(ERR, "Cannot set the indirection table\n"); + return rc; + } + + PMD_DRV_LOG(DEBUG, "RSS configured %d entries for port %d\n", + reta_size, dev->data->port_id); + + return 0; +} + +/* Query redirection table. */ +int ena_rss_reta_query(struct rte_eth_dev *dev, + struct rte_eth_rss_reta_entry64 *reta_conf, + uint16_t reta_size) +{ + uint32_t indirect_table[ENA_RX_RSS_TABLE_SIZE] = { 0 }; + struct ena_adapter *adapter = dev->data->dev_private; + struct ena_com_dev *ena_dev = &adapter->ena_dev; + int rc; + int i; + int reta_conf_idx; + int reta_idx; + + if (reta_size == 0 || reta_conf == NULL || + (reta_size > RTE_RETA_GROUP_SIZE && ((reta_conf + 1) == NULL))) + return -EINVAL; + + if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) { + PMD_DRV_LOG(ERR, + "RSS was not configured for the PMD\n"); + return -ENOTSUP; + } + + rte_spinlock_lock(&adapter->admin_lock); + rc = ena_com_indirect_table_get(ena_dev, indirect_table); + rte_spinlock_unlock(&adapter->admin_lock); + if (unlikely(rc != 0)) { + PMD_DRV_LOG(ERR, "Cannot get indirection table\n"); + return rc; + } + + for (i = 0 ; i < reta_size ; i++) { + reta_conf_idx = i / RTE_RETA_GROUP_SIZE; + reta_idx = i % RTE_RETA_GROUP_SIZE; + if (TEST_BIT(reta_conf[reta_conf_idx].mask, reta_idx)) + reta_conf[reta_conf_idx].reta[reta_idx] = + ENA_IO_RXQ_IDX_REV(indirect_table[i]); + } + + return 0; +} + +static int ena_fill_indirect_table_default(struct ena_com_dev *ena_dev, + size_t tbl_size, + size_t queue_num) +{ + size_t i; + int rc; + uint16_t val; + + for (i = 0; i < tbl_size; ++i) { + val = i % queue_num; + rc = ena_com_indirect_table_fill_entry(ena_dev, i, + ENA_IO_RXQ_IDX(val)); + if (unlikely(rc != 0)) { + PMD_DRV_LOG(DEBUG, + "Failed to set %zu indirection table entry with val %" PRIu16 "\n", + i, val); + return rc; + } + } + + return 0; +} + +static uint64_t ena_admin_hf_to_eth_hf(enum ena_admin_flow_hash_proto proto, + uint16_t fields) +{ + uint64_t rss_hf = 0; + + /* If no fields are activated, then RSS is disabled for this proto */ + if ((fields & ENA_HF_RSS_ALL_L2_L3_L4) == 0) + return 0; + + /* Convert proto to ETH flag */ + switch (proto) { + case ENA_ADMIN_RSS_TCP4: + rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP; + break; + case ENA_ADMIN_RSS_UDP4: + rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP; + break; + case ENA_ADMIN_RSS_TCP6: + rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP; + break; + case ENA_ADMIN_RSS_UDP6: + rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP; + break; + case ENA_ADMIN_RSS_IP4: + rss_hf |= ETH_RSS_IPV4; + break; + case ENA_ADMIN_RSS_IP6: + rss_hf |= ETH_RSS_IPV6; + break; + case ENA_ADMIN_RSS_IP4_FRAG: + rss_hf |= ETH_RSS_FRAG_IPV4; + break; + case ENA_ADMIN_RSS_NOT_IP: + rss_hf |= ETH_RSS_L2_PAYLOAD; + break; + case ENA_ADMIN_RSS_TCP6_EX: + rss_hf |= ETH_RSS_IPV6_TCP_EX; + break; + case ENA_ADMIN_RSS_IP6_EX: + rss_hf |= ETH_RSS_IPV6_EX; + break; + default: + break; + }; + + return rss_hf; +} + +static uint16_t ena_eth_hf_to_admin_hf(enum ena_admin_flow_hash_proto proto) +{ + uint16_t fields_mask = 0; + + /* L2 always uses source and destination addresses. */ + fields_mask = ENA_ADMIN_RSS_L2_DA | + ENA_ADMIN_RSS_L2_SA | + ENA_HF_RSS_ALL_L3 | + ENA_HF_RSS_ALL_L4; + + /* Return appropriate hash fields. */ + switch (proto) { + case ENA_ADMIN_RSS_TCP4: + return ENA_HF_RSS_TCP4 & fields_mask; + case ENA_ADMIN_RSS_UDP4: + return ENA_HF_RSS_UDP4 & fields_mask; + case ENA_ADMIN_RSS_TCP6: + return ENA_HF_RSS_TCP6 & fields_mask; + case ENA_ADMIN_RSS_UDP6: + return ENA_HF_RSS_UDP6 & fields_mask; + case ENA_ADMIN_RSS_IP4: + return ENA_HF_RSS_IP4 & fields_mask; + case ENA_ADMIN_RSS_IP6: + return ENA_HF_RSS_IP6 & fields_mask; + case ENA_ADMIN_RSS_IP4_FRAG: + return ENA_HF_RSS_IP4_FRAG & fields_mask; + case ENA_ADMIN_RSS_NOT_IP: + return ENA_HF_RSS_NOT_IP & fields_mask; + case ENA_ADMIN_RSS_TCP6_EX: + return ENA_HF_RSS_TCP6_EX & fields_mask; + case ENA_ADMIN_RSS_IP6_EX: + return ENA_HF_RSS_IP6_EX & fields_mask; + default: + break; + } + + return 0; +} + +static int ena_set_hash_fields(struct ena_com_dev *ena_dev, uint64_t rss_hf) +{ + struct ena_admin_proto_input selected_fields[ENA_ADMIN_RSS_PROTO_NUM] = {}; + int rc, i; + + /* Turn on appropriate fields for each requested packet type */ + if ((rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) != 0) + selected_fields[ENA_ADMIN_RSS_TCP4].fields = + ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_TCP4); + + if ((rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) != 0) + selected_fields[ENA_ADMIN_RSS_UDP4].fields = + ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_UDP4); + + if ((rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) != 0) + selected_fields[ENA_ADMIN_RSS_TCP6].fields = + ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_TCP6); + + if ((rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) != 0) + selected_fields[ENA_ADMIN_RSS_UDP6].fields = + ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_UDP6); + + if ((rss_hf & ETH_RSS_IPV4) != 0) + selected_fields[ENA_ADMIN_RSS_IP4].fields = + ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_IP4); + + if ((rss_hf & ETH_RSS_IPV6) != 0) + selected_fields[ENA_ADMIN_RSS_IP6].fields = + ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_IP6); + + if ((rss_hf & ETH_RSS_FRAG_IPV4) != 0) + selected_fields[ENA_ADMIN_RSS_IP4_FRAG].fields = + ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_IP4_FRAG); + + if ((rss_hf & ETH_RSS_L2_PAYLOAD) != 0) + selected_fields[ENA_ADMIN_RSS_NOT_IP].fields = + ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_NOT_IP); + + if ((rss_hf & ETH_RSS_IPV6_TCP_EX) != 0) + selected_fields[ENA_ADMIN_RSS_TCP6_EX].fields = + ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_TCP6_EX); + + if ((rss_hf & ETH_RSS_IPV6_EX) != 0) + selected_fields[ENA_ADMIN_RSS_IP6_EX].fields = + ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_IP6_EX); + + /* Try to write them to the device */ + for (i = 0; i < ENA_ADMIN_RSS_PROTO_NUM; i++) { + rc = ena_com_fill_hash_ctrl(ena_dev, + (enum ena_admin_flow_hash_proto)i, + selected_fields[i].fields); + if (unlikely(rc != 0)) { + PMD_DRV_LOG(DEBUG, + "Failed to set ENA HF %d with fields %" PRIu16 "\n", + i, selected_fields[i].fields); + return rc; + } + } + + return 0; +} + +static int ena_rss_hash_set(struct ena_com_dev *ena_dev, + struct rte_eth_rss_conf *rss_conf, + bool default_allowed) +{ + uint8_t hw_rss_key[ENA_HASH_KEY_SIZE]; + uint8_t *rss_key; + int rc; + + if (rss_conf->rss_key != NULL) { + /* Reorder the RSS key bytes for the hardware requirements. */ + ena_reorder_rss_hash_key(hw_rss_key, rss_conf->rss_key, + ENA_HASH_KEY_SIZE); + rss_key = hw_rss_key; + } else { + rss_key = NULL; + } + + /* If the rss_key is NULL, then the randomized key will be used. */ + rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_TOEPLITZ, + rss_key, ENA_HASH_KEY_SIZE, 0); + if (rc != 0 && !(default_allowed && rc == ENA_COM_UNSUPPORTED)) { + PMD_DRV_LOG(ERR, + "Failed to set RSS hash function in the device\n"); + return rc; + } + + rc = ena_set_hash_fields(ena_dev, rss_conf->rss_hf); + if (rc == ENA_COM_UNSUPPORTED) { + if (rss_conf->rss_key == NULL && !default_allowed) { + PMD_DRV_LOG(ERR, + "Setting RSS hash fields is not supported\n"); + return -ENOTSUP; + } + PMD_DRV_LOG(WARNING, + "Setting RSS hash fields is not supported. Using default values: 0x%" PRIx64 "\n", + (uint64_t)(ENA_ALL_RSS_HF)); + } else if (rc != 0) { + PMD_DRV_LOG(ERR, "Failed to set RSS hash fields\n"); + return rc; + } + + return 0; +} + +/* ENA HW interprets the RSS key in reverse bytes order. Because of that, the + * key must be processed upon interaction with ena_com layer. + */ +static void ena_reorder_rss_hash_key(uint8_t *reordered_key, + uint8_t *key, + size_t key_size) +{ + size_t i, rev_i; + + for (i = 0, rev_i = key_size - 1; i < key_size; ++i, --rev_i) + reordered_key[i] = key[rev_i]; +} + +static int ena_get_rss_hash_key(struct ena_com_dev *ena_dev, uint8_t *rss_key) +{ + uint8_t hw_rss_key[ENA_HASH_KEY_SIZE]; + int rc; + + /* The default RSS hash key cannot be retrieved from the HW. Unless it's + * explicitly set, this operation shouldn't be supported. + */ + if (ena_dev->rss.hash_key == NULL) { + PMD_DRV_LOG(WARNING, + "Retrieving default RSS hash key is not supported\n"); + return -ENOTSUP; + } + + rc = ena_com_get_hash_key(ena_dev, hw_rss_key); + if (rc != 0) + return rc; + + ena_reorder_rss_hash_key(rss_key, hw_rss_key, ENA_HASH_KEY_SIZE); + + return 0; +} + +int ena_rss_configure(struct ena_adapter *adapter) +{ + struct rte_eth_rss_conf *rss_conf; + struct ena_com_dev *ena_dev; + int rc; + + ena_dev = &adapter->ena_dev; + rss_conf = &adapter->edev_data->dev_conf.rx_adv_conf.rss_conf; + + if (adapter->edev_data->nb_rx_queues == 0) + return 0; + + /* Restart the indirection table. The number of queues could change + * between start/stop calls, so it must be reinitialized with default + * values. + */ + rc = ena_fill_indirect_table_default(ena_dev, ENA_RX_RSS_TABLE_SIZE, + adapter->edev_data->nb_rx_queues); + if (unlikely(rc != 0)) { + PMD_DRV_LOG(ERR, + "Failed to fill indirection table with default values\n"); + return rc; + } + + rc = ena_com_indirect_table_set(ena_dev); + if (unlikely(rc != 0 && rc != ENA_COM_UNSUPPORTED)) { + PMD_DRV_LOG(ERR, + "Failed to set indirection table in the device\n"); + return rc; + } + + rc = ena_rss_hash_set(ena_dev, rss_conf, true); + if (unlikely(rc != 0)) { + PMD_DRV_LOG(ERR, "Failed to set RSS hash\n"); + return rc; + } + + PMD_DRV_LOG(DEBUG, "RSS configured for port %d\n", + adapter->edev_data->port_id); + + return 0; +} + +int ena_rss_hash_update(struct rte_eth_dev *dev, + struct rte_eth_rss_conf *rss_conf) +{ + struct ena_adapter *adapter = dev->data->dev_private; + int rc; + + rte_spinlock_lock(&adapter->admin_lock); + rc = ena_rss_hash_set(&adapter->ena_dev, rss_conf, false); + rte_spinlock_unlock(&adapter->admin_lock); + if (unlikely(rc != 0)) { + PMD_DRV_LOG(ERR, "Failed to set RSS hash\n"); + return rc; + } + + return 0; +} + +int ena_rss_hash_conf_get(struct rte_eth_dev *dev, + struct rte_eth_rss_conf *rss_conf) +{ + struct ena_adapter *adapter = dev->data->dev_private; + struct ena_com_dev *ena_dev = &adapter->ena_dev; + enum ena_admin_flow_hash_proto proto; + uint64_t rss_hf = 0; + int rc, i; + uint16_t admin_hf; + static bool warn_once; + + if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)) { + PMD_DRV_LOG(ERR, "RSS was not configured for the PMD\n"); + return -ENOTSUP; + } + + if (rss_conf->rss_key != NULL) { + rc = ena_get_rss_hash_key(ena_dev, rss_conf->rss_key); + if (unlikely(rc != 0)) { + PMD_DRV_LOG(ERR, + "Cannot retrieve RSS hash key, err: %d\n", + rc); + return rc; + } + } + + for (i = 0; i < ENA_ADMIN_RSS_PROTO_NUM; ++i) { + proto = (enum ena_admin_flow_hash_proto)i; + rte_spinlock_lock(&adapter->admin_lock); + rc = ena_com_get_hash_ctrl(ena_dev, proto, &admin_hf); + rte_spinlock_unlock(&adapter->admin_lock); + if (rc == ENA_COM_UNSUPPORTED) { + /* As some devices may support only reading rss hash + * key and not the hash ctrl, we want to notify the + * caller that this feature is only partially supported + * and do not return an error - the caller could be + * interested only in the key value. + */ + if (!warn_once) { + PMD_DRV_LOG(WARNING, + "Reading hash control from the device is not supported. .rss_hf will contain a default value.\n"); + warn_once = true; + } + rss_hf = ENA_ALL_RSS_HF; + break; + } else if (rc != 0) { + PMD_DRV_LOG(ERR, + "Failed to retrieve hash ctrl for proto: %d with err: %d\n", + i, rc); + return rc; + } + + rss_hf |= ena_admin_hf_to_eth_hf(proto, admin_hf); + } + + rss_conf->rss_hf = rss_hf; + return 0; +}