admin/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2025-12-07 20:06:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

net/mlx5: Move the vhca event notifier outside of the devlink lock

Browse Source 
The vhca event notifier consists of an atomic notifier for vhca state
changes (used for SF events), multiple workqueues and a blocking
notifier chain for delivering the vhca state change events for further
processing.

This patch moves the vhca notifier head outside of mlx5_init_one() /
mlx5_uninit_one() and into the mlx5_mdev_init() / mlx5_mdev_uninit()
functions.

This allows called notifiers to grab the PF devlink lock which was
previously impossible because it would create a circular lock
dependency.

mlx5_vhca_event_stop() is now called earlier in the cleanup phase and
flushes the workqueues to ensure that after the call, there are no
pending events. This simplifies the cleanup flow for vhca event
consumers.

Signed-off-by: Cosmin Ratiu <cratiu@nvidia.com>
Reviewed-by: Carolina Jubran <cjubran@nvidia.com>
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
Link: https://patch.msgid.link/1763325940-1231508-4-git-send-email-tariqt@nvidia.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Cosmin Ratiu
2025-11-16 22:45:37 +02:00
committed by Jakub Kicinski
parent 3fee828789
commit d3a356db85
6 changed files with 35 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
drivers/net/ethernet/mellanox/mlx5/core/main.c
View File

@@ -1438,12 +1438,12 @@ static void mlx5_unload(struct mlx5_core_dev *dev)
{
mlx5_eswitch_disable(dev->priv.eswitch);
mlx5_devlink_traps_unregister(priv_to_devlink(dev));
mlx5_vhca_event_stop(dev);
mlx5_sf_dev_table_destroy(dev);
mlx5_sriov_detach(dev);
mlx5_lag_remove_mdev(dev);
mlx5_ec_cleanup(dev);
mlx5_sf_hw_table_destroy(dev);
mlx5_vhca_event_stop(dev);
mlx5_fs_core_cleanup(dev);
mlx5_fpga_device_stop(dev);
mlx5_rsc_dump_cleanup(dev);
@@ -1835,6 +1835,7 @@ static int mlx5_notifiers_init(struct mlx5_core_dev *dev)
}
BLOCKING_INIT_NOTIFIER_HEAD(&dev->priv.esw_n_head);
mlx5_vhca_state_notifier_init(dev);
return 0;
}

1
drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.c
View File

@@ -381,7 +381,6 @@ void mlx5_sf_dev_table_destroy(struct mlx5_core_dev *dev)
mlx5_sf_dev_destroy_active_works(table);
mlx5_vhca_event_notifier_unregister(dev, &table->nb);
mlx5_vhca_event_work_queues_flush(dev);
/* Now that event handler is not running, it is safe to destroy
* the sf device without race.

1
drivers/net/ethernet/mellanox/mlx5/core/sf/hw_table.c
View File

@@ -389,6 +389,7 @@ void mlx5_sf_hw_table_destroy(struct mlx5_core_dev *dev)
return;
mlx5_vhca_event_notifier_unregister(dev, &table->vhca_nb);
/* Dealloc SFs whose firmware event has been missed. */
mlx5_sf_hw_table_dealloc_all(table);
}

69
drivers/net/ethernet/mellanox/mlx5/core/sf/vhca_event.c
View File

@@ -9,15 +9,9 @@
#define CREATE_TRACE_POINTS
#include "diag/vhca_tracepoint.h"
struct mlx5_vhca_state_notifier {
struct mlx5_core_dev *dev;
struct mlx5_nb nb;
struct blocking_notifier_head n_head;
};
struct mlx5_vhca_event_work {
struct work_struct work;
struct mlx5_vhca_state_notifier *notifier;
struct mlx5_core_dev *dev;
struct mlx5_vhca_state_event event;
};
@@ -95,16 +89,14 @@ mlx5_vhca_event_notify(struct mlx5_core_dev *dev, struct mlx5_vhca_state_event *
mlx5_vhca_event_arm(dev, event->function_id);
trace_mlx5_sf_vhca_event(dev, event);
blocking_notifier_call_chain(&dev->priv.vhca_state_notifier->n_head, 0, event);
blocking_notifier_call_chain(&dev->priv.vhca_state_n_head, 0, event);
}
static void mlx5_vhca_state_work_handler(struct work_struct *_work)
{
struct mlx5_vhca_event_work *work = container_of(_work, struct mlx5_vhca_event_work, work);
struct mlx5_vhca_state_notifier *notifier = work->notifier;
struct mlx5_core_dev *dev = notifier->dev;
mlx5_vhca_event_notify(dev, &work->event);
mlx5_vhca_event_notify(work->dev, &work->event);
kfree(work);
}
@@ -116,8 +108,8 @@ void mlx5_vhca_events_work_enqueue(struct mlx5_core_dev *dev, int idx, struct wo
static int
mlx5_vhca_state_change_notifier(struct notifier_block *nb, unsigned long type, void *data)
{
struct mlx5_vhca_state_notifier *notifier =
mlx5_nb_cof(nb, struct mlx5_vhca_state_notifier, nb);
struct mlx5_core_dev *dev = mlx5_nb_cof(nb, struct mlx5_core_dev,
priv.vhca_state_nb);
struct mlx5_vhca_event_work *work;
struct mlx5_eqe *eqe = data;
int wq_idx;
@@ -126,10 +118,10 @@ mlx5_vhca_state_change_notifier(struct notifier_block *nb, unsigned long type, v
if (!work)
return NOTIFY_DONE;
INIT_WORK(&work->work, &mlx5_vhca_state_work_handler);
work->notifier = notifier;
work->dev = dev;
work->event.function_id = be16_to_cpu(eqe->data.vhca_state.function_id);
wq_idx = work->event.function_id % MLX5_DEV_MAX_WQS;
mlx5_vhca_events_work_enqueue(notifier->dev, wq_idx, &work->work);
mlx5_vhca_events_work_enqueue(dev, wq_idx, &work->work);
return NOTIFY_OK;
}
@@ -145,9 +137,15 @@ void mlx5_vhca_state_cap_handle(struct mlx5_core_dev *dev, void *set_hca_cap)
MLX5_SET(cmd_hca_cap, set_hca_cap, event_on_vhca_state_teardown_request, 1);
}
void mlx5_vhca_state_notifier_init(struct mlx5_core_dev *dev)
{
BLOCKING_INIT_NOTIFIER_HEAD(&dev->priv.vhca_state_n_head);
MLX5_NB_INIT(&dev->priv.vhca_state_nb, mlx5_vhca_state_change_notifier,
VHCA_STATE_CHANGE);
}
int mlx5_vhca_event_init(struct mlx5_core_dev *dev)
{
struct mlx5_vhca_state_notifier *notifier;
char wq_name[MLX5_CMD_WQ_MAX_NAME];
struct mlx5_vhca_events *events;
int err, i;
@@ -160,7 +158,6 @@ int mlx5_vhca_event_init(struct mlx5_core_dev *dev)
return -ENOMEM;
events->dev = dev;
dev->priv.vhca_events = events;
for (i = 0; i < MLX5_DEV_MAX_WQS; i++) {
snprintf(wq_name, MLX5_CMD_WQ_MAX_NAME, "mlx5_vhca_event%d", i);
events->handler[i].wq = create_singlethread_workqueue(wq_name);
@@ -169,20 +166,10 @@ int mlx5_vhca_event_init(struct mlx5_core_dev *dev)
goto err_create_wq;
}
}
dev->priv.vhca_events = events;
notifier = kzalloc(sizeof(*notifier), GFP_KERNEL);
if (!notifier) {
err = -ENOMEM;
goto err_notifier;
}
dev->priv.vhca_state_notifier = notifier;
notifier->dev = dev;
BLOCKING_INIT_NOTIFIER_HEAD(&notifier->n_head);
MLX5_NB_INIT(&notifier->nb, mlx5_vhca_state_change_notifier, VHCA_STATE_CHANGE);
return 0;
err_notifier:
err_create_wq:
for (--i; i >= 0; i--)
destroy_workqueue(events->handler[i].wq);
@@ -211,8 +198,6 @@ void mlx5_vhca_event_cleanup(struct mlx5_core_dev *dev)
if (!mlx5_vhca_event_supported(dev))
return;
kfree(dev->priv.vhca_state_notifier);
dev->priv.vhca_state_notifier = NULL;
vhca_events = dev->priv.vhca_events;
for (i = 0; i < MLX5_DEV_MAX_WQS; i++)
destroy_workqueue(vhca_events->handler[i].wq);
@@ -221,34 +206,30 @@ void mlx5_vhca_event_cleanup(struct mlx5_core_dev *dev)
void mlx5_vhca_event_start(struct mlx5_core_dev *dev)
{
struct mlx5_vhca_state_notifier *notifier;
if (!dev->priv.vhca_state_notifier)
if (!mlx5_vhca_event_supported(dev))
return;
notifier = dev->priv.vhca_state_notifier;
mlx5_eq_notifier_register(dev, &notifier->nb);
mlx5_eq_notifier_register(dev, &dev->priv.vhca_state_nb);
}
void mlx5_vhca_event_stop(struct mlx5_core_dev *dev)
{
struct mlx5_vhca_state_notifier *notifier;
if (!dev->priv.vhca_state_notifier)
if (!mlx5_vhca_event_supported(dev))
return;
notifier = dev->priv.vhca_state_notifier;
mlx5_eq_notifier_unregister(dev, &notifier->nb);
mlx5_eq_notifier_unregister(dev, &dev->priv.vhca_state_nb);
/* Flush workqueues of all pending events. */
mlx5_vhca_event_work_queues_flush(dev);
}
int mlx5_vhca_event_notifier_register(struct mlx5_core_dev *dev, struct notifier_block *nb)
{
if (!dev->priv.vhca_state_notifier)
return -EOPNOTSUPP;
return blocking_notifier_chain_register(&dev->priv.vhca_state_notifier->n_head, nb);
return blocking_notifier_chain_register(&dev->priv.vhca_state_n_head,
nb);
}
void mlx5_vhca_event_notifier_unregister(struct mlx5_core_dev *dev, struct notifier_block *nb)
{
blocking_notifier_chain_unregister(&dev->priv.vhca_state_notifier->n_head, nb);
blocking_notifier_chain_unregister(&dev->priv.vhca_state_n_head, nb);
}

5
drivers/net/ethernet/mellanox/mlx5/core/sf/vhca_event.h
View File

@@ -18,6 +18,7 @@ static inline bool mlx5_vhca_event_supported(const struct mlx5_core_dev *dev)
}
void mlx5_vhca_state_cap_handle(struct mlx5_core_dev *dev, void *set_hca_cap);
void mlx5_vhca_state_notifier_init(struct mlx5_core_dev *dev);
int mlx5_vhca_event_init(struct mlx5_core_dev *dev);
void mlx5_vhca_event_cleanup(struct mlx5_core_dev *dev);
void mlx5_vhca_event_start(struct mlx5_core_dev *dev);
@@ -37,6 +38,10 @@ static inline void mlx5_vhca_state_cap_handle(struct mlx5_core_dev *dev, void *s
{
}
static inline void mlx5_vhca_state_notifier_init(struct mlx5_core_dev *dev)
{
}
static inline int mlx5_vhca_event_init(struct mlx5_core_dev *dev)
{
return 0;

4
include/linux/mlx5/driver.h
View File

@@ -488,7 +488,6 @@ struct mlx5_devcom_dev;
struct mlx5_fw_reset;
struct mlx5_eq_table;
struct mlx5_irq_table;
struct mlx5_vhca_state_notifier;
struct mlx5_sf_dev_table;
struct mlx5_sf_hw_table;
struct mlx5_sf_table;
@@ -615,7 +614,8 @@ struct mlx5_priv {
struct mlx5_bfreg_data bfregs;
struct mlx5_sq_bfreg bfreg;
#ifdef CONFIG_MLX5_SF
struct mlx5_vhca_state_notifier *vhca_state_notifier;
struct mlx5_nb vhca_state_nb;
struct blocking_notifier_head vhca_state_n_head;
struct mlx5_sf_dev_table *sf_dev_table;
struct mlx5_core_dev *parent_mdev;
#endif