devlink reload destroys resources and allocates resources again.
So, when devices and ports resources are being used, devlink reload
function should not be executed. In order to avoid this race, a new
lock is added and new_port() and del_port() call devlink_reload_disable()
and devlink_reload_enable().
Thread0 Thread1
{new/del}_port() {new/del}_port()
devlink_reload_disable()
devlink_reload_disable()
devlink_reload_enable()
//here
devlink_reload_enable()
Before Thread1's devlink_reload_enable(), the devlink is already allowed
to execute reload because Thread0 allows it. devlink reload disable/enable
variable type is bool. So the above case would exist.
So, disable/enable should be executed atomically.
In order to do that, a new lock is used.
Test commands:
modprobe netdevsim
echo 1 > /sys/bus/netdevsim/new_device
while :
do
echo 1 > /sys/devices/netdevsim1/new_port &
echo 1 > /sys/devices/netdevsim1/del_port &
devlink dev reload netdevsim/netdevsim1 &
done
Splat looks like:
[ 23.342145][ T932] DEBUG_LOCKS_WARN_ON(mutex_is_locked(lock))
[ 23.342159][ T932] WARNING: CPU: 0 PID: 932 at kernel/locking/mutex-debug.c:103 mutex_destroy+0xc7/0xf0
[ 23.344182][ T932] Modules linked in: netdevsim openvswitch nsh nf_conncount nf_nat nf_conntrack nf_defrag_ipv6 nf_dx
[ 23.346485][ T932] CPU: 0 PID: 932 Comm: devlink Not tainted 5.5.0+ #322
[ 23.347696][ T932] Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
[ 23.348893][ T932] RIP: 0010:mutex_destroy+0xc7/0xf0
[ 23.349505][ T932] Code: e0 07 83 c0 03 38 d0 7c 04 84 d2 75 2e 8b 05 00 ac b0 02 85 c0 75 8b 48 c7 c6 00 5e 07 96 40
[ 23.351887][ T932] RSP: 0018:ffff88806208f810 EFLAGS: 00010286
[ 23.353963][ T932] RAX: dffffc0000000008 RBX: ffff888067f6f2c0 RCX: ffffffff942c4bd4
[ 23.355222][ T932] RDX: 0000000000000000 RSI: 0000000000000000 RDI: ffffffff96dac5b4
[ 23.356169][ T932] RBP: ffff888067f6f000 R08: fffffbfff2d235a5 R09: fffffbfff2d235a5
[ 23.357160][ T932] R10: 0000000000000001 R11: fffffbfff2d235a4 R12: ffff888067f6f208
[ 23.358288][ T932] R13: ffff88806208fa70 R14: ffff888067f6f000 R15: ffff888069ce3800
[ 23.359307][ T932] FS: 00007fe2a3876740(0000) GS:ffff88806c000000(0000) knlGS:0000000000000000
[ 23.360473][ T932] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 23.361319][ T932] CR2: 00005561357aa000 CR3: 000000005227a006 CR4: 00000000000606f0
[ 23.362323][ T932] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 23.363417][ T932] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 23.364414][ T932] Call Trace:
[ 23.364828][ T932] nsim_dev_reload_destroy+0x77/0xb0 [netdevsim]
[ 23.365655][ T932] nsim_dev_reload_down+0x84/0xb0 [netdevsim]
[ 23.366433][ T932] devlink_reload+0xb1/0x350
[ 23.367010][ T932] genl_rcv_msg+0x580/0xe90
[ ...]
[ 23.531729][ T1305] kernel BUG at lib/list_debug.c:53!
[ 23.532523][ T1305] invalid opcode: 0000 [#1] SMP DEBUG_PAGEALLOC KASAN PTI
[ 23.533467][ T1305] CPU: 2 PID: 1305 Comm: bash Tainted: G W 5.5.0+ #322
[ 23.534962][ T1305] Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006
[ 23.536503][ T1305] RIP: 0010:__list_del_entry_valid+0xe6/0x150
[ 23.538346][ T1305] Code: 89 ea 48 c7 c7 00 73 1e 96 e8 df f7 4c ff 0f 0b 48 c7 c7 60 73 1e 96 e8 d1 f7 4c ff 0f 0b 44
[ 23.541068][ T1305] RSP: 0018:ffff888047c27b58 EFLAGS: 00010282
[ 23.542001][ T1305] RAX: 0000000000000054 RBX: ffff888067f6f318 RCX: 0000000000000000
[ 23.543051][ T1305] RDX: 0000000000000054 RSI: 0000000000000008 RDI: ffffed1008f84f61
[ 23.544072][ T1305] RBP: ffff88804aa0fca0 R08: ffffed100d940539 R09: ffffed100d940539
[ 23.545085][ T1305] R10: 0000000000000001 R11: ffffed100d940538 R12: ffff888047c27cb0
[ 23.546422][ T1305] R13: ffff88806208b840 R14: ffffffff981976c0 R15: ffff888067f6f2c0
[ 23.547406][ T1305] FS: 00007f76c0431740(0000) GS:ffff88806c800000(0000) knlGS:0000000000000000
[ 23.548527][ T1305] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 23.549389][ T1305] CR2: 00007f5048f1a2f8 CR3: 000000004b310006 CR4: 00000000000606e0
[ 23.550636][ T1305] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 23.551578][ T1305] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 23.552597][ T1305] Call Trace:
[ 23.553004][ T1305] mutex_remove_waiter+0x101/0x520
[ 23.553646][ T1305] __mutex_lock+0xac7/0x14b0
[ 23.554218][ T1305] ? nsim_dev_port_del+0x4e/0x140 [netdevsim]
[ 23.554908][ T1305] ? mutex_lock_io_nested+0x1380/0x1380
[ 23.555570][ T1305] ? _parse_integer+0xf0/0xf0
[ 23.556043][ T1305] ? kstrtouint+0x86/0x110
[ 23.556504][ T1305] ? nsim_dev_port_del+0x4e/0x140 [netdevsim]
[ 23.557133][ T1305] nsim_dev_port_del+0x4e/0x140 [netdevsim]
[ 23.558024][ T1305] del_port_store+0xcc/0xf0 [netdevsim]
[ ... ]
Fixes: 75ba029f3c ("netdevsim: implement proper devlink reload")
Signed-off-by: Taehee Yoo <ap420073@gmail.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
250 lines
6.1 KiB
C
250 lines
6.1 KiB
C
/*
|
|
* Copyright (C) 2017 Netronome Systems, Inc.
|
|
*
|
|
* This software is licensed under the GNU General License Version 2,
|
|
* June 1991 as shown in the file COPYING in the top-level directory of this
|
|
* source tree.
|
|
*
|
|
* THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS"
|
|
* WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
|
|
* BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
|
|
* FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE
|
|
* OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME
|
|
* THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
|
*/
|
|
|
|
#include <linux/device.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/list.h>
|
|
#include <linux/netdevice.h>
|
|
#include <linux/u64_stats_sync.h>
|
|
#include <net/devlink.h>
|
|
#include <net/xdp.h>
|
|
|
|
#define DRV_NAME "netdevsim"
|
|
|
|
#define NSIM_XDP_MAX_MTU 4000
|
|
|
|
#define NSIM_EA(extack, msg) NL_SET_ERR_MSG_MOD((extack), msg)
|
|
|
|
#define NSIM_IPSEC_MAX_SA_COUNT 33
|
|
#define NSIM_IPSEC_VALID BIT(31)
|
|
|
|
struct nsim_sa {
|
|
struct xfrm_state *xs;
|
|
__be32 ipaddr[4];
|
|
u32 key[4];
|
|
u32 salt;
|
|
bool used;
|
|
bool crypt;
|
|
bool rx;
|
|
};
|
|
|
|
struct nsim_ipsec {
|
|
struct nsim_sa sa[NSIM_IPSEC_MAX_SA_COUNT];
|
|
struct dentry *pfile;
|
|
u32 count;
|
|
u32 tx;
|
|
u32 ok;
|
|
};
|
|
|
|
struct netdevsim {
|
|
struct net_device *netdev;
|
|
struct nsim_dev *nsim_dev;
|
|
struct nsim_dev_port *nsim_dev_port;
|
|
|
|
u64 tx_packets;
|
|
u64 tx_bytes;
|
|
struct u64_stats_sync syncp;
|
|
|
|
struct nsim_bus_dev *nsim_bus_dev;
|
|
|
|
struct bpf_prog *bpf_offloaded;
|
|
u32 bpf_offloaded_id;
|
|
|
|
struct xdp_attachment_info xdp;
|
|
struct xdp_attachment_info xdp_hw;
|
|
|
|
bool bpf_tc_accept;
|
|
bool bpf_tc_non_bound_accept;
|
|
bool bpf_xdpdrv_accept;
|
|
bool bpf_xdpoffload_accept;
|
|
|
|
bool bpf_map_accept;
|
|
struct nsim_ipsec ipsec;
|
|
};
|
|
|
|
struct netdevsim *
|
|
nsim_create(struct nsim_dev *nsim_dev, struct nsim_dev_port *nsim_dev_port);
|
|
void nsim_destroy(struct netdevsim *ns);
|
|
|
|
#ifdef CONFIG_BPF_SYSCALL
|
|
int nsim_bpf_dev_init(struct nsim_dev *nsim_dev);
|
|
void nsim_bpf_dev_exit(struct nsim_dev *nsim_dev);
|
|
int nsim_bpf_init(struct netdevsim *ns);
|
|
void nsim_bpf_uninit(struct netdevsim *ns);
|
|
int nsim_bpf(struct net_device *dev, struct netdev_bpf *bpf);
|
|
int nsim_bpf_disable_tc(struct netdevsim *ns);
|
|
int nsim_bpf_setup_tc_block_cb(enum tc_setup_type type,
|
|
void *type_data, void *cb_priv);
|
|
#else
|
|
|
|
static inline int nsim_bpf_dev_init(struct nsim_dev *nsim_dev)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline void nsim_bpf_dev_exit(struct nsim_dev *nsim_dev)
|
|
{
|
|
}
|
|
static inline int nsim_bpf_init(struct netdevsim *ns)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline void nsim_bpf_uninit(struct netdevsim *ns)
|
|
{
|
|
}
|
|
|
|
static inline int nsim_bpf(struct net_device *dev, struct netdev_bpf *bpf)
|
|
{
|
|
return bpf->command == XDP_QUERY_PROG ? 0 : -EOPNOTSUPP;
|
|
}
|
|
|
|
static inline int nsim_bpf_disable_tc(struct netdevsim *ns)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline int
|
|
nsim_bpf_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
|
|
void *cb_priv)
|
|
{
|
|
return -EOPNOTSUPP;
|
|
}
|
|
#endif
|
|
|
|
enum nsim_resource_id {
|
|
NSIM_RESOURCE_NONE, /* DEVLINK_RESOURCE_ID_PARENT_TOP */
|
|
NSIM_RESOURCE_IPV4,
|
|
NSIM_RESOURCE_IPV4_FIB,
|
|
NSIM_RESOURCE_IPV4_FIB_RULES,
|
|
NSIM_RESOURCE_IPV6,
|
|
NSIM_RESOURCE_IPV6_FIB,
|
|
NSIM_RESOURCE_IPV6_FIB_RULES,
|
|
};
|
|
|
|
struct nsim_dev_health {
|
|
struct devlink_health_reporter *empty_reporter;
|
|
struct devlink_health_reporter *dummy_reporter;
|
|
struct dentry *ddir;
|
|
char *recovered_break_msg;
|
|
u32 binary_len;
|
|
bool fail_recover;
|
|
};
|
|
|
|
int nsim_dev_health_init(struct nsim_dev *nsim_dev, struct devlink *devlink);
|
|
void nsim_dev_health_exit(struct nsim_dev *nsim_dev);
|
|
|
|
struct nsim_dev_port {
|
|
struct list_head list;
|
|
struct devlink_port devlink_port;
|
|
unsigned int port_index;
|
|
struct dentry *ddir;
|
|
struct netdevsim *ns;
|
|
};
|
|
|
|
struct nsim_dev {
|
|
struct nsim_bus_dev *nsim_bus_dev;
|
|
struct nsim_fib_data *fib_data;
|
|
struct nsim_trap_data *trap_data;
|
|
struct dentry *ddir;
|
|
struct dentry *ports_ddir;
|
|
struct bpf_offload_dev *bpf_dev;
|
|
bool bpf_bind_accept;
|
|
u32 bpf_bind_verifier_delay;
|
|
struct dentry *ddir_bpf_bound_progs;
|
|
u32 prog_id_gen;
|
|
struct list_head bpf_bound_progs;
|
|
struct list_head bpf_bound_maps;
|
|
struct netdev_phys_item_id switch_id;
|
|
struct list_head port_list;
|
|
struct mutex port_list_lock; /* protects port list */
|
|
bool fw_update_status;
|
|
u32 max_macs;
|
|
bool test1;
|
|
bool dont_allow_reload;
|
|
bool fail_reload;
|
|
struct devlink_region *dummy_region;
|
|
struct nsim_dev_health health;
|
|
};
|
|
|
|
static inline struct net *nsim_dev_net(struct nsim_dev *nsim_dev)
|
|
{
|
|
return devlink_net(priv_to_devlink(nsim_dev));
|
|
}
|
|
|
|
int nsim_dev_init(void);
|
|
void nsim_dev_exit(void);
|
|
int nsim_dev_probe(struct nsim_bus_dev *nsim_bus_dev);
|
|
void nsim_dev_remove(struct nsim_bus_dev *nsim_bus_dev);
|
|
int nsim_dev_port_add(struct nsim_bus_dev *nsim_bus_dev,
|
|
unsigned int port_index);
|
|
int nsim_dev_port_del(struct nsim_bus_dev *nsim_bus_dev,
|
|
unsigned int port_index);
|
|
|
|
struct nsim_fib_data *nsim_fib_create(struct devlink *devlink,
|
|
struct netlink_ext_ack *extack);
|
|
void nsim_fib_destroy(struct devlink *devlink, struct nsim_fib_data *fib_data);
|
|
u64 nsim_fib_get_val(struct nsim_fib_data *fib_data,
|
|
enum nsim_resource_id res_id, bool max);
|
|
|
|
#if IS_ENABLED(CONFIG_XFRM_OFFLOAD)
|
|
void nsim_ipsec_init(struct netdevsim *ns);
|
|
void nsim_ipsec_teardown(struct netdevsim *ns);
|
|
bool nsim_ipsec_tx(struct netdevsim *ns, struct sk_buff *skb);
|
|
#else
|
|
static inline void nsim_ipsec_init(struct netdevsim *ns)
|
|
{
|
|
}
|
|
|
|
static inline void nsim_ipsec_teardown(struct netdevsim *ns)
|
|
{
|
|
}
|
|
|
|
static inline bool nsim_ipsec_tx(struct netdevsim *ns, struct sk_buff *skb)
|
|
{
|
|
return true;
|
|
}
|
|
#endif
|
|
|
|
struct nsim_vf_config {
|
|
int link_state;
|
|
u16 min_tx_rate;
|
|
u16 max_tx_rate;
|
|
u16 vlan;
|
|
__be16 vlan_proto;
|
|
u16 qos;
|
|
u8 vf_mac[ETH_ALEN];
|
|
bool spoofchk_enabled;
|
|
bool trusted;
|
|
bool rss_query_enabled;
|
|
};
|
|
|
|
struct nsim_bus_dev {
|
|
struct device dev;
|
|
struct list_head list;
|
|
unsigned int port_count;
|
|
struct net *initial_net; /* Purpose of this is to carry net pointer
|
|
* during the probe time only.
|
|
*/
|
|
unsigned int num_vfs;
|
|
struct nsim_vf_config *vfconfigs;
|
|
/* Lock for devlink->reload_enabled in netdevsim module */
|
|
struct mutex nsim_bus_reload_lock;
|
|
bool init;
|
|
};
|
|
|
|
int nsim_bus_init(void);
|
|
void nsim_bus_exit(void);
|