1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2/* Copyright (c) 2018 Mellanox Technologies */ 3 4#include <linux/mlx5/vport.h> 5#include <linux/list.h> 6#include "lib/devcom.h" 7#include "mlx5_core.h" 8 9static LIST_HEAD(devcom_dev_list); 10static LIST_HEAD(devcom_comp_list); 11/* protect device list */ 12static DEFINE_MUTEX(dev_list_lock); 13/* protect component list */ 14static DEFINE_MUTEX(comp_list_lock); 15 16#define devcom_for_each_component(iter) \ 17 list_for_each_entry(iter, &devcom_comp_list, comp_list) 18 19struct mlx5_devcom_dev { 20 struct list_head list; 21 struct mlx5_core_dev *dev; 22 struct kref ref; 23}; 24 25struct mlx5_devcom_comp { 26 struct list_head comp_list; 27 enum mlx5_devcom_component id; 28 u64 key; 29 struct list_head comp_dev_list_head; 30 mlx5_devcom_event_handler_t handler; 31 struct kref ref; 32 bool ready; 33 struct rw_semaphore sem; 34 struct lock_class_key lock_key; 35}; 36 37struct mlx5_devcom_comp_dev { 38 struct list_head list; 39 struct mlx5_devcom_comp *comp; 40 struct mlx5_devcom_dev *devc; 41 void __rcu *data; 42}; 43 44static bool devcom_dev_exists(struct mlx5_core_dev *dev) 45{ 46 struct mlx5_devcom_dev *iter; 47 48 list_for_each_entry(iter, &devcom_dev_list, list) 49 if (iter->dev == dev) 50 return true; 51 52 return false; 53} 54 55static struct mlx5_devcom_dev * 56mlx5_devcom_dev_alloc(struct mlx5_core_dev *dev) 57{ 58 struct mlx5_devcom_dev *devc; 59 60 devc = kzalloc(sizeof(*devc), GFP_KERNEL); 61 if (!devc) 62 return NULL; 63 64 devc->dev = dev; 65 kref_init(&devc->ref); 66 return devc; 67} 68 69struct mlx5_devcom_dev * 70mlx5_devcom_register_device(struct mlx5_core_dev *dev) 71{ 72 struct mlx5_devcom_dev *devc; 73 74 mutex_lock(&dev_list_lock); 75 76 if (devcom_dev_exists(dev)) { 77 devc = ERR_PTR(-EEXIST); 78 goto out; 79 } 80 81 devc = mlx5_devcom_dev_alloc(dev); 82 if (!devc) { 83 devc = ERR_PTR(-ENOMEM); 84 goto out; 85 } 86 87 list_add_tail(&devc->list, &devcom_dev_list); 88out: 89 mutex_unlock(&dev_list_lock); 90 return devc; 91} 92 93static void 94mlx5_devcom_dev_release(struct kref *ref) 95{ 96 struct mlx5_devcom_dev *devc = container_of(ref, struct mlx5_devcom_dev, ref); 97 98 mutex_lock(&dev_list_lock); 99 list_del(&devc->list); 100 mutex_unlock(&dev_list_lock); 101 kfree(devc); 102} 103 104void mlx5_devcom_unregister_device(struct mlx5_devcom_dev *devc) 105{ 106 if (!IS_ERR_OR_NULL(devc)) 107 kref_put(&devc->ref, mlx5_devcom_dev_release); 108} 109 110static struct mlx5_devcom_comp * 111mlx5_devcom_comp_alloc(u64 id, u64 key, mlx5_devcom_event_handler_t handler) 112{ 113 struct mlx5_devcom_comp *comp; 114 115 comp = kzalloc(sizeof(*comp), GFP_KERNEL); 116 if (!comp) 117 return ERR_PTR(-ENOMEM); 118 119 comp->id = id; 120 comp->key = key; 121 comp->handler = handler; 122 init_rwsem(&comp->sem); 123 lockdep_register_key(&comp->lock_key); 124 lockdep_set_class(&comp->sem, &comp->lock_key); 125 kref_init(&comp->ref); 126 INIT_LIST_HEAD(&comp->comp_dev_list_head); 127 128 return comp; 129} 130 131static void 132mlx5_devcom_comp_release(struct kref *ref) 133{ 134 struct mlx5_devcom_comp *comp = container_of(ref, struct mlx5_devcom_comp, ref); 135 136 mutex_lock(&comp_list_lock); 137 list_del(&comp->comp_list); 138 mutex_unlock(&comp_list_lock); 139 lockdep_unregister_key(&comp->lock_key); 140 kfree(comp); 141} 142 143static struct mlx5_devcom_comp_dev * 144devcom_alloc_comp_dev(struct mlx5_devcom_dev *devc, 145 struct mlx5_devcom_comp *comp, 146 void *data) 147{ 148 struct mlx5_devcom_comp_dev *devcom; 149 150 devcom = kzalloc(sizeof(*devcom), GFP_KERNEL); 151 if (!devcom) 152 return ERR_PTR(-ENOMEM); 153 154 kref_get(&devc->ref); 155 devcom->devc = devc; 156 devcom->comp = comp; 157 rcu_assign_pointer(devcom->data, data); 158 159 down_write(&comp->sem); 160 list_add_tail(&devcom->list, &comp->comp_dev_list_head); 161 up_write(&comp->sem); 162 163 return devcom; 164} 165 166static void 167devcom_free_comp_dev(struct mlx5_devcom_comp_dev *devcom) 168{ 169 struct mlx5_devcom_comp *comp = devcom->comp; 170 171 down_write(&comp->sem); 172 list_del(&devcom->list); 173 up_write(&comp->sem); 174 175 kref_put(&devcom->devc->ref, mlx5_devcom_dev_release); 176 kfree(devcom); 177 kref_put(&comp->ref, mlx5_devcom_comp_release); 178} 179 180static bool 181devcom_component_equal(struct mlx5_devcom_comp *devcom, 182 enum mlx5_devcom_component id, 183 u64 key) 184{ 185 return devcom->id == id && devcom->key == key; 186} 187 188static struct mlx5_devcom_comp * 189devcom_component_get(struct mlx5_devcom_dev *devc, 190 enum mlx5_devcom_component id, 191 u64 key, 192 mlx5_devcom_event_handler_t handler) 193{ 194 struct mlx5_devcom_comp *comp; 195 196 devcom_for_each_component(comp) { 197 if (devcom_component_equal(comp, id, key)) { 198 if (handler == comp->handler) { 199 kref_get(&comp->ref); 200 return comp; 201 } 202 203 mlx5_core_err(devc->dev, 204 "Cannot register existing devcom component with different handler\n"); 205 return ERR_PTR(-EINVAL); 206 } 207 } 208 209 return NULL; 210} 211 212struct mlx5_devcom_comp_dev * 213mlx5_devcom_register_component(struct mlx5_devcom_dev *devc, 214 enum mlx5_devcom_component id, 215 u64 key, 216 mlx5_devcom_event_handler_t handler, 217 void *data) 218{ 219 struct mlx5_devcom_comp_dev *devcom; 220 struct mlx5_devcom_comp *comp; 221 222 if (IS_ERR_OR_NULL(devc)) 223 return ERR_PTR(-EINVAL); 224 225 mutex_lock(&comp_list_lock); 226 comp = devcom_component_get(devc, id, key, handler); 227 if (IS_ERR(comp)) { 228 devcom = ERR_PTR(-EINVAL); 229 goto out_unlock; 230 } 231 232 if (!comp) { 233 comp = mlx5_devcom_comp_alloc(id, key, handler); 234 if (IS_ERR(comp)) { 235 devcom = ERR_CAST(comp); 236 goto out_unlock; 237 } 238 list_add_tail(&comp->comp_list, &devcom_comp_list); 239 } 240 mutex_unlock(&comp_list_lock); 241 242 devcom = devcom_alloc_comp_dev(devc, comp, data); 243 if (IS_ERR(devcom)) 244 kref_put(&comp->ref, mlx5_devcom_comp_release); 245 246 return devcom; 247 248out_unlock: 249 mutex_unlock(&comp_list_lock); 250 return devcom; 251} 252 253void mlx5_devcom_unregister_component(struct mlx5_devcom_comp_dev *devcom) 254{ 255 if (!IS_ERR_OR_NULL(devcom)) 256 devcom_free_comp_dev(devcom); 257} 258 259int mlx5_devcom_comp_get_size(struct mlx5_devcom_comp_dev *devcom) 260{ 261 struct mlx5_devcom_comp *comp = devcom->comp; 262 263 return kref_read(&comp->ref); 264} 265 266int mlx5_devcom_send_event(struct mlx5_devcom_comp_dev *devcom, 267 int event, int rollback_event, 268 void *event_data) 269{ 270 struct mlx5_devcom_comp_dev *pos; 271 struct mlx5_devcom_comp *comp; 272 int err = 0; 273 void *data; 274 275 if (IS_ERR_OR_NULL(devcom)) 276 return -ENODEV; 277 278 comp = devcom->comp; 279 down_write(&comp->sem); 280 list_for_each_entry(pos, &comp->comp_dev_list_head, list) { 281 data = rcu_dereference_protected(pos->data, lockdep_is_held(&comp->sem)); 282 283 if (pos != devcom && data) { 284 err = comp->handler(event, data, event_data); 285 if (err) 286 goto rollback; 287 } 288 } 289 290 up_write(&comp->sem); 291 return 0; 292 293rollback: 294 if (list_entry_is_head(pos, &comp->comp_dev_list_head, list)) 295 goto out; 296 pos = list_prev_entry(pos, list); 297 list_for_each_entry_from_reverse(pos, &comp->comp_dev_list_head, list) { 298 data = rcu_dereference_protected(pos->data, lockdep_is_held(&comp->sem)); 299 300 if (pos != devcom && data) 301 comp->handler(rollback_event, data, event_data); 302 } 303out: 304 up_write(&comp->sem); 305 return err; 306} 307 308void mlx5_devcom_comp_set_ready(struct mlx5_devcom_comp_dev *devcom, bool ready) 309{ 310 WARN_ON(!rwsem_is_locked(&devcom->comp->sem)); 311 312 WRITE_ONCE(devcom->comp->ready, ready); 313} 314 315bool mlx5_devcom_comp_is_ready(struct mlx5_devcom_comp_dev *devcom) 316{ 317 if (IS_ERR_OR_NULL(devcom)) 318 return false; 319 320 return READ_ONCE(devcom->comp->ready); 321} 322 323bool mlx5_devcom_for_each_peer_begin(struct mlx5_devcom_comp_dev *devcom) 324{ 325 struct mlx5_devcom_comp *comp; 326 327 if (IS_ERR_OR_NULL(devcom)) 328 return false; 329 330 comp = devcom->comp; 331 down_read(&comp->sem); 332 if (!READ_ONCE(comp->ready)) { 333 up_read(&comp->sem); 334 return false; 335 } 336 337 return true; 338} 339 340void mlx5_devcom_for_each_peer_end(struct mlx5_devcom_comp_dev *devcom) 341{ 342 up_read(&devcom->comp->sem); 343} 344 345void *mlx5_devcom_get_next_peer_data(struct mlx5_devcom_comp_dev *devcom, 346 struct mlx5_devcom_comp_dev **pos) 347{ 348 struct mlx5_devcom_comp *comp = devcom->comp; 349 struct mlx5_devcom_comp_dev *tmp; 350 void *data; 351 352 tmp = list_prepare_entry(*pos, &comp->comp_dev_list_head, list); 353 354 list_for_each_entry_continue(tmp, &comp->comp_dev_list_head, list) { 355 if (tmp != devcom) { 356 data = rcu_dereference_protected(tmp->data, lockdep_is_held(&comp->sem)); 357 if (data) 358 break; 359 } 360 } 361 362 if (list_entry_is_head(tmp, &comp->comp_dev_list_head, list)) 363 return NULL; 364 365 *pos = tmp; 366 return data; 367} 368 369void *mlx5_devcom_get_next_peer_data_rcu(struct mlx5_devcom_comp_dev *devcom, 370 struct mlx5_devcom_comp_dev **pos) 371{ 372 struct mlx5_devcom_comp *comp = devcom->comp; 373 struct mlx5_devcom_comp_dev *tmp; 374 void *data; 375 376 tmp = list_prepare_entry(*pos, &comp->comp_dev_list_head, list); 377 378 list_for_each_entry_continue(tmp, &comp->comp_dev_list_head, list) { 379 if (tmp != devcom) { 380 /* This can change concurrently, however 'data' pointer will remain 381 * valid for the duration of RCU read section. 382 */ 383 if (!READ_ONCE(comp->ready)) 384 return NULL; 385 data = rcu_dereference(tmp->data); 386 if (data) 387 break; 388 } 389 } 390 391 if (list_entry_is_head(tmp, &comp->comp_dev_list_head, list)) 392 return NULL; 393 394 *pos = tmp; 395 return data; 396} 397 398void mlx5_devcom_comp_lock(struct mlx5_devcom_comp_dev *devcom) 399{ 400 if (IS_ERR_OR_NULL(devcom)) 401 return; 402 down_write(&devcom->comp->sem); 403} 404 405void mlx5_devcom_comp_unlock(struct mlx5_devcom_comp_dev *devcom) 406{ 407 if (IS_ERR_OR_NULL(devcom)) 408 return; 409 up_write(&devcom->comp->sem); 410} 411 412int mlx5_devcom_comp_trylock(struct mlx5_devcom_comp_dev *devcom) 413{ 414 if (IS_ERR_OR_NULL(devcom)) 415 return 0; 416 return down_write_trylock(&devcom->comp->sem); 417} 418