1302484Smav/*- 2302484Smav * Copyright (c) 2016 Alexander Motin <mav@FreeBSD.org> 3302484Smav * All rights reserved. 4302484Smav * 5302484Smav * Redistribution and use in source and binary forms, with or without 6302484Smav * modification, are permitted provided that the following conditions 7302484Smav * are met: 8302484Smav * 1. Redistributions of source code must retain the above copyright 9302484Smav * notice, this list of conditions and the following disclaimer. 10302484Smav * 2. Redistributions in binary form must reproduce the above copyright 11302484Smav * notice, this list of conditions and the following disclaimer in the 12302484Smav * documentation and/or other materials provided with the distribution. 13302484Smav * 14302484Smav * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15302484Smav * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16302484Smav * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17302484Smav * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18302484Smav * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19302484Smav * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20302484Smav * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21302484Smav * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22302484Smav * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23302484Smav * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24302484Smav * SUCH DAMAGE. 25302484Smav */ 26302484Smav 27302484Smav#include <sys/cdefs.h> 28302484Smav__FBSDID("$FreeBSD$"); 29302484Smav 30302484Smav#include <sys/param.h> 31302484Smav#include <sys/kernel.h> 32302484Smav#include <sys/systm.h> 33302484Smav#include <sys/bus.h> 34304380Smav#include <machine/bus.h> 35304404Smav#include <sys/rmlock.h> 36304404Smav#include <sys/malloc.h> 37302484Smav#include <sys/module.h> 38302484Smav#include <sys/sysctl.h> 39302484Smav 40302484Smav#include "ntb.h" 41302484Smav 42302484Smavdevclass_t ntb_hw_devclass; 43302484SmavSYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW, 0, "NTB sysctls"); 44302484Smav 45304404Smavstruct ntb_child { 46304404Smav device_t dev; 47304404Smav int enabled; 48304404Smav int mwoff; 49304404Smav int mwcnt; 50304404Smav int spadoff; 51304404Smav int spadcnt; 52304404Smav int dboff; 53304404Smav int dbmask; 54304404Smav void *ctx; 55304404Smav const struct ntb_ctx_ops *ctx_ops; 56304404Smav struct rmlock ctx_lock; 57304404Smav struct ntb_child *next; 58304404Smav}; 59304404Smav 60304404Smavint 61304404Smavntb_register_device(device_t dev) 62304404Smav{ 63304404Smav struct ntb_child **cpp = device_get_softc(dev); 64304404Smav struct ntb_child *nc; 65304404Smav int i, mw, mwu, mwt, spad, spadu, spadt, db, dbu, dbt; 66304404Smav char cfg[128] = ""; 67304404Smav char buf[32]; 68304404Smav char *n, *np, *c, *p, *name; 69304404Smav 70304404Smav mwu = 0; 71304404Smav mwt = NTB_MW_COUNT(dev); 72304404Smav spadu = 0; 73304404Smav spadt = NTB_SPAD_COUNT(dev); 74304404Smav dbu = 0; 75304404Smav dbt = flsll(NTB_DB_VALID_MASK(dev)); 76304404Smav 77304404Smav device_printf(dev, "%d memory windows, %d scratchpads, " 78304404Smav "%d doorbells\n", mwt, spadt, dbt); 79304404Smav 80304404Smav snprintf(buf, sizeof(buf), "hint.%s.%d.config", device_get_name(dev), 81304404Smav device_get_unit(dev)); 82304404Smav TUNABLE_STR_FETCH(buf, cfg, sizeof(cfg)); 83304404Smav n = cfg; 84304404Smav i = 0; 85304404Smav while ((c = strsep(&n, ",")) != NULL) { 86304404Smav np = c; 87304404Smav name = strsep(&np, ":"); 88304404Smav if (name != NULL && name[0] == 0) 89304404Smav name = NULL; 90304404Smav p = strsep(&np, ":"); 91304404Smav mw = (p && p[0] != 0) ? strtol(p, NULL, 10) : mwt - mwu; 92304404Smav p = strsep(&np, ":"); 93304404Smav spad = (p && p[0] != 0) ? strtol(p, NULL, 10) : spadt - spadu; 94304404Smav db = (np && np[0] != 0) ? strtol(np, NULL, 10) : dbt - dbu; 95304404Smav 96304404Smav if (mw > mwt - mwu || spad > spadt - spadu || db > dbt - dbu) { 97304404Smav device_printf(dev, "Not enough resources for config\n"); 98304404Smav break; 99304404Smav } 100304404Smav 101304404Smav nc = malloc(sizeof(*nc), M_DEVBUF, M_WAITOK | M_ZERO); 102304404Smav nc->mwoff = mwu; 103304404Smav nc->mwcnt = mw; 104304404Smav nc->spadoff = spadu; 105304404Smav nc->spadcnt = spad; 106304404Smav nc->dboff = dbu; 107304404Smav nc->dbmask = (db == 0) ? 0 : (0xffffffffffffffff >> (64 - db)); 108304404Smav rm_init(&nc->ctx_lock, "ntb ctx"); 109304404Smav nc->dev = device_add_child(dev, name, -1); 110304404Smav if (nc->dev == NULL) { 111304404Smav ntb_unregister_device(dev); 112304404Smav return (ENOMEM); 113304404Smav } 114304404Smav device_set_ivars(nc->dev, nc); 115304404Smav *cpp = nc; 116304404Smav cpp = &nc->next; 117304404Smav 118304404Smav if (bootverbose) { 119304404Smav device_printf(dev, "%d \"%s\":", i, name); 120304404Smav if (mw > 0) { 121304404Smav printf(" memory windows %d", mwu); 122304404Smav if (mw > 1) 123304404Smav printf("-%d", mwu + mw - 1); 124304404Smav } 125304404Smav if (spad > 0) { 126304404Smav printf(" scratchpads %d", spadu); 127304404Smav if (spad > 1) 128304404Smav printf("-%d", spadu + spad - 1); 129304404Smav } 130304404Smav if (db > 0) { 131304404Smav printf(" doorbells %d", dbu); 132304404Smav if (db > 1) 133304404Smav printf("-%d", dbu + db - 1); 134304404Smav } 135304404Smav printf("\n"); 136304404Smav } 137304404Smav 138304404Smav mwu += mw; 139304404Smav spadu += spad; 140304404Smav dbu += db; 141304404Smav i++; 142304404Smav } 143304404Smav 144304404Smav bus_generic_attach(dev); 145304404Smav return (0); 146304404Smav} 147304404Smav 148304404Smavint 149304404Smavntb_unregister_device(device_t dev) 150304404Smav{ 151304404Smav struct ntb_child **cpp = device_get_softc(dev); 152304404Smav struct ntb_child *nc; 153304404Smav int error = 0; 154304404Smav 155304404Smav while ((nc = *cpp) != NULL) { 156304404Smav *cpp = (*cpp)->next; 157304404Smav error = device_delete_child(dev, nc->dev); 158304404Smav if (error) 159304404Smav break; 160304404Smav rm_destroy(&nc->ctx_lock); 161304404Smav free(nc, M_DEVBUF); 162304404Smav } 163304404Smav return (error); 164304404Smav} 165304404Smav 166304404Smavvoid 167304404Smavntb_link_event(device_t dev) 168304404Smav{ 169304404Smav struct ntb_child **cpp = device_get_softc(dev); 170304404Smav struct ntb_child *nc; 171304404Smav struct rm_priotracker ctx_tracker; 172304404Smav 173304404Smav for (nc = *cpp; nc != NULL; nc = nc->next) { 174304404Smav rm_rlock(&nc->ctx_lock, &ctx_tracker); 175304404Smav if (nc->ctx_ops != NULL && nc->ctx_ops->link_event != NULL) 176304404Smav nc->ctx_ops->link_event(nc->ctx); 177304404Smav rm_runlock(&nc->ctx_lock, &ctx_tracker); 178304404Smav } 179304404Smav} 180304404Smav 181304404Smavvoid 182304404Smavntb_db_event(device_t dev, uint32_t vec) 183304404Smav{ 184304404Smav struct ntb_child **cpp = device_get_softc(dev); 185304404Smav struct ntb_child *nc; 186304404Smav struct rm_priotracker ctx_tracker; 187304404Smav 188304404Smav for (nc = *cpp; nc != NULL; nc = nc->next) { 189304404Smav rm_rlock(&nc->ctx_lock, &ctx_tracker); 190304404Smav if (nc->ctx_ops != NULL && nc->ctx_ops->db_event != NULL) 191304404Smav nc->ctx_ops->db_event(nc->ctx, vec); 192304404Smav rm_runlock(&nc->ctx_lock, &ctx_tracker); 193304404Smav } 194304404Smav} 195304404Smav 196304404Smavbool 197304404Smavntb_link_is_up(device_t ntb, enum ntb_speed *speed, enum ntb_width *width) 198304404Smav{ 199304404Smav 200304404Smav return (NTB_LINK_IS_UP(device_get_parent(ntb), speed, width)); 201304404Smav} 202304404Smav 203304404Smavint 204304404Smavntb_link_enable(device_t ntb, enum ntb_speed speed, enum ntb_width width) 205304404Smav{ 206304404Smav struct ntb_child *nc = device_get_ivars(ntb); 207304404Smav struct ntb_child **cpp = device_get_softc(device_get_parent(nc->dev)); 208304404Smav struct ntb_child *nc1; 209304404Smav 210304404Smav for (nc1 = *cpp; nc1 != NULL; nc1 = nc1->next) { 211304404Smav if (nc1->enabled) { 212304404Smav nc->enabled = 1; 213304404Smav return (0); 214304404Smav } 215304404Smav } 216304404Smav nc->enabled = 1; 217304404Smav return (NTB_LINK_ENABLE(device_get_parent(ntb), speed, width)); 218304404Smav} 219304404Smav 220304404Smavint 221304404Smavntb_link_disable(device_t ntb) 222304404Smav{ 223304404Smav struct ntb_child *nc = device_get_ivars(ntb); 224304404Smav struct ntb_child **cpp = device_get_softc(device_get_parent(nc->dev)); 225304404Smav struct ntb_child *nc1; 226304404Smav 227304404Smav if (!nc->enabled) 228304404Smav return (0); 229304404Smav nc->enabled = 0; 230304404Smav for (nc1 = *cpp; nc1 != NULL; nc1 = nc1->next) { 231304404Smav if (nc1->enabled) 232304404Smav return (0); 233304404Smav } 234304404Smav return (NTB_LINK_DISABLE(device_get_parent(ntb))); 235304404Smav} 236304404Smav 237304404Smavbool 238304404Smavntb_link_enabled(device_t ntb) 239304404Smav{ 240304404Smav struct ntb_child *nc = device_get_ivars(ntb); 241304404Smav 242304404Smav return (nc->enabled && NTB_LINK_ENABLED(device_get_parent(ntb))); 243304404Smav} 244304404Smav 245304404Smavint 246304404Smavntb_set_ctx(device_t ntb, void *ctx, const struct ntb_ctx_ops *ctx_ops) 247304404Smav{ 248304404Smav struct ntb_child *nc = device_get_ivars(ntb); 249304404Smav 250304404Smav if (ctx == NULL || ctx_ops == NULL) 251304404Smav return (EINVAL); 252304404Smav 253304404Smav rm_wlock(&nc->ctx_lock); 254304404Smav if (nc->ctx_ops != NULL) { 255304404Smav rm_wunlock(&nc->ctx_lock); 256304404Smav return (EINVAL); 257304404Smav } 258304404Smav nc->ctx = ctx; 259304404Smav nc->ctx_ops = ctx_ops; 260304404Smav rm_wunlock(&nc->ctx_lock); 261304404Smav 262304404Smav return (0); 263304404Smav} 264304404Smav 265304404Smavvoid * 266304404Smavntb_get_ctx(device_t ntb, const struct ntb_ctx_ops **ctx_ops) 267304404Smav{ 268304404Smav struct ntb_child *nc = device_get_ivars(ntb); 269304404Smav 270304404Smav KASSERT(nc->ctx != NULL && nc->ctx_ops != NULL, ("bogus")); 271304404Smav if (ctx_ops != NULL) 272304404Smav *ctx_ops = nc->ctx_ops; 273304404Smav return (nc->ctx); 274304404Smav} 275304404Smav 276304404Smavvoid 277304404Smavntb_clear_ctx(device_t ntb) 278304404Smav{ 279304404Smav struct ntb_child *nc = device_get_ivars(ntb); 280304404Smav 281304404Smav rm_wlock(&nc->ctx_lock); 282304404Smav nc->ctx = NULL; 283304404Smav nc->ctx_ops = NULL; 284304404Smav rm_wunlock(&nc->ctx_lock); 285304404Smav} 286304404Smav 287304404Smavuint8_t 288304404Smavntb_mw_count(device_t ntb) 289304404Smav{ 290304404Smav struct ntb_child *nc = device_get_ivars(ntb); 291304404Smav 292304404Smav return (nc->mwcnt); 293304404Smav} 294304404Smav 295304404Smavint 296304404Smavntb_mw_get_range(device_t ntb, unsigned mw_idx, vm_paddr_t *base, 297304404Smav caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 298304404Smav bus_addr_t *plimit) 299304404Smav{ 300304404Smav struct ntb_child *nc = device_get_ivars(ntb); 301304404Smav 302304404Smav return (NTB_MW_GET_RANGE(device_get_parent(ntb), mw_idx + nc->mwoff, 303304404Smav base, vbase, size, align, align_size, plimit)); 304304404Smav} 305304404Smav 306304404Smavint 307304404Smavntb_mw_set_trans(device_t ntb, unsigned mw_idx, bus_addr_t addr, size_t size) 308304404Smav{ 309304404Smav struct ntb_child *nc = device_get_ivars(ntb); 310304404Smav 311304404Smav return (NTB_MW_SET_TRANS(device_get_parent(ntb), mw_idx + nc->mwoff, 312304404Smav addr, size)); 313304404Smav} 314304404Smav 315304404Smavint 316304404Smavntb_mw_clear_trans(device_t ntb, unsigned mw_idx) 317304404Smav{ 318304404Smav struct ntb_child *nc = device_get_ivars(ntb); 319304404Smav 320304404Smav return (NTB_MW_CLEAR_TRANS(device_get_parent(ntb), mw_idx + nc->mwoff)); 321304404Smav} 322304404Smav 323304404Smavint 324304404Smavntb_mw_get_wc(device_t ntb, unsigned mw_idx, vm_memattr_t *mode) 325304404Smav{ 326304404Smav struct ntb_child *nc = device_get_ivars(ntb); 327304404Smav 328304404Smav return (NTB_MW_GET_WC(device_get_parent(ntb), mw_idx + nc->mwoff, mode)); 329304404Smav} 330304404Smav 331304404Smavint 332304404Smavntb_mw_set_wc(device_t ntb, unsigned mw_idx, vm_memattr_t mode) 333304404Smav{ 334304404Smav struct ntb_child *nc = device_get_ivars(ntb); 335304404Smav 336304404Smav return (NTB_MW_SET_WC(device_get_parent(ntb), mw_idx + nc->mwoff, mode)); 337304404Smav} 338304404Smav 339304404Smavuint8_t 340304404Smavntb_spad_count(device_t ntb) 341304404Smav{ 342304404Smav struct ntb_child *nc = device_get_ivars(ntb); 343304404Smav 344304404Smav return (nc->spadcnt); 345304404Smav} 346304404Smav 347304404Smavvoid 348304404Smavntb_spad_clear(device_t ntb) 349304404Smav{ 350304404Smav struct ntb_child *nc = device_get_ivars(ntb); 351304404Smav unsigned i; 352304404Smav 353304404Smav for (i = 0; i < nc->spadcnt; i++) 354304404Smav NTB_SPAD_WRITE(device_get_parent(ntb), i + nc->spadoff, 0); 355304404Smav} 356304404Smav 357304404Smavint 358304404Smavntb_spad_write(device_t ntb, unsigned int idx, uint32_t val) 359304404Smav{ 360304404Smav struct ntb_child *nc = device_get_ivars(ntb); 361304404Smav 362304404Smav return (NTB_SPAD_WRITE(device_get_parent(ntb), idx + nc->spadoff, val)); 363304404Smav} 364304404Smav 365304404Smavint 366304404Smavntb_spad_read(device_t ntb, unsigned int idx, uint32_t *val) 367304404Smav{ 368304404Smav struct ntb_child *nc = device_get_ivars(ntb); 369304404Smav 370304404Smav return (NTB_SPAD_READ(device_get_parent(ntb), idx + nc->spadoff, val)); 371304404Smav} 372304404Smav 373304404Smavint 374304404Smavntb_peer_spad_write(device_t ntb, unsigned int idx, uint32_t val) 375304404Smav{ 376304404Smav struct ntb_child *nc = device_get_ivars(ntb); 377304404Smav 378304404Smav return (NTB_PEER_SPAD_WRITE(device_get_parent(ntb), idx + nc->spadoff, 379304404Smav val)); 380304404Smav} 381304404Smav 382304404Smavint 383304404Smavntb_peer_spad_read(device_t ntb, unsigned int idx, uint32_t *val) 384304404Smav{ 385304404Smav struct ntb_child *nc = device_get_ivars(ntb); 386304404Smav 387304404Smav return (NTB_PEER_SPAD_READ(device_get_parent(ntb), idx + nc->spadoff, 388304404Smav val)); 389304404Smav} 390304404Smav 391304404Smavuint64_t 392304404Smavntb_db_valid_mask(device_t ntb) 393304404Smav{ 394304404Smav struct ntb_child *nc = device_get_ivars(ntb); 395304404Smav 396304404Smav return (nc->dbmask); 397304404Smav} 398304404Smav 399304404Smavint 400304404Smavntb_db_vector_count(device_t ntb) 401304404Smav{ 402304404Smav 403304404Smav return (NTB_DB_VECTOR_COUNT(device_get_parent(ntb))); 404304404Smav} 405304404Smav 406304404Smavuint64_t 407304404Smavntb_db_vector_mask(device_t ntb, uint32_t vector) 408304404Smav{ 409304404Smav struct ntb_child *nc = device_get_ivars(ntb); 410304404Smav 411304404Smav return ((NTB_DB_VECTOR_MASK(device_get_parent(ntb), vector) 412304404Smav >> nc->dboff) & nc->dbmask); 413304404Smav} 414304404Smav 415304404Smavint 416304404Smavntb_peer_db_addr(device_t ntb, bus_addr_t *db_addr, vm_size_t *db_size) 417304404Smav{ 418304404Smav 419304404Smav return (NTB_PEER_DB_ADDR(device_get_parent(ntb), db_addr, db_size)); 420304404Smav} 421304404Smav 422304404Smavvoid 423304404Smavntb_db_clear(device_t ntb, uint64_t bits) 424304404Smav{ 425304404Smav struct ntb_child *nc = device_get_ivars(ntb); 426304404Smav 427304404Smav return (NTB_DB_CLEAR(device_get_parent(ntb), bits << nc->dboff)); 428304404Smav} 429304404Smav 430304404Smavvoid 431304404Smavntb_db_clear_mask(device_t ntb, uint64_t bits) 432304404Smav{ 433304404Smav struct ntb_child *nc = device_get_ivars(ntb); 434304404Smav 435304404Smav return (NTB_DB_CLEAR_MASK(device_get_parent(ntb), bits << nc->dboff)); 436304404Smav} 437304404Smav 438304404Smavuint64_t 439304404Smavntb_db_read(device_t ntb) 440304404Smav{ 441304404Smav struct ntb_child *nc = device_get_ivars(ntb); 442304404Smav 443304404Smav return ((NTB_DB_READ(device_get_parent(ntb)) >> nc->dboff) 444304404Smav & nc->dbmask); 445304404Smav} 446304404Smav 447304404Smavvoid 448304404Smavntb_db_set_mask(device_t ntb, uint64_t bits) 449304404Smav{ 450304404Smav struct ntb_child *nc = device_get_ivars(ntb); 451304404Smav 452304404Smav return (NTB_DB_SET_MASK(device_get_parent(ntb), bits << nc->dboff)); 453304404Smav} 454304404Smav 455304404Smavvoid 456304404Smavntb_peer_db_set(device_t ntb, uint64_t bits) 457304404Smav{ 458304404Smav struct ntb_child *nc = device_get_ivars(ntb); 459304404Smav 460304404Smav return (NTB_PEER_DB_SET(device_get_parent(ntb), bits << nc->dboff)); 461304404Smav} 462304404Smav 463302484SmavMODULE_VERSION(ntb, 1); 464