phyp_llan.c revision 257292
1179595Sbenno/*- 2179595Sbenno * Copyright 2013 Nathan Whitehorn 3179595Sbenno * All rights reserved. 4179595Sbenno * 5179595Sbenno * Redistribution and use in source and binary forms, with or without 6179595Sbenno * modification, are permitted provided that the following conditions 7179595Sbenno * are met: 8179595Sbenno * 1. Redistributions of source code must retain the above copyright 9179595Sbenno * notice, this list of conditions and the following disclaimer. 10179595Sbenno * 2. Redistributions in binary form must reproduce the above copyright 11179595Sbenno * notice, this list of conditions and the following disclaimer in the 12179595Sbenno * documentation and/or other materials provided with the distribution. 13179595Sbenno * 14179595Sbenno * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15179595Sbenno * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16179595Sbenno * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17179595Sbenno * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18179595Sbenno * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19179595Sbenno * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20179595Sbenno * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21179595Sbenno * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22179595Sbenno * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23179595Sbenno * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24179595Sbenno * SUCH DAMAGE. 25179595Sbenno */ 26179595Sbenno 27179595Sbenno#include <sys/cdefs.h> 28179595Sbenno__FBSDID("$FreeBSD: stable/10/sys/powerpc/pseries/phyp_llan.c 257292 2013-10-28 23:47:52Z nwhitehorn $"); 29179595Sbenno 30179595Sbenno#include <sys/param.h> 31179595Sbenno#include <sys/systm.h> 32179595Sbenno#include <sys/sockio.h> 33179595Sbenno#include <sys/endian.h> 34179595Sbenno#include <sys/mbuf.h> 35179595Sbenno#include <sys/module.h> 36179595Sbenno#include <sys/malloc.h> 37179595Sbenno#include <sys/kernel.h> 38179595Sbenno#include <sys/socket.h> 39179595Sbenno 40179595Sbenno#include <net/bpf.h> 41179595Sbenno#include <net/if.h> 42179595Sbenno#include <net/if_arp.h> 43179595Sbenno#include <net/ethernet.h> 44179595Sbenno#include <net/if_dl.h> 45179595Sbenno#include <net/if_media.h> 46179595Sbenno#include <net/if_types.h> 47179595Sbenno#include <net/if_vlan_var.h> 48179595Sbenno 49179595Sbenno#include <dev/ofw/openfirm.h> 50179595Sbenno#include <dev/ofw/ofw_bus.h> 51179595Sbenno#include <dev/ofw/ofw_bus_subr.h> 52179595Sbenno#include <machine/bus.h> 53179595Sbenno#include <machine/resource.h> 54179595Sbenno#include <sys/bus.h> 55179595Sbenno#include <sys/rman.h> 56227293Sed 57179595Sbenno#include <powerpc/pseries/phyp-hvcall.h> 58179595Sbenno 59179595Sbenno#define LLAN_MAX_RX_PACKETS 100 60179595Sbenno#define LLAN_MAX_TX_PACKETS 100 61179595Sbenno#define LLAN_RX_BUF_LEN 8*PAGE_SIZE 62179595Sbenno 63179595Sbenno#define LLAN_BUFDESC_VALID (1ULL << 63) 64179595Sbenno#define LLAN_ADD_MULTICAST 0x1 65179595Sbenno#define LLAN_DEL_MULTICAST 0x2 66179595Sbenno#define LLAN_CLEAR_MULTICAST 0x3 67179595Sbenno 68278727Sianstruct llan_xfer { 69179595Sbenno struct mbuf *rx_mbuf; 70179595Sbenno bus_dmamap_t rx_dmamap; 71278727Sian uint64_t rx_bufdesc; 72278727Sian}; 73278727Sian 74179595Sbennostruct llan_receive_queue_entry { /* PAPR page 539 */ 75179595Sbenno uint8_t control; 76278727Sian uint8_t reserved; 77278727Sian uint16_t offset; 78179595Sbenno uint32_t length; 79179595Sbenno uint64_t handle; 80278727Sian} __packed; 81179595Sbenno 82179595Sbennostruct llan_softc { 83278727Sian device_t dev; 84278727Sian struct mtx io_lock; 85278727Sian 86278727Sian cell_t unit; 87179595Sbenno uint8_t mac_address[8]; 88179595Sbenno 89278727Sian int irqid; 90278727Sian struct resource *irq; 91278727Sian void *irq_cookie; 92278727Sian 93179595Sbenno bus_dma_tag_t rx_dma_tag; 94179595Sbenno bus_dma_tag_t rxbuf_dma_tag; 95278727Sian bus_dma_tag_t tx_dma_tag; 96278727Sian 97278727Sian bus_dmamap_t tx_dma_map; 98278727Sian 99179595Sbenno struct llan_receive_queue_entry *rx_buf; 100179595Sbenno int rx_dma_slot; 101278727Sian int rx_valid_val; 102278727Sian bus_dmamap_t rx_buf_map; 103278727Sian bus_addr_t rx_buf_phys; 104278727Sian bus_size_t rx_buf_len; 105179595Sbenno bus_addr_t input_buf_phys; 106179595Sbenno bus_addr_t filter_buf_phys; 107278727Sian struct llan_xfer rx_xfer[LLAN_MAX_RX_PACKETS]; 108278727Sian 109278727Sian struct ifnet *ifp; 110278727Sian}; 111179595Sbenno 112179595Sbennostatic int llan_probe(device_t); 113278727Sianstatic int llan_attach(device_t); 114278727Sianstatic void llan_intr(void *xsc); 115278727Sianstatic void llan_init(void *xsc); 116278727Sianstatic void llan_start(struct ifnet *ifp); 117179595Sbennostatic int llan_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data); 118179595Sbennostatic void llan_rx_load_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, 119278727Sian int err); 120278727Sianstatic int llan_add_rxbuf(struct llan_softc *sc, struct llan_xfer *rx); 121278727Sianstatic int llan_set_multicast(struct llan_softc *sc); 122278727Sian 123179595Sbennostatic devclass_t llan_devclass; 124179595Sbennostatic device_method_t llan_methods[] = { 125278727Sian DEVMETHOD(device_probe, llan_probe), 126278727Sian DEVMETHOD(device_attach, llan_attach), 127278727Sian 128278727Sian DEVMETHOD_END 129179595Sbenno}; 130179595Sbennostatic driver_t llan_driver = { 131278727Sian "llan", 132278727Sian llan_methods, 133278727Sian sizeof(struct llan_softc) 134278727Sian}; 135278727SianDRIVER_MODULE(llan, vdevice, llan_driver, llan_devclass, 0, 0); 136278727Sian 137278727Sianstatic int 138278727Sianllan_probe(device_t dev) 139278727Sian{ 140278727Sian if (!ofw_bus_is_compatible(dev,"IBM,l-lan")) 141278727Sian return (ENXIO); 142278727Sian 143278727Sian device_set_desc(dev, "POWER Hypervisor Virtual Ethernet"); 144278727Sian return (0); 145278727Sian} 146278727Sian 147278727Sianstatic int 148278727Sianllan_attach(device_t dev) 149278727Sian{ 150278727Sian struct llan_softc *sc; 151278727Sian phandle_t node; 152278727Sian int error, i; 153278727Sian 154278727Sian sc = device_get_softc(dev); 155278727Sian sc->dev = dev; 156278727Sian 157278727Sian /* Get firmware properties */ 158278727Sian node = ofw_bus_get_node(dev); 159278727Sian OF_getprop(node, "local-mac-address", sc->mac_address, 160278727Sian sizeof(sc->mac_address)); 161278727Sian OF_getprop(node, "reg", &sc->unit, sizeof(sc->unit)); 162278727Sian 163278727Sian mtx_init(&sc->io_lock, "llan", NULL, MTX_DEF); 164278727Sian 165278727Sian /* Setup interrupt */ 166278727Sian sc->irqid = 0; 167278727Sian sc->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->irqid, 168278727Sian RF_ACTIVE); 169278727Sian 170278727Sian if (!sc->irq) { 171179595Sbenno device_printf(dev, "Could not allocate IRQ\n"); 172179595Sbenno mtx_destroy(&sc->io_lock); 173179595Sbenno return (ENXIO); 174179595Sbenno } 175179595Sbenno 176179595Sbenno bus_setup_intr(dev, sc->irq, INTR_TYPE_MISC | INTR_MPSAFE | 177179595Sbenno INTR_ENTROPY, NULL, llan_intr, sc, &sc->irq_cookie); 178179595Sbenno 179179595Sbenno /* Setup DMA */ 180179595Sbenno error = bus_dma_tag_create(bus_get_dma_tag(dev), 16, 0, 181179595Sbenno BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, 182179595Sbenno LLAN_RX_BUF_LEN, 1, BUS_SPACE_MAXSIZE_32BIT, 183278727Sian 0, NULL, NULL, &sc->rx_dma_tag); 184179595Sbenno error = bus_dma_tag_create(bus_get_dma_tag(dev), 4, 0, 185179595Sbenno BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, 186179595Sbenno BUS_SPACE_MAXSIZE, 1, BUS_SPACE_MAXSIZE_32BIT, 187179595Sbenno 0, NULL, NULL, &sc->rxbuf_dma_tag); 188179595Sbenno error = bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0, 189179595Sbenno BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, 190179595Sbenno BUS_SPACE_MAXSIZE, 6, BUS_SPACE_MAXSIZE_32BIT, 0, 191179595Sbenno busdma_lock_mutex, &sc->io_lock, &sc->tx_dma_tag); 192179595Sbenno 193179595Sbenno error = bus_dmamem_alloc(sc->rx_dma_tag, (void **)&sc->rx_buf, 194179595Sbenno BUS_DMA_WAITOK | BUS_DMA_ZERO, &sc->rx_buf_map); 195179595Sbenno error = bus_dmamap_load(sc->rx_dma_tag, sc->rx_buf_map, sc->rx_buf, 196179595Sbenno LLAN_RX_BUF_LEN, llan_rx_load_cb, sc, 0); 197179595Sbenno 198179595Sbenno /* TX DMA maps */ 199278727Sian bus_dmamap_create(sc->tx_dma_tag, 0, &sc->tx_dma_map); 200179595Sbenno 201179595Sbenno /* RX DMA */ 202179595Sbenno for (i = 0; i < LLAN_MAX_RX_PACKETS; i++) { 203179595Sbenno error = bus_dmamap_create(sc->rxbuf_dma_tag, 0, 204179595Sbenno &sc->rx_xfer[i].rx_dmamap); 205179595Sbenno sc->rx_xfer[i].rx_mbuf = NULL; 206179595Sbenno } 207278727Sian 208179595Sbenno /* Attach to network stack */ 209179595Sbenno sc->ifp = if_alloc(IFT_ETHER); 210179595Sbenno sc->ifp->if_softc = sc; 211278727Sian 212179595Sbenno if_initname(sc->ifp, device_get_name(dev), device_get_unit(dev)); 213179595Sbenno sc->ifp->if_mtu = ETHERMTU; /* XXX max-frame-size from OF? */ 214179595Sbenno sc->ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 215179595Sbenno sc->ifp->if_hwassist = 0; /* XXX: ibm,illan-options */ 216179595Sbenno sc->ifp->if_capabilities = 0; 217278727Sian sc->ifp->if_capenable = 0; 218179595Sbenno sc->ifp->if_start = llan_start; 219179595Sbenno sc->ifp->if_ioctl = llan_ioctl; 220179595Sbenno sc->ifp->if_init = llan_init; 221179595Sbenno 222179595Sbenno IFQ_SET_MAXLEN(&sc->ifp->if_snd, LLAN_MAX_TX_PACKETS); 223179595Sbenno sc->ifp->if_snd.ifq_drv_maxlen = LLAN_MAX_TX_PACKETS; 224278727Sian IFQ_SET_READY(&sc->ifp->if_snd); 225179595Sbenno 226179595Sbenno ether_ifattach(sc->ifp, &sc->mac_address[2]); 227179595Sbenno 228278727Sian return (0); 229179595Sbenno} 230179595Sbenno 231179595Sbennostatic void 232179595Sbennollan_rx_load_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int err) 233278727Sian{ 234179595Sbenno struct llan_softc *sc = xsc; 235179595Sbenno 236179595Sbenno sc->rx_buf_phys = segs[0].ds_addr; 237179595Sbenno sc->rx_buf_len = segs[0].ds_len - 2*PAGE_SIZE; 238179595Sbenno sc->input_buf_phys = segs[0].ds_addr + segs[0].ds_len - PAGE_SIZE; 239179595Sbenno sc->filter_buf_phys = segs[0].ds_addr + segs[0].ds_len - 2*PAGE_SIZE; 240278727Sian} 241179595Sbenno 242179595Sbennostatic void 243179595Sbennollan_init(void *xsc) 244278727Sian{ 245179595Sbenno struct llan_softc *sc = xsc; 246179595Sbenno uint64_t rx_buf_desc; 247179595Sbenno uint64_t macaddr; 248179595Sbenno int err, i; 249278727Sian 250179595Sbenno mtx_lock(&sc->io_lock); 251179595Sbenno 252179595Sbenno phyp_hcall(H_FREE_LOGICAL_LAN, sc->unit); 253179595Sbenno 254179595Sbenno /* Create buffers (page 539) */ 255179595Sbenno sc->rx_dma_slot = 0; 256179595Sbenno sc->rx_valid_val = 1; 257278727Sian 258179595Sbenno rx_buf_desc = LLAN_BUFDESC_VALID; 259179595Sbenno rx_buf_desc |= (sc->rx_buf_len << 32); 260179595Sbenno rx_buf_desc |= sc->rx_buf_phys; 261278727Sian memcpy(&macaddr, sc->mac_address, 8); 262179595Sbenno err = phyp_hcall(H_REGISTER_LOGICAL_LAN, sc->unit, sc->input_buf_phys, 263179595Sbenno rx_buf_desc, sc->filter_buf_phys, macaddr); 264179595Sbenno 265179595Sbenno for (i = 0; i < LLAN_MAX_RX_PACKETS; i++) 266278727Sian llan_add_rxbuf(sc, &sc->rx_xfer[i]); 267179595Sbenno 268179595Sbenno phyp_hcall(H_VIO_SIGNAL, sc->unit, 1); /* Enable interrupts */ 269 270 /* Tell stack we're up */ 271 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING; 272 sc->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 273 274 mtx_unlock(&sc->io_lock); 275} 276 277static int 278llan_add_rxbuf(struct llan_softc *sc, struct llan_xfer *rx) 279{ 280 struct mbuf *m; 281 bus_dma_segment_t segs[1]; 282 int error, nsegs; 283 284 mtx_assert(&sc->io_lock, MA_OWNED); 285 286 m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); 287 if (m == NULL) 288 return (ENOBUFS); 289 290 m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; 291 if (rx->rx_mbuf != NULL) { 292 bus_dmamap_sync(sc->rxbuf_dma_tag, rx->rx_dmamap, 293 BUS_DMASYNC_POSTREAD); 294 bus_dmamap_unload(sc->rxbuf_dma_tag, rx->rx_dmamap); 295 } 296 297 /* Save pointer to buffer structure */ 298 m_copyback(m, 0, 8, (void *)&rx); 299 300 error = bus_dmamap_load_mbuf_sg(sc->rxbuf_dma_tag, rx->rx_dmamap, m, 301 segs, &nsegs, BUS_DMA_NOWAIT); 302 if (error != 0) { 303 device_printf(sc->dev, 304 "cannot load RX DMA map %p, error = %d\n", rx, error); 305 m_freem(m); 306 return (error); 307 } 308 309 /* If nsegs is wrong then the stack is corrupt. */ 310 KASSERT(nsegs == 1, 311 ("%s: too many DMA segments (%d)", __func__, nsegs)); 312 rx->rx_mbuf = m; 313 314 bus_dmamap_sync(sc->rxbuf_dma_tag, rx->rx_dmamap, BUS_DMASYNC_PREREAD); 315 316 rx->rx_bufdesc = LLAN_BUFDESC_VALID; 317 rx->rx_bufdesc |= (((uint64_t)segs[0].ds_len) << 32); 318 rx->rx_bufdesc |= segs[0].ds_addr; 319 error = phyp_hcall(H_ADD_LOGICAL_LAN_BUFFER, sc->unit, rx->rx_bufdesc); 320 if (error != 0) { 321 m_freem(m); 322 rx->rx_mbuf = NULL; 323 return (ENOBUFS); 324 } 325 326 return (0); 327} 328 329static void 330llan_intr(void *xsc) 331{ 332 struct llan_softc *sc = xsc; 333 struct llan_xfer *rx; 334 struct mbuf *m; 335 336 mtx_lock(&sc->io_lock); 337 phyp_hcall(H_VIO_SIGNAL, sc->unit, 0); 338 339 while ((sc->rx_buf[sc->rx_dma_slot].control >> 7) == sc->rx_valid_val) { 340 rx = (struct llan_xfer *)sc->rx_buf[sc->rx_dma_slot].handle; 341 m = rx->rx_mbuf; 342 m_adj(m, sc->rx_buf[sc->rx_dma_slot].offset - 8); 343 m->m_len = sc->rx_buf[sc->rx_dma_slot].length; 344 345 /* llan_add_rxbuf does DMA sync and unload as well as requeue */ 346 if (llan_add_rxbuf(sc, rx) != 0) { 347 sc->ifp->if_ierrors++; 348 phyp_hcall(H_ADD_LOGICAL_LAN_BUFFER, sc->unit, 349 rx->rx_bufdesc); 350 continue; 351 } 352 353 sc->ifp->if_ipackets++; 354 m_adj(m, sc->rx_buf[sc->rx_dma_slot].offset); 355 m->m_len = sc->rx_buf[sc->rx_dma_slot].length; 356 m->m_pkthdr.rcvif = sc->ifp; 357 m->m_pkthdr.len = m->m_len; 358 sc->rx_dma_slot++; 359 360 if (sc->rx_dma_slot >= sc->rx_buf_len/sizeof(sc->rx_buf[0])) { 361 sc->rx_dma_slot = 0; 362 sc->rx_valid_val = !sc->rx_valid_val; 363 } 364 365 mtx_unlock(&sc->io_lock); 366 (*sc->ifp->if_input)(sc->ifp, m); 367 mtx_lock(&sc->io_lock); 368 } 369 370 phyp_hcall(H_VIO_SIGNAL, sc->unit, 1); 371 mtx_unlock(&sc->io_lock); 372} 373 374static void 375llan_send_packet(void *xsc, bus_dma_segment_t *segs, int nsegs, 376 bus_size_t mapsize, int error) 377{ 378 struct llan_softc *sc = xsc; 379 uint64_t bufdescs[6]; 380 int i; 381 382 bzero(bufdescs, sizeof(bufdescs)); 383 384 for (i = 0; i < nsegs; i++) { 385 bufdescs[i] = LLAN_BUFDESC_VALID; 386 bufdescs[i] |= (((uint64_t)segs[i].ds_len) << 32); 387 bufdescs[i] |= segs[i].ds_addr; 388 } 389 390 phyp_hcall(H_SEND_LOGICAL_LAN, sc->unit, bufdescs[0], 391 bufdescs[1], bufdescs[2], bufdescs[3], bufdescs[4], bufdescs[5], 0); 392 /* 393 * The hypercall returning implies completion -- or that the call will 394 * not complete. In principle, we should try a few times if we get back 395 * H_BUSY based on the continuation token in R4. For now, just drop 396 * the packet in such cases. 397 */ 398} 399 400static void 401llan_start_locked(struct ifnet *ifp) 402{ 403 struct llan_softc *sc = ifp->if_softc; 404 bus_addr_t first; 405 int nsegs; 406 struct mbuf *mb_head, *m; 407 408 mtx_assert(&sc->io_lock, MA_OWNED); 409 first = 0; 410 411 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 412 IFF_DRV_RUNNING) 413 return; 414 415 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 416 IFQ_DRV_DEQUEUE(&ifp->if_snd, mb_head); 417 418 if (mb_head == NULL) 419 break; 420 421 BPF_MTAP(ifp, mb_head); 422 423 for (m = mb_head, nsegs = 0; m != NULL; m = m->m_next) 424 nsegs++; 425 if (nsegs > 6) { 426 m = m_collapse(mb_head, M_NOWAIT, 6); 427 if (m == NULL) { 428 m_freem(mb_head); 429 continue; 430 } 431 } 432 433 bus_dmamap_load_mbuf(sc->tx_dma_tag, sc->tx_dma_map, 434 mb_head, llan_send_packet, sc, 0); 435 bus_dmamap_unload(sc->tx_dma_tag, sc->tx_dma_map); 436 m_freem(mb_head); 437 } 438} 439 440static void 441llan_start(struct ifnet *ifp) 442{ 443 struct llan_softc *sc = ifp->if_softc; 444 445 mtx_lock(&sc->io_lock); 446 llan_start_locked(ifp); 447 mtx_unlock(&sc->io_lock); 448} 449 450static int 451llan_set_multicast(struct llan_softc *sc) 452{ 453 struct ifnet *ifp = sc->ifp; 454 struct ifmultiaddr *inm; 455 uint64_t macaddr; 456 457 mtx_assert(&sc->io_lock, MA_OWNED); 458 459 phyp_hcall(H_MULTICAST_CTRL, sc->unit, LLAN_CLEAR_MULTICAST, 0); 460 461 if_maddr_rlock(ifp); 462 TAILQ_FOREACH(inm, &ifp->if_multiaddrs, ifma_link) { 463 if (inm->ifma_addr->sa_family != AF_LINK) 464 continue; 465 466 memcpy((uint8_t *)&macaddr + 2, 467 LLADDR((struct sockaddr_dl *)inm->ifma_addr), 6); 468 phyp_hcall(H_MULTICAST_CTRL, sc->unit, LLAN_ADD_MULTICAST, 469 macaddr); 470 } 471 if_maddr_runlock(ifp); 472 473 return (0); 474} 475 476static int 477llan_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 478{ 479 int err = 0; 480 struct llan_softc *sc = ifp->if_softc; 481 482 switch (cmd) { 483 case SIOCADDMULTI: 484 case SIOCDELMULTI: 485 mtx_lock(&sc->io_lock); 486 if ((sc->ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) 487 llan_set_multicast(sc); 488 mtx_unlock(&sc->io_lock); 489 break; 490 case SIOCSIFFLAGS: 491 default: 492 err = ether_ioctl(ifp, cmd, data); 493 break; 494 } 495 496 return (err); 497} 498 499