1/* 2 * Copyright 2017, Data61 3 * Commonwealth Scientific and Industrial Research Organisation (CSIRO) 4 * ABN 41 687 119 230. 5 * 6 * This software may be distributed and modified according to the terms of 7 * the GNU General Public License version 2. Note that NO WARRANTY is provided. 8 * See "LICENSE_GPLv2.txt" for details. 9 * 10 * @TAG(DATA61_GPL) 11 */ 12 13#include <ethdrivers/gen_config.h> 14#include <ethdrivers/intel.h> 15#include <assert.h> 16#include <ethdrivers/helpers.h> 17 18typedef enum e1000_family { 19 e1000_82580 = 1, 20 e1000_82574 21} e1000_family_t; 22 23/* An alignment of 128 bytes is required for most structures by the hardware, except for actual packets */ 24#define DMA_ALIGN 128 25/* This driver is hard coded to use 2k buffers, don't just change this */ 26#define BUF_SIZE 2048 27 28// TX Descriptor Status Bits 29#define TX_DD BIT(0) /* Descriptor Done */ 30/* Descriptor CMD Bits */ 31#define TX_CMD_EOP BIT(0) /* End of Packet */ 32#define TX_CMD_IFCS BIT(1) /* Insert FCS (CRC) */ 33#define TX_CMD_RS BIT(3) /* Report status */ 34#define TX_CMD_IDE BIT(7) /* Interrupt Delay Enable */ 35 36// RX Descriptor Status Bits 37#define RX_DD BIT(0) /* Descriptor Done */ 38#define RX_EOP BIT(1) /* End of Packet */ 39 40#define REG(x,y) (*(volatile uint32_t*)(((uintptr_t)(x)->iobase) + (y))) 41 42#define REG_CTRL(x) REG(x, 0x0) 43#define REG_82580_IMC(x) REG(x, 0x150c) 44#define REG_82574_IMC(x) REG(x, 0xD8) 45#define REG_STATUS(x) REG(x, 0x8) 46#define REG_82580_TXDCTL(x, y) REG(x, 0xE028 + 0x40 * (y)) 47#define REG_82574_TXDCTL(x, y) REG(x, 0x3828 + 0x100 * (y)) 48#define REG_MTA(x, y) REG(x, 0x5200 + 4 * (y)) 49#define REG_RCTL(x) REG(x, 0x100) 50#define REG_EERD(x) REG(x, 0x14) 51#define REG_TCTL(x) REG(x, 0x0400) 52#define REG_RAL(x, y) REG(x, 0x05400 + (y) * 0x8) 53#define REG_RAH(x, y) REG(x, 0x05404 + (y) * 0x8) 54#define REG_82580_TDBAL(x, y) REG(x, 0xE000 + (y) * 0x40) 55#define REG_82574_TDBAL(x, y) REG(x, 0x3800 + (y) * 0x100) 56#define REG_82580_TDBAH(x, y) REG(x, 0xE004 + (y) * 0x40) 57#define REG_82574_TDBAH(x, y) REG(x, 0x3804 + (y) * 0x100) 58#define REG_82580_TDLEN(x, y) REG(x, 0xE008 + (y) * 0x40) 59#define REG_82574_TDLEN(x, y) REG(x, 0x3808 + (y) * 0x100) 60#define REG_82580_TDH(x, y) REG(x, 0xE010 + (y) * 0x40) 61#define REG_82574_TDH(x, y) REG(x, 0x3810 + (y) * 0x100) 62#define REG_82580_TDT(x, y) REG(x, 0xE018 + (y) * 0x40) 63#define REG_82574_TDT(x, y) REG(x, 0x3818 + (y) * 0x100) 64#define REG_82580_RDBAL(x, y) REG(x, 0xC000 + (y) * 0x40) 65#define REG_82574_RDBAL(x, y) REG(x, 0x2800 + (y) * 0x100) 66#define REG_82580_RDBAH(x, y) REG(x, 0xC004 + (y) * 0x40) 67#define REG_82574_RDBAH(x, y) REG(x, 0x2804 + (y) * 0x100) 68#define REG_82580_RDLEN(x, y) REG(x, 0xC008 + (y) * 0x40) 69#define REG_82574_RDLEN(x, y) REG(x, 0x2808 + (y) * 0x100) 70#define REG_82580_RDH(x, y) REG(x, 0xc010 + (y) * 0x40) 71#define REG_82574_RDH(x, y) REG(x, 0x2810 + (y) * 0x100) 72#define REG_82580_RDT(x, y) REG(x, 0xc018 + (y) * 0x40) 73#define REG_82574_RDT(x, y) REG(x, 0x2818 + (y) * 0x100) 74#define REG_82580_RXDCTL(x, y) REG(x, 0xc028 + (y) * 0x40) 75#define REG_82574_RXDCTL(x, y) REG(x, 0x2828 + (y) * 0x100) 76#define REG_82580_IMS(x) REG(x, 0x1508) 77#define REG_82574_IMS(x) REG(x, 0xD0) 78#define REG_82580_ICR(x) REG(x, 0x1500) 79#define REG_82574_ICR(x) REG(x, 0xC0) 80#define REG_TIPG(x) REG(x, 0x410) 81#define REG_82574_RDTR(x) REG(x, 0x2820) 82#define REG_82574_RADV(x) REG(x, 0x282c) 83#define REG_82574_RAID(x) REG(x, 0x2c08) 84#define REG_82574_TIDV(x) REG(x, 0x3820) 85#define REG_82574_TADV(x) REG(x, 0x382c) 86#define REG_82574_PBA(x) REG(x, 0x1000) 87#define REG_MDIC(x) REG(x, 0x20) 88#define REG_82574_GCR(x) REG(x, 0x5B00) 89#define REG_82574_FCT(x) REG(x, 0x030) 90#define REG_82574_FCAL(x) REG(x, 0x028) 91#define REG_82574_FCAH(x) REG(x, 0x02c) 92 93#define IMC_82580_RESERVED_BITS ((uint32_t)(BIT(1) | BIT(3) | BIT(5) | BIT(9) | BIT(15) | BIT(16) | BIT(17) | BIT(21) | BIT(23) | BIT(27) | BIT(31))) 94#define IMC_82574_RESERVED_BITS (BIT(3) | BIT(5) | BIT(8) | (0b11111 << 10) | BIT(19) | (0b1111111 << 25)) 95 96#define CTRL_82580_RESERVED_BITS (BIT(24) | BIT(25)) 97#define CTRL_82574_RESERVED_BITS (BIT(1) | (0b11 << 3) | BIT(7) | BIT(10) | (0b1111111 << 13) | (0b11111 << 21) | BIT(29)) 98#define CTRL_SLU BIT(6) 99#define CTRL_RST BIT(26) 100 101#define STATUS_LU BIT(1) 102 103#define STATUS_82580_LAN_ID_OFFSET 2 104#define STATUS_82580_LAN_ID_MASK (BIT(2) | BIT(3)) 105 106#define RCTL_82580_RESERVED_BITS (BIT(0) | BIT(10) | BIT(11) | BIT(27) | BIT(28) | BIT(29) | BIT(30) | BIT(31)) 107#define RCTL_82574_RESERVED_BITS (BIT(0) | BIT(14) | BIT(21) | BIT(24) | BIT(31)) 108#define RCTL_82574_RDMTS_OFFSET (8) 109#define RCTL_82574_RDMTS_1_4 (0b10 << RCTL_82574_RDMTS_OFFSET) 110#define RCTL_EN BIT(1) 111#define RCTL_UPE BIT(3) 112#define RCTL_MPE BIT(4) 113#define RCTL_BAM BIT(15) 114 115#define TXDCTL_82580_RESERVED_BITS (0) 116#define TXDCTL_82574_RESERVED_BITS (0) 117#define TXDCTL_82580_ENABLE BIT(25) 118#define TXDCTL_82574_BIT_THAT_SHOULD_BE_1 BIT(22) 119#define TXDCTL_82574_GRAN BIT(24) 120#define TXDCTL_82574_PTHRESH_OFFSET 0 121#define TXDCTL_82574_HTHRESH_OFFSET 8 122#define TXDCTL_82574_WTHRESH_OFFSET 16 123 124#define EERD_START BIT(0) 125#define EERD_DONE BIT(1) 126#define EERD_ADDR_OFFSET 2 127 128#define TCTL_82580_RESERVED_BITS (BIT(0) | BIT(2)) 129#define TCTL_82574_RESERVED_BITS (BIT(0) | BIT(2) |BIT(31)) 130#define TCTL_EN BIT(1) 131#define TCTL_PSP BIT(3) 132#define TCTL_CT_BITS(x) ((x) << 4) 133#define TCTL_82574_COLD_BITS(x) (((x) & 0b1111111111) << 12) 134#define TCTL_82574_UNORTX BIT(25) 135#define TCTL_82574_TXDSCMT_BITS(x) (((x) & 3) << 26) 136#define TCTL_82574_RRTHRESH_BITS(x) (((x) & 3) << 29) 137#define TCTL_82574_MULR BIT(28) 138 139#define RXDCTL_82574_RESERVED_BITS (0) 140#define RXDCTL_82574_PTHRESH_OFFSET (0) 141#define RXDCTL_82574_HTHRESH_OFFSET (8) 142#define RXDCTL_82574_WTHRESH_OFFSET (16) 143#define RXDCTL_82574_GRAN BIT(24) 144#define RXDCTL_82580_RESERVED_BITS (0) 145#define RXDCTL_82580_ENABLE BIT(25) 146 147#define IMS_82580_RXDW BIT(7) 148#define IMS_82580_TXDW BIT(0) 149#define IMS_82580_GPHY BIT(10) 150#define IMS_82574_RXQ0 BIT(20) 151#define IMS_82574_RXTO BIT(7) 152#define IMS_82574_RXDMT0 BIT(4) 153#define IMS_82574_TXDW BIT(0) 154#define IMS_82574_ACK BIT(17) 155#define IMS_82574_LSC BIT(2) 156 157#define ICR_82580_RXDW BIT(7) 158#define ICR_82580_TXDW BIT(0) 159#define ICR_82574_RXQ0 BIT(20) 160#define ICR_82580_GPHY BIT(10) 161#define ICR_82574_RXTO BIT(7) 162#define ICR_82574_RXDMT0 BIT(4) 163#define ICR_82574_TXDW BIT(0) 164#define ICR_82574_ACK BIT(17) 165#define ICR_82574_LSC BIT(2) 166 167#define EEPROM_82580_LAN(id, x) ( ((id) ? 0 : 0x40) * (id) + (x)) 168 169#define MTA_LENGTH 128 170 171struct __attribute((packed)) legacy_tx_ldesc { 172 uint64_t bufferAddress; 173 uint32_t length: 16; 174 uint32_t CSO: 8; 175 uint32_t CMD: 8; 176 uint32_t STA: 4; 177 uint32_t ExtCMD : 4; /* This is reserved on the 82580 */ 178 uint32_t CSS: 8; 179 uint32_t VLAN: 16; 180}; 181 182struct __attribute((packed)) legacy_rx_ldesc { 183 uint64_t bufferAddress; 184 uint32_t length: 16; 185 uint32_t packetChecksum: 16; 186 uint32_t status: 8; 187 uint32_t error: 8; 188 uint32_t VLAN: 16; 189}; 190 191typedef struct e1000_dev { 192 e1000_family_t family; 193 void *iobase; 194 /* shadow the value of descriptor tails so we don't have to re-read it to increment */ 195 uint32_t rdt; 196 uint32_t tdt; 197 /* track what we think the values of rdh and tdh are in the hardware so we can find 198 * complete transmit and receive descriptors */ 199 uint32_t tdh; 200 uint32_t rdh; 201 /* descriptor rings */ 202 volatile struct legacy_rx_ldesc *rx_ring; 203 unsigned int rx_size; 204 unsigned int rx_remain; 205 void **rx_cookies; 206 volatile struct legacy_tx_ldesc *tx_ring; 207 unsigned int tx_size; 208 unsigned int tx_remain; 209 void **tx_cookies; 210 unsigned int *tx_lengths; 211 uint32_t tx_cmd_bits; 212 /* whether we believe the link is up or not */ 213 int link_up; 214 /* if the rx ring is empty */ 215 bool need_rx_buffers; 216} e1000_dev_t; 217 218static void disable_all_interrupts(e1000_dev_t *dev) 219{ 220 switch (dev->family) { 221 case e1000_82580: 222 REG_82580_IMC(dev) = ~IMC_82580_RESERVED_BITS; 223 break; 224 case e1000_82574: 225 REG_82574_IMC(dev) = ~IMC_82574_RESERVED_BITS; 226 break; 227 } 228} 229 230static void reset_device(e1000_dev_t *dev) 231{ 232 uint32_t val = CTRL_RST; 233 if (dev->family == e1000_82574) { 234 /* 82574 docs say that bit 3 must be set, so set it and perform the reset */ 235 val |= BIT(3); 236 } 237 REG_CTRL(dev) = val; 238 /* wait approximately 1us before checking for reset completion */ 239 for (volatile int i = 0; i < 10000000; i++); 240 /* wait for reset to complete */ 241 while (REG_CTRL(dev) & CTRL_RST); 242} 243 244static void set_link_up(e1000_dev_t *dev) 245{ 246 uint32_t temp = REG_CTRL(dev); 247 temp |= CTRL_SLU; 248 REG_CTRL(dev) = temp; 249} 250 251static void check_link_status(e1000_dev_t *dev) 252{ 253 dev->link_up = !!(REG_STATUS(dev) & STATUS_LU); 254} 255 256static void configure_pba(e1000_dev_t *dev) 257{ 258 switch (dev->family) { 259 case e1000_82580: 260 /* Leave at defaults */ 261 break; 262 case e1000_82574: 263 /* The default should be 20/20 split for PBA, but set it to that just in case */ 264 REG_82574_PBA(dev) = (20 << 16) | 20; 265 break; 266 default: 267 assert(!"Unknown device"); 268 } 269} 270 271static void phy_write(e1000_dev_t *dev, int phy, int reg, uint16_t data) 272{ 273 REG_MDIC(dev) = data | (reg << 16) | (phy << 21) | (BIT(26)); 274 while ((REG_MDIC(dev) & BIT(28)) == 0); 275} 276 277static uint16_t phy_read(e1000_dev_t *dev, int phy, int reg) 278{ 279 uint32_t mdi; 280 REG_MDIC(dev) = (reg << 16) | (phy << 21) | (2 << 26); 281 while (((mdi = REG_MDIC(dev)) & BIT(28)) == 0); 282 return mdi & MASK(16); 283} 284 285static void reset_phy(e1000_dev_t *dev) 286{ 287 uint32_t temp; 288 switch (dev->family) { 289 case e1000_82580: 290 /* Don't reset the phy for now */ 291 break; 292 case e1000_82574: 293 /* for unknown reasons the 82574 this was tested on cannot seem to perform 294 * well if it ends up being the slave side of a connection. Therefore we 295 * try and force us to negotiate to be the master */ 296 temp = phy_read(dev, 1, 9); 297 temp |= BIT(12) | BIT(11); 298 phy_write(dev, 1, 9, temp); 299 /* write a reset */ 300 phy_write(dev, 1, 0, BIT(15) | BIT(12)); 301 /* wait until it completes */ 302 while (phy_read(dev, 1, 0) & BIT(15)); 303 break; 304 default: 305 assert(!"Unknown device"); 306 } 307} 308 309static void configure_flow_control(e1000_dev_t *dev) 310{ 311 switch (dev->family) { 312 case e1000_82580: 313 /* Nothing to setup. Default values will allow us to receive 314 * pause frames, we don't send them however */ 315 break; 316 case e1000_82574: 317 /* Manual says to set these values for flow control */ 318 REG_82574_FCAL(dev) = 0x00C28001; 319 REG_82574_FCAH(dev) = 0x0100; 320 REG_82574_FCT(dev) = 0x8808; 321 break; 322 default: 323 assert(!"Unknown device"); 324 } 325} 326 327static void initialize(e1000_dev_t *dev) 328{ 329 disable_all_interrupts(dev); 330 reset_device(dev); 331 disable_all_interrupts(dev); 332 333 if (dev->family == e1000_82574) { 334 /* this bit must be set according to the manual */ 335 REG_82574_GCR(dev) |= BIT(22); 336 } 337 338 reset_phy(dev); 339 340 set_link_up(dev); 341 342 configure_flow_control(dev); 343 configure_pba(dev); 344} 345 346static void initialise_TXDCTL(e1000_dev_t *dev) 347{ 348 uint32_t temp; 349 switch (dev->family) { 350 case e1000_82580: 351 /* Enable transmit queue */ 352 temp = REG_82580_TXDCTL(dev, 0); 353 temp &= ~TXDCTL_82580_RESERVED_BITS; 354 temp |= TXDCTL_82580_ENABLE; 355 REG_82580_TXDCTL(dev, 0) = temp; 356 break; 357 case e1000_82574: 358 temp = REG_82574_TXDCTL(dev, 0); 359 temp &= ~TXDCTL_82574_RESERVED_BITS; 360 /* set the bit that we have to set */ 361 temp |= TXDCTL_82574_BIT_THAT_SHOULD_BE_1; 362 /* set gran to 1 */ 363 temp |= TXDCTL_82574_GRAN; 364 /* wthresh and hthresh to 1 to avoid tx stalls */ 365 temp |= 1 << TXDCTL_82574_WTHRESH_OFFSET; 366 temp |= 1 << TXDCTL_82574_HTHRESH_OFFSET; 367 /* prefetch when less than 31 */ 368 temp |= 31 << TXDCTL_82574_PTHRESH_OFFSET; 369 REG_82574_TXDCTL(dev, 0) = temp; 370 break; 371 default: 372 assert(!"Unknown device"); 373 break; 374 } 375} 376 377static void initialise_TCTL(e1000_dev_t *dev) 378{ 379 uint32_t temp = 0; 380 switch (dev->family) { 381 case e1000_82580: 382 break; 383 case e1000_82574: 384 /* set cold to 0x3f for FD (or 0x1FF for HD) */ 385 temp |= TCTL_82574_COLD_BITS(0x3f); 386 /* allow multiple transmit requests from hardware at once */ 387 temp |= TCTL_82574_MULR; 388 break; 389 default: 390 assert(!"Unknown device"); 391 } 392 temp |= TCTL_EN; 393 temp |= TCTL_PSP; 394 temp |= TCTL_CT_BITS(0xf); 395 REG_TCTL(dev) = temp; 396} 397 398static void initialise_TIPG(e1000_dev_t *dev) 399{ 400 switch (dev->family) { 401 case e1000_82580: 402 /* use defaults */ 403 break; 404 case e1000_82574: 405 /* Write in recommended value from manual */ 406 REG_TIPG(dev) = 0x00602006; 407 break; 408 default: 409 assert(!"Unknown device"); 410 } 411} 412 413static void initialise_transmit_timers(e1000_dev_t *dev) 414{ 415 switch (dev->family) { 416 case e1000_82580: 417 break; 418 case e1000_82574: 419 /* Set the transmit notifcations really high. We will cleanup transmits 420 * lazily for the most part and there is no real rush to release buffers quickly */ 421 REG_82574_TIDV(dev) = 20000; 422 REG_82574_TADV(dev) = 60000; 423 break; 424 default: 425 assert(!"Unknown device"); 426 } 427} 428 429static void initialize_transmit(e1000_dev_t *dev) 430{ 431 initialise_TXDCTL(dev); 432 initialise_TIPG(dev); 433 initialise_transmit_timers(dev); 434 initialise_TCTL(dev); 435} 436 437static void initialize_RCTL(e1000_dev_t *dev) 438{ 439 uint32_t temp = 0; 440 switch (dev->family) { 441 case e1000_82580: 442 break; 443 case e1000_82574: 444 /* set free receive descriptor threshold to one quarter */ 445 temp |= RCTL_82574_RDMTS_1_4; 446 break; 447 } 448 /* Enable receive */ 449 temp |= RCTL_EN; 450 /* Accept broadcast packets */ 451 temp |= RCTL_BAM; 452 /* defaults for everything else will give us 2K pages, which is what we want */ 453 REG_RCTL(dev) = temp; 454} 455 456static void enable_prom_mode(e1000_dev_t *dev) 457{ 458 REG_RCTL(dev) |= RCTL_UPE | RCTL_MPE; 459} 460 461static void initialize_RXDCTL(e1000_dev_t *dev) 462{ 463 uint32_t temp; 464 switch (dev->family) { 465 case e1000_82580: 466 temp = REG_82580_RXDCTL(dev, 0); 467 temp &= ~RXDCTL_82580_RESERVED_BITS; 468 temp |= RXDCTL_82580_ENABLE; 469 REG_82580_RXDCTL(dev, 0) = temp; 470 break; 471 case e1000_82574: 472 temp = REG_82574_RXDCTL(dev, 0); 473 temp &= ~RXDCTL_82574_RESERVED_BITS; 474 /* count in descriptors */ 475 temp |= RXDCTL_82574_GRAN; 476 /* prefetch once below 4 */ 477 temp |= 4 << RXDCTL_82574_PTHRESH_OFFSET; 478 /* keep host at 32 free */ 479 temp |= 32 << RXDCTL_82574_HTHRESH_OFFSET; 480 /* write back 4 at a time */ 481 temp |= 4 << RXDCTL_82574_WTHRESH_OFFSET; 482 REG_82574_TXDCTL(dev, 0) = temp; 483 break; 484 default: 485 assert(!"Unknown device"); 486 } 487} 488 489static void initialize_receive_timers(e1000_dev_t *dev) 490{ 491 switch (dev->family) { 492 case e1000_82580: 493 break; 494 case e1000_82574: 495 /* set a base delay of 20 microseconds */ 496 REG_82574_RDTR(dev) = 20; 497 /* force descriptor write back after 20 microseconds */ 498 REG_82574_RADV(dev) = 20; 499 REG_82574_RAID(dev) = 0; 500 break; 501 default: 502 assert(!"Unknown device"); 503 } 504} 505 506static void initialize_receive(e1000_dev_t *dev) 507{ 508 /* zero the MTA */ 509 int i; 510 for (i = 0; i < MTA_LENGTH; i++) { 511 REG_MTA(dev, i) = 0; 512 } 513 initialize_receive_timers(dev); 514 initialize_RXDCTL(dev); 515 initialize_RCTL(dev); 516} 517 518static void enable_interrupts(e1000_dev_t *dev) 519{ 520 switch (dev->family) { 521 case e1000_82580: 522 REG_82580_IMS(dev) = IMS_82580_RXDW | IMS_82580_TXDW | IMS_82580_GPHY; 523 /* enable link status change interrupts in the phy */ 524 phy_write(dev, 0, 24, BIT(2)); 525 break; 526 case e1000_82574: 527 REG_82574_IMS(dev) = IMS_82574_RXQ0 | IMS_82574_RXTO | IMS_82574_RXDMT0 | IMS_82574_ACK | IMS_82574_TXDW | 528 IMS_82574_LSC; 529 break; 530 default: 531 assert(!"Unknown device"); 532 break; 533 } 534} 535 536void print_state(struct eth_driver *eth_driver) 537{ 538} 539 540static uint16_t read_eeprom(e1000_dev_t *dev, uint16_t reg) 541{ 542 REG_EERD(dev) = EERD_START | (reg << EERD_ADDR_OFFSET); 543 uint32_t val; 544 while (((val = REG_EERD(dev)) & EERD_DONE) == 0); 545 return val >> 16; 546} 547 548void eth_get_mac(e1000_dev_t *dev, uint8_t *hwaddr) 549{ 550 /* read the first 3 shorts of the eeprom */ 551 uint16_t mac[3]; 552 uint16_t base = 0x00; 553 switch (dev->family) { 554 case e1000_82580: { 555 /* read our LAN ID so we know what port we are */ 556 int id = (REG_STATUS(dev) & STATUS_82580_LAN_ID_MASK) >> STATUS_82580_LAN_ID_OFFSET; 557 base = EEPROM_82580_LAN(id, 0x00); 558 break; 559 } 560 case e1000_82574: 561 /* Single port, so default base is fine */ 562 break; 563 default: 564 assert(!"Unknown device"); 565 return; 566 } 567 mac[0] = read_eeprom(dev, base + 0x00); 568 mac[1] = read_eeprom(dev, base + 0x01); 569 mac[2] = read_eeprom(dev, base + 0x02); 570 hwaddr[0] = mac[0] & MASK(8); 571 hwaddr[1] = mac[0] >> 8; 572 hwaddr[2] = mac[1] & MASK(8); 573 hwaddr[3] = mac[1] >> 8; 574 hwaddr[4] = mac[2] & MASK(8); 575 hwaddr[5] = mac[2] >> 8; 576} 577 578void low_level_init(struct eth_driver *driver, uint8_t *mac, int *mtu) 579{ 580 e1000_dev_t *dev = (e1000_dev_t *)driver->eth_data; 581 eth_get_mac(dev, mac); 582 /* hardcode MTU for now */ 583 *mtu = 1500; 584} 585 586void get_mac(struct eth_driver *driver, uint8_t *mac) 587{ 588 e1000_dev_t *dev = (e1000_dev_t *)driver->eth_data; 589 uint32_t maclow = REG_RAL(dev, 0); 590 uint32_t machigh = REG_RAH(dev, 0); 591 592 mac[0] = maclow; 593 mac[1] = maclow >> 8; 594 mac[2] = maclow >> 16; 595 mac[3] = maclow >> 24; 596 597 mac[4] = machigh; 598 mac[5] = machigh >> 8; 599} 600 601static void set_tx_ring(e1000_dev_t *dev, uintptr_t phys) 602{ 603 uint32_t phys_low = (uint32_t)phys; 604 uint32_t phys_high = (uint32_t)(sizeof(phys) > 4 ? phys >> 32 : 0); 605 switch (dev->family) { 606 case e1000_82580: 607 REG_82580_TDBAL(dev, 0) = phys_low; 608 REG_82580_TDBAH(dev, 0) = phys_high; 609 break; 610 case e1000_82574: 611 REG_82574_TDBAL(dev, 0) = phys_low; 612 REG_82574_TDBAH(dev, 0) = phys_high; 613 break; 614 default: 615 assert(!"Unknown device"); 616 } 617} 618 619static void set_tdh(e1000_dev_t *dev, uint32_t val) 620{ 621 switch (dev->family) { 622 case e1000_82580: 623 REG_82580_TDH(dev, 0) = val; 624 break; 625 case e1000_82574: 626 REG_82574_TDH(dev, 0) = val; 627 break; 628 default: 629 assert(!"Unknown device"); 630 } 631} 632 633static void set_tdt(e1000_dev_t *dev, uint32_t val) 634{ 635 switch (dev->family) { 636 case e1000_82580: 637 REG_82580_TDT(dev, 0) = val; 638 break; 639 case e1000_82574: 640 REG_82574_TDT(dev, 0) = val; 641 break; 642 default: 643 assert(!"Unknown device"); 644 } 645} 646 647static void set_tdlen(e1000_dev_t *dev, uint32_t val) 648{ 649 /* tdlen must be multiple of 128 */ 650 assert(val % 128 == 0); 651 switch (dev->family) { 652 case e1000_82580: 653 REG_82580_TDLEN(dev, 0) = val; 654 break; 655 case e1000_82574: 656 REG_82574_TDLEN(dev, 0) = val; 657 break; 658 default: 659 assert(!"Unknown device"); 660 } 661} 662 663static void set_rx_ring(e1000_dev_t *dev, uint64_t phys) 664{ 665 uint32_t phys_low = (uint32_t)phys; 666 uint32_t phys_high = (uint32_t)(sizeof(phys) > 4 ? phys >> 32 : 0); 667 switch (dev->family) { 668 case e1000_82580: 669 REG_82580_RDBAL(dev, 0) = phys_low; 670 REG_82580_RDBAH(dev, 0) = phys_high; 671 break; 672 case e1000_82574: 673 REG_82574_RDBAL(dev, 0) = phys_low; 674 REG_82574_RDBAH(dev, 0) = phys_high; 675 break; 676 default: 677 assert(!"Unknown device"); 678 } 679} 680 681static void set_rdlen(e1000_dev_t *dev, uint32_t val) 682{ 683 /* rdlen must be multiple of 128 */ 684 assert(val % 128 == 0); 685 switch (dev->family) { 686 case e1000_82580: 687 REG_82580_RDLEN(dev, 0) = val; 688 break; 689 case e1000_82574: 690 REG_82574_RDLEN(dev, 0) = val; 691 break; 692 default: 693 assert(!"Unknown device"); 694 } 695} 696 697static void set_rdt(e1000_dev_t *dev, uint32_t val) 698{ 699 switch (dev->family) { 700 case e1000_82580: 701 REG_82580_RDT(dev, 0) = val; 702 break; 703 case e1000_82574: 704 REG_82574_RDT(dev, 0) = val; 705 break; 706 default: 707 assert(!"Unknown device"); 708 } 709} 710 711static uint32_t read_rdh(e1000_dev_t *dev) 712{ 713 switch (dev->family) { 714 case e1000_82580: 715 return REG_82580_RDH(dev, 0); 716 case e1000_82574: 717 return REG_82574_RDH(dev, 0); 718 default: 719 assert(!"Unknown device"); 720 return 0; 721 } 722} 723 724static void free_desc_ring(e1000_dev_t *dev, ps_dma_man_t *dma_man) 725{ 726 if (dev->rx_ring) { 727 dma_unpin_free(dma_man, (void *)dev->rx_ring, sizeof(struct legacy_rx_ldesc) * dev->rx_size); 728 dev->rx_ring = NULL; 729 } 730 if (dev->tx_ring) { 731 dma_unpin_free(dma_man, (void *)dev->tx_ring, sizeof(struct legacy_tx_ldesc) * dev->tx_size); 732 dev->tx_ring = NULL; 733 } 734 if (dev->rx_cookies) { 735 free(dev->rx_cookies); 736 dev->rx_cookies = NULL; 737 } 738 if (dev->tx_cookies) { 739 free(dev->tx_cookies); 740 dev->tx_cookies = NULL; 741 } 742 if (dev->tx_lengths) { 743 free(dev->tx_lengths); 744 dev->tx_lengths = NULL; 745 } 746} 747 748static int initialize_desc_ring(e1000_dev_t *dev, ps_dma_man_t *dma_man) 749{ 750 dma_addr_t rx_ring = dma_alloc_pin(dma_man, sizeof(struct legacy_rx_ldesc) * dev->rx_size, 1, DMA_ALIGN); 751 if (!rx_ring.phys) { 752 LOG_ERROR("Failed to allocate rx_ring"); 753 return -1; 754 } 755 dev->rx_ring = rx_ring.virt; 756 dma_addr_t tx_ring = dma_alloc_pin(dma_man, sizeof(struct legacy_tx_ldesc) * dev->tx_size, 1, DMA_ALIGN); 757 if (!tx_ring.phys) { 758 LOG_ERROR("Failed to allocate tx_ring"); 759 free_desc_ring(dev, dma_man); 760 return -1; 761 } 762 dev->rx_cookies = malloc(sizeof(void *) * dev->rx_size); 763 dev->tx_cookies = malloc(sizeof(void *) * dev->tx_size); 764 dev->tx_lengths = malloc(sizeof(unsigned int) * dev->tx_size); 765 if (!dev->rx_cookies || !dev->tx_cookies || !dev->tx_lengths) { 766 if (dev->rx_cookies) { 767 free(dev->rx_cookies); 768 } 769 if (dev->tx_cookies) { 770 free(dev->tx_cookies); 771 } 772 if (dev->tx_lengths) { 773 free(dev->tx_lengths); 774 } 775 LOG_ERROR("Failed to malloc"); 776 free_desc_ring(dev, dma_man); 777 return -1; 778 } 779 dev->tx_ring = tx_ring.virt; 780 /* Remaining needs to be 2 less than size as we cannot actually enqueue size many descriptors, 781 * since then the head and tail pointers would be equal, indicating empty. */ 782 dev->rx_remain = dev->rx_size - 2; 783 dev->tx_remain = dev->tx_size - 2; 784 785 /* Tell the hardware where the rings are and now big they are */ 786 set_tx_ring(dev, tx_ring.phys); 787 set_tdlen(dev, dev->tx_size * sizeof(struct legacy_tx_ldesc)); 788 set_rx_ring(dev, rx_ring.phys); 789 set_rdlen(dev, dev->rx_size * sizeof(struct legacy_rx_ldesc)); 790 791 /* Set transmit ring initially empty */ 792 dev->tdh = dev->tdt = 0; 793 set_tdh(dev, dev->tdh); 794 set_tdt(dev, dev->tdt); 795 796 /* Set receive ring initially empty */ 797 dev->rdh = dev->rdt = read_rdh(dev); 798 set_rdt(dev, dev->rdt); 799 800 return 0; 801} 802 803static void complete_rx(struct eth_driver *driver) 804{ 805 e1000_dev_t *dev = (e1000_dev_t *)driver->eth_data; 806 if (dev->rdh == dev->rdt) { 807 /* We haven't enqueued anything */ 808 return; 809 } 810 unsigned int i, j; 811 unsigned int count = 1; 812 unsigned int rdt = dev->rdt; 813 for (i = dev->rdh; i != rdt; i = (i + 1) % dev->rx_size, count++) { 814 unsigned int status = dev->rx_ring[i].status; 815 /* Ensure no memory references get ordered before we checked the descriptor was written back */ 816 asm volatile("lfence" ::: "memory"); 817 if (!(status & RX_DD)) { 818 /* not complete yet */ 819 break; 820 } 821 if (status & RX_EOP) { 822 void *cookies[count]; 823 unsigned int len[count]; 824 for (j = 0; j < count; j++) { 825 cookies[j] = dev->rx_cookies[(dev->rdh + j) % dev->rx_size]; 826 len[j] = dev->rx_ring[(dev->rdh + j) % dev->rx_size].length; 827 } 828 /* update rdh */ 829 dev->rdh = (dev->rdh + count) % dev->rx_size; 830 dev->rx_remain += count; 831 /* Give the buffers back */ 832 driver->i_cb.rx_complete(driver->cb_cookie, count, cookies, len); 833 count = 0; 834 } 835 } 836} 837 838static void complete_tx(struct eth_driver *driver) 839{ 840 e1000_dev_t *dev = (e1000_dev_t *)driver->eth_data; 841 while (dev->tdh != dev->tdt) { 842 unsigned int i; 843 for (i = 0; i < dev->tx_lengths[dev->tdh]; i++) { 844 if (!(dev->tx_ring[(i + dev->tdh) % dev->tx_size].STA & TX_DD)) { 845 /* not all parts complete */ 846 return; 847 } 848 } 849 /* do not let memory loads happen before our checking of the descriptor write back */ 850 asm volatile("lfence" ::: "memory"); 851 /* increase where we believe tdh to be */ 852 void *cookie = dev->tx_cookies[dev->tdh]; 853 dev->tx_remain += dev->tx_lengths[dev->tdh]; 854 dev->tdh = (dev->tdh + dev->tx_lengths[dev->tdh]) % dev->tx_size; 855 /* give the buffer back */ 856 driver->i_cb.tx_complete(driver->cb_cookie, cookie); 857 } 858} 859 860static int raw_tx(struct eth_driver *driver, unsigned int num, uintptr_t *phys, unsigned int *len, void *cookie) 861{ 862 e1000_dev_t *dev = (e1000_dev_t *)driver->eth_data; 863 if (!dev->link_up) { 864 return ETHIF_TX_FAILED; 865 } 866 /* Ensure we have room */ 867 if (dev->tx_remain < num) { 868 /* try and complete some */ 869 complete_tx(driver); 870 if (dev->tx_remain < num) { 871 return ETHIF_TX_FAILED; 872 } 873 } 874 unsigned int i; 875 for (i = 0; i < num; i++) { 876 dev->tx_ring[(dev->tdt + i) % dev->tx_size] = (struct legacy_tx_ldesc) { 877 .bufferAddress = phys[i], 878 .length = len[i], 879 .CSO = 0, 880 .CMD = dev->tx_cmd_bits | (i + 1 == num ? TX_CMD_EOP : 0), 881 .STA = 0, 882 .ExtCMD = 0, 883 .CSS = 0, 884 .VLAN = 0 885 }; 886 } 887 dev->tx_cookies[dev->tdt] = cookie; 888 dev->tx_lengths[dev->tdt] = num; 889 /* ensure update to descriptors visible before updating tdt */ 890 asm volatile("mfence" ::: "memory"); 891 dev->tdt = (dev->tdt + num) % dev->tx_size; 892 dev->tx_remain -= num; 893 set_tdt(dev, dev->tdt); 894 return ETHIF_TX_ENQUEUED; 895} 896 897static int fill_rx_bufs(struct eth_driver *driver) 898{ 899 e1000_dev_t *dev = (e1000_dev_t *)driver->eth_data; 900 int rdt = dev->rdt; 901 /* We want to install buffers in bursts for performance reasons. 902 * constantly enqueueing single buffers is expensive */ 903 if (dev->rx_remain < 32) { 904 return 0; 905 } 906 while (dev->rx_remain > 0) { 907 /* request a buffer */ 908 void *cookie; 909 uintptr_t phys = driver->i_cb.allocate_rx_buf ? driver->i_cb.allocate_rx_buf(driver->cb_cookie, BUF_SIZE, &cookie) : 0; 910 if (!phys) { 911 dev->need_rx_buffers = true; 912 break; 913 } 914 dev->rx_cookies[dev->rdt] = cookie; 915 /* zery the descriptor */ 916 dev->rx_ring[dev->rdt] = (struct legacy_rx_ldesc) { 917 .bufferAddress = phys, 918 .length = BUF_SIZE, 919 .packetChecksum = 0, 920 .status = 0, 921 .error = 0, 922 .VLAN = 0 923 }; 924 dev->rdt = (dev->rdt + 1) % dev->rx_size; 925 dev->rx_remain--; 926 } 927 if (dev->rdt != rdt) { 928 /* ensure update to descriptor visible before updating rdt */ 929 asm volatile("sfence" ::: "memory"); 930 set_rdt(dev, dev->rdt); 931 } 932 return dev->rx_remain != 0; 933} 934 935static void raw_poll(struct eth_driver *driver) 936{ 937 e1000_dev_t *dev = (e1000_dev_t *)driver->eth_data; 938 if (dev->need_rx_buffers) { 939 dev->need_rx_buffers = false; 940 fill_rx_bufs(driver); 941 } 942 complete_rx(driver); 943 complete_tx(driver); 944 fill_rx_bufs(driver); 945 check_link_status(driver->eth_data); 946} 947 948static void handle_irq(struct eth_driver *driver, int irq) 949{ 950 e1000_dev_t *dev = (e1000_dev_t *)driver->eth_data; 951 uint32_t icr; 952 switch (dev->family) { 953 case e1000_82580: 954 icr = REG_82580_ICR(dev); 955 if (icr & ICR_82580_RXDW) { 956 complete_rx(driver); 957 fill_rx_bufs(driver); 958 } 959 if (icr & ICR_82580_TXDW) { 960 complete_tx(driver); 961 } 962 if (icr & ICR_82580_GPHY) { 963 uint32_t phy = phy_read(dev, 0, 25); 964 if (phy & BIT(3)) { 965 check_link_status(dev); 966 } 967 } 968 break; 969 case e1000_82574: 970 icr = REG_82574_ICR(dev); 971 /* ack */ 972 REG_82574_ICR(dev) = icr; 973 if (icr & (ICR_82574_RXQ0 | ICR_82574_RXTO | ICR_82574_ACK | ICR_82574_RXDMT0)) { 974 complete_rx(driver); 975 fill_rx_bufs(driver); 976 } 977 if (icr & ICR_82574_TXDW) { 978 complete_tx(driver); 979 } 980 if (icr & ICR_82574_LSC) { 981 check_link_status(dev); 982 if (!dev->link_up) { 983 /* should probably remove everything from the TX ring here */ 984 } 985 } 986 break; 987 default: 988 assert(!"Unknown device"); 989 } 990} 991 992static struct raw_iface_funcs iface_fns = { 993 .raw_handleIRQ = handle_irq, 994 .print_state = print_state, 995 .low_level_init = low_level_init, 996 .raw_tx = raw_tx, 997 .raw_poll = raw_poll, 998 .get_mac = get_mac 999}; 1000 1001static void eth_irq_handle(void *data, ps_irq_acknowledge_fn_t acknowledge_fn, void *ack_data) 1002{ 1003 1004 struct eth_driver *eth = data; 1005 1006 handle_irq(eth, 0); 1007 1008 int error = acknowledge_fn(ack_data); 1009 if (error) { 1010 LOG_ERROR("Failed to acknowledge IRQ"); 1011 } 1012 1013} 1014 1015static int common_init(struct eth_driver *driver, ps_io_ops_t io_ops, void *config, e1000_dev_t *dev) 1016{ 1017 int err; 1018 ethif_intel_config_t *eth_config = (ethif_intel_config_t *) config; 1019 dev->iobase = eth_config->bar0; 1020 dev->tx_size = CONFIG_LIB_ETHDRIVER_TX_DESC_COUNT; 1021 dev->rx_size = CONFIG_LIB_ETHDRIVER_RX_DESC_COUNT; 1022 1023 /* technically we support alignemtn of 1, but get better performance with some alignment */ 1024 driver->dma_alignment = 16; 1025 driver->eth_data = dev; 1026 driver->i_fn = iface_fns; 1027 1028 initialize(dev); 1029 err = initialize_desc_ring(dev, &io_ops.dma_manager); 1030 if (err) { 1031 /* Reset device */ 1032 disable_all_interrupts(dev); 1033 reset_device(dev); 1034 /* Free memory */ 1035 free(dev); 1036 return -1; 1037 } 1038 1039 /* If num_irqs are 0 then we assume that this driver is either polled or some external environment 1040 * will call raw_handleIRQ. 1041 */ 1042 if (eth_config->num_irqs == 1) { 1043 irq_id_t irq_id = ps_irq_register(&io_ops.irq_ops, eth_config->irq_info[0], eth_irq_handle, driver); 1044 if (irq_id < 0) { 1045 LOG_ERROR("Failed to register IRQ"); 1046 return -1; 1047 } 1048 1049 } 1050 1051 /* the transmit and receive initialization functions assume 1052 * that we have setup descriptor rings for the transmit receive queues */ 1053 initialize_transmit(dev); 1054 initialize_receive(dev); 1055 1056 if (eth_config->prom_mode) { 1057 enable_prom_mode(dev); 1058 } 1059 1060 /* fill up the receive ring as much as possible */ 1061 fill_rx_bufs(driver); 1062 /* turn interrupts on */ 1063 enable_interrupts(dev); 1064 /* check the current status of the link */ 1065 check_link_status(dev); 1066 return ps_interface_register(&io_ops.interface_registration_ops, PS_ETHERNET_INTERFACE, driver, NULL); 1067} 1068 1069int ethif_e82580_init(struct eth_driver *driver, ps_io_ops_t io_ops, void *config) 1070{ 1071 e1000_dev_t *dev = malloc(sizeof(*dev)); 1072 if (!dev) { 1073 LOG_ERROR("Failed to malloc"); 1074 return -1; 1075 } 1076 dev->family = e1000_82580; 1077 dev->tx_cmd_bits = TX_CMD_IFCS | TX_CMD_RS; 1078 return common_init(driver, io_ops, config, dev); 1079} 1080 1081int ethif_e82574_init(struct eth_driver *driver, ps_io_ops_t io_ops, void *config) 1082{ 1083 e1000_dev_t *dev = malloc(sizeof(*dev)); 1084 if (!dev) { 1085 LOG_ERROR("Failed to malloc"); 1086 return -1; 1087 } 1088 dev->family = e1000_82574; 1089 dev->tx_cmd_bits = TX_CMD_IFCS | TX_CMD_RS | TX_CMD_IDE; 1090 return common_init(driver, io_ops, config, dev); 1091} 1092