1/* 2 * 3 * Copyright (c) 2009, Microsoft Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 * You should have received a copy of the GNU General Public License along with 15 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple 16 * Place - Suite 330, Boston, MA 02111-1307 USA. 17 * 18 * Authors: 19 * Haiyang Zhang <haiyangz@microsoft.com> 20 * Hank Janssen <hjanssen@microsoft.com> 21 * 22 */ 23 24#include <linux/kernel.h> 25#include <linux/mm.h> 26#include "osd.h" 27#include "logging.h" 28#include "ring_buffer.h" 29 30 31/* #defines */ 32 33 34/* Amount of space to write to */ 35#define BYTES_AVAIL_TO_WRITE(r, w, z) ((w) >= (r)) ? ((z) - ((w) - (r))) : ((r) - (w)) 36 37 38/*++ 39 40Name: 41 GetRingBufferAvailBytes() 42 43Description: 44 Get number of bytes available to read and to write to 45 for the specified ring buffer 46 47--*/ 48static inline void 49GetRingBufferAvailBytes(struct hv_ring_buffer_info *rbi, u32 *read, u32 *write) 50{ 51 u32 read_loc, write_loc; 52 53 /* Capture the read/write indices before they changed */ 54 read_loc = rbi->RingBuffer->ReadIndex; 55 write_loc = rbi->RingBuffer->WriteIndex; 56 57 *write = BYTES_AVAIL_TO_WRITE(read_loc, write_loc, rbi->RingDataSize); 58 *read = rbi->RingDataSize - *write; 59} 60 61/*++ 62 63Name: 64 GetNextWriteLocation() 65 66Description: 67 Get the next write location for the specified ring buffer 68 69--*/ 70static inline u32 71GetNextWriteLocation(struct hv_ring_buffer_info *RingInfo) 72{ 73 u32 next = RingInfo->RingBuffer->WriteIndex; 74 75 /* ASSERT(next < RingInfo->RingDataSize); */ 76 77 return next; 78} 79 80/*++ 81 82Name: 83 SetNextWriteLocation() 84 85Description: 86 Set the next write location for the specified ring buffer 87 88--*/ 89static inline void 90SetNextWriteLocation(struct hv_ring_buffer_info *RingInfo, 91 u32 NextWriteLocation) 92{ 93 RingInfo->RingBuffer->WriteIndex = NextWriteLocation; 94} 95 96/*++ 97 98Name: 99 GetNextReadLocation() 100 101Description: 102 Get the next read location for the specified ring buffer 103 104--*/ 105static inline u32 106GetNextReadLocation(struct hv_ring_buffer_info *RingInfo) 107{ 108 u32 next = RingInfo->RingBuffer->ReadIndex; 109 110 /* ASSERT(next < RingInfo->RingDataSize); */ 111 112 return next; 113} 114 115/*++ 116 117Name: 118 GetNextReadLocationWithOffset() 119 120Description: 121 Get the next read location + offset for the specified ring buffer. 122 This allows the caller to skip 123 124--*/ 125static inline u32 126GetNextReadLocationWithOffset(struct hv_ring_buffer_info *RingInfo, u32 Offset) 127{ 128 u32 next = RingInfo->RingBuffer->ReadIndex; 129 130 /* ASSERT(next < RingInfo->RingDataSize); */ 131 next += Offset; 132 next %= RingInfo->RingDataSize; 133 134 return next; 135} 136 137/*++ 138 139Name: 140 SetNextReadLocation() 141 142Description: 143 Set the next read location for the specified ring buffer 144 145--*/ 146static inline void 147SetNextReadLocation(struct hv_ring_buffer_info *RingInfo, u32 NextReadLocation) 148{ 149 RingInfo->RingBuffer->ReadIndex = NextReadLocation; 150} 151 152 153/*++ 154 155Name: 156 GetRingBuffer() 157 158Description: 159 Get the start of the ring buffer 160 161--*/ 162static inline void * 163GetRingBuffer(struct hv_ring_buffer_info *RingInfo) 164{ 165 return (void *)RingInfo->RingBuffer->Buffer; 166} 167 168 169/*++ 170 171Name: 172 GetRingBufferSize() 173 174Description: 175 Get the size of the ring buffer 176 177--*/ 178static inline u32 179GetRingBufferSize(struct hv_ring_buffer_info *RingInfo) 180{ 181 return RingInfo->RingDataSize; 182} 183 184/*++ 185 186Name: 187 GetRingBufferIndices() 188 189Description: 190 Get the read and write indices as u64 of the specified ring buffer 191 192--*/ 193static inline u64 194GetRingBufferIndices(struct hv_ring_buffer_info *RingInfo) 195{ 196 return (u64)RingInfo->RingBuffer->WriteIndex << 32; 197} 198 199 200/*++ 201 202Name: 203 DumpRingInfo() 204 205Description: 206 Dump out to console the ring buffer info 207 208--*/ 209void DumpRingInfo(struct hv_ring_buffer_info *RingInfo, char *Prefix) 210{ 211 u32 bytesAvailToWrite; 212 u32 bytesAvailToRead; 213 214 GetRingBufferAvailBytes(RingInfo, 215 &bytesAvailToRead, 216 &bytesAvailToWrite); 217 218 DPRINT(VMBUS, 219 DEBUG_RING_LVL, 220 "%s <<ringinfo %p buffer %p avail write %u " 221 "avail read %u read idx %u write idx %u>>", 222 Prefix, 223 RingInfo, 224 RingInfo->RingBuffer->Buffer, 225 bytesAvailToWrite, 226 bytesAvailToRead, 227 RingInfo->RingBuffer->ReadIndex, 228 RingInfo->RingBuffer->WriteIndex); 229} 230 231 232/* Internal routines */ 233 234static u32 235CopyToRingBuffer( 236 struct hv_ring_buffer_info *RingInfo, 237 u32 StartWriteOffset, 238 void *Src, 239 u32 SrcLen); 240 241static u32 242CopyFromRingBuffer( 243 struct hv_ring_buffer_info *RingInfo, 244 void *Dest, 245 u32 DestLen, 246 u32 StartReadOffset); 247 248 249 250/*++ 251 252Name: 253 RingBufferGetDebugInfo() 254 255Description: 256 Get various debug metrics for the specified ring buffer 257 258--*/ 259void RingBufferGetDebugInfo(struct hv_ring_buffer_info *RingInfo, 260 struct hv_ring_buffer_debug_info *debug_info) 261{ 262 u32 bytesAvailToWrite; 263 u32 bytesAvailToRead; 264 265 if (RingInfo->RingBuffer) { 266 GetRingBufferAvailBytes(RingInfo, 267 &bytesAvailToRead, 268 &bytesAvailToWrite); 269 270 debug_info->BytesAvailToRead = bytesAvailToRead; 271 debug_info->BytesAvailToWrite = bytesAvailToWrite; 272 debug_info->CurrentReadIndex = RingInfo->RingBuffer->ReadIndex; 273 debug_info->CurrentWriteIndex = RingInfo->RingBuffer->WriteIndex; 274 debug_info->CurrentInterruptMask = RingInfo->RingBuffer->InterruptMask; 275 } 276} 277 278 279/*++ 280 281Name: 282 GetRingBufferInterruptMask() 283 284Description: 285 Get the interrupt mask for the specified ring buffer 286 287--*/ 288u32 GetRingBufferInterruptMask(struct hv_ring_buffer_info *rbi) 289{ 290 return rbi->RingBuffer->InterruptMask; 291} 292 293/*++ 294 295Name: 296 RingBufferInit() 297 298Description: 299 Initialize the ring buffer 300 301--*/ 302int RingBufferInit(struct hv_ring_buffer_info *RingInfo, void *Buffer, u32 BufferLen) 303{ 304 if (sizeof(struct hv_ring_buffer) != PAGE_SIZE) 305 return -EINVAL; 306 307 memset(RingInfo, 0, sizeof(struct hv_ring_buffer_info)); 308 309 RingInfo->RingBuffer = (struct hv_ring_buffer *)Buffer; 310 RingInfo->RingBuffer->ReadIndex = RingInfo->RingBuffer->WriteIndex = 0; 311 312 RingInfo->RingSize = BufferLen; 313 RingInfo->RingDataSize = BufferLen - sizeof(struct hv_ring_buffer); 314 315 spin_lock_init(&RingInfo->ring_lock); 316 317 return 0; 318} 319 320/*++ 321 322Name: 323 RingBufferCleanup() 324 325Description: 326 Cleanup the ring buffer 327 328--*/ 329void RingBufferCleanup(struct hv_ring_buffer_info *RingInfo) 330{ 331} 332 333/*++ 334 335Name: 336 RingBufferWrite() 337 338Description: 339 Write to the ring buffer 340 341--*/ 342int RingBufferWrite(struct hv_ring_buffer_info *OutRingInfo, 343 struct scatterlist *sglist, u32 sgcount) 344{ 345 int i = 0; 346 u32 byteAvailToWrite; 347 u32 byteAvailToRead; 348 u32 totalBytesToWrite = 0; 349 350 struct scatterlist *sg; 351 volatile u32 nextWriteLocation; 352 u64 prevIndices = 0; 353 unsigned long flags; 354 355 for_each_sg(sglist, sg, sgcount, i) 356 { 357 totalBytesToWrite += sg->length; 358 } 359 360 totalBytesToWrite += sizeof(u64); 361 362 spin_lock_irqsave(&OutRingInfo->ring_lock, flags); 363 364 GetRingBufferAvailBytes(OutRingInfo, 365 &byteAvailToRead, 366 &byteAvailToWrite); 367 368 DPRINT_DBG(VMBUS, "Writing %u bytes...", totalBytesToWrite); 369 370 /* DumpRingInfo(OutRingInfo, "BEFORE "); */ 371 372 /* If there is only room for the packet, assume it is full. */ 373 /* Otherwise, the next time around, we think the ring buffer */ 374 /* is empty since the read index == write index */ 375 if (byteAvailToWrite <= totalBytesToWrite) { 376 DPRINT_DBG(VMBUS, 377 "No more space left on outbound ring buffer " 378 "(needed %u, avail %u)", 379 totalBytesToWrite, 380 byteAvailToWrite); 381 382 spin_unlock_irqrestore(&OutRingInfo->ring_lock, flags); 383 return -1; 384 } 385 386 /* Write to the ring buffer */ 387 nextWriteLocation = GetNextWriteLocation(OutRingInfo); 388 389 for_each_sg(sglist, sg, sgcount, i) 390 { 391 nextWriteLocation = CopyToRingBuffer(OutRingInfo, 392 nextWriteLocation, 393 sg_virt(sg), 394 sg->length); 395 } 396 397 /* Set previous packet start */ 398 prevIndices = GetRingBufferIndices(OutRingInfo); 399 400 nextWriteLocation = CopyToRingBuffer(OutRingInfo, 401 nextWriteLocation, 402 &prevIndices, 403 sizeof(u64)); 404 405 /* Make sure we flush all writes before updating the writeIndex */ 406 mb(); 407 408 /* Now, update the write location */ 409 SetNextWriteLocation(OutRingInfo, nextWriteLocation); 410 411 /* DumpRingInfo(OutRingInfo, "AFTER "); */ 412 413 spin_unlock_irqrestore(&OutRingInfo->ring_lock, flags); 414 return 0; 415} 416 417 418/*++ 419 420Name: 421 RingBufferPeek() 422 423Description: 424 Read without advancing the read index 425 426--*/ 427int RingBufferPeek(struct hv_ring_buffer_info *InRingInfo, void *Buffer, u32 BufferLen) 428{ 429 u32 bytesAvailToWrite; 430 u32 bytesAvailToRead; 431 u32 nextReadLocation = 0; 432 unsigned long flags; 433 434 spin_lock_irqsave(&InRingInfo->ring_lock, flags); 435 436 GetRingBufferAvailBytes(InRingInfo, 437 &bytesAvailToRead, 438 &bytesAvailToWrite); 439 440 /* Make sure there is something to read */ 441 if (bytesAvailToRead < BufferLen) { 442 /* DPRINT_DBG(VMBUS, 443 "got callback but not enough to read " 444 "<avail to read %d read size %d>!!", 445 bytesAvailToRead, 446 BufferLen); */ 447 448 spin_unlock_irqrestore(&InRingInfo->ring_lock, flags); 449 450 return -1; 451 } 452 453 /* Convert to byte offset */ 454 nextReadLocation = GetNextReadLocation(InRingInfo); 455 456 nextReadLocation = CopyFromRingBuffer(InRingInfo, 457 Buffer, 458 BufferLen, 459 nextReadLocation); 460 461 spin_unlock_irqrestore(&InRingInfo->ring_lock, flags); 462 463 return 0; 464} 465 466 467/*++ 468 469Name: 470 RingBufferRead() 471 472Description: 473 Read and advance the read index 474 475--*/ 476int RingBufferRead(struct hv_ring_buffer_info *InRingInfo, void *Buffer, 477 u32 BufferLen, u32 Offset) 478{ 479 u32 bytesAvailToWrite; 480 u32 bytesAvailToRead; 481 u32 nextReadLocation = 0; 482 u64 prevIndices = 0; 483 unsigned long flags; 484 485 if (BufferLen <= 0) 486 return -EINVAL; 487 488 spin_lock_irqsave(&InRingInfo->ring_lock, flags); 489 490 GetRingBufferAvailBytes(InRingInfo, 491 &bytesAvailToRead, 492 &bytesAvailToWrite); 493 494 DPRINT_DBG(VMBUS, "Reading %u bytes...", BufferLen); 495 496 /* DumpRingInfo(InRingInfo, "BEFORE "); */ 497 498 /* Make sure there is something to read */ 499 if (bytesAvailToRead < BufferLen) { 500 DPRINT_DBG(VMBUS, 501 "got callback but not enough to read " 502 "<avail to read %d read size %d>!!", 503 bytesAvailToRead, 504 BufferLen); 505 506 spin_unlock_irqrestore(&InRingInfo->ring_lock, flags); 507 508 return -1; 509 } 510 511 nextReadLocation = GetNextReadLocationWithOffset(InRingInfo, Offset); 512 513 nextReadLocation = CopyFromRingBuffer(InRingInfo, 514 Buffer, 515 BufferLen, 516 nextReadLocation); 517 518 nextReadLocation = CopyFromRingBuffer(InRingInfo, 519 &prevIndices, 520 sizeof(u64), 521 nextReadLocation); 522 523 /* Make sure all reads are done before we update the read index since */ 524 /* the writer may start writing to the read area once the read index */ 525 /*is updated */ 526 mb(); 527 528 /* Update the read index */ 529 SetNextReadLocation(InRingInfo, nextReadLocation); 530 531 /* DumpRingInfo(InRingInfo, "AFTER "); */ 532 533 spin_unlock_irqrestore(&InRingInfo->ring_lock, flags); 534 535 return 0; 536} 537 538 539/*++ 540 541Name: 542 CopyToRingBuffer() 543 544Description: 545 Helper routine to copy from source to ring buffer. 546 Assume there is enough room. Handles wrap-around in dest case only!! 547 548--*/ 549static u32 550CopyToRingBuffer( 551 struct hv_ring_buffer_info *RingInfo, 552 u32 StartWriteOffset, 553 void *Src, 554 u32 SrcLen) 555{ 556 void *ringBuffer = GetRingBuffer(RingInfo); 557 u32 ringBufferSize = GetRingBufferSize(RingInfo); 558 u32 fragLen; 559 560 /* wrap-around detected! */ 561 if (SrcLen > ringBufferSize - StartWriteOffset) { 562 DPRINT_DBG(VMBUS, "wrap-around detected!"); 563 564 fragLen = ringBufferSize - StartWriteOffset; 565 memcpy(ringBuffer + StartWriteOffset, Src, fragLen); 566 memcpy(ringBuffer, Src + fragLen, SrcLen - fragLen); 567 } else 568 memcpy(ringBuffer + StartWriteOffset, Src, SrcLen); 569 570 StartWriteOffset += SrcLen; 571 StartWriteOffset %= ringBufferSize; 572 573 return StartWriteOffset; 574} 575 576 577/*++ 578 579Name: 580 CopyFromRingBuffer() 581 582Description: 583 Helper routine to copy to source from ring buffer. 584 Assume there is enough room. Handles wrap-around in src case only!! 585 586--*/ 587static u32 588CopyFromRingBuffer( 589 struct hv_ring_buffer_info *RingInfo, 590 void *Dest, 591 u32 DestLen, 592 u32 StartReadOffset) 593{ 594 void *ringBuffer = GetRingBuffer(RingInfo); 595 u32 ringBufferSize = GetRingBufferSize(RingInfo); 596 597 u32 fragLen; 598 599 /* wrap-around detected at the src */ 600 if (DestLen > ringBufferSize - StartReadOffset) { 601 DPRINT_DBG(VMBUS, "src wrap-around detected!"); 602 603 fragLen = ringBufferSize - StartReadOffset; 604 605 memcpy(Dest, ringBuffer + StartReadOffset, fragLen); 606 memcpy(Dest + fragLen, ringBuffer, DestLen - fragLen); 607 } else 608 609 memcpy(Dest, ringBuffer + StartReadOffset, DestLen); 610 611 612 StartReadOffset += DestLen; 613 StartReadOffset %= ringBufferSize; 614 615 return StartReadOffset; 616} 617 618 619/* eof */ 620