1/* $NetBSD: rf_dagfuncs.c,v 1.35 2021/08/07 16:19:15 thorpej Exp $ */ 2/* 3 * Copyright (c) 1995 Carnegie-Mellon University. 4 * All rights reserved. 5 * 6 * Author: Mark Holland, William V. Courtright II 7 * 8 * Permission to use, copy, modify and distribute this software and 9 * its documentation is hereby granted, provided that both the copyright 10 * notice and this permission notice appear in all copies of the 11 * software, derivative works or modified versions, and any portions 12 * thereof, and that both notices appear in supporting documentation. 13 * 14 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 15 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 16 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 17 * 18 * Carnegie Mellon requests users of this software to return to 19 * 20 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 21 * School of Computer Science 22 * Carnegie Mellon University 23 * Pittsburgh PA 15213-3890 24 * 25 * any improvements or extensions that they make and grant Carnegie the 26 * rights to redistribute these changes. 27 */ 28 29/* 30 * dagfuncs.c -- DAG node execution routines 31 * 32 * Rules: 33 * 1. Every DAG execution function must eventually cause node->status to 34 * get set to "good" or "bad", and "FinishNode" to be called. In the 35 * case of nodes that complete immediately (xor, NullNodeFunc, etc), 36 * the node execution function can do these two things directly. In 37 * the case of nodes that have to wait for some event (a disk read to 38 * complete, a lock to be released, etc) to occur before they can 39 * complete, this is typically achieved by having whatever module 40 * is doing the operation call GenericWakeupFunc upon completion. 41 * 2. DAG execution functions should check the status in the DAG header 42 * and NOP out their operations if the status is not "enable". However, 43 * execution functions that release resources must be sure to release 44 * them even when they NOP out the function that would use them. 45 * Functions that acquire resources should go ahead and acquire them 46 * even when they NOP, so that a downstream release node will not have 47 * to check to find out whether or not the acquire was suppressed. 48 */ 49 50#include <sys/cdefs.h> 51__KERNEL_RCSID(0, "$NetBSD: rf_dagfuncs.c,v 1.35 2021/08/07 16:19:15 thorpej Exp $"); 52 53#include <sys/param.h> 54#include <sys/ioctl.h> 55 56#include "rf_archs.h" 57#include "rf_raid.h" 58#include "rf_dag.h" 59#include "rf_layout.h" 60#include "rf_etimer.h" 61#include "rf_acctrace.h" 62#include "rf_diskqueue.h" 63#include "rf_dagfuncs.h" 64#include "rf_general.h" 65#include "rf_engine.h" 66#include "rf_dagutils.h" 67 68#include "rf_kintf.h" 69 70#if RF_INCLUDE_PARITYLOGGING > 0 71#include "rf_paritylog.h" 72#endif /* RF_INCLUDE_PARITYLOGGING > 0 */ 73 74void (*rf_DiskReadFunc) (RF_DagNode_t *); 75void (*rf_DiskWriteFunc) (RF_DagNode_t *); 76void (*rf_DiskReadUndoFunc) (RF_DagNode_t *); 77void (*rf_DiskWriteUndoFunc) (RF_DagNode_t *); 78void (*rf_RegularXorUndoFunc) (RF_DagNode_t *); 79void (*rf_SimpleXorUndoFunc) (RF_DagNode_t *); 80void (*rf_RecoveryXorUndoFunc) (RF_DagNode_t *); 81 82/***************************************************************************** 83 * main (only) configuration routine for this module 84 ****************************************************************************/ 85int 86rf_ConfigureDAGFuncs(RF_ShutdownList_t **listp) 87{ 88 RF_ASSERT(((sizeof(long) == 8) && RF_LONGSHIFT == 3) || 89 ((sizeof(long) == 4) && RF_LONGSHIFT == 2)); 90 rf_DiskReadFunc = rf_DiskReadFuncForThreads; 91 rf_DiskReadUndoFunc = rf_DiskUndoFunc; 92 rf_DiskWriteFunc = rf_DiskWriteFuncForThreads; 93 rf_DiskWriteUndoFunc = rf_DiskUndoFunc; 94 rf_RegularXorUndoFunc = rf_NullNodeUndoFunc; 95 rf_SimpleXorUndoFunc = rf_NullNodeUndoFunc; 96 rf_RecoveryXorUndoFunc = rf_NullNodeUndoFunc; 97 return (0); 98} 99 100 101 102/***************************************************************************** 103 * the execution function associated with a terminate node 104 ****************************************************************************/ 105void 106rf_TerminateFunc(RF_DagNode_t *node) 107{ 108 RF_ASSERT(node->dagHdr->numCommits == node->dagHdr->numCommitNodes); 109 node->status = rf_good; 110 rf_FinishNode(node, RF_THREAD_CONTEXT); 111} 112 113void 114rf_TerminateUndoFunc(RF_DagNode_t *node) 115{ 116} 117 118 119/***************************************************************************** 120 * execution functions associated with a mirror node 121 * 122 * parameters: 123 * 124 * 0 - physical disk address of data 125 * 1 - buffer for holding read data 126 * 2 - parity stripe ID 127 * 3 - flags 128 * 4 - physical disk address of mirror (parity) 129 * 130 ****************************************************************************/ 131 132void 133rf_DiskReadMirrorIdleFunc(RF_DagNode_t *node) 134{ 135 /* select the mirror copy with the shortest queue and fill in node 136 * parameters with physical disk address */ 137 138 rf_SelectMirrorDiskIdle(node); 139 rf_DiskReadFunc(node); 140} 141 142#if (RF_INCLUDE_CHAINDECLUSTER > 0) || (RF_INCLUDE_INTERDECLUSTER > 0) || (RF_DEBUG_VALIDATE_DAG > 0) 143void 144rf_DiskReadMirrorPartitionFunc(RF_DagNode_t *node) 145{ 146 /* select the mirror copy with the shortest queue and fill in node 147 * parameters with physical disk address */ 148 149 rf_SelectMirrorDiskPartition(node); 150 rf_DiskReadFunc(node); 151} 152#endif 153 154void 155rf_DiskReadMirrorUndoFunc(RF_DagNode_t *node) 156{ 157} 158 159 160 161#if RF_INCLUDE_PARITYLOGGING > 0 162/***************************************************************************** 163 * the execution function associated with a parity log update node 164 ****************************************************************************/ 165void 166rf_ParityLogUpdateFunc(RF_DagNode_t *node) 167{ 168 RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p; 169 void *bf = (void *) node->params[1].p; 170 RF_ParityLogData_t *logData; 171#if RF_ACC_TRACE > 0 172 RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec; 173 RF_Etimer_t timer; 174#endif 175 176 if (node->dagHdr->status == rf_enable) { 177#if RF_ACC_TRACE > 0 178 RF_ETIMER_START(timer); 179#endif 180 logData = rf_CreateParityLogData(RF_UPDATE, pda, bf, 181 (RF_Raid_t *) (node->dagHdr->raidPtr), 182 node->wakeFunc, node, 183 node->dagHdr->tracerec, timer); 184 if (logData) 185 rf_ParityLogAppend(logData, RF_FALSE, NULL, RF_FALSE); 186 else { 187#if RF_ACC_TRACE > 0 188 RF_ETIMER_STOP(timer); 189 RF_ETIMER_EVAL(timer); 190 tracerec->plog_us += RF_ETIMER_VAL_US(timer); 191#endif 192 (node->wakeFunc) (node, ENOMEM); 193 } 194 } 195} 196 197 198/***************************************************************************** 199 * the execution function associated with a parity log overwrite node 200 ****************************************************************************/ 201void 202rf_ParityLogOverwriteFunc(RF_DagNode_t *node) 203{ 204 RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p; 205 void *bf = (void *) node->params[1].p; 206 RF_ParityLogData_t *logData; 207#if RF_ACC_TRACE > 0 208 RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec; 209 RF_Etimer_t timer; 210#endif 211 212 if (node->dagHdr->status == rf_enable) { 213#if RF_ACC_TRACE > 0 214 RF_ETIMER_START(timer); 215#endif 216 logData = rf_CreateParityLogData(RF_OVERWRITE, pda, bf, 217(RF_Raid_t *) (node->dagHdr->raidPtr), 218 node->wakeFunc, node, node->dagHdr->tracerec, timer); 219 if (logData) 220 rf_ParityLogAppend(logData, RF_FALSE, NULL, RF_FALSE); 221 else { 222#if RF_ACC_TRACE > 0 223 RF_ETIMER_STOP(timer); 224 RF_ETIMER_EVAL(timer); 225 tracerec->plog_us += RF_ETIMER_VAL_US(timer); 226#endif 227 (node->wakeFunc) (node, ENOMEM); 228 } 229 } 230} 231 232void 233rf_ParityLogUpdateUndoFunc(RF_DagNode_t *node) 234{ 235} 236 237void 238rf_ParityLogOverwriteUndoFunc(RF_DagNode_t *node) 239{ 240} 241#endif /* RF_INCLUDE_PARITYLOGGING > 0 */ 242 243/***************************************************************************** 244 * the execution function associated with a NOP node 245 ****************************************************************************/ 246void 247rf_NullNodeFunc(RF_DagNode_t *node) 248{ 249 node->status = rf_good; 250 rf_FinishNode(node, RF_THREAD_CONTEXT); 251} 252 253void 254rf_NullNodeUndoFunc(RF_DagNode_t *node) 255{ 256 node->status = rf_undone; 257 rf_FinishNode(node, RF_THREAD_CONTEXT); 258} 259 260 261/***************************************************************************** 262 * the execution function associated with a disk-read node 263 ****************************************************************************/ 264void 265rf_DiskReadFuncForThreads(RF_DagNode_t *node) 266{ 267 RF_DiskQueueData_t *req; 268 RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p; 269 void *bf = (void *) node->params[1].p; 270 RF_StripeNum_t parityStripeID = (RF_StripeNum_t) node->params[2].v; 271 unsigned priority = RF_EXTRACT_PRIORITY(node->params[3].v); 272 unsigned which_ru = RF_EXTRACT_RU(node->params[3].v); 273 RF_IoType_t iotype = (node->dagHdr->status == rf_enable) ? RF_IO_TYPE_READ : RF_IO_TYPE_NOP; 274 RF_DiskQueue_t *dqs = ((RF_Raid_t *) (node->dagHdr->raidPtr))->Queues; 275 276 req = rf_CreateDiskQueueData(iotype, pda->startSector, pda->numSector, 277 bf, parityStripeID, which_ru, node->wakeFunc, node, 278#if RF_ACC_TRACE > 0 279 node->dagHdr->tracerec, 280#else 281 NULL, 282#endif 283 (void *) (node->dagHdr->raidPtr), 0, node->dagHdr->bp); 284 285 node->dagFuncData = (void *) req; 286 rf_DiskIOEnqueue(&(dqs[pda->col]), req, priority); 287} 288 289 290/***************************************************************************** 291 * the execution function associated with a disk-write node 292 ****************************************************************************/ 293void 294rf_DiskWriteFuncForThreads(RF_DagNode_t *node) 295{ 296 RF_DiskQueueData_t *req; 297 RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p; 298 void *bf = (void *) node->params[1].p; 299 RF_StripeNum_t parityStripeID = (RF_StripeNum_t) node->params[2].v; 300 unsigned priority = RF_EXTRACT_PRIORITY(node->params[3].v); 301 unsigned which_ru = RF_EXTRACT_RU(node->params[3].v); 302 RF_IoType_t iotype = (node->dagHdr->status == rf_enable) ? RF_IO_TYPE_WRITE : RF_IO_TYPE_NOP; 303 RF_DiskQueue_t *dqs = ((RF_Raid_t *) (node->dagHdr->raidPtr))->Queues; 304 305 /* normal processing (rollaway or forward recovery) begins here */ 306 req = rf_CreateDiskQueueData(iotype, pda->startSector, pda->numSector, 307 bf, parityStripeID, which_ru, node->wakeFunc, node, 308#if RF_ACC_TRACE > 0 309 node->dagHdr->tracerec, 310#else 311 NULL, 312#endif 313 (void *) (node->dagHdr->raidPtr), 314 0, node->dagHdr->bp); 315 316 node->dagFuncData = (void *) req; 317 rf_DiskIOEnqueue(&(dqs[pda->col]), req, priority); 318} 319/***************************************************************************** 320 * the undo function for disk nodes 321 * Note: this is not a proper undo of a write node, only locks are released. 322 * old data is not restored to disk! 323 ****************************************************************************/ 324void 325rf_DiskUndoFunc(RF_DagNode_t *node) 326{ 327 RF_DiskQueueData_t *req; 328 RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p; 329 RF_DiskQueue_t *dqs = ((RF_Raid_t *) (node->dagHdr->raidPtr))->Queues; 330 331 req = rf_CreateDiskQueueData(RF_IO_TYPE_NOP, 332 0L, 0, NULL, 0L, 0, node->wakeFunc, node, 333#if RF_ACC_TRACE > 0 334 node->dagHdr->tracerec, 335#else 336 NULL, 337#endif 338 (void *) (node->dagHdr->raidPtr), 339 0, NULL); 340 341 node->dagFuncData = (void *) req; 342 rf_DiskIOEnqueue(&(dqs[pda->col]), req, RF_IO_NORMAL_PRIORITY); 343} 344 345/***************************************************************************** 346 * Callback routine for DiskRead and DiskWrite nodes. When the disk 347 * op completes, the routine is called to set the node status and 348 * inform the execution engine that the node has fired. 349 ****************************************************************************/ 350void 351rf_GenericWakeupFunc(void *v, int status) 352{ 353 RF_DagNode_t *node = v; 354 355 switch (node->status) { 356 case rf_fired: 357 if (status) 358 node->status = rf_bad; 359 else 360 node->status = rf_good; 361 break; 362 case rf_recover: 363 /* probably should never reach this case */ 364 if (status) 365 node->status = rf_panic; 366 else 367 node->status = rf_undone; 368 break; 369 default: 370 printf("rf_GenericWakeupFunc:"); 371 printf("node->status is %d,", node->status); 372 printf("status is %d \n", status); 373 RF_PANIC(); 374 break; 375 } 376 if (node->dagFuncData) 377 rf_FreeDiskQueueData((RF_DiskQueueData_t *) node->dagFuncData); 378 rf_FinishNode(node, RF_INTR_CONTEXT); 379} 380 381 382/***************************************************************************** 383 * there are three distinct types of xor nodes: 384 385 * A "regular xor" is used in the fault-free case where the access 386 * spans a complete stripe unit. It assumes that the result buffer is 387 * one full stripe unit in size, and uses the stripe-unit-offset 388 * values that it computes from the PDAs to determine where within the 389 * stripe unit to XOR each argument buffer. 390 * 391 * A "simple xor" is used in the fault-free case where the access 392 * touches only a portion of one (or two, in some cases) stripe 393 * unit(s). It assumes that all the argument buffers are of the same 394 * size and have the same stripe unit offset. 395 * 396 * A "recovery xor" is used in the degraded-mode case. It's similar 397 * to the regular xor function except that it takes the failed PDA as 398 * an additional parameter, and uses it to determine what portions of 399 * the argument buffers need to be xor'd into the result buffer, and 400 * where in the result buffer they should go. 401 ****************************************************************************/ 402 403/* xor the params together and store the result in the result field. 404 * assume the result field points to a buffer that is the size of one 405 * SU, and use the pda params to determine where within the buffer to 406 * XOR the input buffers. */ 407void 408rf_RegularXorFunc(RF_DagNode_t *node) 409{ 410 RF_Raid_t *raidPtr = (RF_Raid_t *) node->params[node->numParams - 1].p; 411#if RF_ACC_TRACE > 0 412 RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec; 413 RF_Etimer_t timer; 414#endif 415 int i, retcode; 416 417 retcode = 0; 418 if (node->dagHdr->status == rf_enable) { 419 /* don't do the XOR if the input is the same as the output */ 420#if RF_ACC_TRACE > 0 421 RF_ETIMER_START(timer); 422#endif 423 for (i = 0; i < node->numParams - 1; i += 2) 424 if (node->params[i + 1].p != node->results[0]) { 425 retcode = rf_XorIntoBuffer(raidPtr, (RF_PhysDiskAddr_t *) node->params[i].p, 426 (char *) node->params[i + 1].p, (char *) node->results[0]); 427 } 428#if RF_ACC_TRACE > 0 429 RF_ETIMER_STOP(timer); 430 RF_ETIMER_EVAL(timer); 431 tracerec->xor_us += RF_ETIMER_VAL_US(timer); 432#endif 433 } 434 rf_GenericWakeupFunc(node, retcode); /* call wake func 435 * explicitly since no 436 * I/O in this node */ 437} 438/* xor the inputs into the result buffer, ignoring placement issues */ 439void 440rf_SimpleXorFunc(RF_DagNode_t *node) 441{ 442 RF_Raid_t *raidPtr = (RF_Raid_t *) node->params[node->numParams - 1].p; 443 int i, retcode = 0; 444#if RF_ACC_TRACE > 0 445 RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec; 446 RF_Etimer_t timer; 447#endif 448 449 if (node->dagHdr->status == rf_enable) { 450#if RF_ACC_TRACE > 0 451 RF_ETIMER_START(timer); 452#endif 453 /* don't do the XOR if the input is the same as the output */ 454 for (i = 0; i < node->numParams - 1; i += 2) 455 if (node->params[i + 1].p != node->results[0]) { 456 retcode = rf_bxor((char *) node->params[i + 1].p, (char *) node->results[0], 457 rf_RaidAddressToByte(raidPtr, ((RF_PhysDiskAddr_t *) node->params[i].p)->numSector)); 458 } 459#if RF_ACC_TRACE > 0 460 RF_ETIMER_STOP(timer); 461 RF_ETIMER_EVAL(timer); 462 tracerec->xor_us += RF_ETIMER_VAL_US(timer); 463#endif 464 } 465 rf_GenericWakeupFunc(node, retcode); /* call wake func 466 * explicitly since no 467 * I/O in this node */ 468} 469/* this xor is used by the degraded-mode dag functions to recover lost 470 * data. the second-to-last parameter is the PDA for the failed 471 * portion of the access. the code here looks at this PDA and assumes 472 * that the xor target buffer is equal in size to the number of 473 * sectors in the failed PDA. It then uses the other PDAs in the 474 * parameter list to determine where within the target buffer the 475 * corresponding data should be xored. */ 476void 477rf_RecoveryXorFunc(RF_DagNode_t *node) 478{ 479 RF_Raid_t *raidPtr = (RF_Raid_t *) node->params[node->numParams - 1].p; 480 RF_RaidLayout_t *layoutPtr = (RF_RaidLayout_t *) & raidPtr->Layout; 481 RF_PhysDiskAddr_t *failedPDA = (RF_PhysDiskAddr_t *) node->params[node->numParams - 2].p; 482 int i, retcode = 0; 483 RF_PhysDiskAddr_t *pda; 484 int suoffset, failedSUOffset = rf_StripeUnitOffset(layoutPtr, failedPDA->startSector); 485 char *srcbuf, *destbuf; 486#if RF_ACC_TRACE > 0 487 RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec; 488 RF_Etimer_t timer; 489#endif 490 491 if (node->dagHdr->status == rf_enable) { 492#if RF_ACC_TRACE > 0 493 RF_ETIMER_START(timer); 494#endif 495 for (i = 0; i < node->numParams - 2; i += 2) 496 if (node->params[i + 1].p != node->results[0]) { 497 pda = (RF_PhysDiskAddr_t *) node->params[i].p; 498 srcbuf = (char *) node->params[i + 1].p; 499 suoffset = rf_StripeUnitOffset(layoutPtr, pda->startSector); 500 destbuf = ((char *) node->results[0]) + rf_RaidAddressToByte(raidPtr, suoffset - failedSUOffset); 501 retcode = rf_bxor(srcbuf, destbuf, rf_RaidAddressToByte(raidPtr, pda->numSector)); 502 } 503#if RF_ACC_TRACE > 0 504 RF_ETIMER_STOP(timer); 505 RF_ETIMER_EVAL(timer); 506 tracerec->xor_us += RF_ETIMER_VAL_US(timer); 507#endif 508 } 509 rf_GenericWakeupFunc(node, retcode); 510} 511/***************************************************************************** 512 * The next three functions are utilities used by the above 513 * xor-execution functions. 514 ****************************************************************************/ 515 516 517/* 518 * this is just a glorified buffer xor. targbuf points to a buffer 519 * that is one full stripe unit in size. srcbuf points to a buffer 520 * that may be less than 1 SU, but never more. When the access 521 * described by pda is one SU in size (which by implication means it's 522 * SU-aligned), all that happens is (targbuf) <- (srcbuf ^ targbuf). 523 * When the access is less than one SU in size the XOR occurs on only 524 * the portion of targbuf identified in the pda. */ 525 526int 527rf_XorIntoBuffer(RF_Raid_t *raidPtr, RF_PhysDiskAddr_t *pda, 528 char *srcbuf, char *targbuf) 529{ 530 char *targptr; 531 int sectPerSU = raidPtr->Layout.sectorsPerStripeUnit; 532 int SUOffset = pda->startSector % sectPerSU; 533 int length, retcode = 0; 534 535 RF_ASSERT(pda->numSector <= sectPerSU); 536 537 targptr = targbuf + rf_RaidAddressToByte(raidPtr, SUOffset); 538 length = rf_RaidAddressToByte(raidPtr, pda->numSector); 539 retcode = rf_bxor(srcbuf, targptr, length); 540 return (retcode); 541} 542/* it really should be the case that the buffer pointers (returned by 543 * malloc) are aligned to the natural word size of the machine, so 544 * this is the only case we optimize for. The length should always be 545 * a multiple of the sector size, so there should be no problem with 546 * leftover bytes at the end. */ 547int 548rf_bxor(char *src, char *dest, int len) 549{ 550 unsigned mask = sizeof(long) - 1, retcode = 0; 551 552 if (!(((unsigned long) src) & mask) && 553 !(((unsigned long) dest) & mask) && !(len & mask)) { 554 retcode = rf_longword_bxor((unsigned long *) src, 555 (unsigned long *) dest, 556 len >> RF_LONGSHIFT); 557 } else { 558 RF_ASSERT(0); 559 } 560 return (retcode); 561} 562 563/* When XORing in kernel mode, we need to map each user page to kernel 564 * space before we can access it. We don't want to assume anything 565 * about which input buffers are in kernel/user space, nor about their 566 * alignment, so in each loop we compute the maximum number of bytes 567 * that we can xor without crossing any page boundaries, and do only 568 * this many bytes before the next remap. 569 * 570 * len - is in longwords 571 */ 572int 573rf_longword_bxor(unsigned long *src, unsigned long *dest, int len) 574{ 575 unsigned long *end = src + len; 576 unsigned long d0, d1, d2, d3, s0, s1, s2, s3; /* temps */ 577 unsigned long *pg_src, *pg_dest; /* per-page source/dest pointers */ 578 int longs_this_time;/* # longwords to xor in the current iteration */ 579 580 pg_src = src; 581 pg_dest = dest; 582 if (!pg_src || !pg_dest) 583 return (EFAULT); 584 585 while (len >= 4) { 586 longs_this_time = RF_MIN(len, RF_MIN(RF_BLIP(pg_src), RF_BLIP(pg_dest)) >> RF_LONGSHIFT); /* note len in longwords */ 587 src += longs_this_time; 588 dest += longs_this_time; 589 len -= longs_this_time; 590 while (longs_this_time >= 4) { 591 d0 = pg_dest[0]; 592 d1 = pg_dest[1]; 593 d2 = pg_dest[2]; 594 d3 = pg_dest[3]; 595 s0 = pg_src[0]; 596 s1 = pg_src[1]; 597 s2 = pg_src[2]; 598 s3 = pg_src[3]; 599 pg_dest[0] = d0 ^ s0; 600 pg_dest[1] = d1 ^ s1; 601 pg_dest[2] = d2 ^ s2; 602 pg_dest[3] = d3 ^ s3; 603 pg_src += 4; 604 pg_dest += 4; 605 longs_this_time -= 4; 606 } 607 while (longs_this_time > 0) { /* cannot cross any page 608 * boundaries here */ 609 *pg_dest++ ^= *pg_src++; 610 longs_this_time--; 611 } 612 613 /* either we're done, or we've reached a page boundary on one 614 * (or possibly both) of the pointers */ 615 if (len) { 616 if (RF_PAGE_ALIGNED(src)) 617 pg_src = src; 618 if (RF_PAGE_ALIGNED(dest)) 619 pg_dest = dest; 620 if (!pg_src || !pg_dest) 621 return (EFAULT); 622 } 623 } 624 while (src < end) { 625 *pg_dest++ ^= *pg_src++; 626 src++; 627 dest++; 628 len--; 629 if (RF_PAGE_ALIGNED(src)) 630 pg_src = src; 631 if (RF_PAGE_ALIGNED(dest)) 632 pg_dest = dest; 633 } 634 RF_ASSERT(len == 0); 635 return (0); 636} 637 638#if 0 639/* 640 dst = a ^ b ^ c; 641 a may equal dst 642 see comment above longword_bxor 643 len is length in longwords 644*/ 645int 646rf_longword_bxor3(unsigned long *dst, unsigned long *a, unsigned long *b, 647 unsigned long *c, int len, void *bp) 648{ 649 unsigned long a0, a1, a2, a3, b0, b1, b2, b3; 650 unsigned long *pg_a, *pg_b, *pg_c, *pg_dst; /* per-page source/dest 651 * pointers */ 652 int longs_this_time;/* # longs to xor in the current iteration */ 653 char dst_is_a = 0; 654 655 pg_a = a; 656 pg_b = b; 657 pg_c = c; 658 if (a == dst) { 659 pg_dst = pg_a; 660 dst_is_a = 1; 661 } else { 662 pg_dst = dst; 663 } 664 665 /* align dest to cache line. Can't cross a pg boundary on dst here. */ 666 while ((((unsigned long) pg_dst) & 0x1f)) { 667 *pg_dst++ = *pg_a++ ^ *pg_b++ ^ *pg_c++; 668 dst++; 669 a++; 670 b++; 671 c++; 672 if (RF_PAGE_ALIGNED(a)) { 673 pg_a = a; 674 if (!pg_a) 675 return (EFAULT); 676 } 677 if (RF_PAGE_ALIGNED(b)) { 678 pg_b = a; 679 if (!pg_b) 680 return (EFAULT); 681 } 682 if (RF_PAGE_ALIGNED(c)) { 683 pg_c = a; 684 if (!pg_c) 685 return (EFAULT); 686 } 687 len--; 688 } 689 690 while (len > 4) { 691 longs_this_time = RF_MIN(len, RF_MIN(RF_BLIP(a), RF_MIN(RF_BLIP(b), RF_MIN(RF_BLIP(c), RF_BLIP(dst)))) >> RF_LONGSHIFT); 692 a += longs_this_time; 693 b += longs_this_time; 694 c += longs_this_time; 695 dst += longs_this_time; 696 len -= longs_this_time; 697 while (longs_this_time >= 4) { 698 a0 = pg_a[0]; 699 longs_this_time -= 4; 700 701 a1 = pg_a[1]; 702 a2 = pg_a[2]; 703 704 a3 = pg_a[3]; 705 pg_a += 4; 706 707 b0 = pg_b[0]; 708 b1 = pg_b[1]; 709 710 b2 = pg_b[2]; 711 b3 = pg_b[3]; 712 /* start dual issue */ 713 a0 ^= b0; 714 b0 = pg_c[0]; 715 716 pg_b += 4; 717 a1 ^= b1; 718 719 a2 ^= b2; 720 a3 ^= b3; 721 722 b1 = pg_c[1]; 723 a0 ^= b0; 724 725 b2 = pg_c[2]; 726 a1 ^= b1; 727 728 b3 = pg_c[3]; 729 a2 ^= b2; 730 731 pg_dst[0] = a0; 732 a3 ^= b3; 733 pg_dst[1] = a1; 734 pg_c += 4; 735 pg_dst[2] = a2; 736 pg_dst[3] = a3; 737 pg_dst += 4; 738 } 739 while (longs_this_time > 0) { /* cannot cross any page 740 * boundaries here */ 741 *pg_dst++ = *pg_a++ ^ *pg_b++ ^ *pg_c++; 742 longs_this_time--; 743 } 744 745 if (len) { 746 if (RF_PAGE_ALIGNED(a)) { 747 pg_a = a; 748 if (!pg_a) 749 return (EFAULT); 750 if (dst_is_a) 751 pg_dst = pg_a; 752 } 753 if (RF_PAGE_ALIGNED(b)) { 754 pg_b = b; 755 if (!pg_b) 756 return (EFAULT); 757 } 758 if (RF_PAGE_ALIGNED(c)) { 759 pg_c = c; 760 if (!pg_c) 761 return (EFAULT); 762 } 763 if (!dst_is_a) 764 if (RF_PAGE_ALIGNED(dst)) { 765 pg_dst = dst; 766 if (!pg_dst) 767 return (EFAULT); 768 } 769 } 770 } 771 while (len) { 772 *pg_dst++ = *pg_a++ ^ *pg_b++ ^ *pg_c++; 773 dst++; 774 a++; 775 b++; 776 c++; 777 if (RF_PAGE_ALIGNED(a)) { 778 pg_a = a; 779 if (!pg_a) 780 return (EFAULT); 781 if (dst_is_a) 782 pg_dst = pg_a; 783 } 784 if (RF_PAGE_ALIGNED(b)) { 785 pg_b = b; 786 if (!pg_b) 787 return (EFAULT); 788 } 789 if (RF_PAGE_ALIGNED(c)) { 790 pg_c = c; 791 if (!pg_c) 792 return (EFAULT); 793 } 794 if (!dst_is_a) 795 if (RF_PAGE_ALIGNED(dst)) { 796 pg_dst = dst; 797 if (!pg_dst) 798 return (EFAULT); 799 } 800 len--; 801 } 802 return (0); 803} 804 805int 806rf_bxor3(unsigned char *dst, unsigned char *a, unsigned char *b, 807 unsigned char *c, unsigned long len, void *bp) 808{ 809 RF_ASSERT(((RF_UL(dst) | RF_UL(a) | RF_UL(b) | RF_UL(c) | len) & 0x7) == 0); 810 811 return (rf_longword_bxor3((unsigned long *) dst, (unsigned long *) a, 812 (unsigned long *) b, (unsigned long *) c, len >> RF_LONGSHIFT, bp)); 813} 814#endif 815