1/*- 2 * Copyright (c) 1990 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 4. Neither the name of the University nor the names of its contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29/*- 30 * Copyright (c) 2001 by Thomas Moestl <tmm@FreeBSD.org>. 31 * All rights reserved. 32 * 33 * Redistribution and use in source and binary forms, with or without 34 * modification, are permitted provided that the following conditions 35 * are met: 36 * 1. Redistributions of source code must retain the above copyright 37 * notice, this list of conditions and the following disclaimer. 38 * 2. Redistributions in binary form must reproduce the above copyright 39 * notice, this list of conditions and the following disclaimer in the 40 * documentation and/or other materials provided with the distribution. 41 * 42 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 43 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 44 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 45 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 46 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 47 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 48 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 49 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 50 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 51 * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 52 * 53 * from tahoe: in_cksum.c 1.2 86/01/05 54 * from: @(#)in_cksum.c 1.3 (Berkeley) 1/19/91 55 * from: FreeBSD: src/sys/i386/i386/in_cksum.c,v 1.22 2000/11/25 56 */ 57 58#include <sys/cdefs.h> 59__FBSDID("$FreeBSD$"); 60 61#include <sys/param.h> 62#include <sys/systm.h> 63#include <sys/mbuf.h> 64 65#include <netinet/in.h> 66#include <netinet/in_systm.h> 67#include <netinet/ip.h> 68 69#include <machine/in_cksum.h> 70 71/* 72 * Checksum routine for Internet Protocol family headers. 73 * 74 * This routine is very heavily used in the network 75 * code and should be modified for each CPU to be as fast as possible. 76 * 77 * This implementation is a sparc64 version. Most code was taken over 78 * and adapted from the i386. Some optimizations were changed to achieve 79 * (hopefully) better performance. 80 * This uses 64-bit loads, but 32-bit additions due to the lack of a 64-bit 81 * add-with-carry operation. 82 */ 83 84/* 85 * REDUCE() is actually not used that frequently... maybe a C implementation 86 * would suffice. 87 */ 88#define REDUCE(sum, tmp) __asm( \ 89 "sll %2, 16, %1\n" \ 90 "addcc %2, %1, %0\n" \ 91 "srl %0, 16, %0\n" \ 92 "addc %0, 0, %0" : "=r" (sum), "=&r" (tmp) : "0" (sum) : "cc") 93 94/* 95 * Note that some of these macros depend on the flags being preserved 96 * between calls, thus they have to be used within a single __asm(). 97 */ 98#define LD64_ADD32(n, mod) \ 99 "ldx [%3 + " #n "], %1\n" \ 100 "add" #mod " %2, %1, %0\n" \ 101 "srlx %1, 32, %1\n" \ 102 "addccc %0, %1, %0\n" 103 104#define LD32_ADD32(n, mod) \ 105 "lduw [%3 + " #n "], %1\n" \ 106 "add" #mod " %2, %1, %0\n" 107 108#define MOP(sum, tmp, addr) \ 109 "addc %2, 0, %0" \ 110 : "=r" (sum), "=&r" (tmp) : "0" (sum), "r" (addr) : "cc" 111 112u_short 113in_cksum_skip(struct mbuf *m, int len, int skip) 114{ 115 u_short *w; 116 unsigned long tmp, sum = 0; 117 int mlen = 0; 118 int byte_swapped = 0; 119 u_short su = 0; 120 121 len -= skip; 122 for (; skip > 0 && m != NULL; m = m->m_next) { 123 if (m->m_len > skip) { 124 mlen = m->m_len - skip; 125 w = (u_short *)(mtod(m, u_char *) + skip); 126 goto skip_start; 127 } else 128 skip -= m->m_len; 129 } 130 131 for (; m != NULL && len > 0; m = m->m_next) { 132 if (m->m_len == 0) 133 continue; 134 w = mtod(m, u_short *); 135 if (mlen == -1) { 136 /* 137 * The first byte of this mbuf is the continuation 138 * of a word spanning between this mbuf and the 139 * last mbuf. 140 * 141 * The high order byte of su is already saved when 142 * scanning previous mbuf. sum was REDUCEd when we 143 * found mlen == -1 144 */ 145 sum += su | *(u_char *)w; 146 w = (u_short *)((u_char *)w + 1); 147 mlen = m->m_len - 1; 148 len--; 149 } else 150 mlen = m->m_len; 151skip_start: 152 if (len < mlen) 153 mlen = len; 154 len -= mlen; 155 /* 156 * Force to a 8-byte boundary first so that we can use 157 * LD64_ADD32. 158 */ 159 if (((u_long)w & 7) != 0) { 160 REDUCE(sum, tmp); 161 if (((u_long)w & 1) != 0 && mlen >= 1) { 162 sum <<= 8; 163 su = *(u_char *)w << 8; 164 w = (u_short *)((u_char *)w + 1); 165 mlen--; 166 byte_swapped = 1; 167 } 168 if (((u_long)w & 2) != 0 && mlen >= 2) { 169 sum += *w++; 170 mlen -= 2; 171 } 172 if (((u_long)w & 4) != 0 && mlen >= 4) { 173 __asm( 174 LD32_ADD32(0, cc) 175 MOP(sum, tmp, w) 176 ); 177 w += 2; 178 mlen -= 4; 179 } 180 } 181 /* 182 * Do as much of the checksum as possible 64 bits at at time. 183 * In fact, this loop is unrolled to make overhead from 184 * branches &c small. 185 */ 186 for (; mlen >= 64; mlen -= 64) { 187 __asm( 188 LD64_ADD32(0, cc) 189 LD64_ADD32(8, ccc) 190 LD64_ADD32(16, ccc) 191 LD64_ADD32(24, ccc) 192 LD64_ADD32(32, ccc) 193 LD64_ADD32(40, ccc) 194 LD64_ADD32(48, ccc) 195 LD64_ADD32(56, ccc) 196 MOP(sum, tmp, w) 197 ); 198 w += 32; 199 } 200 if (mlen >= 32) { 201 __asm( 202 LD64_ADD32(0, cc) 203 LD64_ADD32(8, ccc) 204 LD64_ADD32(16, ccc) 205 LD64_ADD32(24, ccc) 206 MOP(sum, tmp, w) 207 ); 208 w += 16; 209 mlen -= 32; 210 } 211 if (mlen >= 16) { 212 __asm( 213 LD64_ADD32(0, cc) 214 LD64_ADD32(8, ccc) 215 MOP(sum, tmp, w) 216 ); 217 w += 8; 218 mlen -= 16; 219 } 220 if (mlen >= 8) { 221 __asm( 222 LD64_ADD32(0, cc) 223 MOP(sum, tmp, w) 224 ); 225 w += 4; 226 mlen -= 8; 227 } 228 REDUCE(sum, tmp); 229 while ((mlen -= 2) >= 0) 230 sum += *w++; 231 if (byte_swapped) { 232 sum <<= 8; 233 byte_swapped = 0; 234 if (mlen == -1) { 235 su |= *(u_char *)w; 236 sum += su; 237 mlen = 0; 238 } else 239 mlen = -1; 240 } else if (mlen == -1) { 241 /* 242 * This mbuf has odd number of bytes. 243 * There could be a word split between 244 * this mbuf and the next mbuf. 245 * Save the last byte (to prepend to next mbuf). 246 */ 247 su = *(u_char *)w << 8; 248 } 249 } 250 251 if (len) 252 printf("%s: out of data by %d\n", __func__, len); 253 if (mlen == -1) { 254 /* 255 * The last mbuf has odd # of bytes. Follow the 256 * standard (the odd byte is shifted left by 8 bits). 257 */ 258 sum += su & 0xff00; 259 } 260 REDUCE(sum, tmp); 261 return (~sum & 0xffff); 262} 263