in_cksum.c revision 86144
186144Stmm/*- 286144Stmm * Copyright (c) 1990 The Regents of the University of California. 386144Stmm * All rights reserved. 486144Stmm * 586144Stmm * Redistribution and use in source and binary forms, with or without 686144Stmm * modification, are permitted provided that the following conditions 786144Stmm * are met: 886144Stmm * 1. Redistributions of source code must retain the above copyright 986144Stmm * notice, this list of conditions and the following disclaimer. 1086144Stmm * 2. Redistributions in binary form must reproduce the above copyright 1186144Stmm * notice, this list of conditions and the following disclaimer in the 1286144Stmm * documentation and/or other materials provided with the distribution. 1386144Stmm * 3. All advertising materials mentioning features or use of this software 1486144Stmm * must display the following acknowledgement: 1586144Stmm * This product includes software developed by the University of 1686144Stmm * California, Berkeley and its contributors. 1786144Stmm * 4. Neither the name of the University nor the names of its contributors 1886144Stmm * may be used to endorse or promote products derived from this software 1986144Stmm * without specific prior written permission. 2086144Stmm * 2186144Stmm * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 2286144Stmm * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 2386144Stmm * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2486144Stmm * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 2586144Stmm * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2686144Stmm * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2786144Stmm * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2886144Stmm * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2986144Stmm * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 3086144Stmm * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 3186144Stmm * SUCH DAMAGE. 3286144Stmm */ 3386144Stmm/*- 3486144Stmm * Copyright (c) 2001 by Thomas Moestl <tmm@FreeBSD.org>. 3586144Stmm * All rights reserved. 3686144Stmm * 3786144Stmm * Redistribution and use in source and binary forms, with or without 3886144Stmm * modification, are permitted provided that the following conditions 3986144Stmm * are met: 4086144Stmm * 1. Redistributions of source code must retain the above copyright 4186144Stmm * notice, this list of conditions and the following disclaimer. 4286144Stmm * 2. Redistributions in binary form must reproduce the above copyright 4386144Stmm * notice, this list of conditions and the following disclaimer in the 4486144Stmm * documentation and/or other materials provided with the distribution. 4586144Stmm * 4686144Stmm * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 4786144Stmm * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 4886144Stmm * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 4986144Stmm * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 5086144Stmm * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 5186144Stmm * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 5286144Stmm * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 5386144Stmm * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 5486144Stmm * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 5586144Stmm * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 5686144Stmm * 5786144Stmm * from tahoe: in_cksum.c 1.2 86/01/05 5886144Stmm * from: @(#)in_cksum.c 1.3 (Berkeley) 1/19/91 5986144Stmm * from: FreeBSD: src/sys/i386/i386/in_cksum.c,v 1.22 2000/11/25 6086144Stmm * 6186144Stmm * $FreeBSD: head/sys/sparc64/sparc64/in_cksum.c 86144 2001-11-06 20:05:01Z tmm $ 6286144Stmm */ 6386144Stmm 6486144Stmm#include <sys/param.h> 6586144Stmm#include <sys/systm.h> 6686144Stmm#include <sys/mbuf.h> 6786144Stmm 6886144Stmm#include <netinet/in.h> 6986144Stmm#include <netinet/in_systm.h> 7086144Stmm#include <netinet/ip.h> 7186144Stmm 7286144Stmm#include <machine/in_cksum.h> 7386144Stmm 7486144Stmm/* 7586144Stmm * Checksum routine for Internet Protocol family headers. 7686144Stmm * 7786144Stmm * This routine is very heavily used in the network 7886144Stmm * code and should be modified for each CPU to be as fast as possible. 7986144Stmm * 8086144Stmm * This implementation is a sparc64 version. Most code was taken over and 8186144Stmm * adapted from the i386. Some optimizations were changed to achieve (hopefully) 8286144Stmm * better performance. 8386144Stmm * This uses 64 bit loads, but 32 bit additions due to the lack of a 64-bit 8486144Stmm * add-with-carry operation. 8586144Stmm */ 8686144Stmm 8786144Stmm/* 8886144Stmm * REDUCE() is actually not used that frequently... maybe a C implementation 8986144Stmm * would suffice. 9086144Stmm */ 9186144Stmm#define REDUCE(sum, tmp) __asm __volatile( \ 9286144Stmm "sll %2, 16, %1\n" \ 9386144Stmm "addcc %2, %1, %0\n" \ 9486144Stmm "srl %0, 16, %0\n" \ 9586144Stmm "addc %0, 0, %0" : "=r" (sum), "=r" (tmp) : "0" (sum)) 9686144Stmm 9786144Stmm/* 9886144Stmm * Note that some of these macros depend on the flags being preserved between 9986144Stmm * calls, so they should not be intermixed with other C statements. 10086144Stmm */ 10186144Stmm#define LD64_ADD32(sum, tmp, addr, n, mod) __asm __volatile( \ 10286144Stmm "ldx [%3 + " #n "], %1\n" \ 10386144Stmm "add" #mod " %2, %1, %0\n" \ 10486144Stmm "srlx %1, 32, %1\n" \ 10586144Stmm "addccc %0, %1, %0" : "=r" (sum), "=r" (tmp) : "0" (sum), "r" (addr)) 10686144Stmm 10786144Stmm#define LD32_ADD32(sum, tmp, addr, n, mod) __asm __volatile( \ 10886144Stmm "lduw [%3 + " #n "], %1\n" \ 10986144Stmm "add" #mod " %2, %1, %0\n" \ 11086144Stmm : "=r" (sum), "=r" (tmp) : "0" (sum), "r" (addr)) 11186144Stmm 11286144Stmm#define MOP(sum) __asm __volatile( \ 11386144Stmm "addc %1, 0, %0" : "=r" (sum) : "0" (sum)) 11486144Stmm 11586144Stmmu_short 11686144Stmmin_cksum_skip(struct mbuf *m, int len, int skip) 11786144Stmm{ 11886144Stmm u_short *w; 11986144Stmm unsigned long tmp, sum = 0; 12086144Stmm int mlen = 0; 12186144Stmm int byte_swapped = 0; 12286144Stmm u_short su = 0; 12386144Stmm 12486144Stmm len -= skip; 12586144Stmm for (; skip > 0 && m != NULL; m = m->m_next) { 12686144Stmm if (m->m_len > skip) { 12786144Stmm mlen = m->m_len - skip; 12886144Stmm w = (u_short *)(mtod(m, u_char *) + skip); 12986144Stmm goto skip_start; 13086144Stmm } else 13186144Stmm skip -= m->m_len; 13286144Stmm } 13386144Stmm 13486144Stmm for (; m != NULL && len > 0; m = m->m_next) { 13586144Stmm if (m->m_len == 0) 13686144Stmm continue; 13786144Stmm w = mtod(m, u_short *); 13886144Stmm if (mlen == -1) { 13986144Stmm /* 14086144Stmm * The first byte of this mbuf is the continuation 14186144Stmm * of a word spanning between this mbuf and the 14286144Stmm * last mbuf. 14386144Stmm * 14486144Stmm * The high order byte of su is already saved when 14586144Stmm * scanning previous mbuf. sum was REDUCEd when we 14686144Stmm * found mlen == -1 14786144Stmm */ 14886144Stmm sum += su | *(u_char *)w; 14986144Stmm w = (u_short *)((u_char *)w + 1); 15086144Stmm mlen = m->m_len - 1; 15186144Stmm len--; 15286144Stmm } else 15386144Stmm mlen = m->m_len; 15486144Stmmskip_start: 15586144Stmm if (len < mlen) 15686144Stmm mlen = len; 15786144Stmm len -= mlen; 15886144Stmm /* 15986144Stmm * Force to a 8-byte boundary first so that we can use 16086144Stmm * LD64_ADD32. 16186144Stmm */ 16286144Stmm if (((u_long)w & 7) != 0) { 16386144Stmm REDUCE(sum, tmp); 16486144Stmm if (((u_long)w & 1) != 0 && mlen >= 1) { 16586144Stmm sum <<= 8; 16686144Stmm su = *(u_char *)w << 8; 16786144Stmm w = (u_short *)((u_char *)w + 1); 16886144Stmm mlen--; 16986144Stmm byte_swapped = 1; 17086144Stmm } 17186144Stmm if (((u_long)w & 2) != 0 && mlen >= 2) { 17286144Stmm sum += *w++; 17386144Stmm mlen -= 2; 17486144Stmm } 17586144Stmm if (((u_long)w & 4) != 0 && mlen >= 4) { 17686144Stmm LD32_ADD32(sum, tmp, w, 0, cc); 17786144Stmm MOP(sum); 17886144Stmm w += 2; 17986144Stmm mlen -= 4; 18086144Stmm } 18186144Stmm } 18286144Stmm /* 18386144Stmm * Do as much of the checksum as possible 64 bits at at time. 18486144Stmm * In fact, this loop is unrolled to make overhead from 18586144Stmm * branches &c small. 18686144Stmm */ 18786144Stmm for (; mlen >= 64; mlen -= 64) { 18886144Stmm LD64_ADD32(sum, tmp, w, 0, cc); 18986144Stmm LD64_ADD32(sum, tmp, w, 8, ccc); 19086144Stmm LD64_ADD32(sum, tmp, w, 16, ccc); 19186144Stmm LD64_ADD32(sum, tmp, w, 24, ccc); 19286144Stmm LD64_ADD32(sum, tmp, w, 32, ccc); 19386144Stmm LD64_ADD32(sum, tmp, w, 40, ccc); 19486144Stmm LD64_ADD32(sum, tmp, w, 48, ccc); 19586144Stmm LD64_ADD32(sum, tmp, w, 56, ccc); 19686144Stmm MOP(sum); 19786144Stmm w += 32; 19886144Stmm } 19986144Stmm if (mlen >= 32) { 20086144Stmm LD64_ADD32(sum, tmp, w, 0, cc); 20186144Stmm LD64_ADD32(sum, tmp, w, 8, ccc); 20286144Stmm LD64_ADD32(sum, tmp, w, 16, ccc); 20386144Stmm LD64_ADD32(sum, tmp, w, 24, ccc); 20486144Stmm MOP(sum); 20586144Stmm w += 16; 20686144Stmm mlen -= 32; 20786144Stmm } 20886144Stmm if (mlen >= 16) { 20986144Stmm LD64_ADD32(sum, tmp, w, 0, cc); 21086144Stmm LD64_ADD32(sum, tmp, w, 8, ccc); 21186144Stmm MOP(sum); 21286144Stmm w += 8; 21386144Stmm mlen -= 16; 21486144Stmm } 21586144Stmm if (mlen >= 8) { 21686144Stmm LD64_ADD32(sum, tmp, w, 0, cc); 21786144Stmm MOP(sum); 21886144Stmm w += 4; 21986144Stmm mlen -= 8; 22086144Stmm } 22186144Stmm REDUCE(sum, tmp); 22286144Stmm while ((mlen -= 2) >= 0) 22386144Stmm sum += *w++; 22486144Stmm if (byte_swapped) { 22586144Stmm sum <<= 8; 22686144Stmm byte_swapped = 0; 22786144Stmm if (mlen == -1) { 22886144Stmm su |= *(u_char *)w; 22986144Stmm sum += su; 23086144Stmm mlen = 0; 23186144Stmm } else 23286144Stmm mlen = -1; 23386144Stmm } else if (mlen == -1) { 23486144Stmm /* 23586144Stmm * This mbuf has odd number of bytes. 23686144Stmm * There could be a word split betwen 23786144Stmm * this mbuf and the next mbuf. 23886144Stmm * Save the last byte (to prepend to next mbuf). 23986144Stmm */ 24086144Stmm su = *(u_char *)w << 8; 24186144Stmm } 24286144Stmm } 24386144Stmm 24486144Stmm if (len) 24586144Stmm printf("%s: out of data by %d\n", __func__, len); 24686144Stmm if (mlen == -1) { 24786144Stmm /* The last mbuf has odd # of bytes. Follow the 24886144Stmm standard (the odd byte is shifted left by 8 bits) */ 24986144Stmm sum += su & 0xff00; 25086144Stmm } 25186144Stmm REDUCE(sum, tmp); 25286144Stmm return (~sum & 0xffff); 25386144Stmm} 254