1292407Sbr/*- 2292407Sbr * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org> 3292407Sbr * All rights reserved. 4292407Sbr * 5292407Sbr * Redistribution and use in source and binary forms, with or without 6292407Sbr * modification, are permitted provided that the following conditions 7292407Sbr * are met: 8292407Sbr * 1. Redistributions of source code must retain the above copyright 9292407Sbr * notice, this list of conditions and the following disclaimer. 10292407Sbr * 2. Redistributions in binary form must reproduce the above copyright 11292407Sbr * notice, this list of conditions and the following disclaimer in the 12292407Sbr * documentation and/or other materials provided with the distribution. 13292407Sbr * 14292407Sbr * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15292407Sbr * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16292407Sbr * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17292407Sbr * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18292407Sbr * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19292407Sbr * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20292407Sbr * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21292407Sbr * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22292407Sbr * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23292407Sbr * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24292407Sbr * SUCH DAMAGE. 25292407Sbr * 26292407Sbr * $FreeBSD: stable/11/sys/riscv/include/counter.h 328386 2018-01-25 02:45:21Z pkelsey $ 27292407Sbr */ 28292407Sbr 29292407Sbr#ifndef _MACHINE_COUNTER_H_ 30292407Sbr#define _MACHINE_COUNTER_H_ 31292407Sbr 32292407Sbr#include <sys/pcpu.h> 33292407Sbr#ifdef INVARIANTS 34292407Sbr#include <sys/proc.h> 35292407Sbr#endif 36292407Sbr 37292407Sbr#define counter_enter() critical_enter() 38292407Sbr#define counter_exit() critical_exit() 39292407Sbr 40292407Sbr#ifdef IN_SUBR_COUNTER_C 41292407Sbrstatic inline uint64_t 42292407Sbrcounter_u64_read_one(uint64_t *p, int cpu) 43292407Sbr{ 44292407Sbr 45292407Sbr return (*(uint64_t *)((char *)p + sizeof(struct pcpu) * cpu)); 46292407Sbr} 47292407Sbr 48292407Sbrstatic inline uint64_t 49292407Sbrcounter_u64_fetch_inline(uint64_t *p) 50292407Sbr{ 51292407Sbr uint64_t r; 52292407Sbr int i; 53292407Sbr 54292407Sbr r = 0; 55292407Sbr for (i = 0; i < mp_ncpus; i++) 56292407Sbr r += counter_u64_read_one((uint64_t *)p, i); 57292407Sbr 58292407Sbr return (r); 59292407Sbr} 60292407Sbr 61292407Sbr/* XXXKIB might interrupt increment */ 62292407Sbrstatic void 63292407Sbrcounter_u64_zero_one_cpu(void *arg) 64292407Sbr{ 65292407Sbr 66292407Sbr *((uint64_t *)((char *)arg + sizeof(struct pcpu) * 67292407Sbr PCPU_GET(cpuid))) = 0; 68292407Sbr} 69292407Sbr 70292407Sbrstatic inline void 71292407Sbrcounter_u64_zero_inline(counter_u64_t c) 72292407Sbr{ 73292407Sbr 74328386Spkelsey smp_rendezvous(smp_no_rendezvous_barrier, counter_u64_zero_one_cpu, 75328386Spkelsey smp_no_rendezvous_barrier, c); 76292407Sbr} 77292407Sbr#endif 78292407Sbr 79292407Sbr#define counter_u64_add_protected(c, inc) do { \ 80292407Sbr CRITICAL_ASSERT(curthread); \ 81292407Sbr *(uint64_t *)zpcpu_get(c) += (inc); \ 82292407Sbr} while (0) 83292407Sbr 84292407Sbrstatic inline void 85292407Sbrcounter_u64_add(counter_u64_t c, int64_t inc) 86292407Sbr{ 87292407Sbr 88292407Sbr counter_enter(); 89292407Sbr counter_u64_add_protected(c, inc); 90292407Sbr counter_exit(); 91292407Sbr} 92292407Sbr 93292407Sbr#endif /* ! _MACHINE_COUNTER_H_ */ 94