1/* 2 * BSD LICENSE 3 * 4 * Copyright(c) 2017 Cavium, Inc.. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Cavium, Inc. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33/*$FreeBSD: stable/11/sys/dev/liquidio/base/lio_mem_ops.c 325618 2017-11-09 19:52:56Z sbruno $*/ 34 35#include "lio_bsd.h" 36#include "lio_common.h" 37#include "lio_droq.h" 38#include "lio_iq.h" 39#include "lio_response_manager.h" 40#include "lio_device.h" 41#include "lio_mem_ops.h" 42 43#define MEMOPS_IDX LIO_MAX_BAR1_MAP_INDEX 44 45#if BYTE_ORDER == BIG_ENDIAN 46static inline void 47lio_toggle_bar1_swapmode(struct octeon_device *oct, uint32_t idx) 48{ 49 uint32_t mask; 50 51 mask = oct->fn_list.bar1_idx_read(oct, idx); 52 mask = (mask & 0x2) ? (mask & ~2) : (mask | 2); 53 oct->fn_list.bar1_idx_write(oct, idx, mask); 54} 55 56#else /* BYTE_ORDER != BIG_ENDIAN */ 57#define lio_toggle_bar1_swapmode(oct, idx) 58#endif /* BYTE_ORDER == BIG_ENDIAN */ 59 60static inline void 61lio_write_bar1_mem8(struct octeon_device *oct, uint32_t reg, uint64_t val) 62{ 63 64 bus_space_write_1(oct->mem_bus_space[1].tag, 65 oct->mem_bus_space[1].handle, reg, val); 66} 67 68#ifdef __i386__ 69static inline uint32_t 70lio_read_bar1_mem32(struct octeon_device *oct, uint32_t reg) 71{ 72 73 return (bus_space_read_4(oct->mem_bus_space[1].tag, 74 oct->mem_bus_space[1].handle, reg)); 75} 76 77static inline void 78lio_write_bar1_mem32(struct octeon_device *oct, uint32_t reg, uint32_t val) 79{ 80 81 bus_space_write_4(oct->mem_bus_space[1].tag, 82 oct->mem_bus_space[1].handle, reg, val); 83} 84#endif 85 86static inline uint64_t 87lio_read_bar1_mem64(struct octeon_device *oct, uint32_t reg) 88{ 89 90#ifdef __i386__ 91 return (lio_read_bar1_mem32(oct, reg) | 92 ((uint64_t)lio_read_bar1_mem32(oct, reg + 4) << 32)); 93#else 94 return (bus_space_read_8(oct->mem_bus_space[1].tag, 95 oct->mem_bus_space[1].handle, reg)); 96#endif 97} 98 99static inline void 100lio_write_bar1_mem64(struct octeon_device *oct, uint32_t reg, uint64_t val) 101{ 102 103#ifdef __i386__ 104 lio_write_bar1_mem32(oct, reg, (uint32_t)val); 105 lio_write_bar1_mem32(oct, reg + 4, val >> 32); 106#else 107 bus_space_write_8(oct->mem_bus_space[1].tag, 108 oct->mem_bus_space[1].handle, reg, val); 109#endif 110} 111 112static void 113lio_pci_fastwrite(struct octeon_device *oct, uint32_t offset, 114 uint8_t *hostbuf, uint32_t len) 115{ 116 117 while ((len) && ((unsigned long)offset) & 7) { 118 lio_write_bar1_mem8(oct, offset++, *(hostbuf++)); 119 len--; 120 } 121 122 lio_toggle_bar1_swapmode(oct, MEMOPS_IDX); 123 124 while (len >= 8) { 125 lio_write_bar1_mem64(oct, offset, *((uint64_t *)hostbuf)); 126 offset += 8; 127 hostbuf += 8; 128 len -= 8; 129 } 130 131 lio_toggle_bar1_swapmode(oct, MEMOPS_IDX); 132 133 while (len--) 134 lio_write_bar1_mem8(oct, offset++, *(hostbuf++)); 135} 136 137static inline uint64_t 138lio_read_bar1_mem8(struct octeon_device *oct, uint32_t reg) 139{ 140 141 return (bus_space_read_1(oct->mem_bus_space[1].tag, 142 oct->mem_bus_space[1].handle, reg)); 143} 144 145static void 146lio_pci_fastread(struct octeon_device *oct, uint32_t offset, 147 uint8_t *hostbuf, uint32_t len) 148{ 149 150 while ((len) && ((unsigned long)offset) & 7) { 151 *(hostbuf++) = lio_read_bar1_mem8(oct, offset++); 152 len--; 153 } 154 155 lio_toggle_bar1_swapmode(oct, MEMOPS_IDX); 156 157 while (len >= 8) { 158 *((uint64_t *)hostbuf) = lio_read_bar1_mem64(oct, offset); 159 offset += 8; 160 hostbuf += 8; 161 len -= 8; 162 } 163 164 lio_toggle_bar1_swapmode(oct, MEMOPS_IDX); 165 166 while (len--) 167 *(hostbuf++) = lio_read_bar1_mem8(oct, offset++); 168} 169 170/* Core mem read/write with temporary bar1 settings. */ 171/* op = 1 to read, op = 0 to write. */ 172static void 173lio_pci_rw_core_mem(struct octeon_device *oct, uint64_t addr, 174 uint8_t *hostbuf, uint32_t len, uint32_t op) 175{ 176 uint64_t static_mapping_base; 177 uint32_t copy_len = 0, index_reg_val = 0; 178 uint32_t offset; 179 180 static_mapping_base = oct->console_nb_info.dram_region_base; 181 182 if (static_mapping_base && static_mapping_base == 183 (addr & 0xFFFFFFFFFFC00000ULL)) { 184 int bar1_index = oct->console_nb_info.bar1_index; 185 186 offset = (bar1_index << 22) + (addr & 0x3fffff); 187 188 if (op) 189 lio_pci_fastread(oct, offset, hostbuf, len); 190 else 191 lio_pci_fastwrite(oct, offset, hostbuf, len); 192 193 return; 194 } 195 mtx_lock(&oct->mem_access_lock); 196 197 /* Save the original index reg value. */ 198 index_reg_val = oct->fn_list.bar1_idx_read(oct, MEMOPS_IDX); 199 do { 200 oct->fn_list.bar1_idx_setup(oct, addr, MEMOPS_IDX, 1); 201 offset = (MEMOPS_IDX << 22) + (addr & 0x3fffff); 202 203 /* 204 * If operation crosses a 4MB boundary, split the transfer 205 * at the 4MB boundary. 206 */ 207 if (((addr + len - 1) & ~(0x3fffff)) != (addr & ~(0x3fffff))) { 208 copy_len = (uint32_t)(((addr & ~(0x3fffff)) + 209 (MEMOPS_IDX << 22)) - addr); 210 } else { 211 copy_len = len; 212 } 213 214 if (op) { /* read from core */ 215 lio_pci_fastread(oct, offset, hostbuf, 216 copy_len); 217 } else { 218 lio_pci_fastwrite(oct, offset, hostbuf, 219 copy_len); 220 } 221 222 len -= copy_len; 223 addr += copy_len; 224 hostbuf += copy_len; 225 226 } while (len); 227 228 oct->fn_list.bar1_idx_write(oct, MEMOPS_IDX, index_reg_val); 229 230 mtx_unlock(&oct->mem_access_lock); 231} 232 233void 234lio_pci_read_core_mem(struct octeon_device *oct, uint64_t coreaddr, 235 uint8_t *buf, uint32_t len) 236{ 237 238 lio_pci_rw_core_mem(oct, coreaddr, buf, len, 1); 239} 240 241void 242lio_pci_write_core_mem(struct octeon_device *oct, uint64_t coreaddr, 243 uint8_t *buf, uint32_t len) 244{ 245 246 lio_pci_rw_core_mem(oct, coreaddr, buf, len, 0); 247} 248 249uint64_t 250lio_read_device_mem64(struct octeon_device *oct, uint64_t coreaddr) 251{ 252 __be64 ret; 253 254 lio_pci_rw_core_mem(oct, coreaddr, (uint8_t *)&ret, 8, 1); 255 256 return (be64toh(ret)); 257} 258 259uint32_t 260lio_read_device_mem32(struct octeon_device *oct, uint64_t coreaddr) 261{ 262 __be32 ret; 263 264 lio_pci_rw_core_mem(oct, coreaddr, (uint8_t *)&ret, 4, 1); 265 266 return (be32toh(ret)); 267} 268 269void 270lio_write_device_mem32(struct octeon_device *oct, uint64_t coreaddr, 271 uint32_t val) 272{ 273 __be32 t = htobe32(val); 274 275 lio_pci_rw_core_mem(oct, coreaddr, (uint8_t *)&t, 4, 0); 276} 277