1/*- 2 * SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0 3 * 4 * Copyright (c) 2016, Mellanox Technologies. All rights reserved. 5 * Copyright (c) 2017-2018, Broadcom Limited. All rights reserved. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * OpenIB.org BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or 14 * without modification, are permitted provided that the following 15 * conditions are met: 16 * 17 * - Redistributions of source code must retain the above 18 * copyright notice, this list of conditions and the following 19 * disclaimer. 20 * 21 * - Redistributions in binary form must reproduce the above 22 * copyright notice, this list of conditions and the following 23 * disclaimer in the documentation and/or other materials 24 * provided with the distribution. 25 * 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 33 * SOFTWARE. 34 * 35 * $FreeBSD$ 36 */ 37 38/* This file implements Dynamic Interrupt Moderation, DIM */ 39 40#ifndef NET_DIM_H 41#define NET_DIM_H 42 43#include <asm/types.h> 44 45#include <linux/workqueue.h> 46#include <linux/ktime.h> 47 48struct net_dim_cq_moder { 49 u16 usec; 50 u16 pkts; 51 u8 cq_period_mode; 52}; 53 54struct net_dim_sample { 55 ktime_t time; 56 u32 pkt_ctr; 57 u32 byte_ctr; 58 u16 event_ctr; 59}; 60 61struct net_dim_stats { 62 int ppms; /* packets per msec */ 63 int bpms; /* bytes per msec */ 64 int epms; /* events per msec */ 65}; 66 67struct net_dim { /* Adaptive Moderation */ 68 u8 state; 69 struct net_dim_stats prev_stats; 70 struct net_dim_sample start_sample; 71 struct work_struct work; 72 u16 event_ctr; 73 u8 profile_ix; 74 u8 mode; 75 u8 tune_state; 76 u8 steps_right; 77 u8 steps_left; 78 u8 tired; 79}; 80 81enum { 82 NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE = 0x0, 83 NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE = 0x1, 84 NET_DIM_CQ_PERIOD_NUM_MODES = 0x2, 85 NET_DIM_CQ_PERIOD_MODE_DISABLED = 0xFF, 86}; 87 88/* Adaptive moderation logic */ 89enum { 90 NET_DIM_START_MEASURE, 91 NET_DIM_MEASURE_IN_PROGRESS, 92 NET_DIM_APPLY_NEW_PROFILE, 93}; 94 95enum { 96 NET_DIM_PARKING_ON_TOP, 97 NET_DIM_PARKING_TIRED, 98 NET_DIM_GOING_RIGHT, 99 NET_DIM_GOING_LEFT, 100}; 101 102enum { 103 NET_DIM_STATS_WORSE, 104 NET_DIM_STATS_SAME, 105 NET_DIM_STATS_BETTER, 106}; 107 108enum { 109 NET_DIM_STEPPED, 110 NET_DIM_TOO_TIRED, 111 NET_DIM_ON_EDGE, 112}; 113 114#define NET_DIM_PARAMS_NUM_PROFILES 5 115/* Adaptive moderation profiles */ 116#define NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256 117#define NET_DIM_DEF_PROFILE_CQE 1 118#define NET_DIM_DEF_PROFILE_EQE 1 119 120/* All profiles sizes must be NET_PARAMS_DIM_NUM_PROFILES */ 121#define NET_DIM_EQE_PROFILES { \ 122 {1, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ 123 {8, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ 124 {64, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ 125 {128, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ 126 {256, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ 127} 128 129#define NET_DIM_CQE_PROFILES { \ 130 {2, 256}, \ 131 {8, 128}, \ 132 {16, 64}, \ 133 {32, 64}, \ 134 {64, 64} \ 135} 136 137static const struct net_dim_cq_moder 138 net_dim_profile[NET_DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = { 139 NET_DIM_EQE_PROFILES, 140 NET_DIM_CQE_PROFILES, 141}; 142 143static inline struct net_dim_cq_moder 144net_dim_get_profile(u8 cq_period_mode, 145 int ix) 146{ 147 struct net_dim_cq_moder cq_moder; 148 149 cq_moder = net_dim_profile[cq_period_mode][ix]; 150 cq_moder.cq_period_mode = cq_period_mode; 151 return cq_moder; 152} 153 154static inline struct net_dim_cq_moder 155net_dim_get_def_profile(u8 rx_cq_period_mode) 156{ 157 int default_profile_ix; 158 159 if (rx_cq_period_mode == NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE) 160 default_profile_ix = NET_DIM_DEF_PROFILE_CQE; 161 else /* NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE */ 162 default_profile_ix = NET_DIM_DEF_PROFILE_EQE; 163 164 return net_dim_get_profile(rx_cq_period_mode, default_profile_ix); 165} 166 167static inline bool 168net_dim_on_top(struct net_dim *dim) 169{ 170 switch (dim->tune_state) { 171 case NET_DIM_PARKING_ON_TOP: 172 case NET_DIM_PARKING_TIRED: 173 return true; 174 case NET_DIM_GOING_RIGHT: 175 return (dim->steps_left > 1) && (dim->steps_right == 1); 176 default: /* NET_DIM_GOING_LEFT */ 177 return (dim->steps_right > 1) && (dim->steps_left == 1); 178 } 179} 180 181static inline void 182net_dim_turn(struct net_dim *dim) 183{ 184 switch (dim->tune_state) { 185 case NET_DIM_PARKING_ON_TOP: 186 case NET_DIM_PARKING_TIRED: 187 break; 188 case NET_DIM_GOING_RIGHT: 189 dim->tune_state = NET_DIM_GOING_LEFT; 190 dim->steps_left = 0; 191 break; 192 case NET_DIM_GOING_LEFT: 193 dim->tune_state = NET_DIM_GOING_RIGHT; 194 dim->steps_right = 0; 195 break; 196 } 197} 198 199static inline int 200net_dim_step(struct net_dim *dim) 201{ 202 if (dim->tired == (NET_DIM_PARAMS_NUM_PROFILES * 2)) 203 return NET_DIM_TOO_TIRED; 204 205 switch (dim->tune_state) { 206 case NET_DIM_PARKING_ON_TOP: 207 case NET_DIM_PARKING_TIRED: 208 break; 209 case NET_DIM_GOING_RIGHT: 210 if (dim->profile_ix == (NET_DIM_PARAMS_NUM_PROFILES - 1)) 211 return NET_DIM_ON_EDGE; 212 dim->profile_ix++; 213 dim->steps_right++; 214 break; 215 case NET_DIM_GOING_LEFT: 216 if (dim->profile_ix == 0) 217 return NET_DIM_ON_EDGE; 218 dim->profile_ix--; 219 dim->steps_left++; 220 break; 221 } 222 223 dim->tired++; 224 return NET_DIM_STEPPED; 225} 226 227static inline void 228net_dim_park_on_top(struct net_dim *dim) 229{ 230 dim->steps_right = 0; 231 dim->steps_left = 0; 232 dim->tired = 0; 233 dim->tune_state = NET_DIM_PARKING_ON_TOP; 234} 235 236static inline void 237net_dim_park_tired(struct net_dim *dim) 238{ 239 dim->steps_right = 0; 240 dim->steps_left = 0; 241 dim->tune_state = NET_DIM_PARKING_TIRED; 242} 243 244static inline void 245net_dim_exit_parking(struct net_dim *dim) 246{ 247 dim->tune_state = dim->profile_ix ? NET_DIM_GOING_LEFT : 248 NET_DIM_GOING_RIGHT; 249 net_dim_step(dim); 250} 251 252#define IS_SIGNIFICANT_DIFF(val, ref) \ 253 (((100UL * abs((val) - (ref))) / (ref)) > 10) /* more than 10% 254 * difference */ 255 256static inline int 257net_dim_stats_compare(struct net_dim_stats *curr, 258 struct net_dim_stats *prev) 259{ 260 if (!prev->bpms) 261 return curr->bpms ? NET_DIM_STATS_BETTER : 262 NET_DIM_STATS_SAME; 263 264 if (IS_SIGNIFICANT_DIFF(curr->bpms, prev->bpms)) 265 return (curr->bpms > prev->bpms) ? NET_DIM_STATS_BETTER : 266 NET_DIM_STATS_WORSE; 267 268 if (!prev->ppms) 269 return curr->ppms ? NET_DIM_STATS_BETTER : 270 NET_DIM_STATS_SAME; 271 272 if (IS_SIGNIFICANT_DIFF(curr->ppms, prev->ppms)) 273 return (curr->ppms > prev->ppms) ? NET_DIM_STATS_BETTER : 274 NET_DIM_STATS_WORSE; 275 276 if (!prev->epms) 277 return NET_DIM_STATS_SAME; 278 279 if (IS_SIGNIFICANT_DIFF(curr->epms, prev->epms)) 280 return (curr->epms < prev->epms) ? NET_DIM_STATS_BETTER : 281 NET_DIM_STATS_WORSE; 282 283 return NET_DIM_STATS_SAME; 284} 285 286static inline bool 287net_dim_decision(struct net_dim_stats *curr_stats, 288 struct net_dim *dim) 289{ 290 int prev_state = dim->tune_state; 291 int prev_ix = dim->profile_ix; 292 int stats_res; 293 int step_res; 294 295 switch (dim->tune_state) { 296 case NET_DIM_PARKING_ON_TOP: 297 stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats); 298 if (stats_res != NET_DIM_STATS_SAME) 299 net_dim_exit_parking(dim); 300 break; 301 302 case NET_DIM_PARKING_TIRED: 303 dim->tired--; 304 if (!dim->tired) 305 net_dim_exit_parking(dim); 306 break; 307 308 case NET_DIM_GOING_RIGHT: 309 case NET_DIM_GOING_LEFT: 310 stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats); 311 if (stats_res != NET_DIM_STATS_BETTER) 312 net_dim_turn(dim); 313 314 if (net_dim_on_top(dim)) { 315 net_dim_park_on_top(dim); 316 break; 317 } 318 step_res = net_dim_step(dim); 319 switch (step_res) { 320 case NET_DIM_ON_EDGE: 321 net_dim_park_on_top(dim); 322 break; 323 case NET_DIM_TOO_TIRED: 324 net_dim_park_tired(dim); 325 break; 326 } 327 328 break; 329 } 330 331 if ((prev_state != NET_DIM_PARKING_ON_TOP) || 332 (dim->tune_state != NET_DIM_PARKING_ON_TOP)) 333 dim->prev_stats = *curr_stats; 334 335 return dim->profile_ix != prev_ix; 336} 337 338static inline void 339net_dim_sample(u16 event_ctr, 340 u64 packets, 341 u64 bytes, 342 struct net_dim_sample *s) 343{ 344 s->time = ktime_get(); 345 s->pkt_ctr = packets; 346 s->byte_ctr = bytes; 347 s->event_ctr = event_ctr; 348} 349 350#define NET_DIM_NEVENTS 64 351#define BIT_GAP(bits, end, start) ((((end) - (start)) + BIT_ULL(bits)) & (BIT_ULL(bits) - 1)) 352 353static inline void 354net_dim_calc_stats(struct net_dim_sample *start, 355 struct net_dim_sample *end, 356 struct net_dim_stats *curr_stats) 357{ 358 /* u32 holds up to 71 minutes, should be enough */ 359 u32 delta_us = ktime_us_delta(end->time, start->time); 360 u32 npkts = BIT_GAP(BITS_PER_TYPE(u32), end->pkt_ctr, start->pkt_ctr); 361 u32 nbytes = BIT_GAP(BITS_PER_TYPE(u32), end->byte_ctr, 362 start->byte_ctr); 363 364 if (!delta_us) 365 return; 366 367 curr_stats->ppms = DIV_ROUND_UP(npkts * USEC_PER_MSEC, delta_us); 368 curr_stats->bpms = DIV_ROUND_UP(nbytes * USEC_PER_MSEC, delta_us); 369 curr_stats->epms = DIV_ROUND_UP(NET_DIM_NEVENTS * USEC_PER_MSEC, 370 delta_us); 371} 372 373static inline void 374net_dim(struct net_dim *dim, 375 u64 packets, u64 bytes) 376{ 377 struct net_dim_stats curr_stats; 378 struct net_dim_sample end_sample; 379 u16 nevents; 380 381 dim->event_ctr++; 382 383 switch (dim->state) { 384 case NET_DIM_MEASURE_IN_PROGRESS: 385 nevents = BIT_GAP(BITS_PER_TYPE(u16), 386 dim->event_ctr, 387 dim->start_sample.event_ctr); 388 if (nevents < NET_DIM_NEVENTS) 389 break; 390 net_dim_sample(dim->event_ctr, packets, bytes, &end_sample); 391 net_dim_calc_stats(&dim->start_sample, &end_sample, 392 &curr_stats); 393 if (net_dim_decision(&curr_stats, dim)) { 394 dim->state = NET_DIM_APPLY_NEW_PROFILE; 395 schedule_work(&dim->work); 396 break; 397 } 398 /* FALLTHROUGH */ 399 case NET_DIM_START_MEASURE: 400 net_dim_sample(dim->event_ctr, packets, bytes, &dim->start_sample); 401 dim->state = NET_DIM_MEASURE_IN_PROGRESS; 402 break; 403 case NET_DIM_APPLY_NEW_PROFILE: 404 break; 405 default: 406 break; 407 } 408} 409 410#endif /* NET_DIM_H */ 411