1347246Shselasky/*- 2347246Shselasky * SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0 3347246Shselasky * 4347246Shselasky * Copyright (c) 2016, Mellanox Technologies. All rights reserved. 5347246Shselasky * Copyright (c) 2017-2018, Broadcom Limited. All rights reserved. 6347246Shselasky * 7347246Shselasky * This software is available to you under a choice of one of two 8347246Shselasky * licenses. You may choose to be licensed under the terms of the GNU 9347246Shselasky * General Public License (GPL) Version 2, available from the file 10347246Shselasky * COPYING in the main directory of this source tree, or the 11347246Shselasky * OpenIB.org BSD license below: 12347246Shselasky * 13347246Shselasky * Redistribution and use in source and binary forms, with or 14347246Shselasky * without modification, are permitted provided that the following 15347246Shselasky * conditions are met: 16347246Shselasky * 17347246Shselasky * - Redistributions of source code must retain the above 18347246Shselasky * copyright notice, this list of conditions and the following 19347246Shselasky * disclaimer. 20347246Shselasky * 21347246Shselasky * - Redistributions in binary form must reproduce the above 22347246Shselasky * copyright notice, this list of conditions and the following 23347246Shselasky * disclaimer in the documentation and/or other materials 24347246Shselasky * provided with the distribution. 25347246Shselasky * 26347246Shselasky * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 27347246Shselasky * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 28347246Shselasky * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 29347246Shselasky * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 30347246Shselasky * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 31347246Shselasky * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 32347246Shselasky * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 33347246Shselasky * SOFTWARE. 34347246Shselasky * 35347246Shselasky * $FreeBSD: stable/11/sys/compat/linuxkpi/common/include/linux/net_dim.h 347796 2019-05-16 17:09:06Z hselasky $ 36347246Shselasky */ 37347246Shselasky 38347246Shselasky/* This file implements Dynamic Interrupt Moderation, DIM */ 39347246Shselasky 40347246Shselasky#ifndef NET_DIM_H 41347246Shselasky#define NET_DIM_H 42347246Shselasky 43347246Shselasky#include <asm/types.h> 44347246Shselasky 45347246Shselasky#include <linux/workqueue.h> 46347246Shselasky#include <linux/ktime.h> 47347246Shselasky 48347246Shselaskystruct net_dim_cq_moder { 49347246Shselasky u16 usec; 50347246Shselasky u16 pkts; 51347246Shselasky u8 cq_period_mode; 52347246Shselasky}; 53347246Shselasky 54347246Shselaskystruct net_dim_sample { 55347246Shselasky ktime_t time; 56347246Shselasky u32 pkt_ctr; 57347246Shselasky u32 byte_ctr; 58347246Shselasky u16 event_ctr; 59347246Shselasky}; 60347246Shselasky 61347246Shselaskystruct net_dim_stats { 62347246Shselasky int ppms; /* packets per msec */ 63347246Shselasky int bpms; /* bytes per msec */ 64347246Shselasky int epms; /* events per msec */ 65347246Shselasky}; 66347246Shselasky 67347246Shselaskystruct net_dim { /* Adaptive Moderation */ 68347246Shselasky u8 state; 69347246Shselasky struct net_dim_stats prev_stats; 70347246Shselasky struct net_dim_sample start_sample; 71347246Shselasky struct work_struct work; 72347246Shselasky u16 event_ctr; 73347246Shselasky u8 profile_ix; 74347246Shselasky u8 mode; 75347246Shselasky u8 tune_state; 76347246Shselasky u8 steps_right; 77347246Shselasky u8 steps_left; 78347246Shselasky u8 tired; 79347246Shselasky}; 80347246Shselasky 81347246Shselaskyenum { 82347246Shselasky NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE = 0x0, 83347246Shselasky NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE = 0x1, 84347246Shselasky NET_DIM_CQ_PERIOD_NUM_MODES = 0x2, 85347246Shselasky NET_DIM_CQ_PERIOD_MODE_DISABLED = 0xFF, 86347246Shselasky}; 87347246Shselasky 88347246Shselasky/* Adaptive moderation logic */ 89347246Shselaskyenum { 90347246Shselasky NET_DIM_START_MEASURE, 91347246Shselasky NET_DIM_MEASURE_IN_PROGRESS, 92347246Shselasky NET_DIM_APPLY_NEW_PROFILE, 93347246Shselasky}; 94347246Shselasky 95347246Shselaskyenum { 96347246Shselasky NET_DIM_PARKING_ON_TOP, 97347246Shselasky NET_DIM_PARKING_TIRED, 98347246Shselasky NET_DIM_GOING_RIGHT, 99347246Shselasky NET_DIM_GOING_LEFT, 100347246Shselasky}; 101347246Shselasky 102347246Shselaskyenum { 103347246Shselasky NET_DIM_STATS_WORSE, 104347246Shselasky NET_DIM_STATS_SAME, 105347246Shselasky NET_DIM_STATS_BETTER, 106347246Shselasky}; 107347246Shselasky 108347246Shselaskyenum { 109347246Shselasky NET_DIM_STEPPED, 110347246Shselasky NET_DIM_TOO_TIRED, 111347246Shselasky NET_DIM_ON_EDGE, 112347246Shselasky}; 113347246Shselasky 114347246Shselasky#define NET_DIM_PARAMS_NUM_PROFILES 5 115347246Shselasky/* Adaptive moderation profiles */ 116347246Shselasky#define NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256 117347246Shselasky#define NET_DIM_DEF_PROFILE_CQE 1 118347246Shselasky#define NET_DIM_DEF_PROFILE_EQE 1 119347246Shselasky 120347246Shselasky/* All profiles sizes must be NET_PARAMS_DIM_NUM_PROFILES */ 121347246Shselasky#define NET_DIM_EQE_PROFILES { \ 122347246Shselasky {1, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ 123347246Shselasky {8, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ 124347246Shselasky {64, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ 125347246Shselasky {128, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ 126347246Shselasky {256, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ 127347246Shselasky} 128347246Shselasky 129347246Shselasky#define NET_DIM_CQE_PROFILES { \ 130347246Shselasky {2, 256}, \ 131347246Shselasky {8, 128}, \ 132347246Shselasky {16, 64}, \ 133347246Shselasky {32, 64}, \ 134347246Shselasky {64, 64} \ 135347246Shselasky} 136347246Shselasky 137347246Shselaskystatic const struct net_dim_cq_moder 138347246Shselasky net_dim_profile[NET_DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = { 139347246Shselasky NET_DIM_EQE_PROFILES, 140347246Shselasky NET_DIM_CQE_PROFILES, 141347246Shselasky}; 142347246Shselasky 143347246Shselaskystatic inline struct net_dim_cq_moder 144347246Shselaskynet_dim_get_profile(u8 cq_period_mode, 145347246Shselasky int ix) 146347246Shselasky{ 147347246Shselasky struct net_dim_cq_moder cq_moder; 148347246Shselasky 149347246Shselasky cq_moder = net_dim_profile[cq_period_mode][ix]; 150347246Shselasky cq_moder.cq_period_mode = cq_period_mode; 151347246Shselasky return cq_moder; 152347246Shselasky} 153347246Shselasky 154347246Shselaskystatic inline struct net_dim_cq_moder 155347246Shselaskynet_dim_get_def_profile(u8 rx_cq_period_mode) 156347246Shselasky{ 157347246Shselasky int default_profile_ix; 158347246Shselasky 159347246Shselasky if (rx_cq_period_mode == NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE) 160347246Shselasky default_profile_ix = NET_DIM_DEF_PROFILE_CQE; 161347246Shselasky else /* NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE */ 162347246Shselasky default_profile_ix = NET_DIM_DEF_PROFILE_EQE; 163347246Shselasky 164347246Shselasky return net_dim_get_profile(rx_cq_period_mode, default_profile_ix); 165347246Shselasky} 166347246Shselasky 167347246Shselaskystatic inline bool 168347246Shselaskynet_dim_on_top(struct net_dim *dim) 169347246Shselasky{ 170347246Shselasky switch (dim->tune_state) { 171347246Shselasky case NET_DIM_PARKING_ON_TOP: 172347246Shselasky case NET_DIM_PARKING_TIRED: 173347246Shselasky return true; 174347246Shselasky case NET_DIM_GOING_RIGHT: 175347246Shselasky return (dim->steps_left > 1) && (dim->steps_right == 1); 176347246Shselasky default: /* NET_DIM_GOING_LEFT */ 177347246Shselasky return (dim->steps_right > 1) && (dim->steps_left == 1); 178347246Shselasky } 179347246Shselasky} 180347246Shselasky 181347246Shselaskystatic inline void 182347246Shselaskynet_dim_turn(struct net_dim *dim) 183347246Shselasky{ 184347246Shselasky switch (dim->tune_state) { 185347246Shselasky case NET_DIM_PARKING_ON_TOP: 186347246Shselasky case NET_DIM_PARKING_TIRED: 187347246Shselasky break; 188347246Shselasky case NET_DIM_GOING_RIGHT: 189347246Shselasky dim->tune_state = NET_DIM_GOING_LEFT; 190347246Shselasky dim->steps_left = 0; 191347246Shselasky break; 192347246Shselasky case NET_DIM_GOING_LEFT: 193347246Shselasky dim->tune_state = NET_DIM_GOING_RIGHT; 194347246Shselasky dim->steps_right = 0; 195347246Shselasky break; 196347246Shselasky } 197347246Shselasky} 198347246Shselasky 199347246Shselaskystatic inline int 200347246Shselaskynet_dim_step(struct net_dim *dim) 201347246Shselasky{ 202347246Shselasky if (dim->tired == (NET_DIM_PARAMS_NUM_PROFILES * 2)) 203347246Shselasky return NET_DIM_TOO_TIRED; 204347246Shselasky 205347246Shselasky switch (dim->tune_state) { 206347246Shselasky case NET_DIM_PARKING_ON_TOP: 207347246Shselasky case NET_DIM_PARKING_TIRED: 208347246Shselasky break; 209347246Shselasky case NET_DIM_GOING_RIGHT: 210347246Shselasky if (dim->profile_ix == (NET_DIM_PARAMS_NUM_PROFILES - 1)) 211347246Shselasky return NET_DIM_ON_EDGE; 212347246Shselasky dim->profile_ix++; 213347246Shselasky dim->steps_right++; 214347246Shselasky break; 215347246Shselasky case NET_DIM_GOING_LEFT: 216347246Shselasky if (dim->profile_ix == 0) 217347246Shselasky return NET_DIM_ON_EDGE; 218347246Shselasky dim->profile_ix--; 219347246Shselasky dim->steps_left++; 220347246Shselasky break; 221347246Shselasky } 222347246Shselasky 223347246Shselasky dim->tired++; 224347246Shselasky return NET_DIM_STEPPED; 225347246Shselasky} 226347246Shselasky 227347246Shselaskystatic inline void 228347246Shselaskynet_dim_park_on_top(struct net_dim *dim) 229347246Shselasky{ 230347246Shselasky dim->steps_right = 0; 231347246Shselasky dim->steps_left = 0; 232347246Shselasky dim->tired = 0; 233347246Shselasky dim->tune_state = NET_DIM_PARKING_ON_TOP; 234347246Shselasky} 235347246Shselasky 236347246Shselaskystatic inline void 237347246Shselaskynet_dim_park_tired(struct net_dim *dim) 238347246Shselasky{ 239347246Shselasky dim->steps_right = 0; 240347246Shselasky dim->steps_left = 0; 241347246Shselasky dim->tune_state = NET_DIM_PARKING_TIRED; 242347246Shselasky} 243347246Shselasky 244347246Shselaskystatic inline void 245347246Shselaskynet_dim_exit_parking(struct net_dim *dim) 246347246Shselasky{ 247347246Shselasky dim->tune_state = dim->profile_ix ? NET_DIM_GOING_LEFT : 248347246Shselasky NET_DIM_GOING_RIGHT; 249347246Shselasky net_dim_step(dim); 250347246Shselasky} 251347246Shselasky 252347246Shselasky#define IS_SIGNIFICANT_DIFF(val, ref) \ 253347246Shselasky (((100UL * abs((val) - (ref))) / (ref)) > 10) /* more than 10% 254347246Shselasky * difference */ 255347246Shselasky 256347246Shselaskystatic inline int 257347246Shselaskynet_dim_stats_compare(struct net_dim_stats *curr, 258347246Shselasky struct net_dim_stats *prev) 259347246Shselasky{ 260347246Shselasky if (!prev->bpms) 261347246Shselasky return curr->bpms ? NET_DIM_STATS_BETTER : 262347246Shselasky NET_DIM_STATS_SAME; 263347246Shselasky 264347246Shselasky if (IS_SIGNIFICANT_DIFF(curr->bpms, prev->bpms)) 265347246Shselasky return (curr->bpms > prev->bpms) ? NET_DIM_STATS_BETTER : 266347246Shselasky NET_DIM_STATS_WORSE; 267347246Shselasky 268347246Shselasky if (!prev->ppms) 269347246Shselasky return curr->ppms ? NET_DIM_STATS_BETTER : 270347246Shselasky NET_DIM_STATS_SAME; 271347246Shselasky 272347246Shselasky if (IS_SIGNIFICANT_DIFF(curr->ppms, prev->ppms)) 273347246Shselasky return (curr->ppms > prev->ppms) ? NET_DIM_STATS_BETTER : 274347246Shselasky NET_DIM_STATS_WORSE; 275347246Shselasky 276347246Shselasky if (!prev->epms) 277347246Shselasky return NET_DIM_STATS_SAME; 278347246Shselasky 279347246Shselasky if (IS_SIGNIFICANT_DIFF(curr->epms, prev->epms)) 280347246Shselasky return (curr->epms < prev->epms) ? NET_DIM_STATS_BETTER : 281347246Shselasky NET_DIM_STATS_WORSE; 282347246Shselasky 283347246Shselasky return NET_DIM_STATS_SAME; 284347246Shselasky} 285347246Shselasky 286347246Shselaskystatic inline bool 287347246Shselaskynet_dim_decision(struct net_dim_stats *curr_stats, 288347246Shselasky struct net_dim *dim) 289347246Shselasky{ 290347246Shselasky int prev_state = dim->tune_state; 291347246Shselasky int prev_ix = dim->profile_ix; 292347246Shselasky int stats_res; 293347246Shselasky int step_res; 294347246Shselasky 295347246Shselasky switch (dim->tune_state) { 296347246Shselasky case NET_DIM_PARKING_ON_TOP: 297347246Shselasky stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats); 298347246Shselasky if (stats_res != NET_DIM_STATS_SAME) 299347246Shselasky net_dim_exit_parking(dim); 300347246Shselasky break; 301347246Shselasky 302347246Shselasky case NET_DIM_PARKING_TIRED: 303347246Shselasky dim->tired--; 304347246Shselasky if (!dim->tired) 305347246Shselasky net_dim_exit_parking(dim); 306347246Shselasky break; 307347246Shselasky 308347246Shselasky case NET_DIM_GOING_RIGHT: 309347246Shselasky case NET_DIM_GOING_LEFT: 310347246Shselasky stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats); 311347246Shselasky if (stats_res != NET_DIM_STATS_BETTER) 312347246Shselasky net_dim_turn(dim); 313347246Shselasky 314347246Shselasky if (net_dim_on_top(dim)) { 315347246Shselasky net_dim_park_on_top(dim); 316347246Shselasky break; 317347246Shselasky } 318347246Shselasky step_res = net_dim_step(dim); 319347246Shselasky switch (step_res) { 320347246Shselasky case NET_DIM_ON_EDGE: 321347246Shselasky net_dim_park_on_top(dim); 322347246Shselasky break; 323347246Shselasky case NET_DIM_TOO_TIRED: 324347246Shselasky net_dim_park_tired(dim); 325347246Shselasky break; 326347246Shselasky } 327347246Shselasky 328347246Shselasky break; 329347246Shselasky } 330347246Shselasky 331347246Shselasky if ((prev_state != NET_DIM_PARKING_ON_TOP) || 332347246Shselasky (dim->tune_state != NET_DIM_PARKING_ON_TOP)) 333347246Shselasky dim->prev_stats = *curr_stats; 334347246Shselasky 335347246Shselasky return dim->profile_ix != prev_ix; 336347246Shselasky} 337347246Shselasky 338347246Shselaskystatic inline void 339347246Shselaskynet_dim_sample(u16 event_ctr, 340347246Shselasky u64 packets, 341347246Shselasky u64 bytes, 342347246Shselasky struct net_dim_sample *s) 343347246Shselasky{ 344347246Shselasky s->time = ktime_get(); 345347246Shselasky s->pkt_ctr = packets; 346347246Shselasky s->byte_ctr = bytes; 347347246Shselasky s->event_ctr = event_ctr; 348347246Shselasky} 349347246Shselasky 350347246Shselasky#define NET_DIM_NEVENTS 64 351347246Shselasky#define BIT_GAP(bits, end, start) ((((end) - (start)) + BIT_ULL(bits)) & (BIT_ULL(bits) - 1)) 352347246Shselasky 353347246Shselaskystatic inline void 354347246Shselaskynet_dim_calc_stats(struct net_dim_sample *start, 355347246Shselasky struct net_dim_sample *end, 356347246Shselasky struct net_dim_stats *curr_stats) 357347246Shselasky{ 358347246Shselasky /* u32 holds up to 71 minutes, should be enough */ 359347246Shselasky u32 delta_us = ktime_us_delta(end->time, start->time); 360347246Shselasky u32 npkts = BIT_GAP(BITS_PER_TYPE(u32), end->pkt_ctr, start->pkt_ctr); 361347246Shselasky u32 nbytes = BIT_GAP(BITS_PER_TYPE(u32), end->byte_ctr, 362347246Shselasky start->byte_ctr); 363347246Shselasky 364347246Shselasky if (!delta_us) 365347246Shselasky return; 366347246Shselasky 367347246Shselasky curr_stats->ppms = DIV_ROUND_UP(npkts * USEC_PER_MSEC, delta_us); 368347246Shselasky curr_stats->bpms = DIV_ROUND_UP(nbytes * USEC_PER_MSEC, delta_us); 369347246Shselasky curr_stats->epms = DIV_ROUND_UP(NET_DIM_NEVENTS * USEC_PER_MSEC, 370347246Shselasky delta_us); 371347246Shselasky} 372347246Shselasky 373347246Shselaskystatic inline void 374347246Shselaskynet_dim(struct net_dim *dim, 375347246Shselasky u64 packets, u64 bytes) 376347246Shselasky{ 377347246Shselasky struct net_dim_stats curr_stats; 378347246Shselasky struct net_dim_sample end_sample; 379347246Shselasky u16 nevents; 380347246Shselasky 381347246Shselasky dim->event_ctr++; 382347246Shselasky 383347246Shselasky switch (dim->state) { 384347246Shselasky case NET_DIM_MEASURE_IN_PROGRESS: 385347246Shselasky nevents = BIT_GAP(BITS_PER_TYPE(u16), 386347246Shselasky dim->event_ctr, 387347246Shselasky dim->start_sample.event_ctr); 388347246Shselasky if (nevents < NET_DIM_NEVENTS) 389347246Shselasky break; 390347246Shselasky net_dim_sample(dim->event_ctr, packets, bytes, &end_sample); 391347246Shselasky net_dim_calc_stats(&dim->start_sample, &end_sample, 392347246Shselasky &curr_stats); 393347246Shselasky if (net_dim_decision(&curr_stats, dim)) { 394347246Shselasky dim->state = NET_DIM_APPLY_NEW_PROFILE; 395347246Shselasky schedule_work(&dim->work); 396347246Shselasky break; 397347246Shselasky } 398347246Shselasky /* FALLTHROUGH */ 399347246Shselasky case NET_DIM_START_MEASURE: 400347246Shselasky net_dim_sample(dim->event_ctr, packets, bytes, &dim->start_sample); 401347246Shselasky dim->state = NET_DIM_MEASURE_IN_PROGRESS; 402347246Shselasky break; 403347246Shselasky case NET_DIM_APPLY_NEW_PROFILE: 404347246Shselasky break; 405347246Shselasky default: 406347246Shselasky break; 407347246Shselasky } 408347246Shselasky} 409347246Shselasky 410347246Shselasky#endif /* NET_DIM_H */ 411