ethernet/sfc/efx_channels.c

156 	 * We need a channel per event queue, plus a VI per tx queue.
273 			/* Fall back to single channel MSI */
366 	struct efx_channel *channel;
374 	efx_for_each_channel(channel, efx) {
378 		irq_set_affinity_hint(channel->irq, cpumask_of(cpu));
384 	struct efx_channel *channel;
386 	efx_for_each_channel(channel, efx)
387 		irq_set_affinity_hint(channel->irq, NULL);
403 	struct efx_channel *channel;
406 	efx_for_each_channel(channel, efx)
407 		channel->irq = 0;
420  * Event queue memory allocations are done only once.  If the channel
422  * errors during channel reset and also simplifies interrupt handling.
424 int efx_probe_eventq(struct efx_channel *channel)
426 	struct efx_nic *efx = channel->efx;
430 		  "chan %d create event queue\n", channel->channel);
437 	channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1;
439 	return efx_nic_probe_eventq(channel);
442 /* Prepare channel's event queue */
443 int efx_init_eventq(struct efx_channel *channel)
445 	struct efx_nic *efx = channel->efx;
448 	EFX_WARN_ON_PARANOID(channel->eventq_init);
451 		  "chan %d init event queue\n", channel->channel);
453 	rc = efx_nic_init_eventq(channel);
455 		efx->type->push_irq_moderation(channel);
456 		channel->eventq_read_ptr = 0;
457 		channel->eventq_init = true;
463 void efx_start_eventq(struct efx_channel *channel)
465 	netif_dbg(channel->efx, ifup, channel->efx->net_dev,
466 		  "chan %d start event queue\n", channel->channel);
469 	channel->enabled = true;
472 	napi_enable(&channel->napi_str);
473 	efx_nic_eventq_read_ack(channel);
477 void efx_stop_eventq(struct efx_channel *channel)
479 	if (!channel->enabled)
482 	napi_disable(&channel->napi_str);
483 	channel->enabled = false;
486 void efx_fini_eventq(struct efx_channel *channel)
488 	if (!channel->eventq_init)
491 	netif_dbg(channel->efx, drv, channel->efx->net_dev,
492 		  "chan %d fini event queue\n", channel->channel);
494 	efx_nic_fini_eventq(channel);
495 	channel->eventq_init = false;
498 void efx_remove_eventq(struct efx_channel *channel)
500 	netif_dbg(channel->efx, drv, channel->efx->net_dev,
501 		  "chan %d remove event queue\n", channel->channel);
503 	efx_nic_remove_eventq(channel);
516 	struct efx_channel *channel;
519 	channel = container_of(dwork, struct efx_channel, filter_work);
520 	time = jiffies - channel->rfs_last_expiry;
521 	quota = channel->rfs_filter_count * time / (30 * HZ);
522 	if (quota >= 20 && __efx_filter_rfs_expire(channel, min(channel->rfs_filter_count, quota)))
523 		channel->rfs_last_expiry += time;
529 /* Allocate and initialise a channel structure. */
534 	struct efx_channel *channel;
537 	channel = kzalloc(sizeof(*channel), GFP_KERNEL);
538 	if (!channel)
541 	channel->efx = efx;
542 	channel->channel = i;
543 	channel->type = &efx_default_channel_type;
546 		tx_queue = &channel->tx_queue[j];
550 		tx_queue->channel = channel;
554 	INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);
557 	rx_queue = &channel->rx_queue;
561 	return channel;
569 		efx->channel[i] = efx_alloc_channel(efx, i);
570 		if (!efx->channel[i])
591 		if (efx->channel[i]) {
592 			kfree(efx->channel[i]);
593 			efx->channel[i] = NULL;
597 /* Allocate and initialise a channel structure, copying parameters
598  * (but not resources) from an old channel structure.
604 	struct efx_channel *channel;
607 	channel = kmalloc(sizeof(*channel), GFP_KERNEL);
608 	if (!channel)
611 	*channel = *old_channel;
613 	channel->napi_dev = NULL;
614 	INIT_HLIST_NODE(&channel->napi_str.napi_hash_node);
615 	channel->napi_str.napi_id = 0;
616 	channel->napi_str.state = 0;
617 	memset(&channel->eventq, 0, sizeof(channel->eventq));
620 		tx_queue = &channel->tx_queue[j];
621 		if (tx_queue->channel)
622 			tx_queue->channel = channel;
628 	rx_queue = &channel->rx_queue;
633 	INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);
636 	return channel;
639 static int efx_probe_channel(struct efx_channel *channel)
645 	netif_dbg(channel->efx, probe, channel->efx->net_dev,
646 		  "creating channel %d\n", channel->channel);
648 	rc = channel->type->pre_probe(channel);
652 	rc = efx_probe_eventq(channel);
656 	efx_for_each_channel_tx_queue(tx_queue, channel) {
662 	efx_for_each_channel_rx_queue(rx_queue, channel) {
668 	channel->rx_list = NULL;
673 	efx_remove_channel(channel);
677 static void efx_get_channel_name(struct efx_channel *channel, char *buf,
680 	struct efx_nic *efx = channel->efx;
684 	number = channel->channel;
703 	struct efx_channel *channel;
705 	efx_for_each_channel(channel, efx)
706 		channel->type->get_name(channel,
707 					efx->msi_context[channel->channel].name,
713 	struct efx_channel *channel;
721 	efx_for_each_channel_rev(channel, efx) {
722 		rc = efx_probe_channel(channel);
725 				  "failed to create channel %d\n",
726 				  channel->channel);
739 void efx_remove_channel(struct efx_channel *channel)
744 	netif_dbg(channel->efx, drv, channel->efx->net_dev,
745 		  "destroy chan %d\n", channel->channel);
747 	efx_for_each_channel_rx_queue(rx_queue, channel)
749 	efx_for_each_channel_tx_queue(tx_queue, channel)
751 	efx_remove_eventq(channel);
752 	channel->type->post_remove(channel);
757 	struct efx_channel *channel;
759 	efx_for_each_channel(channel, efx)
760 		efx_remove_channel(channel);
773 		  tx_queue->channel->channel, tx_queue->label,
782 	struct efx_channel *channel;
791 	efx_for_each_channel(channel, efx) {
792 		if (channel->channel < efx->tx_channel_offset)
795 		if (efx_channel_is_xdp_tx(channel)) {
796 			efx_for_each_channel_tx_queue(tx_queue, channel) {
804 			efx_for_each_channel_tx_queue(tx_queue, channel) {
808 					  channel->channel, tx_queue->label,
814 			 * first one of the channel
819 				tx_queue = &channel->tx_queue[0];
846 	struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel,
864 		channel = efx->channel[i];
865 		if (channel->type->copy)
866 			channel = channel->type->copy(channel);
867 		if (!channel) {
871 		other_channel[i] = channel;
874 	/* Swap entry counts and channel pointers */
880 		swap(efx->channel[i], other_channel[i]);
883 		channel = efx->channel[i];
884 		if (!channel->type->copy)
886 		rc = efx_probe_channel(channel);
889 		efx_init_napi_channel(efx->channel[i]);
895 	/* Destroy unused channel structures */
897 		channel = other_channel[i];
898 		if (channel && channel->type->copy) {
899 			efx_fini_napi_channel(channel);
900 			efx_remove_channel(channel);
901 			kfree(channel);
910 			  "unable to restart interrupts on channel reallocation\n");
923 		swap(efx->channel[i], other_channel[i]);
930 	struct efx_channel *channel;
944 	efx_for_each_channel(channel, efx) {
945 		if (channel->channel < efx->n_rx_channels)
946 			channel->rx_queue.core_index = channel->channel;
948 			channel->rx_queue.core_index = -1;
959 static bool efx_default_channel_want_txqs(struct efx_channel *channel)
961 	return channel->channel - channel->efx->tx_channel_offset <
962 		channel->efx->n_tx_channels;
971 	struct efx_channel *channel, *end_channel;
979 	efx_for_each_channel(channel, efx) {
980 		if (!channel->type->keep_eventq) {
981 			rc = efx_init_eventq(channel);
985 		efx_start_eventq(channel);
992 	end_channel = channel;
993 	efx_for_each_channel(channel, efx) {
994 		if (channel == end_channel)
996 		efx_stop_eventq(channel);
997 		if (!channel->type->keep_eventq)
998 			efx_fini_eventq(channel);
1006 	struct efx_channel *channel;
1019 	efx_for_each_channel(channel, efx) {
1020 		if (channel->irq)
1021 			synchronize_irq(channel->irq);
1023 		efx_stop_eventq(channel);
1024 		if (!channel->type->keep_eventq)
1025 			efx_fini_eventq(channel);
1034 	struct efx_channel *channel, *end_channel;
1047 	efx_for_each_channel(channel, efx) {
1048 		if (channel->type->keep_eventq) {
1049 			rc = efx_init_eventq(channel);
1062 	end_channel = channel;
1063 	efx_for_each_channel(channel, efx) {
1064 		if (channel == end_channel)
1066 		if (channel->type->keep_eventq)
1067 			efx_fini_eventq(channel);
1077 	struct efx_channel *channel;
1081 	efx_for_each_channel(channel, efx) {
1082 		if (channel->type->keep_eventq)
1083 			efx_fini_eventq(channel);
1093 	struct efx_channel *channel;
1095 	efx_for_each_channel_rev(channel, efx) {
1096 		if (channel->type->start)
1097 			channel->type->start(channel);
1098 		efx_for_each_channel_tx_queue(tx_queue, channel) {
1103 		efx_for_each_channel_rx_queue(rx_queue, channel) {
1106 			efx_stop_eventq(channel);
1108 			efx_start_eventq(channel);
1111 		WARN_ON(channel->rx_pkt_n_frags);
1119 	struct efx_channel *channel;
1123 	 * The channel's stop has to be called first, since it might wait
1124 	 * for a sentinel RX to indicate the channel has fully drained.
1126 	efx_for_each_channel(channel, efx) {
1127 		if (channel->type->stop)
1128 			channel->type->stop(channel);
1129 		efx_for_each_channel_rx_queue(rx_queue, channel)
1133 	efx_for_each_channel(channel, efx) {
1140 		if (efx_channel_has_rx_queue(channel)) {
1141 			efx_stop_eventq(channel);
1142 			efx_start_eventq(channel);
1156 	efx_for_each_channel(channel, efx) {
1157 		efx_for_each_channel_rx_queue(rx_queue, channel)
1159 		efx_for_each_channel_tx_queue(tx_queue, channel)
1170 /* Process channel's event queue
1173  * single channel.  The caller must guarantee that this function will
1174  * never be concurrently called more than once on the same channel,
1177 static int efx_process_channel(struct efx_channel *channel, int budget)
1183 	if (unlikely(!channel->enabled))
1187 	EFX_WARN_ON_PARANOID(channel->rx_list != NULL);
1189 	channel->rx_list = &rx_list;
1191 	efx_for_each_channel_tx_queue(tx_queue, channel) {
1196 	spent = efx_nic_process_eventq(channel, budget);
1197 	if (spent && efx_channel_has_rx_queue(channel)) {
1199 			efx_channel_get_rx_queue(channel);
1201 		efx_rx_flush_packet(channel);
1206 	efx_for_each_channel_tx_queue(tx_queue, channel) {
1215 	netif_receive_skb_list(channel->rx_list);
1216 	channel->rx_list = NULL;
1221 static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel)
1225 	if (channel->irq_mod_score < irq_adapt_low_thresh) {
1226 		if (channel->irq_moderation_us > step) {
1227 			channel->irq_moderation_us -= step;
1228 			efx->type->push_irq_moderation(channel);
1230 	} else if (channel->irq_mod_score > irq_adapt_high_thresh) {
1231 		if (channel->irq_moderation_us <
1233 			channel->irq_moderation_us += step;
1234 			efx->type->push_irq_moderation(channel);
1238 	channel->irq_count = 0;
1239 	channel->irq_mod_score = 0;
1249 	struct efx_channel *channel =
1251 	struct efx_nic *efx = channel->efx;
1258 		   "channel %d NAPI poll executing on CPU %d\n",
1259 		   channel->channel, raw_smp_processor_id());
1261 	spent = efx_process_channel(channel, budget);
1266 		if (efx_channel_has_rx_queue(channel) &&
1268 		    unlikely(++channel->irq_count == 1000)) {
1269 			efx_update_irq_mod(efx, channel);
1274 		time = jiffies - channel->rfs_last_expiry;
1276 		if (channel->rfs_filter_count * time >= 600 * HZ)
1277 			mod_delayed_work(system_wq, &channel->filter_work, 0);
1286 			efx_nic_eventq_read_ack(channel);
1292 void efx_init_napi_channel(struct efx_channel *channel)
1294 	struct efx_nic *efx = channel->efx;
1296 	channel->napi_dev = efx->net_dev;
1297 	netif_napi_add(channel->napi_dev, &channel->napi_str, efx_poll);
1302 	struct efx_channel *channel;
1304 	efx_for_each_channel(channel, efx)
1305 		efx_init_napi_channel(channel);
1308 void efx_fini_napi_channel(struct efx_channel *channel)
1310 	if (channel->napi_dev)
1311 		netif_napi_del(&channel->napi_str);
1313 	channel->napi_dev = NULL;
1318 	struct efx_channel *channel;
1320 	efx_for_each_channel(channel, efx)
1321 		efx_fini_napi_channel(channel);
1328 static int efx_channel_dummy_op_int(struct efx_channel *channel)
1333 void efx_channel_dummy_op_void(struct efx_channel *channel)