1/*
2 * Copyright (c) 2015-2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29/*
30 * Copyright (C) 2013-2014 Universita` di Pisa. All rights reserved.
31 *
32 * Redistribution and use in source and binary forms, with or without
33 * modification, are permitted provided that the following conditions
34 * are met:
35 * 1. Redistributions of source code must retain the above copyright
36 * notice, this list of conditions and the following disclaimer.
37 * 2. Redistributions in binary form must reproduce the above copyright
38 * notice, this list of conditions and the following disclaimer in the
39 * documentation and/or other materials provided with the distribution.
40 *
41 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
42 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
43 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
44 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
45 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
46 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
47 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
50 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
51 * SUCH DAMAGE.
52 */
53
54#include <skywalk/os_skywalk_private.h>
55#include <skywalk/nexus/flowswitch/nx_flowswitch.h>
56#include <skywalk/nexus/flowswitch/fsw_var.h>
57#include <sys/sdt.h>
58
59static void fsw_vp_na_dtor(struct nexus_adapter *);
60static int fsw_vp_na_special(struct nexus_adapter *,
61 struct kern_channel *, struct chreq *, nxspec_cmd_t);
62static struct nexus_vp_adapter *fsw_vp_na_alloc(zalloc_flags_t);
63static void fsw_vp_na_free(struct nexus_adapter *);
64static int fsw_vp_na_channel_event_notify(struct nexus_adapter *vpna,
65 struct __kern_channel_event *ev, uint16_t ev_len);
66
67static SKMEM_TYPE_DEFINE(na_vp_zone, struct nexus_vp_adapter);
68
69static uint16_t fsw_vpna_gencnt = 0;
70
71/* na_activate() callback for flow switch ports */
72int
73fsw_vp_na_activate(struct nexus_adapter *na, na_activate_mode_t mode)
74{
75 int ret = 0;
76 struct nexus_vp_adapter *vpna = (struct nexus_vp_adapter *)(void *)na;
77 struct nx_flowswitch *fsw = vpna->vpna_fsw;
78
79 ASSERT(na->na_type == NA_FLOWSWITCH_VP);
80
81 SK_DF(SK_VERB_FSW, "na \"%s\" (0x%llx) %s", na->na_name,
82 SK_KVA(na), na_activate_mode2str(mode));
83
84 /*
85 * Persistent ports may be put in Skywalk mode
86 * before being attached to a FlowSwitch.
87 */
88 FSW_WLOCK(fsw);
89
90 os_atomic_inc(&fsw_vpna_gencnt, relaxed);
91 vpna->vpna_gencnt = fsw_vpna_gencnt;
92
93 if (mode == NA_ACTIVATE_MODE_ON) {
94 os_atomic_or(&na->na_flags, NAF_ACTIVE, relaxed);
95 }
96
97 ret = fsw_port_na_activate(fsw, vpna, mode);
98 if (ret != 0) {
99 SK_DF(SK_VERB_FSW, "na \"%s\" (0x%llx) %s err(%d)",
100 na->na_name, SK_KVA(na), na_activate_mode2str(mode), ret);
101 if (mode == NA_ACTIVATE_MODE_ON) {
102 os_atomic_andnot(&na->na_flags, NAF_ACTIVE, relaxed);
103 }
104 goto done;
105 }
106
107 if (mode == NA_ACTIVATE_MODE_DEFUNCT ||
108 mode == NA_ACTIVATE_MODE_OFF) {
109 struct skmem_arena_nexus *arn = skmem_arena_nexus(ar: na->na_arena);
110
111 if (mode == NA_ACTIVATE_MODE_OFF) {
112 os_atomic_andnot(&na->na_flags, NAF_ACTIVE, relaxed);
113 }
114
115 AR_LOCK(na->na_arena);
116 if (na->na_type == NA_FLOWSWITCH_VP &&
117 arn->arn_stats_obj != NULL) {
118 fsw_fold_stats(fsw,
119 data: arn->arn_stats_obj, type: na->na_stats_type);
120 }
121 AR_UNLOCK(na->na_arena);
122
123 enum txrx t;
124 uint32_t i;
125 struct __nx_stats_channel_errors stats;
126 for_all_rings(t) {
127 for (i = 0; i < na_get_nrings(na, t); i++) {
128 stats.nxs_cres =
129 &NAKR(na, t)[i].ckr_err_stats;
130 fsw_fold_stats(fsw, data: &stats,
131 type: NEXUS_STATS_TYPE_CHAN_ERRORS);
132 }
133 }
134 }
135
136done:
137 FSW_WUNLOCK(fsw);
138 return ret;
139}
140
141/* na_dtor callback for ephemeral flow switch ports */
142static void
143fsw_vp_na_dtor(struct nexus_adapter *na)
144{
145 struct nexus_vp_adapter *vpna = (struct nexus_vp_adapter *)(void *)na;
146 struct nx_flowswitch *fsw = vpna->vpna_fsw;
147
148 SK_LOCK_ASSERT_HELD();
149 ASSERT(na->na_type == NA_FLOWSWITCH_VP);
150
151 SK_DF(SK_VERB_FSW, "na \"%s\" (0x%llx)", na->na_name, SK_KVA(na));
152
153 if (fsw != NULL) {
154 FSW_WLOCK(fsw);
155 fsw_port_free(fsw, vpna, nx_port: vpna->vpna_nx_port, FALSE);
156 FSW_WUNLOCK(fsw);
157 }
158}
159
160/*
161 * na_krings_create callback for flow switch ports.
162 * Calls the standard na_kr_create(), then adds leases on rx
163 * rings and bdgfwd on tx rings.
164 */
165int
166fsw_vp_na_krings_create(struct nexus_adapter *na, struct kern_channel *ch)
167{
168 ASSERT(na->na_type == NA_FLOWSWITCH_VP);
169
170 return na_rings_mem_setup(na, FALSE, ch);
171}
172
173
174/* na_krings_delete callback for flow switch ports. */
175void
176fsw_vp_na_krings_delete(struct nexus_adapter *na, struct kern_channel *ch,
177 boolean_t defunct)
178{
179 ASSERT(na->na_type == NA_FLOWSWITCH_VP);
180
181 na_rings_mem_teardown(na, ch, defunct);
182}
183
184/* na_txsync callback for flow switch ports */
185int
186fsw_vp_na_txsync(struct __kern_channel_ring *kring, struct proc *p,
187 uint32_t flags)
188{
189#pragma unused(flags)
190 struct nexus_vp_adapter *vpna = VPNA(KRNA(kring));
191 struct nx_flowswitch *fsw = vpna->vpna_fsw;
192 int error = 0;
193
194 /*
195 * Flush packets if and only if the ring isn't in drop mode,
196 * and if the adapter is currently attached to a nexus port;
197 * otherwise we drop them.
198 */
199 if (__probable(!KR_DROP(kring) && fsw != NULL)) {
200 fsw_ring_flush(fsw, skring: kring, p);
201 } else {
202 int dropped_pkts;
203 /* packets between khead to rhead have been dropped */
204 dropped_pkts = kring->ckr_rhead - kring->ckr_khead;
205 if (dropped_pkts < 0) {
206 dropped_pkts += kring->ckr_num_slots;
207 }
208 if (fsw != NULL) {
209 STATS_INC(&fsw->fsw_stats, FSW_STATS_DST_RING_DROPMODE);
210 STATS_ADD(&fsw->fsw_stats, FSW_STATS_DROP,
211 dropped_pkts);
212 }
213 /* we're dropping; claim all */
214 slot_idx_t sidx = kring->ckr_khead;
215 while (sidx != kring->ckr_rhead) {
216 struct __kern_slot_desc *ksd = KR_KSD(kring, sidx);
217 if (KSD_VALID_METADATA(ksd)) {
218 struct __kern_packet *pkt = ksd->sd_pkt;
219 (void) KR_SLOT_DETACH_METADATA(kring, ksd);
220 pp_free_packet_single(pkt);
221 }
222 sidx = SLOT_NEXT(i: sidx, lim: kring->ckr_lim);
223 }
224 kring->ckr_khead = kring->ckr_rhead;
225 kring->ckr_ktail = SLOT_PREV(i: kring->ckr_rhead, lim: kring->ckr_lim);
226 error = ENODEV;
227 SK_ERR("kr \"%s\" (0x%llx) krflags 0x%b in drop mode (err %d)",
228 kring->ckr_name, SK_KVA(kring), kring->ckr_flags,
229 CKRF_BITS, error);
230 }
231
232 SK_DF(SK_VERB_FSW | SK_VERB_SYNC | SK_VERB_TX,
233 "%s(%d) kr \"%s\" (0x%llx) krflags 0x%b ring %u flags 0x%x",
234 sk_proc_name_address(p), sk_proc_pid(p), kring->ckr_name,
235 SK_KVA(kring), kring->ckr_flags, CKRF_BITS, kring->ckr_ring_id,
236 flags);
237
238 return error;
239}
240
241/*
242 * na_rxsync callback for flow switch ports. We're already protected
243 * against concurrent calls from userspace.
244 */
245int
246fsw_vp_na_rxsync(struct __kern_channel_ring *kring, struct proc *p,
247 uint32_t flags)
248{
249#pragma unused(p, flags)
250 slot_idx_t head, khead_prev;
251
252 head = kring->ckr_rhead;
253 ASSERT(head <= kring->ckr_lim);
254
255 /* First part, import newly received packets. */
256 /* actually nothing to do here, they are already in the kring */
257
258 /* Second part, skip past packets that userspace has released. */
259 khead_prev = kring->ckr_khead;
260 kring->ckr_khead = head;
261
262 /* ensure global visibility */
263 os_atomic_thread_fence(seq_cst);
264
265 SK_DF(SK_VERB_FSW | SK_VERB_SYNC | SK_VERB_RX,
266 "%s(%d) kr \"%s\" (0x%llx) krflags 0x%b ring %u "
267 "kh %u (was %u) rh %u flags 0x%x", sk_proc_name_address(p),
268 sk_proc_pid(p), kring->ckr_name, SK_KVA(kring), kring->ckr_flags,
269 CKRF_BITS, kring->ckr_ring_id, kring->ckr_khead, khead_prev,
270 kring->ckr_rhead, flags);
271
272 return 0;
273}
274
275static int
276fsw_vp_na_special(struct nexus_adapter *na, struct kern_channel *ch,
277 struct chreq *chr, nxspec_cmd_t spec_cmd)
278{
279 int error = 0;
280
281 SK_LOCK_ASSERT_HELD();
282 ASSERT(na->na_type == NA_FLOWSWITCH_VP);
283
284 /*
285 * fsw_vp_na_attach() must have created this adapter
286 * exclusively for kernel (NAF_KERNEL); leave this alone.
287 */
288 ASSERT(NA_KERNEL_ONLY(na));
289
290 switch (spec_cmd) {
291 case NXSPEC_CMD_CONNECT:
292 ASSERT(!(na->na_flags & NAF_SPEC_INIT));
293 ASSERT(na->na_channels == 0);
294
295 error = na_bind_channel(na, ch, chr);
296 if (error != 0) {
297 goto done;
298 }
299
300 os_atomic_or(&na->na_flags, NAF_SPEC_INIT, relaxed);
301 break;
302
303 case NXSPEC_CMD_DISCONNECT:
304 ASSERT(na->na_channels > 0);
305 ASSERT(na->na_flags & NAF_SPEC_INIT);
306 os_atomic_andnot(&na->na_flags, NAF_SPEC_INIT, relaxed);
307
308 na_unbind_channel(ch);
309 break;
310
311 case NXSPEC_CMD_START:
312 na_kr_drop(na, FALSE);
313 break;
314
315 case NXSPEC_CMD_STOP:
316 na_kr_drop(na, TRUE);
317 break;
318
319 default:
320 error = EINVAL;
321 break;
322 }
323
324done:
325 SK_DF(error ? SK_VERB_ERROR : SK_VERB_FSW,
326 "ch 0x%llx na \"%s\" (0x%llx) nx 0x%llx spec_cmd %u (err %d)",
327 SK_KVA(ch), na->na_name, SK_KVA(na), SK_KVA(ch->ch_nexus),
328 spec_cmd, error);
329
330 return error;
331}
332
333/*
334 * Create a nexus_vp_adapter that describes a flow switch port.
335 */
336int
337fsw_vp_na_create(struct kern_nexus *nx, struct chreq *chr, struct proc *p,
338 struct nexus_vp_adapter **ret)
339{
340 struct nxprov_params *nxp = NX_PROV(nx)->nxprov_params;
341 struct nx_flowswitch *fsw = NX_FSW_PRIVATE(nx);
342 struct nexus_vp_adapter *vpna;
343 struct nexus_adapter *na;
344 int error;
345
346 SK_LOCK_ASSERT_HELD();
347
348 if ((chr->cr_mode & CHMODE_KERNEL) != 0) {
349 SK_ERR("VP adapter can't be used by kernel");
350 return ENOTSUP;
351 }
352 if ((chr->cr_mode & CHMODE_USER_PACKET_POOL) == 0) {
353 SK_ERR("user packet pool required");
354 return EINVAL;
355 }
356
357 vpna = fsw_vp_na_alloc(Z_WAITOK);
358
359 ASSERT(vpna->vpna_up.na_type == NA_FLOWSWITCH_VP);
360 ASSERT(vpna->vpna_up.na_free == fsw_vp_na_free);
361
362 na = &vpna->vpna_up;
363 (void) snprintf(na->na_name, count: sizeof(na->na_name), "fsw_%s[%u]_%s.%d",
364 fsw->fsw_ifp ? if_name(fsw->fsw_ifp) : "??", chr->cr_port,
365 proc_best_name(p), proc_pid(p));
366 na->na_name[sizeof(na->na_name) - 1] = '\0';
367 uuid_generate_random(out: na->na_uuid);
368
369 /*
370 * Verify upper bounds; for all cases including user pipe nexus,
371 * as well as flow switch-based ones, the parameters must have
372 * already been validated by corresponding nxdom_prov_params()
373 * function defined by each domain. The user pipe nexus would
374 * be checking against the flow switch's parameters there.
375 */
376 na_set_nrings(na, t: NR_TX, v: nxp->nxp_tx_rings);
377 na_set_nrings(na, t: NR_RX, v: nxp->nxp_rx_rings);
378 /*
379 * If the packet pool is configured to be multi-buflet, then we
380 * need 2 pairs of alloc/free rings(for packet and buflet).
381 */
382 na_set_nrings(na, t: NR_A, v: ((nxp->nxp_max_frags > 1) &&
383 (sk_channel_buflet_alloc != 0)) ? 2 : 1);
384 na_set_nslots(na, t: NR_TX, v: nxp->nxp_tx_slots);
385 na_set_nslots(na, t: NR_RX, v: nxp->nxp_rx_slots);
386 na_set_nslots(na, t: NR_A, NX_FSW_AFRINGSIZE);
387 ASSERT(na_get_nrings(na, NR_TX) <= NX_DOM(nx)->nxdom_tx_rings.nb_max);
388 ASSERT(na_get_nrings(na, NR_RX) <= NX_DOM(nx)->nxdom_rx_rings.nb_max);
389 ASSERT(na_get_nslots(na, NR_TX) <= NX_DOM(nx)->nxdom_tx_slots.nb_max);
390 ASSERT(na_get_nslots(na, NR_RX) <= NX_DOM(nx)->nxdom_rx_slots.nb_max);
391
392 os_atomic_or(&na->na_flags, NAF_USER_PKT_POOL, relaxed);
393
394 if (chr->cr_mode & CHMODE_LOW_LATENCY) {
395 os_atomic_or(&na->na_flags, NAF_LOW_LATENCY, relaxed);
396 }
397
398 if (chr->cr_mode & CHMODE_EVENT_RING) {
399 na_set_nrings(na, t: NR_EV, NX_FSW_EVENT_RING_NUM);
400 na_set_nslots(na, t: NR_EV, NX_FSW_EVENT_RING_SIZE);
401 os_atomic_or(&na->na_flags, NAF_EVENT_RING, relaxed);
402 na->na_channel_event_notify = fsw_vp_na_channel_event_notify;
403 }
404 if (nxp->nxp_max_frags > 1 && fsw->fsw_tso_mode != FSW_TSO_MODE_NONE) {
405 na_set_nrings(na, t: NR_LBA, v: 1);
406 na_set_nslots(na, t: NR_LBA, NX_FSW_AFRINGSIZE);
407 }
408 vpna->vpna_nx_port = chr->cr_port;
409 na->na_dtor = fsw_vp_na_dtor;
410 na->na_activate = fsw_vp_na_activate;
411 na->na_txsync = fsw_vp_na_txsync;
412 na->na_rxsync = fsw_vp_na_rxsync;
413 na->na_krings_create = fsw_vp_na_krings_create;
414 na->na_krings_delete = fsw_vp_na_krings_delete;
415 na->na_special = fsw_vp_na_special;
416
417 *(nexus_stats_type_t *)(uintptr_t)&na->na_stats_type =
418 NEXUS_STATS_TYPE_FSW;
419
420 /* other fields are set in the common routine */
421 na_attach_common(na, nx, &nx_fsw_prov_s);
422
423 if ((error = NX_DOM_PROV(nx)->nxdom_prov_mem_new(NX_DOM_PROV(nx),
424 nx, na)) != 0) {
425 ASSERT(na->na_arena == NULL);
426 goto err;
427 }
428 ASSERT(na->na_arena != NULL);
429
430 *(uint32_t *)(uintptr_t)&na->na_flowadv_max = nxp->nxp_flowadv_max;
431 ASSERT(na->na_flowadv_max == 0 ||
432 skmem_arena_nexus(na->na_arena)->arn_flowadv_obj != NULL);
433
434#if SK_LOG
435 uuid_string_t uuidstr;
436 SK_DF(SK_VERB_FSW, "na_name: \"%s\"", na->na_name);
437 SK_DF(SK_VERB_FSW, " UUID: %s", sk_uuid_unparse(na->na_uuid,
438 uuidstr));
439 SK_DF(SK_VERB_FSW, " nx: 0x%llx (\"%s\":\"%s\")",
440 SK_KVA(na->na_nx), NX_DOM(na->na_nx)->nxdom_name,
441 NX_DOM_PROV(na->na_nx)->nxdom_prov_name);
442 SK_DF(SK_VERB_FSW, " flags: 0x%b", na->na_flags, NAF_BITS);
443 SK_DF(SK_VERB_FSW, " stats_type: %u", na->na_stats_type);
444 SK_DF(SK_VERB_FSW, " flowadv_max: %u", na->na_flowadv_max);
445 SK_DF(SK_VERB_FSW, " rings: tx %u rx %u af %u",
446 na_get_nrings(na, NR_TX), na_get_nrings(na, NR_RX),
447 na_get_nrings(na, NR_A));
448 SK_DF(SK_VERB_FSW, " slots: tx %u rx %u af %u",
449 na_get_nslots(na, NR_TX), na_get_nslots(na, NR_RX),
450 na_get_nslots(na, NR_A));
451#if CONFIG_NEXUS_USER_PIPE
452 SK_DF(SK_VERB_FSW, " next_pipe: %u", na->na_next_pipe);
453 SK_DF(SK_VERB_FSW, " max_pipes: %u", na->na_max_pipes);
454#endif /* CONFIG_NEXUS_USER_PIPE */
455 SK_DF(SK_VERB_FSW, " nx_port: %d", (int)vpna->vpna_nx_port);
456#endif /* SK_LOG */
457
458 *ret = vpna;
459 na_retain_locked(na: &vpna->vpna_up);
460
461 return 0;
462
463err:
464 if (na->na_arena != NULL) {
465 skmem_arena_release(na->na_arena);
466 na->na_arena = NULL;
467 }
468 NA_FREE(&vpna->vpna_up);
469 return error;
470}
471
472static struct nexus_vp_adapter *
473fsw_vp_na_alloc(zalloc_flags_t how)
474{
475 struct nexus_vp_adapter *vpna;
476
477 _CASSERT(offsetof(struct nexus_vp_adapter, vpna_up) == 0);
478
479 vpna = zalloc_flags(na_vp_zone, how | Z_ZERO);
480 if (vpna) {
481 vpna->vpna_up.na_type = NA_FLOWSWITCH_VP;
482 vpna->vpna_up.na_free = fsw_vp_na_free;
483 }
484 return vpna;
485}
486
487static void
488fsw_vp_na_free(struct nexus_adapter *na)
489{
490 struct nexus_vp_adapter *vpna = (struct nexus_vp_adapter *)(void *)na;
491
492 ASSERT(vpna->vpna_up.na_refcount == 0);
493 SK_DF(SK_VERB_MEM, "vpna 0x%llx FREE", SK_KVA(vpna));
494 bzero(s: vpna, n: sizeof(*vpna));
495 zfree(na_vp_zone, vpna);
496}
497
498void
499fsw_vp_channel_error_stats_fold(struct fsw_stats *fs,
500 struct __nx_stats_channel_errors *es)
501{
502 STATS_ADD(fs, FSW_STATS_CHAN_ERR_UPP_ALLOC,
503 es->nxs_cres->cres_pkt_alloc_failures);
504}
505
506SK_NO_INLINE_ATTRIBUTE
507static struct __kern_packet *
508nx_fsw_alloc_packet(struct kern_pbufpool *pp, uint32_t sz, kern_packet_t *php)
509{
510 kern_packet_t ph;
511 ph = pp_alloc_packet_by_size(pp, sz, SKMEM_NOSLEEP);
512 if (__improbable(ph == 0)) {
513 DTRACE_SKYWALK2(alloc__fail, struct kern_pbufpool *,
514 pp, size_t, sz);
515 return NULL;
516 }
517 if (php != NULL) {
518 *php = ph;
519 }
520 return SK_PTR_ADDR_KPKT(ph);
521}
522
523SK_NO_INLINE_ATTRIBUTE
524static void
525nx_fsw_free_packet(struct __kern_packet *pkt)
526{
527 pp_free_packet_single(pkt);
528}
529
530static int
531fsw_vp_na_channel_event_notify(struct nexus_adapter *vpna,
532 struct __kern_channel_event *ev, uint16_t ev_len)
533{
534 int err;
535 char *baddr;
536 kern_packet_t ph;
537 kern_buflet_t buf;
538 sk_protect_t protect;
539 kern_channel_slot_t slot;
540 struct __kern_packet *vpna_pkt = NULL;
541 struct __kern_channel_event_metadata *emd;
542 struct __kern_channel_ring *ring = &vpna->na_event_rings[0];
543 struct fsw_stats *fs = &((struct nexus_vp_adapter *)(vpna))->vpna_fsw->fsw_stats;
544
545 if (__probable(ev->ev_type == CHANNEL_EVENT_PACKET_TRANSMIT_STATUS)) {
546 STATS_INC(fs, FSW_STATS_EV_RECV_TX_STATUS);
547 }
548 if (__improbable(ev->ev_type == CHANNEL_EVENT_PACKET_TRANSMIT_EXPIRED)) {
549 STATS_INC(fs, FSW_STATS_EV_RECV_TX_EXPIRED);
550 }
551 STATS_INC(fs, FSW_STATS_EV_RECV);
552
553 if (__improbable(!NA_IS_ACTIVE(vpna))) {
554 STATS_INC(fs, FSW_STATS_EV_DROP_NA_INACTIVE);
555 err = ENXIO;
556 goto error;
557 }
558 if (__improbable(NA_IS_DEFUNCT(vpna))) {
559 STATS_INC(fs, FSW_STATS_EV_DROP_NA_DEFUNCT);
560 err = ENXIO;
561 goto error;
562 }
563 if (!NA_CHANNEL_EVENT_ATTACHED(vpna)) {
564 STATS_INC(fs, FSW_STATS_EV_DROP_KEVENT_INACTIVE);
565 err = ENXIO;
566 goto error;
567 }
568 if (__improbable(KR_DROP(ring))) {
569 STATS_INC(fs, FSW_STATS_EV_DROP_KRDROP_MODE);
570 err = ENXIO;
571 goto error;
572 }
573
574 vpna_pkt = nx_fsw_alloc_packet(pp: ring->ckr_pp, sz: ev_len, php: &ph);
575 if (__improbable(vpna_pkt == NULL)) {
576 STATS_INC(fs, FSW_STATS_EV_DROP_NOMEM_PKT);
577 err = ENOMEM;
578 goto error;
579 }
580 buf = __packet_get_next_buflet(ph, NULL);
581 baddr = __buflet_get_data_address(buf);
582 emd = (struct __kern_channel_event_metadata *)(void *)baddr;
583 emd->emd_etype = ev->ev_type;
584 emd->emd_nevents = 1;
585 bcopy(src: ev, dst: (baddr + __KERN_CHANNEL_EVENT_OFFSET), n: ev_len);
586 err = __buflet_set_data_length(buf,
587 dlen: (ev_len + __KERN_CHANNEL_EVENT_OFFSET));
588 VERIFY(err == 0);
589 err = __packet_finalize(ph);
590 VERIFY(err == 0);
591 kr_enter(ring, TRUE);
592 protect = sk_sync_protect();
593 slot = kern_channel_get_next_slot(kring: ring, NULL, NULL);
594 if (slot == NULL) {
595 sk_sync_unprotect(protect);
596 kr_exit(ring);
597 STATS_INC(fs, FSW_STATS_EV_DROP_KRSPACE);
598 err = ENOSPC;
599 goto error;
600 }
601 err = kern_channel_slot_attach_packet(ring, slot, packet: ph);
602 VERIFY(err == 0);
603 vpna_pkt = NULL;
604 kern_channel_advance_slot(kring: ring, slot);
605 sk_sync_unprotect(protect);
606 kr_exit(ring);
607 kern_channel_event_notify(&vpna->na_tx_rings[0]);
608 STATS_INC(fs, NETIF_STATS_EV_SENT);
609 return 0;
610
611error:
612 ASSERT(err != 0);
613 if (vpna_pkt != NULL) {
614 nx_fsw_free_packet(pkt: vpna_pkt);
615 }
616 STATS_INC(fs, FSW_STATS_EV_DROP);
617 return err;
618}
619
620static inline struct nexus_adapter *
621fsw_find_port_vpna(struct nx_flowswitch *fsw, uint32_t nx_port_id)
622{
623 struct kern_nexus *nx = fsw->fsw_nx;
624 struct nexus_adapter *na = NULL;
625 nexus_port_t port;
626 uint16_t gencnt;
627
628 PKT_DECOMPOSE_NX_PORT_ID(nx_port_id, port, gencnt);
629
630 if (port < FSW_VP_USER_MIN) {
631 SK_ERR("non VPNA port");
632 return NULL;
633 }
634
635 if (__improbable(!nx_port_is_valid(nx, port))) {
636 SK_ERR("%s[%d] port no longer valid",
637 if_name(fsw->fsw_ifp), port);
638 return NULL;
639 }
640
641 na = nx_port_get_na(nx, port);
642 if (na != NULL && VPNA(na)->vpna_gencnt != gencnt) {
643 return NULL;
644 }
645 return na;
646}
647
648errno_t
649fsw_vp_na_channel_event(struct nx_flowswitch *fsw, uint32_t nx_port_id,
650 struct __kern_channel_event *event, uint16_t event_len)
651{
652 int err = 0;
653 struct nexus_adapter *fsw_vpna;
654
655 SK_DF(SK_VERB_EVENTS, "%s[%d] ev: %p ev_len: %hu "
656 "ev_type: %u ev_flags: %u _reserved: %hu ev_dlen: %hu",
657 if_name(fsw->fsw_ifp), nx_port_id, event, event_len,
658 event->ev_type, event->ev_flags, event->_reserved, event->ev_dlen);
659
660 FSW_RLOCK(fsw);
661 struct fsw_stats *fs = &fsw->fsw_stats;
662
663 fsw_vpna = fsw_find_port_vpna(fsw, nx_port_id);
664 if (__improbable(fsw_vpna == NULL)) {
665 err = ENXIO;
666 STATS_INC(fs, FSW_STATS_EV_DROP_DEMUX_ERR);
667 goto error;
668 }
669 if (__improbable(fsw_vpna->na_channel_event_notify == NULL)) {
670 err = ENOTSUP;
671 STATS_INC(fs, FSW_STATS_EV_DROP_EV_VPNA_NOTSUP);
672 goto error;
673 }
674 err = fsw_vpna->na_channel_event_notify(fsw_vpna, event, event_len);
675 FSW_RUNLOCK(fsw);
676 return err;
677
678error:
679 STATS_INC(fs, FSW_STATS_EV_DROP);
680 FSW_RUNLOCK(fsw);
681 return err;
682}
683