1 | /* |
2 | * Copyright (c) 2015-2023 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | /* |
30 | * Copyright (C) 2013-2014 Universita` di Pisa. All rights reserved. |
31 | * |
32 | * Redistribution and use in source and binary forms, with or without |
33 | * modification, are permitted provided that the following conditions |
34 | * are met: |
35 | * 1. Redistributions of source code must retain the above copyright |
36 | * notice, this list of conditions and the following disclaimer. |
37 | * 2. Redistributions in binary form must reproduce the above copyright |
38 | * notice, this list of conditions and the following disclaimer in the |
39 | * documentation and/or other materials provided with the distribution. |
40 | * |
41 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND |
42 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
43 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
44 | * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE |
45 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
46 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
47 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
48 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
49 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
50 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
51 | * SUCH DAMAGE. |
52 | */ |
53 | |
54 | #include <skywalk/os_skywalk_private.h> |
55 | #include <skywalk/nexus/flowswitch/nx_flowswitch.h> |
56 | #include <skywalk/nexus/flowswitch/fsw_var.h> |
57 | #include <sys/sdt.h> |
58 | |
59 | static void fsw_vp_na_dtor(struct nexus_adapter *); |
60 | static int fsw_vp_na_special(struct nexus_adapter *, |
61 | struct kern_channel *, struct chreq *, nxspec_cmd_t); |
62 | static struct nexus_vp_adapter *fsw_vp_na_alloc(zalloc_flags_t); |
63 | static void fsw_vp_na_free(struct nexus_adapter *); |
64 | static int fsw_vp_na_channel_event_notify(struct nexus_adapter *vpna, |
65 | struct __kern_channel_event *ev, uint16_t ev_len); |
66 | |
67 | static SKMEM_TYPE_DEFINE(na_vp_zone, struct nexus_vp_adapter); |
68 | |
69 | static uint16_t fsw_vpna_gencnt = 0; |
70 | |
71 | /* na_activate() callback for flow switch ports */ |
72 | int |
73 | fsw_vp_na_activate(struct nexus_adapter *na, na_activate_mode_t mode) |
74 | { |
75 | int ret = 0; |
76 | struct nexus_vp_adapter *vpna = (struct nexus_vp_adapter *)(void *)na; |
77 | struct nx_flowswitch *fsw = vpna->vpna_fsw; |
78 | |
79 | ASSERT(na->na_type == NA_FLOWSWITCH_VP); |
80 | |
81 | SK_DF(SK_VERB_FSW, "na \"%s\" (0x%llx) %s" , na->na_name, |
82 | SK_KVA(na), na_activate_mode2str(mode)); |
83 | |
84 | /* |
85 | * Persistent ports may be put in Skywalk mode |
86 | * before being attached to a FlowSwitch. |
87 | */ |
88 | FSW_WLOCK(fsw); |
89 | |
90 | os_atomic_inc(&fsw_vpna_gencnt, relaxed); |
91 | vpna->vpna_gencnt = fsw_vpna_gencnt; |
92 | |
93 | if (mode == NA_ACTIVATE_MODE_ON) { |
94 | os_atomic_or(&na->na_flags, NAF_ACTIVE, relaxed); |
95 | } |
96 | |
97 | ret = fsw_port_na_activate(fsw, vpna, mode); |
98 | if (ret != 0) { |
99 | SK_DF(SK_VERB_FSW, "na \"%s\" (0x%llx) %s err(%d)" , |
100 | na->na_name, SK_KVA(na), na_activate_mode2str(mode), ret); |
101 | if (mode == NA_ACTIVATE_MODE_ON) { |
102 | os_atomic_andnot(&na->na_flags, NAF_ACTIVE, relaxed); |
103 | } |
104 | goto done; |
105 | } |
106 | |
107 | if (mode == NA_ACTIVATE_MODE_DEFUNCT || |
108 | mode == NA_ACTIVATE_MODE_OFF) { |
109 | struct skmem_arena_nexus *arn = skmem_arena_nexus(ar: na->na_arena); |
110 | |
111 | if (mode == NA_ACTIVATE_MODE_OFF) { |
112 | os_atomic_andnot(&na->na_flags, NAF_ACTIVE, relaxed); |
113 | } |
114 | |
115 | AR_LOCK(na->na_arena); |
116 | if (na->na_type == NA_FLOWSWITCH_VP && |
117 | arn->arn_stats_obj != NULL) { |
118 | fsw_fold_stats(fsw, |
119 | data: arn->arn_stats_obj, type: na->na_stats_type); |
120 | } |
121 | AR_UNLOCK(na->na_arena); |
122 | |
123 | enum txrx t; |
124 | uint32_t i; |
125 | struct __nx_stats_channel_errors stats; |
126 | for_all_rings(t) { |
127 | for (i = 0; i < na_get_nrings(na, t); i++) { |
128 | stats.nxs_cres = |
129 | &NAKR(na, t)[i].ckr_err_stats; |
130 | fsw_fold_stats(fsw, data: &stats, |
131 | type: NEXUS_STATS_TYPE_CHAN_ERRORS); |
132 | } |
133 | } |
134 | } |
135 | |
136 | done: |
137 | FSW_WUNLOCK(fsw); |
138 | return ret; |
139 | } |
140 | |
141 | /* na_dtor callback for ephemeral flow switch ports */ |
142 | static void |
143 | fsw_vp_na_dtor(struct nexus_adapter *na) |
144 | { |
145 | struct nexus_vp_adapter *vpna = (struct nexus_vp_adapter *)(void *)na; |
146 | struct nx_flowswitch *fsw = vpna->vpna_fsw; |
147 | |
148 | SK_LOCK_ASSERT_HELD(); |
149 | ASSERT(na->na_type == NA_FLOWSWITCH_VP); |
150 | |
151 | SK_DF(SK_VERB_FSW, "na \"%s\" (0x%llx)" , na->na_name, SK_KVA(na)); |
152 | |
153 | if (fsw != NULL) { |
154 | FSW_WLOCK(fsw); |
155 | fsw_port_free(fsw, vpna, nx_port: vpna->vpna_nx_port, FALSE); |
156 | FSW_WUNLOCK(fsw); |
157 | } |
158 | } |
159 | |
160 | /* |
161 | * na_krings_create callback for flow switch ports. |
162 | * Calls the standard na_kr_create(), then adds leases on rx |
163 | * rings and bdgfwd on tx rings. |
164 | */ |
165 | int |
166 | fsw_vp_na_krings_create(struct nexus_adapter *na, struct kern_channel *ch) |
167 | { |
168 | ASSERT(na->na_type == NA_FLOWSWITCH_VP); |
169 | |
170 | return na_rings_mem_setup(na, FALSE, ch); |
171 | } |
172 | |
173 | |
174 | /* na_krings_delete callback for flow switch ports. */ |
175 | void |
176 | fsw_vp_na_krings_delete(struct nexus_adapter *na, struct kern_channel *ch, |
177 | boolean_t defunct) |
178 | { |
179 | ASSERT(na->na_type == NA_FLOWSWITCH_VP); |
180 | |
181 | na_rings_mem_teardown(na, ch, defunct); |
182 | } |
183 | |
184 | /* na_txsync callback for flow switch ports */ |
185 | int |
186 | fsw_vp_na_txsync(struct __kern_channel_ring *kring, struct proc *p, |
187 | uint32_t flags) |
188 | { |
189 | #pragma unused(flags) |
190 | struct nexus_vp_adapter *vpna = VPNA(KRNA(kring)); |
191 | struct nx_flowswitch *fsw = vpna->vpna_fsw; |
192 | int error = 0; |
193 | |
194 | /* |
195 | * Flush packets if and only if the ring isn't in drop mode, |
196 | * and if the adapter is currently attached to a nexus port; |
197 | * otherwise we drop them. |
198 | */ |
199 | if (__probable(!KR_DROP(kring) && fsw != NULL)) { |
200 | fsw_ring_flush(fsw, skring: kring, p); |
201 | } else { |
202 | int dropped_pkts; |
203 | /* packets between khead to rhead have been dropped */ |
204 | dropped_pkts = kring->ckr_rhead - kring->ckr_khead; |
205 | if (dropped_pkts < 0) { |
206 | dropped_pkts += kring->ckr_num_slots; |
207 | } |
208 | if (fsw != NULL) { |
209 | STATS_INC(&fsw->fsw_stats, FSW_STATS_DST_RING_DROPMODE); |
210 | STATS_ADD(&fsw->fsw_stats, FSW_STATS_DROP, |
211 | dropped_pkts); |
212 | } |
213 | /* we're dropping; claim all */ |
214 | slot_idx_t sidx = kring->ckr_khead; |
215 | while (sidx != kring->ckr_rhead) { |
216 | struct __kern_slot_desc *ksd = KR_KSD(kring, sidx); |
217 | if (KSD_VALID_METADATA(ksd)) { |
218 | struct __kern_packet *pkt = ksd->sd_pkt; |
219 | (void) KR_SLOT_DETACH_METADATA(kring, ksd); |
220 | pp_free_packet_single(pkt); |
221 | } |
222 | sidx = SLOT_NEXT(i: sidx, lim: kring->ckr_lim); |
223 | } |
224 | kring->ckr_khead = kring->ckr_rhead; |
225 | kring->ckr_ktail = SLOT_PREV(i: kring->ckr_rhead, lim: kring->ckr_lim); |
226 | error = ENODEV; |
227 | SK_ERR("kr \"%s\" (0x%llx) krflags 0x%b in drop mode (err %d)" , |
228 | kring->ckr_name, SK_KVA(kring), kring->ckr_flags, |
229 | CKRF_BITS, error); |
230 | } |
231 | |
232 | SK_DF(SK_VERB_FSW | SK_VERB_SYNC | SK_VERB_TX, |
233 | "%s(%d) kr \"%s\" (0x%llx) krflags 0x%b ring %u flags 0x%x" , |
234 | sk_proc_name_address(p), sk_proc_pid(p), kring->ckr_name, |
235 | SK_KVA(kring), kring->ckr_flags, CKRF_BITS, kring->ckr_ring_id, |
236 | flags); |
237 | |
238 | return error; |
239 | } |
240 | |
241 | /* |
242 | * na_rxsync callback for flow switch ports. We're already protected |
243 | * against concurrent calls from userspace. |
244 | */ |
245 | int |
246 | fsw_vp_na_rxsync(struct __kern_channel_ring *kring, struct proc *p, |
247 | uint32_t flags) |
248 | { |
249 | #pragma unused(p, flags) |
250 | slot_idx_t head, khead_prev; |
251 | |
252 | head = kring->ckr_rhead; |
253 | ASSERT(head <= kring->ckr_lim); |
254 | |
255 | /* First part, import newly received packets. */ |
256 | /* actually nothing to do here, they are already in the kring */ |
257 | |
258 | /* Second part, skip past packets that userspace has released. */ |
259 | khead_prev = kring->ckr_khead; |
260 | kring->ckr_khead = head; |
261 | |
262 | /* ensure global visibility */ |
263 | os_atomic_thread_fence(seq_cst); |
264 | |
265 | SK_DF(SK_VERB_FSW | SK_VERB_SYNC | SK_VERB_RX, |
266 | "%s(%d) kr \"%s\" (0x%llx) krflags 0x%b ring %u " |
267 | "kh %u (was %u) rh %u flags 0x%x" , sk_proc_name_address(p), |
268 | sk_proc_pid(p), kring->ckr_name, SK_KVA(kring), kring->ckr_flags, |
269 | CKRF_BITS, kring->ckr_ring_id, kring->ckr_khead, khead_prev, |
270 | kring->ckr_rhead, flags); |
271 | |
272 | return 0; |
273 | } |
274 | |
275 | static int |
276 | fsw_vp_na_special(struct nexus_adapter *na, struct kern_channel *ch, |
277 | struct chreq *chr, nxspec_cmd_t spec_cmd) |
278 | { |
279 | int error = 0; |
280 | |
281 | SK_LOCK_ASSERT_HELD(); |
282 | ASSERT(na->na_type == NA_FLOWSWITCH_VP); |
283 | |
284 | /* |
285 | * fsw_vp_na_attach() must have created this adapter |
286 | * exclusively for kernel (NAF_KERNEL); leave this alone. |
287 | */ |
288 | ASSERT(NA_KERNEL_ONLY(na)); |
289 | |
290 | switch (spec_cmd) { |
291 | case NXSPEC_CMD_CONNECT: |
292 | ASSERT(!(na->na_flags & NAF_SPEC_INIT)); |
293 | ASSERT(na->na_channels == 0); |
294 | |
295 | error = na_bind_channel(na, ch, chr); |
296 | if (error != 0) { |
297 | goto done; |
298 | } |
299 | |
300 | os_atomic_or(&na->na_flags, NAF_SPEC_INIT, relaxed); |
301 | break; |
302 | |
303 | case NXSPEC_CMD_DISCONNECT: |
304 | ASSERT(na->na_channels > 0); |
305 | ASSERT(na->na_flags & NAF_SPEC_INIT); |
306 | os_atomic_andnot(&na->na_flags, NAF_SPEC_INIT, relaxed); |
307 | |
308 | na_unbind_channel(ch); |
309 | break; |
310 | |
311 | case NXSPEC_CMD_START: |
312 | na_kr_drop(na, FALSE); |
313 | break; |
314 | |
315 | case NXSPEC_CMD_STOP: |
316 | na_kr_drop(na, TRUE); |
317 | break; |
318 | |
319 | default: |
320 | error = EINVAL; |
321 | break; |
322 | } |
323 | |
324 | done: |
325 | SK_DF(error ? SK_VERB_ERROR : SK_VERB_FSW, |
326 | "ch 0x%llx na \"%s\" (0x%llx) nx 0x%llx spec_cmd %u (err %d)" , |
327 | SK_KVA(ch), na->na_name, SK_KVA(na), SK_KVA(ch->ch_nexus), |
328 | spec_cmd, error); |
329 | |
330 | return error; |
331 | } |
332 | |
333 | /* |
334 | * Create a nexus_vp_adapter that describes a flow switch port. |
335 | */ |
336 | int |
337 | fsw_vp_na_create(struct kern_nexus *nx, struct chreq *chr, struct proc *p, |
338 | struct nexus_vp_adapter **ret) |
339 | { |
340 | struct nxprov_params *nxp = NX_PROV(nx)->nxprov_params; |
341 | struct nx_flowswitch *fsw = NX_FSW_PRIVATE(nx); |
342 | struct nexus_vp_adapter *vpna; |
343 | struct nexus_adapter *na; |
344 | int error; |
345 | |
346 | SK_LOCK_ASSERT_HELD(); |
347 | |
348 | if ((chr->cr_mode & CHMODE_KERNEL) != 0) { |
349 | SK_ERR("VP adapter can't be used by kernel" ); |
350 | return ENOTSUP; |
351 | } |
352 | if ((chr->cr_mode & CHMODE_USER_PACKET_POOL) == 0) { |
353 | SK_ERR("user packet pool required" ); |
354 | return EINVAL; |
355 | } |
356 | |
357 | vpna = fsw_vp_na_alloc(Z_WAITOK); |
358 | |
359 | ASSERT(vpna->vpna_up.na_type == NA_FLOWSWITCH_VP); |
360 | ASSERT(vpna->vpna_up.na_free == fsw_vp_na_free); |
361 | |
362 | na = &vpna->vpna_up; |
363 | (void) snprintf(na->na_name, count: sizeof(na->na_name), "fsw_%s[%u]_%s.%d" , |
364 | fsw->fsw_ifp ? if_name(fsw->fsw_ifp) : "??" , chr->cr_port, |
365 | proc_best_name(p), proc_pid(p)); |
366 | na->na_name[sizeof(na->na_name) - 1] = '\0'; |
367 | uuid_generate_random(out: na->na_uuid); |
368 | |
369 | /* |
370 | * Verify upper bounds; for all cases including user pipe nexus, |
371 | * as well as flow switch-based ones, the parameters must have |
372 | * already been validated by corresponding nxdom_prov_params() |
373 | * function defined by each domain. The user pipe nexus would |
374 | * be checking against the flow switch's parameters there. |
375 | */ |
376 | na_set_nrings(na, t: NR_TX, v: nxp->nxp_tx_rings); |
377 | na_set_nrings(na, t: NR_RX, v: nxp->nxp_rx_rings); |
378 | /* |
379 | * If the packet pool is configured to be multi-buflet, then we |
380 | * need 2 pairs of alloc/free rings(for packet and buflet). |
381 | */ |
382 | na_set_nrings(na, t: NR_A, v: ((nxp->nxp_max_frags > 1) && |
383 | (sk_channel_buflet_alloc != 0)) ? 2 : 1); |
384 | na_set_nslots(na, t: NR_TX, v: nxp->nxp_tx_slots); |
385 | na_set_nslots(na, t: NR_RX, v: nxp->nxp_rx_slots); |
386 | na_set_nslots(na, t: NR_A, NX_FSW_AFRINGSIZE); |
387 | ASSERT(na_get_nrings(na, NR_TX) <= NX_DOM(nx)->nxdom_tx_rings.nb_max); |
388 | ASSERT(na_get_nrings(na, NR_RX) <= NX_DOM(nx)->nxdom_rx_rings.nb_max); |
389 | ASSERT(na_get_nslots(na, NR_TX) <= NX_DOM(nx)->nxdom_tx_slots.nb_max); |
390 | ASSERT(na_get_nslots(na, NR_RX) <= NX_DOM(nx)->nxdom_rx_slots.nb_max); |
391 | |
392 | os_atomic_or(&na->na_flags, NAF_USER_PKT_POOL, relaxed); |
393 | |
394 | if (chr->cr_mode & CHMODE_LOW_LATENCY) { |
395 | os_atomic_or(&na->na_flags, NAF_LOW_LATENCY, relaxed); |
396 | } |
397 | |
398 | if (chr->cr_mode & CHMODE_EVENT_RING) { |
399 | na_set_nrings(na, t: NR_EV, NX_FSW_EVENT_RING_NUM); |
400 | na_set_nslots(na, t: NR_EV, NX_FSW_EVENT_RING_SIZE); |
401 | os_atomic_or(&na->na_flags, NAF_EVENT_RING, relaxed); |
402 | na->na_channel_event_notify = fsw_vp_na_channel_event_notify; |
403 | } |
404 | if (nxp->nxp_max_frags > 1 && fsw->fsw_tso_mode != FSW_TSO_MODE_NONE) { |
405 | na_set_nrings(na, t: NR_LBA, v: 1); |
406 | na_set_nslots(na, t: NR_LBA, NX_FSW_AFRINGSIZE); |
407 | } |
408 | vpna->vpna_nx_port = chr->cr_port; |
409 | na->na_dtor = fsw_vp_na_dtor; |
410 | na->na_activate = fsw_vp_na_activate; |
411 | na->na_txsync = fsw_vp_na_txsync; |
412 | na->na_rxsync = fsw_vp_na_rxsync; |
413 | na->na_krings_create = fsw_vp_na_krings_create; |
414 | na->na_krings_delete = fsw_vp_na_krings_delete; |
415 | na->na_special = fsw_vp_na_special; |
416 | |
417 | *(nexus_stats_type_t *)(uintptr_t)&na->na_stats_type = |
418 | NEXUS_STATS_TYPE_FSW; |
419 | |
420 | /* other fields are set in the common routine */ |
421 | na_attach_common(na, nx, &nx_fsw_prov_s); |
422 | |
423 | if ((error = NX_DOM_PROV(nx)->nxdom_prov_mem_new(NX_DOM_PROV(nx), |
424 | nx, na)) != 0) { |
425 | ASSERT(na->na_arena == NULL); |
426 | goto err; |
427 | } |
428 | ASSERT(na->na_arena != NULL); |
429 | |
430 | *(uint32_t *)(uintptr_t)&na->na_flowadv_max = nxp->nxp_flowadv_max; |
431 | ASSERT(na->na_flowadv_max == 0 || |
432 | skmem_arena_nexus(na->na_arena)->arn_flowadv_obj != NULL); |
433 | |
434 | #if SK_LOG |
435 | uuid_string_t uuidstr; |
436 | SK_DF(SK_VERB_FSW, "na_name: \"%s\"" , na->na_name); |
437 | SK_DF(SK_VERB_FSW, " UUID: %s" , sk_uuid_unparse(na->na_uuid, |
438 | uuidstr)); |
439 | SK_DF(SK_VERB_FSW, " nx: 0x%llx (\"%s\":\"%s\")" , |
440 | SK_KVA(na->na_nx), NX_DOM(na->na_nx)->nxdom_name, |
441 | NX_DOM_PROV(na->na_nx)->nxdom_prov_name); |
442 | SK_DF(SK_VERB_FSW, " flags: 0x%b" , na->na_flags, NAF_BITS); |
443 | SK_DF(SK_VERB_FSW, " stats_type: %u" , na->na_stats_type); |
444 | SK_DF(SK_VERB_FSW, " flowadv_max: %u" , na->na_flowadv_max); |
445 | SK_DF(SK_VERB_FSW, " rings: tx %u rx %u af %u" , |
446 | na_get_nrings(na, NR_TX), na_get_nrings(na, NR_RX), |
447 | na_get_nrings(na, NR_A)); |
448 | SK_DF(SK_VERB_FSW, " slots: tx %u rx %u af %u" , |
449 | na_get_nslots(na, NR_TX), na_get_nslots(na, NR_RX), |
450 | na_get_nslots(na, NR_A)); |
451 | #if CONFIG_NEXUS_USER_PIPE |
452 | SK_DF(SK_VERB_FSW, " next_pipe: %u" , na->na_next_pipe); |
453 | SK_DF(SK_VERB_FSW, " max_pipes: %u" , na->na_max_pipes); |
454 | #endif /* CONFIG_NEXUS_USER_PIPE */ |
455 | SK_DF(SK_VERB_FSW, " nx_port: %d" , (int)vpna->vpna_nx_port); |
456 | #endif /* SK_LOG */ |
457 | |
458 | *ret = vpna; |
459 | na_retain_locked(na: &vpna->vpna_up); |
460 | |
461 | return 0; |
462 | |
463 | err: |
464 | if (na->na_arena != NULL) { |
465 | skmem_arena_release(na->na_arena); |
466 | na->na_arena = NULL; |
467 | } |
468 | NA_FREE(&vpna->vpna_up); |
469 | return error; |
470 | } |
471 | |
472 | static struct nexus_vp_adapter * |
473 | fsw_vp_na_alloc(zalloc_flags_t how) |
474 | { |
475 | struct nexus_vp_adapter *vpna; |
476 | |
477 | _CASSERT(offsetof(struct nexus_vp_adapter, vpna_up) == 0); |
478 | |
479 | vpna = zalloc_flags(na_vp_zone, how | Z_ZERO); |
480 | if (vpna) { |
481 | vpna->vpna_up.na_type = NA_FLOWSWITCH_VP; |
482 | vpna->vpna_up.na_free = fsw_vp_na_free; |
483 | } |
484 | return vpna; |
485 | } |
486 | |
487 | static void |
488 | fsw_vp_na_free(struct nexus_adapter *na) |
489 | { |
490 | struct nexus_vp_adapter *vpna = (struct nexus_vp_adapter *)(void *)na; |
491 | |
492 | ASSERT(vpna->vpna_up.na_refcount == 0); |
493 | SK_DF(SK_VERB_MEM, "vpna 0x%llx FREE" , SK_KVA(vpna)); |
494 | bzero(s: vpna, n: sizeof(*vpna)); |
495 | zfree(na_vp_zone, vpna); |
496 | } |
497 | |
498 | void |
499 | fsw_vp_channel_error_stats_fold(struct fsw_stats *fs, |
500 | struct __nx_stats_channel_errors *es) |
501 | { |
502 | STATS_ADD(fs, FSW_STATS_CHAN_ERR_UPP_ALLOC, |
503 | es->nxs_cres->cres_pkt_alloc_failures); |
504 | } |
505 | |
506 | SK_NO_INLINE_ATTRIBUTE |
507 | static struct __kern_packet * |
508 | nx_fsw_alloc_packet(struct kern_pbufpool *pp, uint32_t sz, kern_packet_t *php) |
509 | { |
510 | kern_packet_t ph; |
511 | ph = pp_alloc_packet_by_size(pp, sz, SKMEM_NOSLEEP); |
512 | if (__improbable(ph == 0)) { |
513 | DTRACE_SKYWALK2(alloc__fail, struct kern_pbufpool *, |
514 | pp, size_t, sz); |
515 | return NULL; |
516 | } |
517 | if (php != NULL) { |
518 | *php = ph; |
519 | } |
520 | return SK_PTR_ADDR_KPKT(ph); |
521 | } |
522 | |
523 | SK_NO_INLINE_ATTRIBUTE |
524 | static void |
525 | nx_fsw_free_packet(struct __kern_packet *pkt) |
526 | { |
527 | pp_free_packet_single(pkt); |
528 | } |
529 | |
530 | static int |
531 | fsw_vp_na_channel_event_notify(struct nexus_adapter *vpna, |
532 | struct __kern_channel_event *ev, uint16_t ev_len) |
533 | { |
534 | int err; |
535 | char *baddr; |
536 | kern_packet_t ph; |
537 | kern_buflet_t buf; |
538 | sk_protect_t protect; |
539 | kern_channel_slot_t slot; |
540 | struct __kern_packet *vpna_pkt = NULL; |
541 | struct __kern_channel_event_metadata *emd; |
542 | struct __kern_channel_ring *ring = &vpna->na_event_rings[0]; |
543 | struct fsw_stats *fs = &((struct nexus_vp_adapter *)(vpna))->vpna_fsw->fsw_stats; |
544 | |
545 | if (__probable(ev->ev_type == CHANNEL_EVENT_PACKET_TRANSMIT_STATUS)) { |
546 | STATS_INC(fs, FSW_STATS_EV_RECV_TX_STATUS); |
547 | } |
548 | if (__improbable(ev->ev_type == CHANNEL_EVENT_PACKET_TRANSMIT_EXPIRED)) { |
549 | STATS_INC(fs, FSW_STATS_EV_RECV_TX_EXPIRED); |
550 | } |
551 | STATS_INC(fs, FSW_STATS_EV_RECV); |
552 | |
553 | if (__improbable(!NA_IS_ACTIVE(vpna))) { |
554 | STATS_INC(fs, FSW_STATS_EV_DROP_NA_INACTIVE); |
555 | err = ENXIO; |
556 | goto error; |
557 | } |
558 | if (__improbable(NA_IS_DEFUNCT(vpna))) { |
559 | STATS_INC(fs, FSW_STATS_EV_DROP_NA_DEFUNCT); |
560 | err = ENXIO; |
561 | goto error; |
562 | } |
563 | if (!NA_CHANNEL_EVENT_ATTACHED(vpna)) { |
564 | STATS_INC(fs, FSW_STATS_EV_DROP_KEVENT_INACTIVE); |
565 | err = ENXIO; |
566 | goto error; |
567 | } |
568 | if (__improbable(KR_DROP(ring))) { |
569 | STATS_INC(fs, FSW_STATS_EV_DROP_KRDROP_MODE); |
570 | err = ENXIO; |
571 | goto error; |
572 | } |
573 | |
574 | vpna_pkt = nx_fsw_alloc_packet(pp: ring->ckr_pp, sz: ev_len, php: &ph); |
575 | if (__improbable(vpna_pkt == NULL)) { |
576 | STATS_INC(fs, FSW_STATS_EV_DROP_NOMEM_PKT); |
577 | err = ENOMEM; |
578 | goto error; |
579 | } |
580 | buf = __packet_get_next_buflet(ph, NULL); |
581 | baddr = __buflet_get_data_address(buf); |
582 | emd = (struct __kern_channel_event_metadata *)(void *)baddr; |
583 | emd->emd_etype = ev->ev_type; |
584 | emd->emd_nevents = 1; |
585 | bcopy(src: ev, dst: (baddr + __KERN_CHANNEL_EVENT_OFFSET), n: ev_len); |
586 | err = __buflet_set_data_length(buf, |
587 | dlen: (ev_len + __KERN_CHANNEL_EVENT_OFFSET)); |
588 | VERIFY(err == 0); |
589 | err = __packet_finalize(ph); |
590 | VERIFY(err == 0); |
591 | kr_enter(ring, TRUE); |
592 | protect = sk_sync_protect(); |
593 | slot = kern_channel_get_next_slot(kring: ring, NULL, NULL); |
594 | if (slot == NULL) { |
595 | sk_sync_unprotect(protect); |
596 | kr_exit(ring); |
597 | STATS_INC(fs, FSW_STATS_EV_DROP_KRSPACE); |
598 | err = ENOSPC; |
599 | goto error; |
600 | } |
601 | err = kern_channel_slot_attach_packet(ring, slot, packet: ph); |
602 | VERIFY(err == 0); |
603 | vpna_pkt = NULL; |
604 | kern_channel_advance_slot(kring: ring, slot); |
605 | sk_sync_unprotect(protect); |
606 | kr_exit(ring); |
607 | kern_channel_event_notify(&vpna->na_tx_rings[0]); |
608 | STATS_INC(fs, NETIF_STATS_EV_SENT); |
609 | return 0; |
610 | |
611 | error: |
612 | ASSERT(err != 0); |
613 | if (vpna_pkt != NULL) { |
614 | nx_fsw_free_packet(pkt: vpna_pkt); |
615 | } |
616 | STATS_INC(fs, FSW_STATS_EV_DROP); |
617 | return err; |
618 | } |
619 | |
620 | static inline struct nexus_adapter * |
621 | fsw_find_port_vpna(struct nx_flowswitch *fsw, uint32_t nx_port_id) |
622 | { |
623 | struct kern_nexus *nx = fsw->fsw_nx; |
624 | struct nexus_adapter *na = NULL; |
625 | nexus_port_t port; |
626 | uint16_t gencnt; |
627 | |
628 | PKT_DECOMPOSE_NX_PORT_ID(nx_port_id, port, gencnt); |
629 | |
630 | if (port < FSW_VP_USER_MIN) { |
631 | SK_ERR("non VPNA port" ); |
632 | return NULL; |
633 | } |
634 | |
635 | if (__improbable(!nx_port_is_valid(nx, port))) { |
636 | SK_ERR("%s[%d] port no longer valid" , |
637 | if_name(fsw->fsw_ifp), port); |
638 | return NULL; |
639 | } |
640 | |
641 | na = nx_port_get_na(nx, port); |
642 | if (na != NULL && VPNA(na)->vpna_gencnt != gencnt) { |
643 | return NULL; |
644 | } |
645 | return na; |
646 | } |
647 | |
648 | errno_t |
649 | fsw_vp_na_channel_event(struct nx_flowswitch *fsw, uint32_t nx_port_id, |
650 | struct __kern_channel_event *event, uint16_t event_len) |
651 | { |
652 | int err = 0; |
653 | struct nexus_adapter *fsw_vpna; |
654 | |
655 | SK_DF(SK_VERB_EVENTS, "%s[%d] ev: %p ev_len: %hu " |
656 | "ev_type: %u ev_flags: %u _reserved: %hu ev_dlen: %hu" , |
657 | if_name(fsw->fsw_ifp), nx_port_id, event, event_len, |
658 | event->ev_type, event->ev_flags, event->_reserved, event->ev_dlen); |
659 | |
660 | FSW_RLOCK(fsw); |
661 | struct fsw_stats *fs = &fsw->fsw_stats; |
662 | |
663 | fsw_vpna = fsw_find_port_vpna(fsw, nx_port_id); |
664 | if (__improbable(fsw_vpna == NULL)) { |
665 | err = ENXIO; |
666 | STATS_INC(fs, FSW_STATS_EV_DROP_DEMUX_ERR); |
667 | goto error; |
668 | } |
669 | if (__improbable(fsw_vpna->na_channel_event_notify == NULL)) { |
670 | err = ENOTSUP; |
671 | STATS_INC(fs, FSW_STATS_EV_DROP_EV_VPNA_NOTSUP); |
672 | goto error; |
673 | } |
674 | err = fsw_vpna->na_channel_event_notify(fsw_vpna, event, event_len); |
675 | FSW_RUNLOCK(fsw); |
676 | return err; |
677 | |
678 | error: |
679 | STATS_INC(fs, FSW_STATS_EV_DROP); |
680 | FSW_RUNLOCK(fsw); |
681 | return err; |
682 | } |
683 | |