1/*
2 * Copyright (c) 2015-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29/*
30 * Copyright (C) 2014 Giuseppe Lettieri. All rights reserved.
31 *
32 * Redistribution and use in source and binary forms, with or without
33 * modification, are permitted provided that the following conditions
34 * are met:
35 * 1. Redistributions of source code must retain the above copyright
36 * notice, this list of conditions and the following disclaimer.
37 * 2. Redistributions in binary form must reproduce the above copyright
38 * notice, this list of conditions and the following disclaimer in the
39 * documentation and/or other materials provided with the distribution.
40 *
41 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
42 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
43 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
44 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
45 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
46 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
47 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
50 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
51 * SUCH DAMAGE.
52 */
53
54#include <skywalk/os_skywalk_private.h>
55#include <skywalk/nexus/upipe/nx_user_pipe.h>
56#include <skywalk/nexus/kpipe/nx_kernel_pipe.h>
57
58/* XXX arbitrary */
59#define NX_KPIPE_RINGSIZE 128 /* default ring size */
60#define NX_KPIPE_MINSLOTS 2
61#define NX_KPIPE_MAXSLOTS 4096
62#define NX_KPIPE_MAXRINGS NX_MAX_NUM_RING_PAIR
63#define NX_KPIPE_BUFSIZE (2 * 1024)
64#define NX_KPIPE_MINBUFSIZE 64
65#define NX_KPIPE_MAXBUFSIZE (16 * 1024)
66
67static int nx_kpipe_na_txsync(struct __kern_channel_ring *, struct proc *,
68 uint32_t);
69static int nx_kpipe_na_rxsync(struct __kern_channel_ring *, struct proc *,
70 uint32_t);
71static int nx_kpipe_na_activate(struct nexus_adapter *, na_activate_mode_t);
72static void nx_kpipe_na_dtor(struct nexus_adapter *);
73static int nx_kpipe_na_krings_create(struct nexus_adapter *,
74 struct kern_channel *);
75static void nx_kpipe_na_krings_delete(struct nexus_adapter *,
76 struct kern_channel *, boolean_t);
77
78static void nx_kpipe_dom_init(struct nxdom *);
79static void nx_kpipe_dom_terminate(struct nxdom *);
80static void nx_kpipe_dom_fini(struct nxdom *);
81static int nx_kpipe_dom_bind_port(struct kern_nexus *, nexus_port_t *,
82 struct nxbind *, void *);
83static int nx_kpipe_dom_unbind_port(struct kern_nexus *, nexus_port_t);
84static int nx_kpipe_dom_connect(struct kern_nexus_domain_provider *,
85 struct kern_nexus *, struct kern_channel *, struct chreq *,
86 struct kern_channel *, struct nxbind *, struct proc *);
87static void nx_kpipe_dom_disconnect(struct kern_nexus_domain_provider *,
88 struct kern_nexus *, struct kern_channel *);
89static void nx_kpipe_dom_defunct(struct kern_nexus_domain_provider *,
90 struct kern_nexus *, struct kern_channel *, struct proc *);
91static void nx_kpipe_dom_defunct_finalize(struct kern_nexus_domain_provider *,
92 struct kern_nexus *, struct kern_channel *, boolean_t);
93
94static int nx_kpipe_prov_init(struct kern_nexus_domain_provider *);
95static int nx_kpipe_prov_params_adjust(
96 const struct kern_nexus_domain_provider *,
97 const struct nxprov_params *, struct nxprov_adjusted_params *);
98static int nx_kpipe_prov_params(struct kern_nexus_domain_provider *,
99 const uint32_t, const struct nxprov_params *, struct nxprov_params *,
100 struct skmem_region_params[SKMEM_REGIONS], uint32_t);
101static int nx_kpipe_prov_mem_new(struct kern_nexus_domain_provider *,
102 struct kern_nexus *, struct nexus_adapter *);
103static void nx_kpipe_prov_fini(struct kern_nexus_domain_provider *);
104static int nx_kpipe_prov_nx_ctor(struct kern_nexus *);
105static void nx_kpipe_prov_nx_dtor(struct kern_nexus *);
106static int nx_kpipe_prov_nx_mem_info(struct kern_nexus *,
107 struct kern_pbufpool **, struct kern_pbufpool **);
108
109static struct nexus_kpipe_adapter *na_kpipe_alloc(zalloc_flags_t);
110static void na_kpipe_free(struct nexus_adapter *);
111
112struct nxdom nx_kpipe_dom_s = {
113 .nxdom_prov_head =
114 STAILQ_HEAD_INITIALIZER(nx_kpipe_dom_s.nxdom_prov_head),
115 .nxdom_type = NEXUS_TYPE_KERNEL_PIPE,
116 .nxdom_md_type = NEXUS_META_TYPE_QUANTUM,
117 .nxdom_md_subtype = NEXUS_META_SUBTYPE_PAYLOAD,
118 .nxdom_name = "kpipe",
119 .nxdom_ports = {
120 .nb_def = 1,
121 .nb_min = 1,
122 .nb_max = 1,
123 },
124 .nxdom_tx_rings = {
125 .nb_def = 1,
126 .nb_min = 1,
127 .nb_max = NX_KPIPE_MAXRINGS,
128 },
129 .nxdom_rx_rings = {
130 .nb_def = 1,
131 .nb_min = 1,
132 .nb_max = NX_KPIPE_MAXRINGS,
133 },
134 .nxdom_tx_slots = {
135 .nb_def = NX_KPIPE_RINGSIZE,
136 .nb_min = NX_KPIPE_MINSLOTS,
137 .nb_max = NX_KPIPE_MAXSLOTS,
138 },
139 .nxdom_rx_slots = {
140 .nb_def = NX_KPIPE_RINGSIZE,
141 .nb_min = NX_KPIPE_MINSLOTS,
142 .nb_max = NX_KPIPE_MAXSLOTS,
143 },
144 .nxdom_buf_size = {
145 .nb_def = NX_KPIPE_BUFSIZE,
146 .nb_min = NX_KPIPE_MINBUFSIZE,
147 .nb_max = NX_KPIPE_MAXBUFSIZE,
148 },
149 .nxdom_large_buf_size = {
150 .nb_def = 0,
151 .nb_min = 0,
152 .nb_max = 0,
153 },
154 .nxdom_meta_size = {
155 .nb_def = NX_METADATA_OBJ_MIN_SZ,
156 .nb_min = NX_METADATA_OBJ_MIN_SZ,
157 .nb_max = NX_METADATA_USR_MAX_SZ,
158 },
159 .nxdom_stats_size = {
160 .nb_def = 0,
161 .nb_min = 0,
162 .nb_max = NX_STATS_MAX_SZ,
163 },
164 .nxdom_pipes = {
165 .nb_def = 0,
166 .nb_min = 0,
167 .nb_max = NX_UPIPE_MAXPIPES,
168 },
169 .nxdom_flowadv_max = {
170 .nb_def = 0,
171 .nb_min = 0,
172 .nb_max = NX_FLOWADV_MAX,
173 },
174 .nxdom_nexusadv_size = {
175 .nb_def = 0,
176 .nb_min = 0,
177 .nb_max = NX_NEXUSADV_MAX_SZ,
178 },
179 .nxdom_capabilities = {
180 .nb_def = NXPCAP_USER_CHANNEL,
181 .nb_min = NXPCAP_USER_CHANNEL,
182 .nb_max = NXPCAP_USER_CHANNEL,
183 },
184 .nxdom_qmap = {
185 .nb_def = NEXUS_QMAP_TYPE_INVALID,
186 .nb_min = NEXUS_QMAP_TYPE_INVALID,
187 .nb_max = NEXUS_QMAP_TYPE_INVALID,
188 },
189 .nxdom_max_frags = {
190 .nb_def = NX_PBUF_FRAGS_DEFAULT,
191 .nb_min = NX_PBUF_FRAGS_MIN,
192 .nb_max = NX_PBUF_FRAGS_DEFAULT,
193 },
194 .nxdom_init = nx_kpipe_dom_init,
195 .nxdom_terminate = nx_kpipe_dom_terminate,
196 .nxdom_fini = nx_kpipe_dom_fini,
197 .nxdom_find_port = NULL,
198 .nxdom_port_is_reserved = NULL,
199 .nxdom_bind_port = nx_kpipe_dom_bind_port,
200 .nxdom_unbind_port = nx_kpipe_dom_unbind_port,
201 .nxdom_connect = nx_kpipe_dom_connect,
202 .nxdom_disconnect = nx_kpipe_dom_disconnect,
203 .nxdom_defunct = nx_kpipe_dom_defunct,
204 .nxdom_defunct_finalize = nx_kpipe_dom_defunct_finalize,
205};
206
207static struct kern_nexus_domain_provider nx_kpipe_prov_s = {
208 .nxdom_prov_name = NEXUS_PROVIDER_KERNEL_PIPE,
209 .nxdom_prov_flags = NXDOMPROVF_DEFAULT,
210 .nxdom_prov_cb = {
211 .dp_cb_init = nx_kpipe_prov_init,
212 .dp_cb_fini = nx_kpipe_prov_fini,
213 .dp_cb_params = nx_kpipe_prov_params,
214 .dp_cb_mem_new = nx_kpipe_prov_mem_new,
215 .dp_cb_config = NULL,
216 .dp_cb_nx_ctor = nx_kpipe_prov_nx_ctor,
217 .dp_cb_nx_dtor = nx_kpipe_prov_nx_dtor,
218 .dp_cb_nx_mem_info = nx_kpipe_prov_nx_mem_info,
219 .dp_cb_nx_mib_get = NULL,
220 .dp_cb_nx_stop = NULL,
221 },
222};
223
224static SKMEM_TYPE_DEFINE(na_kpipe_zone, struct nexus_kpipe_adapter);
225
226static void
227nx_kpipe_dom_init(struct nxdom *nxdom)
228{
229 SK_LOCK_ASSERT_HELD();
230 ASSERT(!(nxdom->nxdom_flags & NEXUSDOMF_INITIALIZED));
231
232 (void) nxdom_prov_add(nxdom, &nx_kpipe_prov_s);
233}
234
235static void
236nx_kpipe_dom_terminate(struct nxdom *nxdom)
237{
238 struct kern_nexus_domain_provider *nxdom_prov, *tnxdp;
239
240 SK_LOCK_ASSERT_HELD();
241
242 STAILQ_FOREACH_SAFE(nxdom_prov, &nxdom->nxdom_prov_head,
243 nxdom_prov_link, tnxdp) {
244 (void) nxdom_prov_del(nxdom_prov);
245 }
246}
247
248static void
249nx_kpipe_dom_fini(struct nxdom *nxdom)
250{
251#pragma unused(nxdom)
252}
253
254static int
255nx_kpipe_dom_bind_port(struct kern_nexus *nx, nexus_port_t *nx_port,
256 struct nxbind *nxb0, void *info)
257{
258#pragma unused(info)
259 struct nxbind *nxb = NULL;
260 int error = 0;
261
262 ASSERT(nx_port != NULL);
263 ASSERT(nxb0 != NULL);
264
265 switch (*nx_port) {
266 case NEXUS_PORT_KERNEL_PIPE_CLIENT:
267 if (nx->nx_arg != NULL) {
268 error = EEXIST;
269 break;
270 }
271
272 nxb = nxb_alloc(Z_WAITOK);
273 nxb_move(nxb0, nxb);
274 nx->nx_arg = nxb;
275
276 ASSERT(error == 0);
277 break;
278
279 default:
280 error = EDOM;
281 break;
282 }
283
284 return error;
285}
286
287static int
288nx_kpipe_dom_unbind_port(struct kern_nexus *nx, nexus_port_t nx_port)
289{
290 struct nxbind *nxb = NULL;
291 int error = 0;
292
293 ASSERT(nx_port != NEXUS_PORT_ANY);
294
295 switch (nx_port) {
296 case NEXUS_PORT_KERNEL_PIPE_CLIENT:
297 if ((nxb = nx->nx_arg) == NULL) {
298 error = ENOENT;
299 break;
300 }
301 nx->nx_arg = NULL;
302 nxb_free(nxb);
303 ASSERT(error == 0);
304 break;
305
306 default:
307 error = EDOM;
308 break;
309 }
310
311 return error;
312}
313
314static int
315nx_kpipe_dom_connect(struct kern_nexus_domain_provider *nxdom_prov,
316 struct kern_nexus *nx, struct kern_channel *ch, struct chreq *chr,
317 struct kern_channel *ch0, struct nxbind *nxb, struct proc *p)
318{
319#pragma unused(nxdom_prov)
320 nexus_port_t port = chr->cr_port;
321 int err = 0;
322
323 SK_DF(SK_VERB_KERNEL_PIPE, "port %d mode 0x%b",
324 (int)port, chr->cr_mode, CHMODE_BITS);
325
326 SK_LOCK_ASSERT_HELD();
327
328 ASSERT(nx->nx_prov->nxprov_params->nxp_type ==
329 nxdom_prov->nxdom_prov_dom->nxdom_type &&
330 nx->nx_prov->nxprov_params->nxp_type == NEXUS_TYPE_KERNEL_PIPE);
331
332 if (port != NEXUS_PORT_KERNEL_PIPE_CLIENT) {
333 err = EINVAL;
334 goto done;
335 }
336
337 /*
338 * XXX: user packet pool is not supported for kernel pipe for now.
339 */
340 if (chr->cr_mode & CHMODE_USER_PACKET_POOL) {
341 SK_ERR("User Packet pool mode not supported for kpipe");
342 err = ENOTSUP;
343 goto done;
344 }
345
346 if (chr->cr_mode & CHMODE_EVENT_RING) {
347 SK_ERR("event ring is not supported for kpipe");
348 err = ENOTSUP;
349 goto done;
350 }
351
352 if (chr->cr_mode & CHMODE_LOW_LATENCY) {
353 SK_ERR("low latency is not supported for kpipe");
354 err = ENOTSUP;
355 goto done;
356 }
357
358 chr->cr_ring_set = RING_SET_DEFAULT;
359 chr->cr_real_endpoint = chr->cr_endpoint = CH_ENDPOINT_KERNEL_PIPE;
360 (void) snprintf(chr->cr_name, count: sizeof(chr->cr_name), "kpipe:%llu:%.*s",
361 nx->nx_id, (int)nx->nx_prov->nxprov_params->nxp_namelen,
362 nx->nx_prov->nxprov_params->nxp_name);
363
364 err = na_connect(nx, ch, chr, ch0, nxb, p);
365 if (err == 0) {
366 /*
367 * Mark the kernel slot descriptor region as busy; this
368 * prevents it from being torn-down at channel defunct
369 * time, as the (external) nexus owner may be calling
370 * KPIs that require accessing the slots.
371 */
372 skmem_arena_nexus_sd_set_noidle(
373 skmem_arena_nexus(ar: ch->ch_na->na_arena), 1);
374 }
375
376done:
377 return err;
378}
379
380static void
381nx_kpipe_dom_disconnect(struct kern_nexus_domain_provider *nxdom_prov,
382 struct kern_nexus *nx, struct kern_channel *ch)
383{
384#pragma unused(nxdom_prov, nx)
385 SK_LOCK_ASSERT_HELD();
386
387 SK_D("channel 0x%llx -!- nexus 0x%llx (%s:\"%s\":%u:%d)", SK_KVA(ch),
388 SK_KVA(nx), nxdom_prov->nxdom_prov_name, ch->ch_na->na_name,
389 ch->ch_info->cinfo_nx_port, (int)ch->ch_info->cinfo_ch_ring_id);
390
391 /*
392 * Release busy assertion held earlier in nx_kpipe_dom_connect();
393 * this allows for the final arena teardown to succeed.
394 */
395 skmem_arena_nexus_sd_set_noidle(
396 skmem_arena_nexus(ar: ch->ch_na->na_arena), -1);
397
398 na_disconnect(nx, ch);
399}
400
401static void
402nx_kpipe_dom_defunct(struct kern_nexus_domain_provider *nxdom_prov,
403 struct kern_nexus *nx, struct kern_channel *ch, struct proc *p)
404{
405#pragma unused(nxdom_prov, nx)
406 struct nexus_adapter *na = ch->ch_na;
407 ring_id_t qfirst;
408 ring_id_t qlast;
409 enum txrx t;
410 uint32_t i;
411
412 LCK_MTX_ASSERT(&ch->ch_lock, LCK_MTX_ASSERT_OWNED);
413 ASSERT(!(ch->ch_flags & CHANF_KERNEL));
414 ASSERT(ch->ch_na->na_type == NA_KERNEL_PIPE);
415
416 /*
417 * Interface drivers like utun & IPsec access the kpipe rings
418 * outside of a kpipe channel sync context. They hold rights
419 * to the ring through kr_enter().
420 */
421 for_rx_tx(t) {
422 qfirst = ch->ch_first[t];
423 qlast = ch->ch_last[t];
424
425 for (i = qfirst; i < qlast; i++) {
426 (void) kr_enter(&NAKR(na, t)[i], TRUE);
427 }
428 }
429
430 na_ch_rings_defunct(ch, p);
431
432 for_rx_tx(t) {
433 qfirst = ch->ch_first[t];
434 qlast = ch->ch_last[t];
435
436 for (i = qfirst; i < qlast; i++) {
437 kr_exit(&NAKR(na, t)[i]);
438 }
439 }
440}
441
442static void
443nx_kpipe_dom_defunct_finalize(struct kern_nexus_domain_provider *nxdom_prov,
444 struct kern_nexus *nx, struct kern_channel *ch, boolean_t locked)
445{
446#pragma unused(nxdom_prov)
447 if (!locked) {
448 SK_LOCK_ASSERT_NOTHELD();
449 SK_LOCK();
450 LCK_MTX_ASSERT(&ch->ch_lock, LCK_MTX_ASSERT_NOTOWNED);
451 } else {
452 SK_LOCK_ASSERT_HELD();
453 LCK_MTX_ASSERT(&ch->ch_lock, LCK_MTX_ASSERT_OWNED);
454 }
455
456 ASSERT(!(ch->ch_flags & CHANF_KERNEL));
457 ASSERT(ch->ch_na->na_type == NA_KERNEL_PIPE);
458
459 na_defunct(nx, ch, ch->ch_na, locked);
460
461 SK_D("%s(%d): ch 0x%llx -/- nx 0x%llx (%s:\"%s\":%u:%d)",
462 ch->ch_name, ch->ch_pid, SK_KVA(ch), SK_KVA(nx),
463 nxdom_prov->nxdom_prov_name, ch->ch_na->na_name,
464 ch->ch_info->cinfo_nx_port, (int)ch->ch_info->cinfo_ch_ring_id);
465
466 if (!locked) {
467 LCK_MTX_ASSERT(&ch->ch_lock, LCK_MTX_ASSERT_NOTOWNED);
468 SK_UNLOCK();
469 } else {
470 LCK_MTX_ASSERT(&ch->ch_lock, LCK_MTX_ASSERT_OWNED);
471 SK_LOCK_ASSERT_HELD();
472 }
473}
474
475static int
476nx_kpipe_prov_init(struct kern_nexus_domain_provider *nxdom_prov)
477{
478#pragma unused(nxdom_prov)
479 SK_D("initializing %s", nxdom_prov->nxdom_prov_name);
480 return 0;
481}
482
483static int
484nx_kpipe_prov_params_adjust(const struct kern_nexus_domain_provider *nxdom_prov,
485 const struct nxprov_params *nxp, struct nxprov_adjusted_params *adj)
486{
487#pragma unused(nxdom_prov, nxp, adj)
488 return 0;
489}
490
491static int
492nx_kpipe_prov_params(struct kern_nexus_domain_provider *nxdom_prov,
493 const uint32_t req, const struct nxprov_params *nxp0,
494 struct nxprov_params *nxp, struct skmem_region_params srp[SKMEM_REGIONS],
495 uint32_t pp_region_config_flags)
496{
497 struct nxdom *nxdom = nxdom_prov->nxdom_prov_dom;
498
499 return nxprov_params_adjust(nxdom_prov, req, nxp0, nxp, srp,
500 nxdom, nxdom, nxdom, pp_region_config_flags,
501 adjust_fn: nx_kpipe_prov_params_adjust);
502}
503
504static int
505nx_kpipe_prov_mem_new(struct kern_nexus_domain_provider *nxdom_prov,
506 struct kern_nexus *nx, struct nexus_adapter *na)
507{
508#pragma unused(nxdom_prov)
509 int err = 0;
510
511 SK_DF(SK_VERB_KERNEL_PIPE,
512 "nx 0x%llx (\"%s\":\"%s\") na \"%s\" (0x%llx)", SK_KVA(nx),
513 NX_DOM(nx)->nxdom_name, nxdom_prov->nxdom_prov_name, na->na_name,
514 SK_KVA(na));
515
516 ASSERT(na->na_arena == NULL);
517 ASSERT(NX_USER_CHANNEL_PROV(nx));
518 /*
519 * Store pp in the nexus to handle kern_nexus_get_pbufpool() calls.
520 */
521 na->na_arena = skmem_arena_create_for_nexus(na,
522 NX_PROV(nx)->nxprov_region_params, &nx->nx_tx_pp,
523 &nx->nx_rx_pp, FALSE, FALSE, NULL, &err);
524 ASSERT(na->na_arena != NULL || err != 0);
525 ASSERT(nx->nx_tx_pp == NULL || (nx->nx_tx_pp->pp_md_type ==
526 NX_DOM(nx)->nxdom_md_type && nx->nx_tx_pp->pp_md_subtype ==
527 NX_DOM(nx)->nxdom_md_subtype));
528 ASSERT(nx->nx_rx_pp == NULL || (nx->nx_rx_pp->pp_md_type ==
529 NX_DOM(nx)->nxdom_md_type && nx->nx_rx_pp->pp_md_subtype ==
530 NX_DOM(nx)->nxdom_md_subtype));
531
532 return err;
533}
534
535static void
536nx_kpipe_prov_fini(struct kern_nexus_domain_provider *nxdom_prov)
537{
538#pragma unused(nxdom_prov)
539 SK_D("destroying %s", nxdom_prov->nxdom_prov_name);
540}
541
542static int
543nx_kpipe_prov_nx_ctor(struct kern_nexus *nx)
544{
545#pragma unused(nx)
546 SK_LOCK_ASSERT_HELD();
547 ASSERT(nx->nx_arg == NULL);
548 return 0;
549}
550
551static void
552nx_kpipe_prov_nx_dtor(struct kern_nexus *nx)
553{
554 struct nxbind *nxb;
555
556 SK_LOCK_ASSERT_HELD();
557
558 if ((nxb = nx->nx_arg) != NULL) {
559 nxb_free(nxb);
560 nx->nx_arg = NULL;
561 }
562}
563
564static int
565nx_kpipe_prov_nx_mem_info(struct kern_nexus *nx, struct kern_pbufpool **tpp,
566 struct kern_pbufpool **rpp)
567{
568 ASSERT(nx->nx_tx_pp != NULL);
569 ASSERT(nx->nx_rx_pp != NULL);
570
571 if (tpp != NULL) {
572 *tpp = nx->nx_tx_pp;
573 }
574 if (rpp != NULL) {
575 *rpp = nx->nx_rx_pp;
576 }
577
578 return 0;
579}
580
581static struct nexus_kpipe_adapter *
582na_kpipe_alloc(zalloc_flags_t how)
583{
584 struct nexus_kpipe_adapter *kna;
585
586 _CASSERT(offsetof(struct nexus_kpipe_adapter, kna_up) == 0);
587
588 kna = zalloc_flags(na_kpipe_zone, how | Z_ZERO);
589 if (kna) {
590 kna->kna_up.na_type = NA_KERNEL_PIPE;
591 kna->kna_up.na_free = na_kpipe_free;
592 }
593 return kna;
594}
595
596static void
597na_kpipe_free(struct nexus_adapter *na)
598{
599 struct nexus_kpipe_adapter *kna = (struct nexus_kpipe_adapter *)na;
600
601 ASSERT(kna->kna_up.na_refcount == 0);
602 SK_DF(SK_VERB_MEM, "kna 0x%llx FREE", SK_KVA(kna));
603 bzero(s: kna, n: sizeof(*kna));
604 zfree(na_kpipe_zone, kna);
605}
606
607static int
608nx_kpipe_na_txsync(struct __kern_channel_ring *kring, struct proc *p,
609 uint32_t flags)
610{
611#pragma unused(p)
612 SK_DF(SK_VERB_KERNEL_PIPE | SK_VERB_SYNC | SK_VERB_TX,
613 "%s(%d) kr \"%s\" (0x%llx) krflags 0x%b ring %u flags 0%x",
614 sk_proc_name_address(p), sk_proc_pid(p), kring->ckr_name,
615 SK_KVA(kring), kring->ckr_flags, CKRF_BITS, kring->ckr_ring_id,
616 flags);
617
618 return nx_sync_tx(kring, commit: (flags & NA_SYNCF_FORCE_RECLAIM));
619}
620
621static int
622nx_kpipe_na_rxsync(struct __kern_channel_ring *kring, struct proc *p,
623 uint32_t flags)
624{
625#pragma unused(p)
626 SK_DF(SK_VERB_KERNEL_PIPE | SK_VERB_SYNC | SK_VERB_RX,
627 "%s(%d) kr \"%s\" (0x%llx) krflags 0x%b ring %u flags 0%x",
628 sk_proc_name_address(p), sk_proc_pid(p), kring->ckr_name,
629 SK_KVA(kring), kring->ckr_flags, CKRF_BITS, kring->ckr_ring_id,
630 flags);
631
632 ASSERT(kring->ckr_rhead <= kring->ckr_lim);
633
634 return nx_sync_rx(kring, commit: (flags & NA_SYNCF_FORCE_READ));
635}
636
637static int
638nx_kpipe_na_activate(struct nexus_adapter *na, na_activate_mode_t mode)
639{
640 ASSERT(na->na_type == NA_KERNEL_PIPE);
641
642 SK_DF(SK_VERB_KERNEL_PIPE, "na \"%s\" (0x%llx) %s", na->na_name,
643 SK_KVA(na), na_activate_mode2str(mode));
644
645 switch (mode) {
646 case NA_ACTIVATE_MODE_ON:
647 os_atomic_or(&na->na_flags, NAF_ACTIVE, relaxed);
648 break;
649
650 case NA_ACTIVATE_MODE_DEFUNCT:
651 break;
652
653 case NA_ACTIVATE_MODE_OFF:
654 os_atomic_andnot(&na->na_flags, NAF_ACTIVE, relaxed);
655 break;
656
657 default:
658 VERIFY(0);
659 /* NOTREACHED */
660 __builtin_unreachable();
661 }
662
663 return 0;
664}
665
666static void
667nx_kpipe_na_dtor(struct nexus_adapter *na)
668{
669#pragma unused(na)
670 ASSERT(na->na_type == NA_KERNEL_PIPE);
671}
672
673static int
674nx_kpipe_na_krings_create(struct nexus_adapter *na, struct kern_channel *ch)
675{
676 ASSERT(na->na_type == NA_KERNEL_PIPE);
677 /*
678 * The assumption here is that all kernel pipe instances
679 * are handled by IOSkywalkFamily, and thus we allocate
680 * the context area for it to store its object references.
681 */
682 return na_rings_mem_setup(na, TRUE, ch);
683}
684
685static void
686nx_kpipe_na_krings_delete(struct nexus_adapter *na, struct kern_channel *ch,
687 boolean_t defunct)
688{
689 ASSERT(na->na_type == NA_KERNEL_PIPE);
690
691 na_rings_mem_teardown(na, ch, defunct);
692}
693
694int
695nx_kpipe_na_find(struct kern_nexus *nx, struct kern_channel *ch,
696 struct chreq *chr, struct nxbind *nxb, struct proc *p,
697 struct nexus_adapter **ret, boolean_t create)
698{
699#pragma unused(ch, p)
700 struct nxprov_params *nxp = NX_PROV(nx)->nxprov_params;
701 struct nexus_kpipe_adapter *kna;
702 ch_endpoint_t ep = chr->cr_endpoint;
703 struct nexus_adapter *na = NULL;
704 int error = 0;
705
706 SK_LOCK_ASSERT_HELD();
707 *ret = NULL;
708
709#if SK_LOG
710 uuid_string_t uuidstr;
711 SK_D("name \"%s\" spec_uuid \"%s\" port %d mode 0x%b pipe_id %u "
712 "ring_id %d ring_set %u ep_type %u:%u create %u%s",
713 chr->cr_name, sk_uuid_unparse(chr->cr_spec_uuid, uuidstr),
714 (int)chr->cr_port, chr->cr_mode, CHMODE_BITS,
715 chr->cr_pipe_id, (int)chr->cr_ring_id, chr->cr_ring_set,
716 chr->cr_real_endpoint, chr->cr_endpoint, create,
717 (ep != CH_ENDPOINT_KERNEL_PIPE) ? " (skipped)" : "");
718#endif /* SK_LOG */
719
720 if (ep != CH_ENDPOINT_KERNEL_PIPE) {
721 return 0;
722 }
723
724 if (!create) {
725 return ENODEV;
726 }
727
728 /*
729 * Check client credentials.
730 */
731 if (!NX_ANONYMOUS_PROV(nx) && (nx->nx_arg == NULL || nxb == NULL ||
732 !nxb_is_equal(nx->nx_arg, nxb))) {
733 return EACCES;
734 }
735
736 kna = na_kpipe_alloc(how: Z_WAITOK);
737
738 na = &kna->kna_up;
739 ASSERT(na->na_type == NA_KERNEL_PIPE);
740 ASSERT(na->na_free == na_kpipe_free);
741
742 (void) snprintf(na->na_name, count: sizeof(na->na_name),
743 "%s{%u", chr->cr_name, NEXUS_PORT_KERNEL_PIPE_CLIENT);
744 uuid_generate_random(out: na->na_uuid);
745
746 na->na_txsync = nx_kpipe_na_txsync;
747 na->na_rxsync = nx_kpipe_na_rxsync;
748 na->na_activate = nx_kpipe_na_activate;
749 na->na_dtor = nx_kpipe_na_dtor;
750 na->na_krings_create = nx_kpipe_na_krings_create;
751 na->na_krings_delete = nx_kpipe_na_krings_delete;
752 na_set_nrings(na, t: NR_TX, v: nxp->nxp_tx_rings);
753 na_set_nrings(na, t: NR_RX, v: nxp->nxp_rx_rings);
754 na_set_nslots(na, t: NR_TX, v: nxp->nxp_tx_slots);
755 na_set_nslots(na, t: NR_RX, v: nxp->nxp_rx_slots);
756 /*
757 * Verify upper bounds; the parameters must have already been
758 * validated by nxdom_prov_params() by the time we get here.
759 */
760 ASSERT(na_get_nrings(na, NR_TX) <= NX_DOM(nx)->nxdom_tx_rings.nb_max);
761 ASSERT(na_get_nrings(na, NR_RX) <= NX_DOM(nx)->nxdom_rx_rings.nb_max);
762 ASSERT(na_get_nslots(na, NR_TX) <= NX_DOM(nx)->nxdom_tx_slots.nb_max);
763 ASSERT(na_get_nslots(na, NR_RX) <= NX_DOM(nx)->nxdom_rx_slots.nb_max);
764
765 *(nexus_stats_type_t *)(uintptr_t)&na->na_stats_type =
766 NEXUS_STATS_TYPE_INVALID;
767
768 na_attach_common(na, nx, &nx_kpipe_prov_s);
769
770 if ((error = NX_DOM_PROV(nx)->nxdom_prov_mem_new(NX_DOM_PROV(nx),
771 nx, na)) != 0) {
772 ASSERT(na->na_arena == NULL);
773 goto err;
774 }
775 ASSERT(na->na_arena != NULL);
776
777 *(uint32_t *)(uintptr_t)&na->na_flowadv_max = nxp->nxp_flowadv_max;
778 ASSERT(na->na_flowadv_max == 0 ||
779 skmem_arena_nexus(na->na_arena)->arn_flowadv_obj != NULL);
780
781#if SK_LOG
782 SK_DF(SK_VERB_KERNEL_PIPE, "created kpipe adapter 0x%llx", SK_KVA(kna));
783 SK_DF(SK_VERB_KERNEL_PIPE, "na_name: \"%s\"", na->na_name);
784 SK_DF(SK_VERB_KERNEL_PIPE, " UUID: %s",
785 sk_uuid_unparse(na->na_uuid, uuidstr));
786 SK_DF(SK_VERB_KERNEL_PIPE, " nx: 0x%llx (\"%s\":\"%s\")",
787 SK_KVA(na->na_nx), NX_DOM(na->na_nx)->nxdom_name,
788 NX_DOM_PROV(na->na_nx)->nxdom_prov_name);
789 SK_DF(SK_VERB_KERNEL_PIPE, " flags: 0x%b",
790 na->na_flags, NAF_BITS);
791 SK_DF(SK_VERB_KERNEL_PIPE, " flowadv_max: %u", na->na_flowadv_max);
792 SK_DF(SK_VERB_KERNEL_PIPE, " rings: tx %u rx %u",
793 na_get_nrings(na, NR_TX),
794 na_get_nrings(na, NR_RX));
795 SK_DF(SK_VERB_KERNEL_PIPE, " slots: tx %u rx %u",
796 na_get_nslots(na, NR_TX),
797 na_get_nslots(na, NR_RX));
798#if CONFIG_NEXUS_USER_PIPE
799 SK_DF(SK_VERB_KERNEL_PIPE, " next_pipe: %u", na->na_next_pipe);
800 SK_DF(SK_VERB_KERNEL_PIPE, " max_pipes: %u", na->na_max_pipes);
801#endif /* CONFIG_NEXUS_USER_PIPE */
802#endif /* SK_LOG */
803
804 *ret = na;
805 na_retain_locked(na: *ret);
806
807 return 0;
808
809err:
810 ASSERT(na != NULL);
811 if (na->na_arena != NULL) {
812 skmem_arena_release(na->na_arena);
813 na->na_arena = NULL;
814 }
815 NA_FREE(na);
816
817 return error;
818}
819
820#if (DEVELOPMENT || DEBUG)
821SYSCTL_NODE(_kern_skywalk, OID_AUTO, kpipe, CTLFLAG_RW | CTLFLAG_LOCKED,
822 0, "Skywalk kpipe tuning");
823#endif
824