1 | /* |
2 | * Copyright (c) 2015-2021 Apple Inc. All rights reserved. |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
5 | * |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License |
8 | * Version 2.0 (the 'License'). You may not use this file except in |
9 | * compliance with the License. The rights granted to you under the License |
10 | * may not be used to create, or enable the creation or redistribution of, |
11 | * unlawful or unlicensed copies of an Apple operating system, or to |
12 | * circumvent, violate, or enable the circumvention or violation of, any |
13 | * terms of an Apple operating system software license agreement. |
14 | * |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. |
17 | * |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and |
24 | * limitations under the License. |
25 | * |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ |
28 | |
29 | /* |
30 | * Copyright (C) 2014 Giuseppe Lettieri. All rights reserved. |
31 | * |
32 | * Redistribution and use in source and binary forms, with or without |
33 | * modification, are permitted provided that the following conditions |
34 | * are met: |
35 | * 1. Redistributions of source code must retain the above copyright |
36 | * notice, this list of conditions and the following disclaimer. |
37 | * 2. Redistributions in binary form must reproduce the above copyright |
38 | * notice, this list of conditions and the following disclaimer in the |
39 | * documentation and/or other materials provided with the distribution. |
40 | * |
41 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND |
42 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
43 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
44 | * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE |
45 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
46 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
47 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
48 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
49 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
50 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
51 | * SUCH DAMAGE. |
52 | */ |
53 | |
54 | #include <skywalk/os_skywalk_private.h> |
55 | #include <skywalk/nexus/upipe/nx_user_pipe.h> |
56 | #include <skywalk/nexus/kpipe/nx_kernel_pipe.h> |
57 | |
58 | /* XXX arbitrary */ |
59 | #define NX_KPIPE_RINGSIZE 128 /* default ring size */ |
60 | #define NX_KPIPE_MINSLOTS 2 |
61 | #define NX_KPIPE_MAXSLOTS 4096 |
62 | #define NX_KPIPE_MAXRINGS NX_MAX_NUM_RING_PAIR |
63 | #define NX_KPIPE_BUFSIZE (2 * 1024) |
64 | #define NX_KPIPE_MINBUFSIZE 64 |
65 | #define NX_KPIPE_MAXBUFSIZE (16 * 1024) |
66 | |
67 | static int nx_kpipe_na_txsync(struct __kern_channel_ring *, struct proc *, |
68 | uint32_t); |
69 | static int nx_kpipe_na_rxsync(struct __kern_channel_ring *, struct proc *, |
70 | uint32_t); |
71 | static int nx_kpipe_na_activate(struct nexus_adapter *, na_activate_mode_t); |
72 | static void nx_kpipe_na_dtor(struct nexus_adapter *); |
73 | static int nx_kpipe_na_krings_create(struct nexus_adapter *, |
74 | struct kern_channel *); |
75 | static void nx_kpipe_na_krings_delete(struct nexus_adapter *, |
76 | struct kern_channel *, boolean_t); |
77 | |
78 | static void nx_kpipe_dom_init(struct nxdom *); |
79 | static void nx_kpipe_dom_terminate(struct nxdom *); |
80 | static void nx_kpipe_dom_fini(struct nxdom *); |
81 | static int nx_kpipe_dom_bind_port(struct kern_nexus *, nexus_port_t *, |
82 | struct nxbind *, void *); |
83 | static int nx_kpipe_dom_unbind_port(struct kern_nexus *, nexus_port_t); |
84 | static int nx_kpipe_dom_connect(struct kern_nexus_domain_provider *, |
85 | struct kern_nexus *, struct kern_channel *, struct chreq *, |
86 | struct kern_channel *, struct nxbind *, struct proc *); |
87 | static void nx_kpipe_dom_disconnect(struct kern_nexus_domain_provider *, |
88 | struct kern_nexus *, struct kern_channel *); |
89 | static void nx_kpipe_dom_defunct(struct kern_nexus_domain_provider *, |
90 | struct kern_nexus *, struct kern_channel *, struct proc *); |
91 | static void nx_kpipe_dom_defunct_finalize(struct kern_nexus_domain_provider *, |
92 | struct kern_nexus *, struct kern_channel *, boolean_t); |
93 | |
94 | static int nx_kpipe_prov_init(struct kern_nexus_domain_provider *); |
95 | static int nx_kpipe_prov_params_adjust( |
96 | const struct kern_nexus_domain_provider *, |
97 | const struct nxprov_params *, struct nxprov_adjusted_params *); |
98 | static int nx_kpipe_prov_params(struct kern_nexus_domain_provider *, |
99 | const uint32_t, const struct nxprov_params *, struct nxprov_params *, |
100 | struct skmem_region_params[SKMEM_REGIONS], uint32_t); |
101 | static int nx_kpipe_prov_mem_new(struct kern_nexus_domain_provider *, |
102 | struct kern_nexus *, struct nexus_adapter *); |
103 | static void nx_kpipe_prov_fini(struct kern_nexus_domain_provider *); |
104 | static int nx_kpipe_prov_nx_ctor(struct kern_nexus *); |
105 | static void nx_kpipe_prov_nx_dtor(struct kern_nexus *); |
106 | static int nx_kpipe_prov_nx_mem_info(struct kern_nexus *, |
107 | struct kern_pbufpool **, struct kern_pbufpool **); |
108 | |
109 | static struct nexus_kpipe_adapter *na_kpipe_alloc(zalloc_flags_t); |
110 | static void na_kpipe_free(struct nexus_adapter *); |
111 | |
112 | struct nxdom nx_kpipe_dom_s = { |
113 | .nxdom_prov_head = |
114 | STAILQ_HEAD_INITIALIZER(nx_kpipe_dom_s.nxdom_prov_head), |
115 | .nxdom_type = NEXUS_TYPE_KERNEL_PIPE, |
116 | .nxdom_md_type = NEXUS_META_TYPE_QUANTUM, |
117 | .nxdom_md_subtype = NEXUS_META_SUBTYPE_PAYLOAD, |
118 | .nxdom_name = "kpipe" , |
119 | .nxdom_ports = { |
120 | .nb_def = 1, |
121 | .nb_min = 1, |
122 | .nb_max = 1, |
123 | }, |
124 | .nxdom_tx_rings = { |
125 | .nb_def = 1, |
126 | .nb_min = 1, |
127 | .nb_max = NX_KPIPE_MAXRINGS, |
128 | }, |
129 | .nxdom_rx_rings = { |
130 | .nb_def = 1, |
131 | .nb_min = 1, |
132 | .nb_max = NX_KPIPE_MAXRINGS, |
133 | }, |
134 | .nxdom_tx_slots = { |
135 | .nb_def = NX_KPIPE_RINGSIZE, |
136 | .nb_min = NX_KPIPE_MINSLOTS, |
137 | .nb_max = NX_KPIPE_MAXSLOTS, |
138 | }, |
139 | .nxdom_rx_slots = { |
140 | .nb_def = NX_KPIPE_RINGSIZE, |
141 | .nb_min = NX_KPIPE_MINSLOTS, |
142 | .nb_max = NX_KPIPE_MAXSLOTS, |
143 | }, |
144 | .nxdom_buf_size = { |
145 | .nb_def = NX_KPIPE_BUFSIZE, |
146 | .nb_min = NX_KPIPE_MINBUFSIZE, |
147 | .nb_max = NX_KPIPE_MAXBUFSIZE, |
148 | }, |
149 | .nxdom_large_buf_size = { |
150 | .nb_def = 0, |
151 | .nb_min = 0, |
152 | .nb_max = 0, |
153 | }, |
154 | .nxdom_meta_size = { |
155 | .nb_def = NX_METADATA_OBJ_MIN_SZ, |
156 | .nb_min = NX_METADATA_OBJ_MIN_SZ, |
157 | .nb_max = NX_METADATA_USR_MAX_SZ, |
158 | }, |
159 | .nxdom_stats_size = { |
160 | .nb_def = 0, |
161 | .nb_min = 0, |
162 | .nb_max = NX_STATS_MAX_SZ, |
163 | }, |
164 | .nxdom_pipes = { |
165 | .nb_def = 0, |
166 | .nb_min = 0, |
167 | .nb_max = NX_UPIPE_MAXPIPES, |
168 | }, |
169 | .nxdom_flowadv_max = { |
170 | .nb_def = 0, |
171 | .nb_min = 0, |
172 | .nb_max = NX_FLOWADV_MAX, |
173 | }, |
174 | .nxdom_nexusadv_size = { |
175 | .nb_def = 0, |
176 | .nb_min = 0, |
177 | .nb_max = NX_NEXUSADV_MAX_SZ, |
178 | }, |
179 | .nxdom_capabilities = { |
180 | .nb_def = NXPCAP_USER_CHANNEL, |
181 | .nb_min = NXPCAP_USER_CHANNEL, |
182 | .nb_max = NXPCAP_USER_CHANNEL, |
183 | }, |
184 | .nxdom_qmap = { |
185 | .nb_def = NEXUS_QMAP_TYPE_INVALID, |
186 | .nb_min = NEXUS_QMAP_TYPE_INVALID, |
187 | .nb_max = NEXUS_QMAP_TYPE_INVALID, |
188 | }, |
189 | .nxdom_max_frags = { |
190 | .nb_def = NX_PBUF_FRAGS_DEFAULT, |
191 | .nb_min = NX_PBUF_FRAGS_MIN, |
192 | .nb_max = NX_PBUF_FRAGS_DEFAULT, |
193 | }, |
194 | .nxdom_init = nx_kpipe_dom_init, |
195 | .nxdom_terminate = nx_kpipe_dom_terminate, |
196 | .nxdom_fini = nx_kpipe_dom_fini, |
197 | .nxdom_find_port = NULL, |
198 | .nxdom_port_is_reserved = NULL, |
199 | .nxdom_bind_port = nx_kpipe_dom_bind_port, |
200 | .nxdom_unbind_port = nx_kpipe_dom_unbind_port, |
201 | .nxdom_connect = nx_kpipe_dom_connect, |
202 | .nxdom_disconnect = nx_kpipe_dom_disconnect, |
203 | .nxdom_defunct = nx_kpipe_dom_defunct, |
204 | .nxdom_defunct_finalize = nx_kpipe_dom_defunct_finalize, |
205 | }; |
206 | |
207 | static struct kern_nexus_domain_provider nx_kpipe_prov_s = { |
208 | .nxdom_prov_name = NEXUS_PROVIDER_KERNEL_PIPE, |
209 | .nxdom_prov_flags = NXDOMPROVF_DEFAULT, |
210 | .nxdom_prov_cb = { |
211 | .dp_cb_init = nx_kpipe_prov_init, |
212 | .dp_cb_fini = nx_kpipe_prov_fini, |
213 | .dp_cb_params = nx_kpipe_prov_params, |
214 | .dp_cb_mem_new = nx_kpipe_prov_mem_new, |
215 | .dp_cb_config = NULL, |
216 | .dp_cb_nx_ctor = nx_kpipe_prov_nx_ctor, |
217 | .dp_cb_nx_dtor = nx_kpipe_prov_nx_dtor, |
218 | .dp_cb_nx_mem_info = nx_kpipe_prov_nx_mem_info, |
219 | .dp_cb_nx_mib_get = NULL, |
220 | .dp_cb_nx_stop = NULL, |
221 | }, |
222 | }; |
223 | |
224 | static SKMEM_TYPE_DEFINE(na_kpipe_zone, struct nexus_kpipe_adapter); |
225 | |
226 | static void |
227 | nx_kpipe_dom_init(struct nxdom *nxdom) |
228 | { |
229 | SK_LOCK_ASSERT_HELD(); |
230 | ASSERT(!(nxdom->nxdom_flags & NEXUSDOMF_INITIALIZED)); |
231 | |
232 | (void) nxdom_prov_add(nxdom, &nx_kpipe_prov_s); |
233 | } |
234 | |
235 | static void |
236 | nx_kpipe_dom_terminate(struct nxdom *nxdom) |
237 | { |
238 | struct kern_nexus_domain_provider *nxdom_prov, *tnxdp; |
239 | |
240 | SK_LOCK_ASSERT_HELD(); |
241 | |
242 | STAILQ_FOREACH_SAFE(nxdom_prov, &nxdom->nxdom_prov_head, |
243 | nxdom_prov_link, tnxdp) { |
244 | (void) nxdom_prov_del(nxdom_prov); |
245 | } |
246 | } |
247 | |
248 | static void |
249 | nx_kpipe_dom_fini(struct nxdom *nxdom) |
250 | { |
251 | #pragma unused(nxdom) |
252 | } |
253 | |
254 | static int |
255 | nx_kpipe_dom_bind_port(struct kern_nexus *nx, nexus_port_t *nx_port, |
256 | struct nxbind *nxb0, void *info) |
257 | { |
258 | #pragma unused(info) |
259 | struct nxbind *nxb = NULL; |
260 | int error = 0; |
261 | |
262 | ASSERT(nx_port != NULL); |
263 | ASSERT(nxb0 != NULL); |
264 | |
265 | switch (*nx_port) { |
266 | case NEXUS_PORT_KERNEL_PIPE_CLIENT: |
267 | if (nx->nx_arg != NULL) { |
268 | error = EEXIST; |
269 | break; |
270 | } |
271 | |
272 | nxb = nxb_alloc(Z_WAITOK); |
273 | nxb_move(nxb0, nxb); |
274 | nx->nx_arg = nxb; |
275 | |
276 | ASSERT(error == 0); |
277 | break; |
278 | |
279 | default: |
280 | error = EDOM; |
281 | break; |
282 | } |
283 | |
284 | return error; |
285 | } |
286 | |
287 | static int |
288 | nx_kpipe_dom_unbind_port(struct kern_nexus *nx, nexus_port_t nx_port) |
289 | { |
290 | struct nxbind *nxb = NULL; |
291 | int error = 0; |
292 | |
293 | ASSERT(nx_port != NEXUS_PORT_ANY); |
294 | |
295 | switch (nx_port) { |
296 | case NEXUS_PORT_KERNEL_PIPE_CLIENT: |
297 | if ((nxb = nx->nx_arg) == NULL) { |
298 | error = ENOENT; |
299 | break; |
300 | } |
301 | nx->nx_arg = NULL; |
302 | nxb_free(nxb); |
303 | ASSERT(error == 0); |
304 | break; |
305 | |
306 | default: |
307 | error = EDOM; |
308 | break; |
309 | } |
310 | |
311 | return error; |
312 | } |
313 | |
314 | static int |
315 | nx_kpipe_dom_connect(struct kern_nexus_domain_provider *nxdom_prov, |
316 | struct kern_nexus *nx, struct kern_channel *ch, struct chreq *chr, |
317 | struct kern_channel *ch0, struct nxbind *nxb, struct proc *p) |
318 | { |
319 | #pragma unused(nxdom_prov) |
320 | nexus_port_t port = chr->cr_port; |
321 | int err = 0; |
322 | |
323 | SK_DF(SK_VERB_KERNEL_PIPE, "port %d mode 0x%b" , |
324 | (int)port, chr->cr_mode, CHMODE_BITS); |
325 | |
326 | SK_LOCK_ASSERT_HELD(); |
327 | |
328 | ASSERT(nx->nx_prov->nxprov_params->nxp_type == |
329 | nxdom_prov->nxdom_prov_dom->nxdom_type && |
330 | nx->nx_prov->nxprov_params->nxp_type == NEXUS_TYPE_KERNEL_PIPE); |
331 | |
332 | if (port != NEXUS_PORT_KERNEL_PIPE_CLIENT) { |
333 | err = EINVAL; |
334 | goto done; |
335 | } |
336 | |
337 | /* |
338 | * XXX: user packet pool is not supported for kernel pipe for now. |
339 | */ |
340 | if (chr->cr_mode & CHMODE_USER_PACKET_POOL) { |
341 | SK_ERR("User Packet pool mode not supported for kpipe" ); |
342 | err = ENOTSUP; |
343 | goto done; |
344 | } |
345 | |
346 | if (chr->cr_mode & CHMODE_EVENT_RING) { |
347 | SK_ERR("event ring is not supported for kpipe" ); |
348 | err = ENOTSUP; |
349 | goto done; |
350 | } |
351 | |
352 | if (chr->cr_mode & CHMODE_LOW_LATENCY) { |
353 | SK_ERR("low latency is not supported for kpipe" ); |
354 | err = ENOTSUP; |
355 | goto done; |
356 | } |
357 | |
358 | chr->cr_ring_set = RING_SET_DEFAULT; |
359 | chr->cr_real_endpoint = chr->cr_endpoint = CH_ENDPOINT_KERNEL_PIPE; |
360 | (void) snprintf(chr->cr_name, count: sizeof(chr->cr_name), "kpipe:%llu:%.*s" , |
361 | nx->nx_id, (int)nx->nx_prov->nxprov_params->nxp_namelen, |
362 | nx->nx_prov->nxprov_params->nxp_name); |
363 | |
364 | err = na_connect(nx, ch, chr, ch0, nxb, p); |
365 | if (err == 0) { |
366 | /* |
367 | * Mark the kernel slot descriptor region as busy; this |
368 | * prevents it from being torn-down at channel defunct |
369 | * time, as the (external) nexus owner may be calling |
370 | * KPIs that require accessing the slots. |
371 | */ |
372 | skmem_arena_nexus_sd_set_noidle( |
373 | skmem_arena_nexus(ar: ch->ch_na->na_arena), 1); |
374 | } |
375 | |
376 | done: |
377 | return err; |
378 | } |
379 | |
380 | static void |
381 | nx_kpipe_dom_disconnect(struct kern_nexus_domain_provider *nxdom_prov, |
382 | struct kern_nexus *nx, struct kern_channel *ch) |
383 | { |
384 | #pragma unused(nxdom_prov, nx) |
385 | SK_LOCK_ASSERT_HELD(); |
386 | |
387 | SK_D("channel 0x%llx -!- nexus 0x%llx (%s:\"%s\":%u:%d)" , SK_KVA(ch), |
388 | SK_KVA(nx), nxdom_prov->nxdom_prov_name, ch->ch_na->na_name, |
389 | ch->ch_info->cinfo_nx_port, (int)ch->ch_info->cinfo_ch_ring_id); |
390 | |
391 | /* |
392 | * Release busy assertion held earlier in nx_kpipe_dom_connect(); |
393 | * this allows for the final arena teardown to succeed. |
394 | */ |
395 | skmem_arena_nexus_sd_set_noidle( |
396 | skmem_arena_nexus(ar: ch->ch_na->na_arena), -1); |
397 | |
398 | na_disconnect(nx, ch); |
399 | } |
400 | |
401 | static void |
402 | nx_kpipe_dom_defunct(struct kern_nexus_domain_provider *nxdom_prov, |
403 | struct kern_nexus *nx, struct kern_channel *ch, struct proc *p) |
404 | { |
405 | #pragma unused(nxdom_prov, nx) |
406 | struct nexus_adapter *na = ch->ch_na; |
407 | ring_id_t qfirst; |
408 | ring_id_t qlast; |
409 | enum txrx t; |
410 | uint32_t i; |
411 | |
412 | LCK_MTX_ASSERT(&ch->ch_lock, LCK_MTX_ASSERT_OWNED); |
413 | ASSERT(!(ch->ch_flags & CHANF_KERNEL)); |
414 | ASSERT(ch->ch_na->na_type == NA_KERNEL_PIPE); |
415 | |
416 | /* |
417 | * Interface drivers like utun & IPsec access the kpipe rings |
418 | * outside of a kpipe channel sync context. They hold rights |
419 | * to the ring through kr_enter(). |
420 | */ |
421 | for_rx_tx(t) { |
422 | qfirst = ch->ch_first[t]; |
423 | qlast = ch->ch_last[t]; |
424 | |
425 | for (i = qfirst; i < qlast; i++) { |
426 | (void) kr_enter(&NAKR(na, t)[i], TRUE); |
427 | } |
428 | } |
429 | |
430 | na_ch_rings_defunct(ch, p); |
431 | |
432 | for_rx_tx(t) { |
433 | qfirst = ch->ch_first[t]; |
434 | qlast = ch->ch_last[t]; |
435 | |
436 | for (i = qfirst; i < qlast; i++) { |
437 | kr_exit(&NAKR(na, t)[i]); |
438 | } |
439 | } |
440 | } |
441 | |
442 | static void |
443 | nx_kpipe_dom_defunct_finalize(struct kern_nexus_domain_provider *nxdom_prov, |
444 | struct kern_nexus *nx, struct kern_channel *ch, boolean_t locked) |
445 | { |
446 | #pragma unused(nxdom_prov) |
447 | if (!locked) { |
448 | SK_LOCK_ASSERT_NOTHELD(); |
449 | SK_LOCK(); |
450 | LCK_MTX_ASSERT(&ch->ch_lock, LCK_MTX_ASSERT_NOTOWNED); |
451 | } else { |
452 | SK_LOCK_ASSERT_HELD(); |
453 | LCK_MTX_ASSERT(&ch->ch_lock, LCK_MTX_ASSERT_OWNED); |
454 | } |
455 | |
456 | ASSERT(!(ch->ch_flags & CHANF_KERNEL)); |
457 | ASSERT(ch->ch_na->na_type == NA_KERNEL_PIPE); |
458 | |
459 | na_defunct(nx, ch, ch->ch_na, locked); |
460 | |
461 | SK_D("%s(%d): ch 0x%llx -/- nx 0x%llx (%s:\"%s\":%u:%d)" , |
462 | ch->ch_name, ch->ch_pid, SK_KVA(ch), SK_KVA(nx), |
463 | nxdom_prov->nxdom_prov_name, ch->ch_na->na_name, |
464 | ch->ch_info->cinfo_nx_port, (int)ch->ch_info->cinfo_ch_ring_id); |
465 | |
466 | if (!locked) { |
467 | LCK_MTX_ASSERT(&ch->ch_lock, LCK_MTX_ASSERT_NOTOWNED); |
468 | SK_UNLOCK(); |
469 | } else { |
470 | LCK_MTX_ASSERT(&ch->ch_lock, LCK_MTX_ASSERT_OWNED); |
471 | SK_LOCK_ASSERT_HELD(); |
472 | } |
473 | } |
474 | |
475 | static int |
476 | nx_kpipe_prov_init(struct kern_nexus_domain_provider *nxdom_prov) |
477 | { |
478 | #pragma unused(nxdom_prov) |
479 | SK_D("initializing %s" , nxdom_prov->nxdom_prov_name); |
480 | return 0; |
481 | } |
482 | |
483 | static int |
484 | nx_kpipe_prov_params_adjust(const struct kern_nexus_domain_provider *nxdom_prov, |
485 | const struct nxprov_params *nxp, struct nxprov_adjusted_params *adj) |
486 | { |
487 | #pragma unused(nxdom_prov, nxp, adj) |
488 | return 0; |
489 | } |
490 | |
491 | static int |
492 | nx_kpipe_prov_params(struct kern_nexus_domain_provider *nxdom_prov, |
493 | const uint32_t req, const struct nxprov_params *nxp0, |
494 | struct nxprov_params *nxp, struct skmem_region_params srp[SKMEM_REGIONS], |
495 | uint32_t pp_region_config_flags) |
496 | { |
497 | struct nxdom *nxdom = nxdom_prov->nxdom_prov_dom; |
498 | |
499 | return nxprov_params_adjust(nxdom_prov, req, nxp0, nxp, srp, |
500 | nxdom, nxdom, nxdom, pp_region_config_flags, |
501 | adjust_fn: nx_kpipe_prov_params_adjust); |
502 | } |
503 | |
504 | static int |
505 | nx_kpipe_prov_mem_new(struct kern_nexus_domain_provider *nxdom_prov, |
506 | struct kern_nexus *nx, struct nexus_adapter *na) |
507 | { |
508 | #pragma unused(nxdom_prov) |
509 | int err = 0; |
510 | |
511 | SK_DF(SK_VERB_KERNEL_PIPE, |
512 | "nx 0x%llx (\"%s\":\"%s\") na \"%s\" (0x%llx)" , SK_KVA(nx), |
513 | NX_DOM(nx)->nxdom_name, nxdom_prov->nxdom_prov_name, na->na_name, |
514 | SK_KVA(na)); |
515 | |
516 | ASSERT(na->na_arena == NULL); |
517 | ASSERT(NX_USER_CHANNEL_PROV(nx)); |
518 | /* |
519 | * Store pp in the nexus to handle kern_nexus_get_pbufpool() calls. |
520 | */ |
521 | na->na_arena = skmem_arena_create_for_nexus(na, |
522 | NX_PROV(nx)->nxprov_region_params, &nx->nx_tx_pp, |
523 | &nx->nx_rx_pp, FALSE, FALSE, NULL, &err); |
524 | ASSERT(na->na_arena != NULL || err != 0); |
525 | ASSERT(nx->nx_tx_pp == NULL || (nx->nx_tx_pp->pp_md_type == |
526 | NX_DOM(nx)->nxdom_md_type && nx->nx_tx_pp->pp_md_subtype == |
527 | NX_DOM(nx)->nxdom_md_subtype)); |
528 | ASSERT(nx->nx_rx_pp == NULL || (nx->nx_rx_pp->pp_md_type == |
529 | NX_DOM(nx)->nxdom_md_type && nx->nx_rx_pp->pp_md_subtype == |
530 | NX_DOM(nx)->nxdom_md_subtype)); |
531 | |
532 | return err; |
533 | } |
534 | |
535 | static void |
536 | nx_kpipe_prov_fini(struct kern_nexus_domain_provider *nxdom_prov) |
537 | { |
538 | #pragma unused(nxdom_prov) |
539 | SK_D("destroying %s" , nxdom_prov->nxdom_prov_name); |
540 | } |
541 | |
542 | static int |
543 | nx_kpipe_prov_nx_ctor(struct kern_nexus *nx) |
544 | { |
545 | #pragma unused(nx) |
546 | SK_LOCK_ASSERT_HELD(); |
547 | ASSERT(nx->nx_arg == NULL); |
548 | return 0; |
549 | } |
550 | |
551 | static void |
552 | nx_kpipe_prov_nx_dtor(struct kern_nexus *nx) |
553 | { |
554 | struct nxbind *nxb; |
555 | |
556 | SK_LOCK_ASSERT_HELD(); |
557 | |
558 | if ((nxb = nx->nx_arg) != NULL) { |
559 | nxb_free(nxb); |
560 | nx->nx_arg = NULL; |
561 | } |
562 | } |
563 | |
564 | static int |
565 | nx_kpipe_prov_nx_mem_info(struct kern_nexus *nx, struct kern_pbufpool **tpp, |
566 | struct kern_pbufpool **rpp) |
567 | { |
568 | ASSERT(nx->nx_tx_pp != NULL); |
569 | ASSERT(nx->nx_rx_pp != NULL); |
570 | |
571 | if (tpp != NULL) { |
572 | *tpp = nx->nx_tx_pp; |
573 | } |
574 | if (rpp != NULL) { |
575 | *rpp = nx->nx_rx_pp; |
576 | } |
577 | |
578 | return 0; |
579 | } |
580 | |
581 | static struct nexus_kpipe_adapter * |
582 | na_kpipe_alloc(zalloc_flags_t how) |
583 | { |
584 | struct nexus_kpipe_adapter *kna; |
585 | |
586 | _CASSERT(offsetof(struct nexus_kpipe_adapter, kna_up) == 0); |
587 | |
588 | kna = zalloc_flags(na_kpipe_zone, how | Z_ZERO); |
589 | if (kna) { |
590 | kna->kna_up.na_type = NA_KERNEL_PIPE; |
591 | kna->kna_up.na_free = na_kpipe_free; |
592 | } |
593 | return kna; |
594 | } |
595 | |
596 | static void |
597 | na_kpipe_free(struct nexus_adapter *na) |
598 | { |
599 | struct nexus_kpipe_adapter *kna = (struct nexus_kpipe_adapter *)na; |
600 | |
601 | ASSERT(kna->kna_up.na_refcount == 0); |
602 | SK_DF(SK_VERB_MEM, "kna 0x%llx FREE" , SK_KVA(kna)); |
603 | bzero(s: kna, n: sizeof(*kna)); |
604 | zfree(na_kpipe_zone, kna); |
605 | } |
606 | |
607 | static int |
608 | nx_kpipe_na_txsync(struct __kern_channel_ring *kring, struct proc *p, |
609 | uint32_t flags) |
610 | { |
611 | #pragma unused(p) |
612 | SK_DF(SK_VERB_KERNEL_PIPE | SK_VERB_SYNC | SK_VERB_TX, |
613 | "%s(%d) kr \"%s\" (0x%llx) krflags 0x%b ring %u flags 0%x" , |
614 | sk_proc_name_address(p), sk_proc_pid(p), kring->ckr_name, |
615 | SK_KVA(kring), kring->ckr_flags, CKRF_BITS, kring->ckr_ring_id, |
616 | flags); |
617 | |
618 | return nx_sync_tx(kring, commit: (flags & NA_SYNCF_FORCE_RECLAIM)); |
619 | } |
620 | |
621 | static int |
622 | nx_kpipe_na_rxsync(struct __kern_channel_ring *kring, struct proc *p, |
623 | uint32_t flags) |
624 | { |
625 | #pragma unused(p) |
626 | SK_DF(SK_VERB_KERNEL_PIPE | SK_VERB_SYNC | SK_VERB_RX, |
627 | "%s(%d) kr \"%s\" (0x%llx) krflags 0x%b ring %u flags 0%x" , |
628 | sk_proc_name_address(p), sk_proc_pid(p), kring->ckr_name, |
629 | SK_KVA(kring), kring->ckr_flags, CKRF_BITS, kring->ckr_ring_id, |
630 | flags); |
631 | |
632 | ASSERT(kring->ckr_rhead <= kring->ckr_lim); |
633 | |
634 | return nx_sync_rx(kring, commit: (flags & NA_SYNCF_FORCE_READ)); |
635 | } |
636 | |
637 | static int |
638 | nx_kpipe_na_activate(struct nexus_adapter *na, na_activate_mode_t mode) |
639 | { |
640 | ASSERT(na->na_type == NA_KERNEL_PIPE); |
641 | |
642 | SK_DF(SK_VERB_KERNEL_PIPE, "na \"%s\" (0x%llx) %s" , na->na_name, |
643 | SK_KVA(na), na_activate_mode2str(mode)); |
644 | |
645 | switch (mode) { |
646 | case NA_ACTIVATE_MODE_ON: |
647 | os_atomic_or(&na->na_flags, NAF_ACTIVE, relaxed); |
648 | break; |
649 | |
650 | case NA_ACTIVATE_MODE_DEFUNCT: |
651 | break; |
652 | |
653 | case NA_ACTIVATE_MODE_OFF: |
654 | os_atomic_andnot(&na->na_flags, NAF_ACTIVE, relaxed); |
655 | break; |
656 | |
657 | default: |
658 | VERIFY(0); |
659 | /* NOTREACHED */ |
660 | __builtin_unreachable(); |
661 | } |
662 | |
663 | return 0; |
664 | } |
665 | |
666 | static void |
667 | nx_kpipe_na_dtor(struct nexus_adapter *na) |
668 | { |
669 | #pragma unused(na) |
670 | ASSERT(na->na_type == NA_KERNEL_PIPE); |
671 | } |
672 | |
673 | static int |
674 | nx_kpipe_na_krings_create(struct nexus_adapter *na, struct kern_channel *ch) |
675 | { |
676 | ASSERT(na->na_type == NA_KERNEL_PIPE); |
677 | /* |
678 | * The assumption here is that all kernel pipe instances |
679 | * are handled by IOSkywalkFamily, and thus we allocate |
680 | * the context area for it to store its object references. |
681 | */ |
682 | return na_rings_mem_setup(na, TRUE, ch); |
683 | } |
684 | |
685 | static void |
686 | nx_kpipe_na_krings_delete(struct nexus_adapter *na, struct kern_channel *ch, |
687 | boolean_t defunct) |
688 | { |
689 | ASSERT(na->na_type == NA_KERNEL_PIPE); |
690 | |
691 | na_rings_mem_teardown(na, ch, defunct); |
692 | } |
693 | |
694 | int |
695 | nx_kpipe_na_find(struct kern_nexus *nx, struct kern_channel *ch, |
696 | struct chreq *chr, struct nxbind *nxb, struct proc *p, |
697 | struct nexus_adapter **ret, boolean_t create) |
698 | { |
699 | #pragma unused(ch, p) |
700 | struct nxprov_params *nxp = NX_PROV(nx)->nxprov_params; |
701 | struct nexus_kpipe_adapter *kna; |
702 | ch_endpoint_t ep = chr->cr_endpoint; |
703 | struct nexus_adapter *na = NULL; |
704 | int error = 0; |
705 | |
706 | SK_LOCK_ASSERT_HELD(); |
707 | *ret = NULL; |
708 | |
709 | #if SK_LOG |
710 | uuid_string_t uuidstr; |
711 | SK_D("name \"%s\" spec_uuid \"%s\" port %d mode 0x%b pipe_id %u " |
712 | "ring_id %d ring_set %u ep_type %u:%u create %u%s" , |
713 | chr->cr_name, sk_uuid_unparse(chr->cr_spec_uuid, uuidstr), |
714 | (int)chr->cr_port, chr->cr_mode, CHMODE_BITS, |
715 | chr->cr_pipe_id, (int)chr->cr_ring_id, chr->cr_ring_set, |
716 | chr->cr_real_endpoint, chr->cr_endpoint, create, |
717 | (ep != CH_ENDPOINT_KERNEL_PIPE) ? " (skipped)" : "" ); |
718 | #endif /* SK_LOG */ |
719 | |
720 | if (ep != CH_ENDPOINT_KERNEL_PIPE) { |
721 | return 0; |
722 | } |
723 | |
724 | if (!create) { |
725 | return ENODEV; |
726 | } |
727 | |
728 | /* |
729 | * Check client credentials. |
730 | */ |
731 | if (!NX_ANONYMOUS_PROV(nx) && (nx->nx_arg == NULL || nxb == NULL || |
732 | !nxb_is_equal(nx->nx_arg, nxb))) { |
733 | return EACCES; |
734 | } |
735 | |
736 | kna = na_kpipe_alloc(how: Z_WAITOK); |
737 | |
738 | na = &kna->kna_up; |
739 | ASSERT(na->na_type == NA_KERNEL_PIPE); |
740 | ASSERT(na->na_free == na_kpipe_free); |
741 | |
742 | (void) snprintf(na->na_name, count: sizeof(na->na_name), |
743 | "%s{%u" , chr->cr_name, NEXUS_PORT_KERNEL_PIPE_CLIENT); |
744 | uuid_generate_random(out: na->na_uuid); |
745 | |
746 | na->na_txsync = nx_kpipe_na_txsync; |
747 | na->na_rxsync = nx_kpipe_na_rxsync; |
748 | na->na_activate = nx_kpipe_na_activate; |
749 | na->na_dtor = nx_kpipe_na_dtor; |
750 | na->na_krings_create = nx_kpipe_na_krings_create; |
751 | na->na_krings_delete = nx_kpipe_na_krings_delete; |
752 | na_set_nrings(na, t: NR_TX, v: nxp->nxp_tx_rings); |
753 | na_set_nrings(na, t: NR_RX, v: nxp->nxp_rx_rings); |
754 | na_set_nslots(na, t: NR_TX, v: nxp->nxp_tx_slots); |
755 | na_set_nslots(na, t: NR_RX, v: nxp->nxp_rx_slots); |
756 | /* |
757 | * Verify upper bounds; the parameters must have already been |
758 | * validated by nxdom_prov_params() by the time we get here. |
759 | */ |
760 | ASSERT(na_get_nrings(na, NR_TX) <= NX_DOM(nx)->nxdom_tx_rings.nb_max); |
761 | ASSERT(na_get_nrings(na, NR_RX) <= NX_DOM(nx)->nxdom_rx_rings.nb_max); |
762 | ASSERT(na_get_nslots(na, NR_TX) <= NX_DOM(nx)->nxdom_tx_slots.nb_max); |
763 | ASSERT(na_get_nslots(na, NR_RX) <= NX_DOM(nx)->nxdom_rx_slots.nb_max); |
764 | |
765 | *(nexus_stats_type_t *)(uintptr_t)&na->na_stats_type = |
766 | NEXUS_STATS_TYPE_INVALID; |
767 | |
768 | na_attach_common(na, nx, &nx_kpipe_prov_s); |
769 | |
770 | if ((error = NX_DOM_PROV(nx)->nxdom_prov_mem_new(NX_DOM_PROV(nx), |
771 | nx, na)) != 0) { |
772 | ASSERT(na->na_arena == NULL); |
773 | goto err; |
774 | } |
775 | ASSERT(na->na_arena != NULL); |
776 | |
777 | *(uint32_t *)(uintptr_t)&na->na_flowadv_max = nxp->nxp_flowadv_max; |
778 | ASSERT(na->na_flowadv_max == 0 || |
779 | skmem_arena_nexus(na->na_arena)->arn_flowadv_obj != NULL); |
780 | |
781 | #if SK_LOG |
782 | SK_DF(SK_VERB_KERNEL_PIPE, "created kpipe adapter 0x%llx" , SK_KVA(kna)); |
783 | SK_DF(SK_VERB_KERNEL_PIPE, "na_name: \"%s\"" , na->na_name); |
784 | SK_DF(SK_VERB_KERNEL_PIPE, " UUID: %s" , |
785 | sk_uuid_unparse(na->na_uuid, uuidstr)); |
786 | SK_DF(SK_VERB_KERNEL_PIPE, " nx: 0x%llx (\"%s\":\"%s\")" , |
787 | SK_KVA(na->na_nx), NX_DOM(na->na_nx)->nxdom_name, |
788 | NX_DOM_PROV(na->na_nx)->nxdom_prov_name); |
789 | SK_DF(SK_VERB_KERNEL_PIPE, " flags: 0x%b" , |
790 | na->na_flags, NAF_BITS); |
791 | SK_DF(SK_VERB_KERNEL_PIPE, " flowadv_max: %u" , na->na_flowadv_max); |
792 | SK_DF(SK_VERB_KERNEL_PIPE, " rings: tx %u rx %u" , |
793 | na_get_nrings(na, NR_TX), |
794 | na_get_nrings(na, NR_RX)); |
795 | SK_DF(SK_VERB_KERNEL_PIPE, " slots: tx %u rx %u" , |
796 | na_get_nslots(na, NR_TX), |
797 | na_get_nslots(na, NR_RX)); |
798 | #if CONFIG_NEXUS_USER_PIPE |
799 | SK_DF(SK_VERB_KERNEL_PIPE, " next_pipe: %u" , na->na_next_pipe); |
800 | SK_DF(SK_VERB_KERNEL_PIPE, " max_pipes: %u" , na->na_max_pipes); |
801 | #endif /* CONFIG_NEXUS_USER_PIPE */ |
802 | #endif /* SK_LOG */ |
803 | |
804 | *ret = na; |
805 | na_retain_locked(na: *ret); |
806 | |
807 | return 0; |
808 | |
809 | err: |
810 | ASSERT(na != NULL); |
811 | if (na->na_arena != NULL) { |
812 | skmem_arena_release(na->na_arena); |
813 | na->na_arena = NULL; |
814 | } |
815 | NA_FREE(na); |
816 | |
817 | return error; |
818 | } |
819 | |
820 | #if (DEVELOPMENT || DEBUG) |
821 | SYSCTL_NODE(_kern_skywalk, OID_AUTO, kpipe, CTLFLAG_RW | CTLFLAG_LOCKED, |
822 | 0, "Skywalk kpipe tuning" ); |
823 | #endif |
824 | |